From 3341938bb644e6fc39afa6a8fcb1306604999e42 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Wed, 12 Aug 2026 22:37:49 +0800 Subject: [RISCV] Split the GlobalISel target components --- backend/include/rcc/CodeGen/InlineAsm.h | 12 +- .../include/rcc/Target/RISCV/RISCVCallLowering.h | 43 - backend/include/rcc/Target/RISCV/RISCVCodeGen.h | 36 - .../include/rcc/Target/RISCV/RISCVFunctionInfo.h | 13 +- backend/include/rcc/Target/RISCV/RISCVPipeline.h | 10 +- backend/include/rcc/Target/RISCV/RISCVSubtarget.h | 40 +- backend/lib/CodeGen/InstructionDescriptor.cpp | 7 +- backend/lib/CodeGen/MachineFunctionCloner.cpp | 3 +- backend/lib/CodeGen/MachineVerifier.cpp | 2 +- backend/lib/MIR/EmbeddedIR.cpp | 4 +- backend/lib/MIR/MIRFunctionRecordsParser.cpp | 6 +- backend/lib/MIR/MIRMetadataParser.cpp | 8 +- backend/lib/Support/YAML.cpp | 6 +- backend/lib/Target/RISCV/CMakeLists.txt | 24 +- .../lib/Target/RISCV/GISel/RISCVCallLowering.cpp | 1495 +++++ backend/lib/Target/RISCV/GISel/RISCVCallLowering.h | 53 + .../RISCV/GISel/RISCVInstructionSelector.cpp | 2627 ++++++++ .../lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 3818 ++++++++++++ .../lib/Target/RISCV/GISel/RISCVLegalizerInfo.h | 29 + .../Target/RISCV/GISel/RISCVRegisterBankInfo.cpp | 157 + .../lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h | 28 + backend/lib/Target/RISCV/RISCV.h | 38 + backend/lib/Target/RISCV/RISCVCallLowering.cpp | 1512 ----- backend/lib/Target/RISCV/RISCVCodeGen.cpp | 6536 -------------------- backend/lib/Target/RISCV/RISCVFrameLowering.cpp | 4 +- backend/lib/Target/RISCV/RISCVFunctionInfo.cpp | 38 +- .../Target/RISCV/RISCVImmediateMaterialization.cpp | 27 +- .../Target/RISCV/RISCVImmediateMaterialization.h | 6 +- backend/lib/Target/RISCV/RISCVMachineCombiner.cpp | 2 +- .../lib/Target/RISCV/RISCVMachineMemoryFolding.cpp | 2 +- .../lib/Target/RISCV/RISCVOptimizationPipeline.cpp | 2 +- backend/lib/Target/RISCV/RISCVPipeline.cpp | 40 +- backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp | 2 +- .../lib/Target/RISCV/RISCVRegisterAllocation.cpp | 2 +- backend/lib/Target/RISCV/RISCVScheduling.cpp | 2 +- backend/lib/Target/RISCV/RISCVSubtarget.cpp | 34 + backend/lib/Tooling/MIRFuzzer.cpp | 6 +- backend/lib/Tooling/PipelineDriver.cpp | 32 +- backend/tools/MIRTool/MIRToolMain.cpp | 7 +- backend/unittests/CodeGen/CMakeLists.txt | 8 + .../unittests/CodeGen/GlobalISel/CMakeLists.txt | 2 + .../CodeGen/GlobalISel/InstructionSelectTest.cpp | 26 +- .../unittests/CodeGen/GlobalISel/LegalizerTest.cpp | 9 +- .../CodeGen/GlobalISel/RegBankSelectTest.cpp | 10 +- .../unittests/CodeGen/MachineOptimizationTest.cpp | 2 +- .../unittests/CodeGen/MachineSchedulingTest.cpp | 2 +- backend/unittests/Target/CMakeLists.txt | 10 +- backend/unittests/Target/RISCVCallLoweringTest.cpp | 73 +- backend/unittests/Target/RISCVCodeGenTest.cpp | 279 +- .../unittests/Target/RISCVFrameLoweringTest.cpp | 4 +- backend/unittests/Target/RISCVLateCodeGenTest.cpp | 2 +- .../Target/RISCVRegisterAllocationTest.cpp | 2 +- backend/unittests/Target/RISCVSSAExitTest.cpp | 2 +- 53 files changed, 8652 insertions(+), 8492 deletions(-) delete mode 100644 backend/include/rcc/Target/RISCV/RISCVCallLowering.h delete mode 100644 backend/include/rcc/Target/RISCV/RISCVCodeGen.h create mode 100644 backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp create mode 100644 backend/lib/Target/RISCV/GISel/RISCVCallLowering.h create mode 100644 backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp create mode 100644 backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp create mode 100644 backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h create mode 100644 backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp create mode 100644 backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h create mode 100644 backend/lib/Target/RISCV/RISCV.h delete mode 100644 backend/lib/Target/RISCV/RISCVCallLowering.cpp delete mode 100644 backend/lib/Target/RISCV/RISCVCodeGen.cpp (limited to 'backend') diff --git a/backend/include/rcc/CodeGen/InlineAsm.h b/backend/include/rcc/CodeGen/InlineAsm.h index 492161f..15c0de6 100644 --- a/backend/include/rcc/CodeGen/InlineAsm.h +++ b/backend/include/rcc/CodeGen/InlineAsm.h @@ -28,8 +28,7 @@ enum class InlineAsmExtraInfoFlag : std::uint32_t { class InlineAsmExtraInfo final { public: - explicit constexpr InlineAsmExtraInfo(std::uint32_t RawValue = 0) noexcept - : Value(RawValue) {} + explicit constexpr InlineAsmExtraInfo(std::uint32_t RawValue = 0) noexcept : Value(RawValue) {} [[nodiscard]] constexpr std::uint32_t rawValue() const noexcept { return Value; @@ -193,8 +192,7 @@ public: } [[nodiscard]] constexpr std::optional matchingOperand() const noexcept { - return isMatched() ? std::optional((Value >> PayloadShift) & MatchedOperandMask) - : std::nullopt; + return isMatched() ? std::optional((Value >> PayloadShift) & MatchedOperandMask) : std::nullopt; } [[nodiscard]] constexpr std::optional registerClass() const noexcept { @@ -226,8 +224,7 @@ public: return true; } - [[nodiscard]] constexpr bool - setMemoryConstraint(InlineAsmMemoryConstraint Constraint) noexcept { + [[nodiscard]] constexpr bool setMemoryConstraint(InlineAsmMemoryConstraint Constraint) noexcept { const std::uint32_t Encoded = static_cast(Constraint); if (Encoded == 0 || Encoded > MatchedOperandMask || isMatched()) return false; @@ -253,8 +250,7 @@ public: return std::nullopt; } - [[nodiscard]] static constexpr std::string_view - kindName(InlineAsmOperandKind Kind) noexcept { + [[nodiscard]] static constexpr std::string_view kindName(InlineAsmOperandKind Kind) noexcept { switch (Kind) { case InlineAsmOperandKind::RegisterUse: return "reguse"; diff --git a/backend/include/rcc/Target/RISCV/RISCVCallLowering.h b/backend/include/rcc/Target/RISCV/RISCVCallLowering.h deleted file mode 100644 index 445c922..0000000 --- a/backend/include/rcc/Target/RISCV/RISCVCallLowering.h +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef RCC_TARGET_RISCV_RISCVCALLLOWERING_H -#define RCC_TARGET_RISCV_RISCVCALLLOWERING_H - -#include "rcc/CodeGen/TargetCallLowering.h" -#include "rcc/Target/RISCV/RISCVInstructions.h" - -#include - -namespace rcc::riscv { - -enum class CallTargetFlag : std::uint32_t { - DirectCall = 1U << 0U, -}; - -enum class CallStackObjectFlag : std::uint64_t { - IncomingArgument = 1ULL << 0U, - OutgoingArgument = 1ULL << 1U, - ByValCopy = 1ULL << 2U, - VarArgSaveArea = 1ULL << 3U, - VarArgAlignmentPadding = 1ULL << 4U, -}; - -enum class CallValueTransform : std::uint8_t { - Full, - AnyExtend, - SignExtend, - ZeroExtend, - Truncate, - Bitcast, -}; - -[[nodiscard]] constexpr CallingConventionID -callingConventionID(CallingConvention Convention) noexcept { - return CallingConventionID::fromValue(static_cast(Convention)); -} - -[[nodiscard]] const TargetCallLowering &callLowering() noexcept; - -} // namespace rcc::riscv - -#endif // RCC_TARGET_RISCV_RISCVCALLLOWERING_H diff --git a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h deleted file mode 100644 index 37f1c97..0000000 --- a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef RCC_TARGET_RISCV_RISCVCODEGEN_H -#define RCC_TARGET_RISCV_RISCVCODEGEN_H - -#include "rcc/CodeGen/GlobalISel/InstructionSelector.h" -#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" -#include "rcc/CodeGen/MachineCombinerPattern.h" -#include "rcc/CodeGen/MachineOptimizationPipeline.h" -#include "rcc/CodeGen/RegisterBankInfo.h" -#include "rcc/CodeGen/TargetFrameLowering.h" -#include "rcc/CodeGen/TargetMemoryFolding.h" -#include "rcc/CodeGen/TargetPseudoExpansion.h" -#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" -#include "rcc/CodeGen/TargetSchedulingModel.h" -#include "rcc/Target/RISCV/RISCVSubtarget.h" - -#include - -namespace rcc::riscv { - -[[nodiscard]] const LegalizerInfo &legalizerInfo() noexcept; -[[nodiscard]] std::unique_ptr createLegalizerInfo(RISCVSubtargetFeatures Features); -[[nodiscard]] const RegisterBankInfo ®isterBankInfo() noexcept; -[[nodiscard]] const InstructionSelector &instructionSelector() noexcept; -[[nodiscard]] const TargetRegisterAllocationInfo ®isterAllocationInfo() noexcept; -[[nodiscard]] const TargetFrameLowering &frameLowering() noexcept; -[[nodiscard]] const TargetSchedulingModel &schedulingModel() noexcept; -[[nodiscard]] const TargetMachineCombinerInfo &machineCombinerInfo() noexcept; -[[nodiscard]] const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept; -[[nodiscard]] const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept; -[[nodiscard]] MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept; - -} // namespace rcc::riscv - -#endif // RCC_TARGET_RISCV_RISCVCODEGEN_H diff --git a/backend/include/rcc/Target/RISCV/RISCVFunctionInfo.h b/backend/include/rcc/Target/RISCV/RISCVFunctionInfo.h index 2720830..6a05096 100644 --- a/backend/include/rcc/Target/RISCV/RISCVFunctionInfo.h +++ b/backend/include/rcc/Target/RISCV/RISCVFunctionInfo.h @@ -19,8 +19,7 @@ namespace riscv { class RISCVFunctionInfo final { public: - RISCVFunctionInfo(std::int32_t VarArgsFrameIndex = 0, - std::int32_t VarArgsSaveSize = 0) noexcept + RISCVFunctionInfo(std::int32_t VarArgsFrameIndex = 0, std::int32_t VarArgsSaveSize = 0) noexcept : VarArgsFrameIndex(VarArgsFrameIndex), VarArgsSaveSize(VarArgsSaveSize) {} [[nodiscard]] static constexpr TargetFunctionStateTypeID stableTypeID() noexcept { @@ -32,12 +31,11 @@ public: } [[nodiscard]] Expected verify() const; - [[nodiscard]] Expected - clone(const MachineFunctionCloneContext &Context) const; + [[nodiscard]] Expected clone(const MachineFunctionCloneContext &Context) const; [[nodiscard]] std::string print() const; [[nodiscard]] Expected> encode() const; - [[nodiscard]] static Expected - decode(std::uint32_t Version, std::span Payload); + [[nodiscard]] static Expected decode(std::uint32_t Version, + std::span Payload); [[nodiscard]] std::int32_t varArgsFrameIndex() const noexcept { return VarArgsFrameIndex; @@ -47,8 +45,7 @@ public: return VarArgsSaveSize; } - friend bool operator==(const RISCVFunctionInfo &Left, - const RISCVFunctionInfo &Right) = default; + friend bool operator==(const RISCVFunctionInfo &Left, const RISCVFunctionInfo &Right) = default; private: std::int32_t VarArgsFrameIndex = 0; diff --git a/backend/include/rcc/Target/RISCV/RISCVPipeline.h b/backend/include/rcc/Target/RISCV/RISCVPipeline.h index 9d871a9..ba0428b 100644 --- a/backend/include/rcc/Target/RISCV/RISCVPipeline.h +++ b/backend/include/rcc/Target/RISCV/RISCVPipeline.h @@ -19,17 +19,13 @@ class TargetDescription; namespace riscv { -[[nodiscard]] Expected addCanonicalPipeline(MachinePassManager &Pipeline, - const MachineFunction &Function); [[nodiscard]] Expected addCanonicalPipeline(MachinePassManager &Pipeline, const MachineFunction &Function, - RISCVSubtargetFeatures Features); -[[nodiscard]] Expected> -createCanonicalPipelinePass(std::string_view Name); + const RISCVSubtarget &Subtarget); [[nodiscard]] Expected> -createCanonicalPipelinePass(std::string_view Name, RISCVSubtargetFeatures Features); +createCanonicalPipelinePass(std::string_view Name, const RISCVSubtarget &Subtarget); [[nodiscard]] Expected> -canonicalPipelinePassNames(const MachineFunction &Function); +canonicalPipelinePassNames(const MachineFunction &Function, const RISCVSubtarget &Subtarget); } // namespace riscv } // namespace rcc diff --git a/backend/include/rcc/Target/RISCV/RISCVSubtarget.h b/backend/include/rcc/Target/RISCV/RISCVSubtarget.h index d5f088f..28718f0 100644 --- a/backend/include/rcc/Target/RISCV/RISCVSubtarget.h +++ b/backend/include/rcc/Target/RISCV/RISCVSubtarget.h @@ -6,10 +6,19 @@ #include "rcc/Support/Error.h" #include +#include #include #include -namespace rcc::riscv { +namespace rcc { + +class InstructionSelector; +class LegalizerInfo; +class TargetCallLowering; + +namespace riscv { + +class RISCVRegisterBankInfo; enum class RISCVExtension : std::uint8_t { I, @@ -42,6 +51,33 @@ private: std::uint16_t Enabled = 0; }; -} // namespace rcc::riscv +class RISCVSubtarget final { +public: + explicit RISCVSubtarget(RISCVSubtargetFeatures Features); + RISCVSubtarget(const RISCVSubtarget &) = delete; + RISCVSubtarget &operator=(const RISCVSubtarget &) = delete; + RISCVSubtarget(RISCVSubtarget &&) = delete; + RISCVSubtarget &operator=(RISCVSubtarget &&) = delete; + ~RISCVSubtarget(); + + [[nodiscard]] const RISCVSubtargetFeatures &features() const noexcept { + return Features; + } + + [[nodiscard]] const TargetCallLowering *getCallLowering() const; + [[nodiscard]] InstructionSelector *getInstructionSelector() const; + [[nodiscard]] const LegalizerInfo *getLegalizerInfo() const; + [[nodiscard]] const RISCVRegisterBankInfo *getRegBankInfo() const; + +private: + RISCVSubtargetFeatures Features; + mutable std::unique_ptr CallLoweringInfo; + mutable std::unique_ptr InstSelector; + mutable std::unique_ptr Legalizer; + mutable std::unique_ptr RegBankInfo; +}; + +} // namespace riscv +} // namespace rcc #endif // RCC_TARGET_RISCV_RISCVSUBTARGET_H diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index fa00537..2c99939 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -396,10 +396,9 @@ Expected InstructionSet::validateExplicitOperands(const InstructionDescrip const OperandDescriptor &OperandDesc = descriptorForOperand(Descriptor, *Shape, Index); const auto Validation = validateOperand(OperandDesc, Operand, RegInfo); if (!Validation) { - return makeUnexpected( - Validation.error().code(), - "operand " + std::to_string(Index) + " of '" + Descriptor.Name + "': " + - std::string(Validation.error().message())); + return makeUnexpected(Validation.error().code(), + "operand " + std::to_string(Index) + " of '" + Descriptor.Name + + "': " + std::string(Validation.error().message())); } if (Operand.isRegister()) { if (isDefRole(OperandDesc.RegisterRole)) { diff --git a/backend/lib/CodeGen/MachineFunctionCloner.cpp b/backend/lib/CodeGen/MachineFunctionCloner.cpp index 1a66e66..bbd70b7 100644 --- a/backend/lib/CodeGen/MachineFunctionCloner.cpp +++ b/backend/lib/CodeGen/MachineFunctionCloner.cpp @@ -203,8 +203,7 @@ cloneMemOperand(const MachineMemOperand &SourceOperand, CloneState &State) { return std::unexpected(DestinationReg.error()); if (SourceInfo.Name) { - auto DestinationInfo = - State.Destination.registerInfo().virtualRegisterInfo(*DestinationReg); + auto DestinationInfo = State.Destination.registerInfo().virtualRegisterInfo(*DestinationReg); if (!DestinationInfo) return std::unexpected(DestinationInfo.error()); DestinationInfo->get().Name = SourceInfo.Name; diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp index 3d9fbdb..00be72f 100644 --- a/backend/lib/CodeGen/MachineVerifier.cpp +++ b/backend/lib/CodeGen/MachineVerifier.cpp @@ -30,7 +30,7 @@ namespace rcc { namespace { [[nodiscard]] std::string formatDebugInstruction(const MachineInstr &Instruction, - const TargetDescription &Target) { + const TargetDescription &Target) { const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); std::string Result = Descriptor == nullptr ? "" : Descriptor->Name; for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { diff --git a/backend/lib/MIR/EmbeddedIR.cpp b/backend/lib/MIR/EmbeddedIR.cpp index 93cd534..25aea05 100644 --- a/backend/lib/MIR/EmbeddedIR.cpp +++ b/backend/lib/MIR/EmbeddedIR.cpp @@ -741,8 +741,8 @@ private: bool InQuote = false; for (std::size_t Position = Begin; Position < End; ++Position) { const char Character = Line[Position]; - if (InQuote && Character == '\\' && Position + 2 < End && - isHexDigit(Line[Position + 1]) && isHexDigit(Line[Position + 2])) { + if (InQuote && Character == '\\' && Position + 2 < End && isHexDigit(Line[Position + 1]) && + isHexDigit(Line[Position + 2])) { Position += 2; continue; } diff --git a/backend/lib/MIR/MIRFunctionRecordsParser.cpp b/backend/lib/MIR/MIRFunctionRecordsParser.cpp index aed6947..04857bb 100644 --- a/backend/lib/MIR/MIRFunctionRecordsParser.cpp +++ b/backend/lib/MIR/MIRFunctionRecordsParser.cpp @@ -240,16 +240,14 @@ bool MIRDocumentParser::parseFunctionRecords(const yaml::Node &Root, MachineFunc TargetNode->scalarStyle() == yaml::ScalarStyle::DoubleQuoted) { ++ContentOffset; } - const auto ErrorRange = [this, ContentOffset](std::size_t Begin, - std::size_t End = 1U) { + const auto ErrorRange = [this, ContentOffset](std::size_t Begin, std::size_t End = 1U) { return Result.Source.range(ContentOffset + Begin, ContentOffset + Begin + End); }; const auto SkipSpace = [&Text](std::size_t &Cursor) { while (Cursor < Text.size() && (Text[Cursor] == ' ' || Text[Cursor] == '\t')) ++Cursor; }; - const auto ParseNumber = [&Text, &SkipSpace](std::size_t &Cursor, - std::uint32_t &Value) { + const auto ParseNumber = [&Text, &SkipSpace](std::size_t &Cursor, std::uint32_t &Value) { SkipSpace(Cursor); const std::size_t Begin = Cursor; while (Cursor < Text.size() && Text[Cursor] >= '0' && Text[Cursor] <= '9') diff --git a/backend/lib/MIR/MIRMetadataParser.cpp b/backend/lib/MIR/MIRMetadataParser.cpp index 911edb6..63727ee 100644 --- a/backend/lib/MIR/MIRMetadataParser.cpp +++ b/backend/lib/MIR/MIRMetadataParser.cpp @@ -187,8 +187,7 @@ bool MIRDocumentParser::parseMetadata(const yaml::Node &Root, MachineFunction &F MILexer Lexer("machine-metadata", std::string(Text)); MIToken Token = Lexer.next(); - const auto LexerFailed = [this, &Lexer, DefinitionNode, &TranslateRange, - &Token]() -> bool { + const auto LexerFailed = [this, &Lexer, DefinitionNode, &TranslateRange, &Token]() -> bool { if (!Token.is(MITokenKind::Error)) return false; const auto Diagnostics = Lexer.diagnostics(); @@ -203,7 +202,7 @@ bool MIRDocumentParser::parseMetadata(const yaml::Node &Root, MachineFunction &F return LexerFailed(); }; const auto FailAtToken = [this, DefinitionNode, &TranslateRange](const MIToken &At, - std::string Message) { + std::string Message) { return fail(TranslateRange(*DefinitionNode, At.range()), std::move(Message)); }; @@ -214,8 +213,7 @@ bool MIRDocumentParser::parseMetadata(const yaml::Node &Root, MachineFunction &F if (Advance()) return true; std::uint32_t Slot = 0; - if (!Token.is(MITokenKind::IntegerLiteral) || - !parseUnsigned32(Token.integerValue(), Slot)) { + if (!Token.is(MITokenKind::IntegerLiteral) || !parseUnsigned32(Token.integerValue(), Slot)) { return FailAtToken(Token, "expected metadata id after '!'"); } if (Advance()) diff --git a/backend/lib/Support/YAML.cpp b/backend/lib/Support/YAML.cpp index 1d67114..1ee7131 100644 --- a/backend/lib/Support/YAML.cpp +++ b/backend/lib/Support/YAML.cpp @@ -247,7 +247,7 @@ private: } void diagnose(std::size_t Begin, std::size_t End, std::string Message, - std::string SourceName = {}) { + std::string SourceName = {}) { if (Failed) return; Failed = true; @@ -255,8 +255,8 @@ private: .Severity = DiagnosticSeverity::Error, .Category = "yaml", .Message = std::move(Message), - .SourceName = SourceName.empty() ? std::string(Parsed.Source.name()) - : std::move(SourceName), + .SourceName = + SourceName.empty() ? std::string(Parsed.Source.name()) : std::move(SourceName), .Range = Parsed.Source.range(Begin, End), }); } diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index d28cdf3..5783e91 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -47,9 +47,11 @@ set_source_files_properties( add_library( RCCBackendRISCV + GISel/RISCVCallLowering.cpp + GISel/RISCVInstructionSelector.cpp + GISel/RISCVLegalizerInfo.cpp + GISel/RISCVRegisterBankInfo.cpp RISCVInstrInfo.cpp - RISCVCodeGen.cpp - RISCVCallLowering.cpp RISCVAssemblyPrinter.cpp RISCVFrameLowering.cpp RISCVFunctionInfo.cpp @@ -118,10 +120,16 @@ install( ) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVCallLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVCallLowering.h" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVInstructionSelector.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVLegalizerInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVLegalizerInfo.h" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVRegisterBankInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVRegisterBankInfo.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCV.h" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVInstrInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVAssemblyPrinter.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFunctionInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVImmediateMaterialization.cpp" @@ -141,10 +149,8 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVSubtarget.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTargetConstant.cpp" - "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVInstrInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVAssemblyPrinter.h" - "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVInstPrinter.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVAsmBackend.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVFixupKinds.h" @@ -159,10 +165,12 @@ rcc_backend_register_format_files( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTargetRelocationKinds.h" ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVCallLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVInstructionSelector.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVLegalizerInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISel/RISCVRegisterBankInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVInstrInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVAssemblyPrinter.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFunctionInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVImmediateMaterialization.cpp" diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp new file mode 100644 index 0000000..cfe8bb6 --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp @@ -0,0 +1,1495 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RISCVCallLowering.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::string_view PlanTarget = "riscv64-unknown-linux-gnu"; +constexpr std::uint64_t XLenBytes = 8; +constexpr std::uint64_t ABIStackAlignmentBytes = 16; +constexpr std::uint32_t AllowedDirectCallFlags = + static_cast(CallTargetFlag::DirectCall); + +[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::uint64_t stackObjectFlag(CallStackObjectFlag Flag) noexcept { + return static_cast(Flag); +} + +[[nodiscard]] Expected alignment(std::uint64_t Bytes) { + return Alignment::create(Bytes); +} + +[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t AlignmentBytes) { + const std::uint64_t Remainder = Value % AlignmentBytes; + if (Remainder == 0) + return Value; + const std::uint64_t Increment = AlignmentBytes - Remainder; + if (Value > std::numeric_limits::max() - Increment) { + return makeUnexpected(ErrorCode::Unsupported, "RISC-V ABI stack-area size overflows uint64"); + } + return Value + Increment; +} + +[[nodiscard]] std::uint64_t valueSizeBytes(LLT Type) noexcept { + return (static_cast(Type.scalarBitWidth()) + 7U) / 8U; +} + +[[nodiscard]] bool isSupportedFloat(LLT Type) noexcept { + if (!Type.isFloat()) + return false; + const auto Semantics = Type.floatSemantics(); + return Semantics && + (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +} + +[[nodiscard]] Expected floatingRegisterView(PhysicalRegister DoubleView, + LLT Type) { + if (!isSupportedFloat(Type)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V floating-register view requires f32 or f64"); + } + if (Type.scalarBitWidth() == 64) + return DoubleView; + + constexpr std::uint32_t FirstDouble = static_cast(PhysicalRegisterID::F0_D); + constexpr std::uint32_t LastDouble = static_cast(PhysicalRegisterID::F31_D); + constexpr std::uint32_t FirstSingle = static_cast(PhysicalRegisterID::F0_F); + const std::uint32_t Number = DoubleView.number(); + if (Number < FirstDouble || Number > LastDouble) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V ABI floating register is not an FPR64 view"); + } + return PhysicalRegister::create(FirstSingle + Number - FirstDouble); +} + +[[nodiscard]] bool requiresIntegerTransport(const ABIValue &Value) noexcept { + return !isSupportedFloat(Value.Type) || Value.Flags.has(ABIValueFlag::ByVal) || + Value.Flags.has(ABIValueFlag::StructReturn) || Value.Flags.has(ABIValueFlag::Indirect); +} + +[[nodiscard]] Expected verifyABIValue(const ABIValue &Value, bool RequireRegister) { + if (RequireRegister && Value.Value.isNoRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V ABI value requires a source or destination register"); + } + if (!Value.Type.isValid() || !Value.Type.isScalarValue() || Value.Type.scalarBitWidth() > 64) { + return makeUnexpected( + ErrorCode::Unsupported, + "RISC-V ABI accepts explicitly classified integer, pointer, f32, and f64 scalar fragments"); + } + if (Value.Type.isFloat() && !isSupportedFloat(Value.Type)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V LP64 ABIs only support f32 and f64 floating fragments"); + } + if (Value.Type.isPointer() && Value.Type.scalarBitWidth() != 64) { + return makeUnexpected(ErrorCode::Unsupported, "RISC-V LP64 ABIs require 64-bit pointers"); + } + if ((Value.Type.isInteger() || Value.Type.isAnyScalar()) && Value.Type.scalarBitWidth() < 64) { + if (Value.Extension == ABIExtensionKind::None) { + return makeUnexpected(ErrorCode::InvalidArgument, + "narrow RISC-V ABI integers require an explicit extension kind"); + } + } else if (Value.Extension != ABIExtensionKind::None) { + return makeUnexpected(ErrorCode::InvalidArgument, + "only narrow integer ABI fragments may carry an extension kind"); + } + if (Value.Extension >= ABIExtensionKind::Count || Value.FragmentCount == 0 || + Value.FragmentIndex >= Value.FragmentCount || Value.FragmentCount > 2) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V ABI fragment coordinates or extension are invalid"); + } + + const bool IsByVal = Value.Flags.has(ABIValueFlag::ByVal); + const bool IsSRet = Value.Flags.has(ABIValueFlag::StructReturn); + const bool IsIndirect = Value.Flags.has(ABIValueFlag::Indirect); + const unsigned SpecialFlagCount = static_cast(IsByVal) + static_cast(IsSRet) + + static_cast(IsIndirect); + if (SpecialFlagCount > 1) { + return makeUnexpected(ErrorCode::InvalidArgument, + "byval, sret, and indirect ABI flags are mutually exclusive"); + } + if ((IsByVal || IsSRet || IsIndirect) && + (!Value.Type.isPointer() || Value.FragmentCount != 1 || Value.FragmentIndex != 0 || + Value.FragmentOffsetBytes != 0)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "byval, sret, and indirect values require one explicit pointer fragment"); + } + if (IsByVal) { + if (Value.ByValSizeBytes == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "byval arguments require a nonzero copy size"); + } + } else if (Value.ByValSizeBytes != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-byval values cannot carry a byval copy size"); + } + return {}; +} + +[[nodiscard]] Expected verifyValueSequence(std::span Values, + bool RequireRegisters) { + std::size_t ExpectedOriginal = 0; + std::size_t Index = 0; + while (Index < Values.size()) { + const ABIValue &First = Values[Index]; + auto ValueVerification = verifyABIValue(First, RequireRegisters); + if (!ValueVerification) + return std::unexpected(ValueVerification.error()); + if (First.OriginalValue != ExpectedOriginal || First.FragmentIndex != 0 || + Index + First.FragmentCount > Values.size()) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "RISC-V ABI values must form contiguous, zero-based original-value groups"); + } + + std::uint64_t PreviousEnd = 0; + for (std::size_t Fragment = 0; Fragment < First.FragmentCount; ++Fragment) { + const ABIValue &Value = Values[Index + Fragment]; + auto FragmentVerification = verifyABIValue(Value, RequireRegisters); + if (!FragmentVerification) + return std::unexpected(FragmentVerification.error()); + if (Value.OriginalValue != First.OriginalValue || Value.FragmentIndex != Fragment || + Value.FragmentCount != First.FragmentCount || + Value.OriginalAlignment != First.OriginalAlignment || Value.Flags != First.Flags || + Value.ByValSizeBytes != First.ByValSizeBytes || + Value.ByValAlignment != First.ByValAlignment || Value.FragmentOffsetBytes < PreviousEnd) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V ABI fragments of one value have inconsistent metadata"); + } + PreviousEnd = + static_cast(Value.FragmentOffsetBytes) + valueSizeBytes(Value.Type); + } + Index += First.FragmentCount; + ++ExpectedOriginal; + } + return ExpectedOriginal; +} + +[[nodiscard]] CCAssignmentFlags assignmentFlags(const ABIValue &Value, bool IsVariadic, + bool IsBitcast) { + CCAssignmentFlags Result; + if (IsVariadic) + Result.set(CCAssignmentFlag::Variadic); + if (Value.Flags.has(ABIValueFlag::ByVal)) + Result.set(CCAssignmentFlag::ByVal); + if (Value.Flags.has(ABIValueFlag::StructReturn)) + Result.set(CCAssignmentFlag::StructReturn); + if (Value.Flags.has(ABIValueFlag::Indirect)) + Result.set(CCAssignmentFlag::Indirect); + if (Value.Flags.has(ABIValueFlag::InRegister)) + Result.set(CCAssignmentFlag::InRegister); + if (IsBitcast) + Result.set(CCAssignmentFlag::Bitcast); + return Result; +} + +[[nodiscard]] CCLocationKind locationKind(const ABIValue &Value, bool IsRegister) noexcept { + if (Value.Flags.has(ABIValueFlag::Indirect)) + return CCLocationKind::Indirect; + return IsRegister ? CCLocationKind::Register : CCLocationKind::Stack; +} + +struct ArgumentAssignmentResult final { + std::vector Assignments; + StackArgumentArea StackArea; + std::size_t FirstUnallocatedGPR = 0; +}; + +struct AssignmentState final { + std::size_t GPR = 0; + std::size_t FPR = 0; + std::uint64_t StackOffset = 0; + bool VariadicStackOnly = false; +}; + +[[nodiscard]] Expected registerAssignment(const ABIValue &Value, + std::size_t ValueIndex, + PhysicalRegister Location, LLT LocationType, + bool IsVariadic, bool IsBitcast) { + auto Align = + alignment(std::min(Value.OriginalAlignment.bytes(), ABIStackAlignmentBytes)); + if (!Align) + return std::unexpected(Align.error()); + return CCAssignment{ + .ValueIndex = ValueIndex, + .OriginalValue = Value.Value, + .OriginalValueIndex = Value.OriginalValue, + .FragmentIndex = Value.FragmentIndex, + .FragmentCount = Value.FragmentCount, + .ValueType = Value.Type, + .LocationType = LocationType, + .LocationKind = locationKind(Value, true), + .LocationRegister = Location, + .StackOffsetBytes = std::nullopt, + .LocationSizeBytes = valueSizeBytes(LocationType), + .StackSlotSizeBytes = XLenBytes, + .LocationAlignment = *Align, + .Extension = Value.Extension, + .CustomFlags = assignmentFlags(Value, IsVariadic, IsBitcast), + }; +} + +[[nodiscard]] Expected stackAssignment(const ABIValue &Value, std::size_t ValueIndex, + std::uint64_t StackOffset, LLT LocationType, + bool IsVariadic, + std::uint64_t StackAlignment) { + auto Align = alignment(StackAlignment); + if (!Align) + return std::unexpected(Align.error()); + return CCAssignment{ + .ValueIndex = ValueIndex, + .OriginalValue = Value.Value, + .OriginalValueIndex = Value.OriginalValue, + .FragmentIndex = Value.FragmentIndex, + .FragmentCount = Value.FragmentCount, + .ValueType = Value.Type, + .LocationType = LocationType, + .LocationKind = locationKind(Value, false), + .LocationRegister = std::nullopt, + .StackOffsetBytes = StackOffset, + .LocationSizeBytes = valueSizeBytes(LocationType), + .StackSlotSizeBytes = XLenBytes, + .LocationAlignment = *Align, + .Extension = Value.Extension, + .CustomFlags = assignmentFlags(Value, IsVariadic, false), + }; +} + +[[nodiscard]] Expected +assignArguments(std::span Values, std::size_t FixedArgumentCount, bool IsVariadic, + const CallingConventionInfo &Convention) { + auto I64 = LLT::makeInteger(64); + auto StackAlign = alignment(Convention.StackArgumentAlignmentBytes); + if (!I64 || !StackAlign) + return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid"); + + ArgumentAssignmentResult Result; + AssignmentState State; + State.StackOffset = Convention.ShadowSpaceBytes; + Result.Assignments.reserve(Values.size()); + + std::size_t Index = 0; + while (Index < Values.size()) { + const ABIValue &First = Values[Index]; + const std::size_t GroupSize = First.FragmentCount; + const bool VariadicArgument = IsVariadic && First.OriginalValue >= FixedArgumentCount; + bool HasPreferredFloat = false; + std::size_t RequiredGPRs = 0; + std::size_t RequiredFPRs = 0; + for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { + const ABIValue &Value = Values[Index + Fragment]; + if (!VariadicArgument && !requiresIntegerTransport(Value)) { + HasPreferredFloat = true; + ++RequiredFPRs; + } else { + ++RequiredGPRs; + } + } + + const bool CanUsePreferredAggregate = + HasPreferredFloat && + State.GPR + RequiredGPRs <= Convention.IntegerArgumentRegisters.size() && + State.FPR + RequiredFPRs <= Convention.FloatingArgumentRegisters.size(); + if (CanUsePreferredAggregate) { + for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { + const ABIValue &Value = Values[Index + Fragment]; + Expected Assignment = makeUnexpected( + ErrorCode::InternalError, "RISC-V ABI argument assignment was not initialized"); + if (requiresIntegerTransport(Value)) { + Assignment = registerAssignment(Value, Index + Fragment, + Convention.IntegerArgumentRegisters[State.GPR++], *I64, + false, false); + } else { + auto View = + floatingRegisterView(Convention.FloatingArgumentRegisters[State.FPR++], Value.Type); + if (!View) + return std::unexpected(View.error()); + Assignment = registerAssignment(Value, Index + Fragment, *View, Value.Type, false, false); + } + if (!Assignment) + return std::unexpected(Assignment.error()); + Result.Assignments.push_back(*Assignment); + } + Index += GroupSize; + continue; + } + + const bool RequiresAlignedPair = + VariadicArgument && GroupSize == 2 && First.OriginalAlignment.bytes() >= 16; + if (RequiresAlignedPair && State.GPR % 2 != 0 && + State.GPR < Convention.IntegerArgumentRegisters.size()) { + ++State.GPR; + } + bool ForceEntireGroupToStack = State.VariadicStackOnly; + if (RequiresAlignedPair && State.GPR + GroupSize > Convention.IntegerArgumentRegisters.size()) { + ForceEntireGroupToStack = true; + State.GPR = Convention.IntegerArgumentRegisters.size(); + } + + bool FirstStackFragment = true; + for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { + const ABIValue &Value = Values[Index + Fragment]; + const bool HasGPR = + !ForceEntireGroupToStack && State.GPR < Convention.IntegerArgumentRegisters.size(); + if (HasGPR) { + const bool Bitcast = isSupportedFloat(Value.Type); + auto Assignment = registerAssignment(Value, Index + Fragment, + Convention.IntegerArgumentRegisters[State.GPR++], *I64, + VariadicArgument, Bitcast); + if (!Assignment) + return std::unexpected(Assignment.error()); + Result.Assignments.push_back(*Assignment); + continue; + } + + if (Value.Flags.has(ABIValueFlag::InRegister)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V inreg argument cannot be assigned to the stack"); + } + std::uint64_t RequiredAlignment = XLenBytes; + if (FirstStackFragment && ForceEntireGroupToStack) { + RequiredAlignment = std::min( + std::max(XLenBytes, First.OriginalAlignment.bytes()), + ABIStackAlignmentBytes); + } + auto AlignedOffset = alignTo(State.StackOffset, RequiredAlignment); + if (!AlignedOffset) + return std::unexpected(AlignedOffset.error()); + State.StackOffset = *AlignedOffset; + const LLT LocationType = isSupportedFloat(Value.Type) ? Value.Type : *I64; + auto Assignment = stackAssignment(Value, Index + Fragment, State.StackOffset, LocationType, + VariadicArgument, RequiredAlignment); + if (!Assignment) + return std::unexpected(Assignment.error()); + Result.Assignments.push_back(*Assignment); + if (State.StackOffset > std::numeric_limits::max() - XLenBytes) { + return makeUnexpected(ErrorCode::Unsupported, "RISC-V argument area overflows uint64"); + } + State.StackOffset += XLenBytes; + FirstStackFragment = false; + if (VariadicArgument) + State.VariadicStackOnly = true; + } + Index += GroupSize; + } + + auto FinalSize = alignTo(State.StackOffset, Convention.StackArgumentAlignmentBytes); + if (!FinalSize) + return std::unexpected(FinalSize.error()); + Result.StackArea = StackArgumentArea{*FinalSize, Convention.ShadowSpaceBytes, *StackAlign}; + Result.FirstUnallocatedGPR = State.GPR; + return Result; +} + +[[nodiscard]] Expected> +assignReturns(std::span Values, bool RequireRegisters, + const CallingConventionInfo &Convention) { + auto Sequence = verifyValueSequence(Values, RequireRegisters); + if (!Sequence) + return std::unexpected(Sequence.error()); + if (Values.size() > 2) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V direct returns are limited to two explicit fragments"); + } + auto I64 = LLT::makeInteger(64); + if (!I64) + return std::unexpected(I64.error()); + + std::vector Result; + Result.reserve(Values.size()); + std::size_t GPR = 0; + std::size_t FPR = 0; + for (std::size_t Index = 0; Index < Values.size(); ++Index) { + const ABIValue &Value = Values[Index]; + if (Value.Flags.rawValue() != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "return fragments cannot carry argument-only ABI flags"); + } + const bool UseFloatingRegister = + isSupportedFloat(Value.Type) && FPR < Convention.FloatingReturnRegisters.size(); + Expected Assignment = makeUnexpected( + ErrorCode::Unsupported, "RISC-V calling convention has no return register available"); + if (UseFloatingRegister) { + auto View = floatingRegisterView(Convention.FloatingReturnRegisters[FPR++], Value.Type); + if (!View) + return std::unexpected(View.error()); + Assignment = registerAssignment(Value, Index, *View, Value.Type, false, false); + } else if (GPR < Convention.IntegerReturnRegisters.size()) { + Assignment = registerAssignment(Value, Index, Convention.IntegerReturnRegisters[GPR++], *I64, + false, isSupportedFloat(Value.Type)); + } + if (!Assignment) + return std::unexpected(Assignment.error()); + Result.push_back(*Assignment); + } + return Result; +} + +[[nodiscard]] bool hasStructReturn(std::span Arguments) noexcept { + return std::ranges::any_of( + Arguments, [](const ABIValue &Value) { return Value.Flags.has(ABIValueFlag::StructReturn); }); +} + +[[nodiscard]] bool equivalentReturnLocations(std::span Left, + std::span Right) noexcept { + if (Left.size() != Right.size()) + return false; + for (std::size_t Index = 0; Index < Left.size(); ++Index) { + if (Left[Index].ValueType != Right[Index].ValueType || + Left[Index].LocationType != Right[Index].LocationType || + Left[Index].LocationKind != Right[Index].LocationKind || + Left[Index].LocationRegister != Right[Index].LocationRegister || + Left[Index].Extension != Right[Index].Extension || + Left[Index].CustomFlags != Right[Index].CustomFlags) { + return false; + } + } + return true; +} + +[[nodiscard]] Expected +tailCallRejection(const CallLoweringRequest &Request, const ArgumentAssignmentResult &Arguments, + std::span Results, + std::span ByValCopies, + const CallingConventionInfo &Convention) { + if (!Convention.SupportsTailCalls) + return std::string("the calling convention does not support tail calls"); + if (!Request.TailContext) + return std::string("tail-call context is missing"); + const TailCallContext &Context = *Request.TailContext; + if (Context.CallerConvention != Request.Convention) + return std::string("caller and callee calling conventions differ"); + if (Context.CallerIsVariadic != Request.IsVariadic || + (Request.IsVariadic && Context.CallerFixedArgumentCount != Request.FixedArgumentCount)) { + return std::string("caller and callee variadic signatures differ"); + } + if (Context.CallerHasDynamicStackAllocation) + return std::string("the caller has a dynamic stack allocation"); + if (!ByValCopies.empty()) + return std::string("tail calls cannot create byval copies"); + if (Context.CallerUsesStructReturn != hasStructReturn(Request.Arguments)) + return std::string("caller and callee sret conventions differ"); + if (Arguments.StackArea.SizeBytes > Context.ReusableIncomingArgumentBytes) + return std::string("outgoing stack arguments exceed the reusable incoming area"); + if (std::ranges::any_of(Request.Results, + [](const ABIValue &Value) { return !Value.Value.isNoRegister(); })) { + return std::string("tail-call result signatures must not request local result copies"); + } + auto CallerReturns = assignReturns(Context.CallerReturns, false, Convention); + if (!CallerReturns) + return std::unexpected(CallerReturns.error()); + if (!equivalentReturnLocations(Results, *CallerReturns)) + return std::string("caller and callee return locations differ"); + return std::string{}; +} + +[[nodiscard]] Expected +riscvConvention(CallingConventionID Convention) { + CallingConvention TargetConvention = CallingConvention::None; + if (Convention == callingConventionID(CallingConvention::LP64D)) + TargetConvention = CallingConvention::LP64D; + else if (Convention == callingConventionID(CallingConvention::LP64)) + TargetConvention = CallingConvention::LP64; + else + return makeUnexpected(ErrorCode::Unsupported, "RISC-V backend only supports LP64 and LP64D"); + const CallingConventionInfo *Info = callingConvention(TargetConvention); + if (Info == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V calling-convention target facts are unavailable"); + return Info; +} + +} // namespace + +Expected +RISCVCallLowering::planFormalArguments(const FormalArgumentRequest &Request) const { + auto Convention = riscvConvention(Request.Convention); + if (!Convention) + return std::unexpected(Convention.error()); + auto OriginalCount = verifyValueSequence(Request.Arguments, true); + if (!OriginalCount) + return std::unexpected(OriginalCount.error()); + if (Request.FixedArgumentCount != *OriginalCount) { + return makeUnexpected(ErrorCode::InvalidArgument, + "formal-argument request must identify every declared argument as fixed"); + } + if (!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-variadic formal request has an invalid fixed-argument boundary"); + } + if (hasStructReturn(Request.Arguments) && + (Request.Arguments.empty() || Request.Arguments.front().OriginalValue != 0 || + !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V sret must be the first formal argument"); + } + + auto Assignments = assignArguments(Request.Arguments, Request.FixedArgumentCount, + Request.IsVariadic, **Convention); + if (!Assignments) + return std::unexpected(Assignments.error()); + + VarArgLoweringState VarArgs; + if (Request.IsVariadic) { + VarArgs.IsVariadic = true; + VarArgs.FixedArgumentCount = Request.FixedArgumentCount; + VarArgs.FirstUnallocatedIntegerRegister = Assignments->FirstUnallocatedGPR; + VarArgs.OverflowAreaOffsetBytes = Assignments->StackArea.SizeBytes; + const std::size_t First = Assignments->FirstUnallocatedGPR; + const std::size_t RegisterCount = (*Convention)->IntegerArgumentRegisters.size(); + VarArgs.RegisterSaveAreaSizeBytes = (RegisterCount - First) * XLenBytes; + VarArgs.RegisterSaveAreaOffsetBytes = + VarArgs.RegisterSaveAreaSizeBytes == 0 + ? static_cast(VarArgs.OverflowAreaOffsetBytes) + : -static_cast(VarArgs.RegisterSaveAreaSizeBytes); + if (VarArgs.RegisterSaveAreaSizeBytes != 0 && First % 2 != 0) + VarArgs.RegisterSavePaddingBytes = XLenBytes; + for (std::size_t Index = First; Index < RegisterCount; ++Index) { + VarArgs.RegistersToSave.push_back( + {(*Convention)->IntegerArgumentRegisters[Index], (Index - First) * XLenBytes}); + } + } + + CallLoweringPlanDesc Plan; + Plan.Kind = CallLoweringPlanKind::FormalArguments; + Plan.Target = PlanTarget; + Plan.Convention = Request.Convention; + Plan.ArgumentAssignments = std::move(Assignments->Assignments); + Plan.StackArguments = Assignments->StackArea; + Plan.VarArgs = std::move(VarArgs); + return CallLoweringPlan::create(std::move(Plan)); +} + +Expected RISCVCallLowering::planCall(const CallLoweringRequest &Request) const { + auto Convention = riscvConvention(Request.Convention); + if (!Convention) + return std::unexpected(Convention.error()); + auto OriginalCount = verifyValueSequence(Request.Arguments, true); + if (!OriginalCount) + return std::unexpected(OriginalCount.error()); + if ((!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) || + (Request.IsVariadic && Request.FixedArgumentCount > *OriginalCount)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call request has an invalid fixed/variadic argument boundary"); + } + if (!Request.Callee.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "call request requires a valid callee"); + if (hasStructReturn(Request.Arguments) && + (Request.Arguments.empty() || + !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V sret must be the first call argument"); + } + if (hasStructReturn(Request.Arguments) && !Request.Results.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V sret calls cannot also request direct return values"); + } + + MachineCallTarget EffectiveCallee = Request.Callee; + switch (Request.Callee.kind()) { + case MachineCallTargetKind::GlobalSymbol: { + if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "direct RISC-V call uses unknown target flags"); + } + auto Callee = MachineCallTarget::globalSymbol( + std::string(Request.Callee.symbolName()), + Request.Callee.targetFlags() | static_cast(CallTargetFlag::DirectCall)); + if (!Callee) + return std::unexpected(Callee.error()); + EffectiveCallee = std::move(*Callee); + break; + } + case MachineCallTargetKind::ExternalSymbol: { + if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "direct RISC-V call uses unknown target flags"); + } + auto Callee = MachineCallTarget::externalSymbol( + std::string(Request.Callee.symbolName()), + Request.Callee.targetFlags() | static_cast(CallTargetFlag::DirectCall)); + if (!Callee) + return std::unexpected(Callee.error()); + EffectiveCallee = std::move(*Callee); + break; + } + case MachineCallTargetKind::IndirectRegister: + if (Request.Callee.targetFlags() != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "indirect RISC-V calls cannot carry direct-call target flags"); + } + break; + case MachineCallTargetKind::Invalid: + return makeUnexpected(ErrorCode::InvalidArgument, "call target is invalid"); + } + + const bool RequireResultRegisters = Request.TailKind == TailCallKind::NotTail; + auto ResultSequence = verifyValueSequence(Request.Results, RequireResultRegisters); + if (!ResultSequence) + return std::unexpected(ResultSequence.error()); + auto Arguments = assignArguments(Request.Arguments, Request.FixedArgumentCount, + Request.IsVariadic, **Convention); + if (!Arguments) + return std::unexpected(Arguments.error()); + auto Results = assignReturns(Request.Results, RequireResultRegisters, **Convention); + if (!Results) + return std::unexpected(Results.error()); + + auto Align16 = alignment(ABIStackAlignmentBytes); + if (!Align16) + return std::unexpected(Align16.error()); + std::uint64_t CallFrameCursor = Arguments->StackArea.SizeBytes; + std::vector ByValCopies; + for (std::size_t Index = 0; Index < Arguments->Assignments.size(); ++Index) { + const CCAssignment &Assignment = Arguments->Assignments[Index]; + if (!Assignment.CustomFlags.has(CCAssignmentFlag::ByVal)) + continue; + const ABIValue &Value = Request.Arguments[Assignment.ValueIndex]; + const std::uint64_t CopyAlignment = + std::min(Value.ByValAlignment.bytes(), ABIStackAlignmentBytes); + auto CopyOffset = alignTo(CallFrameCursor, CopyAlignment); + if (!CopyOffset) + return std::unexpected(CopyOffset.error()); + if (*CopyOffset > std::numeric_limits::max() - Value.ByValSizeBytes) { + return makeUnexpected(ErrorCode::Unsupported, "RISC-V byval area overflows uint64"); + } + auto CopyAlign = alignment(CopyAlignment); + if (!CopyAlign) + return std::unexpected(CopyAlign.error()); + ByValCopies.push_back( + {Index, Value.Value, Value.Type, *CopyOffset, Value.ByValSizeBytes, *CopyAlign}); + CallFrameCursor = *CopyOffset + Value.ByValSizeBytes; + } + auto CallFrameSize = alignTo(CallFrameCursor, ABIStackAlignmentBytes); + if (!CallFrameSize) + return std::unexpected(CallFrameSize.error()); + + TailCallKind Decision = TailCallKind::NotTail; + std::string RejectionReason; + if (Request.TailKind != TailCallKind::NotTail) { + auto Rejection = tailCallRejection(Request, *Arguments, *Results, ByValCopies, **Convention); + if (!Rejection) + return std::unexpected(Rejection.error()); + RejectionReason = std::move(*Rejection); + if (RejectionReason.empty()) { + Decision = Request.TailKind; + } else if (Request.TailKind == TailCallKind::MustTail) { + return makeUnexpected(ErrorCode::Unsupported, + "musttail call is not ABI-compatible: " + RejectionReason); + } else { + auto NormalResults = verifyValueSequence(Request.Results, true); + if (!NormalResults) + return std::unexpected(NormalResults.error()); + } + } + + CallLoweringPlanDesc Plan; + Plan.Kind = CallLoweringPlanKind::Call; + Plan.Target = PlanTarget; + Plan.Convention = Request.Convention; + Plan.ArgumentAssignments = std::move(Arguments->Assignments); + Plan.ReturnAssignments = std::move(*Results); + Plan.StackArguments = Arguments->StackArea; + Plan.CallFrame = CallFrameRequirements{ + Decision == TailCallKind::NotTail ? *CallFrameSize : Arguments->StackArea.SizeBytes, *Align16, + Decision == TailCallKind::NotTail, Decision == TailCallKind::NotTail, + Decision != TailCallKind::NotTail}; + if (Request.IsVariadic) { + Plan.VarArgs.IsVariadic = true; + Plan.VarArgs.FixedArgumentCount = Request.FixedArgumentCount; + } + Plan.ByValCopies = std::move(ByValCopies); + Plan.PreservedMask = createCallPreservedMask((*Convention)->ID); + if (!Plan.PreservedMask) + return makeUnexpected(ErrorCode::InternalError, "RISC-V call-preserved mask is unavailable"); + Plan.Callee = std::move(EffectiveCallee); + Plan.TailDecision = Decision; + Plan.TailRejectionReason = std::move(RejectionReason); + return CallLoweringPlan::create(std::move(Plan)); +} + +Expected +RISCVCallLowering::planReturn(const ReturnLoweringRequest &Request) const { + auto Convention = riscvConvention(Request.Convention); + if (!Convention) + return std::unexpected(Convention.error()); + auto Values = assignReturns(Request.Values, true, **Convention); + if (!Values) + return std::unexpected(Values.error()); + + CallLoweringPlanDesc Plan; + Plan.Kind = CallLoweringPlanKind::Return; + Plan.Target = PlanTarget; + Plan.Convention = Request.Convention; + Plan.ReturnAssignments = std::move(*Values); + return CallLoweringPlan::create(std::move(Plan)); +} + +[[nodiscard]] CallValueTransform outgoingTransform(const CCAssignment &Assignment) noexcept { + if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast)) + return CallValueTransform::Bitcast; + switch (Assignment.Extension) { + case ABIExtensionKind::None: + return CallValueTransform::Full; + case ABIExtensionKind::AnyExtend: + return CallValueTransform::AnyExtend; + case ABIExtensionKind::SignExtend: + return CallValueTransform::SignExtend; + case ABIExtensionKind::ZeroExtend: + return CallValueTransform::ZeroExtend; + case ABIExtensionKind::Count: + break; + } + return CallValueTransform::Full; +} + +[[nodiscard]] CallValueTransform incomingTransform(const CCAssignment &Assignment) noexcept { + if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast)) + return CallValueTransform::Bitcast; + if (Assignment.ValueType.scalarBitWidth() < Assignment.LocationType.scalarBitWidth()) + return CallValueTransform::Truncate; + return CallValueTransform::Full; +} + +[[nodiscard]] Expected addUnsignedImmediate(MachineInstrBuilder &Builder, + std::uint64_t Value) { + auto Immediate = IntegerImmediate::fromUnsigned(64, Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Builder.addIntegerImmediate(std::move(*Immediate)); +} + +[[nodiscard]] Expected +createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::uint64_t SizeBytes, + Alignment Align, MemoryBase Base, std::int64_t Offset = 0) { + auto Size = MemorySize::fixed(SizeBytes); + if (!Size) + return std::unexpected(Size.error()); + MachineMemOperandDesc Description; + Description.Accesses = Accesses; + Description.Size = *Size; + Description.Align = Align; + Description.Base = std::move(Base); + Description.Offset = Offset; + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] Expected emitCCCopy(MachineIRBuilder &Builder, Register Destination, + Register Source, CallValueTransform Transform) { + const Opcode CopyOpcode = Transform == CallValueTransform::Full + ? opcode(GenericOpcode::COPY) + : targetOpcode(InstructionOpcode::PseudoCC_COPY); + auto Instruction = Builder.buildInstr(CopyOpcode); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Def = Instruction->addDef(Destination); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Instruction->addUse(Source); + if (!Use) + return std::unexpected(Use.error()); + if (Transform != CallValueTransform::Full) { + auto Action = Instruction->addImmediate(static_cast(Transform)); + if (!Action) + return std::unexpected(Action.error()); + } + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected emitStackLoad(MachineIRBuilder &Builder, + const CCAssignment &Assignment, FrameIndex Object, + std::uint64_t ObjectOffset, + const MachineMemOperand &Memory) { + auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoLOAD_STACK_ARG)); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Def = Instruction->addDef(Assignment.OriginalValue); + if (!Def) + return std::unexpected(Def.error()); + auto Frame = Instruction->addFrameIndex(Object); + if (!Frame) + return std::unexpected(Frame.error()); + auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset); + if (!Offset) + return std::unexpected(Offset.error()); + auto Transform = + Instruction->addImmediate(static_cast(incomingTransform(Assignment))); + if (!Transform) + return std::unexpected(Transform.error()); + auto Mem = Instruction->addMemOperand(Memory); + if (!Mem) + return std::unexpected(Mem.error()); + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected emitStackStore(MachineIRBuilder &Builder, Register Source, + FrameIndex Object, std::uint64_t ObjectOffset, + const MachineMemOperand &Memory, + CallValueTransform Transform) { + auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoSTORE_STACK_ARG)); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Use = Instruction->addUse(Source); + if (!Use) + return std::unexpected(Use.error()); + auto Frame = Instruction->addFrameIndex(Object); + if (!Frame) + return std::unexpected(Frame.error()); + auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset); + if (!Offset) + return std::unexpected(Offset.error()); + auto Action = Instruction->addImmediate(static_cast(Transform)); + if (!Action) + return std::unexpected(Action.error()); + auto Mem = Instruction->addMemOperand(Memory); + if (!Mem) + return std::unexpected(Mem.error()); + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected ensureFunctionLiveIn(MachineRegisterInfo &Registers, + PhysicalRegister Reg, + std::optional InitialValue) { + const auto Existing = + std::ranges::find(Registers.liveIns(), Reg, &PhysicalRegisterLiveIn::PhysReg); + if (Existing == Registers.liveIns().end()) + return Registers.addLiveIn(Reg, InitialValue); + if (InitialValue && Existing->InitialVirtualRegister && + Existing->InitialVirtualRegister != InitialValue) { + return makeUnexpected(ErrorCode::VerificationError, + "physical formal register already maps to another virtual live-in"); + } + return {}; +} + +[[nodiscard]] Expected ensureBlockLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg) { + return Block.addLiveIn(Reg, LaneMask::all()); +} + +[[nodiscard]] Expected verifyPlanRegisters(const CallLoweringPlan &Plan, + const MachineFunction &Function, + const TargetDescription &Target) { + const MachineRegisterInfo &Registers = Function.registerInfo(); + const auto Verify = [&Registers, &Target](const CCAssignment &Assignment) -> Expected { + if (Assignment.LocationRegister && + !Target.registers().isKnownPhysicalRegister(*Assignment.LocationRegister)) { + return makeUnexpected(ErrorCode::VerificationError, + "call-lowering plan names an unknown ABI location register"); + } + const Register Reg = Assignment.OriginalValue; + if (Reg.isNoRegister()) + return {}; + if (Reg.isPhysical()) { + if (!Target.registers().isKnownPhysicalRegister(*Reg.physicalRegister())) { + return makeUnexpected(ErrorCode::VerificationError, + "call-lowering plan names an unknown physical value register"); + } + return {}; + } + auto Info = Registers.virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) { + return makeUnexpected(ErrorCode::VerificationError, + "call-lowering plan register LLT differs from MachineRegisterInfo"); + } + return {}; + }; + for (const CCAssignment &Assignment : Plan.argumentAssignments()) { + auto Result = Verify(Assignment); + if (!Result) + return std::unexpected(Result.error()); + } + for (const CCAssignment &Assignment : Plan.returnAssignments()) { + auto Result = Verify(Assignment); + if (!Result) + return std::unexpected(Result.error()); + } + if (const MachineCallTarget *Callee = Plan.callee(); + Callee != nullptr && Callee->kind() == MachineCallTargetKind::IndirectRegister) { + const Register CalleeReg = Callee->indirectRegister(); + if (CalleeReg.isPhysical()) { + const std::optional Physical = CalleeReg.physicalRegister(); + if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) { + return makeUnexpected(ErrorCode::VerificationError, + "indirect RISC-V callee names an unknown physical register"); + } + } else { + auto Info = Registers.virtualRegisterInfo(CalleeReg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional Type = Info->get().Type; + if (!Type || ((!Type->isPointer() && !Type->isInteger()) || Type->scalarBitWidth() != 64)) { + return makeUnexpected(ErrorCode::VerificationError, + "indirect RISC-V callee requires a 64-bit pointer or integer LLT"); + } + } + } + return {}; +} + +[[nodiscard]] Expected verifyRequiredOpcodes(const MachineIRBuilder &Builder) { + static constexpr std::array Required{ + InstructionOpcode::PseudoCC_COPY, + InstructionOpcode::PseudoLOAD_STACK_ARG, + InstructionOpcode::PseudoSTORE_STACK_ARG, + InstructionOpcode::PseudoBYVAL_COPY, + InstructionOpcode::ADJCALLSTACKDOWN, + InstructionOpcode::ADJCALLSTACKUP, + InstructionOpcode::PseudoCALL, + InstructionOpcode::PseudoTAIL, + InstructionOpcode::PseudoRET, + }; + for (const InstructionOpcode OpcodeValue : Required) { + if (Builder.target().instructionDescriptor(targetOpcode(OpcodeValue)) == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V target description lacks a call-lowering pseudo"); + } + } + return {}; +} + +[[nodiscard]] Expected verifyFormalLiveIns(const CallLoweringPlan &Plan, + const MachineFunction &Function) { + if (Plan.kind() != CallLoweringPlanKind::FormalArguments) + return {}; + + const auto LiveIns = Function.registerInfo().liveIns(); + for (const CCAssignment &Assignment : Plan.argumentAssignments()) { + if (!Assignment.LocationRegister || !Assignment.OriginalValue.isVirtual()) + continue; + const auto Existing = + std::ranges::find(LiveIns, *Assignment.LocationRegister, &PhysicalRegisterLiveIn::PhysReg); + if (Existing != LiveIns.end() && Existing->InitialVirtualRegister && + *Existing->InitialVirtualRegister != Assignment.OriginalValue) { + return makeUnexpected(ErrorCode::VerificationError, + "physical formal register already maps to another virtual live-in"); + } + } + return {}; +} + +[[nodiscard]] Expected preflightPlan(MachineIRBuilder &Builder, + const CallLoweringPlan &Plan) { + auto Convention = riscvConvention(Plan.convention()); + if (!Convention) + return std::unexpected(Convention.error()); + if (Plan.target() != PlanTarget) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering plan belongs to another target or ABI"); + } + auto InsertPoint = Builder.verifyInsertPoint(); + if (!InsertPoint) + return std::unexpected(InsertPoint.error()); + auto Opcodes = verifyRequiredOpcodes(Builder); + if (!Opcodes) + return std::unexpected(Opcodes.error()); + auto Registers = verifyPlanRegisters(Plan, Builder.function(), Builder.target()); + if (!Registers) + return std::unexpected(Registers.error()); + auto LiveIns = verifyFormalLiveIns(Plan, Builder.function()); + if (!LiveIns) + return std::unexpected(LiveIns.error()); + for (const CCAssignment &Assignment : Plan.argumentAssignments()) { + if (Assignment.StackOffsetBytes && + *Assignment.StackOffsetBytes > + static_cast(std::numeric_limits::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V stack assignment offset exceeds the frame-index range"); + } + } + for (const ByValCopyAssignment &Copy : Plan.byValCopies()) { + if (Copy.StackOffsetBytes > + static_cast(std::numeric_limits::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V byval offset exceeds the frame-index range"); + } + } + if (Plan.kind() != CallLoweringPlanKind::Return && Builder.function().frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::VerificationError, + "call lowering cannot modify a frozen machine frame"); + } + if (Plan.kind() == CallLoweringPlanKind::FormalArguments && + (Builder.function().firstBlock() == nullptr || + Builder.insertionBlock() != Builder.function().firstBlock())) { + return makeUnexpected(ErrorCode::InvalidArgument, + "formal arguments must be lowered in the entry machine block"); + } + return {}; +} + +[[nodiscard]] Expected updateMaximumFrameAlignment(MachineFrameState &State, + Alignment Required) { + if (!State.MaxAlignment || State.MaxAlignment->bytes() < Required.bytes()) + State.MaxAlignment = Required; + return {}; +} + +[[nodiscard]] Expected emitFormalArguments(MachineIRBuilder &Builder, + const CallLoweringPlan &Plan) { + MachineFunction &Function = Builder.function(); + MachineBasicBlock &Entry = *Function.firstBlock(); + MachineRegisterInfo &Registers = Function.registerInfo(); + MachineFrameInfo &Frame = Function.frameInfo(); + Alignment RequiredFrameAlignment; + + for (const CCAssignment &Assignment : Plan.argumentAssignments()) { + if (Assignment.LocationRegister) { + const PhysicalRegister Location = *Assignment.LocationRegister; + const std::optional Initial = + Assignment.OriginalValue.isVirtual() ? std::optional(Assignment.OriginalValue) + : std::nullopt; + auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Location, Initial); + if (!FunctionLiveIn) + return std::unexpected(FunctionLiveIn.error()); + auto BlockLiveIn = ensureBlockLiveIn(Entry, Location); + if (!BlockLiveIn) + return std::unexpected(BlockLiveIn.error()); + auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(Location), + incomingTransform(Assignment)); + if (!Copy) + return std::unexpected(Copy.error()); + continue; + } + + if (!Assignment.StackOffsetBytes) { + return makeUnexpected(ErrorCode::InternalError, + "verified formal stack assignment has no stack offset"); + } + StackObjectDesc Object; + Object.Kind = StackObjectKind::FixedABI; + Object.Size = Assignment.StackSlotSizeBytes; + Object.Align = Assignment.LocationAlignment; + Object.Offset = static_cast(*Assignment.StackOffsetBytes); + Object.IsImmutable = true; + Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::IncomingArgument); + auto FrameObject = Frame.createObject(std::move(Object)); + if (!FrameObject) + return std::unexpected(FrameObject.error()); + auto Memory = + createMemoryOperand(Function, {MemoryAccessFlag::Load}, Assignment.LocationSizeBytes, + Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject)); + if (!Memory) + return std::unexpected(Memory.error()); + auto Load = emitStackLoad(Builder, Assignment, *FrameObject, 0, **Memory); + if (!Load) + return std::unexpected(Load.error()); + if (RequiredFrameAlignment.bytes() < Assignment.LocationAlignment.bytes()) + RequiredFrameAlignment = Assignment.LocationAlignment; + } + + const VarArgLoweringState &VarArgs = Plan.varArgs(); + if (!VarArgs.IsVariadic) { + if (RequiredFrameAlignment.bytes() == 1) + return {}; + MachineFrameState State = Frame.state(); + auto AlignmentUpdate = updateMaximumFrameAlignment(State, RequiredFrameAlignment); + if (!AlignmentUpdate) + return std::unexpected(AlignmentUpdate.error()); + return Frame.setState(std::move(State)); + } + + auto Align8 = alignment(XLenBytes); + auto Align16 = alignment(ABIStackAlignmentBytes); + if (!Align8 || !Align16) + return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid"); + + FrameIndex SaveObject; + if (VarArgs.RegisterSaveAreaSizeBytes == 0) { + StackObjectDesc Object; + Object.Kind = StackObjectKind::FixedABI; + Object.Size = XLenBytes; + Object.Align = *Align8; + Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes; + Object.IsImmutable = true; + Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea); + auto Created = Frame.createObject(std::move(Object)); + if (!Created) + return std::unexpected(Created.error()); + SaveObject = *Created; + } else { + StackObjectDesc Object; + Object.Kind = StackObjectKind::FixedABI; + Object.Size = VarArgs.RegisterSaveAreaSizeBytes; + Object.Align = VarArgs.FirstUnallocatedIntegerRegister % 2 == 0 ? *Align16 : *Align8; + Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes; + Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea); + auto Created = Frame.createObject(std::move(Object)); + if (!Created) + return std::unexpected(Created.error()); + SaveObject = *Created; + + if (VarArgs.RegisterSavePaddingBytes != 0) { + StackObjectDesc Padding; + Padding.Kind = StackObjectKind::FixedABI; + Padding.Size = VarArgs.RegisterSavePaddingBytes; + Padding.Align = *Align8; + Padding.Offset = VarArgs.RegisterSaveAreaOffsetBytes - + static_cast(VarArgs.RegisterSavePaddingBytes); + Padding.IsImmutable = true; + Padding.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgAlignmentPadding); + auto CreatedPadding = Frame.createObject(std::move(Padding)); + if (!CreatedPadding) + return std::unexpected(CreatedPadding.error()); + } + } + + for (const VarArgRegisterSave Save : VarArgs.RegistersToSave) { + auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Save.Reg, std::nullopt); + if (!FunctionLiveIn) + return std::unexpected(FunctionLiveIn.error()); + auto BlockLiveIn = ensureBlockLiveIn(Entry, Save.Reg); + if (!BlockLiveIn) + return std::unexpected(BlockLiveIn.error()); + auto Memory = createMemoryOperand(Function, {MemoryAccessFlag::Store}, XLenBytes, *Align8, + MemoryBase::frameIndex(SaveObject), + static_cast(Save.OffsetBytes)); + if (!Memory) + return std::unexpected(Memory.error()); + auto Store = emitStackStore(Builder, Register::fromPhysical(Save.Reg), SaveObject, + Save.OffsetBytes, **Memory, CallValueTransform::Full); + if (!Store) + return std::unexpected(Store.error()); + } + + MachineFrameState State = Frame.state(); + const Alignment VarArgAlignment = + RequiredFrameAlignment.bytes() > Align16->bytes() ? RequiredFrameAlignment : *Align16; + auto AlignmentUpdate = updateMaximumFrameAlignment(State, VarArgAlignment); + if (!AlignmentUpdate) + return std::unexpected(AlignmentUpdate.error()); + return Frame.setState(std::move(State)); +} + +[[nodiscard]] Expected emitCallFrameAdjustment(MachineIRBuilder &Builder, + InstructionOpcode OpcodeValue, + const CallFrameRequirements &Frame) { + if (Frame.SizeBytes > static_cast(std::numeric_limits::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V call-frame size does not fit a MIR immediate"); + } + auto Instruction = Builder.buildInstr(targetOpcode(OpcodeValue)); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Size = Instruction->addImmediate(static_cast(Frame.SizeBytes)); + if (!Size) + return std::unexpected(Size.error()); + auto PreparedBytes = Instruction->addImmediate(0); + if (!PreparedBytes) + return std::unexpected(PreparedBytes.error()); + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected> +emitByValCopies(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { + MachineFunction &Function = Builder.function(); + MachineFrameInfo &Frame = Function.frameInfo(); + std::unordered_map Replacements; + for (const ByValCopyAssignment Copy : Plan.byValCopies()) { + StackObjectDesc Object; + Object.Kind = StackObjectKind::Target; + Object.Size = Copy.SizeBytes; + Object.Align = Copy.Align; + Object.LocalOffset = static_cast(Copy.StackOffsetBytes); + Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::ByValCopy); + auto FrameObject = Frame.createObject(std::move(Object)); + if (!FrameObject) + return std::unexpected(FrameObject.error()); + auto Temporary = Function.registerInfo().createVirtualRegister(Copy.PointerType); + if (!Temporary) + return std::unexpected(Temporary.error()); + + auto SourceMemory = createMemoryOperand(Function, {MemoryAccessFlag::Load}, Copy.SizeBytes, + Copy.Align, MemoryBase{}); + if (!SourceMemory) + return std::unexpected(SourceMemory.error()); + auto DestinationMemory = + createMemoryOperand(Function, {MemoryAccessFlag::Store}, Copy.SizeBytes, Copy.Align, + MemoryBase::frameIndex(*FrameObject)); + if (!DestinationMemory) + return std::unexpected(DestinationMemory.error()); + + auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoBYVAL_COPY)); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Def = Instruction->addDef(*Temporary); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Instruction->addUse(Copy.Source); + if (!Use) + return std::unexpected(Use.error()); + auto FrameOperand = Instruction->addFrameIndex(*FrameObject); + if (!FrameOperand) + return std::unexpected(FrameOperand.error()); + auto Size = addUnsignedImmediate(*Instruction, Copy.SizeBytes); + if (!Size) + return std::unexpected(Size.error()); + auto Align = addUnsignedImmediate(*Instruction, Copy.Align.bytes()); + if (!Align) + return std::unexpected(Align.error()); + auto SourceMem = Instruction->addMemOperand(**SourceMemory); + if (!SourceMem) + return std::unexpected(SourceMem.error()); + auto DestinationMem = Instruction->addMemOperand(**DestinationMemory); + if (!DestinationMem) + return std::unexpected(DestinationMem.error()); + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + Replacements.emplace(Copy.ArgumentAssignmentIndex, *Temporary); + } + return Replacements; +} + +[[nodiscard]] Expected +emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan, + const std::unordered_map &Replacements) { + MachineFunction &Function = Builder.function(); + MachineFrameInfo &Frame = Function.frameInfo(); + const auto Assignments = Plan.argumentAssignments(); + for (std::size_t Index = 0; Index < Assignments.size(); ++Index) { + const CCAssignment &Assignment = Assignments[Index]; + const auto Replacement = Replacements.find(Index); + const Register Source = + Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second; + if (Assignment.LocationRegister) { + auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Assignment.LocationRegister), Source, + outgoingTransform(Assignment)); + if (!Copy) + return std::unexpected(Copy.error()); + continue; + } + + if (!Assignment.StackOffsetBytes) { + return makeUnexpected(ErrorCode::InternalError, + "verified outgoing stack assignment has no stack offset"); + } + StackObjectDesc Object; + Object.Size = Assignment.StackSlotSizeBytes; + Object.Align = Assignment.LocationAlignment; + Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::OutgoingArgument); + if (Plan.tailDecision() == TailCallKind::NotTail) { + Object.Kind = StackObjectKind::Target; + Object.LocalOffset = static_cast(*Assignment.StackOffsetBytes); + } else { + Object.Kind = StackObjectKind::FixedABI; + Object.Offset = static_cast(*Assignment.StackOffsetBytes); + } + auto FrameObject = Frame.createObject(std::move(Object)); + if (!FrameObject) + return std::unexpected(FrameObject.error()); + auto Memory = + createMemoryOperand(Function, {MemoryAccessFlag::Store}, Assignment.LocationSizeBytes, + Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject)); + if (!Memory) + return std::unexpected(Memory.error()); + auto Store = + emitStackStore(Builder, Source, *FrameObject, 0, **Memory, outgoingTransform(Assignment)); + if (!Store) + return std::unexpected(Store.error()); + } + return {}; +} + +[[nodiscard]] Expected addCallTarget(MachineInstrBuilder &Instruction, + const MachineCallTarget &Callee) { + switch (Callee.kind()) { + case MachineCallTargetKind::GlobalSymbol: + return Instruction.addGlobalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags()); + case MachineCallTargetKind::ExternalSymbol: + return Instruction.addExternalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags()); + case MachineCallTargetKind::IndirectRegister: + return Instruction.addUse(Callee.indirectRegister()); + case MachineCallTargetKind::Invalid: + break; + } + return makeUnexpected(ErrorCode::InternalError, + "verified call-lowering plan has an invalid callee"); +} + +[[nodiscard]] Expected requiredLocationRegister(const CCAssignment &Assignment) { + if (!Assignment.LocationRegister) { + return makeUnexpected(ErrorCode::InternalError, + "verified return assignment has no location register"); + } + return *Assignment.LocationRegister; +} + +[[nodiscard]] Expected emitCallInstruction(MachineIRBuilder &Builder, + const CallLoweringPlan &Plan) { + const bool IsTail = Plan.tailDecision() != TailCallKind::NotTail; + auto Instruction = Builder.buildInstr( + targetOpcode(IsTail ? InstructionOpcode::PseudoTAIL : InstructionOpcode::PseudoCALL)); + if (!Instruction) + return std::unexpected(Instruction.error()); + auto Callee = addCallTarget(*Instruction, *Plan.callee()); + if (!Callee) + return std::unexpected(Callee.error()); + auto Mask = Instruction->addRegisterMask(Plan.preservedMask()); + if (!Mask) + return std::unexpected(Mask.error()); + + std::unordered_set DynamicUses; + for (const CCAssignment &Assignment : Plan.argumentAssignments()) { + if (Assignment.LocationRegister && DynamicUses.insert(*Assignment.LocationRegister).second) { + auto Use = Instruction->addImplicitUse(Register::fromPhysical(*Assignment.LocationRegister)); + if (!Use) + return std::unexpected(Use.error()); + } + } + std::unordered_set DynamicDefs; + if (!IsTail) { + for (const CCAssignment &Assignment : Plan.returnAssignments()) { + if (Assignment.LocationRegister && DynamicDefs.insert(*Assignment.LocationRegister).second) { + auto Def = + Instruction->addImplicitDef(Register::fromPhysical(*Assignment.LocationRegister)); + if (!Def) + return std::unexpected(Def.error()); + } + } + } + auto Built = Instruction->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected emitCall(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { + if (Plan.callFrame().EmitSetup) { + auto Down = + emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKDOWN, Plan.callFrame()); + if (!Down) + return std::unexpected(Down.error()); + } + + auto Replacements = emitByValCopies(Builder, Plan); + if (!Replacements) + return std::unexpected(Replacements.error()); + auto Arguments = emitOutgoingArguments(Builder, Plan, *Replacements); + if (!Arguments) + return std::unexpected(Arguments.error()); + auto Call = emitCallInstruction(Builder, Plan); + if (!Call) + return std::unexpected(Call.error()); + + if (Plan.callFrame().EmitDestroy) { + auto Up = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKUP, Plan.callFrame()); + if (!Up) + return std::unexpected(Up.error()); + } + if (Plan.tailDecision() == TailCallKind::NotTail) { + for (const CCAssignment &Assignment : Plan.returnAssignments()) { + auto Location = requiredLocationRegister(Assignment); + if (!Location) + return std::unexpected(Location.error()); + auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(*Location), + incomingTransform(Assignment)); + if (!Copy) + return std::unexpected(Copy.error()); + } + } + + MachineFrameInfo &Frame = Builder.function().frameInfo(); + MachineFrameState State = Frame.state(); + State.HasCalls = true; + State.HasTailCall = State.HasTailCall || Plan.tailDecision() != TailCallKind::NotTail; + State.AdjustsStack = + State.AdjustsStack || (Plan.callFrame().EmitSetup && Plan.callFrame().SizeBytes != 0); + State.HasMustTailInVarArgFunc = + State.HasMustTailInVarArgFunc || + (Plan.tailDecision() == TailCallKind::MustTail && Plan.varArgs().IsVariadic); + if (Plan.tailDecision() == TailCallKind::NotTail) { + State.MaxCallFrameSize = + std::max(State.MaxCallFrameSize.value_or(0), Plan.callFrame().SizeBytes); + } + auto AlignmentUpdate = updateMaximumFrameAlignment(State, Plan.callFrame().Align); + if (!AlignmentUpdate) + return std::unexpected(AlignmentUpdate.error()); + return Frame.setState(std::move(State)); +} + +[[nodiscard]] Expected emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { + for (const CCAssignment &Assignment : Plan.returnAssignments()) { + auto Location = requiredLocationRegister(Assignment); + if (!Location) + return std::unexpected(Location.error()); + auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Location), Assignment.OriginalValue, + outgoingTransform(Assignment)); + if (!Copy) + return std::unexpected(Copy.error()); + } + + auto Return = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoRET)); + if (!Return) + return std::unexpected(Return.error()); + for (const CCAssignment &Assignment : Plan.returnAssignments()) { + auto Location = requiredLocationRegister(Assignment); + if (!Location) + return std::unexpected(Location.error()); + auto Use = Return->addImplicitUse(Register::fromPhysical(*Location)); + if (!Use) + return std::unexpected(Use.error()); + } + auto Built = Return->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +Expected RISCVCallLowering::emitPlan(MachineIRBuilder &Builder, + const CallLoweringPlan &Plan) const { + auto Preflight = preflightPlan(Builder, Plan); + if (!Preflight) + return std::unexpected(Preflight.error()); + switch (Plan.kind()) { + case CallLoweringPlanKind::FormalArguments: + return emitFormalArguments(Builder, Plan); + case CallLoweringPlanKind::Call: + return emitCall(Builder, Plan); + case CallLoweringPlanKind::Return: + return emitReturn(Builder, Plan); + } + return makeUnexpected(ErrorCode::InternalError, "call-lowering plan kind is invalid"); +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h new file mode 100644 index 0000000..dad0875 --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H +#define RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H + +#include "rcc/CodeGen/TargetCallLowering.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" + +#include + +namespace rcc::riscv { + +enum class CallTargetFlag : std::uint32_t { + DirectCall = 1U << 0U, +}; + +enum class CallStackObjectFlag : std::uint64_t { + IncomingArgument = 1ULL << 0U, + OutgoingArgument = 1ULL << 1U, + ByValCopy = 1ULL << 2U, + VarArgSaveArea = 1ULL << 3U, + VarArgAlignmentPadding = 1ULL << 4U, +}; + +enum class CallValueTransform : std::uint8_t { + Full, + AnyExtend, + SignExtend, + ZeroExtend, + Truncate, + Bitcast, +}; + +[[nodiscard]] constexpr CallingConventionID +callingConventionID(CallingConvention Convention) noexcept { + return CallingConventionID::fromValue(static_cast(Convention)); +} + +class RISCVCallLowering final : public TargetCallLowering { +public: + [[nodiscard]] Expected + planFormalArguments(const FormalArgumentRequest &Request) const override; + [[nodiscard]] Expected + planCall(const CallLoweringRequest &Request) const override; + [[nodiscard]] Expected + planReturn(const ReturnLoweringRequest &Request) const override; + [[nodiscard]] Expected emitPlan(MachineIRBuilder &Builder, + const CallLoweringPlan &Plan) const override; +}; + +} // namespace rcc::riscv + +#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp new file mode 100644 index 0000000..a2c7194 --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -0,0 +1,2627 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RISCV.h" +#include "RISCVRegisterBankInfo.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/InstructionSelector.h" +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::optional registerType(const MachineFunction &Function, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::nullopt; + return Info->get().Type; +} + +[[nodiscard]] std::optional resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { + const MachineOperand *Result = Instruction.operandAt(0); + return Result == nullptr ? std::nullopt : registerType(Function, *Result); +} + +[[nodiscard]] std::optional operandType(const MachineFunction &Function, + const MachineInstr &Instruction, std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); +} + +[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { + if (!Type.isFloat()) + return false; + const std::optional Semantics = Type.floatSemantics(); + return Semantics && + (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +} + +[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64; +} + +[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept { + const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U)); + const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1; + return Value >= Minimum && Value <= Maximum; +} + +[[nodiscard]] std::optional +memoryAccessBytes(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return std::nullopt; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (!Description.Size.isKnown() || Description.Size.isScalable()) + return std::nullopt; + return Description.Size.minimumBytes(); +} + +[[nodiscard]] Expected appendImmediate(MachineInstr &Instruction, std::int64_t Value) { + return Instruction.addOperand(MachineOperand::immediate(Value)); +} + +[[nodiscard]] Expected insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index, + PhysicalRegister Reg, + RegisterOperandFlag Flag) { + auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag}); + if (!Operand) + return std::unexpected(Operand.error()); + return Instruction.insertOperand(Index, std::move(*Operand)); +} + +[[nodiscard]] Expected insertVirtualUse(MachineInstr &Instruction, std::size_t Index, + Register Reg) { + auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + return Instruction.insertOperand(Index, std::move(*Operand)); +} + +[[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf()) + return *Value; + const auto *Integer = Operand.getIf(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an immediate no wider than 64 bits"); + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast(Raw); +} + +class RISCVInstructionSelector final : public InstructionSelector { +public: + RISCVInstructionSelector(const RISCVSubtarget &Subtarget, + const RISCVRegisterBankInfo &RegisterBanks) + : Subtarget(Subtarget), RegisterBanks(RegisterBanks) {} + + [[nodiscard]] Expected + constrainSelectedInstruction(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const override { + return RegisterBanks.constrainInstruction(Instruction, Function, Target); + } + + [[nodiscard]] Expected canSelect(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const override { + if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) { + const std::optional Type = resultType(Function, Instruction); + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isScalarGPRType(*Type) && Type->scalarBitWidth() == 64 && + *Left == *Type && *Right == *Type + ? Expected{} + : makeUnexpected(ErrorCode::Unsupported, + "RISC-V G_SLLW requires three 64-bit GPR operands"); + } + if (!isGenericOpcode(Instruction.opcode())) + return {}; + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; + const auto Float = [Type] { return Type && isRVFloatType(*Type); }; + auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target); + if (!Classification) + return std::unexpected(Classification.error()); + if (*Classification != LegalizationAction::Legal) { + std::string Message = "RISC-V selector cannot select opcode '" + + std::string(getGenericOpcodeName(Generic)) + "'"; + if (Type) + Message += " with result type '" + Type->toString() + "'"; + Message += ": the legalization rule explicitly marks this combination unsupported"; + return makeUnexpected(ErrorCode::Unsupported, std::move(Message)); + } + switch (Generic) { + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_UREM: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: + case GenericOpcode::G_PTR_ADD: + case GenericOpcode::G_PTRMASK: + case GenericOpcode::G_CONSTANT: + if (!Integer()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires an i1-i64 integer or 64-bit pointer value"); + if (Generic == GenericOpcode::G_CONSTANT) { + const MachineOperand *Immediate = Instruction.operandAt(1); + if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && + Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer constant requires an immediate operand"); + } + return {}; + case GenericOpcode::G_LOAD: + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: + case GenericOpcode::G_FPEXTLOAD: + case GenericOpcode::G_FPTRUNCSTORE: + case GenericOpcode::G_STORE: + case GenericOpcode::G_BR: + case GenericOpcode::G_BRCOND: + return {}; + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: + case GenericOpcode::G_FMA: + case GenericOpcode::G_FSQRT: + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FCOPYSIGN: + if (!Float()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires f32 or f64 floating values"); + return {}; + case GenericOpcode::G_FCONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; + if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires a matching floating constant"); + return {}; + } + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + case GenericOpcode::G_PTRTOINT: + case GenericOpcode::G_INTTOPTR: + case GenericOpcode::G_BITCAST: + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_SEXT_INREG: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ICMP: + case GenericOpcode::G_FCMP: + case GenericOpcode::G_FPEXT: + case GenericOpcode::G_FPTRUNC: + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + case GenericOpcode::G_SELECT: + case GenericOpcode::G_ABS: + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + case GenericOpcode::G_FENCE: + case GenericOpcode::G_BRINDIRECT: + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + case GenericOpcode::G_UBSANTRAP: + case GenericOpcode::G_PHI: + case GenericOpcode::G_IMPLICIT_DEF: + return {}; + default: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + } + + [[nodiscard]] Expected select(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const override { + auto Match = canSelect(Instruction, Function, Target); + if (!Match) + return std::unexpected(Match.error()); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLW)); + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + + const auto SelectDirect = [&Instruction](InstructionOpcode Selected) { + return Instruction.setOpcode(targetOpcode(Selected)); + }; + switch (Generic) { + case GenericOpcode::G_ADD: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD, + InstructionOpcode::ADDW); + case GenericOpcode::G_SUB: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB, + InstructionOpcode::SUBW); + case GenericOpcode::G_MUL: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL, + InstructionOpcode::MULW); + case GenericOpcode::G_SDIV: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::DIV, InstructionOpcode::DIVW, true, + true); + case GenericOpcode::G_UDIV: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false, + true); + case GenericOpcode::G_SREM: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::REM, InstructionOpcode::REMW, true, + true); + case GenericOpcode::G_UREM: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::REMU, InstructionOpcode::REMUW, false, + true); + case GenericOpcode::G_SDIVREM: + return selectDivRem(Instruction, Function, Target, Type, true); + case GenericOpcode::G_UDIVREM: + return selectDivRem(Instruction, Function, Target, Type, false); + case GenericOpcode::G_AND: + return SelectDirect(InstructionOpcode::AND); + case GenericOpcode::G_OR: + return SelectDirect(InstructionOpcode::OR); + case GenericOpcode::G_XOR: + return SelectDirect(InstructionOpcode::XOR); + case GenericOpcode::G_SHL: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL, + InstructionOpcode::SLLW); + case GenericOpcode::G_LSHR: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::SRL, InstructionOpcode::SRLW, false, + false); + case GenericOpcode::G_ASHR: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::SRA, InstructionOpcode::SRAW, true, + false); + case GenericOpcode::G_PTR_ADD: + return SelectDirect(InstructionOpcode::ADD); + case GenericOpcode::G_PTRMASK: + return SelectDirect(InstructionOpcode::AND); + case GenericOpcode::G_LOAD: + return selectLoad(Instruction, Type); + case GenericOpcode::G_SEXTLOAD: + return selectExtLoad(Instruction, Type, true); + case GenericOpcode::G_ZEXTLOAD: + return selectExtLoad(Instruction, Type, false); + case GenericOpcode::G_FPEXTLOAD: + return selectFloatExtLoad(Instruction, Function, Target); + case GenericOpcode::G_STORE: + return selectStore(Instruction, Function); + case GenericOpcode::G_FPTRUNCSTORE: + return selectFloatTruncStore(Instruction, Function, Target); + case GenericOpcode::G_BR: + return selectBranch(Instruction); + case GenericOpcode::G_BRCOND: + return selectConditionalBranch(Instruction); + case GenericOpcode::G_CONSTANT: + return selectIntegerConstant(Instruction, Function, Target, Type); + case GenericOpcode::G_FCONSTANT: + return selectFloatingConstant(Instruction, Function, Target, Type); + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + if (Instruction.operandCount() == 3) { + auto EraseImmediate = Instruction.eraseOperand(2); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + } + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + case GenericOpcode::G_PTRTOINT: + case GenericOpcode::G_INTTOPTR: + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_BITCAST: + return selectBitcast(Instruction, Function); + case GenericOpcode::G_FADD: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S, + InstructionOpcode::FADD_D); + case GenericOpcode::G_FSUB: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S, + InstructionOpcode::FSUB_D); + case GenericOpcode::G_FMUL: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S, + InstructionOpcode::FMUL_D); + case GenericOpcode::G_FDIV: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S, + InstructionOpcode::FDIV_D); + case GenericOpcode::G_FMA: + return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S, + InstructionOpcode::FMADD_D); + case GenericOpcode::G_FSQRT: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S, + InstructionOpcode::FSQRT_D); + case GenericOpcode::G_FNEG: + return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S, + InstructionOpcode::FSGNJN_D); + case GenericOpcode::G_FABS: + return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S, + InstructionOpcode::FSGNJX_D); + case GenericOpcode::G_FCOPYSIGN: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S, + InstructionOpcode::FSGNJ_D); + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: + return selectIntegerExtension(Instruction, Function, Target, Generic, Type); + case GenericOpcode::G_TRUNC: + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_SEXT_INREG: + return selectSignExtendInRegister(Instruction, Function, Target, Type); + case GenericOpcode::G_ICMP: + return selectIntegerCompare(Instruction, Function, Target); + case GenericOpcode::G_FCMP: + return selectFloatingCompare(Instruction, Function, Target); + case GenericOpcode::G_FPEXT: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S, + InstructionOpcode::FCVT_D_S); + case GenericOpcode::G_FPTRUNC: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D, + InstructionOpcode::FCVT_S_D); + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI); + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + return selectIntegerToFloat(Instruction, Function, Target, Type, + Generic == GenericOpcode::G_UITOFP); + case GenericOpcode::G_FMINNUM: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S, + InstructionOpcode::FMIN_D); + case GenericOpcode::G_FMAXNUM: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S, + InstructionOpcode::FMAX_D); + case GenericOpcode::G_SMULH: + return selectMultiplyHigh(Instruction, Function, Target, Type, true); + case GenericOpcode::G_UMULH: + return selectMultiplyHigh(Instruction, Function, Target, Type, false); + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + return selectIntegerMinMax(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_SELECT: + return selectValue(Instruction, Function, Target, Type); + case GenericOpcode::G_ABS: + return selectAbsoluteValue(Instruction, Function, Target, Type); + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: + return selectOverflow(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + return selectExtendedOverflow(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + return selectAtomicRMW(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_FENCE: + return selectFence(Instruction); + case GenericOpcode::G_BRINDIRECT: + return selectIndirectBranch(Instruction); + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + case GenericOpcode::G_UBSANTRAP: + return selectTrap(Instruction); + case GenericOpcode::G_PHI: + return Instruction.setOpcode(opcode(GenericOpcode::PHI)); + case GenericOpcode::G_IMPLICIT_DEF: + return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF)); + case GenericOpcode::COPY: + case GenericOpcode::IMPLICIT_DEF: + case GenericOpcode::PHI: + case GenericOpcode::REG_SEQUENCE: + case GenericOpcode::INSERT_SUBREG: + case GenericOpcode::EXTRACT_SUBREG: + case GenericOpcode::SUBREG_TO_REG: + case GenericOpcode::KILL: + case GenericOpcode::BUNDLE: + case GenericOpcode::INLINEASM: + case GenericOpcode::CORE_PSEUDO_END: + break; + default: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + +private: + [[nodiscard]] Expected createTemporary(MachineFunction &Function, LLT Type, + const TargetDescription &Target) const { + auto Temporary = Function.registerInfo().createVirtualRegister(Type); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Constraint = RegisterBanks.constrainRegister(Function, *Temporary, Target); + if (!Constraint) + return std::unexpected(Constraint.error()); + return *Temporary; + } + + [[nodiscard]] Expected replaceImmediate(MachineInstr &Instruction, std::size_t Index, + std::int64_t Value) const { + if (Instruction.operandAt(Index) == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction is missing an immediate operand"); + auto Erased = Instruction.eraseOperand(Index); + if (!Erased) + return std::unexpected(Erased.error()); + return Instruction.insertOperand(Index, MachineOperand::immediate(Value)); + } + + [[nodiscard]] Expected buildBinaryAfter(MachineIRBuilder &Builder, + MachineInstr &Anchor, + InstructionOpcode Opcode, Register Def, + Register Left, Register Right) const { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto LeftResult = Pending->addUse(Left); + if (!LeftResult) + return std::unexpected(LeftResult.error()); + auto RightResult = Pending->addUse(Right); + if (!RightResult) + return std::unexpected(RightResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] Expected + buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, + Register Def, Register Source, std::int64_t Value) const { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto ImmediateResult = Pending->addImmediate(Value); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] Expected + buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, + Register Def, Register Source, std::int64_t Value) const { + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto ImmediateResult = Pending->addImmediate(Value); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] Expected normalizeIntegerBefore(MachineInstr &Anchor, + MachineFunction &Function, + const TargetDescription &Target, + Register Source, LLT Type, + bool Signed) const { + if (!isSupportedIntegerScalar(Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer normalization requires a scalar i1-i64 type"); + const std::uint32_t Bits = Type.scalarBitWidth(); + if (Bits == 64) + return Source; + + MachineIRBuilder Builder(Function, Target); + if (Signed && Bits == 32) { + auto Result = createTemporary(Function, Type, Target); + if (!Result) + return std::unexpected(Result.error()); + auto Extended = + buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0); + if (!Extended) + return std::unexpected(Extended.error()); + return *Result; + } + + const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits - Bits); + auto Shifted = createTemporary(Function, Type, Target); + auto Result = createTemporary(Function, Type, Target); + if (!Shifted || !Result) + return std::unexpected(!Shifted ? Shifted.error() : Result.error()); + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) + : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); + auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift); + if (!Left) + return std::unexpected(Left.error()); + auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift); + if (!Right) + return std::unexpected(Right.error()); + return *Result; + } + + [[nodiscard]] Expected buildUnaryAfter(MachineIRBuilder &Builder, + MachineInstr &Anchor, + InstructionOpcode Opcode, Register Def, + Register Source) const { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] Expected + buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, + const TargetDescription &Target, LLT ValueType, Register Destination, + Register Condition, Register TrueValue, Register FalseValue) const { + auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth()); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto Mask = createTemporary(Function, *IntegerType, Target); + auto Difference = createTemporary(Function, *IntegerType, Target); + auto MaskedDifference = createTemporary(Function, *IntegerType, Target); + if (!Mask || !Difference || !MaskedDifference) { + return std::unexpected( + (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error()))); + } + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + const InstructionOpcode Subtract = + ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB; + auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask, + Register::fromPhysical(*Zero), Condition); + if (!MaskInstruction) + return std::unexpected(MaskInstruction.error()); + auto DifferenceInstruction = buildBinaryAfter( + Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue); + if (!DifferenceInstruction) + return std::unexpected(DifferenceInstruction.error()); + auto MaskInstructionResult = + buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND, + *MaskedDifference, *Difference, *Mask); + if (!MaskInstructionResult) + return std::unexpected(MaskInstructionResult.error()); + return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination, + FalseValue, *MaskedDifference); + } + + [[nodiscard]] Expected selectDivRem(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Signed) const { + const MachineOperand *RemainderOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Remainder = + RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V division-with-remainder has invalid operands"); + const Register RemainderRegister = *Remainder; + auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); + auto RightRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); + if (!LeftRegister || !RightRegister) + return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); + auto RemoveRemainder = Instruction.eraseOperand(1); + if (!RemoveRemainder) + return std::unexpected(RemoveRemainder.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister); + if (!SetRight) + return std::unexpected(SetRight.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + const InstructionOpcode Division = + IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW) + : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU); + const InstructionOpcode RemainderOpcode = + IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW) + : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU); + auto SetDivision = Instruction.setOpcode(targetOpcode(Division)); + if (!SetDivision) + return std::unexpected(SetDivision.error()); + MachineIRBuilder Builder(Function, Target); + auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister, + *LeftRegister, *RightRegister); + if (!BuiltRemainder) + return std::unexpected(BuiltRemainder.error()); + return {}; + } + + [[nodiscard]] Expected selectMultiplyHigh(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Signed) const { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V high multiplication has no result type"); + if (Type->scalarBitWidth() == 64) { + return Instruction.setOpcode( + targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU)); + } + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V high multiplication has invalid register operands"); + const Register DestinationRegister = *Destination; + const Register RightRegister = *Right; + auto I64 = LLT::makeInteger(64); + if (!I64) + return std::unexpected(I64.error()); + auto ExtendedLeft = createTemporary(Function, *I64, Target); + auto ExtendedRight = createTemporary(Function, *I64, Target); + auto Product = createTemporary(Function, *I64, Target); + if (!ExtendedLeft || !ExtendedRight || !Product) { + return std::unexpected(!ExtendedLeft + ? ExtendedLeft.error() + : (!ExtendedRight ? ExtendedRight.error() : Product.error())); + } + + auto RemoveRight = Instruction.eraseOperand(2); + if (!RemoveRight) + return std::unexpected(RemoveRight.error()); + MachineIRBuilder Builder(Function, Target); + MachineInstr *Anchor = &Instruction; + if (Signed) { + auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, + *ExtendedRight, RightRegister, 0); + if (!ExtendRight) + return std::unexpected(ExtendRight.error()); + Anchor = *ExtendRight; + } else { + auto ShiftedLeft = createTemporary(Function, *I64, Target); + auto ShiftedRight = createTemporary(Function, *I64, Target); + if (!ShiftedLeft || !ShiftedRight) + return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 32); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI, + *ExtendedLeft, *ShiftedLeft, 32); + if (!ExtendLeft) + return std::unexpected(ExtendLeft.error()); + auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI, + *ShiftedRight, RightRegister, 32); + if (!ShiftRight) + return std::unexpected(ShiftRight.error()); + auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI, + *ExtendedRight, *ShiftedRight, 32); + if (!ExtendRight) + return std::unexpected(ExtendRight.error()); + Anchor = *ExtendRight; + } + auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product, + *ExtendedLeft, *ExtendedRight); + if (!Multiply) + return std::unexpected(Multiply.error()); + auto Extract = buildImmediateAfter(Builder, **Multiply, + Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI, + DestinationRegister, *Product, 32); + if (!Extract) + return std::unexpected(Extract.error()); + return {}; + } + + [[nodiscard]] Expected selectIntegerMinMax(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) const { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer min/max has invalid operands"); + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto Condition = createTemporary(Function, *I1, Target); + if (!Condition) + return std::unexpected(Condition.error()); + const Register DestinationRegister = *Destination; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX; + auto NormalizedLeft = + normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned); + auto NormalizedRight = + normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned); + if (!NormalizedLeft || !NormalizedRight) + return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error()); + auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition); + if (!SetCondition) + return std::unexpected(SetCondition.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + auto SetCompare = Instruction.setOpcode( + targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT)); + if (!SetCompare) + return std::unexpected(SetCompare.error()); + const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX; + const Register TrueValue = IsMaximum ? RightRegister : LeftRegister; + const Register FalseValue = IsMaximum ? LeftRegister : RightRegister; + MachineIRBuilder Builder(Function, Target); + auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, + DestinationRegister, *Condition, TrueValue, FalseValue); + if (!Selected) + return std::unexpected(Selected.error()); + return {}; + } + + [[nodiscard]] Expected selectValue(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) const { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ConditionOperand = Instruction.operandAt(1); + const MachineOperand *TrueOperand = Instruction.operandAt(2); + const MachineOperand *FalseOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Condition = + ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); + const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf(); + const Register *FalseValue = + FalseOperand == nullptr ? nullptr : FalseOperand->getIf(); + if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr || + FalseValue == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V value selection has invalid operands"); + } + const Register DestinationRegister = *Destination; + const Register TrueRegister = *TrueValue; + const Register FalseRegister = *FalseValue; + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto NormalizedCondition = createTemporary(Function, *I1, Target); + if (!NormalizedCondition) + return std::unexpected(NormalizedCondition.error()); + auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition); + if (!SetCondition) + return std::unexpected(SetCondition.error()); + auto EraseFalse = Instruction.eraseOperand(3); + if (!EraseFalse) + return std::unexpected(EraseFalse.error()); + auto EraseTrue = Instruction.eraseOperand(2); + if (!EraseTrue) + return std::unexpected(EraseTrue.error()); + auto Mask = appendImmediate(Instruction, 1); + if (!Mask) + return std::unexpected(Mask.error()); + auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI)); + if (!SetNormalize) + return std::unexpected(SetNormalize.error()); + + MachineIRBuilder Builder(Function, Target); + if (!Type->isFloat()) { + auto Selected = + buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, + *NormalizedCondition, TrueRegister, FalseRegister); + if (!Selected) + return std::unexpected(Selected.error()); + return {}; + } + + auto IntegerType = LLT::makeInteger(Type->scalarBitWidth()); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto IntegerTrue = createTemporary(Function, *IntegerType, Target); + auto IntegerFalse = createTemporary(Function, *IntegerType, Target); + auto IntegerResult = createTemporary(Function, *IntegerType, Target); + if (!IntegerTrue || !IntegerFalse || !IntegerResult) { + return std::unexpected(!IntegerTrue + ? IntegerTrue.error() + : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error())); + } + const InstructionOpcode ToInteger = + Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; + const InstructionOpcode ToFloat = + Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister); + if (!MoveTrue) + return std::unexpected(MoveTrue.error()); + auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister); + if (!MoveFalse) + return std::unexpected(MoveFalse.error()); + auto Selected = + buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult, + *NormalizedCondition, *IntegerTrue, *IntegerFalse); + if (!Selected) + return std::unexpected(Selected.error()); + auto MoveResult = + buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult); + if (!MoveResult) + return std::unexpected(MoveResult.error()); + return {}; + } + + [[nodiscard]] Expected selectAbsoluteValue(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) const { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V absolute value has invalid operands"); + const Register DestinationRegister = *Destination; + const Register SourceRegister = *Source; + auto NormalizedSource = + normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true); + if (!NormalizedSource) + return std::unexpected(NormalizedSource.error()); + auto SignMask = createTemporary(Function, *Type, Target); + auto Flipped = createTemporary(Function, *Type, Target); + if (!SignMask || !Flipped) + return std::unexpected(!SignMask ? SignMask.error() : Flipped.error()); + auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask); + if (!SetMask) + return std::unexpected(SetMask.error()); + auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource); + if (!SetSource) + return std::unexpected(SetSource.error()); + const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64; + auto Shift = appendImmediate(Instruction, ContainerBits - 1U); + if (!Shift) + return std::unexpected(Shift.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + auto SetShift = Instruction.setOpcode( + targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)); + if (!SetShift) + return std::unexpected(SetShift.error()); + MachineIRBuilder Builder(Function, Target); + auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped, + *NormalizedSource, *SignMask); + if (!Flip) + return std::unexpected(Flip.error()); + auto Result = + buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB, + DestinationRegister, *Flipped, *SignMask); + if (!Result) + return std::unexpected(Result.error()); + return {}; + } + + [[nodiscard]] Expected> + buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, + MachineFunction &Function, const TargetDescription &Target, LLT Type, + Register Result, Register Left, Register Right, bool IsAdd) const { + auto FirstDifference = createTemporary(Function, Type, Target); + auto SecondDifference = createTemporary(Function, Type, Target); + auto SignBits = createTemporary(Function, Type, Target); + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto Overflow = createTemporary(Function, *I1, Target); + if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) { + return std::unexpected(!FirstDifference ? FirstDifference.error() + : !SecondDifference ? SecondDifference.error() + : !SignBits ? SignBits.error() + : Overflow.error()); + } + const Register FirstLeft = IsAdd ? Result : Left; + const Register FirstRight = IsAdd ? Left : Right; + const Register SecondLeft = Result; + const Register SecondRight = IsAdd ? Right : Left; + auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference, + FirstLeft, FirstRight); + if (!First) + return std::unexpected(First.error()); + auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, + SecondLeft, SecondRight); + if (!Second) + return std::unexpected(Second.error()); + auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, + *FirstDifference, *SecondDifference); + if (!Both) + return std::unexpected(Both.error()); + const InstructionOpcode Shift = + Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI; + auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits, + Type.scalarBitWidth() - 1U); + if (!Extract) + return std::unexpected(Extract.error()); + return std::pair{*Overflow, *Extract}; + } + + [[nodiscard]] Expected selectOverflow(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) const { + const MachineOperand *ResultOperand = Instruction.operandAt(0); + const MachineOperand *OverflowOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + const Register *Overflow = + OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V overflow operation has invalid operands"); + } + const Register ResultRegister = *Result; + const Register OverflowRegister = *Overflow; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + auto RemoveOverflow = Instruction.eraseOperand(1); + if (!RemoveOverflow) + return std::unexpected(RemoveOverflow.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO; + const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO; + const bool IsUnsigned = Generic == GenericOpcode::G_UADDO || + Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO; + const InstructionOpcode Arithmetic = + IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) + : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB) + : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL); + auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); + if (!SetArithmetic) + return std::unexpected(SetArithmetic.error()); + MachineIRBuilder Builder(Function, Target); + if (IsUnsigned && IsAdd) { + auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister, + ResultRegister, LeftRegister); + if (!Carry) + return std::unexpected(Carry.error()); + return {}; + } + if (IsUnsigned && IsSub) { + auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, + OverflowRegister, LeftRegister, RightRegister); + if (!Borrow) + return std::unexpected(Borrow.error()); + return {}; + } + if (IsAdd || IsSub) { + auto FirstDifference = createTemporary(Function, *Type, Target); + auto SecondDifference = createTemporary(Function, *Type, Target); + auto SignBits = createTemporary(Function, *Type, Target); + if (!FirstDifference || !SecondDifference || !SignBits) { + return std::unexpected( + !FirstDifference ? FirstDifference.error() + : (!SecondDifference ? SecondDifference.error() : SignBits.error())); + } + const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister; + const Register FirstRight = IsAdd ? LeftRegister : RightRegister; + const Register SecondLeft = ResultRegister; + const Register SecondRight = IsAdd ? RightRegister : LeftRegister; + auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference, + FirstLeft, FirstRight); + if (!First) + return std::unexpected(First.error()); + auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, + SecondLeft, SecondRight); + if (!Second) + return std::unexpected(Second.error()); + auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, + *FirstDifference, *SecondDifference); + if (!Both) + return std::unexpected(Both.error()); + auto OverflowResult = buildImmediateAfter( + Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI, + OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + auto I64 = LLT::makeInteger(64); + if (!I64) + return std::unexpected(I64.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + if (!IsWord) { + auto High = createTemporary(Function, *I64, Target); + if (!High) + return std::unexpected(High.error()); + auto HighProduct = buildBinaryAfter( + Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH, + *High, LeftRegister, RightRegister); + if (!HighProduct) + return std::unexpected(HighProduct.error()); + if (IsUnsigned) { + auto OverflowResult = + buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *High); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + auto ExpectedHigh = createTemporary(Function, *I64, Target); + auto Difference = createTemporary(Function, *I64, Target); + if (!ExpectedHigh || !Difference) + return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error()); + auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI, + *ExpectedHigh, ResultRegister, 63); + if (!Sign) + return std::unexpected(Sign.error()); + auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High, + *ExpectedHigh); + if (!Compare) + return std::unexpected(Compare.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *Difference); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + auto ExtendedLeft = createTemporary(Function, *I64, Target); + auto ExtendedRight = createTemporary(Function, *I64, Target); + auto FullProduct = createTemporary(Function, *I64, Target); + if (!ExtendedLeft || !ExtendedRight || !FullProduct) { + return std::unexpected(!ExtendedLeft + ? ExtendedLeft.error() + : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error())); + } + MachineInstr *Anchor = &Instruction; + if (IsUnsigned) { + auto ShiftedLeft = createTemporary(Function, *I64, Target); + auto ShiftedRight = createTemporary(Function, *I64, Target); + if (!ShiftedLeft || !ShiftedRight) + return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); + auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft, + LeftRegister, 32); + if (!LeftShift) + return std::unexpected(LeftShift.error()); + auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI, + *ExtendedLeft, *ShiftedLeft, 32); + if (!LeftExtend) + return std::unexpected(LeftExtend.error()); + auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI, + *ShiftedRight, RightRegister, 32); + if (!RightShift) + return std::unexpected(RightShift.error()); + auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI, + *ExtendedRight, *ShiftedRight, 32); + if (!RightExtend) + return std::unexpected(RightExtend.error()); + Anchor = *RightExtend; + } else { + auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, + *ExtendedLeft, LeftRegister, 0); + if (!LeftExtend) + return std::unexpected(LeftExtend.error()); + auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW, + *ExtendedRight, RightRegister, 0); + if (!RightExtend) + return std::unexpected(RightExtend.error()); + Anchor = *RightExtend; + } + auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct, + *ExtendedLeft, *ExtendedRight); + if (!Product) + return std::unexpected(Product.error()); + if (IsUnsigned) { + auto High = createTemporary(Function, *I64, Target); + if (!High) + return std::unexpected(High.error()); + auto ExtractHigh = + buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32); + if (!ExtractHigh) + return std::unexpected(ExtractHigh.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *High); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + auto Difference = createTemporary(Function, *I64, Target); + if (!Difference) + return std::unexpected(Difference.error()); + auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference, + *FullProduct, ResultRegister); + if (!Compare) + return std::unexpected(Compare.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *Difference); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + [[nodiscard]] Expected selectExtendedOverflow(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) const { + const MachineOperand *ResultOperand = Instruction.operandAt(0); + const MachineOperand *CarryOutOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const MachineOperand *CarryInOperand = Instruction.operandAt(4); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + const Register *CarryOut = + CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const Register *CarryIn = + CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf(); + if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr || + CarryIn == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V extended overflow operation has invalid operands"); + } + const Register ResultRegister = *Result; + const Register CarryOutRegister = *CarryOut; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const Register CarryInRegister = *CarryIn; + const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE; + const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE; + const bool IsWord = Type->scalarBitWidth() <= 32; + const InstructionOpcode Arithmetic = + IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) + : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB); + auto Partial = createTemporary(Function, *Type, Target); + auto I1 = LLT::makeInteger(1); + if (!Partial || !I1) + return std::unexpected(!Partial ? Partial.error() : I1.error()); + auto NormalizedCarry = createTemporary(Function, *I1, Target); + if (!NormalizedCarry) + return std::unexpected(NormalizedCarry.error()); + auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial); + if (!SetPartial) + return std::unexpected(SetPartial.error()); + auto RemoveCarryOut = Instruction.eraseOperand(1); + if (!RemoveCarryOut) + return std::unexpected(RemoveCarryOut.error()); + auto RemoveCarryIn = Instruction.eraseOperand(3); + if (!RemoveCarryIn) + return std::unexpected(RemoveCarryIn.error()); + auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); + if (!SetArithmetic) + return std::unexpected(SetArithmetic.error()); + MachineIRBuilder Builder(Function, Target); + auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI, + *NormalizedCarry, CarryInRegister, 1); + if (!Normalize) + return std::unexpected(Normalize.error()); + auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial, + *NormalizedCarry); + if (!Final) + return std::unexpected(Final.error()); + + if (IsUnsigned) { + auto FirstCarry = createTemporary(Function, *I1, Target); + auto SecondCarry = createTemporary(Function, *I1, Target); + if (!FirstCarry || !SecondCarry) + return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error()); + auto First = + buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry, + IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister); + if (!First) + return std::unexpected(First.error()); + auto Second = + buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry, + IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry); + if (!Second) + return std::unexpected(Second.error()); + auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister, + *FirstCarry, *SecondCarry); + if (!Combined) + return std::unexpected(Combined.error()); + return {}; + } + + auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type, + *Partial, LeftRegister, RightRegister, IsAdd); + if (!FirstOverflow) + return std::unexpected(FirstOverflow.error()); + auto SecondOverflow = + buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type, + ResultRegister, *Partial, *NormalizedCarry, IsAdd); + if (!SecondOverflow) + return std::unexpected(SecondOverflow.error()); + auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR, + CarryOutRegister, FirstOverflow->first, SecondOverflow->first); + if (!Combined) + return std::unexpected(Combined.error()); + return {}; + } + + [[nodiscard]] Expected selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) const { + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const MachineOperand *ValueOperand = Instruction.operandAt(2); + const Register *Address = + AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + if (!Type || Address == nullptr || Value == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands"); + Register SelectedValue = *Value; + const bool IsWord = Type->scalarBitWidth() == 32; + if (Generic == GenericOpcode::G_ATOMICRMW_SUB) { + auto Negated = createTemporary(Function, *Type, Target); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Negated || !Zero) + return std::unexpected(!Negated ? Negated.error() : Zero.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr( + targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(*Negated); + if (!Def) + return std::unexpected(Def.error()); + auto Left = Pending->addUse(Register::fromPhysical(*Zero)); + if (!Left) + return std::unexpected(Left.error()); + auto Right = Pending->addUse(*Value); + if (!Right) + return std::unexpected(Right.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + SelectedValue = *Negated; + } + const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode { + switch (Generic) { + case GenericOpcode::G_ATOMICRMW_XCHG: + return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D; + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D; + case GenericOpcode::G_ATOMICRMW_AND: + return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D; + case GenericOpcode::G_ATOMICRMW_OR: + return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D; + case GenericOpcode::G_ATOMICRMW_XOR: + return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D; + case GenericOpcode::G_ATOMICRMW_MAX: + return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D; + case GenericOpcode::G_ATOMICRMW_MIN: + return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D; + case GenericOpcode::G_ATOMICRMW_UMAX: + return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D; + case GenericOpcode::G_ATOMICRMW_UMIN: + return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D; + default: + return InstructionOpcode::Invalid; + } + }; + const InstructionOpcode Selected = SelectOpcode(); + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V atomic RMW selection reached an unhandled opcode"); + auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue); + if (!SetValue) + return std::unexpected(SetValue.error()); + auto SetAddress = Instruction.operandAt(2)->setRegister(*Address); + if (!SetAddress) + return std::unexpected(SetAddress.error()); + return Instruction.setOpcode(targetOpcode(Selected)); + } + + [[nodiscard]] Expected selectIntegerExtension(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + GenericOpcode Generic, + std::optional Type) const { + const std::optional SourceType = operandType(Function, Instruction, 1); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || !SourceType || Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer extension has invalid register operands"); + const Register DestinationRegister = *Destination; + if (Generic == GenericOpcode::G_ANYEXT) + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + + const std::uint32_t DestinationBits = Type->scalarBitWidth(); + const std::uint32_t SourceBits = SourceType->scalarBitWidth(); + if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) { + auto ImmediateResult = appendImmediate(Instruction, 0); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + } + + const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits - SourceBits); + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + Generic == GenericOpcode::G_SEXT + ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) + : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ShiftImmediate = appendImmediate(Instruction, Shift); + if (!ShiftImmediate) + return std::unexpected(ShiftImmediate.error()); + + MachineIRBuilder Builder(Function, Target); + auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, + *Temporary, Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + return {}; + } + + [[nodiscard]] Expected selectSignExtendInRegister(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) const { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + std::optional Width; + if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) { + auto ParsedWidth = signedImmediate(*WidthOperand); + if (ParsedWidth) + Width = *ParsedWidth; + } + if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 || + static_cast(*Width) > Type->scalarBitWidth()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V sign-extension-in-register has invalid operands"); + const Register DestinationRegister = *Destination; + if (static_cast(*Width) == Type->scalarBitWidth()) { + auto EraseWidth = Instruction.eraseOperand(2); + if (!EraseWidth) + return std::unexpected(EraseWidth.error()); + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + } + + const std::uint32_t TypeBits = Type->scalarBitWidth(); + if (TypeBits == 64 && *Width == 32) { + auto ImmediateResult = replaceImmediate(Instruction, 2, 0); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + } + const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits) - *Width; + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI; + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift); + if (!ShiftImmediate) + return std::unexpected(ShiftImmediate.error()); + + MachineIRBuilder Builder(Function, Target); + auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, + *Temporary, Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + return {}; + } + + [[nodiscard]] Expected selectIntegerCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + const MachineOperand *PredicateMO = Instruction.operandAt(1); + const auto *Predicate = + PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const std::optional LeftType = operandType(Function, Instruction, 2); + if (Predicate == nullptr || Predicate->integerPredicate() == nullptr || + Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer comparison has invalid operands"); + const IntegerPredicate Condition = *Predicate->integerPredicate(); + const Register DestinationRegister = *Destination; + const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan || + Condition == IntegerPredicate::SignedGreaterEqual || + Condition == IntegerPredicate::SignedLessThan || + Condition == IntegerPredicate::SignedLessEqual; + auto LeftRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands); + auto RightRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands); + if (!LeftRegister || !RightRegister) + return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); + auto ErasePredicate = Instruction.eraseOperand(1); + if (!ErasePredicate) + return std::unexpected(ErasePredicate.error()); + + const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected { + auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + return Instruction.operandAt(2)->setRegister(NewRight); + }; + const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional { + switch (Value) { + case IntegerPredicate::SignedLessThan: + return InstructionOpcode::SLT; + case IntegerPredicate::UnsignedLessThan: + return InstructionOpcode::SLTU; + default: + return std::nullopt; + } + }; + + IntegerPredicate StrictCondition = Condition; + bool Swap = false; + bool Invert = false; + switch (Condition) { + case IntegerPredicate::Equal: + case IntegerPredicate::NotEqual: + break; + case IntegerPredicate::SignedGreaterThan: + StrictCondition = IntegerPredicate::SignedLessThan; + Swap = true; + break; + case IntegerPredicate::SignedGreaterEqual: + StrictCondition = IntegerPredicate::SignedLessThan; + Invert = true; + break; + case IntegerPredicate::SignedLessThan: + StrictCondition = IntegerPredicate::SignedLessThan; + break; + case IntegerPredicate::SignedLessEqual: + StrictCondition = IntegerPredicate::SignedLessThan; + Swap = true; + Invert = true; + break; + case IntegerPredicate::UnsignedGreaterThan: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Swap = true; + break; + case IntegerPredicate::UnsignedGreaterEqual: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Invert = true; + break; + case IntegerPredicate::UnsignedLessThan: + StrictCondition = IntegerPredicate::UnsignedLessThan; + break; + case IntegerPredicate::UnsignedLessEqual: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Swap = true; + Invert = true; + break; + case IntegerPredicate::Count: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer comparison predicate is invalid"); + } + + MachineIRBuilder Builder(Function, Target); + if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) { + auto Temporary = createTemporary(Function, *LeftType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + if (Condition == IntegerPredicate::Equal) { + auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU, + DestinationRegister, *Temporary, 1); + if (!Result) + return std::unexpected(Result.error()); + } else { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Result = + buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister, + Register::fromPhysical(*Zero), *Temporary); + if (!Result) + return std::unexpected(Result.error()); + } + return {}; + } + + const std::optional OpcodeValue = StrictOpcode(StrictCondition); + if (!OpcodeValue) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer comparison predicate is invalid"); + if (Invert) { + auto Temporary = createTemporary(Function, *LeftType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + const Register Base = *Temporary; + if (Swap) { + auto Set = SetSources(*RightRegister, *LeftRegister); + if (!Set) + return std::unexpected(Set.error()); + } else { + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + } + auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, + DestinationRegister, Base, 1); + if (!Result) + return std::unexpected(Result.error()); + return {}; + } + if (Swap) { + auto Set = SetSources(*RightRegister, *LeftRegister); + if (!Set) + return std::unexpected(Set.error()); + } else { + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + } + return Instruction.setOpcode(targetOpcode(*OpcodeValue)); + } + + [[nodiscard]] Expected selectFloatingCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + const MachineOperand *PredicateMO = Instruction.operandAt(1); + const auto *Predicate = + PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const std::optional FloatType = operandType(Function, Instruction, 2); + const std::optional ResultType = resultType(Function, Instruction); + if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr || + Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating comparison has invalid operands"); + const FloatingPredicate Condition = *Predicate->floatingPredicate(); + const Register DestinationRegister = *Destination; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const InstructionOpcode EqualOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D; + const InstructionOpcode LessOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D; + const InstructionOpcode LessEqualOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D; + + auto ErasePredicate = Instruction.eraseOperand(1); + if (!ErasePredicate) + return std::unexpected(ErasePredicate.error()); + const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def, + Register NewLeft, Register NewRight) -> Expected { + auto SetDestination = Instruction.operandAt(0)->setRegister(Def); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(NewRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + return Instruction.setOpcode(targetOpcode(OpcodeValue)); + }; + const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft, + Register BaseRight) -> Expected { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, + DestinationRegister, *Temporary, 1); + if (!Invert) + return std::unexpected(Invert.error()); + return {}; + }; + const auto BuildSelfAnd = [&](bool Invert) -> Expected { + auto First = createTemporary(Function, *ResultType, Target); + auto Second = createTemporary(Function, *ResultType, Target); + if (!First || !Second) + return std::unexpected((!First ? First.error() : Second.error())); + auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second, + RightRegister, RightRegister); + if (!SecondInstruction) + return std::unexpected(SecondInstruction.error()); + Register CombinedDestination = DestinationRegister; + if (Invert) { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + CombinedDestination = *Temporary; + } + auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND, + CombinedDestination, *First, *Second); + if (!Combined) + return std::unexpected(Combined.error()); + if (!Invert) + return {}; + auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, + DestinationRegister, CombinedDestination, 1); + if (!Inverted) + return std::unexpected(Inverted.error()); + return {}; + }; + const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected { + auto First = createTemporary(Function, *ResultType, Target); + auto Second = createTemporary(Function, *ResultType, Target); + if (!First || !Second) + return std::unexpected((!First ? First.error() : Second.error())); + auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto SecondInstruction = + buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister); + if (!SecondInstruction) + return std::unexpected(SecondInstruction.error()); + Register CombinedDestination = DestinationRegister; + if (Invert) { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + CombinedDestination = *Temporary; + } + auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR, + CombinedDestination, *First, *Second); + if (!Combined) + return std::unexpected(Combined.error()); + if (!Invert) + return {}; + auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, + DestinationRegister, CombinedDestination, 1); + if (!Inverted) + return std::unexpected(Inverted.error()); + return {}; + }; + + switch (Condition) { + case FloatingPredicate::AlwaysFalse: + case FloatingPredicate::AlwaysTrue: { + auto EraseRight = Instruction.eraseOperand(2); + if (!EraseRight) + return std::unexpected(EraseRight.error()); + auto EraseLeft = Instruction.eraseOperand(1); + if (!EraseLeft) + return std::unexpected(EraseLeft.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!InsertZero) + return std::unexpected(InsertZero.error()); + auto Immediate = + appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI)); + } + case FloatingPredicate::OrderedEqual: + return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedGreaterThan: + return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister); + case FloatingPredicate::OrderedGreaterEqual: + return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister); + case FloatingPredicate::OrderedLessThan: + return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedLessEqual: + return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedNotEqual: + return BuildOrderedNotEqual(false); + case FloatingPredicate::Ordered: + return BuildSelfAnd(false); + case FloatingPredicate::Unordered: + return BuildSelfAnd(true); + case FloatingPredicate::UnorderedEqual: + return BuildOrderedNotEqual(true); + case FloatingPredicate::UnorderedGreaterThan: + return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister); + case FloatingPredicate::UnorderedGreaterEqual: + return BuildInverted(LessOpcode, LeftRegister, RightRegister); + case FloatingPredicate::UnorderedLessThan: + return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister); + case FloatingPredicate::UnorderedLessEqual: + return BuildInverted(LessOpcode, RightRegister, LeftRegister); + case FloatingPredicate::UnorderedNotEqual: + return BuildInverted(EqualOpcode, LeftRegister, RightRegister); + case FloatingPredicate::Count: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V floating comparison predicate is invalid"); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V floating comparison predicate is invalid"); + } + + [[nodiscard]] Expected selectFloatToInteger(MachineInstr &Instruction, + MachineFunction &Function, + std::optional Type, bool Unsigned) const { + const std::optional Source = operandType(Function, Instruction, 1); + if (!Type || !Source) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating-to-integer conversion has invalid types"); + auto Rounding = appendImmediate(Instruction, 1); + if (!Rounding) + return std::unexpected(Rounding.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + if (Source->scalarBitWidth() == 32) { + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S) + : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S))); + } + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D) + : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D))); + } + + [[nodiscard]] Expected selectIntegerToFloat(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Unsigned) const { + const std::optional Source = operandType(Function, Instruction, 1); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *SourceRegister = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || !Source || SourceRegister == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer-to-floating conversion has invalid types"); + auto Normalized = + normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned); + if (!Normalized) + return std::unexpected(Normalized.error()); + auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized); + if (!SetSource) + return std::unexpected(SetSource.error()); + auto Rounding = appendImmediate(Instruction, 0); + if (!Rounding) + return std::unexpected(Rounding.error()); + const bool IsWord = Source->scalarBitWidth() <= 32; + if (Type->scalarBitWidth() == 32) { + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W) + : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L))); + } + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W) + : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L))); + } + + [[nodiscard]] Expected selectFloatingMinMax(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) const { + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V cannot select this floating min/max type"); + } + + [[nodiscard]] Expected selectFence(MachineInstr &Instruction) const { + const auto Ordering = signedImmediate(*Instruction.operandAt(0)); + if (!Ordering) + return std::unexpected(Ordering.error()); + std::int64_t Pred = 0; + std::int64_t Succ = 0; + switch (*Ordering) { + case 4: + Pred = 1; + Succ = 3; + break; + case 5: + Pred = 3; + Succ = 2; + break; + case 6: + case 7: + Pred = 3; + Succ = 3; + break; + default: + return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported"); + } + auto SetPred = replaceImmediate(Instruction, 0, Pred); + if (!SetPred) + return std::unexpected(SetPred.error()); + auto SetSucc = replaceImmediate(Instruction, 1, Succ); + if (!SetSucc) + return std::unexpected(SetSucc.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE)); + } + + [[nodiscard]] Expected selectIndirectBranch(MachineInstr &Instruction) const { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Inserted) + return std::unexpected(Inserted.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR)); + } + + [[nodiscard]] Expected selectTrap(MachineInstr &Instruction) const { + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return std::unexpected(Erased.error()); + } + return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK)); + } + + [[nodiscard]] Expected selectBitcast(MachineInstr &Instruction, + MachineFunction &Function) const { + const std::optional DestinationType = resultType(Function, Instruction); + const std::optional SourceType = operandType(Function, Instruction, 1); + if (!DestinationType || !SourceType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V bitcast has invalid register types"); + if (DestinationType->isInteger() && SourceType->isFloat()) { + return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32 + ? InstructionOpcode::FMV_X_W + : InstructionOpcode::FMV_X_D)); + } + if (DestinationType->isFloat() && SourceType->isInteger()) { + return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32 + ? InstructionOpcode::FMV_W_X + : InstructionOpcode::FMV_D_X)); + } + return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported"); + } + + [[nodiscard]] Expected selectIntegerBinary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode DoubleWord, + InstructionOpcode Word) const { + if (!Type || !isSupportedIntegerType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer instruction requires a supported scalar integer"); + const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); + } + + [[nodiscard]] Expected + selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, std::optional Type, + InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed, + bool NormalizeRight) const { + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V normalized integer operation has invalid operands"); + } + auto NormalizedLeft = + normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); + if (!NormalizedLeft) + return std::unexpected(NormalizedLeft.error()); + Register SelectedRight = *Right; + if (NormalizeRight) { + auto NormalizedRight = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); + if (!NormalizedRight) + return std::unexpected(NormalizedRight.error()); + SelectedRight = *NormalizedRight; + } + auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); + } + + [[nodiscard]] Expected selectLoad(MachineInstr &Instruction, + std::optional Type) const { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "load result has no virtual-register type"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V load size does not match its result type"); + if (Instruction.memOperands().front()->isAtomic()) { + if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V atomic load requires a 32-bit or 64-bit GPR value"); + return Instruction.setOpcode( + targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D)); + } + auto AppendOffset = appendImmediate(Instruction, 0); + if (!AppendOffset) + return std::unexpected(AppendOffset.error()); + if (isSupportedIntegerType(*Type)) { + const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB + : *Bytes == 2 ? InstructionOpcode::LH + : *Bytes == 4 ? InstructionOpcode::LW + : *Bytes == 8 ? InstructionOpcode::LD + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported"); + return Instruction.setOpcode(targetOpcode(Selected)); + } + if (Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); + if (Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type"); + } + + [[nodiscard]] Expected selectExtLoad(MachineInstr &Instruction, std::optional Type, + bool Signed) const { + if (!Type || !isSupportedIntegerType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V extension load requires an integer result"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width"); + const InstructionOpcode Selected = + *Bytes == 1 ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU) + : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU) + : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU) + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported"); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(Selected)); + } + + [[nodiscard]] Expected selectFloatExtLoad(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating extension load has no destination register"); + const Register DestinationRegister = *Destination; + auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + if (!F32) + return std::unexpected(F32.error()); + auto Temporary = createTemporary(Function, *F32, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + MachineIRBuilder Builder(Function, Target); + auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S, + DestinationRegister, *Temporary, 0); + if (!Conversion) + return std::unexpected(Conversion.error()); + return {}; + } + + [[nodiscard]] Expected selectFloatTruncStore(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating truncation store has no value register"); + const Register ValueRegister = *Value; + auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + if (!F32) + return std::unexpected(F32.error()); + auto Temporary = createTemporary(Function, *F32, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D)); + if (!Conversion) + return std::unexpected(Conversion.error()); + auto Def = Conversion->addDef(*Temporary); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Conversion->addUse(ValueRegister); + if (!Use) + return std::unexpected(Use.error()); + auto Rounding = Conversion->addImmediate(0); + if (!Rounding) + return std::unexpected(Rounding.error()); + auto Built = Conversion->build(); + if (!Built) + return std::unexpected(Built.error()); + auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetValue) + return std::unexpected(SetValue.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); + } + + [[nodiscard]] Expected selectStore(MachineInstr &Instruction, + MachineFunction &Function) const { + const MachineOperand *Value = Instruction.operandAt(0); + const std::optional Type = + Value == nullptr ? std::nullopt : registerType(Function, *Value); + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "store value has no virtual-register type"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V store size does not match its value type"); + if (Instruction.memOperands().front()->isAtomic()) { + if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V atomic store requires a 32-bit or 64-bit GPR value"); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Destination) + return std::unexpected(Destination.error()); + return Instruction.setOpcode( + targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D)); + } + auto AppendOffset = appendImmediate(Instruction, 0); + if (!AppendOffset) + return std::unexpected(AppendOffset.error()); + if (isSupportedIntegerType(*Type)) { + const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB + : *Bytes == 2 ? InstructionOpcode::SH + : *Bytes == 4 ? InstructionOpcode::SW + : *Bytes == 8 ? InstructionOpcode::SD + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported"); + return Instruction.setOpcode(targetOpcode(Selected)); + } + if (Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); + if (Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type"); + } + + [[nodiscard]] Expected selectBranch(MachineInstr &Instruction) const { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL)); + } + + [[nodiscard]] Expected selectConditionalBranch(MachineInstr &Instruction) const { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE)); + } + + [[nodiscard]] Expected selectIntegerConstant(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) const { + if (!Type || !isSupportedIntegerType(*Type)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an i1-i64 or pointer result"); + } + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + if (ImmediateOperand == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V generic constant has no immediate operand"); + auto ImmediateValue = signedImmediate(*ImmediateOperand); + if (!ImmediateValue) + return std::unexpected(ImmediateValue.error()); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *DestinationValue = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (DestinationValue == nullptr || !DestinationValue->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V generic constant must define a virtual register"); + } + const Register Destination = *DestinationValue; + const std::int64_t Value = *ImmediateValue; + const std::uint32_t ImmediateTargetFlags = ImmediateOperand->targetFlags(); + const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + const InstructionOpcode AddImmediate = + IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI; + const InstructionOpcode ShiftLeftImmediate = + IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + if (fitsSigned(Value, 12)) { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!Source) + return std::unexpected(Source.error()); + auto EraseImmediate = Instruction.eraseOperand(2); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + auto NormalizeImmediate = + Instruction.insertOperand(2, MachineOperand::immediate(Value, ImmediateTargetFlags)); + if (!NormalizeImmediate) + return std::unexpected(NormalizeImmediate.error()); + return Instruction.setOpcode(targetOpcode(AddImmediate)); + } + + // Recursively decompose the value into signed 12-bit chunks. The + // quotient is computed from an unsigned bit pattern with explicit sign + // extension so both INT64_MIN and INT64_MAX remain well-defined. + std::uint64_t Remaining = static_cast(Value); + std::vector Digits; + while (!fitsSigned(std::bit_cast(Remaining), 12)) { + const std::uint64_t LowBits = Remaining & 0xFFFU; + std::int64_t Digit = static_cast(LowBits); + if (Digit >= 0x800) + Digit -= 0x1000; + Digits.push_back(Digit); + + const std::uint64_t Adjusted = Remaining - static_cast(Digit); + std::uint64_t Quotient = Adjusted >> 12U; + if ((Remaining & (std::uint64_t{1} << 63U)) != 0) + Quotient |= ~std::uint64_t{0} << 52U; + Remaining = Quotient; + } + const std::int64_t Base = std::bit_cast(Remaining); + + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto *Parent = Instruction.parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant selection received a detached instruction"); + auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto EraseImmediate = Instruction.eraseOperand(1); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!AddSource) + return std::unexpected(AddSource.error()); + auto AddBase = appendImmediate(Instruction, Base); + if (!AddBase) + return std::unexpected(AddBase.error()); + + MachineIRBuilder Builder(Function, Target); + auto InsertPoint = Builder.setInsertPointAfter(Instruction); + if (!InsertPoint) + return std::unexpected(InsertPoint.error()); + + Register Current = *Temporary; + for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) { + const std::size_t Index = Reverse - 1U; + auto ShiftResult = createTemporary(Function, *Type, Target); + if (!ShiftResult) + return std::unexpected(ShiftResult.error()); + auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate)); + if (!Shift) + return std::unexpected(Shift.error()); + auto ShiftDef = Shift->addDef(*ShiftResult); + if (!ShiftDef) + return std::unexpected(ShiftDef.error()); + auto ShiftUse = Shift->addUse(Current); + if (!ShiftUse) + return std::unexpected(ShiftUse.error()); + auto ShiftAmount = Shift->addImmediate(12); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto BuiltShift = Shift->build(); + if (!BuiltShift) + return std::unexpected(BuiltShift.error()); + + const bool IsFinal = Index == 0; + Register AddDestination; + if (IsFinal) { + AddDestination = Destination; + } else { + auto AddTemporary = createTemporary(Function, *Type, Target); + if (!AddTemporary) + return std::unexpected(AddTemporary.error()); + AddDestination = *AddTemporary; + } + auto Add = Builder.setInsertPointAfter(**BuiltShift); + if (!Add) + return std::unexpected(Add.error()); + auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate)); + if (!AddInstruction) + return std::unexpected(AddInstruction.error()); + auto AddDef = AddInstruction->addDef(AddDestination); + if (!AddDef) + return std::unexpected(AddDef.error()); + auto AddUse = AddInstruction->addUse(*ShiftResult); + if (!AddUse) + return std::unexpected(AddUse.error()); + auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]); + if (!AddImmediateResult) + return std::unexpected(AddImmediateResult.error()); + auto BuiltAdd = AddInstruction->build(); + if (!BuiltAdd) + return std::unexpected(BuiltAdd.error()); + Current = AddDestination; + } + return {}; + } + + [[nodiscard]] Expected selectFloatingConstant(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) const { + if (!Type || !isRVFloatType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64"); + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type->floatSemantics(); + if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating constant has mismatched bit semantics"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || !Destination->isVirtual()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating constant must define a virtual register"); + const Register DestinationRegister = *Destination; + + const std::uint32_t BitWidth = Type->scalarBitWidth(); + const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front(); + auto Integer = IntegerImmediate::fromUnsigned(64, Bits); + if (!Integer) + return std::unexpected(Integer.error()); + auto IntegerType = LLT::makeInteger(64); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto Temporary = createTemporary(Function, *IntegerType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto EraseImmediate = Instruction.eraseOperand(1); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer)); + if (!AddInteger) + return std::unexpected(AddInteger.error()); + auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType); + if (!IntegerSelection) + return std::unexpected(IntegerSelection.error()); + + MachineInstr *IntegerDefinition = nullptr; + for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) { + IntegerDefinition = DefOperand.parent(); + break; + } + if (IntegerDefinition == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V floating constant lost its integer materialization"); + MachineIRBuilder Builder(Function, Target); + auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition); + if (!InsertPoint) + return std::unexpected(InsertPoint.error()); + const InstructionOpcode Move = + BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + auto MoveInstruction = Builder.buildInstr(targetOpcode(Move)); + if (!MoveInstruction) + return std::unexpected(MoveInstruction.error()); + auto MoveDef = MoveInstruction->addDef(DestinationRegister); + if (!MoveDef) + return std::unexpected(MoveDef.error()); + auto MoveUse = MoveInstruction->addUse(*Temporary); + if (!MoveUse) + return std::unexpected(MoveUse.error()); + auto BuiltMove = MoveInstruction->build(); + if (!BuiltMove) + return std::unexpected(BuiltMove.error()); + return {}; + } + + [[nodiscard]] Expected selectRoundingBinary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) const { + auto RoundingMode = appendImmediate(Instruction, 0); + if (!RoundingMode) + return std::unexpected(RoundingMode.error()); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type"); + } + + [[nodiscard]] Expected selectRoundingTernary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) const { + return selectRoundingBinary(Instruction, Type, Single, Double); + } + + [[nodiscard]] Expected selectRoundingUnary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) const { + return selectRoundingBinary(Instruction, Type, Single, Double); + } + + [[nodiscard]] Expected selectSignInjection(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) const { + const MachineOperand *Source = Instruction.operandAt(1); + const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf(); + if (SourceReg == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating sign operation requires a register source"); + auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg); + if (!Duplicate) + return std::unexpected(Duplicate.error()); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type"); + } + const RISCVSubtarget &Subtarget; + const RISCVRegisterBankInfo &RegisterBanks; +}; + +} // namespace + +InstructionSelector *createRISCVInstructionSelector(const RISCVSubtarget &Subtarget, + const RISCVRegisterBankInfo &RegisterBanks) { + return new RISCVInstructionSelector(Subtarget, RegisterBanks); +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp new file mode 100644 index 0000000..e9d72fd --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -0,0 +1,3818 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RISCVLegalizerInfo.h" + +#include "RISCVCallLowering.h" +#include "RISCVImmediateMaterialization.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::optional registerType(const MachineFunction &Function, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::nullopt; + return Info->get().Type; +} + +[[nodiscard]] std::optional resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { + const MachineOperand *Result = Instruction.operandAt(0); + return Result == nullptr ? std::nullopt : registerType(Function, *Result); +} + +[[nodiscard]] std::optional operandType(const MachineFunction &Function, + const MachineInstr &Instruction, std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); +} + +[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { + if (!Type.isFloat()) + return false; + const std::optional Semantics = Type.floatSemantics(); + return Semantics && + (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +} + +[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64; +} + +[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && + (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isI1Type(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; +} + +[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { + return (Destination.isInteger() || Destination.isAnyScalar()) && + (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && + Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && + Destination.scalarBitWidth() <= 64; +} + +[[nodiscard]] Expected extendIntegerImmediate(const IntegerImmediate &Value, + std::uint32_t DestinationWidth, + bool SignExtend) { + if (DestinationWidth == 0 || DestinationWidth > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires a destination no wider than 64 bits"); + + std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front(); + const std::uint32_t SourceWidth = Value.bitWidth(); + if (SignExtend && Value.isNegative() && SourceWidth < 64) + Raw |= ~std::uint64_t{0} << SourceWidth; + if (DestinationWidth < 64) + Raw &= (std::uint64_t{1} << DestinationWidth) - 1U; + return IntegerImmediate::create(DestinationWidth, std::span(&Raw, 1)); +} + +[[nodiscard]] Expected +foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) { + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || + !Destination->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension requires virtual-register operands"); + } + const Register SourceRegister = *Source; + const Register DestinationRegister = *Destination; + const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister); + const MachineInstr *SourceDef = + SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); + if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires a single constant definition"); + const MachineOperand *ConstantOperand = SourceDef->operandAt(1); + const auto *Integer = + ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf(); + if (Integer == nullptr) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires an integer constant"); + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + if (!DestinationInfo || !DestinationInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension destination has no type"); + const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth(); + const bool SignExtend = Generic != GenericOpcode::G_ZEXT; + auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend); + if (!Extended) + return std::unexpected(Extended.error()); + + if (Function.registerInfo().hasOneUse(SourceRegister)) { + MachineInstr *Def = const_cast(SourceDef); + if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced) + return std::unexpected(Replaced.error()); + auto Erased = Def->eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + MachineBasicBlock *Parent = Instruction.parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension is detached during folding"); + return Parent->removeInstruction(Instruction); + } + + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + + if (Function.registerInfo().useEmpty(SourceRegister)) { + MachineInstr *Def = const_cast(SourceDef); + MachineBasicBlock *Parent = Def->parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension source definition is detached"); + auto Removed = Parent->removeInstruction(*Def); + if (!Removed) + return std::unexpected(Removed.error()); + // Keep the source register record as a tombstone-free historical vreg until + // the legalizer worklist transaction has completed. + } + return {}; +} + +struct WideConstantUnmergePlan final { + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = nullptr; + std::vector Outputs; + bool SignExtend = false; +}; + +[[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { + MachineInstr *UseInstruction = nullptr; + auto &RegisterInfo = const_cast(Function.registerInfo()); + for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { + if (Operand.parent() == nullptr) + continue; + if (UseInstruction != nullptr) + return nullptr; + UseInstruction = Operand.parent(); + } + return UseInstruction; +} + +[[nodiscard]] std::optional +wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { + const MachineOperand *ResultOperand = Constant.operandAt(0); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + if (Result == nullptr || !Result->isVirtual()) + return std::nullopt; + + MachineInstr *FirstUse = singleRegisterUse(Function, *Result); + if (FirstUse == nullptr) + return std::nullopt; + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = FirstUse; + if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || + FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || + FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { + Extension = FirstUse; + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + if (Extended == nullptr || !Extended->isVirtual()) + return std::nullopt; + Unmerge = singleRegisterUse(Function, *Extended); + if (Unmerge == nullptr) + return std::nullopt; + } + if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3) + return std::nullopt; + + WideConstantUnmergePlan Plan; + Plan.Extension = Extension; + Plan.Unmerge = Unmerge; + Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); + for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { + const MachineOperand *Operand = Unmerge->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) + return std::nullopt; + const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 || + Info->get().Type->scalarBitWidth() > 64 || + (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar())) + return std::nullopt; + Plan.Outputs.push_back(*Reg); + } + const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + const Register *ExpectedSource = Result; + if (Extension != nullptr) { + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + } + if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) + return std::nullopt; + return Plan; +} + +[[nodiscard]] Expected wideConstantWord(const IntegerImmediate &Value, + std::size_t WordIndex, + std::uint32_t Width, bool SignExtend) { + std::uint64_t Raw = 0; + if (WordIndex < Value.words().size()) + Raw = Value.words()[WordIndex]; + else if (SignExtend && Value.isNegative()) + Raw = std::numeric_limits::max(); + const std::size_t WordBitOffset = WordIndex * 64U; + if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { + const std::uint32_t Remaining = Value.bitWidth() - static_cast(WordBitOffset); + if (Remaining < 64) + Raw |= ~std::uint64_t{0} << Remaining; + } + if (Width < 64) + Raw &= (std::uint64_t{1} << Width) - 1U; + return IntegerImmediate::create(Width, std::span(&Raw, 1)); +} + +[[nodiscard]] Expected foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = wideConstantUnmergePlan(Instruction, Function); + if (!Plan) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide constant requires a G_UNMERGE_VALUES use"); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Integer = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); + if (Integer == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant has no integer immediate"); + + std::vector Words; + Words.reserve(Plan->Outputs.size()); + for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { + const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant output has no virtual-register type"); + auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(), + Plan->SignExtend); + if (!Word) + return std::unexpected(Word.error()); + Words.push_back(std::move(*Word)); + } + + std::vector Representatives(Plan->Outputs.size()); + for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { + Representatives[Index] = Index; + for (std::size_t Previous = 0; Previous < Index; ++Previous) { + if (Words[Previous] == Words[Index]) { + Representatives[Index] = Previous; + break; + } + } + } + + const Register FirstOutput = Plan->Outputs.front(); + auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); + if (!Added) + return std::unexpected(Added.error()); + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant is detached during legalization"); + if (Plan->Unmerge != &Instruction) { + auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); + if (!RemovedUnmerge) + return std::unexpected(RemovedUnmerge.error()); + } + if (Plan->Extension != nullptr) { + auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); + if (!RemovedExtension) + return std::unexpected(RemovedExtension.error()); + } + + for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { + if (Representatives[Index] == Index) + continue; + auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index], + Plan->Outputs[Representatives[Index]]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + + MachineInstr *Last = &Instruction; + for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { + if (Representatives[Index] != Index) + continue; + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointAfter(*Last); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Plan->Outputs[Index]); + if (!Def) + return std::unexpected(Def.error()); + auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); + if (!Value) + return std::unexpected(Value.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + Last = *Built; + } + return {}; +} + +[[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::uint32_t Width, + std::uint64_t Value); + +[[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Left, + Register Right); + +[[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Source); + +[[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction); + +[[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Source); + +struct NarrowAnyExtBinaryPlan final { + MachineInstr *Binary = nullptr; + GenericOpcode Generic = GenericOpcode::Invalid; + std::array WideInputs{}; + std::vector Artifacts; + std::vector Constants; + std::uint32_t SourceWidth = 0; +}; + +[[nodiscard]] std::optional +narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { + const MachineOperand *DestinationOperand = Extension.operandAt(0); + const MachineOperand *SourceOperand = Extension.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) + return std::nullopt; + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); + if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type) + return std::nullopt; + const LLT DestinationType = *DestinationInfo->get().Type; + const LLT SourceType = *SourceInfo->get().Type; + if (!isScalarIntegerExtension(DestinationType, SourceType) || + DestinationType.scalarBitWidth() != 64) + return std::nullopt; + + const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); + MachineInstr *Binary = SourceDefinition == nullptr + ? nullptr + : const_cast(SourceDefinition->parent()); + if (Binary == nullptr || Binary->operandCount() != 3) + return std::nullopt; + const GenericOpcode Generic = static_cast(Binary->opcode().value()); + if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && + Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && + Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && + Generic != GenericOpcode::G_SHL) + return std::nullopt; + + NarrowAnyExtBinaryPlan Plan; + Plan.Binary = Binary; + Plan.Generic = Generic; + Plan.SourceWidth = SourceType.scalarBitWidth(); + const auto AddArtifact = [&Plan](MachineInstr *Artifact) { + if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) + Plan.Artifacts.push_back(Artifact); + }; + const auto TruncatedWideSource = [&](MachineInstr *Truncation, + MachineInstr *OnlyUse) -> std::optional { + if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || + Truncation->operandCount() != 2) + return std::nullopt; + const MachineOperand *NarrowOperand = Truncation->operandAt(0); + const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); + const MachineOperand *WideOperand = Truncation->operandAt(1); + const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); + const std::optional WideType = + WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); + if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || + *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) + return std::nullopt; + return *Wide; + }; + for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { + const MachineOperand *Operand = Binary->operandAt(Index + 1U); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info || !Info->get().Type) + return std::nullopt; + if (*Info->get().Type == DestinationType) { + Plan.WideInputs[Index] = *Reg; + continue; + } + if (*Info->get().Type != SourceType) + return std::nullopt; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (auto Wide = TruncatedWideSource(Def, Binary)) { + Plan.WideInputs[Index] = *Wide; + AddArtifact(Def); + continue; + } + if (Def != nullptr && + (Def->opcode() == opcode(GenericOpcode::COPY) || + Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && + Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { + const MachineOperand *NarrowOperand = Def->operandAt(1); + const Register *Narrow = + NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); + const MachineOperand *NarrowDefinition = + Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); + MachineInstr *Truncation = NarrowDefinition == nullptr + ? nullptr + : const_cast(NarrowDefinition->parent()); + if (auto Wide = TruncatedWideSource(Truncation, Def)) { + Plan.WideInputs[Index] = *Wide; + AddArtifact(Def); + AddArtifact(Truncation); + continue; + } + } + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, *Reg) == Binary) { + Plan.WideInputs[Index] = *Reg; + Plan.Constants.push_back(Def); + continue; + } + return std::nullopt; + } + return Plan; +} + +[[nodiscard]] Expected widenConstantToRV64(MachineInstr &Instruction, + MachineFunction &Function) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Immediate = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); + if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow constant legalization has invalid operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto SetType = Function.registerInfo().setType(*Destination, *Type); + if (!SetType) + return std::unexpected(SetType.error()); + auto Extended = extendIntegerImmediate(Immediate->Value, 64, true); + if (!Extended) + return std::unexpected(Extended.error()); + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + return {}; +} + +[[nodiscard]] Expected widenAtomicCompareExchangeSource(MachineInstr &Instruction, + MachineFunction &Function, + MachineIRBuilder &Builder, + Register Source, LLT WideType) { + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo || !SourceInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange source has no scalar type"); + if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) + return Source; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, Source) == &Instruction) { + auto Widened = widenConstantToRV64(*Def, Function); + if (!Widened) + return std::unexpected(Widened.error()); + return Source; + } + return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); +} + +[[nodiscard]] Expected lowerAtomicCompareExchange(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const std::optional OriginalType = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *CompareOperand = Instruction.operandAt(2); + const MachineOperand *NewValueOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); + const Register *NewValue = + NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); + if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || + OriginalType->scalarBitWidth() >= 64) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow compare-exchange has invalid operands"); + const Register OriginalDestination = *Destination; + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideCompare = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); + if (!WideCompare) + return std::unexpected(WideCompare.error()); + auto WideNewValue = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); + if (!WideNewValue) + return std::unexpected(WideNewValue.error()); + + Register WideDestination; + MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); + if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && + operandType(Function, *FoldedExtension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); + const Register *FoldedDestination = + WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf(); + if (FoldedDestination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange extension has no destination"); + WideDestination = *FoldedDestination; + } else { + auto Created = Function.registerInfo().createVirtualRegister(*WideType); + if (!Created) + return std::unexpected(Created.error()); + WideDestination = *Created; + FoldedExtension = nullptr; + } + + if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) + return std::unexpected(Replaced.error()); + + if (FoldedExtension != nullptr) + return eraseInstruction(*FoldedExtension); + + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(OriginalDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(WideDestination); !Added) + return std::unexpected(Added.error()); + auto Built = Truncation->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected +lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *OldValueOperand = Instruction.operandAt(0); + const MachineOperand *SuccessOperand = Instruction.operandAt(1); + const MachineOperand *AddressOperand = Instruction.operandAt(2); + const MachineOperand *CompareOperand = Instruction.operandAt(3); + const MachineOperand *NewValueOperand = Instruction.operandAt(4); + const Register *OldValue = + OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf(); + const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); + const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); + const Register *NewValue = + NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); + const std::optional ValueType = resultType(Function, Instruction); + if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || + NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange-with-success has invalid operands"); + + auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType); + if (!AtomicResult) + return std::unexpected(AtomicResult.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); + if (!Atomic) + return std::unexpected(Atomic.error()); + if (auto Added = Atomic->addDef(*AtomicResult); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*Compare); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*NewValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) + return std::unexpected(Added.error()); + auto BuiltAtomic = Atomic->build(); + if (!BuiltAtomic) + return std::unexpected(BuiltAtomic.error()); + + Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); + if (!CompareResult) + return std::unexpected(CompareResult.error()); + if (auto Added = CompareResult->addDef(*Success); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addUse(*AtomicResult); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addUse(*Compare); !Added) + return std::unexpected(Added.error()); + auto BuiltCompare = CompareResult->build(); + if (!BuiltCompare) + return std::unexpected(BuiltCompare.error()); + + while (!Instruction.memOperands().empty()) { + auto Erased = Instruction.eraseMemOperand(0); + if (!Erased) + return std::unexpected(Erased.error()); + } + return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult); +} + +[[nodiscard]] Expected widenIntegerCompareSource(MachineInstr &Instruction, + MachineFunction &Function, + MachineIRBuilder &Builder, + Register Source, LLT WideType) { + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo || !SourceInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare source has no scalar type"); + const LLT SourceType = *SourceInfo->get().Type; + if (SourceType.scalarBitWidth() == WideType.scalarBitWidth()) + return Source; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + std::optional ExistingExtension; + bool AmbiguousExtension = false; + auto &RegisterInfo = Function.registerInfo(); + for (MachineOperand &Use : RegisterInfo.useOperands(Source)) { + MachineInstr *User = Use.parent(); + if (User == nullptr || User == &Instruction || User->parent() == nullptr || + (User->opcode() != opcode(GenericOpcode::G_ANYEXT) && + User->opcode() != opcode(GenericOpcode::G_SEXT))) { + continue; + } + const MachineOperand *ExtendedOperand = User->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + const std::optional ExtendedType = + ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); + if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || + (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && + User->opcode() != opcode(GenericOpcode::G_SEXT)))) { + continue; + } + if (ExistingExtension) { + AmbiguousExtension = true; + break; + } + ExistingExtension = *Extended; + } + if (ExistingExtension && !AmbiguousExtension) + return *ExistingExtension; + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, Source) == &Instruction) { + auto Widened = widenConstantToRV64(*Def, Function); + if (!Widened) + return std::unexpected(Widened.error()); + return Source; + } + + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && + Def->operandCount() == 2) { + const MachineOperand *WideOperand = Def->operandAt(1); + const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf(); + const std::optional ExistingWideType = + WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); + if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { + auto Result = Function.registerInfo().createVirtualRegister(WideType); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(*WideSource); !Added) + return std::unexpected(Added.error()); + if (auto Added = + Pending->addImmediate(static_cast(SourceType.scalarBitWidth())); + !Added) { + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; + } + } + return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source); +} + +[[nodiscard]] Expected lowerIntegerCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare has invalid register operands"); + const Register OriginalDestination = *Destination; + const Register OriginalLeft = *Left; + const Register OriginalRight = *Right; + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideLeft = + widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + auto WideRight = + widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType); + if (!WideRight) + return std::unexpected(WideRight.error()); + if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced) + return std::unexpected(Replaced.error()); + + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination); + if (!DestinationInfo || !DestinationInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare destination has no scalar type"); + if (DestinationInfo->get().Type->scalarBitWidth() == 64) + return {}; + + MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); + if (FoldedExtension != nullptr && + (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) || + FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || + FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && + operandType(Function, *FoldedExtension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + if (Extended == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare extension has no destination"); + const Register WideDestination = *Extended; + if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + return eraseInstruction(*FoldedExtension); + } + + auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + if (Function.registerInfo().useEmpty(OriginalDestination)) + return {}; + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(OriginalDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(*WideDestination); !Added) + return std::unexpected(Added.error()); + auto Built = Truncation->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected foldNarrowAnyExtBinary(MachineInstr &Extension, + MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); + if (!Plan) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V narrow any-extension does not have a legal binary chain"); + for (MachineInstr *Constant : Plan->Constants) { + auto Widened = widenConstantToRV64(*Constant, Function); + if (!Widened) + return std::unexpected(Widened.error()); + } + for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { + auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + for (MachineInstr *Artifact : Plan->Artifacts) { + if (Artifact->parent() == nullptr) + continue; + auto Removed = Artifact->parent()->removeInstruction(*Artifact); + if (!Removed) + return std::unexpected(Removed.error()); + } + + if (Plan->Generic == GenericOpcode::G_SHL) { + Register Amount = Plan->WideInputs[1]; + if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { + const std::uint64_t Mask = Plan->SourceWidth == 64 + ? std::numeric_limits::max() + : (std::uint64_t{1} << Plan->SourceWidth) - 1U; + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask); + if (!MaskValue) + return std::unexpected(MaskValue.error()); + auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType, + Amount, *MaskValue); + if (!MaskedAmount) + return std::unexpected(MaskedAmount.error()); + Amount = *MaskedAmount; + } + auto SetOpcode = + Extension.setOpcode(Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW) + : opcode(GenericOpcode::G_SHL)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Extension.operandCount() > 1) { + auto Erased = Extension.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + for (Register Input : {Plan->WideInputs[0], Amount}) { + auto Operand = + MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + auto Added = Extension.addOperand(std::move(*Operand)); + if (!Added) + return std::unexpected(Added.error()); + } + MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); + if (BinaryBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow shift is detached during legalization"); + return BinaryBlock->removeInstruction(*Plan->Binary); + } + + if (Plan->SourceWidth == 32 && + (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); + if (!ReplacedSource) + return std::unexpected(ReplacedSource.error()); + auto Width = Extension.addOperand(MachineOperand::immediate(32)); + if (!Width) + return std::unexpected(Width.error()); + return {}; + } + + auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Extension.operandCount() > 1) { + auto Erased = Extension.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + for (Register Input : Plan->WideInputs) { + auto Operand = + MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + auto Added = Extension.addOperand(std::move(*Operand)); + if (!Added) + return std::unexpected(Added.error()); + } + MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); + if (BinaryBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow binary instruction is detached during legalization"); + auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); + if (!Removed) + return std::unexpected(Removed.error()); + (void)Target; + return {}; +} + +[[nodiscard]] Expected lowerConstantToPool(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Immediate = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); + if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant-pool legalization has invalid operands"); + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + if (!DestinationInfo || !DestinationInfo->get().Type || + DestinationInfo->get().Type->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant-pool legalization requires a 64-bit destination"); + const LLT DestinationType = *DestinationInfo->get().Type; + auto Value = GenericConstantData::integer(Immediate->Value); + if (!Value) + return std::unexpected(Value.error()); + auto Align = Alignment::create(8); + if (!Align) + return std::unexpected(Align.error()); + MachineConstantPoolEntryDesc Description; + Description.Value = std::move(*Value); + Description.Size = 8; + Description.Align = *Align; + auto PoolIndex = Function.constantPool().addEntry(std::move(Description)); + if (!PoolIndex) + return std::unexpected(PoolIndex.error()); + + auto PointerType = LLT::makePointer(0, 64); + if (!PointerType) + return std::unexpected(PointerType.error()); + auto Address = Function.registerInfo().createVirtualRegister(*PointerType); + if (!Address) + return std::unexpected(Address.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL)); + if (!AddressInstruction) + return std::unexpected(AddressInstruction.error()); + auto AddressDef = AddressInstruction->addDef(*Address); + if (!AddressDef) + return std::unexpected(AddressDef.error()); + auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex); + if (!AddressPool) + return std::unexpected(AddressPool.error()); + auto BuiltAddress = AddressInstruction->build(); + if (!BuiltAddress) + return std::unexpected(BuiltAddress.error()); + + MachineMemOperandDesc MemoryDescription; + MemoryDescription.Accesses = {MemoryAccessFlag::Load}; + MemoryDescription.Size = *MemorySize::fixed(8); + MemoryDescription.MemoryType = DestinationType; + MemoryDescription.Align = *Align; + MemoryDescription.Base = MemoryBase::constantPoolPseudo(); + auto Memory = Function.createMemOperand(std::move(MemoryDescription)); + if (!Memory) + return std::unexpected(Memory.error()); + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto AddressUse = + MachineOperand::registerOperand(*Address, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!AddressUse) + return std::unexpected(AddressUse.error()); + auto AddedAddress = Instruction.addOperand(std::move(*AddressUse)); + if (!AddedAddress) + return std::unexpected(AddedAddress.error()); + auto AddedMemory = Instruction.addMemOperand(**Memory); + if (!AddedMemory) + return std::unexpected(AddedMemory.error()); + return {}; +} + +[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && + !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; +} + +[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t Bytes) noexcept { + if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return Description.AtomicKind == Operation && Description.AddressSpace == 0 && + Description.Size.isKnown() && !Description.Size.isScalable() && + Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && + Description.SyncScope.value() == 0; +} + +[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t ContainerBytes) noexcept { + if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) + return false; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || + !Description.Size.isKnown() || Description.Size.isScalable() || + Description.SyncScope.value() != 0) + return false; + const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); + return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && + MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; +} + +[[nodiscard]] std::optional +memoryAccessBytes(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return std::nullopt; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (!Description.Size.isKnown() || Description.Size.isScalable()) + return std::nullopt; + return Description.Size.minimumBytes(); +} + +[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) + return false; + if (isRVFloatType(Type)) + return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; + const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; + return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; +} + +[[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf()) + return *Value; + const auto *Integer = Operand.getIf(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an immediate no wider than 64 bits"); + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast(Raw); +} + +[[nodiscard]] Expected replaceRegisterUses(MachineFunction &Function, Register OldReg, + Register NewReg) { + std::vector Uses; + for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) + Uses.push_back(&Operand); + for (MachineOperand *Operand : Uses) { + auto Replaced = Operand->setRegister(NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + return {}; +} + +[[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V legalizer cannot erase a detached instruction"); + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && + !Function.registerInfo().useEmpty(*Reg)) + return false; + } + return true; +} + +void eraseDeadArtifacts(std::span Candidates, + const MachineFunction &Function) { + bool Changed = false; + do { + Changed = false; + for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { + MachineInstr *Instruction = *It; + if (Instruction == nullptr || Instruction->parent() == nullptr || + !allDefsUnused(*Instruction, Function)) + continue; + const Opcode Value = Instruction->opcode(); + if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) && + Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && + Value != opcode(GenericOpcode::G_MERGE_VALUES) && + Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY)) + continue; + if (eraseInstruction(*Instruction)) + Changed = true; + } + } while (Changed); +} + +[[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::uint32_t Width, + std::uint64_t Value) { + auto Type = LLT::makeAnyScalar(Width); + if (!Type) + return std::unexpected(Type.error()); + auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Immediate = IntegerImmediate::fromUnsigned(Width, Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, + Register Source) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Left, + Register Right) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Source) { + auto SetOpcode = Instruction.setOpcode(opcode(Generic)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Def = MachineOperand::registerOperand(Destination, + RegisterOperandFlags{RegisterOperandFlag::Define}); + if (!Def) + return std::unexpected(Def.error()); + if (auto Added = Instruction.addOperand(std::move(*Def)); !Added) + return std::unexpected(Added.error()); + auto Use = + MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Use) + return std::unexpected(Use.error()); + return Instruction.addOperand(std::move(*Use)); +} + +[[nodiscard]] Expected widenNarrowMultiply(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const std::optional Type = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || + Type->scalarBitWidth() >= 64) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow multiply has invalid operands"); + } + + MachineInstr *Extension = singleRegisterUse(Function, *Destination); + if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { + const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); + if (Plan && Plan->Binary == &Instruction) + return foldNarrowAnyExtBinary(*Extension, Function, Target); + } + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + auto WideRight = + buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); + if (!WideRight) + return std::unexpected(WideRight.error()); + auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, + *WideRight); + if (!Product) + return std::unexpected(Product.error()); + return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); +} + +[[nodiscard]] Expected lowerMultiplyLibcall(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + CallingConvention Convention, + const TargetCallLowering &CallLowering) { + const std::optional Type = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || + Type->scalarBitWidth() != 64) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V multiply libcall requires three s64 registers"); + } + auto ABIAlignment = Alignment::create(8); + auto Callee = MachineCallTarget::externalSymbol("__muldi3"); + if (!ABIAlignment || !Callee) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V multiply libcall ABI facts are unavailable"); + const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { + return ABIValue{ + .Value = Reg, + .Type = Type, + .OriginalValue = Number, + .FragmentIndex = 0, + .FragmentCount = 1, + .FragmentOffsetBytes = 0, + .OriginalAlignment = Alignment, + .Extension = ABIExtensionKind::None, + .Flags = {}, + .ByValSizeBytes = 0, + .ByValAlignment = Alignment, + }; + }; + + CallLoweringRequest Request; + Request.Convention = callingConventionID(Convention); + Request.Callee = std::move(*Callee); + Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; + Request.Results = {Value(*Destination, 0)}; + Request.FixedArgumentCount = Request.Arguments.size(); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Lowered = CallLowering.lowerCall(Builder, Request); + if (!Lowered) + return std::unexpected(Lowered.error()); + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Left, Register Right) { + auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Use) + return std::unexpected(Use.error()); + return Instruction.addOperand(std::move(*Use)); +} + +[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, + std::uint32_t ActiveBits, unsigned Depth = 0) { + if (Depth > 8 || !Reg.isVirtual()) + return false; + const auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info || !Info->get().Type) + return false; + const std::uint32_t Width = Info->get().Type->scalarBitWidth(); + if (Width <= ActiveBits) + return true; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); + const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); + if (Def == nullptr) + return false; + if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { + const MachineOperand *BitsOperand = Def->operandAt(2); + auto Bits = BitsOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) + : signedImmediate(*BitsOperand); + return Bits && *Bits >= 0 && static_cast(*Bits) <= ActiveBits; + } + if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return false; + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); + return SourceInfo && SourceInfo->get().Type && + SourceInfo->get().Type->scalarBitWidth() <= ActiveBits; + } + if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); + } + return false; +} + +[[nodiscard]] Expected zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, + MachineFunction &Function, Register Source, + std::vector &DeadCandidates) { + const auto Info = Function.registerInfo().virtualRegisterInfo(Source); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); + const LLT SourceType = *Info->get().Type; + if (SourceType.scalarBitWidth() == 64) + return Source; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *WideOperand = Def->operandAt(1); + const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); + if (Wide != nullptr) { + const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide); + if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 && + isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) { + DeadCandidates.push_back(Def); + return *Wide; + } + } + } + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); +} + +[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { + MachineInstr *Result = nullptr; + auto &RegisterInfo = const_cast(Function.registerInfo()); + for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { + MachineInstr *Instruction = Operand.parent(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + if (Result != nullptr && Result != Instruction) + return nullptr; + Result = Instruction; + } + return Result; +} + +[[nodiscard]] Expected lowerWideMergeValues(MachineInstr &Instruction, + MachineFunction &Function) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const MachineOperand *LowOperand = Instruction.operandAt(1); + const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); + if (Destination == nullptr || Low == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide merge has invalid register operands"); + + std::vector Truncations; + auto &RegisterInfo = Function.registerInfo(); + std::vector Uses; + for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + MachineInstr *Trunc = Use->parent(); + if (Trunc == nullptr || Trunc->parent() == nullptr || + Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge is only representable through low-part truncation"); + const MachineOperand *TruncResultOperand = Trunc->operandAt(0); + const Register *TruncResult = + TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf(); + const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low); + const auto ResultInfo = + TruncResult == nullptr + ? Expected>( + std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) + : RegisterInfo.virtualRegisterInfo(*TruncResult); + if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type || + ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge truncation spans more than its low source"); + if (*ResultInfo->get().Type == *LowInfo->get().Type) { + auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); + if (!Replaced) + return std::unexpected(Replaced.error()); + Truncations.push_back(Trunc); + } else { + auto Replaced = Use->setRegister(*Low); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + for (MachineInstr *Trunc : Truncations) { + auto Erased = eraseInstruction(*Trunc); + if (!Erased) + return std::unexpected(Erased.error()); + } + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const std::optional DestinationType = resultType(Function, Instruction); + if (!DestinationType || DestinationType->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V merge destination has no scalar type"); + if (DestinationType->scalarBitWidth() > 64) + return lowerWideMergeValues(Instruction, Function); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || Instruction.operandCount() < 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V merge has invalid register operands"); + + Register Result = *Destination; + MachineInstr *FoldedExtension = nullptr; + if (DestinationType->scalarBitWidth() < 64) { + MachineInstr *Use = singleUseInstruction(Function, *Destination); + if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && + operandType(Function, *Use, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *ExtendedOperand = Use->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + if (Extended != nullptr) { + Result = *Extended; + FoldedExtension = Use; + } + } + } + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + std::vector DeadCandidates; + std::vector Terms; + Terms.reserve(Instruction.operandCount() - 1U); + for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register"); + auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); + if (!WideSource) + return std::unexpected(WideSource.error()); + if (Index == 1) { + Terms.push_back(*WideSource); + continue; + } + const std::optional SourceType = registerType(Function, *Operand); + if (!SourceType) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); + const std::uint64_t Shift = + static_cast(SourceType->scalarBitWidth()) * (Index - 1U); + auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); + if (!Amount) + return std::unexpected(Amount.error()); + auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, + *WideSource, *Amount); + if (!Shifted) + return std::unexpected(Shifted.error()); + Terms.push_back(*Shifted); + } + + Register Accumulator = Terms.front(); + for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { + auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, + Accumulator, Terms[Index]); + if (!Combined) + return std::unexpected(Combined.error()); + Accumulator = *Combined; + } + if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { + auto Rewritten = + rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back()); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } else { + auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, + Accumulator, Terms.back()); + if (!Combined) + return std::unexpected(Combined.error()); + auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + if (FoldedExtension != nullptr) { + auto Erased = eraseInstruction(*FoldedExtension); + if (!Erased) + return std::unexpected(Erased.error()); + } + eraseDeadArtifacts(DeadCandidates, Function); + return {}; +} + +[[nodiscard]] Expected foldWideExtendedUnmerge(MachineInstr &Instruction, + MachineFunction &Function) { + const std::size_t OutputCount = Instruction.operandCount() - 1U; + const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + const MachineOperand *Definition = + Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); + MachineInstr *Extension = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && + Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && + Extension->opcode() != opcode(GenericOpcode::G_SEXT))) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge requires an extension-defined source"); + const MachineOperand *NarrowOperand = Extension->operandAt(1); + const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); + if (Narrow == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide unmerge extension has no register source"); + const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow); + if (!NarrowInfo || !NarrowInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide unmerge extension source has no type"); + const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth(); + + std::vector DeadUsers; + for (std::size_t Index = 0; Index < OutputCount; ++Index) { + const MachineOperand *OutputOperand = Instruction.operandAt(Index); + const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); + if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) + continue; + const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output); + if (!OutputInfo || !OutputInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type"); + const std::uint64_t Offset = + static_cast(Index) * OutputInfo->get().Type->scalarBitWidth(); + if (Offset != 0) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge cannot materialize a used high wide part"); + if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) { + auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + continue; + } + if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge output unexpectedly narrows its source"); + std::vector Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + MachineInstr *Trunc = Use->parent(); + if (Trunc == nullptr || Trunc->parent() == nullptr || + Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) + return makeUnexpected( + ErrorCode::Unsupported, + "RISC-V wide unmerge result is only representable through truncation"); + const MachineOperand *ResultOperand = Trunc->operandAt(0); + const Register *Result = + ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + const auto ResultInfo = + Result == nullptr + ? Expected>( + std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) + : Function.registerInfo().virtualRegisterInfo(*Result); + if (!ResultInfo || !ResultInfo->get().Type || + ResultInfo->get().Type->scalarBitWidth() > NarrowWidth) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge truncation exceeds its source width"); + if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) { + auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + DeadUsers.push_back(Trunc); + } else { + auto Replaced = Use->setRegister(*Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + for (MachineInstr *User : DeadUsers) { + auto Erased = eraseInstruction(*User); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Erased = eraseInstruction(Instruction); + if (!Erased) + return std::unexpected(Erased.error()); + if (Function.registerInfo().useEmpty(*Source)) + return eraseInstruction(*Extension); + return {}; +} + +[[nodiscard]] Expected lowerUnmergeValues(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + if (Instruction.operandCount() < 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge has invalid register operands"); + const std::size_t OutputCount = Instruction.operandCount() - 1U; + const std::optional OutputType = operandType(Function, Instruction, 0); + const std::optional SourceType = operandType(Function, Instruction, OutputCount); + if (!OutputType || !SourceType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge operands have no scalar types"); + if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) + return foldWideExtendedUnmerge(Instruction, Function); + + const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + std::vector FoldedExtensions; + for (std::size_t Index = 0; Index < OutputCount; ++Index) { + const MachineOperand *OutputOperand = Instruction.operandAt(Index); + const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); + if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) + continue; + Register Extracted = *Source; + const std::uint64_t Offset = static_cast(Index) * OutputType->scalarBitWidth(); + if (Offset != 0) { + auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); + if (!Amount) + return std::unexpected(Amount.error()); + auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, + *Source, *Amount); + if (!Shifted) + return std::unexpected(Shifted.error()); + Extracted = *Shifted; + } + + MachineInstr *Extension = singleUseInstruction(Function, *Output); + const bool FoldZExt = Extension != nullptr && + Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && + operandType(Function, *Extension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false); + if (FoldZExt) { + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + if (Extended == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge extension has no destination"); + const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 + ? std::numeric_limits::max() + : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; + auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); + if (!MaskReg) + return std::unexpected(MaskReg.error()); + auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, + Extracted, *MaskReg); + if (!Masked) + return std::unexpected(Masked.error()); + auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); + if (!Replaced) + return std::unexpected(Replaced.error()); + FoldedExtensions.push_back(Extension); + continue; + } + auto Truncated = + buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted); + if (!Truncated) + return std::unexpected(Truncated.error()); + auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + for (MachineInstr *Extension : FoldedExtensions) { + auto Erased = eraseInstruction(*Extension); + if (!Erased) + return std::unexpected(Erased.error()); + } + return eraseInstruction(Instruction); +} + +struct WideValueParts final { + std::uint32_t BitWidth = 0; + std::vector Parts; + std::vector Artifacts; +}; + +[[nodiscard]] Expected +collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) { + if (Depth > 16) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep"); + const auto Info = Function.registerInfo().virtualRegisterInfo(Value); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); + const LLT Type = *Info->get().Type; + if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide-value decomposition requires an integer scalar"); + if (Type.scalarBitWidth() <= 64) + return WideValueParts{.BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Def == nullptr) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide value has no decomposable definition"); + if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { + WideValueParts Result; + Result.BitWidth = Type.scalarBitWidth(); + for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { + const MachineOperand *Operand = Def->operandAt(Index); + const Register *Part = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional PartType = + Operand == nullptr ? std::nullopt : registerType(Function, *Operand); + if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge must already use 64-bit limbs"); + Result.Parts.push_back(*Part); + } + Result.Artifacts.push_back(Def); + return Result; + } + if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide truncation has no source register"); + auto Result = collectWideValueParts(*Source, Function, Depth + 1U); + if (!Result) + return std::unexpected(Result.error()); + Result->BitWidth = Type.scalarBitWidth(); + const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U; + if (Result->Parts.size() < RequiredParts) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide truncation source has too few limbs"); + Result->Parts.resize(RequiredParts); + Result->Artifacts.push_back(Def); + return Result; + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide value is not defined by merge/trunc artifacts"); +} + +struct WideResultPlan final { + std::vector Outputs; + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = nullptr; +}; + +[[nodiscard]] Expected wideResultPlan(Register Result, MachineFunction &Function, + std::size_t RequiredParts) { + MachineInstr *Use = singleUseInstruction(Function, Result); + WideResultPlan Plan; + if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || + Use->opcode() == opcode(GenericOpcode::G_SEXT) || + Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { + Plan.Extension = Use; + const MachineOperand *ExtendedOperand = Use->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); + Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); + } + if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide result must feed an unmerge artifact"); + Plan.Unmerge = Use; + const std::size_t OutputCount = Use->operandCount() - 1U; + if (OutputCount < RequiredParts) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide result unmerge has too few outputs"); + Plan.Outputs.reserve(RequiredParts); + for (std::size_t Index = 0; Index < RequiredParts; ++Index) { + const MachineOperand *Operand = Use->operandAt(Index); + const Register *Output = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Type = + Operand == nullptr ? std::nullopt : registerType(Function, *Operand); + if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide result must unmerge to 64-bit limbs"); + Plan.Outputs.push_back(*Output); + } + return Plan; +} + +[[nodiscard]] Expected buildIntegerCompareBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + IntegerPredicate Predicate, + Register Left, Register Right) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + LLT Type, Register Condition, Register TrueValue, + Register FalseValue) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Condition); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(TrueValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(FalseValue); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +struct TwoLimbShiftResult final { + Register Low; + Register High; +}; + +[[nodiscard]] Expected +emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, + Register High, Register Amount, Register AmountExcess, Register AmountLack, + Register Zero, Register IsShort, Register IsZero) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); + if (!LowShort) + return std::unexpected(LowShort.error()); + auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); + if (!CrossLow) + return std::unexpected(CrossLow.error()); + auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto ShortHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort); + if (!ShortHigh) + return std::unexpected(ShortHigh.error()); + auto LongHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); + if (!LongHigh) + return std::unexpected(LongHigh.error()); + auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); + if (!ResultLow) + return std::unexpected(ResultLow.error()); + auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); + if (!ShiftedHigh) + return std::unexpected(ShiftedHigh.error()); + auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); + if (!ResultHigh) + return std::unexpected(ResultHigh.error()); + return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; +} + +[[nodiscard]] Expected emitTwoLimbLeftShift(MachineIRBuilder &Builder, + MachineInstr &Anchor, Register Low, + Register High, Register Amount) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); + if (!Boundary) + return std::unexpected(Boundary.error()); + auto AmountExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary); + if (!AmountExcess) + return std::unexpected(AmountExcess.error()); + auto AmountLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount); + if (!AmountLack) + return std::unexpected(AmountLack.error()); + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *Boundary); + if (!IsShort) + return std::unexpected(IsShort.error()); + auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack, + *Zero, *IsShort, *IsZero); +} + +[[nodiscard]] Expected +emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, + Register High, Register Amount, Register Boundary, + Register Zero) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto AmountExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary); + if (!AmountExcess) + return std::unexpected(AmountExcess.error()); + auto AmountLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount); + if (!AmountLack) + return std::unexpected(AmountLack.error()); + auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, Boundary); + if (!IsShort) + return std::unexpected(IsShort.error()); + auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); + if (!LowShort) + return std::unexpected(LowShort.error()); + auto HighShort = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto ShortValue = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort); + if (!ShortValue) + return std::unexpected(ShortValue.error()); + auto LongValue = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess); + if (!LongValue) + return std::unexpected(LongValue.error()); + auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue); + if (!Shifted) + return std::unexpected(Shifted.error()); + return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); +} + +[[nodiscard]] Expected> +emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, + std::span Parts, Register Amount) { + if (Parts.size() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V three-limb shift requires exactly three limbs"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + + auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); + if (!OuterBoundary) + return std::unexpected(OuterBoundary.error()); + auto OuterExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary); + if (!OuterExcess) + return std::unexpected(OuterExcess.error()); + auto OuterLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount); + if (!OuterLack) + return std::unexpected(OuterLack.error()); + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *OuterBoundary); + if (!OuterIsShort) + return std::unexpected(OuterIsShort.error()); + auto OuterIsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!OuterIsZero) + return std::unexpected(OuterIsZero.error()); + + auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); + if (!InnerBoundary) + return std::unexpected(InnerBoundary.error()); + auto InnerExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary); + if (!InnerExcess) + return std::unexpected(InnerExcess.error()); + auto InnerLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount); + if (!InnerLack) + return std::unexpected(InnerLack.error()); + auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *InnerBoundary); + if (!InnerIsShort) + return std::unexpected(InnerIsShort.error()); + auto InnerIsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!InnerIsZero) + return std::unexpected(InnerIsZero.error()); + auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, + *InnerLack, *Zero, *InnerIsShort, *InnerIsZero); + if (!Lower) + return std::unexpected(Lower.error()); + + auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1], + *OuterLack, *InnerBoundary, *Zero); + if (!Cross) + return std::unexpected(Cross.error()); + auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *InnerBoundary); + if (!HighIsShort) + return std::unexpected(HighIsShort.error()); + auto HighShort = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero); + if (!BoundedHigh) + return std::unexpected(BoundedHigh.error()); + auto ShortHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh); + if (!ShortHigh) + return std::unexpected(ShortHigh.error()); + + auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + *OuterExcess, *InnerBoundary); + if (!LongIsShort) + return std::unexpected(LongIsShort.error()); + auto LongShift = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess); + if (!LongShift) + return std::unexpected(LongShift.error()); + auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero); + if (!LongHigh) + return std::unexpected(LongHigh.error()); + + auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero); + if (!ResultLow) + return std::unexpected(ResultLow.error()); + auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero); + if (!ResultMiddle) + return std::unexpected(ResultMiddle.error()); + auto ShiftedHigh = + buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh); + if (!ShiftedHigh) + return std::unexpected(ShiftedHigh.error()); + auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh); + if (!ResultHigh) + return std::unexpected(ResultHigh.error()); + return std::vector{*ResultLow, *ResultMiddle, *ResultHigh}; +} + +[[nodiscard]] Expected lowerWideShiftLeft(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ValueOperand = Instruction.operandAt(1); + const MachineOperand *AmountOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf(); + if (Destination == nullptr || Value == nullptr || Amount == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide shift has invalid register operands"); + auto ValueParts = collectWideValueParts(*Value, Function); + if (!ValueParts) + return std::unexpected(ValueParts.error()); + auto AmountParts = collectWideValueParts(*Amount, Function); + if (!AmountParts) + return std::unexpected(AmountParts.error()); + if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide shift has an invalid limb decomposition"); + auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size()); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + + MachineIRBuilder Builder(Function, Target); + Expected> Results = + ValueParts->Parts.size() == 2 + ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1], + AmountParts->Parts[0]) + .transform([](TwoLimbShiftResult Result) { + return std::vector{Result.Low, Result.High}; + }) + : ValueParts->Parts.size() == 3 + ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0]) + : Expected>(std::unexpected( + Error(ErrorCode::Unsupported, + "RISC-V wide shift currently requires two or three 64-bit limbs"))); + if (!Results) + return std::unexpected(Results.error()); + for (std::size_t Index = 0; Index < Results->size(); ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { + const MachineOperand *Definition = + Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); + if (Definition != nullptr && Definition->parent() != nullptr) + AmountParts->Artifacts.push_back(const_cast(Definition->parent())); + } + ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(), + AmountParts->Artifacts.end()); + eraseDeadArtifacts(ValueParts->Artifacts, Function); + return {}; +} + +[[nodiscard]] Expected> emitWideAdd(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::span Left, + std::span Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide add requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector Results; + Results.reserve(Left.size()); + auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); + if (!Low) + return std::unexpected(Low.error()); + auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low, + Right[0]); + if (!Carry) + return std::unexpected(Carry.error()); + auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); + if (!LowCopy) + return std::unexpected(LowCopy.error()); + Results.push_back(*LowCopy); + for (std::size_t Index = 1; Index < Left.size(); ++Index) { + auto Base = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); + if (!Base) + return std::unexpected(Base.error()); + std::optional BaseCarry; + if (Index + 1U < Left.size()) { + auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + *Base, Left[Index]); + if (!Compared) + return std::unexpected(Compared.error()); + BaseCarry = *Compared; + } + auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); + if (!Sum) + return std::unexpected(Sum.error()); + if (Index + 1U < Left.size()) { + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto CarryFromIncoming = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); + if (!CarryFromIncoming) + return std::unexpected(CarryFromIncoming.error()); + auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry, + *CarryFromIncoming); + if (!NextCarry) + return std::unexpected(NextCarry.error()); + Carry = *NextCarry; + } + auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); + if (!Copy) + return std::unexpected(Copy.error()); + Results.push_back(*Copy); + } + return Results; +} + +[[nodiscard]] Expected> emitWideSub(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::span Left, + std::span Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide sub requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector Results; + Results.reserve(Left.size()); + auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); + if (!Low) + return std::unexpected(Low.error()); + Results.push_back(*Low); + auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Left[0], Right[0]); + if (!Borrow) + return std::unexpected(Borrow.error()); + for (std::size_t Index = 1; Index < Left.size(); ++Index) { + auto Base = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); + if (!Base) + return std::unexpected(Base.error()); + std::optional BaseBorrow; + if (Index + 1U < Left.size()) { + auto Compared = buildIntegerCompareBefore( + Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]); + if (!Compared) + return std::unexpected(Compared.error()); + BaseBorrow = *Compared; + } + auto Difference = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); + if (!Difference) + return std::unexpected(Difference.error()); + Results.push_back(*Difference); + if (Index + 1U < Left.size()) { + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto BorrowFromIncoming = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); + if (!BorrowFromIncoming) + return std::unexpected(BorrowFromIncoming.error()); + auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow, + *BorrowFromIncoming); + if (!NextBorrow) + return std::unexpected(NextBorrow.error()); + Borrow = *NextBorrow; + } + } + return Results; +} + +[[nodiscard]] Expected> +emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, + std::span Left, std::span Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide bitwise operation requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector Results; + Results.reserve(Left.size()); + for (std::size_t Index = 0; Index < Left.size(); ++Index) { + auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); + if (!Part) + return std::unexpected(Part.error()); + Results.push_back(*Part); + } + return Results; +} + +[[nodiscard]] Expected lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + GenericOpcode Generic) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide binary instruction has invalid operands"); + auto LeftParts = collectWideValueParts(*Left, Function); + if (!LeftParts) + return std::unexpected(LeftParts.error()); + auto RightParts = collectWideValueParts(*Right, Function); + if (!RightParts) + return std::unexpected(RightParts.error()); + if (LeftParts->Parts.size() != RightParts->Parts.size()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide binary operands have different limb counts"); + auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + + MachineIRBuilder Builder(Function, Target); + Expected> Results = + Generic == GenericOpcode::G_ADD + ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) + : Generic == GenericOpcode::G_SUB + ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) + : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); + if (!Results) + return std::unexpected(Results.error()); + for (std::size_t Index = 0; Index < Results->size(); ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), + RightParts->Artifacts.end()); + eraseDeadArtifacts(LeftParts->Artifacts, Function); + return {}; +} + +[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2) + return false; + const MachineOperand *ConditionOperand = Instruction.operandAt(0); + const Register *Condition = + ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); + if (Condition == nullptr || !Condition->isVirtual() || + singleRegisterUse(Function, *Condition) != &Instruction) + return false; + const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition); + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition); + const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent(); + if (!ConditionInfo || !ConditionInfo->get().Type || + ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr || + Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) + return false; + return operandType(Function, *Truncation, 1) + .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) + .value_or(false); +} + +[[nodiscard]] Expected lowerNarrowBranchCondition(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + if (!canLowerNarrowBranchCondition(Instruction, Function)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V branch condition is not a narrow truncation"); + const MachineOperand *ConditionOperand = Instruction.operandAt(0); + const Register Condition = *ConditionOperand->getIf(); + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); + MachineInstr *Truncation = const_cast(Definition->parent()); + const Register Source = *Truncation->operandAt(1)->getIf(); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + MachineIRBuilder Builder(Function, Target); + auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); + if (!One) + return std::unexpected(One.error()); + auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One); + if (!Masked) + return std::unexpected(Masked.error()); + auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); + if (!Replaced) + return std::unexpected(Replaced.error()); + if (Function.registerInfo().useEmpty(Condition)) + return eraseInstruction(*Truncation); + return {}; +} + +struct NarrowPhiIncoming final { + Register Value; + MachineBasicBlock *Block = nullptr; + MachineInstr *ReusableTruncation = nullptr; +}; + +[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { + MachineInstr *Instruction = Block.firstInstruction(); + while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY)) + Instruction = Instruction->next(); + return Instruction; +} + +[[nodiscard]] Expected +buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, + MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, + LLT Type, Register Source) { + auto Result = Function.registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = + Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const std::optional OriginalType = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || + Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); + auto LegalType = LLT::makeAnyScalar(32); + if (!LegalType) + return std::unexpected(LegalType.error()); + + std::vector Incoming; + Incoming.reserve((Instruction.operandCount() - 1U) / 2U); + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const MachineOperand *ValueOperand = Instruction.operandAt(Index); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const auto *Block = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); + if (Value == nullptr || Block == nullptr || Block->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI has an invalid incoming edge"); + MachineInstr *Reusable = nullptr; + const MachineOperand *Definition = + Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && + Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && + operandType(Function, *Def, 1) + .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) + .value_or(false)) { + Reusable = Def; + } + Incoming.push_back( + NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable}); + } + for (NarrowPhiIncoming &Entry : Incoming) { + if (Entry.ReusableTruncation == nullptr) + continue; + const std::size_t Count = static_cast(std::ranges::count( + Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation)); + if (Count != 1) + Entry.ReusableTruncation = nullptr; + } + for (NarrowPhiIncoming &Entry : Incoming) { + if (Entry.ReusableTruncation == nullptr) + continue; + MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); + if (OriginalBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI input truncation is detached"); + auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); + if (!Removed) + return std::unexpected(Removed.error()); + } + + std::vector> Anchors; + for (const NarrowPhiIncoming &Entry : Incoming) { + if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) == + Anchors.end()) { + Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block)); + } + } + std::vector LegalIncoming; + LegalIncoming.reserve(Incoming.size()); + for (NarrowPhiIncoming &Entry : Incoming) { + const auto AnchorPosition = + std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first); + MachineInstr *Anchor = AnchorPosition->second; + if (Entry.ReusableTruncation != nullptr) { + auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); + if (!SetType) + return std::unexpected(SetType.error()); + auto Inserted = + Anchor == nullptr + ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) + : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation); + if (!Inserted) + return std::unexpected(Inserted.error()); + LegalIncoming.push_back(Entry.Value); + continue; + } + const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value); + if (!ValueInfo || !ValueInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI input has no scalar type"); + const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth(); + const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT + : Width > 32 ? GenericOpcode::G_TRUNC + : GenericOpcode::COPY; + auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion, + *LegalType, Entry.Value); + if (!Converted) + return std::unexpected(Converted.error()); + LegalIncoming.push_back(*Converted); + } + + auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); + if (!LegalResult) + return std::unexpected(LegalResult.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*LegalResult); !Added) + return std::unexpected(Added.error()); + for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { + if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); + if (!Replaced) + return std::unexpected(Replaced.error()); + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || Instruction.operandCount() < 5 || + (Instruction.operandCount() % 2U) == 0) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands"); + + std::vector IncomingParts; + std::vector IncomingBlocks; + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const MachineOperand *ValueOperand = Instruction.operandAt(Index); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const auto *Block = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); + if (Value == nullptr || Block == nullptr || Block->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide PHI has an invalid incoming edge"); + auto Parts = collectWideValueParts(*Value, Function); + if (!Parts) + return std::unexpected(Parts.error()); + if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide PHI incoming values have different limb counts"); + IncomingParts.push_back(std::move(*Parts)); + IncomingBlocks.push_back(Block->Block); + } + const std::size_t PartCount = IncomingParts.front().Parts.size(); + auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + MachineIRBuilder Builder(Function, Target); + std::vector Results; + Results.reserve(PartCount); + for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { + auto Result = Function.registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) { + if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + Results.push_back(*Result); + } + for (std::size_t Index = 0; Index < PartCount; ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + std::vector Artifacts; + for (WideValueParts &Parts : IncomingParts) + Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); + eraseDeadArtifacts(Artifacts, Function); + return {}; +} + +[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, + const MachineFunction &Function) { + const MachineOperand *LowOperand = Instruction.operandAt(1); + const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); + if (Low == nullptr) + return false; + const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low); + if (!LowInfo || !LowInfo->get().Type) + return false; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr) + return false; + auto &RegisterInfo = const_cast(Function.registerInfo()); + for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { + MachineInstr *Trunc = Use.parent(); + if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || + Trunc->operandAt(1) != &Use) + return false; + const std::optional Result = resultType(Function, *Trunc); + if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) + return false; + } + return true; +} + +struct RV64ScalarLayout final { + std::uint64_t StoreSize = 0; + std::uint64_t AllocationSize = 0; + std::uint64_t ABIAlignment = 0; +}; + +[[nodiscard]] std::optional rv64ScalarLayout(LLT Type) noexcept { + if (Type.isPointer()) { + return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 + ? std::optional{{8, 8, 8}} + : std::nullopt; + } + if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) { + return std::nullopt; + } + const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; + const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes); + return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min(AllocationBytes, 8)}; +} + +[[nodiscard]] Expected +createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, + std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { + auto Size = MemorySize::fixed(SizeBytes); + auto Align = Alignment::create(AlignmentBytes); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid"); + MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.MemoryType = Type; + Description.Align = *Align; + return Function.createMemOperand(std::move(Description)); +} + +struct VAArgPlan final { + Register Destination; + Register ListPointer; + LLT DestinationType; + LLT PointerType; + std::uint64_t Alignment = 0; + RV64ScalarLayout ValueLayout; +}; + +[[nodiscard]] std::optional vaargPlan(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || + !Instruction.memOperands().empty()) { + return std::nullopt; + } + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ListOperand = Instruction.operandAt(1); + const MachineOperand *AlignmentOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf(); + const std::optional DestinationType = resultType(Function, Instruction); + const std::optional PointerType = operandType(Function, Instruction, 1); + const auto AlignmentValue = AlignmentOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing alignment"))) + : signedImmediate(*AlignmentOperand); + const std::optional ValueLayout = + DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt; + if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || + !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || + !DestinationType || !PointerType || !PointerType->isPointer() || + PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue || + *AlignmentValue <= 0 || !std::has_single_bit(static_cast(*AlignmentValue)) || + !ValueLayout) { + return std::nullopt; + } + + return VAArgPlan{*Destination, + *ListPointer, + *DestinationType, + *PointerType, + static_cast(*AlignmentValue), + *ValueLayout}; +} + +[[nodiscard]] Expected lowerVAArg(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = vaargPlan(Instruction, Function); + if (!Plan) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V va_arg legalization requires a scalar result, p0 list, and " + "power-of-two alignment"); + } + + auto ListLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8); + if (!ListLoadMemory) + return std::unexpected(ListLoadMemory.error()); + auto ListStoreMemory = + createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8); + if (!ListStoreMemory) + return std::unexpected(ListStoreMemory.error()); + + auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType); + if (!CurrentList) + return std::unexpected(CurrentList.error()); + MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ListLoad) + return std::unexpected(ListLoad.error()); + if (auto Added = ListLoad->addDef(*CurrentList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListLoad->build(); !Built) + return std::unexpected(Built.error()); + + Register ValueAddress = *CurrentList; + if (Plan->Alignment > 1) { + auto AlignmentOffset = + buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U); + if (!AlignmentOffset) + return std::unexpected(AlignmentOffset.error()); + auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, + Plan->PointerType, ValueAddress, *AlignmentOffset); + if (!BiasedAddress) + return std::unexpected(BiasedAddress.error()); + auto AlignmentMask = + buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1})); + if (!AlignmentMask) + return std::unexpected(AlignmentMask.error()); + auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK, + Plan->PointerType, *BiasedAddress, *AlignmentMask); + if (!AlignedAddress) + return std::unexpected(AlignedAddress.error()); + ValueAddress = *AlignedAddress; + } + + auto Increment = + buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize); + if (!Increment) + return std::unexpected(Increment.error()); + auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, + Plan->PointerType, ValueAddress, *Increment); + if (!NextList) + return std::unexpected(NextList.error()); + auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!ListStore) + return std::unexpected(ListStore.error()); + if (auto Added = ListStore->addUse(*NextList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addUse(Plan->ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListStore->build(); !Built) + return std::unexpected(Built.error()); + + if (!Function.registerInfo().useEmpty(Plan->Destination)) { + auto ValueLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType, + Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment); + if (!ValueLoadMemory) + return std::unexpected(ValueLoadMemory.error()); + auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ValueLoad) + return std::unexpected(ValueLoad.error()); + if (auto Added = ValueLoad->addDef(Plan->Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addUse(ValueAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ValueLoad->build(); !Built) + return std::unexpected(Built.error()); + } + return eraseInstruction(Instruction); +} + +struct VACopyPlan final { + Register Destination; + Register Source; + LLT PointerType; +}; + +[[nodiscard]] std::optional vacopyPlan(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || + Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { + return std::nullopt; + } + const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); + const MachineOperand *DestinationOperand = Instruction.operandAt(1); + const MachineOperand *SourceOperand = Instruction.operandAt(2); + const auto *Intrinsic = + IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + const std::optional DestinationType = operandType(Function, Instruction, 1); + const std::optional SourceType = operandType(Function, Instruction, 2); + if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || + Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || + !DestinationType || !SourceType || *DestinationType != *SourceType || + !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || + DestinationType->scalarBitWidth() != 64) { + return std::nullopt; + } + return VACopyPlan{*Destination, *Source, *DestinationType}; +} + +[[nodiscard]] Expected lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = vacopyPlan(Instruction, Function); + if (!Plan) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V va_copy legalization requires two p0 operands"); + } + auto Size = MemorySize::fixed(8); + auto Align = Alignment::create(8); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); + + MachineMemOperandDesc LoadDescription; + LoadDescription.Accesses = {MemoryAccessFlag::Load}; + LoadDescription.Size = *Size; + LoadDescription.MemoryType = Plan->PointerType; + LoadDescription.Align = *Align; + auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); + if (!LoadMemory) + return std::unexpected(LoadMemory.error()); + + MachineMemOperandDesc StoreDescription; + StoreDescription.Accesses = {MemoryAccessFlag::Store}; + StoreDescription.Size = *Size; + StoreDescription.MemoryType = Plan->PointerType; + StoreDescription.Align = *Align; + auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); + if (!StoreMemory) + return std::unexpected(StoreMemory.error()); + + auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); + if (!Value) + return std::unexpected(Value.error()); + MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(Plan->Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(**LoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + + auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(Plan->Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(**StoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + return eraseInstruction(Instruction); +} + +} // namespace + +Expected RISCVLegalizerInfo::classify(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &) const { + if (!Subtarget.features().has(RISCVExtension::I)) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V legalizer requires the base I extension"); + } + if (!isGenericOpcode(Instruction.opcode())) + return LegalizationAction::Legal; + + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + switch (Generic) { + case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: + return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + case GenericOpcode::G_VAARG: + return vaargPlan(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + case GenericOpcode::G_FRAME_INDEX: + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_BR: + return LegalizationAction::Legal; + case GenericOpcode::G_BRCOND: { + const std::optional Condition = operandType(Function, Instruction, 0); + if (!Condition || !isScalarGPRType(*Condition)) + return LegalizationAction::Unsupported; + return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Legal; + } + case GenericOpcode::G_PHI: { + if (!Type) + return LegalizationAction::Unsupported; + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const std::optional Incoming = operandType(Function, Instruction, Index); + if (!Incoming || *Incoming != *Type) + return LegalizationAction::Unsupported; + } + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) + return LegalizationAction::Custom; + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) + return LegalizationAction::Custom; + if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) + return LegalizationAction::Unsupported; + return LegalizationAction::Legal; + } + case GenericOpcode::G_IMPLICIT_DEF: + return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_STORE: + if (Instruction.operandCount() == 0) + return LegalizationAction::Unsupported; + if (const std::optional ValueType = registerType(Function, *Instruction.operandAt(0)); + ValueType) { + const std::optional AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; + return isSupportedIntegerType(*ValueType) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && + isSupportedNonAtomicMemoryAccess(Instruction) && + memoryMatchesType(Instruction, *ValueType)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && + *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && + (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && *Bytes == 4 && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTRUNCSTORE: { + const std::optional ValueType = operandType(Function, Instruction, 0); + const std::optional AddressType = operandType(Function, Instruction, 1); + const auto Bytes = memoryAccessBytes(Instruction); + return ValueType && AddressType && Bytes && ValueType->isFloat() && + ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && + AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && + *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_LOAD: + if (Type) { + const std::optional AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; + return isSupportedIntegerType(*Type) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && + isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMA: { + const std::optional First = operandType(Function, Instruction, 1); + const std::optional Second = operandType(Function, Instruction, 2); + const std::optional Third = operandType(Function, Instruction, 3); + return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && + *Second == *Type && *Third == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FSQRT: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && *Source == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCOPYSIGN: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; + return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && + Floating->Bits.semantics() == *Semantics + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && + Instruction.operandCount() == 3 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRTOINT: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && + Source->isPointer() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_INTTOPTR: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && + Source->isInteger() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BITCAST: { + const std::optional Source = operandType(Function, Instruction, 1); + const bool IntegerFloatPair = + Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && + ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && + (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); + return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: { + const std::optional Source = operandType(Function, Instruction, 1); + if (!Type || !Source) + return LegalizationAction::Unsupported; + if (!isScalarIntegerExtension(*Type, *Source)) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + MachineInstr *OnlyUse = + Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); + return (Type->isInteger() || Type->isAnyScalar()) && + (Source->isInteger() || Source->isAnyScalar()) && + Type->scalarBitWidth() > 64 && + Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && + OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *SourceRegister = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + const MachineOperand *SourceDefinition = + SourceRegister == nullptr || !SourceRegister->isVirtual() + ? nullptr + : Function.registerInfo().getUniqueVRegDef(*SourceRegister); + const MachineInstr *SourceDef = + SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); + if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) + return LegalizationAction::Custom; + if (Generic == GenericOpcode::G_ANYEXT && + narrowAnyExtBinaryPlan(Instruction, Function).has_value()) + return LegalizationAction::Custom; + return LegalizationAction::Legal; + } + case GenericOpcode::G_TRUNC: { + const std::optional Source = operandType(Function, Instruction, 1); + if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && + (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && + Source->scalarBitWidth() > Type->scalarBitWidth()) + return LegalizationAction::Legal; + return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && + Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_UNMERGE_VALUES: { + if (Instruction.operandCount() < 3) + return LegalizationAction::Unsupported; + const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; + const std::size_t BigTypeIndex = + Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; + const std::optional SmallType = operandType(Function, Instruction, SmallTypeIndex); + const std::optional BigType = operandType(Function, Instruction, BigTypeIndex); + if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && + BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) + return LegalizationAction::Legal; + return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) && + (BigType->isInteger() || BigType->isAnyScalar()) && + SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SEXT_INREG: { + const std::optional Source = operandType(Function, Instruction, 1); + const MachineOperand *WidthOperand = Instruction.operandAt(2); + const auto Width = + WidthOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) + : signedImmediate(*WidthOperand); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && + static_cast(*Width) <= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ICMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *IntegerPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf(); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || + IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || + !isSupportedIntegerType(*Left)) { + return LegalizationAction::Unsupported; + } + return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Custom; + } + case GenericOpcode::G_FCMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *FloatingPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf(); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && isI1Type(*Type) && FloatingPredicate != nullptr && + FloatingPredicate->floatingPredicate() != nullptr && Left && Right && + *Left == *Right && isRVFloatType(*Left) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXT: + case GenericOpcode::G_FPTRUNC: { + const std::optional Source = operandType(Function, Instruction, 1); + const bool IsExtension = Generic == GenericOpcode::G_FPEXT; + return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && + ((IsExtension && Source->scalarBitWidth() == 32 && + Type->scalarBitWidth() == 64) || + (!IsExtension && Source->scalarBitWidth() == 64 && + Type->scalarBitWidth() == 32)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SELECT: { + const std::optional Condition = operandType(Function, Instruction, 1); + const std::optional TrueValue = operandType(Function, Instruction, 2); + const std::optional FalseValue = operandType(Function, Instruction, 3); + return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && + *TrueValue == *Type && *FalseValue == *Type && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRMASK: { + const std::optional Pointer = operandType(Function, Instruction, 1); + const std::optional Mask = operandType(Function, Instruction, 2); + return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && + isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ABS: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: { + const std::optional Remainder = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && + *Remainder == *Type && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: { + const std::optional Overflow = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && + isI1Type(*Overflow) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: { + const std::optional CarryOut = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + const std::optional CarryIn = operandType(Function, Instruction, 4); + return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && + isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: { + const std::optional Address = operandType(Function, Instruction, 1); + const std::optional Value = operandType(Function, Instruction, 2); + const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; + return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && + Address->isPointer() && Address->addressSpace() == 0 && + Address->scalarBitWidth() == 64 && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, + Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ATOMIC_CMPXCHG: { + const std::optional Address = operandType(Function, Instruction, 1); + const std::optional Compare = operandType(Function, Instruction, 2); + const std::optional NewValue = operandType(Function, Instruction, 3); + if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || + (!Type->isInteger() && !Type->isAnyScalar()) || + (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && + Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || + !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || + !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, + (Type->scalarBitWidth() + 7U) / 8U)) { + return LegalizationAction::Unsupported; + } + return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; + } + case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { + const std::optional Success = operandType(Function, Instruction, 1); + const std::optional Address = operandType(Function, Instruction, 2); + const std::optional Compare = operandType(Function, Instruction, 3); + const std::optional NewValue = operandType(Function, Instruction, 4); + return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && + *Compare == *Type && *NewValue == *Type && + (Type->isInteger() || Type->isAnyScalar()) && + (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || + Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && + Address->isPointer() && Address->addressSpace() == 0 && + Address->scalarBitWidth() == 64 && + isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, + (Type->scalarBitWidth() + 7U) / 8U) + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FENCE: { + const MachineOperand *OrderingOperand = Instruction.operandAt(0); + const MachineOperand *ScopeOperand = Instruction.operandAt(1); + const auto Ordering = OrderingOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence ordering"))) + : signedImmediate(*OrderingOperand); + const auto Scope = ScopeOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence scope"))) + : signedImmediate(*ScopeOperand); + return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_CONSTANT_POOL: { + const MachineOperand *PoolOperand = Instruction.operandAt(1); + const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf(); + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && + Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_GLOBAL_VALUE: { + const MachineOperand *GlobalOperand = Instruction.operandAt(1); + const auto *Global = + GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf(); + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && + Global != nullptr && !Global->Name.empty() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BRINDIRECT: + return operandType(Function, Instruction, 0) + .transform([](LLT Value) { + return Value.isPointer() && Value.addressSpace() == 0 && + Value.scalarBitWidth() == 64; + }) + .value_or(false) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + return Instruction.operandCount() == 0 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_UBSANTRAP: + return Instruction.operandCount() == 1 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_CONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + if (!Type || Immediate == nullptr || + (Immediate->kind() != MachineOperand::Kind::Immediate && + Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) + return LegalizationAction::Unsupported; + if (Type->scalarBitWidth() > 64) + return wideConstantUnmergePlan(Instruction, Function).has_value() + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { + auto Value = signedImmediate(*Immediate); + if (Value && shouldUseRV64IntegerConstantPool(*Value)) + return LegalizationAction::Custom; + } + return isSupportedIntegerType(*Type) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_UREM: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || + (!Type->isInteger() && !Type->isAnyScalar())) + return LegalizationAction::Unsupported; + if (Type->scalarBitWidth() > 64) { + return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || + Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || + Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) { + if (Type->scalarBitWidth() == 64) + return LegalizationAction::Libcall; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + MachineInstr *Extension = + Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); + const std::optional ExtensionType = + Extension == nullptr ? std::nullopt : resultType(Function, *Extension); + if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && + ExtensionType && ExtensionType->scalarBitWidth() == 64) { + return LegalizationAction::Legal; + } + return LegalizationAction::Custom; + } + return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTR_ADD: { + const std::optional Pointer = operandType(Function, Instruction, 1); + const std::optional Offset = operandType(Function, Instruction, 2); + return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && + isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + default: + break; + } + + // Keep the unsupported policy mechanically synchronized with the complete + // target-independent opcode inventory. Supported cases return above. + switch (Generic) { +#define RCC_CORE_OPCODE(Name) +#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: +#include "rcc/CodeGen/GenericOpcode.def" +#undef RCC_GENERIC_OPCODE +#undef RCC_CORE_OPCODE + return LegalizationAction::Unsupported; + case GenericOpcode::Invalid: + case GenericOpcode::CORE_PSEUDO_END: + case GenericOpcode::G_GENERIC_END: + default: + break; + } + return makeUnexpected(ErrorCode::InternalError, + "RISC-V legalizer received an invalid generic opcode value"); +} + +Expected RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + LegalizationAction Action) const { + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { + return lowerVACopy(Instruction, Function, Target); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) { + return lowerVAArg(Instruction, Function, Target); + } + if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && + !Subtarget.features().has(RISCVExtension::M)) { + if (Action == LegalizationAction::Custom) + return widenNarrowMultiply(Instruction, Function, Target); + if (Action == LegalizationAction::Libcall) + return lowerMultiplyLibcall(Instruction, Function, Target, + Subtarget.features().has(RISCVExtension::D) + ? CallingConvention::LP64D + : CallingConvention::LP64, + *Subtarget.getCallLowering()); + } + if (Action == LegalizationAction::Custom && + resultType(Function, Instruction) + .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) + .value_or(false)) { + const Opcode Value = Instruction.opcode(); + if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || + Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || + Value == opcode(GenericOpcode::G_XOR)) { + return lowerWideBinary(Instruction, Function, Target, + static_cast(Value.value())); + } + if (Value == opcode(GenericOpcode::G_SHL)) + return lowerWideShiftLeft(Instruction, Function, Target); + if (Value == opcode(GenericOpcode::G_PHI)) + return lowerWidePhi(Instruction, Function, Target); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) + return lowerMergeValues(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) + return lowerUnmergeValues(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) + return lowerNarrowBranchCondition(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) + return lowerNarrowPhi(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) + return lowerIntegerCompare(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) + return lowerAtomicCompareExchange(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) + return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { + if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && + narrowAnyExtBinaryPlan(Instruction, Function).has_value()) + return foldNarrowAnyExtBinary(Instruction, Function, Target); + return foldConstantExtension(Instruction, Function, + static_cast(Instruction.opcode().value())); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { + const MachineOperand *Immediate = Instruction.operandAt(1); + const std::optional Type = resultType(Function, Instruction); + if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && + Type->scalarBitWidth() == 64) { + auto Value = signedImmediate(*Immediate); + if (Value && shouldUseRV64IntegerConstantPool(*Value)) + return lowerConstantToPool(Instruction, Function, Target); + } + return foldWideConstant(Instruction, Function, Target); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V legalizer has no lowering rule for this instruction"); +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h new file mode 100644 index 0000000..f6ba000 --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H +#define RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H + +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" + +namespace rcc::riscv { + +class RISCVSubtarget; + +class RISCVLegalizerInfo final : public LegalizerInfo { +public: + explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {} + + [[nodiscard]] Expected + classify(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) const override; + [[nodiscard]] Expected lower(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + LegalizationAction Action) const override; + +private: + const RISCVSubtarget &Subtarget; +}; + +} // namespace rcc::riscv + +#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp new file mode 100644 index 0000000..6e38d52 --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RISCVRegisterBankInfo.h" + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" + +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { + return Type.isFloat() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { + return (Type.isPointer() && Type.addressSpace() == 0 && Type.scalarBitWidth() == 64) || + ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() >= 1 && + Type.scalarBitWidth() <= 64); +} + +[[nodiscard]] Expected requiredRegisterClass(const MachineFunction &Function, + Register Reg, + const TargetDescription &Target) { + if (!Reg.isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V register-class selection requires a virtual register"); + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V selection requires a type for every virtual register"); + } + const VirtualRegisterInfo &RegisterInfo = Info->get(); + const LLT Type = RegisterInfo.Type.value_or(LLT{}); + RegisterClassID RequiredClass; + if (isScalarGPRType(Type)) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::GPR)); + } else if (Type.isFloat() && Type.scalarBitWidth() == 32) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR32)); + } else if (Type.isFloat() && Type.scalarBitWidth() == 64) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR64)); + } else { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V RV64GC has no register class for type '" + Type.toString() + "'"); + } + + if (Target.registers().registerClass(RequiredClass) == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V target description is missing a required register class"); + } + if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V virtual register bank cannot cover its required class"); + } + if (RegisterInfo.RegClass && + !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V virtual register has a conflicting register class"); + } + return RequiredClass; +} + +} // namespace + +Expected RISCVRegisterBankInfo::selectBank(const MachineFunction &, Register, + const VirtualRegisterInfo &Info, + const MachineInstr *, + const TargetDescription &) const { + if (!Info.Type) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V register-bank selection requires a virtual-register type"); + } + if (isScalarGPRType(*Info.Type)) + return RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); + if (isRVFloatType(*Info.Type)) + return RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V RV64GC has no register bank for virtual register type '" + + Info.Type->toString() + "'"); +} + +Expected RISCVRegisterBankInfo::constrainInstruction(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + std::vector> PlannedClasses; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Required = requiredRegisterClass(Function, *Reg, Target); + if (!Required) + return std::unexpected(Required.error()); + if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) { + return Entry.first == *Reg; + }) == PlannedClasses.end()) + PlannedClasses.emplace_back(*Reg, *Required); + } + for (const auto &[Reg, Required] : PlannedClasses) { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().RegClass) + continue; + auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + return {}; +} + +Expected RISCVRegisterBankInfo::constrainRegister(MachineFunction &Function, Register Reg, + const TargetDescription &Target) const { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + auto Bank = selectBank(Function, Reg, Info->get(), nullptr, Target); + if (!Bank) + return std::unexpected(Bank.error()); + auto RequiredClass = requiredRegisterClass(Function, Reg, Target); + if (!RequiredClass) + return std::unexpected(RequiredClass.error()); + if (!Info->get().Bank) { + auto SetBank = Function.registerInfo().setRegisterBank(Reg, *Bank); + if (!SetBank) + return std::unexpected(SetBank.error()); + } + if (!Info->get().RegClass) { + auto SetClass = Function.registerInfo().setRegisterClass(Reg, *RequiredClass); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + return {}; +} + +Expected RISCVRegisterBankInfo::validateInstruction(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Required = requiredRegisterClass(Function, *Reg, Target); + if (!Required) + return std::unexpected(Required.error()); + } + return {}; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h new file mode 100644 index 0000000..543bc4d --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H +#define RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H + +#include "rcc/CodeGen/RegisterBankInfo.h" + +namespace rcc::riscv { + +class RISCVRegisterBankInfo final : public RegisterBankInfo { +public: + [[nodiscard]] Expected selectBank(const MachineFunction &Function, Register Reg, + const VirtualRegisterInfo &Info, + const MachineInstr *Instruction, + const TargetDescription &Target) const override; + [[nodiscard]] Expected constrainInstruction(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const override; + [[nodiscard]] Expected constrainRegister(MachineFunction &Function, Register Reg, + const TargetDescription &Target) const; + [[nodiscard]] Expected validateInstruction(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const override; +}; + +} // namespace rcc::riscv + +#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H diff --git a/backend/lib/Target/RISCV/RISCV.h b/backend/lib/Target/RISCV/RISCV.h new file mode 100644 index 0000000..1a81f59 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCV.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_TARGET_RISCV_RISCV_H +#define RCC_LIB_TARGET_RISCV_RISCV_H + +#include "rcc/CodeGen/MachineCombinerPattern.h" +#include "rcc/CodeGen/MachineOptimizationPipeline.h" +#include "rcc/CodeGen/TargetFrameLowering.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" +#include "rcc/CodeGen/TargetPseudoExpansion.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" + +namespace rcc { + +class InstructionSelector; + +namespace riscv { + +class RISCVRegisterBankInfo; +class RISCVSubtarget; + +[[nodiscard]] InstructionSelector * +createRISCVInstructionSelector(const RISCVSubtarget &Subtarget, + const RISCVRegisterBankInfo &RegisterBanks); +[[nodiscard]] const TargetRegisterAllocationInfo ®isterAllocationInfo() noexcept; +[[nodiscard]] const TargetFrameLowering &frameLowering() noexcept; +[[nodiscard]] const TargetSchedulingModel &schedulingModel() noexcept; +[[nodiscard]] const TargetMachineCombinerInfo &machineCombinerInfo() noexcept; +[[nodiscard]] const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept; +[[nodiscard]] const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept; +[[nodiscard]] MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept; + +} // namespace riscv +} // namespace rcc + +#endif // RCC_LIB_TARGET_RISCV_RISCV_H diff --git a/backend/lib/Target/RISCV/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/RISCVCallLowering.cpp deleted file mode 100644 index 5da3fc8..0000000 --- a/backend/lib/Target/RISCV/RISCVCallLowering.cpp +++ /dev/null @@ -1,1512 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "rcc/Target/RISCV/RISCVCallLowering.h" - -#include "rcc/CodeGen/GenericOpcodes.h" -#include "rcc/CodeGen/MachineBasicBlock.h" -#include "rcc/CodeGen/MachineFrameInfo.h" -#include "rcc/CodeGen/MachineFunction.h" -#include "rcc/CodeGen/MachineIRBuilder.h" -#include "rcc/CodeGen/MachineInstr.h" -#include "rcc/CodeGen/MachineMemOperand.h" -#include "rcc/CodeGen/MachineRegisterInfo.h" -#include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVTarget.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rcc::riscv { -namespace { - -constexpr std::string_view PlanTarget = "riscv64-unknown-linux-gnu"; -constexpr std::uint64_t XLenBytes = 8; -constexpr std::uint64_t ABIStackAlignmentBytes = 16; -constexpr std::uint32_t AllowedDirectCallFlags = - static_cast(CallTargetFlag::DirectCall); - -[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { - return Opcode::fromValue(static_cast(Value)); -} - -[[nodiscard]] std::uint64_t stackObjectFlag(CallStackObjectFlag Flag) noexcept { - return static_cast(Flag); -} - -[[nodiscard]] Expected alignment(std::uint64_t Bytes) { - return Alignment::create(Bytes); -} - -[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t AlignmentBytes) { - const std::uint64_t Remainder = Value % AlignmentBytes; - if (Remainder == 0) - return Value; - const std::uint64_t Increment = AlignmentBytes - Remainder; - if (Value > std::numeric_limits::max() - Increment) { - return makeUnexpected(ErrorCode::Unsupported, "RISC-V ABI stack-area size overflows uint64"); - } - return Value + Increment; -} - -[[nodiscard]] std::uint64_t valueSizeBytes(LLT Type) noexcept { - return (static_cast(Type.scalarBitWidth()) + 7U) / 8U; -} - -[[nodiscard]] bool isSupportedFloat(LLT Type) noexcept { - if (!Type.isFloat()) - return false; - const auto Semantics = Type.floatSemantics(); - return Semantics && - (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); -} - -[[nodiscard]] Expected floatingRegisterView(PhysicalRegister DoubleView, - LLT Type) { - if (!isSupportedFloat(Type)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V floating-register view requires f32 or f64"); - } - if (Type.scalarBitWidth() == 64) - return DoubleView; - - constexpr std::uint32_t FirstDouble = static_cast(PhysicalRegisterID::F0_D); - constexpr std::uint32_t LastDouble = static_cast(PhysicalRegisterID::F31_D); - constexpr std::uint32_t FirstSingle = static_cast(PhysicalRegisterID::F0_F); - const std::uint32_t Number = DoubleView.number(); - if (Number < FirstDouble || Number > LastDouble) { - return makeUnexpected(ErrorCode::InternalError, - "RISC-V ABI floating register is not an FPR64 view"); - } - return PhysicalRegister::create(FirstSingle + Number - FirstDouble); -} - -[[nodiscard]] bool requiresIntegerTransport(const ABIValue &Value) noexcept { - return !isSupportedFloat(Value.Type) || Value.Flags.has(ABIValueFlag::ByVal) || - Value.Flags.has(ABIValueFlag::StructReturn) || Value.Flags.has(ABIValueFlag::Indirect); -} - -[[nodiscard]] Expected verifyABIValue(const ABIValue &Value, bool RequireRegister) { - if (RequireRegister && Value.Value.isNoRegister()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V ABI value requires a source or destination register"); - } - if (!Value.Type.isValid() || !Value.Type.isScalarValue() || Value.Type.scalarBitWidth() > 64) { - return makeUnexpected( - ErrorCode::Unsupported, - "RISC-V ABI accepts explicitly classified integer, pointer, f32, and f64 scalar fragments"); - } - if (Value.Type.isFloat() && !isSupportedFloat(Value.Type)) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V LP64 ABIs only support f32 and f64 floating fragments"); - } - if (Value.Type.isPointer() && Value.Type.scalarBitWidth() != 64) { - return makeUnexpected(ErrorCode::Unsupported, "RISC-V LP64 ABIs require 64-bit pointers"); - } - if ((Value.Type.isInteger() || Value.Type.isAnyScalar()) && Value.Type.scalarBitWidth() < 64) { - if (Value.Extension == ABIExtensionKind::None) { - return makeUnexpected(ErrorCode::InvalidArgument, - "narrow RISC-V ABI integers require an explicit extension kind"); - } - } else if (Value.Extension != ABIExtensionKind::None) { - return makeUnexpected(ErrorCode::InvalidArgument, - "only narrow integer ABI fragments may carry an extension kind"); - } - if (Value.Extension >= ABIExtensionKind::Count || Value.FragmentCount == 0 || - Value.FragmentIndex >= Value.FragmentCount || Value.FragmentCount > 2) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V ABI fragment coordinates or extension are invalid"); - } - - const bool IsByVal = Value.Flags.has(ABIValueFlag::ByVal); - const bool IsSRet = Value.Flags.has(ABIValueFlag::StructReturn); - const bool IsIndirect = Value.Flags.has(ABIValueFlag::Indirect); - const unsigned SpecialFlagCount = static_cast(IsByVal) + static_cast(IsSRet) + - static_cast(IsIndirect); - if (SpecialFlagCount > 1) { - return makeUnexpected(ErrorCode::InvalidArgument, - "byval, sret, and indirect ABI flags are mutually exclusive"); - } - if ((IsByVal || IsSRet || IsIndirect) && - (!Value.Type.isPointer() || Value.FragmentCount != 1 || Value.FragmentIndex != 0 || - Value.FragmentOffsetBytes != 0)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "byval, sret, and indirect values require one explicit pointer fragment"); - } - if (IsByVal) { - if (Value.ByValSizeBytes == 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "byval arguments require a nonzero copy size"); - } - } else if (Value.ByValSizeBytes != 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "non-byval values cannot carry a byval copy size"); - } - return {}; -} - -[[nodiscard]] Expected verifyValueSequence(std::span Values, - bool RequireRegisters) { - std::size_t ExpectedOriginal = 0; - std::size_t Index = 0; - while (Index < Values.size()) { - const ABIValue &First = Values[Index]; - auto ValueVerification = verifyABIValue(First, RequireRegisters); - if (!ValueVerification) - return std::unexpected(ValueVerification.error()); - if (First.OriginalValue != ExpectedOriginal || First.FragmentIndex != 0 || - Index + First.FragmentCount > Values.size()) { - return makeUnexpected( - ErrorCode::InvalidArgument, - "RISC-V ABI values must form contiguous, zero-based original-value groups"); - } - - std::uint64_t PreviousEnd = 0; - for (std::size_t Fragment = 0; Fragment < First.FragmentCount; ++Fragment) { - const ABIValue &Value = Values[Index + Fragment]; - auto FragmentVerification = verifyABIValue(Value, RequireRegisters); - if (!FragmentVerification) - return std::unexpected(FragmentVerification.error()); - if (Value.OriginalValue != First.OriginalValue || Value.FragmentIndex != Fragment || - Value.FragmentCount != First.FragmentCount || - Value.OriginalAlignment != First.OriginalAlignment || Value.Flags != First.Flags || - Value.ByValSizeBytes != First.ByValSizeBytes || - Value.ByValAlignment != First.ByValAlignment || Value.FragmentOffsetBytes < PreviousEnd) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V ABI fragments of one value have inconsistent metadata"); - } - PreviousEnd = - static_cast(Value.FragmentOffsetBytes) + valueSizeBytes(Value.Type); - } - Index += First.FragmentCount; - ++ExpectedOriginal; - } - return ExpectedOriginal; -} - -[[nodiscard]] CCAssignmentFlags assignmentFlags(const ABIValue &Value, bool IsVariadic, - bool IsBitcast) { - CCAssignmentFlags Result; - if (IsVariadic) - Result.set(CCAssignmentFlag::Variadic); - if (Value.Flags.has(ABIValueFlag::ByVal)) - Result.set(CCAssignmentFlag::ByVal); - if (Value.Flags.has(ABIValueFlag::StructReturn)) - Result.set(CCAssignmentFlag::StructReturn); - if (Value.Flags.has(ABIValueFlag::Indirect)) - Result.set(CCAssignmentFlag::Indirect); - if (Value.Flags.has(ABIValueFlag::InRegister)) - Result.set(CCAssignmentFlag::InRegister); - if (IsBitcast) - Result.set(CCAssignmentFlag::Bitcast); - return Result; -} - -[[nodiscard]] CCLocationKind locationKind(const ABIValue &Value, bool IsRegister) noexcept { - if (Value.Flags.has(ABIValueFlag::Indirect)) - return CCLocationKind::Indirect; - return IsRegister ? CCLocationKind::Register : CCLocationKind::Stack; -} - -struct ArgumentAssignmentResult final { - std::vector Assignments; - StackArgumentArea StackArea; - std::size_t FirstUnallocatedGPR = 0; -}; - -struct AssignmentState final { - std::size_t GPR = 0; - std::size_t FPR = 0; - std::uint64_t StackOffset = 0; - bool VariadicStackOnly = false; -}; - -[[nodiscard]] Expected registerAssignment(const ABIValue &Value, - std::size_t ValueIndex, - PhysicalRegister Location, LLT LocationType, - bool IsVariadic, bool IsBitcast) { - auto Align = - alignment(std::min(Value.OriginalAlignment.bytes(), ABIStackAlignmentBytes)); - if (!Align) - return std::unexpected(Align.error()); - return CCAssignment{ - .ValueIndex = ValueIndex, - .OriginalValue = Value.Value, - .OriginalValueIndex = Value.OriginalValue, - .FragmentIndex = Value.FragmentIndex, - .FragmentCount = Value.FragmentCount, - .ValueType = Value.Type, - .LocationType = LocationType, - .LocationKind = locationKind(Value, true), - .LocationRegister = Location, - .StackOffsetBytes = std::nullopt, - .LocationSizeBytes = valueSizeBytes(LocationType), - .StackSlotSizeBytes = XLenBytes, - .LocationAlignment = *Align, - .Extension = Value.Extension, - .CustomFlags = assignmentFlags(Value, IsVariadic, IsBitcast), - }; -} - -[[nodiscard]] Expected stackAssignment(const ABIValue &Value, std::size_t ValueIndex, - std::uint64_t StackOffset, LLT LocationType, - bool IsVariadic, - std::uint64_t StackAlignment) { - auto Align = alignment(StackAlignment); - if (!Align) - return std::unexpected(Align.error()); - return CCAssignment{ - .ValueIndex = ValueIndex, - .OriginalValue = Value.Value, - .OriginalValueIndex = Value.OriginalValue, - .FragmentIndex = Value.FragmentIndex, - .FragmentCount = Value.FragmentCount, - .ValueType = Value.Type, - .LocationType = LocationType, - .LocationKind = locationKind(Value, false), - .LocationRegister = std::nullopt, - .StackOffsetBytes = StackOffset, - .LocationSizeBytes = valueSizeBytes(LocationType), - .StackSlotSizeBytes = XLenBytes, - .LocationAlignment = *Align, - .Extension = Value.Extension, - .CustomFlags = assignmentFlags(Value, IsVariadic, false), - }; -} - -[[nodiscard]] Expected -assignArguments(std::span Values, std::size_t FixedArgumentCount, bool IsVariadic, - const CallingConventionInfo &Convention) { - auto I64 = LLT::makeInteger(64); - auto StackAlign = alignment(Convention.StackArgumentAlignmentBytes); - if (!I64 || !StackAlign) - return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid"); - - ArgumentAssignmentResult Result; - AssignmentState State; - State.StackOffset = Convention.ShadowSpaceBytes; - Result.Assignments.reserve(Values.size()); - - std::size_t Index = 0; - while (Index < Values.size()) { - const ABIValue &First = Values[Index]; - const std::size_t GroupSize = First.FragmentCount; - const bool VariadicArgument = IsVariadic && First.OriginalValue >= FixedArgumentCount; - bool HasPreferredFloat = false; - std::size_t RequiredGPRs = 0; - std::size_t RequiredFPRs = 0; - for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { - const ABIValue &Value = Values[Index + Fragment]; - if (!VariadicArgument && !requiresIntegerTransport(Value)) { - HasPreferredFloat = true; - ++RequiredFPRs; - } else { - ++RequiredGPRs; - } - } - - const bool CanUsePreferredAggregate = - HasPreferredFloat && - State.GPR + RequiredGPRs <= Convention.IntegerArgumentRegisters.size() && - State.FPR + RequiredFPRs <= Convention.FloatingArgumentRegisters.size(); - if (CanUsePreferredAggregate) { - for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { - const ABIValue &Value = Values[Index + Fragment]; - Expected Assignment = makeUnexpected( - ErrorCode::InternalError, "RISC-V ABI argument assignment was not initialized"); - if (requiresIntegerTransport(Value)) { - Assignment = registerAssignment(Value, Index + Fragment, - Convention.IntegerArgumentRegisters[State.GPR++], *I64, - false, false); - } else { - auto View = - floatingRegisterView(Convention.FloatingArgumentRegisters[State.FPR++], Value.Type); - if (!View) - return std::unexpected(View.error()); - Assignment = registerAssignment(Value, Index + Fragment, *View, Value.Type, false, false); - } - if (!Assignment) - return std::unexpected(Assignment.error()); - Result.Assignments.push_back(*Assignment); - } - Index += GroupSize; - continue; - } - - const bool RequiresAlignedPair = - VariadicArgument && GroupSize == 2 && First.OriginalAlignment.bytes() >= 16; - if (RequiresAlignedPair && State.GPR % 2 != 0 && - State.GPR < Convention.IntegerArgumentRegisters.size()) { - ++State.GPR; - } - bool ForceEntireGroupToStack = State.VariadicStackOnly; - if (RequiresAlignedPair && State.GPR + GroupSize > Convention.IntegerArgumentRegisters.size()) { - ForceEntireGroupToStack = true; - State.GPR = Convention.IntegerArgumentRegisters.size(); - } - - bool FirstStackFragment = true; - for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) { - const ABIValue &Value = Values[Index + Fragment]; - const bool HasGPR = - !ForceEntireGroupToStack && State.GPR < Convention.IntegerArgumentRegisters.size(); - if (HasGPR) { - const bool Bitcast = isSupportedFloat(Value.Type); - auto Assignment = registerAssignment(Value, Index + Fragment, - Convention.IntegerArgumentRegisters[State.GPR++], *I64, - VariadicArgument, Bitcast); - if (!Assignment) - return std::unexpected(Assignment.error()); - Result.Assignments.push_back(*Assignment); - continue; - } - - if (Value.Flags.has(ABIValueFlag::InRegister)) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V inreg argument cannot be assigned to the stack"); - } - std::uint64_t RequiredAlignment = XLenBytes; - if (FirstStackFragment && ForceEntireGroupToStack) { - RequiredAlignment = std::min( - std::max(XLenBytes, First.OriginalAlignment.bytes()), - ABIStackAlignmentBytes); - } - auto AlignedOffset = alignTo(State.StackOffset, RequiredAlignment); - if (!AlignedOffset) - return std::unexpected(AlignedOffset.error()); - State.StackOffset = *AlignedOffset; - const LLT LocationType = isSupportedFloat(Value.Type) ? Value.Type : *I64; - auto Assignment = stackAssignment(Value, Index + Fragment, State.StackOffset, LocationType, - VariadicArgument, RequiredAlignment); - if (!Assignment) - return std::unexpected(Assignment.error()); - Result.Assignments.push_back(*Assignment); - if (State.StackOffset > std::numeric_limits::max() - XLenBytes) { - return makeUnexpected(ErrorCode::Unsupported, "RISC-V argument area overflows uint64"); - } - State.StackOffset += XLenBytes; - FirstStackFragment = false; - if (VariadicArgument) - State.VariadicStackOnly = true; - } - Index += GroupSize; - } - - auto FinalSize = alignTo(State.StackOffset, Convention.StackArgumentAlignmentBytes); - if (!FinalSize) - return std::unexpected(FinalSize.error()); - Result.StackArea = StackArgumentArea{*FinalSize, Convention.ShadowSpaceBytes, *StackAlign}; - Result.FirstUnallocatedGPR = State.GPR; - return Result; -} - -[[nodiscard]] Expected> -assignReturns(std::span Values, bool RequireRegisters, - const CallingConventionInfo &Convention) { - auto Sequence = verifyValueSequence(Values, RequireRegisters); - if (!Sequence) - return std::unexpected(Sequence.error()); - if (Values.size() > 2) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V direct returns are limited to two explicit fragments"); - } - auto I64 = LLT::makeInteger(64); - if (!I64) - return std::unexpected(I64.error()); - - std::vector Result; - Result.reserve(Values.size()); - std::size_t GPR = 0; - std::size_t FPR = 0; - for (std::size_t Index = 0; Index < Values.size(); ++Index) { - const ABIValue &Value = Values[Index]; - if (Value.Flags.rawValue() != 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "return fragments cannot carry argument-only ABI flags"); - } - const bool UseFloatingRegister = - isSupportedFloat(Value.Type) && FPR < Convention.FloatingReturnRegisters.size(); - Expected Assignment = makeUnexpected( - ErrorCode::Unsupported, "RISC-V calling convention has no return register available"); - if (UseFloatingRegister) { - auto View = floatingRegisterView(Convention.FloatingReturnRegisters[FPR++], Value.Type); - if (!View) - return std::unexpected(View.error()); - Assignment = registerAssignment(Value, Index, *View, Value.Type, false, false); - } else if (GPR < Convention.IntegerReturnRegisters.size()) { - Assignment = registerAssignment(Value, Index, Convention.IntegerReturnRegisters[GPR++], *I64, - false, isSupportedFloat(Value.Type)); - } - if (!Assignment) - return std::unexpected(Assignment.error()); - Result.push_back(*Assignment); - } - return Result; -} - -[[nodiscard]] bool hasStructReturn(std::span Arguments) noexcept { - return std::ranges::any_of( - Arguments, [](const ABIValue &Value) { return Value.Flags.has(ABIValueFlag::StructReturn); }); -} - -[[nodiscard]] bool equivalentReturnLocations(std::span Left, - std::span Right) noexcept { - if (Left.size() != Right.size()) - return false; - for (std::size_t Index = 0; Index < Left.size(); ++Index) { - if (Left[Index].ValueType != Right[Index].ValueType || - Left[Index].LocationType != Right[Index].LocationType || - Left[Index].LocationKind != Right[Index].LocationKind || - Left[Index].LocationRegister != Right[Index].LocationRegister || - Left[Index].Extension != Right[Index].Extension || - Left[Index].CustomFlags != Right[Index].CustomFlags) { - return false; - } - } - return true; -} - -[[nodiscard]] Expected -tailCallRejection(const CallLoweringRequest &Request, const ArgumentAssignmentResult &Arguments, - std::span Results, - std::span ByValCopies, - const CallingConventionInfo &Convention) { - if (!Convention.SupportsTailCalls) - return std::string("the calling convention does not support tail calls"); - if (!Request.TailContext) - return std::string("tail-call context is missing"); - const TailCallContext &Context = *Request.TailContext; - if (Context.CallerConvention != Request.Convention) - return std::string("caller and callee calling conventions differ"); - if (Context.CallerIsVariadic != Request.IsVariadic || - (Request.IsVariadic && Context.CallerFixedArgumentCount != Request.FixedArgumentCount)) { - return std::string("caller and callee variadic signatures differ"); - } - if (Context.CallerHasDynamicStackAllocation) - return std::string("the caller has a dynamic stack allocation"); - if (!ByValCopies.empty()) - return std::string("tail calls cannot create byval copies"); - if (Context.CallerUsesStructReturn != hasStructReturn(Request.Arguments)) - return std::string("caller and callee sret conventions differ"); - if (Arguments.StackArea.SizeBytes > Context.ReusableIncomingArgumentBytes) - return std::string("outgoing stack arguments exceed the reusable incoming area"); - if (std::ranges::any_of(Request.Results, - [](const ABIValue &Value) { return !Value.Value.isNoRegister(); })) { - return std::string("tail-call result signatures must not request local result copies"); - } - auto CallerReturns = assignReturns(Context.CallerReturns, false, Convention); - if (!CallerReturns) - return std::unexpected(CallerReturns.error()); - if (!equivalentReturnLocations(Results, *CallerReturns)) - return std::string("caller and callee return locations differ"); - return std::string{}; -} - -[[nodiscard]] Expected -riscvConvention(CallingConventionID Convention) { - CallingConvention TargetConvention = CallingConvention::None; - if (Convention == callingConventionID(CallingConvention::LP64D)) - TargetConvention = CallingConvention::LP64D; - else if (Convention == callingConventionID(CallingConvention::LP64)) - TargetConvention = CallingConvention::LP64; - else - return makeUnexpected(ErrorCode::Unsupported, "RISC-V backend only supports LP64 and LP64D"); - const CallingConventionInfo *Info = callingConvention(TargetConvention); - if (Info == nullptr) - return makeUnexpected(ErrorCode::InternalError, - "RISC-V calling-convention target facts are unavailable"); - return Info; -} - -class RISCVCallLowering final : public TargetCallLowering { -public: - [[nodiscard]] Expected - planFormalArguments(const FormalArgumentRequest &Request) const override; - [[nodiscard]] Expected - planCall(const CallLoweringRequest &Request) const override; - [[nodiscard]] Expected - planReturn(const ReturnLoweringRequest &Request) const override; - [[nodiscard]] Expected emitPlan(MachineIRBuilder &Builder, - const CallLoweringPlan &Plan) const override; -}; - -Expected -RISCVCallLowering::planFormalArguments(const FormalArgumentRequest &Request) const { - auto Convention = riscvConvention(Request.Convention); - if (!Convention) - return std::unexpected(Convention.error()); - auto OriginalCount = verifyValueSequence(Request.Arguments, true); - if (!OriginalCount) - return std::unexpected(OriginalCount.error()); - if (Request.FixedArgumentCount != *OriginalCount) { - return makeUnexpected(ErrorCode::InvalidArgument, - "formal-argument request must identify every declared argument as fixed"); - } - if (!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) { - return makeUnexpected(ErrorCode::InvalidArgument, - "non-variadic formal request has an invalid fixed-argument boundary"); - } - if (hasStructReturn(Request.Arguments) && - (Request.Arguments.empty() || Request.Arguments.front().OriginalValue != 0 || - !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V sret must be the first formal argument"); - } - - auto Assignments = assignArguments(Request.Arguments, Request.FixedArgumentCount, - Request.IsVariadic, **Convention); - if (!Assignments) - return std::unexpected(Assignments.error()); - - VarArgLoweringState VarArgs; - if (Request.IsVariadic) { - VarArgs.IsVariadic = true; - VarArgs.FixedArgumentCount = Request.FixedArgumentCount; - VarArgs.FirstUnallocatedIntegerRegister = Assignments->FirstUnallocatedGPR; - VarArgs.OverflowAreaOffsetBytes = Assignments->StackArea.SizeBytes; - const std::size_t First = Assignments->FirstUnallocatedGPR; - const std::size_t RegisterCount = (*Convention)->IntegerArgumentRegisters.size(); - VarArgs.RegisterSaveAreaSizeBytes = (RegisterCount - First) * XLenBytes; - VarArgs.RegisterSaveAreaOffsetBytes = - VarArgs.RegisterSaveAreaSizeBytes == 0 - ? static_cast(VarArgs.OverflowAreaOffsetBytes) - : -static_cast(VarArgs.RegisterSaveAreaSizeBytes); - if (VarArgs.RegisterSaveAreaSizeBytes != 0 && First % 2 != 0) - VarArgs.RegisterSavePaddingBytes = XLenBytes; - for (std::size_t Index = First; Index < RegisterCount; ++Index) { - VarArgs.RegistersToSave.push_back( - {(*Convention)->IntegerArgumentRegisters[Index], (Index - First) * XLenBytes}); - } - } - - CallLoweringPlanDesc Plan; - Plan.Kind = CallLoweringPlanKind::FormalArguments; - Plan.Target = PlanTarget; - Plan.Convention = Request.Convention; - Plan.ArgumentAssignments = std::move(Assignments->Assignments); - Plan.StackArguments = Assignments->StackArea; - Plan.VarArgs = std::move(VarArgs); - return CallLoweringPlan::create(std::move(Plan)); -} - -Expected RISCVCallLowering::planCall(const CallLoweringRequest &Request) const { - auto Convention = riscvConvention(Request.Convention); - if (!Convention) - return std::unexpected(Convention.error()); - auto OriginalCount = verifyValueSequence(Request.Arguments, true); - if (!OriginalCount) - return std::unexpected(OriginalCount.error()); - if ((!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) || - (Request.IsVariadic && Request.FixedArgumentCount > *OriginalCount)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "call request has an invalid fixed/variadic argument boundary"); - } - if (!Request.Callee.isValid()) - return makeUnexpected(ErrorCode::InvalidArgument, "call request requires a valid callee"); - if (hasStructReturn(Request.Arguments) && - (Request.Arguments.empty() || - !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V sret must be the first call argument"); - } - if (hasStructReturn(Request.Arguments) && !Request.Results.empty()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V sret calls cannot also request direct return values"); - } - - MachineCallTarget EffectiveCallee = Request.Callee; - switch (Request.Callee.kind()) { - case MachineCallTargetKind::GlobalSymbol: { - if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "direct RISC-V call uses unknown target flags"); - } - auto Callee = MachineCallTarget::globalSymbol( - std::string(Request.Callee.symbolName()), - Request.Callee.targetFlags() | static_cast(CallTargetFlag::DirectCall)); - if (!Callee) - return std::unexpected(Callee.error()); - EffectiveCallee = std::move(*Callee); - break; - } - case MachineCallTargetKind::ExternalSymbol: { - if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "direct RISC-V call uses unknown target flags"); - } - auto Callee = MachineCallTarget::externalSymbol( - std::string(Request.Callee.symbolName()), - Request.Callee.targetFlags() | static_cast(CallTargetFlag::DirectCall)); - if (!Callee) - return std::unexpected(Callee.error()); - EffectiveCallee = std::move(*Callee); - break; - } - case MachineCallTargetKind::IndirectRegister: - if (Request.Callee.targetFlags() != 0) { - return makeUnexpected(ErrorCode::InvalidArgument, - "indirect RISC-V calls cannot carry direct-call target flags"); - } - break; - case MachineCallTargetKind::Invalid: - return makeUnexpected(ErrorCode::InvalidArgument, "call target is invalid"); - } - - const bool RequireResultRegisters = Request.TailKind == TailCallKind::NotTail; - auto ResultSequence = verifyValueSequence(Request.Results, RequireResultRegisters); - if (!ResultSequence) - return std::unexpected(ResultSequence.error()); - auto Arguments = assignArguments(Request.Arguments, Request.FixedArgumentCount, - Request.IsVariadic, **Convention); - if (!Arguments) - return std::unexpected(Arguments.error()); - auto Results = assignReturns(Request.Results, RequireResultRegisters, **Convention); - if (!Results) - return std::unexpected(Results.error()); - - auto Align16 = alignment(ABIStackAlignmentBytes); - if (!Align16) - return std::unexpected(Align16.error()); - std::uint64_t CallFrameCursor = Arguments->StackArea.SizeBytes; - std::vector ByValCopies; - for (std::size_t Index = 0; Index < Arguments->Assignments.size(); ++Index) { - const CCAssignment &Assignment = Arguments->Assignments[Index]; - if (!Assignment.CustomFlags.has(CCAssignmentFlag::ByVal)) - continue; - const ABIValue &Value = Request.Arguments[Assignment.ValueIndex]; - const std::uint64_t CopyAlignment = - std::min(Value.ByValAlignment.bytes(), ABIStackAlignmentBytes); - auto CopyOffset = alignTo(CallFrameCursor, CopyAlignment); - if (!CopyOffset) - return std::unexpected(CopyOffset.error()); - if (*CopyOffset > std::numeric_limits::max() - Value.ByValSizeBytes) { - return makeUnexpected(ErrorCode::Unsupported, "RISC-V byval area overflows uint64"); - } - auto CopyAlign = alignment(CopyAlignment); - if (!CopyAlign) - return std::unexpected(CopyAlign.error()); - ByValCopies.push_back( - {Index, Value.Value, Value.Type, *CopyOffset, Value.ByValSizeBytes, *CopyAlign}); - CallFrameCursor = *CopyOffset + Value.ByValSizeBytes; - } - auto CallFrameSize = alignTo(CallFrameCursor, ABIStackAlignmentBytes); - if (!CallFrameSize) - return std::unexpected(CallFrameSize.error()); - - TailCallKind Decision = TailCallKind::NotTail; - std::string RejectionReason; - if (Request.TailKind != TailCallKind::NotTail) { - auto Rejection = tailCallRejection(Request, *Arguments, *Results, ByValCopies, **Convention); - if (!Rejection) - return std::unexpected(Rejection.error()); - RejectionReason = std::move(*Rejection); - if (RejectionReason.empty()) { - Decision = Request.TailKind; - } else if (Request.TailKind == TailCallKind::MustTail) { - return makeUnexpected(ErrorCode::Unsupported, - "musttail call is not ABI-compatible: " + RejectionReason); - } else { - auto NormalResults = verifyValueSequence(Request.Results, true); - if (!NormalResults) - return std::unexpected(NormalResults.error()); - } - } - - CallLoweringPlanDesc Plan; - Plan.Kind = CallLoweringPlanKind::Call; - Plan.Target = PlanTarget; - Plan.Convention = Request.Convention; - Plan.ArgumentAssignments = std::move(Arguments->Assignments); - Plan.ReturnAssignments = std::move(*Results); - Plan.StackArguments = Arguments->StackArea; - Plan.CallFrame = CallFrameRequirements{ - Decision == TailCallKind::NotTail ? *CallFrameSize : Arguments->StackArea.SizeBytes, *Align16, - Decision == TailCallKind::NotTail, Decision == TailCallKind::NotTail, - Decision != TailCallKind::NotTail}; - if (Request.IsVariadic) { - Plan.VarArgs.IsVariadic = true; - Plan.VarArgs.FixedArgumentCount = Request.FixedArgumentCount; - } - Plan.ByValCopies = std::move(ByValCopies); - Plan.PreservedMask = createCallPreservedMask((*Convention)->ID); - if (!Plan.PreservedMask) - return makeUnexpected(ErrorCode::InternalError, "RISC-V call-preserved mask is unavailable"); - Plan.Callee = std::move(EffectiveCallee); - Plan.TailDecision = Decision; - Plan.TailRejectionReason = std::move(RejectionReason); - return CallLoweringPlan::create(std::move(Plan)); -} - -Expected -RISCVCallLowering::planReturn(const ReturnLoweringRequest &Request) const { - auto Convention = riscvConvention(Request.Convention); - if (!Convention) - return std::unexpected(Convention.error()); - auto Values = assignReturns(Request.Values, true, **Convention); - if (!Values) - return std::unexpected(Values.error()); - - CallLoweringPlanDesc Plan; - Plan.Kind = CallLoweringPlanKind::Return; - Plan.Target = PlanTarget; - Plan.Convention = Request.Convention; - Plan.ReturnAssignments = std::move(*Values); - return CallLoweringPlan::create(std::move(Plan)); -} - -[[nodiscard]] CallValueTransform outgoingTransform(const CCAssignment &Assignment) noexcept { - if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast)) - return CallValueTransform::Bitcast; - switch (Assignment.Extension) { - case ABIExtensionKind::None: - return CallValueTransform::Full; - case ABIExtensionKind::AnyExtend: - return CallValueTransform::AnyExtend; - case ABIExtensionKind::SignExtend: - return CallValueTransform::SignExtend; - case ABIExtensionKind::ZeroExtend: - return CallValueTransform::ZeroExtend; - case ABIExtensionKind::Count: - break; - } - return CallValueTransform::Full; -} - -[[nodiscard]] CallValueTransform incomingTransform(const CCAssignment &Assignment) noexcept { - if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast)) - return CallValueTransform::Bitcast; - if (Assignment.ValueType.scalarBitWidth() < Assignment.LocationType.scalarBitWidth()) - return CallValueTransform::Truncate; - return CallValueTransform::Full; -} - -[[nodiscard]] Expected addUnsignedImmediate(MachineInstrBuilder &Builder, - std::uint64_t Value) { - auto Immediate = IntegerImmediate::fromUnsigned(64, Value); - if (!Immediate) - return std::unexpected(Immediate.error()); - return Builder.addIntegerImmediate(std::move(*Immediate)); -} - -[[nodiscard]] Expected -createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::uint64_t SizeBytes, - Alignment Align, MemoryBase Base, std::int64_t Offset = 0) { - auto Size = MemorySize::fixed(SizeBytes); - if (!Size) - return std::unexpected(Size.error()); - MachineMemOperandDesc Description; - Description.Accesses = Accesses; - Description.Size = *Size; - Description.Align = Align; - Description.Base = std::move(Base); - Description.Offset = Offset; - return Function.createMemOperand(std::move(Description)); -} - -[[nodiscard]] Expected emitCCCopy(MachineIRBuilder &Builder, Register Destination, - Register Source, CallValueTransform Transform) { - const Opcode CopyOpcode = Transform == CallValueTransform::Full - ? opcode(GenericOpcode::COPY) - : targetOpcode(InstructionOpcode::PseudoCC_COPY); - auto Instruction = Builder.buildInstr(CopyOpcode); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Def = Instruction->addDef(Destination); - if (!Def) - return std::unexpected(Def.error()); - auto Use = Instruction->addUse(Source); - if (!Use) - return std::unexpected(Use.error()); - if (Transform != CallValueTransform::Full) { - auto Action = Instruction->addImmediate(static_cast(Transform)); - if (!Action) - return std::unexpected(Action.error()); - } - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected emitStackLoad(MachineIRBuilder &Builder, - const CCAssignment &Assignment, FrameIndex Object, - std::uint64_t ObjectOffset, - const MachineMemOperand &Memory) { - auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoLOAD_STACK_ARG)); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Def = Instruction->addDef(Assignment.OriginalValue); - if (!Def) - return std::unexpected(Def.error()); - auto Frame = Instruction->addFrameIndex(Object); - if (!Frame) - return std::unexpected(Frame.error()); - auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset); - if (!Offset) - return std::unexpected(Offset.error()); - auto Transform = - Instruction->addImmediate(static_cast(incomingTransform(Assignment))); - if (!Transform) - return std::unexpected(Transform.error()); - auto Mem = Instruction->addMemOperand(Memory); - if (!Mem) - return std::unexpected(Mem.error()); - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected emitStackStore(MachineIRBuilder &Builder, Register Source, - FrameIndex Object, std::uint64_t ObjectOffset, - const MachineMemOperand &Memory, - CallValueTransform Transform) { - auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoSTORE_STACK_ARG)); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Use = Instruction->addUse(Source); - if (!Use) - return std::unexpected(Use.error()); - auto Frame = Instruction->addFrameIndex(Object); - if (!Frame) - return std::unexpected(Frame.error()); - auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset); - if (!Offset) - return std::unexpected(Offset.error()); - auto Action = Instruction->addImmediate(static_cast(Transform)); - if (!Action) - return std::unexpected(Action.error()); - auto Mem = Instruction->addMemOperand(Memory); - if (!Mem) - return std::unexpected(Mem.error()); - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected ensureFunctionLiveIn(MachineRegisterInfo &Registers, - PhysicalRegister Reg, - std::optional InitialValue) { - const auto Existing = - std::ranges::find(Registers.liveIns(), Reg, &PhysicalRegisterLiveIn::PhysReg); - if (Existing == Registers.liveIns().end()) - return Registers.addLiveIn(Reg, InitialValue); - if (InitialValue && Existing->InitialVirtualRegister && - Existing->InitialVirtualRegister != InitialValue) { - return makeUnexpected(ErrorCode::VerificationError, - "physical formal register already maps to another virtual live-in"); - } - return {}; -} - -[[nodiscard]] Expected ensureBlockLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg) { - return Block.addLiveIn(Reg, LaneMask::all()); -} - -[[nodiscard]] Expected verifyPlanRegisters(const CallLoweringPlan &Plan, - const MachineFunction &Function, - const TargetDescription &Target) { - const MachineRegisterInfo &Registers = Function.registerInfo(); - const auto Verify = [&Registers, &Target](const CCAssignment &Assignment) -> Expected { - if (Assignment.LocationRegister && - !Target.registers().isKnownPhysicalRegister(*Assignment.LocationRegister)) { - return makeUnexpected(ErrorCode::VerificationError, - "call-lowering plan names an unknown ABI location register"); - } - const Register Reg = Assignment.OriginalValue; - if (Reg.isNoRegister()) - return {}; - if (Reg.isPhysical()) { - if (!Target.registers().isKnownPhysicalRegister(*Reg.physicalRegister())) { - return makeUnexpected(ErrorCode::VerificationError, - "call-lowering plan names an unknown physical value register"); - } - return {}; - } - auto Info = Registers.virtualRegisterInfo(Reg); - if (!Info) - return std::unexpected(Info.error()); - if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) { - return makeUnexpected(ErrorCode::VerificationError, - "call-lowering plan register LLT differs from MachineRegisterInfo"); - } - return {}; - }; - for (const CCAssignment &Assignment : Plan.argumentAssignments()) { - auto Result = Verify(Assignment); - if (!Result) - return std::unexpected(Result.error()); - } - for (const CCAssignment &Assignment : Plan.returnAssignments()) { - auto Result = Verify(Assignment); - if (!Result) - return std::unexpected(Result.error()); - } - if (const MachineCallTarget *Callee = Plan.callee(); - Callee != nullptr && Callee->kind() == MachineCallTargetKind::IndirectRegister) { - const Register CalleeReg = Callee->indirectRegister(); - if (CalleeReg.isPhysical()) { - const std::optional Physical = CalleeReg.physicalRegister(); - if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) { - return makeUnexpected(ErrorCode::VerificationError, - "indirect RISC-V callee names an unknown physical register"); - } - } else { - auto Info = Registers.virtualRegisterInfo(CalleeReg); - if (!Info) - return std::unexpected(Info.error()); - const std::optional Type = Info->get().Type; - if (!Type || ((!Type->isPointer() && !Type->isInteger()) || Type->scalarBitWidth() != 64)) { - return makeUnexpected(ErrorCode::VerificationError, - "indirect RISC-V callee requires a 64-bit pointer or integer LLT"); - } - } - } - return {}; -} - -[[nodiscard]] Expected verifyRequiredOpcodes(const MachineIRBuilder &Builder) { - static constexpr std::array Required{ - InstructionOpcode::PseudoCC_COPY, - InstructionOpcode::PseudoLOAD_STACK_ARG, - InstructionOpcode::PseudoSTORE_STACK_ARG, - InstructionOpcode::PseudoBYVAL_COPY, - InstructionOpcode::ADJCALLSTACKDOWN, - InstructionOpcode::ADJCALLSTACKUP, - InstructionOpcode::PseudoCALL, - InstructionOpcode::PseudoTAIL, - InstructionOpcode::PseudoRET, - }; - for (const InstructionOpcode OpcodeValue : Required) { - if (Builder.target().instructionDescriptor(targetOpcode(OpcodeValue)) == nullptr) { - return makeUnexpected(ErrorCode::InternalError, - "RISC-V target description lacks a call-lowering pseudo"); - } - } - return {}; -} - -[[nodiscard]] Expected verifyFormalLiveIns(const CallLoweringPlan &Plan, - const MachineFunction &Function) { - if (Plan.kind() != CallLoweringPlanKind::FormalArguments) - return {}; - - const auto LiveIns = Function.registerInfo().liveIns(); - for (const CCAssignment &Assignment : Plan.argumentAssignments()) { - if (!Assignment.LocationRegister || !Assignment.OriginalValue.isVirtual()) - continue; - const auto Existing = - std::ranges::find(LiveIns, *Assignment.LocationRegister, &PhysicalRegisterLiveIn::PhysReg); - if (Existing != LiveIns.end() && Existing->InitialVirtualRegister && - *Existing->InitialVirtualRegister != Assignment.OriginalValue) { - return makeUnexpected(ErrorCode::VerificationError, - "physical formal register already maps to another virtual live-in"); - } - } - return {}; -} - -[[nodiscard]] Expected preflightPlan(MachineIRBuilder &Builder, - const CallLoweringPlan &Plan) { - auto Convention = riscvConvention(Plan.convention()); - if (!Convention) - return std::unexpected(Convention.error()); - if (Plan.target() != PlanTarget) { - return makeUnexpected(ErrorCode::InvalidArgument, - "call-lowering plan belongs to another target or ABI"); - } - auto InsertPoint = Builder.verifyInsertPoint(); - if (!InsertPoint) - return std::unexpected(InsertPoint.error()); - auto Opcodes = verifyRequiredOpcodes(Builder); - if (!Opcodes) - return std::unexpected(Opcodes.error()); - auto Registers = verifyPlanRegisters(Plan, Builder.function(), Builder.target()); - if (!Registers) - return std::unexpected(Registers.error()); - auto LiveIns = verifyFormalLiveIns(Plan, Builder.function()); - if (!LiveIns) - return std::unexpected(LiveIns.error()); - for (const CCAssignment &Assignment : Plan.argumentAssignments()) { - if (Assignment.StackOffsetBytes && - *Assignment.StackOffsetBytes > - static_cast(std::numeric_limits::max())) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V stack assignment offset exceeds the frame-index range"); - } - } - for (const ByValCopyAssignment &Copy : Plan.byValCopies()) { - if (Copy.StackOffsetBytes > - static_cast(std::numeric_limits::max())) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V byval offset exceeds the frame-index range"); - } - } - if (Plan.kind() != CallLoweringPlanKind::Return && Builder.function().frameInfo().isFrozen()) { - return makeUnexpected(ErrorCode::VerificationError, - "call lowering cannot modify a frozen machine frame"); - } - if (Plan.kind() == CallLoweringPlanKind::FormalArguments && - (Builder.function().firstBlock() == nullptr || - Builder.insertionBlock() != Builder.function().firstBlock())) { - return makeUnexpected(ErrorCode::InvalidArgument, - "formal arguments must be lowered in the entry machine block"); - } - return {}; -} - -[[nodiscard]] Expected updateMaximumFrameAlignment(MachineFrameState &State, - Alignment Required) { - if (!State.MaxAlignment || State.MaxAlignment->bytes() < Required.bytes()) - State.MaxAlignment = Required; - return {}; -} - -[[nodiscard]] Expected emitFormalArguments(MachineIRBuilder &Builder, - const CallLoweringPlan &Plan) { - MachineFunction &Function = Builder.function(); - MachineBasicBlock &Entry = *Function.firstBlock(); - MachineRegisterInfo &Registers = Function.registerInfo(); - MachineFrameInfo &Frame = Function.frameInfo(); - Alignment RequiredFrameAlignment; - - for (const CCAssignment &Assignment : Plan.argumentAssignments()) { - if (Assignment.LocationRegister) { - const PhysicalRegister Location = *Assignment.LocationRegister; - const std::optional Initial = - Assignment.OriginalValue.isVirtual() ? std::optional(Assignment.OriginalValue) - : std::nullopt; - auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Location, Initial); - if (!FunctionLiveIn) - return std::unexpected(FunctionLiveIn.error()); - auto BlockLiveIn = ensureBlockLiveIn(Entry, Location); - if (!BlockLiveIn) - return std::unexpected(BlockLiveIn.error()); - auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(Location), - incomingTransform(Assignment)); - if (!Copy) - return std::unexpected(Copy.error()); - continue; - } - - if (!Assignment.StackOffsetBytes) { - return makeUnexpected(ErrorCode::InternalError, - "verified formal stack assignment has no stack offset"); - } - StackObjectDesc Object; - Object.Kind = StackObjectKind::FixedABI; - Object.Size = Assignment.StackSlotSizeBytes; - Object.Align = Assignment.LocationAlignment; - Object.Offset = static_cast(*Assignment.StackOffsetBytes); - Object.IsImmutable = true; - Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::IncomingArgument); - auto FrameObject = Frame.createObject(std::move(Object)); - if (!FrameObject) - return std::unexpected(FrameObject.error()); - auto Memory = - createMemoryOperand(Function, {MemoryAccessFlag::Load}, Assignment.LocationSizeBytes, - Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject)); - if (!Memory) - return std::unexpected(Memory.error()); - auto Load = emitStackLoad(Builder, Assignment, *FrameObject, 0, **Memory); - if (!Load) - return std::unexpected(Load.error()); - if (RequiredFrameAlignment.bytes() < Assignment.LocationAlignment.bytes()) - RequiredFrameAlignment = Assignment.LocationAlignment; - } - - const VarArgLoweringState &VarArgs = Plan.varArgs(); - if (!VarArgs.IsVariadic) { - if (RequiredFrameAlignment.bytes() == 1) - return {}; - MachineFrameState State = Frame.state(); - auto AlignmentUpdate = updateMaximumFrameAlignment(State, RequiredFrameAlignment); - if (!AlignmentUpdate) - return std::unexpected(AlignmentUpdate.error()); - return Frame.setState(std::move(State)); - } - - auto Align8 = alignment(XLenBytes); - auto Align16 = alignment(ABIStackAlignmentBytes); - if (!Align8 || !Align16) - return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid"); - - FrameIndex SaveObject; - if (VarArgs.RegisterSaveAreaSizeBytes == 0) { - StackObjectDesc Object; - Object.Kind = StackObjectKind::FixedABI; - Object.Size = XLenBytes; - Object.Align = *Align8; - Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes; - Object.IsImmutable = true; - Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea); - auto Created = Frame.createObject(std::move(Object)); - if (!Created) - return std::unexpected(Created.error()); - SaveObject = *Created; - } else { - StackObjectDesc Object; - Object.Kind = StackObjectKind::FixedABI; - Object.Size = VarArgs.RegisterSaveAreaSizeBytes; - Object.Align = VarArgs.FirstUnallocatedIntegerRegister % 2 == 0 ? *Align16 : *Align8; - Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes; - Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea); - auto Created = Frame.createObject(std::move(Object)); - if (!Created) - return std::unexpected(Created.error()); - SaveObject = *Created; - - if (VarArgs.RegisterSavePaddingBytes != 0) { - StackObjectDesc Padding; - Padding.Kind = StackObjectKind::FixedABI; - Padding.Size = VarArgs.RegisterSavePaddingBytes; - Padding.Align = *Align8; - Padding.Offset = VarArgs.RegisterSaveAreaOffsetBytes - - static_cast(VarArgs.RegisterSavePaddingBytes); - Padding.IsImmutable = true; - Padding.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgAlignmentPadding); - auto CreatedPadding = Frame.createObject(std::move(Padding)); - if (!CreatedPadding) - return std::unexpected(CreatedPadding.error()); - } - } - - for (const VarArgRegisterSave Save : VarArgs.RegistersToSave) { - auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Save.Reg, std::nullopt); - if (!FunctionLiveIn) - return std::unexpected(FunctionLiveIn.error()); - auto BlockLiveIn = ensureBlockLiveIn(Entry, Save.Reg); - if (!BlockLiveIn) - return std::unexpected(BlockLiveIn.error()); - auto Memory = createMemoryOperand(Function, {MemoryAccessFlag::Store}, XLenBytes, *Align8, - MemoryBase::frameIndex(SaveObject), - static_cast(Save.OffsetBytes)); - if (!Memory) - return std::unexpected(Memory.error()); - auto Store = emitStackStore(Builder, Register::fromPhysical(Save.Reg), SaveObject, - Save.OffsetBytes, **Memory, CallValueTransform::Full); - if (!Store) - return std::unexpected(Store.error()); - } - - MachineFrameState State = Frame.state(); - const Alignment VarArgAlignment = - RequiredFrameAlignment.bytes() > Align16->bytes() ? RequiredFrameAlignment : *Align16; - auto AlignmentUpdate = updateMaximumFrameAlignment(State, VarArgAlignment); - if (!AlignmentUpdate) - return std::unexpected(AlignmentUpdate.error()); - return Frame.setState(std::move(State)); -} - -[[nodiscard]] Expected emitCallFrameAdjustment(MachineIRBuilder &Builder, - InstructionOpcode OpcodeValue, - const CallFrameRequirements &Frame) { - if (Frame.SizeBytes > static_cast(std::numeric_limits::max())) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V call-frame size does not fit a MIR immediate"); - } - auto Instruction = Builder.buildInstr(targetOpcode(OpcodeValue)); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Size = Instruction->addImmediate(static_cast(Frame.SizeBytes)); - if (!Size) - return std::unexpected(Size.error()); - auto PreparedBytes = Instruction->addImmediate(0); - if (!PreparedBytes) - return std::unexpected(PreparedBytes.error()); - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected> -emitByValCopies(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { - MachineFunction &Function = Builder.function(); - MachineFrameInfo &Frame = Function.frameInfo(); - std::unordered_map Replacements; - for (const ByValCopyAssignment Copy : Plan.byValCopies()) { - StackObjectDesc Object; - Object.Kind = StackObjectKind::Target; - Object.Size = Copy.SizeBytes; - Object.Align = Copy.Align; - Object.LocalOffset = static_cast(Copy.StackOffsetBytes); - Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::ByValCopy); - auto FrameObject = Frame.createObject(std::move(Object)); - if (!FrameObject) - return std::unexpected(FrameObject.error()); - auto Temporary = Function.registerInfo().createVirtualRegister(Copy.PointerType); - if (!Temporary) - return std::unexpected(Temporary.error()); - - auto SourceMemory = createMemoryOperand(Function, {MemoryAccessFlag::Load}, Copy.SizeBytes, - Copy.Align, MemoryBase{}); - if (!SourceMemory) - return std::unexpected(SourceMemory.error()); - auto DestinationMemory = - createMemoryOperand(Function, {MemoryAccessFlag::Store}, Copy.SizeBytes, Copy.Align, - MemoryBase::frameIndex(*FrameObject)); - if (!DestinationMemory) - return std::unexpected(DestinationMemory.error()); - - auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoBYVAL_COPY)); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Def = Instruction->addDef(*Temporary); - if (!Def) - return std::unexpected(Def.error()); - auto Use = Instruction->addUse(Copy.Source); - if (!Use) - return std::unexpected(Use.error()); - auto FrameOperand = Instruction->addFrameIndex(*FrameObject); - if (!FrameOperand) - return std::unexpected(FrameOperand.error()); - auto Size = addUnsignedImmediate(*Instruction, Copy.SizeBytes); - if (!Size) - return std::unexpected(Size.error()); - auto Align = addUnsignedImmediate(*Instruction, Copy.Align.bytes()); - if (!Align) - return std::unexpected(Align.error()); - auto SourceMem = Instruction->addMemOperand(**SourceMemory); - if (!SourceMem) - return std::unexpected(SourceMem.error()); - auto DestinationMem = Instruction->addMemOperand(**DestinationMemory); - if (!DestinationMem) - return std::unexpected(DestinationMem.error()); - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - Replacements.emplace(Copy.ArgumentAssignmentIndex, *Temporary); - } - return Replacements; -} - -[[nodiscard]] Expected -emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan, - const std::unordered_map &Replacements) { - MachineFunction &Function = Builder.function(); - MachineFrameInfo &Frame = Function.frameInfo(); - const auto Assignments = Plan.argumentAssignments(); - for (std::size_t Index = 0; Index < Assignments.size(); ++Index) { - const CCAssignment &Assignment = Assignments[Index]; - const auto Replacement = Replacements.find(Index); - const Register Source = - Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second; - if (Assignment.LocationRegister) { - auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Assignment.LocationRegister), Source, - outgoingTransform(Assignment)); - if (!Copy) - return std::unexpected(Copy.error()); - continue; - } - - if (!Assignment.StackOffsetBytes) { - return makeUnexpected(ErrorCode::InternalError, - "verified outgoing stack assignment has no stack offset"); - } - StackObjectDesc Object; - Object.Size = Assignment.StackSlotSizeBytes; - Object.Align = Assignment.LocationAlignment; - Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::OutgoingArgument); - if (Plan.tailDecision() == TailCallKind::NotTail) { - Object.Kind = StackObjectKind::Target; - Object.LocalOffset = static_cast(*Assignment.StackOffsetBytes); - } else { - Object.Kind = StackObjectKind::FixedABI; - Object.Offset = static_cast(*Assignment.StackOffsetBytes); - } - auto FrameObject = Frame.createObject(std::move(Object)); - if (!FrameObject) - return std::unexpected(FrameObject.error()); - auto Memory = - createMemoryOperand(Function, {MemoryAccessFlag::Store}, Assignment.LocationSizeBytes, - Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject)); - if (!Memory) - return std::unexpected(Memory.error()); - auto Store = - emitStackStore(Builder, Source, *FrameObject, 0, **Memory, outgoingTransform(Assignment)); - if (!Store) - return std::unexpected(Store.error()); - } - return {}; -} - -[[nodiscard]] Expected addCallTarget(MachineInstrBuilder &Instruction, - const MachineCallTarget &Callee) { - switch (Callee.kind()) { - case MachineCallTargetKind::GlobalSymbol: - return Instruction.addGlobalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags()); - case MachineCallTargetKind::ExternalSymbol: - return Instruction.addExternalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags()); - case MachineCallTargetKind::IndirectRegister: - return Instruction.addUse(Callee.indirectRegister()); - case MachineCallTargetKind::Invalid: - break; - } - return makeUnexpected(ErrorCode::InternalError, - "verified call-lowering plan has an invalid callee"); -} - -[[nodiscard]] Expected requiredLocationRegister(const CCAssignment &Assignment) { - if (!Assignment.LocationRegister) { - return makeUnexpected(ErrorCode::InternalError, - "verified return assignment has no location register"); - } - return *Assignment.LocationRegister; -} - -[[nodiscard]] Expected emitCallInstruction(MachineIRBuilder &Builder, - const CallLoweringPlan &Plan) { - const bool IsTail = Plan.tailDecision() != TailCallKind::NotTail; - auto Instruction = Builder.buildInstr( - targetOpcode(IsTail ? InstructionOpcode::PseudoTAIL : InstructionOpcode::PseudoCALL)); - if (!Instruction) - return std::unexpected(Instruction.error()); - auto Callee = addCallTarget(*Instruction, *Plan.callee()); - if (!Callee) - return std::unexpected(Callee.error()); - auto Mask = Instruction->addRegisterMask(Plan.preservedMask()); - if (!Mask) - return std::unexpected(Mask.error()); - - std::unordered_set DynamicUses; - for (const CCAssignment &Assignment : Plan.argumentAssignments()) { - if (Assignment.LocationRegister && DynamicUses.insert(*Assignment.LocationRegister).second) { - auto Use = Instruction->addImplicitUse(Register::fromPhysical(*Assignment.LocationRegister)); - if (!Use) - return std::unexpected(Use.error()); - } - } - std::unordered_set DynamicDefs; - if (!IsTail) { - for (const CCAssignment &Assignment : Plan.returnAssignments()) { - if (Assignment.LocationRegister && DynamicDefs.insert(*Assignment.LocationRegister).second) { - auto Def = - Instruction->addImplicitDef(Register::fromPhysical(*Assignment.LocationRegister)); - if (!Def) - return std::unexpected(Def.error()); - } - } - } - auto Built = Instruction->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected emitCall(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { - if (Plan.callFrame().EmitSetup) { - auto Down = - emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKDOWN, Plan.callFrame()); - if (!Down) - return std::unexpected(Down.error()); - } - - auto Replacements = emitByValCopies(Builder, Plan); - if (!Replacements) - return std::unexpected(Replacements.error()); - auto Arguments = emitOutgoingArguments(Builder, Plan, *Replacements); - if (!Arguments) - return std::unexpected(Arguments.error()); - auto Call = emitCallInstruction(Builder, Plan); - if (!Call) - return std::unexpected(Call.error()); - - if (Plan.callFrame().EmitDestroy) { - auto Up = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKUP, Plan.callFrame()); - if (!Up) - return std::unexpected(Up.error()); - } - if (Plan.tailDecision() == TailCallKind::NotTail) { - for (const CCAssignment &Assignment : Plan.returnAssignments()) { - auto Location = requiredLocationRegister(Assignment); - if (!Location) - return std::unexpected(Location.error()); - auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(*Location), - incomingTransform(Assignment)); - if (!Copy) - return std::unexpected(Copy.error()); - } - } - - MachineFrameInfo &Frame = Builder.function().frameInfo(); - MachineFrameState State = Frame.state(); - State.HasCalls = true; - State.HasTailCall = State.HasTailCall || Plan.tailDecision() != TailCallKind::NotTail; - State.AdjustsStack = - State.AdjustsStack || (Plan.callFrame().EmitSetup && Plan.callFrame().SizeBytes != 0); - State.HasMustTailInVarArgFunc = - State.HasMustTailInVarArgFunc || - (Plan.tailDecision() == TailCallKind::MustTail && Plan.varArgs().IsVariadic); - if (Plan.tailDecision() == TailCallKind::NotTail) { - State.MaxCallFrameSize = - std::max(State.MaxCallFrameSize.value_or(0), Plan.callFrame().SizeBytes); - } - auto AlignmentUpdate = updateMaximumFrameAlignment(State, Plan.callFrame().Align); - if (!AlignmentUpdate) - return std::unexpected(AlignmentUpdate.error()); - return Frame.setState(std::move(State)); -} - -[[nodiscard]] Expected emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { - for (const CCAssignment &Assignment : Plan.returnAssignments()) { - auto Location = requiredLocationRegister(Assignment); - if (!Location) - return std::unexpected(Location.error()); - auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Location), Assignment.OriginalValue, - outgoingTransform(Assignment)); - if (!Copy) - return std::unexpected(Copy.error()); - } - - auto Return = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoRET)); - if (!Return) - return std::unexpected(Return.error()); - for (const CCAssignment &Assignment : Plan.returnAssignments()) { - auto Location = requiredLocationRegister(Assignment); - if (!Location) - return std::unexpected(Location.error()); - auto Use = Return->addImplicitUse(Register::fromPhysical(*Location)); - if (!Use) - return std::unexpected(Use.error()); - } - auto Built = Return->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -Expected RISCVCallLowering::emitPlan(MachineIRBuilder &Builder, - const CallLoweringPlan &Plan) const { - auto Preflight = preflightPlan(Builder, Plan); - if (!Preflight) - return std::unexpected(Preflight.error()); - switch (Plan.kind()) { - case CallLoweringPlanKind::FormalArguments: - return emitFormalArguments(Builder, Plan); - case CallLoweringPlanKind::Call: - return emitCall(Builder, Plan); - case CallLoweringPlanKind::Return: - return emitReturn(Builder, Plan); - } - return makeUnexpected(ErrorCode::InternalError, "call-lowering plan kind is invalid"); -} - -} // namespace - -const TargetCallLowering &callLowering() noexcept { - static const RISCVCallLowering Lowering; - return Lowering; -} - -} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp deleted file mode 100644 index b610d81..0000000 --- a/backend/lib/Target/RISCV/RISCVCodeGen.cpp +++ /dev/null @@ -1,6536 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "rcc/Target/RISCV/RISCVCodeGen.h" - -#include "RISCVImmediateMaterialization.h" -#include "rcc/CodeGen/GenericOpcodes.h" -#include "rcc/CodeGen/LowLevelType.h" -#include "rcc/CodeGen/MachineBasicBlock.h" -#include "rcc/CodeGen/MachineFunction.h" -#include "rcc/CodeGen/MachineIRBuilder.h" -#include "rcc/CodeGen/MachineInstr.h" -#include "rcc/CodeGen/MachineMemOperand.h" -#include "rcc/CodeGen/MachineOperand.h" -#include "rcc/CodeGen/MachineRegisterInfo.h" -#include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVCallLowering.h" -#include "rcc/Target/RISCV/RISCVTarget.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rcc::riscv { -namespace { - -[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { - return Opcode::fromValue(static_cast(Value)); -} - -[[nodiscard]] std::optional registerType(const MachineFunction &Function, - const MachineOperand &Operand) { - const Register *Reg = Operand.getIf(); - if (Reg == nullptr || !Reg->isVirtual()) - return std::nullopt; - auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); - if (!Info) - return std::nullopt; - return Info->get().Type; -} - -[[nodiscard]] std::optional resultType(const MachineFunction &Function, - const MachineInstr &Instruction) { - const MachineOperand *Result = Instruction.operandAt(0); - return Result == nullptr ? std::nullopt : registerType(Function, *Result); -} - -[[nodiscard]] std::optional operandType(const MachineFunction &Function, - const MachineInstr &Instruction, std::size_t Index) { - const MachineOperand *Operand = Instruction.operandAt(Index); - return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); -} - -[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { - if (!Type.isFloat()) - return false; - const std::optional Semantics = Type.floatSemantics(); - return Semantics && - (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); -} - -[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64) || - (Type.isPointer() && Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64; -} - -[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && - (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64) || - (Type.isPointer() && Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isI1Type(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; -} - -[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { - return (Destination.isInteger() || Destination.isAnyScalar()) && - (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && - Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && - Destination.scalarBitWidth() <= 64; -} - -[[nodiscard]] Expected -extendIntegerImmediate(const IntegerImmediate &Value, std::uint32_t DestinationWidth, - bool SignExtend) { - if (DestinationWidth == 0 || DestinationWidth > 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V constant extension requires a destination no wider than 64 bits"); - - std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front(); - const std::uint32_t SourceWidth = Value.bitWidth(); - if (SignExtend && Value.isNegative() && SourceWidth < 64) - Raw |= ~std::uint64_t{0} << SourceWidth; - if (DestinationWidth < 64) - Raw &= (std::uint64_t{1} << DestinationWidth) - 1U; - return IntegerImmediate::create(DestinationWidth, std::span(&Raw, 1)); -} - -[[nodiscard]] Expected foldConstantExtension(MachineInstr &Instruction, - MachineFunction &Function, - GenericOpcode Generic) { - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || - !Destination->isVirtual()) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant extension requires virtual-register operands"); - } - const Register SourceRegister = *Source; - const Register DestinationRegister = *Destination; - const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister); - const MachineInstr *SourceDef = SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); - if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V constant extension requires a single constant definition"); - const MachineOperand *ConstantOperand = SourceDef->operandAt(1); - const auto *Integer = ConstantOperand == nullptr - ? nullptr - : ConstantOperand->getIf(); - if (Integer == nullptr) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V constant extension requires an integer constant"); - const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); - if (!DestinationInfo || !DestinationInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant extension destination has no type"); - const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth(); - const bool SignExtend = Generic != GenericOpcode::G_ZEXT; - auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend); - if (!Extended) - return std::unexpected(Extended.error()); - - if (Function.registerInfo().hasOneUse(SourceRegister)) { - MachineInstr *Def = const_cast(SourceDef); - if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced) - return std::unexpected(Replaced.error()); - auto Erased = Def->eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended))); - if (!Added) - return std::unexpected(Added.error()); - MachineBasicBlock *Parent = Instruction.parent(); - if (Parent == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant extension is detached during folding"); - return Parent->removeInstruction(Instruction); - } - - auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Instruction.operandCount() > 1) { - auto Erased = Instruction.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); - if (!Added) - return std::unexpected(Added.error()); - - if (Function.registerInfo().useEmpty(SourceRegister)) { - MachineInstr *Def = const_cast(SourceDef); - MachineBasicBlock *Parent = Def->parent(); - if (Parent == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant extension source definition is detached"); - auto Removed = Parent->removeInstruction(*Def); - if (!Removed) - return std::unexpected(Removed.error()); - // Keep the source register record as a tombstone-free historical vreg until - // the legalizer worklist transaction has completed. - } - return {}; -} - -struct WideConstantUnmergePlan final { - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = nullptr; - std::vector Outputs; - bool SignExtend = false; -}; - -[[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { - MachineInstr *UseInstruction = nullptr; - auto &RegisterInfo = const_cast(Function.registerInfo()); - for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { - if (Operand.parent() == nullptr) - continue; - if (UseInstruction != nullptr) - return nullptr; - UseInstruction = Operand.parent(); - } - return UseInstruction; -} - -[[nodiscard]] std::optional -wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { - const MachineOperand *ResultOperand = Constant.operandAt(0); - const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); - if (Result == nullptr || !Result->isVirtual()) - return std::nullopt; - - MachineInstr *FirstUse = singleRegisterUse(Function, *Result); - if (FirstUse == nullptr) - return std::nullopt; - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = FirstUse; - if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || - FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || - FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { - Extension = FirstUse; - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - if (Extended == nullptr || !Extended->isVirtual()) - return std::nullopt; - Unmerge = singleRegisterUse(Function, *Extended); - if (Unmerge == nullptr) - return std::nullopt; - } - if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || - Unmerge->operandCount() < 3) - return std::nullopt; - - WideConstantUnmergePlan Plan; - Plan.Extension = Extension; - Plan.Unmerge = Unmerge; - Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); - for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { - const MachineOperand *Operand = Unmerge->operandAt(Index); - const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); - if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) - return std::nullopt; - const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); - if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 || - Info->get().Type->scalarBitWidth() > 64 || - (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar())) - return std::nullopt; - Plan.Outputs.push_back(*Reg); - } - const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - const Register *ExpectedSource = Result; - if (Extension != nullptr) { - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - } - if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) - return std::nullopt; - return Plan; -} - -[[nodiscard]] Expected -wideConstantWord(const IntegerImmediate &Value, std::size_t WordIndex, std::uint32_t Width, - bool SignExtend) { - std::uint64_t Raw = 0; - if (WordIndex < Value.words().size()) - Raw = Value.words()[WordIndex]; - else if (SignExtend && Value.isNegative()) - Raw = std::numeric_limits::max(); - const std::size_t WordBitOffset = WordIndex * 64U; - if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { - const std::uint32_t Remaining = - Value.bitWidth() - static_cast(WordBitOffset); - if (Remaining < 64) - Raw |= ~std::uint64_t{0} << Remaining; - } - if (Width < 64) - Raw &= (std::uint64_t{1} << Width) - 1U; - return IntegerImmediate::create(Width, std::span(&Raw, 1)); -} - -[[nodiscard]] Expected foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = wideConstantUnmergePlan(Instruction, Function); - if (!Plan) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide constant requires a G_UNMERGE_VALUES use"); - const MachineOperand *ImmediateOperand = Instruction.operandAt(1); - const auto *Integer = ImmediateOperand == nullptr - ? nullptr - : ImmediateOperand->getIf(); - if (Integer == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant has no integer immediate"); - - std::vector Words; - Words.reserve(Plan->Outputs.size()); - for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { - const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]); - if (!Info || !Info->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant output has no virtual-register type"); - auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(), - Plan->SignExtend); - if (!Word) - return std::unexpected(Word.error()); - Words.push_back(std::move(*Word)); - } - - std::vector Representatives(Plan->Outputs.size()); - for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { - Representatives[Index] = Index; - for (std::size_t Previous = 0; Previous < Index; ++Previous) { - if (Words[Previous] == Words[Index]) { - Representatives[Index] = Previous; - break; - } - } - } - - const Register FirstOutput = Plan->Outputs.front(); - auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); - if (!Retargeted) - return std::unexpected(Retargeted.error()); - auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Instruction.operandCount() > 1) { - auto Erased = Instruction.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); - if (!Added) - return std::unexpected(Added.error()); - - MachineBasicBlock *Block = Instruction.parent(); - if (Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant is detached during legalization"); - if (Plan->Unmerge != &Instruction) { - auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); - if (!RemovedUnmerge) - return std::unexpected(RemovedUnmerge.error()); - } - if (Plan->Extension != nullptr) { - auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); - if (!RemovedExtension) - return std::unexpected(RemovedExtension.error()); - } - - for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { - if (Representatives[Index] == Index) - continue; - auto Replaced = Function.registerInfo().replaceRegWith( - Plan->Outputs[Index], Plan->Outputs[Representatives[Index]]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - - MachineInstr *Last = &Instruction; - for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { - if (Representatives[Index] != Index) - continue; - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointAfter(*Last); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); - if (!Pending) - return std::unexpected(Pending.error()); - auto Def = Pending->addDef(Plan->Outputs[Index]); - if (!Def) - return std::unexpected(Def.error()); - auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); - if (!Value) - return std::unexpected(Value.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - Last = *Built; - } - return {}; -} - -[[nodiscard]] Expected -buildIntegerConstantBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - std::uint32_t Width, std::uint64_t Value); - -[[nodiscard]] Expected -buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Left, - Register Right); - -[[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Source); - -[[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction); - -[[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, - Register Destination, Register Source); - -struct NarrowAnyExtBinaryPlan final { - MachineInstr *Binary = nullptr; - GenericOpcode Generic = GenericOpcode::Invalid; - std::array WideInputs{}; - std::vector Artifacts; - std::vector Constants; - std::uint32_t SourceWidth = 0; -}; - -[[nodiscard]] std::optional -narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { - const MachineOperand *DestinationOperand = Extension.operandAt(0); - const MachineOperand *SourceOperand = Extension.operandAt(1); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || - !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) - return std::nullopt; - const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); - const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); - if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type) - return std::nullopt; - const LLT DestinationType = *DestinationInfo->get().Type; - const LLT SourceType = *SourceInfo->get().Type; - if (!isScalarIntegerExtension(DestinationType, SourceType) || - DestinationType.scalarBitWidth() != 64) - return std::nullopt; - - const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); - MachineInstr *Binary = - SourceDefinition == nullptr ? nullptr : const_cast(SourceDefinition->parent()); - if (Binary == nullptr || Binary->operandCount() != 3) - return std::nullopt; - const GenericOpcode Generic = static_cast(Binary->opcode().value()); - if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && - Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && - Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && - Generic != GenericOpcode::G_SHL) - return std::nullopt; - - NarrowAnyExtBinaryPlan Plan; - Plan.Binary = Binary; - Plan.Generic = Generic; - Plan.SourceWidth = SourceType.scalarBitWidth(); - const auto AddArtifact = [&Plan](MachineInstr *Artifact) { - if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) - Plan.Artifacts.push_back(Artifact); - }; - const auto TruncatedWideSource = [&](MachineInstr *Truncation, - MachineInstr *OnlyUse) -> std::optional { - if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || - Truncation->operandCount() != 2) - return std::nullopt; - const MachineOperand *NarrowOperand = Truncation->operandAt(0); - const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); - const MachineOperand *WideOperand = Truncation->operandAt(1); - const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); - const std::optional WideType = - WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); - if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || - *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) - return std::nullopt; - return *Wide; - }; - for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { - const MachineOperand *Operand = Binary->operandAt(Index + 1U); - const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); - if (Reg == nullptr || !Reg->isVirtual()) - return std::nullopt; - const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); - if (!Info || !Info->get().Type) - return std::nullopt; - if (*Info->get().Type == DestinationType) { - Plan.WideInputs[Index] = *Reg; - continue; - } - if (*Info->get().Type != SourceType) - return std::nullopt; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); - MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (auto Wide = TruncatedWideSource(Def, Binary)) { - Plan.WideInputs[Index] = *Wide; - AddArtifact(Def); - continue; - } - if (Def != nullptr && - (Def->opcode() == opcode(GenericOpcode::COPY) || - Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && - Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { - const MachineOperand *NarrowOperand = Def->operandAt(1); - const Register *Narrow = - NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); - const MachineOperand *NarrowDefinition = - Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); - MachineInstr *Truncation = NarrowDefinition == nullptr - ? nullptr - : const_cast(NarrowDefinition->parent()); - if (auto Wide = TruncatedWideSource(Truncation, Def)) { - Plan.WideInputs[Index] = *Wide; - AddArtifact(Def); - AddArtifact(Truncation); - continue; - } - } - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && - singleRegisterUse(Function, *Reg) == Binary) { - Plan.WideInputs[Index] = *Reg; - Plan.Constants.push_back(Def); - continue; - } - return std::nullopt; - } - return Plan; -} - -[[nodiscard]] Expected widenConstantToRV64(MachineInstr &Instruction, - MachineFunction &Function) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const MachineOperand *ImmediateOperand = Instruction.operandAt(1); - const auto *Immediate = ImmediateOperand == nullptr - ? nullptr - : ImmediateOperand->getIf(); - if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow constant legalization has invalid operands"); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto SetType = Function.registerInfo().setType(*Destination, *Type); - if (!SetType) - return std::unexpected(SetType.error()); - auto Extended = extendIntegerImmediate(Immediate->Value, 64, true); - if (!Extended) - return std::unexpected(Extended.error()); - auto Erased = Instruction.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); - if (!Added) - return std::unexpected(Added.error()); - return {}; -} - -[[nodiscard]] Expected widenAtomicCompareExchangeSource(MachineInstr &Instruction, - MachineFunction &Function, - MachineIRBuilder &Builder, - Register Source, LLT WideType) { - const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); - if (!SourceInfo || !SourceInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V compare-exchange source has no scalar type"); - if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) - return Source; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && - singleRegisterUse(Function, Source) == &Instruction) { - auto Widened = widenConstantToRV64(*Def, Function); - if (!Widened) - return std::unexpected(Widened.error()); - return Source; - } - return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); -} - -[[nodiscard]] Expected lowerAtomicCompareExchange(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const std::optional OriginalType = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *CompareOperand = Instruction.operandAt(2); - const MachineOperand *NewValueOperand = Instruction.operandAt(3); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); - const Register *NewValue = - NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); - if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || - OriginalType->scalarBitWidth() >= 64) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow compare-exchange has invalid operands"); - const Register OriginalDestination = *Destination; - - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto WideCompare = - widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); - if (!WideCompare) - return std::unexpected(WideCompare.error()); - auto WideNewValue = - widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); - if (!WideNewValue) - return std::unexpected(WideNewValue.error()); - - Register WideDestination; - MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); - if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && - operandType(Function, *FoldedExtension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false)) { - const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); - const Register *FoldedDestination = - WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf(); - if (FoldedDestination == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V compare-exchange extension has no destination"); - WideDestination = *FoldedDestination; - } else { - auto Created = Function.registerInfo().createVirtualRegister(*WideType); - if (!Created) - return std::unexpected(Created.error()); - WideDestination = *Created; - FoldedExtension = nullptr; - } - - if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) - return std::unexpected(Replaced.error()); - if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) - return std::unexpected(Replaced.error()); - if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) - return std::unexpected(Replaced.error()); - - if (FoldedExtension != nullptr) - return eraseInstruction(*FoldedExtension); - - auto Point = Builder.setInsertPointAfter(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); - if (!Truncation) - return std::unexpected(Truncation.error()); - if (auto Added = Truncation->addDef(OriginalDestination); !Added) - return std::unexpected(Added.error()); - if (auto Added = Truncation->addUse(WideDestination); !Added) - return std::unexpected(Added.error()); - auto Built = Truncation->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected -lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *OldValueOperand = Instruction.operandAt(0); - const MachineOperand *SuccessOperand = Instruction.operandAt(1); - const MachineOperand *AddressOperand = Instruction.operandAt(2); - const MachineOperand *CompareOperand = Instruction.operandAt(3); - const MachineOperand *NewValueOperand = Instruction.operandAt(4); - const Register *OldValue = - OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf(); - const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf(); - const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); - const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); - const Register *NewValue = - NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); - const std::optional ValueType = resultType(Function, Instruction); - if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || - NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V compare-exchange-with-success has invalid operands"); - - auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType); - if (!AtomicResult) - return std::unexpected(AtomicResult.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); - if (!Atomic) - return std::unexpected(Atomic.error()); - if (auto Added = Atomic->addDef(*AtomicResult); !Added) - return std::unexpected(Added.error()); - if (auto Added = Atomic->addUse(*Address); !Added) - return std::unexpected(Added.error()); - if (auto Added = Atomic->addUse(*Compare); !Added) - return std::unexpected(Added.error()); - if (auto Added = Atomic->addUse(*NewValue); !Added) - return std::unexpected(Added.error()); - if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) - return std::unexpected(Added.error()); - auto BuiltAtomic = Atomic->build(); - if (!BuiltAtomic) - return std::unexpected(BuiltAtomic.error()); - - Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); - if (!CompareResult) - return std::unexpected(CompareResult.error()); - if (auto Added = CompareResult->addDef(*Success); !Added) - return std::unexpected(Added.error()); - if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added) - return std::unexpected(Added.error()); - if (auto Added = CompareResult->addUse(*AtomicResult); !Added) - return std::unexpected(Added.error()); - if (auto Added = CompareResult->addUse(*Compare); !Added) - return std::unexpected(Added.error()); - auto BuiltCompare = CompareResult->build(); - if (!BuiltCompare) - return std::unexpected(BuiltCompare.error()); - - while (!Instruction.memOperands().empty()) { - auto Erased = Instruction.eraseMemOperand(0); - if (!Erased) - return std::unexpected(Erased.error()); - } - return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult); -} - -[[nodiscard]] Expected widenIntegerCompareSource(MachineInstr &Instruction, - MachineFunction &Function, - MachineIRBuilder &Builder, - Register Source, LLT WideType) { - const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); - if (!SourceInfo || !SourceInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer compare source has no scalar type"); - const LLT SourceType = *SourceInfo->get().Type; - if (SourceType.scalarBitWidth() == WideType.scalarBitWidth()) - return Source; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast(Definition->parent()); - std::optional ExistingExtension; - bool AmbiguousExtension = false; - auto &RegisterInfo = Function.registerInfo(); - for (MachineOperand &Use : RegisterInfo.useOperands(Source)) { - MachineInstr *User = Use.parent(); - if (User == nullptr || User == &Instruction || User->parent() == nullptr || - (User->opcode() != opcode(GenericOpcode::G_ANYEXT) && - User->opcode() != opcode(GenericOpcode::G_SEXT))) { - continue; - } - const MachineOperand *ExtendedOperand = User->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - const std::optional ExtendedType = - ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); - if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || - (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && - User->opcode() != opcode(GenericOpcode::G_SEXT)))) { - continue; - } - if (ExistingExtension) { - AmbiguousExtension = true; - break; - } - ExistingExtension = *Extended; - } - if (ExistingExtension && !AmbiguousExtension) - return *ExistingExtension; - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && - singleRegisterUse(Function, Source) == &Instruction) { - auto Widened = widenConstantToRV64(*Def, Function); - if (!Widened) - return std::unexpected(Widened.error()); - return Source; - } - - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && - Def->operandCount() == 2) { - const MachineOperand *WideOperand = Def->operandAt(1); - const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf(); - const std::optional ExistingWideType = - WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); - if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { - auto Result = Function.registerInfo().createVirtualRegister(WideType); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(*WideSource); !Added) - return std::unexpected(Added.error()); - if (auto Added = - Pending->addImmediate(static_cast(SourceType.scalarBitWidth())); - !Added) { - return std::unexpected(Added.error()); - } - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; - } - } - return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source); -} - -[[nodiscard]] Expected lowerIntegerCompare(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (Destination == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer compare has invalid register operands"); - const Register OriginalDestination = *Destination; - const Register OriginalLeft = *Left; - const Register OriginalRight = *Right; - - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto WideLeft = - widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType); - if (!WideLeft) - return std::unexpected(WideLeft.error()); - auto WideRight = - widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType); - if (!WideRight) - return std::unexpected(WideRight.error()); - if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced) - return std::unexpected(Replaced.error()); - if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced) - return std::unexpected(Replaced.error()); - - const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination); - if (!DestinationInfo || !DestinationInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer compare destination has no scalar type"); - if (DestinationInfo->get().Type->scalarBitWidth() == 64) - return {}; - - MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); - if (FoldedExtension != nullptr && - (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) || - FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || - FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && - operandType(Function, *FoldedExtension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false)) { - const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - if (Extended == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer compare extension has no destination"); - const Register WideDestination = *Extended; - if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) - return std::unexpected(Replaced.error()); - return eraseInstruction(*FoldedExtension); - } - - auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType); - if (!WideDestination) - return std::unexpected(WideDestination.error()); - if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced) - return std::unexpected(Replaced.error()); - if (Function.registerInfo().useEmpty(OriginalDestination)) - return {}; - auto Point = Builder.setInsertPointAfter(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); - if (!Truncation) - return std::unexpected(Truncation.error()); - if (auto Added = Truncation->addDef(OriginalDestination); !Added) - return std::unexpected(Added.error()); - if (auto Added = Truncation->addUse(*WideDestination); !Added) - return std::unexpected(Added.error()); - auto Built = Truncation->build(); - if (!Built) - return std::unexpected(Built.error()); - return {}; -} - -[[nodiscard]] Expected foldNarrowAnyExtBinary(MachineInstr &Extension, - MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); - if (!Plan) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V narrow any-extension does not have a legal binary chain"); - for (MachineInstr *Constant : Plan->Constants) { - auto Widened = widenConstantToRV64(*Constant, Function); - if (!Widened) - return std::unexpected(Widened.error()); - } - for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { - auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - for (MachineInstr *Artifact : Plan->Artifacts) { - if (Artifact->parent() == nullptr) - continue; - auto Removed = Artifact->parent()->removeInstruction(*Artifact); - if (!Removed) - return std::unexpected(Removed.error()); - } - - if (Plan->Generic == GenericOpcode::G_SHL) { - Register Amount = Plan->WideInputs[1]; - if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { - const std::uint64_t Mask = - Plan->SourceWidth == 64 - ? std::numeric_limits::max() - : (std::uint64_t{1} << Plan->SourceWidth) - 1U; - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto MaskValue = - buildIntegerConstantBefore(Builder, Extension, 64, Mask); - if (!MaskValue) - return std::unexpected(MaskValue.error()); - auto MaskedAmount = buildBinaryBefore(Builder, Extension, - GenericOpcode::G_AND, *WideType, - Amount, *MaskValue); - if (!MaskedAmount) - return std::unexpected(MaskedAmount.error()); - Amount = *MaskedAmount; - } - auto SetOpcode = Extension.setOpcode( - Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW) - : opcode(GenericOpcode::G_SHL)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Extension.operandCount() > 1) { - auto Erased = Extension.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - for (Register Input : {Plan->WideInputs[0], Amount}) { - auto Operand = MachineOperand::registerOperand( - Input, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Operand) - return std::unexpected(Operand.error()); - auto Added = Extension.addOperand(std::move(*Operand)); - if (!Added) - return std::unexpected(Added.error()); - } - MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); - if (BinaryBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow shift is detached during legalization"); - return BinaryBlock->removeInstruction(*Plan->Binary); - } - - if (Plan->SourceWidth == 32 && - (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); - if (!Retargeted) - return std::unexpected(Retargeted.error()); - auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); - if (!ReplacedSource) - return std::unexpected(ReplacedSource.error()); - auto Width = Extension.addOperand(MachineOperand::immediate(32)); - if (!Width) - return std::unexpected(Width.error()); - return {}; - } - - auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Extension.operandCount() > 1) { - auto Erased = Extension.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - for (Register Input : Plan->WideInputs) { - auto Operand = MachineOperand::registerOperand( - Input, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Operand) - return std::unexpected(Operand.error()); - auto Added = Extension.addOperand(std::move(*Operand)); - if (!Added) - return std::unexpected(Added.error()); - } - MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); - if (BinaryBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow binary instruction is detached during legalization"); - auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); - if (!Removed) - return std::unexpected(Removed.error()); - (void)Target; - return {}; -} - -[[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept { - const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U)); - const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1; - return Value >= Minimum && Value <= Maximum; -} - -[[nodiscard]] Expected lowerConstantToPool(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const MachineOperand *ImmediateOperand = Instruction.operandAt(1); - const auto *Immediate = ImmediateOperand == nullptr - ? nullptr - : ImmediateOperand->getIf(); - if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant-pool legalization has invalid operands"); - const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); - if (!DestinationInfo || !DestinationInfo->get().Type || - DestinationInfo->get().Type->scalarBitWidth() != 64) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant-pool legalization requires a 64-bit destination"); - const LLT DestinationType = *DestinationInfo->get().Type; - auto Value = GenericConstantData::integer(Immediate->Value); - if (!Value) - return std::unexpected(Value.error()); - auto Align = Alignment::create(8); - if (!Align) - return std::unexpected(Align.error()); - MachineConstantPoolEntryDesc Description; - Description.Value = std::move(*Value); - Description.Size = 8; - Description.Align = *Align; - auto PoolIndex = Function.constantPool().addEntry(std::move(Description)); - if (!PoolIndex) - return std::unexpected(PoolIndex.error()); - - auto PointerType = LLT::makePointer(0, 64); - if (!PointerType) - return std::unexpected(PointerType.error()); - auto Address = Function.registerInfo().createVirtualRegister(*PointerType); - if (!Address) - return std::unexpected(Address.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL)); - if (!AddressInstruction) - return std::unexpected(AddressInstruction.error()); - auto AddressDef = AddressInstruction->addDef(*Address); - if (!AddressDef) - return std::unexpected(AddressDef.error()); - auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex); - if (!AddressPool) - return std::unexpected(AddressPool.error()); - auto BuiltAddress = AddressInstruction->build(); - if (!BuiltAddress) - return std::unexpected(BuiltAddress.error()); - - MachineMemOperandDesc MemoryDescription; - MemoryDescription.Accesses = {MemoryAccessFlag::Load}; - MemoryDescription.Size = *MemorySize::fixed(8); - MemoryDescription.MemoryType = DestinationType; - MemoryDescription.Align = *Align; - MemoryDescription.Base = MemoryBase::constantPoolPseudo(); - auto Memory = Function.createMemOperand(std::move(MemoryDescription)); - if (!Memory) - return std::unexpected(Memory.error()); - auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Instruction.operandCount() > 1) { - auto Erased = Instruction.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto AddressUse = MachineOperand::registerOperand( - *Address, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!AddressUse) - return std::unexpected(AddressUse.error()); - auto AddedAddress = Instruction.addOperand(std::move(*AddressUse)); - if (!AddedAddress) - return std::unexpected(AddedAddress.error()); - auto AddedMemory = Instruction.addMemOperand(**Memory); - if (!AddedMemory) - return std::unexpected(AddedMemory.error()); - return {}; -} - -[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { - if (Instruction.memOperands().size() != 1) - return false; - const MachineMemOperand &Memory = *Instruction.memOperands().front(); - const MachineMemOperandDesc &Description = Memory.description(); - return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && - !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; -} - -[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, - AtomicOperation Operation, - std::uint64_t Bytes) noexcept { - if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) - return false; - const MachineMemOperand &Memory = *Instruction.memOperands().front(); - const MachineMemOperandDesc &Description = Memory.description(); - return Description.AtomicKind == Operation && Description.AddressSpace == 0 && - Description.Size.isKnown() && !Description.Size.isScalable() && - Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && - Description.SyncScope.value() == 0; -} - -[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, - AtomicOperation Operation, - std::uint64_t ContainerBytes) noexcept { - if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) - return false; - const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); - if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || - !Description.Size.isKnown() || Description.Size.isScalable() || - Description.SyncScope.value() != 0) - return false; - const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); - return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && - MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; -} - -[[nodiscard]] std::optional -memoryAccessBytes(const MachineInstr &Instruction) noexcept { - if (Instruction.memOperands().size() != 1) - return std::nullopt; - const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); - if (!Description.Size.isKnown() || Description.Size.isScalable()) - return std::nullopt; - return Description.Size.minimumBytes(); -} - -[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { - const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) - return false; - if (isRVFloatType(Type)) - return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; - const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; - return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; -} - -[[nodiscard]] Expected requiredRegisterClass(const MachineFunction &Function, - Register Reg, - const TargetDescription &Target) { - if (!Reg.isVirtual()) - return makeUnexpected(ErrorCode::InvalidArgument, - "RISC-V register-class selection requires a virtual register"); - auto Info = Function.registerInfo().virtualRegisterInfo(Reg); - if (!Info) - return std::unexpected(Info.error()); - if (!Info->get().Type) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V selection requires a type for every virtual register"); - } - const VirtualRegisterInfo &RegisterInfo = Info->get(); - const LLT Type = RegisterInfo.Type.value_or(LLT{}); - RegisterClassID RequiredClass; - if (isScalarGPRType(Type)) { - RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::GPR)); - } else if (Type.isFloat() && Type.scalarBitWidth() == 32) { - RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR32)); - } else if (Type.isFloat() && Type.scalarBitWidth() == 64) { - RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR64)); - } else { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V RV64GC has no register class for type '" + Type.toString() + "'"); - } - - if (Target.registers().registerClass(RequiredClass) == nullptr) { - return makeUnexpected(ErrorCode::InternalError, - "RISC-V target description is missing a required register class"); - } - if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V virtual register bank cannot cover its required class"); - } - if (RegisterInfo.RegClass && - !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V virtual register has a conflicting register class"); - } - return RequiredClass; -} - -[[nodiscard]] Expected appendImmediate(MachineInstr &Instruction, std::int64_t Value) { - return Instruction.addOperand(MachineOperand::immediate(Value)); -} - -[[nodiscard]] Expected insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index, - PhysicalRegister Reg, - RegisterOperandFlag Flag) { - auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag}); - if (!Operand) - return std::unexpected(Operand.error()); - return Instruction.insertOperand(Index, std::move(*Operand)); -} - -[[nodiscard]] Expected insertVirtualUse(MachineInstr &Instruction, std::size_t Index, - Register Reg) { - auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use}); - if (!Operand) - return std::unexpected(Operand.error()); - return Instruction.insertOperand(Index, std::move(*Operand)); -} - -[[nodiscard]] Expected createTemporary(MachineFunction &Function, LLT Type, - const TargetDescription &Target) { - auto Temporary = Function.registerInfo().createVirtualRegister(Type); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto RequiredClass = requiredRegisterClass(Function, *Temporary, Target); - if (!RequiredClass) - return std::unexpected(RequiredClass.error()); - const RegisterBank BankKind = Type.isFloat() ? RegisterBank::FPR : RegisterBank::GPR; - auto Bank = Function.registerInfo().setRegisterBank( - *Temporary, RegisterBankID::fromValue(static_cast(BankKind))); - if (!Bank) - return std::unexpected(Bank.error()); - auto Class = Function.registerInfo().setRegisterClass(*Temporary, *RequiredClass); - if (!Class) - return std::unexpected(Class.error()); - return *Temporary; -} - -[[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { - if (const std::int64_t *Value = Operand.getIf()) - return *Value; - const auto *Integer = Operand.getIf(); - if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V constant selection requires an immediate no wider than 64 bits"); - std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); - const std::uint32_t Width = Integer->Value.bitWidth(); - if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) - Raw |= ~std::uint64_t{0} << Width; - return std::bit_cast(Raw); -} - -[[nodiscard]] Expected replaceRegisterUses(MachineFunction &Function, Register OldReg, - Register NewReg) { - std::vector Uses; - for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) - Uses.push_back(&Operand); - for (MachineOperand *Operand : Uses) { - auto Replaced = Operand->setRegister(NewReg); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - return {}; -} - -[[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction) { - MachineBasicBlock *Block = Instruction.parent(); - if (Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V legalizer cannot erase a detached instruction"); - return Block->removeInstruction(Instruction); -} - -[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, - const MachineFunction &Function) { - for (const MachineOperand &Operand : Instruction.operands()) { - const Register *Reg = Operand.getIf(); - if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && - !Function.registerInfo().useEmpty(*Reg)) - return false; - } - return true; -} - -void eraseDeadArtifacts(std::span Candidates, - const MachineFunction &Function) { - bool Changed = false; - do { - Changed = false; - for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { - MachineInstr *Instruction = *It; - if (Instruction == nullptr || Instruction->parent() == nullptr || - !allDefsUnused(*Instruction, Function)) - continue; - const Opcode Value = Instruction->opcode(); - if (Value != opcode(GenericOpcode::G_TRUNC) && - Value != opcode(GenericOpcode::G_ANYEXT) && - Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && - Value != opcode(GenericOpcode::G_MERGE_VALUES) && - Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && - Value != opcode(GenericOpcode::COPY)) - continue; - if (eraseInstruction(*Instruction)) - Changed = true; - } - } while (Changed); -} - -[[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, - MachineInstr &Anchor, - std::uint32_t Width, - std::uint64_t Value) { - auto Type = LLT::makeAnyScalar(Width); - if (!Type) - return std::unexpected(Type.error()); - auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Immediate = IntegerImmediate::fromUnsigned(Width, Value); - if (!Immediate) - return std::unexpected(Immediate.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, - Register Source) { - auto Result = Builder.function().registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(Generic)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Source); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Left, - Register Right) { - auto Result = Builder.function().registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(Generic)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Left); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Right); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, - Register Destination, Register Source) { - auto SetOpcode = Instruction.setOpcode(opcode(Generic)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Instruction.operandCount() != 0) { - auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Def = MachineOperand::registerOperand( - Destination, RegisterOperandFlags{RegisterOperandFlag::Define}); - if (!Def) - return std::unexpected(Def.error()); - if (auto Added = Instruction.addOperand(std::move(*Def)); !Added) - return std::unexpected(Added.error()); - auto Use = - MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Use) - return std::unexpected(Use.error()); - return Instruction.addOperand(std::move(*Use)); -} - -[[nodiscard]] Expected widenNarrowMultiply(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const std::optional Type = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || - Type->scalarBitWidth() >= 64) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow multiply has invalid operands"); - } - - MachineInstr *Extension = singleRegisterUse(Function, *Destination); - if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { - const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); - if (Plan && Plan->Binary == &Instruction) - return foldNarrowAnyExtBinary(*Extension, Function, Target); - } - - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); - if (!WideLeft) - return std::unexpected(WideLeft.error()); - auto WideRight = - buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); - if (!WideRight) - return std::unexpected(WideRight.error()); - auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, - *WideRight); - if (!Product) - return std::unexpected(Product.error()); - return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); -} - -[[nodiscard]] Expected lowerMultiplyLibcall(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - CallingConvention Convention) { - const std::optional Type = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || - Type->scalarBitWidth() != 64) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V multiply libcall requires three s64 registers"); - } - auto ABIAlignment = Alignment::create(8); - auto Callee = MachineCallTarget::externalSymbol("__muldi3"); - if (!ABIAlignment || !Callee) - return makeUnexpected(ErrorCode::InternalError, - "RISC-V multiply libcall ABI facts are unavailable"); - const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { - return ABIValue{ - .Value = Reg, - .Type = Type, - .OriginalValue = Number, - .FragmentIndex = 0, - .FragmentCount = 1, - .FragmentOffsetBytes = 0, - .OriginalAlignment = Alignment, - .Extension = ABIExtensionKind::None, - .Flags = {}, - .ByValSizeBytes = 0, - .ByValAlignment = Alignment, - }; - }; - - CallLoweringRequest Request; - Request.Convention = callingConventionID(Convention); - Request.Callee = std::move(*Callee); - Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; - Request.Results = {Value(*Destination, 0)}; - Request.FixedArgumentCount = Request.Arguments.size(); - - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Lowered = callLowering().lowerCall(Builder, Request); - if (!Lowered) - return std::unexpected(Lowered.error()); - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, - Register Destination, Register Left, Register Right) { - auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Use) - return std::unexpected(Use.error()); - return Instruction.addOperand(std::move(*Use)); -} - -[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, - std::uint32_t ActiveBits, unsigned Depth = 0) { - if (Depth > 8 || !Reg.isVirtual()) - return false; - const auto Info = Function.registerInfo().virtualRegisterInfo(Reg); - if (!Info || !Info->get().Type) - return false; - const std::uint32_t Width = Info->get().Type->scalarBitWidth(); - if (Width <= ActiveBits) - return true; - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); - const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); - if (Def == nullptr) - return false; - if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { - const MachineOperand *BitsOperand = Def->operandAt(2); - auto Bits = BitsOperand == nullptr - ? Expected(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) - : signedImmediate(*BitsOperand); - return Bits && *Bits >= 0 && static_cast(*Bits) <= ActiveBits; - } - if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (Source == nullptr) - return false; - const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); - return SourceInfo && SourceInfo->get().Type && - SourceInfo->get().Type->scalarBitWidth() <= ActiveBits; - } - if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); - } - return false; -} - -[[nodiscard]] Expected -zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, - Register Source, std::vector &DeadCandidates) { - const auto Info = Function.registerInfo().virtualRegisterInfo(Source); - if (!Info || !Info->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge source has no scalar type"); - const LLT SourceType = *Info->get().Type; - if (SourceType.scalarBitWidth() == 64) - return Source; - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); - MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *WideOperand = Def->operandAt(1); - const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); - if (Wide != nullptr) { - const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide); - if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 && - isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) { - DeadCandidates.push_back(Def); - return *Wide; - } - } - } - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); -} - -[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { - MachineInstr *Result = nullptr; - auto &RegisterInfo = const_cast(Function.registerInfo()); - for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { - MachineInstr *Instruction = Operand.parent(); - if (Instruction == nullptr || Instruction->parent() == nullptr) - continue; - if (Result != nullptr && Result != Instruction) - return nullptr; - Result = Instruction; - } - return Result; -} - -[[nodiscard]] Expected lowerWideMergeValues(MachineInstr &Instruction, - MachineFunction &Function) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const MachineOperand *LowOperand = Instruction.operandAt(1); - const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); - if (Destination == nullptr || Low == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide merge has invalid register operands"); - - std::vector Truncations; - auto &RegisterInfo = Function.registerInfo(); - std::vector Uses; - for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) - Uses.push_back(&Use); - for (MachineOperand *Use : Uses) { - MachineInstr *Trunc = Use->parent(); - if (Trunc == nullptr || Trunc->parent() == nullptr || - Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge is only representable through low-part truncation"); - const MachineOperand *TruncResultOperand = Trunc->operandAt(0); - const Register *TruncResult = - TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf(); - const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low); - const auto ResultInfo = TruncResult == nullptr - ? Expected>( - std::unexpected(Error(ErrorCode::InvalidArgument, - "missing truncation result"))) - : RegisterInfo.virtualRegisterInfo(*TruncResult); - if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type || - ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge truncation spans more than its low source"); - if (*ResultInfo->get().Type == *LowInfo->get().Type) { - auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); - if (!Replaced) - return std::unexpected(Replaced.error()); - Truncations.push_back(Trunc); - } else { - auto Replaced = Use->setRegister(*Low); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - } - for (MachineInstr *Trunc : Truncations) { - auto Erased = eraseInstruction(*Trunc); - if (!Erased) - return std::unexpected(Erased.error()); - } - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected lowerMergeValues(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const std::optional DestinationType = resultType(Function, Instruction); - if (!DestinationType || DestinationType->scalarBitWidth() == 0) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge destination has no scalar type"); - if (DestinationType->scalarBitWidth() > 64) - return lowerWideMergeValues(Instruction, Function); - - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Destination == nullptr || Instruction.operandCount() < 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge has invalid register operands"); - - Register Result = *Destination; - MachineInstr *FoldedExtension = nullptr; - if (DestinationType->scalarBitWidth() < 64) { - MachineInstr *Use = singleUseInstruction(Function, *Destination); - if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && - operandType(Function, *Use, 0).transform([](LLT Type) { - return Type.scalarBitWidth() == 64; - }).value_or(false)) { - const MachineOperand *ExtendedOperand = Use->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - if (Extended != nullptr) { - Result = *Extended; - FoldedExtension = Use; - } - } - } - - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - std::vector DeadCandidates; - std::vector Terms; - Terms.reserve(Instruction.operandCount() - 1U); - for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { - const MachineOperand *Operand = Instruction.operandAt(Index); - const Register *Source = Operand == nullptr ? nullptr : Operand->getIf(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge source is not a register"); - auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); - if (!WideSource) - return std::unexpected(WideSource.error()); - if (Index == 1) { - Terms.push_back(*WideSource); - continue; - } - const std::optional SourceType = registerType(Function, *Operand); - if (!SourceType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge source has no scalar type"); - const std::uint64_t Shift = - static_cast(SourceType->scalarBitWidth()) * (Index - 1U); - auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); - if (!Amount) - return std::unexpected(Amount.error()); - auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, - *WideSource, *Amount); - if (!Shifted) - return std::unexpected(Shifted.error()); - Terms.push_back(*Shifted); - } - - Register Accumulator = Terms.front(); - for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { - auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, - Accumulator, Terms[Index]); - if (!Combined) - return std::unexpected(Combined.error()); - Accumulator = *Combined; - } - if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { - auto Rewritten = rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, - Terms.back()); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - } else { - auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, - Accumulator, Terms.back()); - if (!Combined) - return std::unexpected(Combined.error()); - auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - } - if (FoldedExtension != nullptr) { - auto Erased = eraseInstruction(*FoldedExtension); - if (!Erased) - return std::unexpected(Erased.error()); - } - eraseDeadArtifacts(DeadCandidates, Function); - return {}; -} - -[[nodiscard]] Expected -foldWideExtendedUnmerge(MachineInstr &Instruction, MachineFunction &Function) { - const std::size_t OutputCount = Instruction.operandCount() - 1U; - const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - const MachineOperand *Definition = - Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); - MachineInstr *Extension = - Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (Extension == nullptr || - (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && - Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && - Extension->opcode() != opcode(GenericOpcode::G_SEXT))) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge requires an extension-defined source"); - const MachineOperand *NarrowOperand = Extension->operandAt(1); - const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); - if (Narrow == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge extension has no register source"); - const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow); - if (!NarrowInfo || !NarrowInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge extension source has no type"); - const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth(); - - std::vector DeadUsers; - for (std::size_t Index = 0; Index < OutputCount; ++Index) { - const MachineOperand *OutputOperand = Instruction.operandAt(Index); - const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); - if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) - continue; - const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output); - if (!OutputInfo || !OutputInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge output has no type"); - const std::uint64_t Offset = - static_cast(Index) * OutputInfo->get().Type->scalarBitWidth(); - if (Offset != 0) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge cannot materialize a used high wide part"); - if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) { - auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - continue; - } - if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge output unexpectedly narrows its source"); - std::vector Uses; - for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) - Uses.push_back(&Use); - for (MachineOperand *Use : Uses) { - MachineInstr *Trunc = Use->parent(); - if (Trunc == nullptr || Trunc->parent() == nullptr || - Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge result is only representable through truncation"); - const MachineOperand *ResultOperand = Trunc->operandAt(0); - const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); - const auto ResultInfo = Result == nullptr - ? Expected>( - std::unexpected(Error(ErrorCode::InvalidArgument, - "missing truncation result"))) - : Function.registerInfo().virtualRegisterInfo(*Result); - if (!ResultInfo || !ResultInfo->get().Type || - ResultInfo->get().Type->scalarBitWidth() > NarrowWidth) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge truncation exceeds its source width"); - if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) { - auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - DeadUsers.push_back(Trunc); - } else { - auto Replaced = Use->setRegister(*Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - } - } - for (MachineInstr *User : DeadUsers) { - auto Erased = eraseInstruction(*User); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Erased = eraseInstruction(Instruction); - if (!Erased) - return std::unexpected(Erased.error()); - if (Function.registerInfo().useEmpty(*Source)) - return eraseInstruction(*Extension); - return {}; -} - -[[nodiscard]] Expected lowerUnmergeValues(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - if (Instruction.operandCount() < 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge has invalid register operands"); - const std::size_t OutputCount = Instruction.operandCount() - 1U; - const std::optional OutputType = operandType(Function, Instruction, 0); - const std::optional SourceType = operandType(Function, Instruction, OutputCount); - if (!OutputType || !SourceType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge operands have no scalar types"); - if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) - return foldWideExtendedUnmerge(Instruction, Function); - - const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge source is not a register"); - auto WideType = LLT::makeAnyScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - std::vector FoldedExtensions; - for (std::size_t Index = 0; Index < OutputCount; ++Index) { - const MachineOperand *OutputOperand = Instruction.operandAt(Index); - const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); - if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) - continue; - Register Extracted = *Source; - const std::uint64_t Offset = - static_cast(Index) * OutputType->scalarBitWidth(); - if (Offset != 0) { - auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); - if (!Amount) - return std::unexpected(Amount.error()); - auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, - *Source, *Amount); - if (!Shifted) - return std::unexpected(Shifted.error()); - Extracted = *Shifted; - } - - MachineInstr *Extension = singleUseInstruction(Function, *Output); - const bool FoldZExt = Extension != nullptr && - Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && - operandType(Function, *Extension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false); - if (FoldZExt) { - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - if (Extended == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge extension has no destination"); - const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 - ? std::numeric_limits::max() - : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; - auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); - if (!MaskReg) - return std::unexpected(MaskReg.error()); - auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, - Extracted, *MaskReg); - if (!Masked) - return std::unexpected(Masked.error()); - auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); - if (!Replaced) - return std::unexpected(Replaced.error()); - FoldedExtensions.push_back(Extension); - continue; - } - auto Truncated = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, - Extracted); - if (!Truncated) - return std::unexpected(Truncated.error()); - auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - for (MachineInstr *Extension : FoldedExtensions) { - auto Erased = eraseInstruction(*Extension); - if (!Erased) - return std::unexpected(Erased.error()); - } - return eraseInstruction(Instruction); -} - -struct WideValueParts final { - std::uint32_t BitWidth = 0; - std::vector Parts; - std::vector Artifacts; -}; - -[[nodiscard]] Expected collectWideValueParts(Register Value, - MachineFunction &Function, - unsigned Depth = 0) { - if (Depth > 16) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide-value artifact chain is too deep"); - const auto Info = Function.registerInfo().virtualRegisterInfo(Value); - if (!Info || !Info->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide value has no scalar type"); - const LLT Type = *Info->get().Type; - if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide-value decomposition requires an integer scalar"); - if (Type.scalarBitWidth() <= 64) - return WideValueParts{ - .BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (Def == nullptr) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide value has no decomposable definition"); - if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { - WideValueParts Result; - Result.BitWidth = Type.scalarBitWidth(); - for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { - const MachineOperand *Operand = Def->operandAt(Index); - const Register *Part = Operand == nullptr ? nullptr : Operand->getIf(); - const std::optional PartType = - Operand == nullptr ? std::nullopt : registerType(Function, *Operand); - if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge must already use 64-bit limbs"); - Result.Parts.push_back(*Part); - } - Result.Artifacts.push_back(Def); - return Result; - } - if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide truncation has no source register"); - auto Result = collectWideValueParts(*Source, Function, Depth + 1U); - if (!Result) - return std::unexpected(Result.error()); - Result->BitWidth = Type.scalarBitWidth(); - const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U; - if (Result->Parts.size() < RequiredParts) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide truncation source has too few limbs"); - Result->Parts.resize(RequiredParts); - Result->Artifacts.push_back(Def); - return Result; - } - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide value is not defined by merge/trunc artifacts"); -} - -struct WideResultPlan final { - std::vector Outputs; - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = nullptr; -}; - -[[nodiscard]] Expected wideResultPlan(Register Result, - MachineFunction &Function, - std::size_t RequiredParts) { - MachineInstr *Use = singleUseInstruction(Function, Result); - WideResultPlan Plan; - if (Use != nullptr && - (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || - Use->opcode() == opcode(GenericOpcode::G_SEXT) || - Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { - Plan.Extension = Use; - const MachineOperand *ExtendedOperand = Use->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); - Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); - } - if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide result must feed an unmerge artifact"); - Plan.Unmerge = Use; - const std::size_t OutputCount = Use->operandCount() - 1U; - if (OutputCount < RequiredParts) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide result unmerge has too few outputs"); - Plan.Outputs.reserve(RequiredParts); - for (std::size_t Index = 0; Index < RequiredParts; ++Index) { - const MachineOperand *Operand = Use->operandAt(Index); - const Register *Output = Operand == nullptr ? nullptr : Operand->getIf(); - const std::optional Type = - Operand == nullptr ? std::nullopt : registerType(Function, *Operand); - if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide result must unmerge to 64-bit limbs"); - Plan.Outputs.push_back(*Output); - } - return Plan; -} - -[[nodiscard]] Expected -buildIntegerCompareBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - IntegerPredicate Predicate, Register Left, Register Right) { - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Left); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Right); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected -buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, LLT Type, - Register Condition, Register TrueValue, Register FalseValue) { - auto Result = Builder.function().registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Condition); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(TrueValue); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(FalseValue); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -struct TwoLimbShiftResult final { - Register Low; - Register High; -}; - -[[nodiscard]] Expected -emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, - Register Low, Register High, Register Amount, - Register AmountExcess, Register AmountLack, - Register Zero, Register IsShort, Register IsZero) { - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto LowShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); - if (!LowShort) - return std::unexpected(LowShort.error()); - auto CrossLow = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); - if (!CrossLow) - return std::unexpected(CrossLow.error()); - auto HighShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, - *CrossLow, *HighShort); - if (!ShortHigh) - return std::unexpected(ShortHigh.error()); - auto LongHigh = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); - if (!LongHigh) - return std::unexpected(LongHigh.error()); - auto ResultLow = - buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); - if (!ResultLow) - return std::unexpected(ResultLow.error()); - auto ShiftedHigh = - buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); - if (!ShiftedHigh) - return std::unexpected(ShiftedHigh.error()); - auto ResultHigh = - buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); - if (!ResultHigh) - return std::unexpected(ResultHigh.error()); - return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; -} - -[[nodiscard]] Expected -emitTwoLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, - Register Low, Register High, Register Amount) { - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); - if (!Boundary) - return std::unexpected(Boundary.error()); - auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - Amount, *Boundary); - if (!AmountExcess) - return std::unexpected(AmountExcess.error()); - auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - *Boundary, Amount); - if (!AmountLack) - return std::unexpected(AmountLack.error()); - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsShort = buildIntegerCompareBefore(Builder, Anchor, - IntegerPredicate::UnsignedLessThan, - Amount, *Boundary); - if (!IsShort) - return std::unexpected(IsShort.error()); - auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, - Amount, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, - *AmountExcess, *AmountLack, *Zero, *IsShort, - *IsZero); -} - -[[nodiscard]] Expected -emitLowLimbOfTwoLimbLogicalRightShift( - MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, - Register Amount, Register Boundary, Register Zero) { - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - Amount, Boundary); - if (!AmountExcess) - return std::unexpected(AmountExcess.error()); - auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - Boundary, Amount); - if (!AmountLack) - return std::unexpected(AmountLack.error()); - auto IsShort = buildIntegerCompareBefore(Builder, Anchor, - IntegerPredicate::UnsignedLessThan, - Amount, Boundary); - if (!IsShort) - return std::unexpected(IsShort.error()); - auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, - Amount, Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto LowShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); - if (!LowShort) - return std::unexpected(LowShort.error()); - auto HighShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto ShortValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, - *LowShort, *HighShort); - if (!ShortValue) - return std::unexpected(ShortValue.error()); - auto LongValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, - High, *AmountExcess); - if (!LongValue) - return std::unexpected(LongValue.error()); - auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, - *LongValue); - if (!Shifted) - return std::unexpected(Shifted.error()); - return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); -} - -[[nodiscard]] Expected> -emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, - std::span Parts, Register Amount) { - if (Parts.size() != 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V three-limb shift requires exactly three limbs"); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - - auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); - if (!OuterBoundary) - return std::unexpected(OuterBoundary.error()); - auto OuterExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - Amount, *OuterBoundary); - if (!OuterExcess) - return std::unexpected(OuterExcess.error()); - auto OuterLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - *OuterBoundary, Amount); - if (!OuterLack) - return std::unexpected(OuterLack.error()); - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto OuterIsShort = buildIntegerCompareBefore( - Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *OuterBoundary); - if (!OuterIsShort) - return std::unexpected(OuterIsShort.error()); - auto OuterIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, - Amount, *Zero); - if (!OuterIsZero) - return std::unexpected(OuterIsZero.error()); - - auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); - if (!InnerBoundary) - return std::unexpected(InnerBoundary.error()); - auto InnerExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - Amount, *InnerBoundary); - if (!InnerExcess) - return std::unexpected(InnerExcess.error()); - auto InnerLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, - *InnerBoundary, Amount); - if (!InnerLack) - return std::unexpected(InnerLack.error()); - auto InnerIsShort = buildIntegerCompareBefore( - Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); - if (!InnerIsShort) - return std::unexpected(InnerIsShort.error()); - auto InnerIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, - Amount, *Zero); - if (!InnerIsZero) - return std::unexpected(InnerIsZero.error()); - auto Lower = emitTwoLimbLeftShiftBody( - Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, *InnerLack, - *Zero, *InnerIsShort, *InnerIsZero); - if (!Lower) - return std::unexpected(Lower.error()); - - auto Cross = emitLowLimbOfTwoLimbLogicalRightShift( - Builder, Anchor, Parts[0], Parts[1], *OuterLack, *InnerBoundary, *Zero); - if (!Cross) - return std::unexpected(Cross.error()); - auto HighIsShort = buildIntegerCompareBefore( - Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); - if (!HighIsShort) - return std::unexpected(HighIsShort.error()); - auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, - Parts[2], Amount); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, - *HighShort, *Zero); - if (!BoundedHigh) - return std::unexpected(BoundedHigh.error()); - auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, - *Cross, *BoundedHigh); - if (!ShortHigh) - return std::unexpected(ShortHigh.error()); - - auto LongIsShort = buildIntegerCompareBefore( - Builder, Anchor, IntegerPredicate::UnsignedLessThan, *OuterExcess, - *InnerBoundary); - if (!LongIsShort) - return std::unexpected(LongIsShort.error()); - auto LongShift = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, - Parts[0], *OuterExcess); - if (!LongShift) - return std::unexpected(LongShift.error()); - auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, - *LongShift, *Zero); - if (!LongHigh) - return std::unexpected(LongHigh.error()); - - auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, - Lower->Low, *Zero); - if (!ResultLow) - return std::unexpected(ResultLow.error()); - auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, - Lower->High, *Zero); - if (!ResultMiddle) - return std::unexpected(ResultMiddle.error()); - auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, - *ShortHigh, *LongHigh); - if (!ShiftedHigh) - return std::unexpected(ShiftedHigh.error()); - auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, - Parts[2], *ShiftedHigh); - if (!ResultHigh) - return std::unexpected(ResultHigh.error()); - return std::vector{*ResultLow, *ResultMiddle, *ResultHigh}; -} - -[[nodiscard]] Expected lowerWideShiftLeft(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *ValueOperand = Instruction.operandAt(1); - const MachineOperand *AmountOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); - const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf(); - if (Destination == nullptr || Value == nullptr || Amount == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide shift has invalid register operands"); - auto ValueParts = collectWideValueParts(*Value, Function); - if (!ValueParts) - return std::unexpected(ValueParts.error()); - auto AmountParts = collectWideValueParts(*Amount, Function); - if (!AmountParts) - return std::unexpected(AmountParts.error()); - if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide shift has an invalid limb decomposition"); - auto ResultPlan = - wideResultPlan(*Destination, Function, ValueParts->Parts.size()); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - - MachineIRBuilder Builder(Function, Target); - Expected> Results = - ValueParts->Parts.size() == 2 - ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], - ValueParts->Parts[1], AmountParts->Parts[0]) - .transform([](TwoLimbShiftResult Result) { - return std::vector{Result.Low, Result.High}; - }) - : ValueParts->Parts.size() == 3 - ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, - AmountParts->Parts[0]) - : Expected>(std::unexpected( - Error(ErrorCode::Unsupported, - "RISC-V wide shift currently requires two or three 64-bit limbs"))); - if (!Results) - return std::unexpected(Results.error()); - for (std::size_t Index = 0; Index < Results->size(); ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], - (*Results)[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { - const MachineOperand *Definition = - Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); - if (Definition != nullptr && Definition->parent() != nullptr) - AmountParts->Artifacts.push_back( - const_cast(Definition->parent())); - } - ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), - AmountParts->Artifacts.begin(), - AmountParts->Artifacts.end()); - eraseDeadArtifacts(ValueParts->Artifacts, Function); - return {}; -} - -[[nodiscard]] Expected> -emitWideAdd(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, - std::span Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide add requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector Results; - Results.reserve(Left.size()); - auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); - if (!Low) - return std::unexpected(Low.error()); - auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - *Low, Right[0]); - if (!Carry) - return std::unexpected(Carry.error()); - auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); - if (!LowCopy) - return std::unexpected(LowCopy.error()); - Results.push_back(*LowCopy); - for (std::size_t Index = 1; Index < Left.size(); ++Index) { - auto Base = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); - if (!Base) - return std::unexpected(Base.error()); - std::optional BaseCarry; - if (Index + 1U < Left.size()) { - auto Compared = buildIntegerCompareBefore(Builder, Anchor, - IntegerPredicate::UnsignedLessThan, *Base, - Left[Index]); - if (!Compared) - return std::unexpected(Compared.error()); - BaseCarry = *Compared; - } - auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); - if (!Sum) - return std::unexpected(Sum.error()); - if (Index + 1U < Left.size()) { - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto CarryFromIncoming = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); - if (!CarryFromIncoming) - return std::unexpected(CarryFromIncoming.error()); - auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry, - *CarryFromIncoming); - if (!NextCarry) - return std::unexpected(NextCarry.error()); - Carry = *NextCarry; - } - auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); - if (!Copy) - return std::unexpected(Copy.error()); - Results.push_back(*Copy); - } - return Results; -} - -[[nodiscard]] Expected> -emitWideSub(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, - std::span Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide sub requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector Results; - Results.reserve(Left.size()); - auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); - if (!Low) - return std::unexpected(Low.error()); - Results.push_back(*Low); - auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Left[0], Right[0]); - if (!Borrow) - return std::unexpected(Borrow.error()); - for (std::size_t Index = 1; Index < Left.size(); ++Index) { - auto Base = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); - if (!Base) - return std::unexpected(Base.error()); - std::optional BaseBorrow; - if (Index + 1U < Left.size()) { - auto Compared = buildIntegerCompareBefore(Builder, Anchor, - IntegerPredicate::UnsignedGreaterThan, *Base, - Left[Index]); - if (!Compared) - return std::unexpected(Compared.error()); - BaseBorrow = *Compared; - } - auto Difference = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); - if (!Difference) - return std::unexpected(Difference.error()); - Results.push_back(*Difference); - if (Index + 1U < Left.size()) { - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsZero = - buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto BorrowFromIncoming = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); - if (!BorrowFromIncoming) - return std::unexpected(BorrowFromIncoming.error()); - auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow, - *BorrowFromIncoming); - if (!NextBorrow) - return std::unexpected(NextBorrow.error()); - Borrow = *NextBorrow; - } - } - return Results; -} - -[[nodiscard]] Expected> -emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, - std::span Left, std::span Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide bitwise operation requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector Results; - Results.reserve(Left.size()); - for (std::size_t Index = 0; Index < Left.size(); ++Index) { - auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); - if (!Part) - return std::unexpected(Part.error()); - Results.push_back(*Part); - } - return Results; -} - -[[nodiscard]] Expected lowerWideBinary(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - GenericOpcode Generic) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (Destination == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide binary instruction has invalid operands"); - auto LeftParts = collectWideValueParts(*Left, Function); - if (!LeftParts) - return std::unexpected(LeftParts.error()); - auto RightParts = collectWideValueParts(*Right, Function); - if (!RightParts) - return std::unexpected(RightParts.error()); - if (LeftParts->Parts.size() != RightParts->Parts.size()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide binary operands have different limb counts"); - auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - - MachineIRBuilder Builder(Function, Target); - Expected> Results = - Generic == GenericOpcode::G_ADD - ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) - : Generic == GenericOpcode::G_SUB - ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) - : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); - if (!Results) - return std::unexpected(Results.error()); - for (std::size_t Index = 0; Index < Results->size(); ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), - RightParts->Artifacts.end()); - eraseDeadArtifacts(LeftParts->Artifacts, Function); - return {}; -} - -[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || - Instruction.operandCount() < 2) - return false; - const MachineOperand *ConditionOperand = Instruction.operandAt(0); - const Register *Condition = - ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); - if (Condition == nullptr || !Condition->isVirtual() || - singleRegisterUse(Function, *Condition) != &Instruction) - return false; - const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition); - const MachineOperand *Definition = - Function.registerInfo().getUniqueVRegDef(*Condition); - const MachineInstr *Truncation = - Definition == nullptr ? nullptr : Definition->parent(); - if (!ConditionInfo || !ConditionInfo->get().Type || - ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr || - Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || - Truncation->operandCount() != 2) - return false; - return operandType(Function, *Truncation, 1) - .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) - .value_or(false); -} - -[[nodiscard]] Expected -lowerNarrowBranchCondition(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - if (!canLowerNarrowBranchCondition(Instruction, Function)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V branch condition is not a narrow truncation"); - const MachineOperand *ConditionOperand = Instruction.operandAt(0); - const Register Condition = *ConditionOperand->getIf(); - const MachineOperand *Definition = - Function.registerInfo().getUniqueVRegDef(Condition); - MachineInstr *Truncation = - const_cast(Definition->parent()); - const Register Source = *Truncation->operandAt(1)->getIf(); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - MachineIRBuilder Builder(Function, Target); - auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); - if (!One) - return std::unexpected(One.error()); - auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, - Source, *One); - if (!Masked) - return std::unexpected(Masked.error()); - auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); - if (!Replaced) - return std::unexpected(Replaced.error()); - if (Function.registerInfo().useEmpty(Condition)) - return eraseInstruction(*Truncation); - return {}; -} - -struct NarrowPhiIncoming final { - Register Value; - MachineBasicBlock *Block = nullptr; - MachineInstr *ReusableTruncation = nullptr; -}; - -[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { - MachineInstr *Instruction = Block.firstInstruction(); - while (Instruction != nullptr && - Instruction->opcode() == opcode(GenericOpcode::COPY)) - Instruction = Instruction->next(); - return Instruction; -} - -[[nodiscard]] Expected -buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, - MachineBasicBlock &Block, MachineInstr *Anchor, - GenericOpcode Generic, LLT Type, Register Source) { - auto Result = Function.registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Anchor == nullptr ? Builder.setInsertPoint(Block) - : Builder.setInsertPointBefore(*Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(Generic)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Source); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected lowerNarrowPhi(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const std::optional OriginalType = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || - Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI has invalid operands"); - auto LegalType = LLT::makeAnyScalar(32); - if (!LegalType) - return std::unexpected(LegalType.error()); - - std::vector Incoming; - Incoming.reserve((Instruction.operandCount() - 1U) / 2U); - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const MachineOperand *ValueOperand = Instruction.operandAt(Index); - const Register *Value = - ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); - const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); - const auto *Block = - BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); - if (Value == nullptr || Block == nullptr || Block->Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI has an invalid incoming edge"); - MachineInstr *Reusable = nullptr; - const MachineOperand *Definition = - Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast(Definition->parent()); - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && - Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && - operandType(Function, *Def, 1) - .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) - .value_or(false)) { - Reusable = Def; - } - Incoming.push_back( - NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, - .ReusableTruncation = Reusable}); - } - for (NarrowPhiIncoming &Entry : Incoming) { - if (Entry.ReusableTruncation == nullptr) - continue; - const std::size_t Count = static_cast(std::ranges::count( - Incoming, Entry.ReusableTruncation, - &NarrowPhiIncoming::ReusableTruncation)); - if (Count != 1) - Entry.ReusableTruncation = nullptr; - } - for (NarrowPhiIncoming &Entry : Incoming) { - if (Entry.ReusableTruncation == nullptr) - continue; - MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); - if (OriginalBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI input truncation is detached"); - auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); - if (!Removed) - return std::unexpected(Removed.error()); - } - - std::vector> Anchors; - for (const NarrowPhiIncoming &Entry : Incoming) { - if (std::ranges::find(Anchors, Entry.Block, - &decltype(Anchors)::value_type::first) == Anchors.end()) { - Anchors.emplace_back(Entry.Block, - firstInstructionAfterLeadingCopies(*Entry.Block)); - } - } - std::vector LegalIncoming; - LegalIncoming.reserve(Incoming.size()); - for (NarrowPhiIncoming &Entry : Incoming) { - const auto AnchorPosition = std::ranges::find( - Anchors, Entry.Block, &decltype(Anchors)::value_type::first); - MachineInstr *Anchor = AnchorPosition->second; - if (Entry.ReusableTruncation != nullptr) { - auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); - if (!SetType) - return std::unexpected(SetType.error()); - auto Inserted = Anchor == nullptr - ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) - : Entry.Block->insertInstructionBefore( - *Anchor, *Entry.ReusableTruncation); - if (!Inserted) - return std::unexpected(Inserted.error()); - LegalIncoming.push_back(Entry.Value); - continue; - } - const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value); - if (!ValueInfo || !ValueInfo->get().Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI input has no scalar type"); - const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth(); - const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT - : Width > 32 ? GenericOpcode::G_TRUNC - : GenericOpcode::COPY; - auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, - Conversion, *LegalType, Entry.Value); - if (!Converted) - return std::unexpected(Converted.error()); - LegalIncoming.push_back(*Converted); - } - - auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); - if (!LegalResult) - return std::unexpected(LegalResult.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*LegalResult); !Added) - return std::unexpected(Added.error()); - for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { - if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) - return std::unexpected(Added.error()); - } - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); - if (!Replaced) - return std::unexpected(Replaced.error()); - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected lowerWidePhi(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Destination == nullptr || Instruction.operandCount() < 5 || - (Instruction.operandCount() % 2U) == 0) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide PHI has invalid operands"); - - std::vector IncomingParts; - std::vector IncomingBlocks; - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const MachineOperand *ValueOperand = Instruction.operandAt(Index); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); - const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); - const auto *Block = BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); - if (Value == nullptr || Block == nullptr || Block->Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide PHI has an invalid incoming edge"); - auto Parts = collectWideValueParts(*Value, Function); - if (!Parts) - return std::unexpected(Parts.error()); - if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide PHI incoming values have different limb counts"); - IncomingParts.push_back(std::move(*Parts)); - IncomingBlocks.push_back(Block->Block); - } - const std::size_t PartCount = IncomingParts.front().Parts.size(); - auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - auto Type = LLT::makeAnyScalar(64); - if (!Type) - return std::unexpected(Type.error()); - MachineIRBuilder Builder(Function, Target); - std::vector Results; - Results.reserve(PartCount); - for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { - auto Result = Function.registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); - ++IncomingIndex) { - if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) - return std::unexpected(Added.error()); - } - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - Results.push_back(*Result); - } - for (std::size_t Index = 0; Index < PartCount; ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - std::vector Artifacts; - for (WideValueParts &Parts : IncomingParts) - Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); - eraseDeadArtifacts(Artifacts, Function); - return {}; -} - -[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, - const MachineFunction &Function) { - const MachineOperand *LowOperand = Instruction.operandAt(1); - const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); - if (Low == nullptr) - return false; - const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low); - if (!LowInfo || !LowInfo->get().Type) - return false; - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Destination == nullptr) - return false; - auto &RegisterInfo = const_cast(Function.registerInfo()); - for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { - MachineInstr *Trunc = Use.parent(); - if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || - Trunc->operandAt(1) != &Use) - return false; - const std::optional Result = resultType(Function, *Trunc); - if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) - return false; - } - return true; -} - -struct RV64ScalarLayout final { - std::uint64_t StoreSize = 0; - std::uint64_t AllocationSize = 0; - std::uint64_t ABIAlignment = 0; -}; - -[[nodiscard]] std::optional rv64ScalarLayout(LLT Type) noexcept { - if (Type.isPointer()) { - return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 - ? std::optional{{8, 8, 8}} - : std::nullopt; - } - if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) { - return std::nullopt; - } - const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; - const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes); - return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min(AllocationBytes, 8)}; -} - -[[nodiscard]] Expected -createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, - std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { - auto Size = MemorySize::fixed(SizeBytes); - auto Align = Alignment::create(AlignmentBytes); - if (!Size || !Align) - return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid"); - MachineMemOperandDesc Description; - Description.Accesses = {Access}; - Description.Size = *Size; - Description.MemoryType = Type; - Description.Align = *Align; - return Function.createMemOperand(std::move(Description)); -} - -struct VAArgPlan final { - Register Destination; - Register ListPointer; - LLT DestinationType; - LLT PointerType; - std::uint64_t Alignment = 0; - RV64ScalarLayout ValueLayout; -}; - -[[nodiscard]] std::optional vaargPlan(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || - !Instruction.memOperands().empty()) { - return std::nullopt; - } - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *ListOperand = Instruction.operandAt(1); - const MachineOperand *AlignmentOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf(); - const std::optional DestinationType = resultType(Function, Instruction); - const std::optional PointerType = operandType(Function, Instruction, 1); - const auto AlignmentValue = AlignmentOperand == nullptr - ? Expected(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing alignment"))) - : signedImmediate(*AlignmentOperand); - const std::optional ValueLayout = - DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt; - if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || - !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || - !DestinationType || !PointerType || !PointerType->isPointer() || - PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue || - *AlignmentValue <= 0 || !std::has_single_bit(static_cast(*AlignmentValue)) || - !ValueLayout) { - return std::nullopt; - } - - return VAArgPlan{*Destination, - *ListPointer, - *DestinationType, - *PointerType, - static_cast(*AlignmentValue), - *ValueLayout}; -} - -[[nodiscard]] Expected lowerVAArg(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = vaargPlan(Instruction, Function); - if (!Plan) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V va_arg legalization requires a scalar result, p0 list, and " - "power-of-two alignment"); - } - - auto ListLoadMemory = - createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8); - if (!ListLoadMemory) - return std::unexpected(ListLoadMemory.error()); - auto ListStoreMemory = - createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8); - if (!ListStoreMemory) - return std::unexpected(ListStoreMemory.error()); - - auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType); - if (!CurrentList) - return std::unexpected(CurrentList.error()); - MachineIRBuilder Builder(Function, Target); - if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) - return std::unexpected(Point.error()); - auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!ListLoad) - return std::unexpected(ListLoad.error()); - if (auto Added = ListLoad->addDef(*CurrentList); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ListLoad->build(); !Built) - return std::unexpected(Built.error()); - - Register ValueAddress = *CurrentList; - if (Plan->Alignment > 1) { - auto AlignmentOffset = - buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U); - if (!AlignmentOffset) - return std::unexpected(AlignmentOffset.error()); - auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, - Plan->PointerType, ValueAddress, *AlignmentOffset); - if (!BiasedAddress) - return std::unexpected(BiasedAddress.error()); - auto AlignmentMask = - buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1})); - if (!AlignmentMask) - return std::unexpected(AlignmentMask.error()); - auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK, - Plan->PointerType, *BiasedAddress, *AlignmentMask); - if (!AlignedAddress) - return std::unexpected(AlignedAddress.error()); - ValueAddress = *AlignedAddress; - } - - auto Increment = - buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize); - if (!Increment) - return std::unexpected(Increment.error()); - auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, - Plan->PointerType, ValueAddress, *Increment); - if (!NextList) - return std::unexpected(NextList.error()); - auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); - if (!ListStore) - return std::unexpected(ListStore.error()); - if (auto Added = ListStore->addUse(*NextList); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListStore->addUse(Plan->ListPointer); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ListStore->build(); !Built) - return std::unexpected(Built.error()); - - if (!Function.registerInfo().useEmpty(Plan->Destination)) { - auto ValueLoadMemory = - createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType, - Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment); - if (!ValueLoadMemory) - return std::unexpected(ValueLoadMemory.error()); - auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!ValueLoad) - return std::unexpected(ValueLoad.error()); - if (auto Added = ValueLoad->addDef(Plan->Destination); !Added) - return std::unexpected(Added.error()); - if (auto Added = ValueLoad->addUse(ValueAddress); !Added) - return std::unexpected(Added.error()); - if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ValueLoad->build(); !Built) - return std::unexpected(Built.error()); - } - return eraseInstruction(Instruction); -} - -struct VACopyPlan final { - Register Destination; - Register Source; - LLT PointerType; -}; - -[[nodiscard]] std::optional vacopyPlan(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || - Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { - return std::nullopt; - } - const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); - const MachineOperand *DestinationOperand = Instruction.operandAt(1); - const MachineOperand *SourceOperand = Instruction.operandAt(2); - const auto *Intrinsic = - IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf(); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - const std::optional DestinationType = operandType(Function, Instruction, 1); - const std::optional SourceType = operandType(Function, Instruction, 2); - if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || - Destination == nullptr || Source == nullptr || !Destination->isVirtual() || - !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || - !DestinationType || !SourceType || *DestinationType != *SourceType || - !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || - DestinationType->scalarBitWidth() != 64) { - return std::nullopt; - } - return VACopyPlan{*Destination, *Source, *DestinationType}; -} - -[[nodiscard]] Expected lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = vacopyPlan(Instruction, Function); - if (!Plan) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V va_copy legalization requires two p0 operands"); - } - auto Size = MemorySize::fixed(8); - auto Align = Alignment::create(8); - if (!Size || !Align) - return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); - - MachineMemOperandDesc LoadDescription; - LoadDescription.Accesses = {MemoryAccessFlag::Load}; - LoadDescription.Size = *Size; - LoadDescription.MemoryType = Plan->PointerType; - LoadDescription.Align = *Align; - auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); - if (!LoadMemory) - return std::unexpected(LoadMemory.error()); - - MachineMemOperandDesc StoreDescription; - StoreDescription.Accesses = {MemoryAccessFlag::Store}; - StoreDescription.Size = *Size; - StoreDescription.MemoryType = Plan->PointerType; - StoreDescription.Align = *Align; - auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); - if (!StoreMemory) - return std::unexpected(StoreMemory.error()); - - auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); - if (!Value) - return std::unexpected(Value.error()); - MachineIRBuilder Builder(Function, Target); - if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) - return std::unexpected(Point.error()); - - auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!Load) - return std::unexpected(Load.error()); - if (auto Added = Load->addDef(*Value); !Added) - return std::unexpected(Added.error()); - if (auto Added = Load->addUse(Plan->Source); !Added) - return std::unexpected(Added.error()); - if (auto Added = Load->addMemOperand(**LoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = Load->build(); !Built) - return std::unexpected(Built.error()); - - auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); - if (!Store) - return std::unexpected(Store.error()); - if (auto Added = Store->addUse(*Value); !Added) - return std::unexpected(Added.error()); - if (auto Added = Store->addUse(Plan->Destination); !Added) - return std::unexpected(Added.error()); - if (auto Added = Store->addMemOperand(**StoreMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = Store->build(); !Built) - return std::unexpected(Built.error()); - return eraseInstruction(Instruction); -} - -class RISCVLegalizerInfo final : public LegalizerInfo { -public: - explicit RISCVLegalizerInfo(RISCVSubtargetFeatures Features) : Features(Features) {} - - [[nodiscard]] Expected classify(const MachineInstr &Instruction, - const MachineFunction &Function, - const TargetDescription &) const override { - if (!Features.has(RISCVExtension::I)) { - return makeUnexpected(ErrorCode::InternalError, - "RISC-V legalizer requires the base I extension"); - } - if (!isGenericOpcode(Instruction.opcode())) - return LegalizationAction::Legal; - - const GenericOpcode Generic = static_cast(Instruction.opcode().value()); - const std::optional Type = resultType(Function, Instruction); - switch (Generic) { - case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - case GenericOpcode::G_VAARG: - return vaargPlan(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - case GenericOpcode::G_FRAME_INDEX: - return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_BR: - return LegalizationAction::Legal; - case GenericOpcode::G_BRCOND: { - const std::optional Condition = operandType(Function, Instruction, 0); - if (!Condition || !isScalarGPRType(*Condition)) - return LegalizationAction::Unsupported; - return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Legal; - } - case GenericOpcode::G_PHI: { - if (!Type) - return LegalizationAction::Unsupported; - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const std::optional Incoming = operandType(Function, Instruction, Index); - if (!Incoming || *Incoming != *Type) - return LegalizationAction::Unsupported; - } - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) - return LegalizationAction::Custom; - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) - return LegalizationAction::Custom; - if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) - return LegalizationAction::Unsupported; - return LegalizationAction::Legal; - } - case GenericOpcode::G_IMPLICIT_DEF: - return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_STORE: - if (Instruction.operandCount() == 0) - return LegalizationAction::Unsupported; - if (const std::optional ValueType = registerType(Function, *Instruction.operandAt(0)); - ValueType) { - const std::optional AddressType = operandType(Function, Instruction, 1); - if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || - AddressType->scalarBitWidth() != 64) { - return LegalizationAction::Unsupported; - } - if (Instruction.memOperands().size() == 1 && - Instruction.memOperands().front()->isAtomic()) { - const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; - return isSupportedIntegerType(*ValueType) && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && - isSupportedNonAtomicMemoryAccess(Instruction) && - memoryMatchesType(Instruction, *ValueType)) { - return LegalizationAction::Legal; - } - } - return LegalizationAction::Unsupported; - case GenericOpcode::G_SEXTLOAD: - case GenericOpcode::G_ZEXTLOAD: { - const auto Bytes = memoryAccessBytes(Instruction); - const std::optional AddressType = operandType(Function, Instruction, 1); - return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && - AddressType->isPointer() && AddressType->addressSpace() == 0 && - AddressType->scalarBitWidth() == 64 && - *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && - (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && - isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPEXTLOAD: { - const auto Bytes = memoryAccessBytes(Instruction); - const std::optional AddressType = operandType(Function, Instruction, 1); - return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && - AddressType->isPointer() && AddressType->addressSpace() == 0 && - AddressType->scalarBitWidth() == 64 && *Bytes == 4 && - isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPTRUNCSTORE: { - const std::optional ValueType = operandType(Function, Instruction, 0); - const std::optional AddressType = operandType(Function, Instruction, 1); - const auto Bytes = memoryAccessBytes(Instruction); - return ValueType && AddressType && Bytes && ValueType->isFloat() && - ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && - AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && - *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_LOAD: - if (Type) { - const std::optional AddressType = operandType(Function, Instruction, 1); - if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || - AddressType->scalarBitWidth() != 64) { - return LegalizationAction::Unsupported; - } - if (Instruction.memOperands().size() == 1 && - Instruction.memOperands().front()->isAtomic()) { - const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; - return isSupportedIntegerType(*Type) && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && - isSupportedNonAtomicMemoryAccess(Instruction) && - memoryMatchesType(Instruction, *Type)) { - return LegalizationAction::Legal; - } - } - return LegalizationAction::Unsupported; - case GenericOpcode::G_FADD: - case GenericOpcode::G_FSUB: - case GenericOpcode::G_FMUL: - case GenericOpcode::G_FDIV: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FMA: { - const std::optional First = operandType(Function, Instruction, 1); - const std::optional Second = operandType(Function, Instruction, 2); - const std::optional Third = operandType(Function, Instruction, 3); - return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && - *Second == *Type && *Third == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FNEG: - case GenericOpcode::G_FABS: - case GenericOpcode::G_FSQRT: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && isRVFloatType(*Type) && *Source == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FCOPYSIGN: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FCONSTANT: { - const MachineOperand *Immediate = Instruction.operandAt(1); - const auto *Floating = - Immediate == nullptr ? nullptr : Immediate->getIf(); - const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; - return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && - Floating->Bits.semantics() == *Semantics - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ASSERT_SEXT: - case GenericOpcode::G_ASSERT_ZEXT: - case GenericOpcode::G_ASSERT_ALIGN: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && *Type == *Source && isScalarGPRType(*Type) && - Instruction.operandCount() == 3 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FREEZE: - case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && *Type == *Source && - (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTRTOINT: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && - Source->isPointer() && Source->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_INTTOPTR: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && - Source->isInteger() && Source->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_BITCAST: { - const std::optional Source = operandType(Function, Instruction, 1); - const bool IntegerFloatPair = - Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && - ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && - (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); - return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ANYEXT: - case GenericOpcode::G_SEXT: - case GenericOpcode::G_ZEXT: { - const std::optional Source = operandType(Function, Instruction, 1); - if (!Type || !Source) - return LegalizationAction::Unsupported; - if (!isScalarIntegerExtension(*Type, *Source)) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = DestinationOperand == nullptr - ? nullptr - : DestinationOperand->getIf(); - MachineInstr *OnlyUse = Destination == nullptr - ? nullptr - : singleUseInstruction(Function, *Destination); - return (Type->isInteger() || Type->isAnyScalar()) && - (Source->isInteger() || Source->isAnyScalar()) && - Type->scalarBitWidth() > 64 && - Source->scalarBitWidth() < Type->scalarBitWidth() && - OnlyUse != nullptr && - OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *SourceRegister = - SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - const MachineOperand *SourceDefinition = - SourceRegister == nullptr || !SourceRegister->isVirtual() - ? nullptr - : Function.registerInfo().getUniqueVRegDef(*SourceRegister); - const MachineInstr *SourceDef = - SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); - if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) - return LegalizationAction::Custom; - if (Generic == GenericOpcode::G_ANYEXT && - narrowAnyExtBinaryPlan(Instruction, Function).has_value()) - return LegalizationAction::Custom; - return LegalizationAction::Legal; - } - case GenericOpcode::G_TRUNC: { - const std::optional Source = operandType(Function, Instruction, 1); - if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && - (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && - Source->scalarBitWidth() > Type->scalarBitWidth()) - return LegalizationAction::Legal; - return Type && Source && isSupportedIntegerScalar(*Type) && - isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 && - Source->scalarBitWidth() >= Type->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_MERGE_VALUES: - case GenericOpcode::G_UNMERGE_VALUES: { - if (Instruction.operandCount() < 3) - return LegalizationAction::Unsupported; - const std::size_t SmallTypeIndex = - Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; - const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES - ? 0 - : Instruction.operandCount() - 1U; - const std::optional SmallType = operandType(Function, Instruction, SmallTypeIndex); - const std::optional BigType = operandType(Function, Instruction, BigTypeIndex); - if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && - BigType->scalarBitWidth() > 64 && - !canFoldWideMergeImmediately(Instruction, Function)) - return LegalizationAction::Legal; - return SmallType && BigType && - (SmallType->isInteger() || SmallType->isAnyScalar()) && - (BigType->isInteger() || BigType->isAnyScalar()) && - SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SEXT_INREG: { - const std::optional Source = operandType(Function, Instruction, 1); - const MachineOperand *WidthOperand = Instruction.operandAt(2); - const auto Width = - WidthOperand == nullptr - ? Expected(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) - : signedImmediate(*WidthOperand); - return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && - static_cast(*Width) <= Type->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ICMP: { - const MachineOperand *Predicate = Instruction.operandAt(1); - const auto *IntegerPredicate = - Predicate == nullptr ? nullptr : Predicate->getIf(); - const std::optional Left = operandType(Function, Instruction, 2); - const std::optional Right = operandType(Function, Instruction, 3); - if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || - IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || - !isSupportedIntegerType(*Left)) { - return LegalizationAction::Unsupported; - } - return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Custom; - } - case GenericOpcode::G_FCMP: { - const MachineOperand *Predicate = Instruction.operandAt(1); - const auto *FloatingPredicate = - Predicate == nullptr ? nullptr : Predicate->getIf(); - const std::optional Left = operandType(Function, Instruction, 2); - const std::optional Right = operandType(Function, Instruction, 3); - return Type && isI1Type(*Type) && FloatingPredicate != nullptr && - FloatingPredicate->floatingPredicate() != nullptr && Left && Right && - *Left == *Right && isRVFloatType(*Left) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPEXT: - case GenericOpcode::G_FPTRUNC: { - const std::optional Source = operandType(Function, Instruction, 1); - const bool IsExtension = Generic == GenericOpcode::G_FPEXT; - return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && - ((IsExtension && Source->scalarBitWidth() == 32 && - Type->scalarBitWidth() == 64) || - (!IsExtension && Source->scalarBitWidth() == 64 && - Type->scalarBitWidth() == 32)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPTOSI: - case GenericOpcode::G_FPTOUI: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SITOFP: - case GenericOpcode::G_UITOFP: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FMINNUM: - case GenericOpcode::G_FMAXNUM: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SELECT: { - const std::optional Condition = operandType(Function, Instruction, 1); - const std::optional TrueValue = operandType(Function, Instruction, 2); - const std::optional FalseValue = operandType(Function, Instruction, 3); - return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && - *TrueValue == *Type && *FalseValue == *Type && - (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTRMASK: { - const std::optional Pointer = operandType(Function, Instruction, 1); - const std::optional Mask = operandType(Function, Instruction, 2); - return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && *Pointer == *Type && - isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ABS: { - const std::optional Source = operandType(Function, Instruction, 1); - return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SDIVREM: - case GenericOpcode::G_UDIVREM: { - const std::optional Remainder = operandType(Function, Instruction, 1); - const std::optional Left = operandType(Function, Instruction, 2); - const std::optional Right = operandType(Function, Instruction, 3); - return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && - *Remainder == *Type && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SMULH: - case GenericOpcode::G_UMULH: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SMIN: - case GenericOpcode::G_SMAX: - case GenericOpcode::G_UMIN: - case GenericOpcode::G_UMAX: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_UADDO: - case GenericOpcode::G_USUBO: - case GenericOpcode::G_SADDO: - case GenericOpcode::G_SSUBO: - case GenericOpcode::G_UMULO: - case GenericOpcode::G_SMULO: { - const std::optional Overflow = operandType(Function, Instruction, 1); - const std::optional Left = operandType(Function, Instruction, 2); - const std::optional Right = operandType(Function, Instruction, 3); - return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && - isI1Type(*Overflow) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_UADDE: - case GenericOpcode::G_USUBE: - case GenericOpcode::G_SADDE: - case GenericOpcode::G_SSUBE: { - const std::optional CarryOut = operandType(Function, Instruction, 1); - const std::optional Left = operandType(Function, Instruction, 2); - const std::optional Right = operandType(Function, Instruction, 3); - const std::optional CarryIn = operandType(Function, Instruction, 4); - return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && - isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ATOMICRMW_XCHG: - case GenericOpcode::G_ATOMICRMW_ADD: - case GenericOpcode::G_ATOMICRMW_SUB: - case GenericOpcode::G_ATOMICRMW_AND: - case GenericOpcode::G_ATOMICRMW_OR: - case GenericOpcode::G_ATOMICRMW_XOR: - case GenericOpcode::G_ATOMICRMW_MAX: - case GenericOpcode::G_ATOMICRMW_MIN: - case GenericOpcode::G_ATOMICRMW_UMAX: - case GenericOpcode::G_ATOMICRMW_UMIN: { - const std::optional Address = operandType(Function, Instruction, 1); - const std::optional Value = operandType(Function, Instruction, 2); - const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; - return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && - Address->isPointer() && Address->addressSpace() == 0 && - Address->scalarBitWidth() == 64 && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, - Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ATOMIC_CMPXCHG: { - const std::optional Address = operandType(Function, Instruction, 1); - const std::optional Compare = operandType(Function, Instruction, 2); - const std::optional NewValue = operandType(Function, Instruction, 3); - if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || - (!Type->isInteger() && !Type->isAnyScalar()) || - (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && - Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || - !Address->isPointer() || Address->addressSpace() != 0 || - Address->scalarBitWidth() != 64 || - !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, - (Type->scalarBitWidth() + 7U) / 8U)) { - return LegalizationAction::Unsupported; - } - return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; - } - case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { - const std::optional Success = operandType(Function, Instruction, 1); - const std::optional Address = operandType(Function, Instruction, 2); - const std::optional Compare = operandType(Function, Instruction, 3); - const std::optional NewValue = operandType(Function, Instruction, 4); - return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && - *Compare == *Type && *NewValue == *Type && - (Type->isInteger() || Type->isAnyScalar()) && - (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || - Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && - Address->isPointer() && Address->addressSpace() == 0 && - Address->scalarBitWidth() == 64 && - isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, - (Type->scalarBitWidth() + 7U) / 8U) - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FENCE: { - const MachineOperand *OrderingOperand = Instruction.operandAt(0); - const MachineOperand *ScopeOperand = Instruction.operandAt(1); - const auto Ordering = OrderingOperand == nullptr - ? Expected(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing fence ordering"))) - : signedImmediate(*OrderingOperand); - const auto Scope = ScopeOperand == nullptr - ? Expected(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing fence scope"))) - : signedImmediate(*ScopeOperand); - return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_CONSTANT_POOL: { - const MachineOperand *PoolOperand = Instruction.operandAt(1); - const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf(); - return Type && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && Pool != nullptr && - Function.constantPool().entry(Pool->Index) != nullptr - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_GLOBAL_VALUE: { - const MachineOperand *GlobalOperand = Instruction.operandAt(1); - const auto *Global = GlobalOperand == nullptr - ? nullptr - : GlobalOperand->getIf(); - return Type && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && Global != nullptr && - !Global->Name.empty() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_BRINDIRECT: - return operandType(Function, Instruction, 0) - .transform([](LLT Value) { - return Value.isPointer() && Value.addressSpace() == 0 && - Value.scalarBitWidth() == 64; - }) - .value_or(false) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_TRAP: - case GenericOpcode::G_DEBUGTRAP: - return Instruction.operandCount() == 0 ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_UBSANTRAP: - return Instruction.operandCount() == 1 ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_CONSTANT: { - const MachineOperand *Immediate = Instruction.operandAt(1); - if (!Type || Immediate == nullptr || - (Immediate->kind() != MachineOperand::Kind::Immediate && - Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) - return LegalizationAction::Unsupported; - if (Type->scalarBitWidth() > 64) - return wideConstantUnmergePlan(Instruction, Function).has_value() - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { - auto Value = signedImmediate(*Immediate); - if (Value && shouldUseRV64IntegerConstantPool(*Value)) - return LegalizationAction::Custom; - } - return isSupportedIntegerType(*Type) ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ADD: - case GenericOpcode::G_SUB: - case GenericOpcode::G_MUL: - case GenericOpcode::G_SDIV: - case GenericOpcode::G_UDIV: - case GenericOpcode::G_SREM: - case GenericOpcode::G_UREM: - case GenericOpcode::G_AND: - case GenericOpcode::G_OR: - case GenericOpcode::G_XOR: - case GenericOpcode::G_SHL: - case GenericOpcode::G_LSHR: - case GenericOpcode::G_ASHR: { - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || - (!Type->isInteger() && !Type->isAnyScalar())) - return LegalizationAction::Unsupported; - if (Type->scalarBitWidth() > 64) { - return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || - Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || - Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - if (Generic == GenericOpcode::G_MUL && !Features.has(RISCVExtension::M)) { - if (Type->scalarBitWidth() == 64) - return LegalizationAction::Libcall; - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - MachineInstr *Extension = - Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); - const std::optional ExtensionType = - Extension == nullptr ? std::nullopt : resultType(Function, *Extension); - if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && - ExtensionType && ExtensionType->scalarBitWidth() == 64) { - return LegalizationAction::Legal; - } - return LegalizationAction::Custom; - } - return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTR_ADD: { - const std::optional Pointer = operandType(Function, Instruction, 1); - const std::optional Offset = operandType(Function, Instruction, 2); - return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && *Pointer == *Type && - isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - default: - break; - } - - // Keep the unsupported policy mechanically synchronized with the complete - // target-independent opcode inventory. Supported cases return above. - switch (Generic) { -#define RCC_CORE_OPCODE(Name) -#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: -#include "rcc/CodeGen/GenericOpcode.def" -#undef RCC_GENERIC_OPCODE -#undef RCC_CORE_OPCODE - return LegalizationAction::Unsupported; - case GenericOpcode::Invalid: - case GenericOpcode::CORE_PSEUDO_END: - case GenericOpcode::G_GENERIC_END: - default: - break; - } - return makeUnexpected(ErrorCode::InternalError, - "RISC-V legalizer received an invalid generic opcode value"); - } - - [[nodiscard]] Expected lower(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, - LegalizationAction Action) const override { - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { - return lowerVACopy(Instruction, Function, Target); - } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) { - return lowerVAArg(Instruction, Function, Target); - } - if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && !Features.has(RISCVExtension::M)) { - if (Action == LegalizationAction::Custom) - return widenNarrowMultiply(Instruction, Function, Target); - if (Action == LegalizationAction::Libcall) - return lowerMultiplyLibcall(Instruction, Function, Target, - Features.has(RISCVExtension::D) ? CallingConvention::LP64D - : CallingConvention::LP64); - } - if (Action == LegalizationAction::Custom && - resultType(Function, Instruction) - .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) - .value_or(false)) { - const Opcode Value = Instruction.opcode(); - if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || - Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || - Value == opcode(GenericOpcode::G_XOR)) { - return lowerWideBinary(Instruction, Function, Target, - static_cast(Value.value())); - } - if (Value == opcode(GenericOpcode::G_SHL)) - return lowerWideShiftLeft(Instruction, Function, Target); - if (Value == opcode(GenericOpcode::G_PHI)) - return lowerWidePhi(Instruction, Function, Target); - } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) - return lowerMergeValues(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) - return lowerUnmergeValues(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) - return lowerNarrowBranchCondition(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_PHI)) - return lowerNarrowPhi(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) - return lowerIntegerCompare(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) - return lowerAtomicCompareExchange(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) - return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || - Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || - Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { - if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && - narrowAnyExtBinaryPlan(Instruction, Function).has_value()) - return foldNarrowAnyExtBinary(Instruction, Function, Target); - return foldConstantExtension( - Instruction, Function, - static_cast(Instruction.opcode().value())); - } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { - const MachineOperand *Immediate = Instruction.operandAt(1); - const std::optional Type = resultType(Function, Instruction); - if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && - Type->scalarBitWidth() == 64) { - auto Value = signedImmediate(*Immediate); - if (Value && shouldUseRV64IntegerConstantPool(*Value)) - return lowerConstantToPool(Instruction, Function, Target); - } - return foldWideConstant(Instruction, Function, Target); - } - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V legalizer has no lowering rule for this instruction"); - } - -private: - RISCVSubtargetFeatures Features; -}; - -class RISCVRegisterBankInfo final : public RegisterBankInfo { -public: - [[nodiscard]] Expected selectBank(const MachineFunction &, Register, - const VirtualRegisterInfo &Info, - const MachineInstr *, - const TargetDescription &) const override { - if (!Info.Type) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V register-bank selection requires a virtual-register type"); - } - if (isScalarGPRType(*Info.Type)) { - return RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); - } - if (isRVFloatType(*Info.Type)) - return RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V RV64GC has no register bank for virtual register type '" + - Info.Type->toString() + "'"); - } - - [[nodiscard]] Expected - constrainInstruction(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) const override { - std::vector> PlannedClasses; - for (const MachineOperand &Operand : Instruction.operands()) { - const Register *Reg = Operand.getIf(); - if (Reg == nullptr || !Reg->isVirtual()) - continue; - auto Required = requiredRegisterClass(Function, *Reg, Target); - if (!Required) - return std::unexpected(Required.error()); - if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) { - return Entry.first == *Reg; - }) == PlannedClasses.end()) - PlannedClasses.emplace_back(*Reg, *Required); - } - for (const auto &[Reg, Required] : PlannedClasses) { - auto Info = Function.registerInfo().virtualRegisterInfo(Reg); - if (!Info) - return std::unexpected(Info.error()); - if (Info->get().RegClass) - continue; - auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required); - if (!SetClass) - return std::unexpected(SetClass.error()); - } - return {}; - } - - [[nodiscard]] Expected validateInstruction(const MachineInstr &Instruction, - const MachineFunction &Function, - const TargetDescription &Target) const override { - for (const MachineOperand &Operand : Instruction.operands()) { - const Register *Reg = Operand.getIf(); - if (Reg == nullptr || !Reg->isVirtual()) - continue; - auto Required = requiredRegisterClass(Function, *Reg, Target); - if (!Required) - return std::unexpected(Required.error()); - } - return {}; - } -}; - -class RISCVInstructionSelector final : public InstructionSelector { -public: - [[nodiscard]] Expected - constrainSelectedInstruction(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) const override { - return registerBankInfo().constrainInstruction(Instruction, Function, Target); - } - - [[nodiscard]] Expected canSelect(const MachineInstr &Instruction, - const MachineFunction &Function, - const TargetDescription &Target) const override { - if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) { - const std::optional Type = resultType(Function, Instruction); - const std::optional Left = operandType(Function, Instruction, 1); - const std::optional Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isScalarGPRType(*Type) && - Type->scalarBitWidth() == 64 && *Left == *Type && *Right == *Type - ? Expected{} - : makeUnexpected(ErrorCode::Unsupported, - "RISC-V G_SLLW requires three 64-bit GPR operands"); - } - if (!isGenericOpcode(Instruction.opcode())) - return {}; - const GenericOpcode Generic = static_cast(Instruction.opcode().value()); - const std::optional Type = resultType(Function, Instruction); - const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; - const auto Float = [Type] { return Type && isRVFloatType(*Type); }; - auto Classification = legalizerInfo().classify(Instruction, Function, Target); - if (!Classification) - return std::unexpected(Classification.error()); - if (*Classification != LegalizationAction::Legal) { - std::string Message = "RISC-V selector cannot select opcode '" + - std::string(getGenericOpcodeName(Generic)) + "'"; - if (Type) - Message += " with result type '" + Type->toString() + "'"; - Message += ": the legalization rule explicitly marks this combination unsupported"; - return makeUnexpected(ErrorCode::Unsupported, std::move(Message)); - } - switch (Generic) { - case GenericOpcode::G_ADD: - case GenericOpcode::G_SUB: - case GenericOpcode::G_MUL: - case GenericOpcode::G_SDIV: - case GenericOpcode::G_UDIV: - case GenericOpcode::G_SREM: - case GenericOpcode::G_UREM: - case GenericOpcode::G_AND: - case GenericOpcode::G_OR: - case GenericOpcode::G_XOR: - case GenericOpcode::G_SHL: - case GenericOpcode::G_LSHR: - case GenericOpcode::G_ASHR: - case GenericOpcode::G_PTR_ADD: - case GenericOpcode::G_PTRMASK: - case GenericOpcode::G_CONSTANT: - if (!Integer()) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V selector requires an i1-i64 integer or 64-bit pointer value"); - if (Generic == GenericOpcode::G_CONSTANT) { - const MachineOperand *Immediate = Instruction.operandAt(1); - if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && - Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer constant requires an immediate operand"); - } - return {}; - case GenericOpcode::G_LOAD: - case GenericOpcode::G_SEXTLOAD: - case GenericOpcode::G_ZEXTLOAD: - case GenericOpcode::G_FPEXTLOAD: - case GenericOpcode::G_FPTRUNCSTORE: - case GenericOpcode::G_STORE: - case GenericOpcode::G_BR: - case GenericOpcode::G_BRCOND: - return {}; - case GenericOpcode::G_FADD: - case GenericOpcode::G_FSUB: - case GenericOpcode::G_FMUL: - case GenericOpcode::G_FDIV: - case GenericOpcode::G_FMA: - case GenericOpcode::G_FSQRT: - case GenericOpcode::G_FNEG: - case GenericOpcode::G_FABS: - case GenericOpcode::G_FCOPYSIGN: - if (!Float()) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V selector requires f32 or f64 floating values"); - return {}; - case GenericOpcode::G_FCONSTANT: { - const MachineOperand *Immediate = Instruction.operandAt(1); - const auto *Floating = - Immediate == nullptr ? nullptr : Immediate->getIf(); - const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; - if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V selector requires a matching floating constant"); - return {}; - } - case GenericOpcode::G_ANYEXT: - case GenericOpcode::G_ASSERT_SEXT: - case GenericOpcode::G_ASSERT_ZEXT: - case GenericOpcode::G_ASSERT_ALIGN: - case GenericOpcode::G_FREEZE: - case GenericOpcode::G_CONSTANT_FOLD_BARRIER: - case GenericOpcode::G_PTRTOINT: - case GenericOpcode::G_INTTOPTR: - case GenericOpcode::G_BITCAST: - case GenericOpcode::G_TRUNC: - case GenericOpcode::G_SEXT: - case GenericOpcode::G_SEXT_INREG: - case GenericOpcode::G_ZEXT: - case GenericOpcode::G_ICMP: - case GenericOpcode::G_FCMP: - case GenericOpcode::G_FPEXT: - case GenericOpcode::G_FPTRUNC: - case GenericOpcode::G_FPTOSI: - case GenericOpcode::G_FPTOUI: - case GenericOpcode::G_SITOFP: - case GenericOpcode::G_UITOFP: - case GenericOpcode::G_FMINNUM: - case GenericOpcode::G_FMAXNUM: - case GenericOpcode::G_SDIVREM: - case GenericOpcode::G_UDIVREM: - case GenericOpcode::G_SMULH: - case GenericOpcode::G_UMULH: - case GenericOpcode::G_SMIN: - case GenericOpcode::G_SMAX: - case GenericOpcode::G_UMIN: - case GenericOpcode::G_UMAX: - case GenericOpcode::G_SELECT: - case GenericOpcode::G_ABS: - case GenericOpcode::G_UADDO: - case GenericOpcode::G_USUBO: - case GenericOpcode::G_SADDO: - case GenericOpcode::G_SSUBO: - case GenericOpcode::G_UMULO: - case GenericOpcode::G_SMULO: - case GenericOpcode::G_UADDE: - case GenericOpcode::G_USUBE: - case GenericOpcode::G_SADDE: - case GenericOpcode::G_SSUBE: - case GenericOpcode::G_ATOMICRMW_XCHG: - case GenericOpcode::G_ATOMICRMW_ADD: - case GenericOpcode::G_ATOMICRMW_SUB: - case GenericOpcode::G_ATOMICRMW_AND: - case GenericOpcode::G_ATOMICRMW_OR: - case GenericOpcode::G_ATOMICRMW_XOR: - case GenericOpcode::G_ATOMICRMW_MAX: - case GenericOpcode::G_ATOMICRMW_MIN: - case GenericOpcode::G_ATOMICRMW_UMAX: - case GenericOpcode::G_ATOMICRMW_UMIN: - case GenericOpcode::G_FENCE: - case GenericOpcode::G_BRINDIRECT: - case GenericOpcode::G_TRAP: - case GenericOpcode::G_DEBUGTRAP: - case GenericOpcode::G_UBSANTRAP: - case GenericOpcode::G_PHI: - case GenericOpcode::G_IMPLICIT_DEF: - return {}; - default: - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V instruction selector has no rule for generic opcode"); - } - } - - [[nodiscard]] Expected select(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) const override { - auto Match = canSelect(Instruction, Function, Target); - if (!Match) - return std::unexpected(Match.error()); - if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) - return Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLW)); - const GenericOpcode Generic = static_cast(Instruction.opcode().value()); - const std::optional Type = resultType(Function, Instruction); - - const auto SelectDirect = [&Instruction](InstructionOpcode Selected) { - return Instruction.setOpcode(targetOpcode(Selected)); - }; - switch (Generic) { - case GenericOpcode::G_ADD: - return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD, - InstructionOpcode::ADDW); - case GenericOpcode::G_SUB: - return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB, - InstructionOpcode::SUBW); - case GenericOpcode::G_MUL: - return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL, - InstructionOpcode::MULW); - case GenericOpcode::G_SDIV: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::DIV, InstructionOpcode::DIVW, true, - true); - case GenericOpcode::G_UDIV: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false, - true); - case GenericOpcode::G_SREM: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::REM, InstructionOpcode::REMW, true, - true); - case GenericOpcode::G_UREM: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::REMU, InstructionOpcode::REMUW, false, - true); - case GenericOpcode::G_SDIVREM: - return selectDivRem(Instruction, Function, Target, Type, true); - case GenericOpcode::G_UDIVREM: - return selectDivRem(Instruction, Function, Target, Type, false); - case GenericOpcode::G_AND: - return SelectDirect(InstructionOpcode::AND); - case GenericOpcode::G_OR: - return SelectDirect(InstructionOpcode::OR); - case GenericOpcode::G_XOR: - return SelectDirect(InstructionOpcode::XOR); - case GenericOpcode::G_SHL: - return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL, - InstructionOpcode::SLLW); - case GenericOpcode::G_LSHR: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::SRL, InstructionOpcode::SRLW, false, - false); - case GenericOpcode::G_ASHR: - return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, - InstructionOpcode::SRA, InstructionOpcode::SRAW, true, - false); - case GenericOpcode::G_PTR_ADD: - return SelectDirect(InstructionOpcode::ADD); - case GenericOpcode::G_PTRMASK: - return SelectDirect(InstructionOpcode::AND); - case GenericOpcode::G_LOAD: - return selectLoad(Instruction, Type); - case GenericOpcode::G_SEXTLOAD: - return selectExtLoad(Instruction, Type, true); - case GenericOpcode::G_ZEXTLOAD: - return selectExtLoad(Instruction, Type, false); - case GenericOpcode::G_FPEXTLOAD: - return selectFloatExtLoad(Instruction, Function, Target); - case GenericOpcode::G_STORE: - return selectStore(Instruction, Function); - case GenericOpcode::G_FPTRUNCSTORE: - return selectFloatTruncStore(Instruction, Function, Target); - case GenericOpcode::G_BR: - return selectBranch(Instruction); - case GenericOpcode::G_BRCOND: - return selectConditionalBranch(Instruction); - case GenericOpcode::G_CONSTANT: - return selectIntegerConstant(Instruction, Function, Target, Type); - case GenericOpcode::G_FCONSTANT: - return selectFloatingConstant(Instruction, Function, Target, Type); - case GenericOpcode::G_ASSERT_SEXT: - case GenericOpcode::G_ASSERT_ZEXT: - case GenericOpcode::G_ASSERT_ALIGN: - if (Instruction.operandCount() == 3) { - auto EraseImmediate = Instruction.eraseOperand(2); - if (!EraseImmediate) - return std::unexpected(EraseImmediate.error()); - } - return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - case GenericOpcode::G_FREEZE: - case GenericOpcode::G_CONSTANT_FOLD_BARRIER: - case GenericOpcode::G_PTRTOINT: - case GenericOpcode::G_INTTOPTR: - return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - case GenericOpcode::G_BITCAST: - return selectBitcast(Instruction, Function); - case GenericOpcode::G_FADD: - return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S, - InstructionOpcode::FADD_D); - case GenericOpcode::G_FSUB: - return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S, - InstructionOpcode::FSUB_D); - case GenericOpcode::G_FMUL: - return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S, - InstructionOpcode::FMUL_D); - case GenericOpcode::G_FDIV: - return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S, - InstructionOpcode::FDIV_D); - case GenericOpcode::G_FMA: - return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S, - InstructionOpcode::FMADD_D); - case GenericOpcode::G_FSQRT: - return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S, - InstructionOpcode::FSQRT_D); - case GenericOpcode::G_FNEG: - return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S, - InstructionOpcode::FSGNJN_D); - case GenericOpcode::G_FABS: - return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S, - InstructionOpcode::FSGNJX_D); - case GenericOpcode::G_FCOPYSIGN: - return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S, - InstructionOpcode::FSGNJ_D); - case GenericOpcode::G_ANYEXT: - case GenericOpcode::G_SEXT: - case GenericOpcode::G_ZEXT: - return selectIntegerExtension(Instruction, Function, Target, Generic, Type); - case GenericOpcode::G_TRUNC: - return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - case GenericOpcode::G_SEXT_INREG: - return selectSignExtendInRegister(Instruction, Function, Target, Type); - case GenericOpcode::G_ICMP: - return selectIntegerCompare(Instruction, Function, Target); - case GenericOpcode::G_FCMP: - return selectFloatingCompare(Instruction, Function, Target); - case GenericOpcode::G_FPEXT: - return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S, - InstructionOpcode::FCVT_D_S); - case GenericOpcode::G_FPTRUNC: - return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D, - InstructionOpcode::FCVT_S_D); - case GenericOpcode::G_FPTOSI: - case GenericOpcode::G_FPTOUI: - return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI); - case GenericOpcode::G_SITOFP: - case GenericOpcode::G_UITOFP: - return selectIntegerToFloat(Instruction, Function, Target, Type, - Generic == GenericOpcode::G_UITOFP); - case GenericOpcode::G_FMINNUM: - return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S, - InstructionOpcode::FMIN_D); - case GenericOpcode::G_FMAXNUM: - return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S, - InstructionOpcode::FMAX_D); - case GenericOpcode::G_SMULH: - return selectMultiplyHigh(Instruction, Function, Target, Type, true); - case GenericOpcode::G_UMULH: - return selectMultiplyHigh(Instruction, Function, Target, Type, false); - case GenericOpcode::G_SMIN: - case GenericOpcode::G_SMAX: - case GenericOpcode::G_UMIN: - case GenericOpcode::G_UMAX: - return selectIntegerMinMax(Instruction, Function, Target, Type, Generic); - case GenericOpcode::G_SELECT: - return selectValue(Instruction, Function, Target, Type); - case GenericOpcode::G_ABS: - return selectAbsoluteValue(Instruction, Function, Target, Type); - case GenericOpcode::G_UADDO: - case GenericOpcode::G_USUBO: - case GenericOpcode::G_SADDO: - case GenericOpcode::G_SSUBO: - case GenericOpcode::G_UMULO: - case GenericOpcode::G_SMULO: - return selectOverflow(Instruction, Function, Target, Type, Generic); - case GenericOpcode::G_UADDE: - case GenericOpcode::G_USUBE: - case GenericOpcode::G_SADDE: - case GenericOpcode::G_SSUBE: - return selectExtendedOverflow(Instruction, Function, Target, Type, Generic); - case GenericOpcode::G_ATOMICRMW_XCHG: - case GenericOpcode::G_ATOMICRMW_ADD: - case GenericOpcode::G_ATOMICRMW_SUB: - case GenericOpcode::G_ATOMICRMW_AND: - case GenericOpcode::G_ATOMICRMW_OR: - case GenericOpcode::G_ATOMICRMW_XOR: - case GenericOpcode::G_ATOMICRMW_MAX: - case GenericOpcode::G_ATOMICRMW_MIN: - case GenericOpcode::G_ATOMICRMW_UMAX: - case GenericOpcode::G_ATOMICRMW_UMIN: - return selectAtomicRMW(Instruction, Function, Target, Type, Generic); - case GenericOpcode::G_FENCE: - return selectFence(Instruction); - case GenericOpcode::G_BRINDIRECT: - return selectIndirectBranch(Instruction); - case GenericOpcode::G_TRAP: - case GenericOpcode::G_DEBUGTRAP: - case GenericOpcode::G_UBSANTRAP: - return selectTrap(Instruction); - case GenericOpcode::G_PHI: - return Instruction.setOpcode(opcode(GenericOpcode::PHI)); - case GenericOpcode::G_IMPLICIT_DEF: - return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF)); - case GenericOpcode::COPY: - case GenericOpcode::IMPLICIT_DEF: - case GenericOpcode::PHI: - case GenericOpcode::REG_SEQUENCE: - case GenericOpcode::INSERT_SUBREG: - case GenericOpcode::EXTRACT_SUBREG: - case GenericOpcode::SUBREG_TO_REG: - case GenericOpcode::KILL: - case GenericOpcode::BUNDLE: - case GenericOpcode::INLINEASM: - case GenericOpcode::CORE_PSEUDO_END: - break; - default: - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V instruction selector has no rule for generic opcode"); - } - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V instruction selector has no rule for generic opcode"); - } - -private: - [[nodiscard]] static Expected replaceImmediate(MachineInstr &Instruction, std::size_t Index, - std::int64_t Value) { - if (Instruction.operandAt(Index) == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V instruction is missing an immediate operand"); - auto Erased = Instruction.eraseOperand(Index); - if (!Erased) - return std::unexpected(Erased.error()); - return Instruction.insertOperand(Index, MachineOperand::immediate(Value)); - } - - [[nodiscard]] static Expected - buildBinaryAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, - Register Def, Register Left, Register Right) { - auto Point = Builder.setInsertPointAfter(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(targetOpcode(Opcode)); - if (!Pending) - return std::unexpected(Pending.error()); - auto DefResult = Pending->addDef(Def); - if (!DefResult) - return std::unexpected(DefResult.error()); - auto LeftResult = Pending->addUse(Left); - if (!LeftResult) - return std::unexpected(LeftResult.error()); - auto RightResult = Pending->addUse(Right); - if (!RightResult) - return std::unexpected(RightResult.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Built; - } - - [[nodiscard]] static Expected - buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, - Register Def, Register Source, std::int64_t Value) { - auto Point = Builder.setInsertPointAfter(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(targetOpcode(Opcode)); - if (!Pending) - return std::unexpected(Pending.error()); - auto DefResult = Pending->addDef(Def); - if (!DefResult) - return std::unexpected(DefResult.error()); - auto SourceResult = Pending->addUse(Source); - if (!SourceResult) - return std::unexpected(SourceResult.error()); - auto ImmediateResult = Pending->addImmediate(Value); - if (!ImmediateResult) - return std::unexpected(ImmediateResult.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Built; - } - - [[nodiscard]] static Expected - buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, - Register Def, Register Source, std::int64_t Value) { - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(targetOpcode(Opcode)); - if (!Pending) - return std::unexpected(Pending.error()); - auto DefResult = Pending->addDef(Def); - if (!DefResult) - return std::unexpected(DefResult.error()); - auto SourceResult = Pending->addUse(Source); - if (!SourceResult) - return std::unexpected(SourceResult.error()); - auto ImmediateResult = Pending->addImmediate(Value); - if (!ImmediateResult) - return std::unexpected(ImmediateResult.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Built; - } - - [[nodiscard]] static Expected - normalizeIntegerBefore(MachineInstr &Anchor, MachineFunction &Function, - const TargetDescription &Target, Register Source, LLT Type, bool Signed) { - if (!isSupportedIntegerScalar(Type)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V integer normalization requires a scalar i1-i64 type"); - const std::uint32_t Bits = Type.scalarBitWidth(); - if (Bits == 64) - return Source; - - MachineIRBuilder Builder(Function, Target); - if (Signed && Bits == 32) { - auto Result = createTemporary(Function, Type, Target); - if (!Result) - return std::unexpected(Result.error()); - auto Extended = - buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0); - if (!Extended) - return std::unexpected(Extended.error()); - return *Result; - } - - const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64; - const std::int64_t Shift = static_cast(ContainerBits - Bits); - auto Shifted = createTemporary(Function, Type, Target); - auto Result = createTemporary(Function, Type, Target); - if (!Shifted || !Result) - return std::unexpected(!Shifted ? Shifted.error() : Result.error()); - const InstructionOpcode ShiftLeft = - ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; - const InstructionOpcode ShiftRight = - Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) - : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); - auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift); - if (!Left) - return std::unexpected(Left.error()); - auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift); - if (!Right) - return std::unexpected(Right.error()); - return *Result; - } - - [[nodiscard]] static Expected buildUnaryAfter(MachineIRBuilder &Builder, - MachineInstr &Anchor, - InstructionOpcode Opcode, - Register Def, Register Source) { - auto Point = Builder.setInsertPointAfter(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(targetOpcode(Opcode)); - if (!Pending) - return std::unexpected(Pending.error()); - auto DefResult = Pending->addDef(Def); - if (!DefResult) - return std::unexpected(DefResult.error()); - auto SourceResult = Pending->addUse(Source); - if (!SourceResult) - return std::unexpected(SourceResult.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Built; - } - - [[nodiscard]] static Expected - buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, - const TargetDescription &Target, LLT ValueType, Register Destination, - Register Condition, Register TrueValue, Register FalseValue) { - auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth()); - if (!IntegerType) - return std::unexpected(IntegerType.error()); - auto Mask = createTemporary(Function, *IntegerType, Target); - auto Difference = createTemporary(Function, *IntegerType, Target); - auto MaskedDifference = createTemporary(Function, *IntegerType, Target); - if (!Mask || !Difference || !MaskedDifference) { - return std::unexpected( - (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error()))); - } - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - const InstructionOpcode Subtract = - ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB; - auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask, - Register::fromPhysical(*Zero), Condition); - if (!MaskInstruction) - return std::unexpected(MaskInstruction.error()); - auto DifferenceInstruction = buildBinaryAfter( - Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue); - if (!DifferenceInstruction) - return std::unexpected(DifferenceInstruction.error()); - auto MaskInstructionResult = - buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND, - *MaskedDifference, *Difference, *Mask); - if (!MaskInstructionResult) - return std::unexpected(MaskInstructionResult.error()); - return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination, - FalseValue, *MaskedDifference); - } - - [[nodiscard]] static Expected selectDivRem(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type, bool Signed) { - const MachineOperand *RemainderOperand = Instruction.operandAt(1); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const Register *Remainder = - RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V division-with-remainder has invalid operands"); - const Register RemainderRegister = *Remainder; - auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); - auto RightRegister = - normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); - if (!LeftRegister || !RightRegister) - return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); - auto RemoveRemainder = Instruction.eraseOperand(1); - if (!RemoveRemainder) - return std::unexpected(RemoveRemainder.error()); - auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister); - if (!SetLeft) - return std::unexpected(SetLeft.error()); - auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister); - if (!SetRight) - return std::unexpected(SetRight.error()); - const bool IsWord = Type->scalarBitWidth() <= 32; - const InstructionOpcode Division = - IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW) - : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU); - const InstructionOpcode RemainderOpcode = - IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW) - : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU); - auto SetDivision = Instruction.setOpcode(targetOpcode(Division)); - if (!SetDivision) - return std::unexpected(SetDivision.error()); - MachineIRBuilder Builder(Function, Target); - auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister, - *LeftRegister, *RightRegister); - if (!BuiltRemainder) - return std::unexpected(BuiltRemainder.error()); - return {}; - } - - [[nodiscard]] static Expected selectMultiplyHigh(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type, bool Signed) { - if (!Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V high multiplication has no result type"); - if (Type->scalarBitWidth() == 64) { - return Instruction.setOpcode( - targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU)); - } - - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (Destination == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V high multiplication has invalid register operands"); - const Register DestinationRegister = *Destination; - const Register RightRegister = *Right; - auto I64 = LLT::makeInteger(64); - if (!I64) - return std::unexpected(I64.error()); - auto ExtendedLeft = createTemporary(Function, *I64, Target); - auto ExtendedRight = createTemporary(Function, *I64, Target); - auto Product = createTemporary(Function, *I64, Target); - if (!ExtendedLeft || !ExtendedRight || !Product) { - return std::unexpected(!ExtendedLeft - ? ExtendedLeft.error() - : (!ExtendedRight ? ExtendedRight.error() : Product.error())); - } - - auto RemoveRight = Instruction.eraseOperand(2); - if (!RemoveRight) - return std::unexpected(RemoveRight.error()); - MachineIRBuilder Builder(Function, Target); - MachineInstr *Anchor = &Instruction; - if (Signed) { - auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto Immediate = appendImmediate(Instruction, 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, - *ExtendedRight, RightRegister, 0); - if (!ExtendRight) - return std::unexpected(ExtendRight.error()); - Anchor = *ExtendRight; - } else { - auto ShiftedLeft = createTemporary(Function, *I64, Target); - auto ShiftedRight = createTemporary(Function, *I64, Target); - if (!ShiftedLeft || !ShiftedRight) - return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto Immediate = appendImmediate(Instruction, 32); - if (!Immediate) - return std::unexpected(Immediate.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI, - *ExtendedLeft, *ShiftedLeft, 32); - if (!ExtendLeft) - return std::unexpected(ExtendLeft.error()); - auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI, - *ShiftedRight, RightRegister, 32); - if (!ShiftRight) - return std::unexpected(ShiftRight.error()); - auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI, - *ExtendedRight, *ShiftedRight, 32); - if (!ExtendRight) - return std::unexpected(ExtendRight.error()); - Anchor = *ExtendRight; - } - auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product, - *ExtendedLeft, *ExtendedRight); - if (!Multiply) - return std::unexpected(Multiply.error()); - auto Extract = buildImmediateAfter(Builder, **Multiply, - Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI, - DestinationRegister, *Product, 32); - if (!Extract) - return std::unexpected(Extract.error()); - return {}; - } - - [[nodiscard]] static Expected selectIntegerMinMax(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type, - GenericOpcode Generic) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer min/max has invalid operands"); - auto I1 = LLT::makeInteger(1); - if (!I1) - return std::unexpected(I1.error()); - auto Condition = createTemporary(Function, *I1, Target); - if (!Condition) - return std::unexpected(Condition.error()); - const Register DestinationRegister = *Destination; - const Register LeftRegister = *Left; - const Register RightRegister = *Right; - const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX; - auto NormalizedLeft = - normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned); - auto NormalizedRight = - normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned); - if (!NormalizedLeft || !NormalizedRight) - return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error()); - auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition); - if (!SetCondition) - return std::unexpected(SetCondition.error()); - auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); - if (!SetLeft) - return std::unexpected(SetLeft.error()); - auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight); - if (!SetRight) - return std::unexpected(SetRight.error()); - auto SetCompare = Instruction.setOpcode( - targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT)); - if (!SetCompare) - return std::unexpected(SetCompare.error()); - const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX; - const Register TrueValue = IsMaximum ? RightRegister : LeftRegister; - const Register FalseValue = IsMaximum ? LeftRegister : RightRegister; - MachineIRBuilder Builder(Function, Target); - auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, - DestinationRegister, *Condition, TrueValue, FalseValue); - if (!Selected) - return std::unexpected(Selected.error()); - return {}; - } - - [[nodiscard]] static Expected selectValue(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *ConditionOperand = Instruction.operandAt(1); - const MachineOperand *TrueOperand = Instruction.operandAt(2); - const MachineOperand *FalseOperand = Instruction.operandAt(3); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Condition = - ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); - const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf(); - const Register *FalseValue = - FalseOperand == nullptr ? nullptr : FalseOperand->getIf(); - if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr || - FalseValue == nullptr) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V value selection has invalid operands"); - } - const Register DestinationRegister = *Destination; - const Register TrueRegister = *TrueValue; - const Register FalseRegister = *FalseValue; - auto I1 = LLT::makeInteger(1); - if (!I1) - return std::unexpected(I1.error()); - auto NormalizedCondition = createTemporary(Function, *I1, Target); - if (!NormalizedCondition) - return std::unexpected(NormalizedCondition.error()); - auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition); - if (!SetCondition) - return std::unexpected(SetCondition.error()); - auto EraseFalse = Instruction.eraseOperand(3); - if (!EraseFalse) - return std::unexpected(EraseFalse.error()); - auto EraseTrue = Instruction.eraseOperand(2); - if (!EraseTrue) - return std::unexpected(EraseTrue.error()); - auto Mask = appendImmediate(Instruction, 1); - if (!Mask) - return std::unexpected(Mask.error()); - auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI)); - if (!SetNormalize) - return std::unexpected(SetNormalize.error()); - - MachineIRBuilder Builder(Function, Target); - if (!Type->isFloat()) { - auto Selected = - buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, - *NormalizedCondition, TrueRegister, FalseRegister); - if (!Selected) - return std::unexpected(Selected.error()); - return {}; - } - - auto IntegerType = LLT::makeInteger(Type->scalarBitWidth()); - if (!IntegerType) - return std::unexpected(IntegerType.error()); - auto IntegerTrue = createTemporary(Function, *IntegerType, Target); - auto IntegerFalse = createTemporary(Function, *IntegerType, Target); - auto IntegerResult = createTemporary(Function, *IntegerType, Target); - if (!IntegerTrue || !IntegerFalse || !IntegerResult) { - return std::unexpected(!IntegerTrue - ? IntegerTrue.error() - : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error())); - } - const InstructionOpcode ToInteger = - Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; - const InstructionOpcode ToFloat = - Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; - auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister); - if (!MoveTrue) - return std::unexpected(MoveTrue.error()); - auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister); - if (!MoveFalse) - return std::unexpected(MoveFalse.error()); - auto Selected = - buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult, - *NormalizedCondition, *IntegerTrue, *IntegerFalse); - if (!Selected) - return std::unexpected(Selected.error()); - auto MoveResult = - buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult); - if (!MoveResult) - return std::unexpected(MoveResult.error()); - return {}; - } - - [[nodiscard]] static Expected selectAbsoluteValue(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (!Type || Destination == nullptr || Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V absolute value has invalid operands"); - const Register DestinationRegister = *Destination; - const Register SourceRegister = *Source; - auto NormalizedSource = - normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true); - if (!NormalizedSource) - return std::unexpected(NormalizedSource.error()); - auto SignMask = createTemporary(Function, *Type, Target); - auto Flipped = createTemporary(Function, *Type, Target); - if (!SignMask || !Flipped) - return std::unexpected(!SignMask ? SignMask.error() : Flipped.error()); - auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask); - if (!SetMask) - return std::unexpected(SetMask.error()); - auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource); - if (!SetSource) - return std::unexpected(SetSource.error()); - const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64; - auto Shift = appendImmediate(Instruction, ContainerBits - 1U); - if (!Shift) - return std::unexpected(Shift.error()); - const bool IsWord = Type->scalarBitWidth() <= 32; - auto SetShift = Instruction.setOpcode( - targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)); - if (!SetShift) - return std::unexpected(SetShift.error()); - MachineIRBuilder Builder(Function, Target); - auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped, - *NormalizedSource, *SignMask); - if (!Flip) - return std::unexpected(Flip.error()); - auto Result = - buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB, - DestinationRegister, *Flipped, *SignMask); - if (!Result) - return std::unexpected(Result.error()); - return {}; - } - - [[nodiscard]] static Expected> - buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, - MachineFunction &Function, const TargetDescription &Target, LLT Type, - Register Result, Register Left, Register Right, bool IsAdd) { - auto FirstDifference = createTemporary(Function, Type, Target); - auto SecondDifference = createTemporary(Function, Type, Target); - auto SignBits = createTemporary(Function, Type, Target); - auto I1 = LLT::makeInteger(1); - if (!I1) - return std::unexpected(I1.error()); - auto Overflow = createTemporary(Function, *I1, Target); - if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) { - return std::unexpected(!FirstDifference ? FirstDifference.error() - : !SecondDifference ? SecondDifference.error() - : !SignBits ? SignBits.error() - : Overflow.error()); - } - const Register FirstLeft = IsAdd ? Result : Left; - const Register FirstRight = IsAdd ? Left : Right; - const Register SecondLeft = Result; - const Register SecondRight = IsAdd ? Right : Left; - auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference, - FirstLeft, FirstRight); - if (!First) - return std::unexpected(First.error()); - auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, - SecondLeft, SecondRight); - if (!Second) - return std::unexpected(Second.error()); - auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, - *FirstDifference, *SecondDifference); - if (!Both) - return std::unexpected(Both.error()); - const InstructionOpcode Shift = - Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI; - auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits, - Type.scalarBitWidth() - 1U); - if (!Extract) - return std::unexpected(Extract.error()); - return std::pair{*Overflow, *Extract}; - } - - [[nodiscard]] static Expected - selectOverflow(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { - const MachineOperand *ResultOperand = Instruction.operandAt(0); - const MachineOperand *OverflowOperand = Instruction.operandAt(1); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); - const Register *Overflow = - OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V overflow operation has invalid operands"); - } - const Register ResultRegister = *Result; - const Register OverflowRegister = *Overflow; - const Register LeftRegister = *Left; - const Register RightRegister = *Right; - auto RemoveOverflow = Instruction.eraseOperand(1); - if (!RemoveOverflow) - return std::unexpected(RemoveOverflow.error()); - const bool IsWord = Type->scalarBitWidth() <= 32; - const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO; - const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO; - const bool IsUnsigned = Generic == GenericOpcode::G_UADDO || - Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO; - const InstructionOpcode Arithmetic = - IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) - : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB) - : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL); - auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); - if (!SetArithmetic) - return std::unexpected(SetArithmetic.error()); - MachineIRBuilder Builder(Function, Target); - if (IsUnsigned && IsAdd) { - auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister, - ResultRegister, LeftRegister); - if (!Carry) - return std::unexpected(Carry.error()); - return {}; - } - if (IsUnsigned && IsSub) { - auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, - OverflowRegister, LeftRegister, RightRegister); - if (!Borrow) - return std::unexpected(Borrow.error()); - return {}; - } - if (IsAdd || IsSub) { - auto FirstDifference = createTemporary(Function, *Type, Target); - auto SecondDifference = createTemporary(Function, *Type, Target); - auto SignBits = createTemporary(Function, *Type, Target); - if (!FirstDifference || !SecondDifference || !SignBits) { - return std::unexpected( - !FirstDifference ? FirstDifference.error() - : (!SecondDifference ? SecondDifference.error() : SignBits.error())); - } - const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister; - const Register FirstRight = IsAdd ? LeftRegister : RightRegister; - const Register SecondLeft = ResultRegister; - const Register SecondRight = IsAdd ? RightRegister : LeftRegister; - auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference, - FirstLeft, FirstRight); - if (!First) - return std::unexpected(First.error()); - auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, - SecondLeft, SecondRight); - if (!Second) - return std::unexpected(Second.error()); - auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, - *FirstDifference, *SecondDifference); - if (!Both) - return std::unexpected(Both.error()); - auto OverflowResult = buildImmediateAfter( - Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI, - OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U); - if (!OverflowResult) - return std::unexpected(OverflowResult.error()); - return {}; - } - - auto I64 = LLT::makeInteger(64); - if (!I64) - return std::unexpected(I64.error()); - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - if (!IsWord) { - auto High = createTemporary(Function, *I64, Target); - if (!High) - return std::unexpected(High.error()); - auto HighProduct = buildBinaryAfter( - Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH, - *High, LeftRegister, RightRegister); - if (!HighProduct) - return std::unexpected(HighProduct.error()); - if (IsUnsigned) { - auto OverflowResult = - buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister, - Register::fromPhysical(*Zero), *High); - if (!OverflowResult) - return std::unexpected(OverflowResult.error()); - return {}; - } - auto ExpectedHigh = createTemporary(Function, *I64, Target); - auto Difference = createTemporary(Function, *I64, Target); - if (!ExpectedHigh || !Difference) - return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error()); - auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI, - *ExpectedHigh, ResultRegister, 63); - if (!Sign) - return std::unexpected(Sign.error()); - auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High, - *ExpectedHigh); - if (!Compare) - return std::unexpected(Compare.error()); - auto OverflowResult = - buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, - Register::fromPhysical(*Zero), *Difference); - if (!OverflowResult) - return std::unexpected(OverflowResult.error()); - return {}; - } - - auto ExtendedLeft = createTemporary(Function, *I64, Target); - auto ExtendedRight = createTemporary(Function, *I64, Target); - auto FullProduct = createTemporary(Function, *I64, Target); - if (!ExtendedLeft || !ExtendedRight || !FullProduct) { - return std::unexpected(!ExtendedLeft - ? ExtendedLeft.error() - : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error())); - } - MachineInstr *Anchor = &Instruction; - if (IsUnsigned) { - auto ShiftedLeft = createTemporary(Function, *I64, Target); - auto ShiftedRight = createTemporary(Function, *I64, Target); - if (!ShiftedLeft || !ShiftedRight) - return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); - auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft, - LeftRegister, 32); - if (!LeftShift) - return std::unexpected(LeftShift.error()); - auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI, - *ExtendedLeft, *ShiftedLeft, 32); - if (!LeftExtend) - return std::unexpected(LeftExtend.error()); - auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI, - *ShiftedRight, RightRegister, 32); - if (!RightShift) - return std::unexpected(RightShift.error()); - auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI, - *ExtendedRight, *ShiftedRight, 32); - if (!RightExtend) - return std::unexpected(RightExtend.error()); - Anchor = *RightExtend; - } else { - auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, - *ExtendedLeft, LeftRegister, 0); - if (!LeftExtend) - return std::unexpected(LeftExtend.error()); - auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW, - *ExtendedRight, RightRegister, 0); - if (!RightExtend) - return std::unexpected(RightExtend.error()); - Anchor = *RightExtend; - } - auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct, - *ExtendedLeft, *ExtendedRight); - if (!Product) - return std::unexpected(Product.error()); - if (IsUnsigned) { - auto High = createTemporary(Function, *I64, Target); - if (!High) - return std::unexpected(High.error()); - auto ExtractHigh = - buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32); - if (!ExtractHigh) - return std::unexpected(ExtractHigh.error()); - auto OverflowResult = - buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister, - Register::fromPhysical(*Zero), *High); - if (!OverflowResult) - return std::unexpected(OverflowResult.error()); - return {}; - } - auto Difference = createTemporary(Function, *I64, Target); - if (!Difference) - return std::unexpected(Difference.error()); - auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference, - *FullProduct, ResultRegister); - if (!Compare) - return std::unexpected(Compare.error()); - auto OverflowResult = - buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, - Register::fromPhysical(*Zero), *Difference); - if (!OverflowResult) - return std::unexpected(OverflowResult.error()); - return {}; - } - - [[nodiscard]] static Expected selectExtendedOverflow(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type, - GenericOpcode Generic) { - const MachineOperand *ResultOperand = Instruction.operandAt(0); - const MachineOperand *CarryOutOperand = Instruction.operandAt(1); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const MachineOperand *CarryInOperand = Instruction.operandAt(4); - const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); - const Register *CarryOut = - CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - const Register *CarryIn = - CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf(); - if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr || - CarryIn == nullptr) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V extended overflow operation has invalid operands"); - } - const Register ResultRegister = *Result; - const Register CarryOutRegister = *CarryOut; - const Register LeftRegister = *Left; - const Register RightRegister = *Right; - const Register CarryInRegister = *CarryIn; - const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE; - const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE; - const bool IsWord = Type->scalarBitWidth() <= 32; - const InstructionOpcode Arithmetic = - IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) - : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB); - auto Partial = createTemporary(Function, *Type, Target); - auto I1 = LLT::makeInteger(1); - if (!Partial || !I1) - return std::unexpected(!Partial ? Partial.error() : I1.error()); - auto NormalizedCarry = createTemporary(Function, *I1, Target); - if (!NormalizedCarry) - return std::unexpected(NormalizedCarry.error()); - auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial); - if (!SetPartial) - return std::unexpected(SetPartial.error()); - auto RemoveCarryOut = Instruction.eraseOperand(1); - if (!RemoveCarryOut) - return std::unexpected(RemoveCarryOut.error()); - auto RemoveCarryIn = Instruction.eraseOperand(3); - if (!RemoveCarryIn) - return std::unexpected(RemoveCarryIn.error()); - auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); - if (!SetArithmetic) - return std::unexpected(SetArithmetic.error()); - MachineIRBuilder Builder(Function, Target); - auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI, - *NormalizedCarry, CarryInRegister, 1); - if (!Normalize) - return std::unexpected(Normalize.error()); - auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial, - *NormalizedCarry); - if (!Final) - return std::unexpected(Final.error()); - - if (IsUnsigned) { - auto FirstCarry = createTemporary(Function, *I1, Target); - auto SecondCarry = createTemporary(Function, *I1, Target); - if (!FirstCarry || !SecondCarry) - return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error()); - auto First = - buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry, - IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister); - if (!First) - return std::unexpected(First.error()); - auto Second = - buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry, - IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry); - if (!Second) - return std::unexpected(Second.error()); - auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister, - *FirstCarry, *SecondCarry); - if (!Combined) - return std::unexpected(Combined.error()); - return {}; - } - - auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type, - *Partial, LeftRegister, RightRegister, IsAdd); - if (!FirstOverflow) - return std::unexpected(FirstOverflow.error()); - auto SecondOverflow = - buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type, - ResultRegister, *Partial, *NormalizedCarry, IsAdd); - if (!SecondOverflow) - return std::unexpected(SecondOverflow.error()); - auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR, - CarryOutRegister, FirstOverflow->first, SecondOverflow->first); - if (!Combined) - return std::unexpected(Combined.error()); - return {}; - } - - [[nodiscard]] static Expected - selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { - const MachineOperand *AddressOperand = Instruction.operandAt(1); - const MachineOperand *ValueOperand = Instruction.operandAt(2); - const Register *Address = - AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); - if (!Type || Address == nullptr || Value == nullptr) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands"); - Register SelectedValue = *Value; - const bool IsWord = Type->scalarBitWidth() == 32; - if (Generic == GenericOpcode::G_ATOMICRMW_SUB) { - auto Negated = createTemporary(Function, *Type, Target); - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Negated || !Zero) - return std::unexpected(!Negated ? Negated.error() : Zero.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr( - targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)); - if (!Pending) - return std::unexpected(Pending.error()); - auto Def = Pending->addDef(*Negated); - if (!Def) - return std::unexpected(Def.error()); - auto Left = Pending->addUse(Register::fromPhysical(*Zero)); - if (!Left) - return std::unexpected(Left.error()); - auto Right = Pending->addUse(*Value); - if (!Right) - return std::unexpected(Right.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - SelectedValue = *Negated; - } - const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode { - switch (Generic) { - case GenericOpcode::G_ATOMICRMW_XCHG: - return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D; - case GenericOpcode::G_ATOMICRMW_ADD: - case GenericOpcode::G_ATOMICRMW_SUB: - return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D; - case GenericOpcode::G_ATOMICRMW_AND: - return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D; - case GenericOpcode::G_ATOMICRMW_OR: - return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D; - case GenericOpcode::G_ATOMICRMW_XOR: - return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D; - case GenericOpcode::G_ATOMICRMW_MAX: - return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D; - case GenericOpcode::G_ATOMICRMW_MIN: - return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D; - case GenericOpcode::G_ATOMICRMW_UMAX: - return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D; - case GenericOpcode::G_ATOMICRMW_UMIN: - return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D; - default: - return InstructionOpcode::Invalid; - } - }; - const InstructionOpcode Selected = SelectOpcode(); - if (Selected == InstructionOpcode::Invalid) - return makeUnexpected(ErrorCode::InternalError, - "RISC-V atomic RMW selection reached an unhandled opcode"); - auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue); - if (!SetValue) - return std::unexpected(SetValue.error()); - auto SetAddress = Instruction.operandAt(2)->setRegister(*Address); - if (!SetAddress) - return std::unexpected(SetAddress.error()); - return Instruction.setOpcode(targetOpcode(Selected)); - } - - [[nodiscard]] static Expected selectIntegerExtension(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - GenericOpcode Generic, - std::optional Type) { - const std::optional SourceType = operandType(Function, Instruction, 1); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (!Type || !SourceType || Destination == nullptr || Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer extension has invalid register operands"); - const Register DestinationRegister = *Destination; - if (Generic == GenericOpcode::G_ANYEXT) - return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - - const std::uint32_t DestinationBits = Type->scalarBitWidth(); - const std::uint32_t SourceBits = SourceType->scalarBitWidth(); - if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) { - auto ImmediateResult = appendImmediate(Instruction, 0); - if (!ImmediateResult) - return std::unexpected(ImmediateResult.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); - } - - const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64; - const std::int64_t Shift = static_cast(ContainerBits - SourceBits); - const InstructionOpcode ShiftLeft = - ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; - const InstructionOpcode ShiftRight = - Generic == GenericOpcode::G_SEXT - ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) - : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); - auto Temporary = createTemporary(Function, *Type, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ShiftImmediate = appendImmediate(Instruction, Shift); - if (!ShiftImmediate) - return std::unexpected(ShiftImmediate.error()); - - MachineIRBuilder Builder(Function, Target); - auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, - *Temporary, Shift); - if (!Shifted) - return std::unexpected(Shifted.error()); - return {}; - } - - [[nodiscard]] static Expected selectSignExtendInRegister(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - std::optional Width; - if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) { - auto ParsedWidth = signedImmediate(*WidthOperand); - if (ParsedWidth) - Width = *ParsedWidth; - } - if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 || - static_cast(*Width) > Type->scalarBitWidth()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V sign-extension-in-register has invalid operands"); - const Register DestinationRegister = *Destination; - if (static_cast(*Width) == Type->scalarBitWidth()) { - auto EraseWidth = Instruction.eraseOperand(2); - if (!EraseWidth) - return std::unexpected(EraseWidth.error()); - return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - } - - const std::uint32_t TypeBits = Type->scalarBitWidth(); - if (TypeBits == 64 && *Width == 32) { - auto ImmediateResult = replaceImmediate(Instruction, 2, 0); - if (!ImmediateResult) - return std::unexpected(ImmediateResult.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); - } - const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64; - const std::int64_t Shift = static_cast(ContainerBits) - *Width; - const InstructionOpcode ShiftLeft = - ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; - const InstructionOpcode ShiftRight = - ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI; - auto Temporary = createTemporary(Function, *Type, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift); - if (!ShiftImmediate) - return std::unexpected(ShiftImmediate.error()); - - MachineIRBuilder Builder(Function, Target); - auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, - *Temporary, Shift); - if (!Shifted) - return std::unexpected(Shifted.error()); - return {}; - } - - [[nodiscard]] static Expected selectIntegerCompare(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *PredicateMO = Instruction.operandAt(1); - const auto *Predicate = - PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - const std::optional LeftType = operandType(Function, Instruction, 2); - if (Predicate == nullptr || Predicate->integerPredicate() == nullptr || - Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer comparison has invalid operands"); - const IntegerPredicate Condition = *Predicate->integerPredicate(); - const Register DestinationRegister = *Destination; - const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan || - Condition == IntegerPredicate::SignedGreaterEqual || - Condition == IntegerPredicate::SignedLessThan || - Condition == IntegerPredicate::SignedLessEqual; - auto LeftRegister = - normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands); - auto RightRegister = - normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands); - if (!LeftRegister || !RightRegister) - return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); - auto ErasePredicate = Instruction.eraseOperand(1); - if (!ErasePredicate) - return std::unexpected(ErasePredicate.error()); - - const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected { - auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); - if (!SetLeft) - return std::unexpected(SetLeft.error()); - return Instruction.operandAt(2)->setRegister(NewRight); - }; - const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional { - switch (Value) { - case IntegerPredicate::SignedLessThan: - return InstructionOpcode::SLT; - case IntegerPredicate::UnsignedLessThan: - return InstructionOpcode::SLTU; - default: - return std::nullopt; - } - }; - - IntegerPredicate StrictCondition = Condition; - bool Swap = false; - bool Invert = false; - switch (Condition) { - case IntegerPredicate::Equal: - case IntegerPredicate::NotEqual: - break; - case IntegerPredicate::SignedGreaterThan: - StrictCondition = IntegerPredicate::SignedLessThan; - Swap = true; - break; - case IntegerPredicate::SignedGreaterEqual: - StrictCondition = IntegerPredicate::SignedLessThan; - Invert = true; - break; - case IntegerPredicate::SignedLessThan: - StrictCondition = IntegerPredicate::SignedLessThan; - break; - case IntegerPredicate::SignedLessEqual: - StrictCondition = IntegerPredicate::SignedLessThan; - Swap = true; - Invert = true; - break; - case IntegerPredicate::UnsignedGreaterThan: - StrictCondition = IntegerPredicate::UnsignedLessThan; - Swap = true; - break; - case IntegerPredicate::UnsignedGreaterEqual: - StrictCondition = IntegerPredicate::UnsignedLessThan; - Invert = true; - break; - case IntegerPredicate::UnsignedLessThan: - StrictCondition = IntegerPredicate::UnsignedLessThan; - break; - case IntegerPredicate::UnsignedLessEqual: - StrictCondition = IntegerPredicate::UnsignedLessThan; - Swap = true; - Invert = true; - break; - case IntegerPredicate::Count: - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V integer comparison predicate is invalid"); - } - - MachineIRBuilder Builder(Function, Target); - if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) { - auto Temporary = createTemporary(Function, *LeftType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto Set = SetSources(*LeftRegister, *RightRegister); - if (!Set) - return std::unexpected(Set.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - if (Condition == IntegerPredicate::Equal) { - auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU, - DestinationRegister, *Temporary, 1); - if (!Result) - return std::unexpected(Result.error()); - } else { - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Result = - buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister, - Register::fromPhysical(*Zero), *Temporary); - if (!Result) - return std::unexpected(Result.error()); - } - return {}; - } - - const std::optional OpcodeValue = StrictOpcode(StrictCondition); - if (!OpcodeValue) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V integer comparison predicate is invalid"); - if (Invert) { - auto Temporary = createTemporary(Function, *LeftType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - const Register Base = *Temporary; - if (Swap) { - auto Set = SetSources(*RightRegister, *LeftRegister); - if (!Set) - return std::unexpected(Set.error()); - } else { - auto Set = SetSources(*LeftRegister, *RightRegister); - if (!Set) - return std::unexpected(Set.error()); - } - auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, - DestinationRegister, Base, 1); - if (!Result) - return std::unexpected(Result.error()); - return {}; - } - if (Swap) { - auto Set = SetSources(*RightRegister, *LeftRegister); - if (!Set) - return std::unexpected(Set.error()); - } else { - auto Set = SetSources(*LeftRegister, *RightRegister); - if (!Set) - return std::unexpected(Set.error()); - } - return Instruction.setOpcode(targetOpcode(*OpcodeValue)); - } - - [[nodiscard]] static Expected selectFloatingCompare(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *PredicateMO = Instruction.operandAt(1); - const auto *Predicate = - PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(2); - const MachineOperand *RightOperand = Instruction.operandAt(3); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - const std::optional FloatType = operandType(Function, Instruction, 2); - const std::optional ResultType = resultType(Function, Instruction); - if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr || - Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating comparison has invalid operands"); - const FloatingPredicate Condition = *Predicate->floatingPredicate(); - const Register DestinationRegister = *Destination; - const Register LeftRegister = *Left; - const Register RightRegister = *Right; - const InstructionOpcode EqualOpcode = - FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D; - const InstructionOpcode LessOpcode = - FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D; - const InstructionOpcode LessEqualOpcode = - FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D; - - auto ErasePredicate = Instruction.eraseOperand(1); - if (!ErasePredicate) - return std::unexpected(ErasePredicate.error()); - const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def, - Register NewLeft, Register NewRight) -> Expected { - auto SetDestination = Instruction.operandAt(0)->setRegister(Def); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); - if (!SetLeft) - return std::unexpected(SetLeft.error()); - auto SetRight = Instruction.operandAt(2)->setRegister(NewRight); - if (!SetRight) - return std::unexpected(SetRight.error()); - return Instruction.setOpcode(targetOpcode(OpcodeValue)); - }; - const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft, - Register BaseRight) -> Expected { - auto Temporary = createTemporary(Function, *ResultType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight); - if (!Base) - return std::unexpected(Base.error()); - MachineIRBuilder Builder(Function, Target); - auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, - DestinationRegister, *Temporary, 1); - if (!Invert) - return std::unexpected(Invert.error()); - return {}; - }; - const auto BuildSelfAnd = [&](bool Invert) -> Expected { - auto First = createTemporary(Function, *ResultType, Target); - auto Second = createTemporary(Function, *ResultType, Target); - if (!First || !Second) - return std::unexpected((!First ? First.error() : Second.error())); - auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister); - if (!Base) - return std::unexpected(Base.error()); - MachineIRBuilder Builder(Function, Target); - auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second, - RightRegister, RightRegister); - if (!SecondInstruction) - return std::unexpected(SecondInstruction.error()); - Register CombinedDestination = DestinationRegister; - if (Invert) { - auto Temporary = createTemporary(Function, *ResultType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - CombinedDestination = *Temporary; - } - auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND, - CombinedDestination, *First, *Second); - if (!Combined) - return std::unexpected(Combined.error()); - if (!Invert) - return {}; - auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, - DestinationRegister, CombinedDestination, 1); - if (!Inverted) - return std::unexpected(Inverted.error()); - return {}; - }; - const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected { - auto First = createTemporary(Function, *ResultType, Target); - auto Second = createTemporary(Function, *ResultType, Target); - if (!First || !Second) - return std::unexpected((!First ? First.error() : Second.error())); - auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister); - if (!Base) - return std::unexpected(Base.error()); - MachineIRBuilder Builder(Function, Target); - auto SecondInstruction = - buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister); - if (!SecondInstruction) - return std::unexpected(SecondInstruction.error()); - Register CombinedDestination = DestinationRegister; - if (Invert) { - auto Temporary = createTemporary(Function, *ResultType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - CombinedDestination = *Temporary; - } - auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR, - CombinedDestination, *First, *Second); - if (!Combined) - return std::unexpected(Combined.error()); - if (!Invert) - return {}; - auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, - DestinationRegister, CombinedDestination, 1); - if (!Inverted) - return std::unexpected(Inverted.error()); - return {}; - }; - - switch (Condition) { - case FloatingPredicate::AlwaysFalse: - case FloatingPredicate::AlwaysTrue: { - auto EraseRight = Instruction.eraseOperand(2); - if (!EraseRight) - return std::unexpected(EraseRight.error()); - auto EraseLeft = Instruction.eraseOperand(1); - if (!EraseLeft) - return std::unexpected(EraseLeft.error()); - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); - if (!InsertZero) - return std::unexpected(InsertZero.error()); - auto Immediate = - appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI)); - } - case FloatingPredicate::OrderedEqual: - return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister); - case FloatingPredicate::OrderedGreaterThan: - return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister); - case FloatingPredicate::OrderedGreaterEqual: - return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister); - case FloatingPredicate::OrderedLessThan: - return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister); - case FloatingPredicate::OrderedLessEqual: - return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister); - case FloatingPredicate::OrderedNotEqual: - return BuildOrderedNotEqual(false); - case FloatingPredicate::Ordered: - return BuildSelfAnd(false); - case FloatingPredicate::Unordered: - return BuildSelfAnd(true); - case FloatingPredicate::UnorderedEqual: - return BuildOrderedNotEqual(true); - case FloatingPredicate::UnorderedGreaterThan: - return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister); - case FloatingPredicate::UnorderedGreaterEqual: - return BuildInverted(LessOpcode, LeftRegister, RightRegister); - case FloatingPredicate::UnorderedLessThan: - return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister); - case FloatingPredicate::UnorderedLessEqual: - return BuildInverted(LessOpcode, RightRegister, LeftRegister); - case FloatingPredicate::UnorderedNotEqual: - return BuildInverted(EqualOpcode, LeftRegister, RightRegister); - case FloatingPredicate::Count: - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V floating comparison predicate is invalid"); - } - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V floating comparison predicate is invalid"); - } - - [[nodiscard]] static Expected selectFloatToInteger(MachineInstr &Instruction, - MachineFunction &Function, - std::optional Type, bool Unsigned) { - const std::optional Source = operandType(Function, Instruction, 1); - if (!Type || !Source) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating-to-integer conversion has invalid types"); - auto Rounding = appendImmediate(Instruction, 1); - if (!Rounding) - return std::unexpected(Rounding.error()); - const bool IsWord = Type->scalarBitWidth() <= 32; - if (Source->scalarBitWidth() == 32) { - return Instruction.setOpcode(targetOpcode( - IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S) - : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S))); - } - return Instruction.setOpcode(targetOpcode( - IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D) - : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D))); - } - - [[nodiscard]] static Expected selectIntegerToFloat(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type, bool Unsigned) { - const std::optional Source = operandType(Function, Instruction, 1); - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *SourceRegister = - SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); - if (!Type || !Source || SourceRegister == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V integer-to-floating conversion has invalid types"); - auto Normalized = - normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned); - if (!Normalized) - return std::unexpected(Normalized.error()); - auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized); - if (!SetSource) - return std::unexpected(SetSource.error()); - auto Rounding = appendImmediate(Instruction, 0); - if (!Rounding) - return std::unexpected(Rounding.error()); - const bool IsWord = Source->scalarBitWidth() <= 32; - if (Type->scalarBitWidth() == 32) { - return Instruction.setOpcode(targetOpcode( - IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W) - : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L))); - } - return Instruction.setOpcode(targetOpcode( - IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W) - : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L))); - } - - [[nodiscard]] static Expected selectFloatingMinMax(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode Single, - InstructionOpcode Double) { - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) - return Instruction.setOpcode(targetOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) - return Instruction.setOpcode(targetOpcode(Double)); - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V cannot select this floating min/max type"); - } - - [[nodiscard]] static Expected selectFence(MachineInstr &Instruction) { - const auto Ordering = signedImmediate(*Instruction.operandAt(0)); - if (!Ordering) - return std::unexpected(Ordering.error()); - std::int64_t Pred = 0; - std::int64_t Succ = 0; - switch (*Ordering) { - case 4: - Pred = 1; - Succ = 3; - break; - case 5: - Pred = 3; - Succ = 2; - break; - case 6: - case 7: - Pred = 3; - Succ = 3; - break; - default: - return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported"); - } - auto SetPred = replaceImmediate(Instruction, 0, Pred); - if (!SetPred) - return std::unexpected(SetPred.error()); - auto SetSucc = replaceImmediate(Instruction, 1, Succ); - if (!SetSucc) - return std::unexpected(SetSucc.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE)); - } - - [[nodiscard]] static Expected selectIndirectBranch(MachineInstr &Instruction) { - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); - if (!Inserted) - return std::unexpected(Inserted.error()); - auto Immediate = appendImmediate(Instruction, 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR)); - } - - [[nodiscard]] static Expected selectTrap(MachineInstr &Instruction) { - while (Instruction.operandCount() != 0) { - auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); - if (!Erased) - return std::unexpected(Erased.error()); - } - return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK)); - } - - [[nodiscard]] static Expected selectBitcast(MachineInstr &Instruction, - MachineFunction &Function) { - const std::optional DestinationType = resultType(Function, Instruction); - const std::optional SourceType = operandType(Function, Instruction, 1); - if (!DestinationType || !SourceType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V bitcast has invalid register types"); - if (DestinationType->isInteger() && SourceType->isFloat()) { - return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32 - ? InstructionOpcode::FMV_X_W - : InstructionOpcode::FMV_X_D)); - } - if (DestinationType->isFloat() && SourceType->isInteger()) { - return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32 - ? InstructionOpcode::FMV_W_X - : InstructionOpcode::FMV_D_X)); - } - return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported"); - } - - [[nodiscard]] static Expected selectIntegerBinary(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode DoubleWord, - InstructionOpcode Word) { - if (!Type || !isSupportedIntegerType(*Type)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V integer instruction requires a supported scalar integer"); - const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; - return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); - } - - [[nodiscard]] static Expected - selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, std::optional Type, - InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed, - bool NormalizeRight) { - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); - if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V normalized integer operation has invalid operands"); - } - auto NormalizedLeft = - normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); - if (!NormalizedLeft) - return std::unexpected(NormalizedLeft.error()); - Register SelectedRight = *Right; - if (NormalizeRight) { - auto NormalizedRight = - normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); - if (!NormalizedRight) - return std::unexpected(NormalizedRight.error()); - SelectedRight = *NormalizedRight; - } - auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); - if (!SetLeft) - return std::unexpected(SetLeft.error()); - auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight); - if (!SetRight) - return std::unexpected(SetRight.error()); - const bool IsWord = Type->scalarBitWidth() <= 32; - return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); - } - - [[nodiscard]] static Expected selectLoad(MachineInstr &Instruction, - std::optional Type) { - if (!Type) - return makeUnexpected(ErrorCode::VerificationError, - "load result has no virtual-register type"); - const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V load size does not match its result type"); - if (Instruction.memOperands().front()->isAtomic()) { - if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V atomic load requires a 32-bit or 64-bit GPR value"); - return Instruction.setOpcode( - targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D)); - } - auto AppendOffset = appendImmediate(Instruction, 0); - if (!AppendOffset) - return std::unexpected(AppendOffset.error()); - if (isSupportedIntegerType(*Type)) { - const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB - : *Bytes == 2 ? InstructionOpcode::LH - : *Bytes == 4 ? InstructionOpcode::LW - : *Bytes == 8 ? InstructionOpcode::LD - : InstructionOpcode::Invalid; - if (Selected == InstructionOpcode::Invalid) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported"); - return Instruction.setOpcode(targetOpcode(Selected)); - } - if (Type->isFloat() && Type->scalarBitWidth() == 32) - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); - if (Type->isFloat() && Type->scalarBitWidth() == 64) - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD)); - return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type"); - } - - [[nodiscard]] static Expected selectExtLoad(MachineInstr &Instruction, - std::optional Type, bool Signed) { - if (!Type || !isSupportedIntegerType(*Type)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V extension load requires an integer result"); - const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width"); - const InstructionOpcode Selected = - *Bytes == 1 ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU) - : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU) - : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU) - : InstructionOpcode::Invalid; - if (Selected == InstructionOpcode::Invalid) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported"); - auto Immediate = appendImmediate(Instruction, 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - return Instruction.setOpcode(targetOpcode(Selected)); - } - - [[nodiscard]] static Expected selectFloatExtLoad(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Destination == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating extension load has no destination register"); - const Register DestinationRegister = *Destination; - auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); - if (!F32) - return std::unexpected(F32.error()); - auto Temporary = createTemporary(Function, *F32, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto Immediate = appendImmediate(Instruction, 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - MachineIRBuilder Builder(Function, Target); - auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S, - DestinationRegister, *Temporary, 0); - if (!Conversion) - return std::unexpected(Conversion.error()); - return {}; - } - - [[nodiscard]] static Expected selectFloatTruncStore(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *ValueOperand = Instruction.operandAt(0); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); - if (Value == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating truncation store has no value register"); - const Register ValueRegister = *Value; - auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); - if (!F32) - return std::unexpected(F32.error()); - auto Temporary = createTemporary(Function, *F32, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D)); - if (!Conversion) - return std::unexpected(Conversion.error()); - auto Def = Conversion->addDef(*Temporary); - if (!Def) - return std::unexpected(Def.error()); - auto Use = Conversion->addUse(ValueRegister); - if (!Use) - return std::unexpected(Use.error()); - auto Rounding = Conversion->addImmediate(0); - if (!Rounding) - return std::unexpected(Rounding.error()); - auto Built = Conversion->build(); - if (!Built) - return std::unexpected(Built.error()); - auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetValue) - return std::unexpected(SetValue.error()); - auto Immediate = appendImmediate(Instruction, 0); - if (!Immediate) - return std::unexpected(Immediate.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); - } - - [[nodiscard]] static Expected selectStore(MachineInstr &Instruction, - MachineFunction &Function) { - const MachineOperand *Value = Instruction.operandAt(0); - const std::optional Type = - Value == nullptr ? std::nullopt : registerType(Function, *Value); - if (!Type) - return makeUnexpected(ErrorCode::VerificationError, - "store value has no virtual-register type"); - const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V store size does not match its value type"); - if (Instruction.memOperands().front()->isAtomic()) { - if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V atomic store requires a 32-bit or 64-bit GPR value"); - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); - if (!Destination) - return std::unexpected(Destination.error()); - return Instruction.setOpcode( - targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D)); - } - auto AppendOffset = appendImmediate(Instruction, 0); - if (!AppendOffset) - return std::unexpected(AppendOffset.error()); - if (isSupportedIntegerType(*Type)) { - const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB - : *Bytes == 2 ? InstructionOpcode::SH - : *Bytes == 4 ? InstructionOpcode::SW - : *Bytes == 8 ? InstructionOpcode::SD - : InstructionOpcode::Invalid; - if (Selected == InstructionOpcode::Invalid) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported"); - return Instruction.setOpcode(targetOpcode(Selected)); - } - if (Type->isFloat() && Type->scalarBitWidth() == 32) - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); - if (Type->isFloat() && Type->scalarBitWidth() == 64) - return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD)); - return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type"); - } - - [[nodiscard]] static Expected selectBranch(MachineInstr &Instruction) { - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); - if (!Inserted) - return std::unexpected(Inserted.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL)); - } - - [[nodiscard]] static Expected selectConditionalBranch(MachineInstr &Instruction) { - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); - if (!Inserted) - return std::unexpected(Inserted.error()); - return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE)); - } - - [[nodiscard]] static Expected selectIntegerConstant(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type) { - if (!Type || !isSupportedIntegerType(*Type)) { - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V constant selection requires an i1-i64 or pointer result"); - } - const MachineOperand *ImmediateOperand = Instruction.operandAt(1); - if (ImmediateOperand == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V generic constant has no immediate operand"); - auto ImmediateValue = signedImmediate(*ImmediateOperand); - if (!ImmediateValue) - return std::unexpected(ImmediateValue.error()); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *DestinationValue = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (DestinationValue == nullptr || !DestinationValue->isVirtual()) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V generic constant must define a virtual register"); - } - const Register Destination = *DestinationValue; - const std::int64_t Value = *ImmediateValue; - const std::uint32_t ImmediateTargetFlags = ImmediateOperand->targetFlags(); - const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; - const InstructionOpcode AddImmediate = - IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI; - const InstructionOpcode ShiftLeftImmediate = - IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; - if (fitsSigned(Value, 12)) { - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); - if (!Source) - return std::unexpected(Source.error()); - auto EraseImmediate = Instruction.eraseOperand(2); - if (!EraseImmediate) - return std::unexpected(EraseImmediate.error()); - auto NormalizeImmediate = Instruction.insertOperand( - 2, MachineOperand::immediate(Value, ImmediateTargetFlags)); - if (!NormalizeImmediate) - return std::unexpected(NormalizeImmediate.error()); - return Instruction.setOpcode(targetOpcode(AddImmediate)); - } - - // Recursively decompose the value into signed 12-bit chunks. The - // quotient is computed from an unsigned bit pattern with explicit sign - // extension so both INT64_MIN and INT64_MAX remain well-defined. - std::uint64_t Remaining = static_cast(Value); - std::vector Digits; - while (!fitsSigned(std::bit_cast(Remaining), 12)) { - const std::uint64_t LowBits = Remaining & 0xFFFU; - std::int64_t Digit = static_cast(LowBits); - if (Digit >= 0x800) - Digit -= 0x1000; - Digits.push_back(Digit); - - const std::uint64_t Adjusted = Remaining - static_cast(Digit); - std::uint64_t Quotient = Adjusted >> 12U; - if ((Remaining & (std::uint64_t{1} << 63U)) != 0) - Quotient |= ~std::uint64_t{0} << 52U; - Remaining = Quotient; - } - const std::int64_t Base = std::bit_cast(Remaining); - - auto Temporary = createTemporary(Function, *Type, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto *Parent = Instruction.parent(); - if (Parent == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V constant selection received a detached instruction"); - auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto EraseImmediate = Instruction.eraseOperand(1); - if (!EraseImmediate) - return std::unexpected(EraseImmediate.error()); - auto Zero = physicalRegister(PhysicalRegisterID::X0); - if (!Zero) - return std::unexpected(Zero.error()); - auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); - if (!AddSource) - return std::unexpected(AddSource.error()); - auto AddBase = appendImmediate(Instruction, Base); - if (!AddBase) - return std::unexpected(AddBase.error()); - - MachineIRBuilder Builder(Function, Target); - auto InsertPoint = Builder.setInsertPointAfter(Instruction); - if (!InsertPoint) - return std::unexpected(InsertPoint.error()); - - Register Current = *Temporary; - for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) { - const std::size_t Index = Reverse - 1U; - auto ShiftResult = createTemporary(Function, *Type, Target); - if (!ShiftResult) - return std::unexpected(ShiftResult.error()); - auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate)); - if (!Shift) - return std::unexpected(Shift.error()); - auto ShiftDef = Shift->addDef(*ShiftResult); - if (!ShiftDef) - return std::unexpected(ShiftDef.error()); - auto ShiftUse = Shift->addUse(Current); - if (!ShiftUse) - return std::unexpected(ShiftUse.error()); - auto ShiftAmount = Shift->addImmediate(12); - if (!ShiftAmount) - return std::unexpected(ShiftAmount.error()); - auto BuiltShift = Shift->build(); - if (!BuiltShift) - return std::unexpected(BuiltShift.error()); - - const bool IsFinal = Index == 0; - Register AddDestination; - if (IsFinal) { - AddDestination = Destination; - } else { - auto AddTemporary = createTemporary(Function, *Type, Target); - if (!AddTemporary) - return std::unexpected(AddTemporary.error()); - AddDestination = *AddTemporary; - } - auto Add = Builder.setInsertPointAfter(**BuiltShift); - if (!Add) - return std::unexpected(Add.error()); - auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate)); - if (!AddInstruction) - return std::unexpected(AddInstruction.error()); - auto AddDef = AddInstruction->addDef(AddDestination); - if (!AddDef) - return std::unexpected(AddDef.error()); - auto AddUse = AddInstruction->addUse(*ShiftResult); - if (!AddUse) - return std::unexpected(AddUse.error()); - auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]); - if (!AddImmediateResult) - return std::unexpected(AddImmediateResult.error()); - auto BuiltAdd = AddInstruction->build(); - if (!BuiltAdd) - return std::unexpected(BuiltAdd.error()); - Current = AddDestination; - } - return {}; - } - - [[nodiscard]] static Expected selectFloatingConstant(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - std::optional Type) { - if (!Type || !isRVFloatType(*Type)) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64"); - const MachineOperand *Immediate = Instruction.operandAt(1); - const auto *Floating = - Immediate == nullptr ? nullptr : Immediate->getIf(); - const std::optional Semantics = Type->floatSemantics(); - if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating constant has mismatched bit semantics"); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); - if (Destination == nullptr || !Destination->isVirtual()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating constant must define a virtual register"); - const Register DestinationRegister = *Destination; - - const std::uint32_t BitWidth = Type->scalarBitWidth(); - const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front(); - auto Integer = IntegerImmediate::fromUnsigned(64, Bits); - if (!Integer) - return std::unexpected(Integer.error()); - auto IntegerType = LLT::makeInteger(64); - if (!IntegerType) - return std::unexpected(IntegerType.error()); - auto Temporary = createTemporary(Function, *IntegerType, Target); - if (!Temporary) - return std::unexpected(Temporary.error()); - - auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); - if (!SetDestination) - return std::unexpected(SetDestination.error()); - auto EraseImmediate = Instruction.eraseOperand(1); - if (!EraseImmediate) - return std::unexpected(EraseImmediate.error()); - auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer)); - if (!AddInteger) - return std::unexpected(AddInteger.error()); - auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType); - if (!IntegerSelection) - return std::unexpected(IntegerSelection.error()); - - MachineInstr *IntegerDefinition = nullptr; - for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) { - IntegerDefinition = DefOperand.parent(); - break; - } - if (IntegerDefinition == nullptr) - return makeUnexpected(ErrorCode::InternalError, - "RISC-V floating constant lost its integer materialization"); - MachineIRBuilder Builder(Function, Target); - auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition); - if (!InsertPoint) - return std::unexpected(InsertPoint.error()); - const InstructionOpcode Move = - BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; - auto MoveInstruction = Builder.buildInstr(targetOpcode(Move)); - if (!MoveInstruction) - return std::unexpected(MoveInstruction.error()); - auto MoveDef = MoveInstruction->addDef(DestinationRegister); - if (!MoveDef) - return std::unexpected(MoveDef.error()); - auto MoveUse = MoveInstruction->addUse(*Temporary); - if (!MoveUse) - return std::unexpected(MoveUse.error()); - auto BuiltMove = MoveInstruction->build(); - if (!BuiltMove) - return std::unexpected(BuiltMove.error()); - return {}; - } - - [[nodiscard]] static Expected selectRoundingBinary(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode Single, - InstructionOpcode Double) { - auto RoundingMode = appendImmediate(Instruction, 0); - if (!RoundingMode) - return std::unexpected(RoundingMode.error()); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) - return Instruction.setOpcode(targetOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) - return Instruction.setOpcode(targetOpcode(Double)); - return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type"); - } - - [[nodiscard]] static Expected selectRoundingTernary(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode Single, - InstructionOpcode Double) { - return selectRoundingBinary(Instruction, Type, Single, Double); - } - - [[nodiscard]] static Expected selectRoundingUnary(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode Single, - InstructionOpcode Double) { - return selectRoundingBinary(Instruction, Type, Single, Double); - } - - [[nodiscard]] static Expected selectSignInjection(MachineInstr &Instruction, - std::optional Type, - InstructionOpcode Single, - InstructionOpcode Double) { - const MachineOperand *Source = Instruction.operandAt(1); - const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf(); - if (SourceReg == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V floating sign operation requires a register source"); - auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg); - if (!Duplicate) - return std::unexpected(Duplicate.error()); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) - return Instruction.setOpcode(targetOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) - return Instruction.setOpcode(targetOpcode(Double)); - return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type"); - } -}; - -} // namespace - -const LegalizerInfo &legalizerInfo() noexcept { - static const RISCVLegalizerInfo Value(RISCVSubtargetFeatures::rv64gc()); - return Value; -} - -std::unique_ptr createLegalizerInfo(RISCVSubtargetFeatures Features) { - return std::make_unique(Features); -} - -const RegisterBankInfo ®isterBankInfo() noexcept { - static const RISCVRegisterBankInfo Value; - return Value; -} - -const InstructionSelector &instructionSelector() noexcept { - static const RISCVInstructionSelector Value; - return Value; -} - -} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp index 513788f..108f6da 100644 --- a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "GISel/RISCVCallLowering.h" +#include "RISCV.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/InstructionDescriptor.h" #include "rcc/CodeGen/MachineBasicBlock.h" @@ -13,8 +15,6 @@ #include "rcc/CodeGen/RegisterMask.h" #include "rcc/CodeGen/RegisterScavenger.h" #include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVCallLowering.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/lib/Target/RISCV/RISCVFunctionInfo.cpp b/backend/lib/Target/RISCV/RISCVFunctionInfo.cpp index f25bb8c..bd72103 100644 --- a/backend/lib/Target/RISCV/RISCVFunctionInfo.cpp +++ b/backend/lib/Target/RISCV/RISCVFunctionInfo.cpp @@ -21,24 +21,22 @@ namespace { [[nodiscard]] Expected parseSignedField(const MIRTargetNode &Node, std::string_view Name) { if (Node.Kind != MIRTargetNodeKind::Scalar) - return makeUnexpected(ErrorCode::ParseError, - "RISC-V machineFunctionInfo field '" + std::string(Name) + - "' must be a scalar"); + return makeUnexpected(ErrorCode::ParseError, "RISC-V machineFunctionInfo field '" + + std::string(Name) + "' must be a scalar"); std::int32_t Value = 0; const auto Result = std::from_chars(Node.Scalar.data(), Node.Scalar.data() + Node.Scalar.size(), Value, 10); if (Result.ec != std::errc{} || Result.ptr != Node.Scalar.data() + Node.Scalar.size()) { - return makeUnexpected(ErrorCode::ParseError, - "RISC-V machineFunctionInfo field '" + std::string(Name) + - "' must be a 32-bit signed integer"); + return makeUnexpected(ErrorCode::ParseError, "RISC-V machineFunctionInfo field '" + + std::string(Name) + + "' must be a 32-bit signed integer"); } return Value; } [[nodiscard]] Expected parseFunctionInfo(const MIRTargetNode &Node) { if (Node.Kind != MIRTargetNodeKind::Mapping) - return makeUnexpected(ErrorCode::ParseError, - "RISC-V machineFunctionInfo must be a mapping"); + return makeUnexpected(ErrorCode::ParseError, "RISC-V machineFunctionInfo must be a mapping"); std::int32_t VarArgsFrameIndex = 0; std::int32_t VarArgsSaveSize = 0; @@ -60,21 +58,19 @@ namespace { return makeUnexpected(ErrorCode::ParseError, "unknown RISC-V machineFunctionInfo field '" + Entry.Key + "'"); } - return TargetFunctionState::create( - RISCVFunctionInfo(VarArgsFrameIndex, VarArgsSaveSize)); + return TargetFunctionState::create(RISCVFunctionInfo(VarArgsFrameIndex, VarArgsSaveSize)); } -[[nodiscard]] Expected -printFunctionInfo(const TargetFunctionState &State) { +[[nodiscard]] Expected printFunctionInfo(const TargetFunctionState &State) { const auto *Info = State.getIf(); if (Info == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "target function state is not RISC-V function information"); std::vector Entries; - Entries.push_back({"varArgsFrameIndex", - MIRTargetNode::scalar(std::to_string(Info->varArgsFrameIndex()))}); - Entries.push_back({"varArgsSaveSize", - MIRTargetNode::scalar(std::to_string(Info->varArgsSaveSize()))}); + Entries.push_back( + {"varArgsFrameIndex", MIRTargetNode::scalar(std::to_string(Info->varArgsFrameIndex()))}); + Entries.push_back( + {"varArgsSaveSize", MIRTargetNode::scalar(std::to_string(Info->varArgsSaveSize()))}); return MIRTargetNode::mapping(std::move(Entries)); } @@ -99,8 +95,7 @@ Expected RISCVFunctionInfo::verify() const { return {}; } -Expected -RISCVFunctionInfo::clone(const MachineFunctionCloneContext &) const { +Expected RISCVFunctionInfo::clone(const MachineFunctionCloneContext &) const { return RISCVFunctionInfo(VarArgsFrameIndex, VarArgsSaveSize); } @@ -117,11 +112,10 @@ Expected> RISCVFunctionInfo::encode() const { return Payload; } -Expected -RISCVFunctionInfo::decode(std::uint32_t Version, std::span Payload) { +Expected RISCVFunctionInfo::decode(std::uint32_t Version, + std::span Payload) { if (Version != schemaVersion()) - return makeUnexpected(ErrorCode::ParseError, - "unsupported RISC-V function-info schema version"); + return makeUnexpected(ErrorCode::ParseError, "unsupported RISC-V function-info schema version"); if (Payload.size() != 8) return makeUnexpected(ErrorCode::ParseError, "RISC-V function-info payload must contain eight bytes"); diff --git a/backend/lib/Target/RISCV/RISCVImmediateMaterialization.cpp b/backend/lib/Target/RISCV/RISCVImmediateMaterialization.cpp index 56bce25..d25a609 100644 --- a/backend/lib/Target/RISCV/RISCVImmediateMaterialization.cpp +++ b/backend/lib/Target/RISCV/RISCVImmediateMaterialization.cpp @@ -18,8 +18,7 @@ namespace { return Value >= Minimum && Value <= Maximum; } -[[nodiscard]] constexpr std::int64_t signExtend(std::uint64_t Value, - std::uint32_t Bits) noexcept { +[[nodiscard]] constexpr std::int64_t signExtend(std::uint64_t Value, std::uint32_t Bits) noexcept { const std::uint64_t Mask = (std::uint64_t{1} << Bits) - 1U; const std::uint64_t Sign = std::uint64_t{1} << (Bits - 1U); Value &= Mask; @@ -59,13 +58,11 @@ namespace { static_cast(Low12 != 0); } -[[nodiscard]] std::optional -leadingZeroBuildCost(std::uint64_t Value) noexcept { +[[nodiscard]] std::optional leadingZeroBuildCost(std::uint64_t Value) noexcept { if (Value == 0 || (Value & (std::uint64_t{1} << 63U)) != 0) return std::nullopt; - const std::uint32_t LeadingZeros = - static_cast(std::countl_zero(Value)); + const std::uint32_t LeadingZeros = static_cast(std::countl_zero(Value)); const std::uint64_t Shifted = Value << LeadingZeros; const std::uint64_t LowMask = (std::uint64_t{1} << LeadingZeros) - 1U; std::optional Best; @@ -79,8 +76,7 @@ leadingZeroBuildCost(std::uint64_t Value) noexcept { return Best; } -[[nodiscard]] std::optional -twoRegisterBuildCost(std::int64_t Value) noexcept { +[[nodiscard]] std::optional twoRegisterBuildCost(std::int64_t Value) noexcept { const std::uint64_t Bits = static_cast(Value); const std::int64_t Low = signExtend(Bits, 32); if (Low == 0) @@ -91,8 +87,7 @@ twoRegisterBuildCost(std::int64_t Value) noexcept { return std::nullopt; const std::uint32_t LowZeros = static_cast(std::countr_zero(static_cast(Low))); - const std::uint32_t HighZeros = - static_cast(std::countr_zero(HighContribution)); + const std::uint32_t HighZeros = static_cast(std::countr_zero(HighContribution)); if (LowZeros >= 32 || HighZeros < 32 || HighZeros < LowZeros) return std::nullopt; const std::uint32_t Shift = HighZeros - LowZeros; @@ -108,8 +103,7 @@ std::size_t estimateRV64IntegerBuildCost(std::int64_t Value) noexcept { std::size_t Best = baseBuildCost(Value); if ((Bits & 0xFFFU) != 0 && (Bits & 1U) == 0 && Best >= 2) { - const std::uint32_t TrailingZeros = - static_cast(std::countr_zero(Bits)); + const std::uint32_t TrailingZeros = static_cast(std::countr_zero(Bits)); const std::int64_t Shifted = Value >> TrailingZeros; const std::size_t Candidate = baseBuildCost(Shifted) + 1U; if (Candidate < Best || fitsSigned(Shifted, 6)) @@ -119,8 +113,7 @@ std::size_t estimateRV64IntegerBuildCost(std::int64_t Value) noexcept { return Best; if ((Bits & 0xFFFU) != 0 && (Bits & 0x1800U) == 0x1000U) { - const std::int64_t LastAdd = - -static_cast(0x800U - (Bits & 0xFFFU)); + const std::int64_t LastAdd = -static_cast(0x800U - (Bits & 0xFFFU)); const std::uint64_t AdjustedBits = Bits - static_cast(LastAdd); Best = std::min(Best, baseBuildCost(std::bit_cast(AdjustedBits)) + 1U); } @@ -129,16 +122,14 @@ std::size_t estimateRV64IntegerBuildCost(std::int64_t Value) noexcept { if (const auto Candidate = leadingZeroBuildCost(Bits); Candidate && *Candidate < Best) Best = *Candidate; } else if (Value < 0 && Best > 3) { - if (const auto Candidate = leadingZeroBuildCost(~Bits); - Candidate && *Candidate + 1U < Best) { + if (const auto Candidate = leadingZeroBuildCost(~Bits); Candidate && *Candidate + 1U < Best) { Best = *Candidate + 1U; } } return Best; } -bool shouldUseRV64IntegerConstantPool(std::int64_t Value, - std::size_t MaximumBuildCost) noexcept { +bool shouldUseRV64IntegerConstantPool(std::int64_t Value, std::size_t MaximumBuildCost) noexcept { if (fitsSigned(Value, 32) || estimateRV64IntegerBuildCost(Value) <= MaximumBuildCost) return false; const auto TwoRegisterCost = twoRegisterBuildCost(Value); diff --git a/backend/lib/Target/RISCV/RISCVImmediateMaterialization.h b/backend/lib/Target/RISCV/RISCVImmediateMaterialization.h index 99d3876..e38f752 100644 --- a/backend/lib/Target/RISCV/RISCVImmediateMaterialization.h +++ b/backend/lib/Target/RISCV/RISCVImmediateMaterialization.h @@ -12,10 +12,8 @@ inline constexpr std::size_t DefaultRV64IntegerBuildCost = 5; [[nodiscard]] std::size_t estimateRV64IntegerBuildCost(std::int64_t Value) noexcept; -[[nodiscard]] bool -shouldUseRV64IntegerConstantPool( - std::int64_t Value, - std::size_t MaximumBuildCost = DefaultRV64IntegerBuildCost) noexcept; +[[nodiscard]] bool shouldUseRV64IntegerConstantPool( + std::int64_t Value, std::size_t MaximumBuildCost = DefaultRV64IntegerBuildCost) noexcept; } // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp index 5c22dde..f41e77a 100644 --- a/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp +++ b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "RISCV.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include diff --git a/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp index 2cc5f24..ef72fe1 100644 --- a/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp +++ b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/MachineFunction.h" #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include #include diff --git a/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp index 9301757..5bacc9c 100644 --- a/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp +++ b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "RISCVInstrInfo.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" namespace rcc::riscv { diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp index 7c778ea..5212838 100644 --- a/backend/lib/Target/RISCV/RISCVPipeline.cpp +++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp @@ -2,6 +2,8 @@ #include "rcc/Target/RISCV/RISCVPipeline.h" +#include "GISel/RISCVRegisterBankInfo.h" +#include "RISCV.h" #include "RISCVInstrInfo.h" #include "rcc/CodeGen/GlobalISel/InstructionSelect.h" #include "rcc/CodeGen/GlobalISel/Legalizer.h" @@ -12,7 +14,6 @@ #include "rcc/CodeGen/MachineSSAExit.h" #include "rcc/CodeGen/RegAllocPBQP.h" #include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include #include @@ -30,18 +31,18 @@ template } [[nodiscard]] Expected addInitialPipeline(MachinePassManager &Pipeline, - RISCVSubtargetFeatures Features) { + const RISCVSubtarget &Subtarget) { auto Added = addPass(Pipeline, machineCombinerInfo(), MachineCombinerStage::PreLegalizer); if (!Added) return std::unexpected(Added.error()); - Added = addPass(Pipeline, createLegalizerInfo(Features)); + Added = addPass(Pipeline, Subtarget.getLegalizerInfo()); if (!Added) return std::unexpected(Added.error()); - Added = addPass(Pipeline, registerBankInfo()); + Added = addPass(Pipeline, Subtarget.getRegBankInfo()); if (!Added) return std::unexpected(Added.error()); - return addPass(Pipeline, instructionSelector()); + return addPass(Pipeline, Subtarget.getInstructionSelector()); } [[nodiscard]] Expected addSSAExitPipeline(MachinePassManager &Pipeline) { @@ -62,8 +63,8 @@ template } [[nodiscard]] Expected addCompleteRegistry(MachinePassManager &Pipeline, - RISCVSubtargetFeatures Features) { - auto Added = addInitialPipeline(Pipeline, Features); + const RISCVSubtarget &Subtarget) { + auto Added = addInitialPipeline(Pipeline, Subtarget); if (!Added) return std::unexpected(Added.error()); Added = addSSAExitPipeline(Pipeline); @@ -77,28 +78,24 @@ template } // namespace -Expected addCanonicalPipeline(MachinePassManager &Pipeline, const MachineFunction &Function) { - return addCanonicalPipeline(Pipeline, Function, RISCVSubtargetFeatures::rv64gc()); -} - Expected addCanonicalPipeline(MachinePassManager &Pipeline, const MachineFunction &Function, - RISCVSubtargetFeatures Features) { + const RISCVSubtarget &Subtarget) { if (!Function.properties().has(MachineFunctionProperty::Legalized)) { auto Added = addPass(Pipeline, machineCombinerInfo(), MachineCombinerStage::PreLegalizer); if (!Added) return std::unexpected(Added.error()); - Added = addPass(Pipeline, createLegalizerInfo(Features)); + Added = addPass(Pipeline, Subtarget.getLegalizerInfo()); if (!Added) return std::unexpected(Added.error()); } if (!Function.properties().has(MachineFunctionProperty::RegBankSelected)) { - auto Added = addPass(Pipeline, registerBankInfo()); + auto Added = addPass(Pipeline, Subtarget.getRegBankInfo()); if (!Added) return std::unexpected(Added.error()); } if (!Function.properties().has(MachineFunctionProperty::Selected)) { - auto Added = addPass(Pipeline, instructionSelector()); + auto Added = addPass(Pipeline, Subtarget.getInstructionSelector()); if (!Added) return std::unexpected(Added.error()); } @@ -130,16 +127,12 @@ Expected addCanonicalPipeline(MachinePassManager &Pipeline, const MachineF return {}; } -Expected> createCanonicalPipelinePass(std::string_view Name) { - return createCanonicalPipelinePass(Name, RISCVSubtargetFeatures::rv64gc()); -} - Expected> -createCanonicalPipelinePass(std::string_view Name, RISCVSubtargetFeatures Features) { +createCanonicalPipelinePass(std::string_view Name, const RISCVSubtarget &Subtarget) { if (Name.empty()) return makeUnexpected(ErrorCode::InvalidArgument, "pipeline pass name cannot be empty"); MachinePassManager Registry; - auto Added = addCompleteRegistry(Registry, Features); + auto Added = addCompleteRegistry(Registry, Subtarget); if (!Added) return std::unexpected(Added.error()); MachinePassManager::PassList Passes = std::move(Registry).takePasses(); @@ -153,9 +146,10 @@ createCanonicalPipelinePass(std::string_view Name, RISCVSubtargetFeatures Featur return std::move(*Position); } -Expected> canonicalPipelinePassNames(const MachineFunction &Function) { +Expected> canonicalPipelinePassNames(const MachineFunction &Function, + const RISCVSubtarget &Subtarget) { MachinePassManager Pipeline; - auto Added = addCanonicalPipeline(Pipeline, Function); + auto Added = addCanonicalPipeline(Pipeline, Function, Subtarget); if (!Added) return std::unexpected(Added.error()); MachinePassManager::PassList Passes = std::move(Pipeline).takePasses(); diff --git a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp index d0de28e..1f3564c 100644 --- a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp +++ b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" @@ -9,7 +10,6 @@ #include "rcc/CodeGen/MachineOperand.h" #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/TargetDescription.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp index d14e802..de5900d 100644 --- a/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp +++ b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/InstructionDescriptor.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" @@ -7,7 +8,6 @@ #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/MachineMemOperand.h" #include "rcc/CodeGen/MachineOperand.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/lib/Target/RISCV/RISCVScheduling.cpp b/backend/lib/Target/RISCV/RISCVScheduling.cpp index 8ef8d91..90b6620 100644 --- a/backend/lib/Target/RISCV/RISCVScheduling.cpp +++ b/backend/lib/Target/RISCV/RISCVScheduling.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "RISCV.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include diff --git a/backend/lib/Target/RISCV/RISCVSubtarget.cpp b/backend/lib/Target/RISCV/RISCVSubtarget.cpp index 3a3d9e2..89f6c61 100644 --- a/backend/lib/Target/RISCV/RISCVSubtarget.cpp +++ b/backend/lib/Target/RISCV/RISCVSubtarget.cpp @@ -2,6 +2,12 @@ #include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "GISel/RISCVCallLowering.h" +#include "GISel/RISCVLegalizerInfo.h" +#include "GISel/RISCVRegisterBankInfo.h" +#include "RISCV.h" +#include "rcc/CodeGen/GlobalISel/InstructionSelector.h" + #include #include #include @@ -91,4 +97,32 @@ bool RISCVSubtargetFeatures::has(RISCVExtension Extension) const noexcept { return (Enabled & extensionBit(Extension)) != 0; } +RISCVSubtarget::RISCVSubtarget(RISCVSubtargetFeatures Features) : Features(Features) {} + +RISCVSubtarget::~RISCVSubtarget() = default; + +const TargetCallLowering *RISCVSubtarget::getCallLowering() const { + if (!CallLoweringInfo) + CallLoweringInfo = std::make_unique(); + return CallLoweringInfo.get(); +} + +InstructionSelector *RISCVSubtarget::getInstructionSelector() const { + if (!InstSelector) + InstSelector.reset(createRISCVInstructionSelector(*this, *getRegBankInfo())); + return InstSelector.get(); +} + +const LegalizerInfo *RISCVSubtarget::getLegalizerInfo() const { + if (!Legalizer) + Legalizer = std::make_unique(*this); + return Legalizer.get(); +} + +const RISCVRegisterBankInfo *RISCVSubtarget::getRegBankInfo() const { + if (!RegBankInfo) + RegBankInfo = std::make_unique(); + return RegBankInfo.get(); +} + } // namespace rcc::riscv diff --git a/backend/lib/Tooling/MIRFuzzer.cpp b/backend/lib/Tooling/MIRFuzzer.cpp index 653b9fd..e75e9f9 100644 --- a/backend/lib/Tooling/MIRFuzzer.cpp +++ b/backend/lib/Tooling/MIRFuzzer.cpp @@ -290,7 +290,11 @@ private: const MachineFunction *Function = Parsed->Document.unit().functionAt(0); if (Function == nullptr) return std::unexpected(Error(ErrorCode::InternalError, "pipeline fuzz input has no function")); - auto Names = riscv::canonicalPipelinePassNames(*Function); + auto Features = riscv::RISCVSubtargetFeatures::fromFeatureList(Invocation.Features); + if (!Features) + return std::unexpected(Features.error()); + const riscv::RISCVSubtarget Subtarget(*Features); + auto Names = riscv::canonicalPipelinePassNames(*Function, Subtarget); if (!Names) return std::unexpected(Names.error()); PipelineControls Controls; diff --git a/backend/lib/Tooling/PipelineDriver.cpp b/backend/lib/Tooling/PipelineDriver.cpp index 3d137d8..495842a 100644 --- a/backend/lib/Tooling/PipelineDriver.cpp +++ b/backend/lib/Tooling/PipelineDriver.cpp @@ -81,10 +81,10 @@ struct FunctionRunResult final { [[nodiscard]] Expected createPasses(const MachineFunction &Function, const BackendInvocation &Invocation, - riscv::RISCVSubtargetFeatures Features, MachinePassManagerOptions Options) { + const riscv::RISCVSubtarget &Subtarget, MachinePassManagerOptions Options) { MachinePassManager Pipeline(Options); if (Invocation.PassPipeline.size() == 1 && Invocation.PassPipeline.front() == "default") { - auto Added = riscv::addCanonicalPipeline(Pipeline, Function, Features); + auto Added = riscv::addCanonicalPipeline(Pipeline, Function, Subtarget); if (!Added) return std::unexpected(Added.error()); } else { @@ -93,7 +93,7 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio "the default pipeline marker cannot be mixed with named passes"); } for (const std::string &Name : Invocation.PassPipeline) { - auto Pass = riscv::createCanonicalPipelinePass(Name, Features); + auto Pass = riscv::createCanonicalPipelinePass(Name, Subtarget); if (!Pass) return std::unexpected(Pass.error()); auto Added = Pipeline.addPass(std::move(*Pass)); @@ -104,7 +104,8 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio return std::move(Pipeline).takePasses(); } -[[nodiscard]] Expected validateSelector(std::string_view Selector, bool AllowAll) { +[[nodiscard]] Expected validateSelector(std::string_view Selector, bool AllowAll, + const riscv::RISCVSubtarget &Subtarget) { if (Selector == "all" && AllowAll) return {}; const std::size_t Comma = Selector.find(','); @@ -112,7 +113,7 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio Comma == std::string_view::npos ? Selector : Selector.substr(0, Comma); if (Name.empty()) return makeUnexpected(ErrorCode::InvalidInvocation, "empty pipeline pass selector"); - auto Pass = riscv::createCanonicalPipelinePass(Name); + auto Pass = riscv::createCanonicalPipelinePass(Name, Subtarget); if (!Pass) return std::unexpected(Pass.error()); if (Comma != std::string_view::npos) { @@ -129,7 +130,9 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio return {}; } -[[nodiscard]] Expected validateControls(const PipelineControls &Controls) { +[[nodiscard]] Expected validateControls(const PipelineControls &Controls, + riscv::RISCVSubtargetFeatures Features) { + const riscv::RISCVSubtarget Subtarget(Features); if (Controls.StartBefore && Controls.StartAfter) { return makeUnexpected(ErrorCode::InvalidInvocation, "-start-before and -start-after cannot be combined"); @@ -139,32 +142,32 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio "-stop-before and -stop-after cannot be combined"); } if (Controls.StartBefore) { - auto Result = validateSelector(*Controls.StartBefore, false); + auto Result = validateSelector(*Controls.StartBefore, false, Subtarget); if (!Result) return Result; } if (Controls.StartAfter) { - auto Result = validateSelector(*Controls.StartAfter, false); + auto Result = validateSelector(*Controls.StartAfter, false, Subtarget); if (!Result) return Result; } if (Controls.StopBefore) { - auto Result = validateSelector(*Controls.StopBefore, false); + auto Result = validateSelector(*Controls.StopBefore, false, Subtarget); if (!Result) return Result; } if (Controls.StopAfter) { - auto Result = validateSelector(*Controls.StopAfter, false); + auto Result = validateSelector(*Controls.StopAfter, false, Subtarget); if (!Result) return Result; } for (const std::string &Selector : Controls.PrintBefore) { - auto Result = validateSelector(Selector, true); + auto Result = validateSelector(Selector, true, Subtarget); if (!Result) return Result; } for (const std::string &Selector : Controls.PrintAfter) { - auto Result = validateSelector(Selector, true); + auto Result = validateSelector(Selector, true, Subtarget); if (!Result) return Result; } @@ -200,12 +203,13 @@ runFunction(MachineFunction &Function, mir::MIRDocument &Document, const TargetD riscv::RISCVSubtargetFeatures Features, bool AllowSnapshots, MachinePassInstrumentation *Instrumentation) { FunctionRunResult Result; + const riscv::RISCVSubtarget Subtarget(Features); MachinePassManagerOptions ManagerOptions{ .VerifyBefore = true, .VerifyEach = Controls.VerifyEach, .DetectUnreportedChanges = Controls.DetectUnreportedChanges, }; - auto Passes = createPasses(Function, Invocation, Features, ManagerOptions); + auto Passes = createPasses(Function, Invocation, Subtarget, ManagerOptions); if (!Passes) return std::unexpected(Passes.error()); MachineAnalysisManager Analyses; @@ -408,7 +412,7 @@ Expected runRV64GCPipeline(mir::MIRDocument &Document, auto Features = riscv::RISCVSubtargetFeatures::fromFeatureList(Invocation.Features); if (!Features) return std::unexpected(Features.error()); - auto ControlVerification = validateControls(Controls); + auto ControlVerification = validateControls(Controls, *Features); if (!ControlVerification) return std::unexpected(ControlVerification.error()); auto UnitVerification = Document.unit().verify(); diff --git a/backend/tools/MIRTool/MIRToolMain.cpp b/backend/tools/MIRTool/MIRToolMain.cpp index 2c0fe8d..a932a89 100644 --- a/backend/tools/MIRTool/MIRToolMain.cpp +++ b/backend/tools/MIRTool/MIRToolMain.cpp @@ -857,12 +857,17 @@ runWithCache(const CommandLine &Command, CanonicalMIR Input, const rcc::TargetDe return {}; } if (Command.Action == Operation::ListPasses) { + auto Features = + rcc::riscv::RISCVSubtargetFeatures::fromFeatureList(Command.Invocation.Features); + if (!Features) + return std::unexpected(Features.error()); + const rcc::riscv::RISCVSubtarget Subtarget(*Features); std::vector Names; for (std::size_t Index = 0; Index < Input->Document.unit().functionCount(); ++Index) { const rcc::MachineFunction *Function = Input->Document.unit().functionAt(Index); if (Function == nullptr) return rcc::makeUnexpected(ErrorCode::InternalError, "MIR input contains a null function"); - auto FunctionNames = rcc::riscv::canonicalPipelinePassNames(*Function); + auto FunctionNames = rcc::riscv::canonicalPipelinePassNames(*Function, Subtarget); if (!FunctionNames) return std::unexpected(FunctionNames.error()); for (std::string &Name : *FunctionNames) { diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt index 5f73d3b..3ae0f3d 100644 --- a/backend/unittests/CodeGen/CMakeLists.txt +++ b/backend/unittests/CodeGen/CMakeLists.txt @@ -348,6 +348,10 @@ target_link_libraries( add_test(NAME RCCBackend.CodeGen.MachineAnalyses COMMAND RCCBackendCodeGenMachineAnalysesTests) add_executable(RCCBackendCodeGenMachineOptimizationTests MachineOptimizationTest.cpp) +target_include_directories( + RCCBackendCodeGenMachineOptimizationTests + PRIVATE "${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV" +) rcc_backend_configure_target(RCCBackendCodeGenMachineOptimizationTests) target_link_libraries( RCCBackendCodeGenMachineOptimizationTests @@ -356,6 +360,10 @@ target_link_libraries( add_test(NAME RCCBackend.CodeGen.MachineOptimization COMMAND RCCBackendCodeGenMachineOptimizationTests) add_executable(RCCBackendCodeGenMachineSchedulingTests MachineSchedulingTest.cpp) +target_include_directories( + RCCBackendCodeGenMachineSchedulingTests + PRIVATE "${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV" +) rcc_backend_configure_target(RCCBackendCodeGenMachineSchedulingTests) target_link_libraries( RCCBackendCodeGenMachineSchedulingTests diff --git a/backend/unittests/CodeGen/GlobalISel/CMakeLists.txt b/backend/unittests/CodeGen/GlobalISel/CMakeLists.txt index 319e31f..0ff962b 100644 --- a/backend/unittests/CodeGen/GlobalISel/CMakeLists.txt +++ b/backend/unittests/CodeGen/GlobalISel/CMakeLists.txt @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later +include_directories("${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV") + add_executable(RCCBackendCodeGenLegalizerTests LegalizerTest.cpp) rcc_backend_configure_target(RCCBackendCodeGenLegalizerTests) target_link_libraries( diff --git a/backend/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp b/backend/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp index 9e09934..e98f121 100644 --- a/backend/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp +++ b/backend/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp @@ -2,11 +2,13 @@ #include "rcc/CodeGen/GlobalISel/InstructionSelect.h" +#include "GISel/RISCVRegisterBankInfo.h" #include "GISelMITest.h" #include "rcc/CodeGen/GlobalISel/Legalizer.h" #include "rcc/CodeGen/GlobalISel/RegBankSelect.h" #include "rcc/CodeGen/MachineMemOperand.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" #include #include @@ -117,16 +119,16 @@ void testPipeline(TestContext &Context) { Fixture.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, rcc::MachineFunctionProperty::TracksLiveness}; + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); rcc::MachinePassManager Pipeline; Context.expect( - Pipeline.addPass(std::make_unique(rcc::riscv::legalizerInfo())).has_value(), + Pipeline.addPass(std::make_unique(Subtarget.getLegalizerInfo())).has_value(), "legalizer must be accepted by the pass manager"); + Context.expect(Pipeline.addPass(std::make_unique(Subtarget.getRegBankInfo())) + .has_value(), + "register-bank selector must be accepted by the pass manager"); Context.expect( - Pipeline.addPass(std::make_unique(rcc::riscv::registerBankInfo())) - .has_value(), - "register-bank selector must be accepted by the pass manager"); - Context.expect( - Pipeline.addPass(std::make_unique(rcc::riscv::instructionSelector())) + Pipeline.addPass(std::make_unique(Subtarget.getInstructionSelector())) .has_value(), "instruction selector must be accepted by the pass manager"); rcc::MachineAnalysisManager Analyses; @@ -198,7 +200,8 @@ void testInstructionSelectionOrder(TestContext &Context) { } Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized, rcc::MachineFunctionProperty::RegBankSelected}; - RecordingSelector Selector(rcc::riscv::instructionSelector()); + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); + RecordingSelector Selector(*Subtarget.getInstructionSelector()); rcc::InstructionSelect Pass(Selector); rcc::MachineAnalysisManager Analyses; auto Run = Pass.run(*Fixture.Function, *Target, Analyses); @@ -322,11 +325,12 @@ void testExtendedRV64GCPipeline(TestContext &Context) { Fixture.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, rcc::MachineFunctionProperty::TracksLiveness}; + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); rcc::MachinePassManager Pipeline; - if (!Pipeline.addPass(std::make_unique(rcc::riscv::legalizerInfo())) || - !Pipeline.addPass(std::make_unique(rcc::riscv::registerBankInfo())) || + if (!Pipeline.addPass(std::make_unique(Subtarget.getLegalizerInfo())) || + !Pipeline.addPass(std::make_unique(Subtarget.getRegBankInfo())) || !Pipeline.addPass( - std::make_unique(rcc::riscv::instructionSelector()))) { + std::make_unique(Subtarget.getInstructionSelector()))) { Context.expect(false, "extended RV64GC pipeline construction must succeed"); return; } diff --git a/backend/unittests/CodeGen/GlobalISel/LegalizerTest.cpp b/backend/unittests/CodeGen/GlobalISel/LegalizerTest.cpp index 6280182..9cb8a67 100644 --- a/backend/unittests/CodeGen/GlobalISel/LegalizerTest.cpp +++ b/backend/unittests/CodeGen/GlobalISel/LegalizerTest.cpp @@ -3,7 +3,8 @@ #include "rcc/CodeGen/GlobalISel/Legalizer.h" #include "GISelMITest.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" #include #include @@ -79,7 +80,8 @@ void testNullAndMissingContracts(TestContext &Context) { Context.expect(Fixture.Function->properties().empty(), "a rejected pass must not publish function properties"); - rcc::Legalizer ValidPass(rcc::riscv::legalizerInfo()); + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); + rcc::Legalizer ValidPass(Subtarget.getLegalizerInfo()); auto MissingResult = ValidPass.run(*Fixture.Function, *Target, Analyses); Context.expect(!MissingResult && MissingResult.error().code() == rcc::ErrorCode::VerificationError, @@ -114,9 +116,10 @@ void testUnsupportedRollback(TestContext &Context) { "unsupported floating operation must be constructible"); Unsupported.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, rcc::MachineFunctionProperty::TracksLiveness}; + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); rcc::MachinePassManager OnlyLegalizer; Context.expect( - OnlyLegalizer.addPass(std::make_unique(rcc::riscv::legalizerInfo())) + OnlyLegalizer.addPass(std::make_unique(Subtarget.getLegalizerInfo())) .has_value(), "standalone legalizer must be accepted"); rcc::MachineAnalysisManager UnsupportedAnalyses; diff --git a/backend/unittests/CodeGen/GlobalISel/RegBankSelectTest.cpp b/backend/unittests/CodeGen/GlobalISel/RegBankSelectTest.cpp index 3cfa888..d4925eb 100644 --- a/backend/unittests/CodeGen/GlobalISel/RegBankSelectTest.cpp +++ b/backend/unittests/CodeGen/GlobalISel/RegBankSelectTest.cpp @@ -2,8 +2,10 @@ #include "rcc/CodeGen/GlobalISel/RegBankSelect.h" +#include "GISel/RISCVRegisterBankInfo.h" #include "GISelMITest.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" #include @@ -29,7 +31,8 @@ void testBankConflict(TestContext &Context) { .has_value(), "conflicting bank assignment must be constructible"); Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized}; - rcc::RegBankSelect Pass(rcc::riscv::registerBankInfo()); + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); + rcc::RegBankSelect Pass(Subtarget.getRegBankInfo()); rcc::MachineAnalysisManager Analyses; auto Result = Pass.run(*Fixture.Function, *Target, Analyses); Context.expect(!Result && Result.error().code() == rcc::ErrorCode::VerificationError, @@ -61,7 +64,8 @@ void testBankClassOnlyChange(TestContext &Context) { Context.expect(Fixture.Function->registerInfo().setRegisterBank(Value, GPR).has_value(), "pre-existing GPR bank must be assignable"); Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized}; - rcc::RegBankSelect Pass(rcc::riscv::registerBankInfo()); + const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); + rcc::RegBankSelect Pass(Subtarget.getRegBankInfo()); rcc::MachineAnalysisManager Analyses; auto Result = Pass.run(*Fixture.Function, *Target, Analyses); Context.expect(Result.has_value() && Result->Changed && diff --git a/backend/unittests/CodeGen/MachineOptimizationTest.cpp b/backend/unittests/CodeGen/MachineOptimizationTest.cpp index d4a39fa..f4233f8 100644 --- a/backend/unittests/CodeGen/MachineOptimizationTest.cpp +++ b/backend/unittests/CodeGen/MachineOptimizationTest.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineCFGUpdater.h" @@ -17,7 +18,6 @@ #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/MachineSink.h" #include "rcc/CodeGen/TargetMemoryFolding.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include diff --git a/backend/unittests/CodeGen/MachineSchedulingTest.cpp b/backend/unittests/CodeGen/MachineSchedulingTest.cpp index a485146..83f7000 100644 --- a/backend/unittests/CodeGen/MachineSchedulingTest.cpp +++ b/backend/unittests/CodeGen/MachineSchedulingTest.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" #include "rcc/CodeGen/MachineIRBuilder.h" @@ -11,7 +12,6 @@ #include "rcc/CodeGen/ScheduleDAG.h" #include "rcc/CodeGen/ScheduleHazardRecognizer.h" #include "rcc/CodeGen/TargetSchedulingModel.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include diff --git a/backend/unittests/Target/CMakeLists.txt b/backend/unittests/Target/CMakeLists.txt index 0b4ed74..3377f82 100644 --- a/backend/unittests/Target/CMakeLists.txt +++ b/backend/unittests/Target/CMakeLists.txt @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later +include_directories("${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV") + add_library(RCCBackendTestTarget TestTarget.cpp) add_library(RCCBackend::TestTarget ALIAS RCCBackendTestTarget) @@ -34,10 +36,6 @@ target_link_libraries(RCCBackendRISCVCodeGenTests PRIVATE RCCBackend::RISCV RCCB add_test(NAME RCCBackendRISCVCodeGenTests COMMAND RCCBackendRISCVCodeGenTests) add_executable(RCCBackendRISCVSSAExitTests RISCVSSAExitTest.cpp) -target_include_directories( - RCCBackendRISCVSSAExitTests - PRIVATE "${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV" -) rcc_backend_configure_target(RCCBackendRISCVSSAExitTests) target_link_libraries(RCCBackendRISCVSSAExitTests PRIVATE RCCBackend::RISCV RCCBackend::CodeGen) add_test(NAME RCCBackendRISCVSSAExitTests COMMAND RCCBackendRISCVSSAExitTests) @@ -65,10 +63,6 @@ add_test( ) add_executable(RCCBackendRISCVLateCodeGenTests RISCVLateCodeGenTest.cpp) -target_include_directories( - RCCBackendRISCVLateCodeGenTests - PRIVATE "${RCC_BACKEND_SOURCE_DIR}/lib/Target/RISCV" -) rcc_backend_configure_target(RCCBackendRISCVLateCodeGenTests) target_link_libraries( RCCBackendRISCVLateCodeGenTests diff --git a/backend/unittests/Target/RISCVCallLoweringTest.cpp b/backend/unittests/Target/RISCVCallLoweringTest.cpp index 33723b1..eee10ab 100644 --- a/backend/unittests/Target/RISCVCallLoweringTest.cpp +++ b/backend/unittests/Target/RISCVCallLoweringTest.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "rcc/Target/RISCV/RISCVCallLowering.h" +#include "GISel/RISCVCallLowering.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" @@ -11,6 +11,7 @@ #include "rcc/CodeGen/MachineMemOperand.h" #include "rcc/CodeGen/MachineOperand.h" #include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include @@ -29,6 +30,9 @@ namespace { +const rcc::riscv::RISCVSubtarget Subtarget(rcc::riscv::RISCVSubtargetFeatures::rv64gc()); +const rcc::TargetCallLowering &CallLowering = *Subtarget.getCallLowering(); + class TestContext final { public: void expect(bool Condition, std::string_view Message) { @@ -186,7 +190,7 @@ abiValue(rcc::LLT Type, std::uint32_t OriginalValue, rcc::Register Value = plann void testIntegerAndFloatingAssignments(TestContext &Context) { auto Integers = callRequest(integerValues(9), 9); - auto IntegerPlan = rcc::riscv::callLowering().planCall(Integers); + auto IntegerPlan = CallLowering.planCall(Integers); Context.expect(IntegerPlan.has_value(), "nine integer arguments must produce an LP64D call plan"); if (!IntegerPlan) return; @@ -207,7 +211,7 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { "LP64D integer arguments must use a0-a7 before a 16-byte stack area"); auto Floats = callRequest(floatingValues(17), 17); - auto FloatPlan = rcc::riscv::callLowering().planCall(Floats); + auto FloatPlan = CallLowering.planCall(Floats); Context.expect(FloatPlan.has_value(), "seventeen floating arguments must produce an LP64D call plan"); if (!FloatPlan) @@ -228,7 +232,7 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { auto Single = callRequest({abiValue(floating(rcc::FloatSemantics::IEEESingle), 0)}, 1); Single.Results = {abiValue(floating(rcc::FloatSemantics::IEEESingle), 0)}; - auto SinglePlan = rcc::riscv::callLowering().planCall(Single); + auto SinglePlan = CallLowering.planCall(Single); Context.expect(SinglePlan && SinglePlan->argumentAssignments().size() == 1 && SinglePlan->returnAssignments().size() == 1 && isLocation(SinglePlan->argumentAssignments()[0], @@ -246,7 +250,7 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { IntegerFragment.FragmentOffsetBytes = 8; IntegerFragment.OriginalAlignment = align(16); auto Mixed = callRequest({FloatFragment, IntegerFragment}, 1); - auto MixedPlan = rcc::riscv::callLowering().planCall(Mixed); + auto MixedPlan = CallLowering.planCall(Mixed); Context.expect(MixedPlan && MixedPlan->argumentAssignments().size() == 2 && isLocation(MixedPlan->argumentAssignments()[0], argumentFPR(0)) && isLocation(MixedPlan->argumentAssignments()[1], argumentGPR(0)), @@ -254,7 +258,7 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { auto GenericScalar = callRequest({abiValue(anyScalar(64), 0)}, 1); GenericScalar.Results = {abiValue(anyScalar(64), 0)}; - auto GenericScalarPlan = rcc::riscv::callLowering().planCall(GenericScalar); + auto GenericScalarPlan = CallLowering.planCall(GenericScalar); Context.expect(GenericScalarPlan && GenericScalarPlan->argumentAssignments().size() == 1 && GenericScalarPlan->returnAssignments().size() == 1 && GenericScalarPlan->argumentAssignments()[0].ValueType == anyScalar(64) && @@ -264,7 +268,7 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { auto LP64Float = callRequest({abiValue(floating(rcc::FloatSemantics::IEEEDouble), 0)}, 1); LP64Float.Convention = rcc::riscv::callingConventionID(rcc::riscv::CallingConvention::LP64); LP64Float.Results = {abiValue(floating(rcc::FloatSemantics::IEEEDouble), 0)}; - auto LP64Plan = rcc::riscv::callLowering().planCall(LP64Float); + auto LP64Plan = CallLowering.planCall(LP64Float); const auto LP64Mask = rcc::riscv::createCallPreservedMask(rcc::riscv::CallingConvention::LP64); Context.expect( LP64Plan && LP64Plan->argumentAssignments().size() == 1 && @@ -279,11 +283,11 @@ void testIntegerAndFloatingAssignments(TestContext &Context) { auto Narrow = abiValue(integer(32), 0); auto InvalidNarrow = callRequest({Narrow}, 1); - Context.expect(!rcc::riscv::callLowering().planCall(InvalidNarrow), + Context.expect(!CallLowering.planCall(InvalidNarrow), "a narrow integer without an explicit ABI extension must be rejected"); Narrow.Extension = rcc::ABIExtensionKind::SignExtend; auto ExtendedNarrow = callRequest({Narrow}, 1); - auto NarrowPlan = rcc::riscv::callLowering().planCall(ExtendedNarrow); + auto NarrowPlan = CallLowering.planCall(ExtendedNarrow); Context.expect(NarrowPlan && NarrowPlan->argumentAssignments()[0].LocationType == integer(64) && NarrowPlan->argumentAssignments()[0].Extension == rcc::ABIExtensionKind::SignExtend, @@ -294,7 +298,7 @@ void testVariadicAssignments(TestContext &Context) { auto FixedFloat = abiValue(floating(rcc::FloatSemantics::IEEEDouble), 0); auto VariableFloat = abiValue(floating(rcc::FloatSemantics::IEEEDouble), 1); auto Request = callRequest({FixedFloat, VariableFloat}, 1, true); - auto Plan = rcc::riscv::callLowering().planCall(Request); + auto Plan = CallLowering.planCall(Request); Context.expect( Plan && isLocation(Plan->argumentAssignments()[0], argumentFPR(0)) && isLocation(Plan->argumentAssignments()[1], argumentGPR(0)) && @@ -312,7 +316,7 @@ void testVariadicAssignments(TestContext &Context) { PairHigh.FragmentOffsetBytes = 8; PairHigh.OriginalAlignment = align(16); auto PairRequest = callRequest({FixedInteger, PairLow, PairHigh}, 1, true); - auto PairPlan = rcc::riscv::callLowering().planCall(PairRequest); + auto PairPlan = CallLowering.planCall(PairRequest); Context.expect(PairPlan && isLocation(PairPlan->argumentAssignments()[0], argumentGPR(0)) && isLocation(PairPlan->argumentAssignments()[1], argumentGPR(2)) && isLocation(PairPlan->argumentAssignments()[2], argumentGPR(3)), @@ -323,7 +327,7 @@ void testVariadicAssignments(TestContext &Context) { Formal.Arguments = {FixedInteger}; Formal.FixedArgumentCount = 1; Formal.IsVariadic = true; - auto FormalPlan = rcc::riscv::callLowering().planFormalArguments(Formal); + auto FormalPlan = CallLowering.planFormalArguments(Formal); Context.expect(FormalPlan && FormalPlan->varArgs().IsVariadic && FormalPlan->varArgs().FirstUnallocatedIntegerRegister == 1 && FormalPlan->varArgs().RegisterSaveAreaSizeBytes == 56 && @@ -341,7 +345,7 @@ void testSpecialArgumentsAndReturns(TestContext &Context) { ByVal.ByValSizeBytes = 24; ByVal.ByValAlignment = align(16); auto ByValRequest = callRequest({ByVal}, 1); - auto ByValPlan = rcc::riscv::callLowering().planCall(ByValRequest); + auto ByValPlan = CallLowering.planCall(ByValRequest); Context.expect( ByValPlan && ByValPlan->byValCopies().size() == 1 && ByValPlan->byValCopies()[0].StackOffsetBytes == 0 && @@ -351,23 +355,23 @@ void testSpecialArgumentsAndReturns(TestContext &Context) { auto SRet = abiValue(pointer(), 0); SRet.Flags.set(rcc::ABIValueFlag::StructReturn); - auto SRetPlan = rcc::riscv::callLowering().planCall(callRequest({SRet}, 1)); + auto SRetPlan = CallLowering.planCall(callRequest({SRet}, 1)); Context.expect( SRetPlan && isLocation(SRetPlan->argumentAssignments()[0], argumentGPR(0)) && SRetPlan->argumentAssignments()[0].CustomFlags.has(rcc::CCAssignmentFlag::StructReturn), "sret must be carried as the first integer argument"); auto SRetWithDirectResult = callRequest({SRet}, 1); SRetWithDirectResult.Results = {abiValue(integer(64), 0)}; - Context.expect(!rcc::riscv::callLowering().planCall(SRetWithDirectResult), + Context.expect(!CallLowering.planCall(SRetWithDirectResult), "an sret call cannot also expose direct ABI return registers"); auto Plain = abiValue(integer(64), 0); SRet.OriginalValue = 1; - Context.expect(!rcc::riscv::callLowering().planCall(callRequest({Plain, SRet}, 2)), + Context.expect(!CallLowering.planCall(callRequest({Plain, SRet}, 2)), "sret outside the first argument position must be rejected"); auto Indirect = abiValue(pointer(), 0); Indirect.Flags.set(rcc::ABIValueFlag::Indirect); - auto IndirectPlan = rcc::riscv::callLowering().planCall(callRequest({Indirect}, 1)); + auto IndirectPlan = CallLowering.planCall(callRequest({Indirect}, 1)); Context.expect( IndirectPlan && IndirectPlan->argumentAssignments()[0].LocationKind == rcc::CCLocationKind::Indirect && @@ -376,14 +380,14 @@ void testSpecialArgumentsAndReturns(TestContext &Context) { auto InRegister = integerValues(9); InRegister.back().Flags.set(rcc::ABIValueFlag::InRegister); - Context.expect(!rcc::riscv::callLowering().planCall(callRequest(std::move(InRegister), 9)), + Context.expect(!CallLowering.planCall(callRequest(std::move(InRegister), 9)), "an inreg argument that would spill to the stack must be rejected"); rcc::ReturnLoweringRequest Return; Return.Convention = convention(); Return.Values = {abiValue(integer(64), 0), abiValue(floating(rcc::FloatSemantics::IEEEDouble), 1)}; - auto ReturnPlan = rcc::riscv::callLowering().planReturn(Return); + auto ReturnPlan = CallLowering.planReturn(Return); Context.expect(ReturnPlan && ReturnPlan->returnAssignments().size() == 2 && isLocation(ReturnPlan->returnAssignments()[0], returnGPR(0)) && isLocation(ReturnPlan->returnAssignments()[1], returnFPR(0)), @@ -402,14 +406,14 @@ void testTailCalls(TestContext &Context) { auto Tail = callRequest({}, 0); Tail.TailKind = rcc::TailCallKind::Tail; Tail.TailContext = emptyTailContext(); - auto TailPlan = rcc::riscv::callLowering().planCall(Tail); + auto TailPlan = CallLowering.planCall(Tail); Context.expect(TailPlan && TailPlan->tailDecision() == rcc::TailCallKind::Tail && TailPlan->callFrame().ReusesIncomingArea && !TailPlan->callFrame().EmitSetup, "an ABI-compatible tail call must reuse the incoming argument area"); auto Fallback = callRequest({}, 0); Fallback.TailKind = rcc::TailCallKind::Tail; - auto FallbackPlan = rcc::riscv::callLowering().planCall(Fallback); + auto FallbackPlan = CallLowering.planCall(Fallback); Context.expect(FallbackPlan && FallbackPlan->tailDecision() == rcc::TailCallKind::NotTail && !FallbackPlan->tailRejectionReason().empty() && FallbackPlan->callFrame().EmitSetup, @@ -417,7 +421,7 @@ void testTailCalls(TestContext &Context) { auto MustTail = callRequest({}, 0); MustTail.TailKind = rcc::TailCallKind::MustTail; - auto MustTailPlan = rcc::riscv::callLowering().planCall(MustTail); + auto MustTailPlan = CallLowering.planCall(MustTail); Context.expect(!MustTailPlan && MustTailPlan.error().code() == rcc::ErrorCode::Unsupported, "an incompatible musttail call must fail instead of falling back"); @@ -427,7 +431,7 @@ void testTailCalls(TestContext &Context) { StackContext.CallerFixedArgumentCount = 9; StackContext.ReusableIncomingArgumentBytes = 16; StackTail.TailContext = std::move(StackContext); - auto StackTailPlan = rcc::riscv::callLowering().planCall(StackTail); + auto StackTailPlan = CallLowering.planCall(StackTail); Context.expect(StackTailPlan && StackTailPlan->tailDecision() == rcc::TailCallKind::Tail && StackTailPlan->callFrame().SizeBytes == 16, "a tail call may reuse a sufficiently large incoming stack argument area"); @@ -506,7 +510,7 @@ void testOrdinaryAndIndirectEmission(TestContext &Context) { rcc::MachineIRBuilder Builder(*Function, *Target); Context.expect(Builder.setInsertPoint(*Block).has_value(), "ordinary-call insertion point must be valid"); - auto Lowered = rcc::riscv::callLowering().lowerCall(Builder, Request); + auto Lowered = CallLowering.lowerCall(Builder, Request); Context.expect(Lowered.has_value(), "ordinary direct call emission must succeed"); if (!Lowered) return; @@ -602,7 +606,7 @@ void testOrdinaryAndIndirectEmission(TestContext &Context) { rcc::MachineIRBuilder IndirectBuilder(*IndirectFunction, *Target); if (!IndirectBuilder.setInsertPoint(*IndirectBlock)) return; - auto IndirectLowered = rcc::riscv::callLowering().lowerCall(IndirectBuilder, IndirectRequest); + auto IndirectLowered = CallLowering.lowerCall(IndirectBuilder, IndirectRequest); rcc::MachineInstr *IndirectCall = findInstruction(*IndirectBlock, rcc::riscv::InstructionOpcode::PseudoCALL); Context.expect(IndirectLowered && IndirectCall != nullptr && @@ -632,7 +636,7 @@ void testStackAndByValEmission(TestContext &Context) { rcc::MachineIRBuilder Builder(*Function, *Target); if (!Builder.setInsertPoint(*Block)) return; - auto Lowered = rcc::riscv::callLowering().lowerCall(Builder, Request); + auto Lowered = CallLowering.lowerCall(Builder, Request); rcc::MachineInstr *ByValCopy = findInstruction(*Block, rcc::riscv::InstructionOpcode::PseudoBYVAL_COPY); rcc::MachineInstr *StackStore = @@ -679,7 +683,7 @@ void testFormalReturnAndTailEmission(TestContext &Context) { rcc::MachineIRBuilder FormalBuilder(*FormalFunction, *Target); if (!FormalBuilder.setInsertPoint(*FormalBlock)) return; - auto FormalLowered = rcc::riscv::callLowering().lowerFormalArguments(FormalBuilder, Formal); + auto FormalLowered = CallLowering.lowerFormalArguments(FormalBuilder, Formal); std::size_t SaveStores = 0; for (rcc::MachineInstr *Instruction : instructions(*FormalBlock)) { if (Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::PseudoSTORE_STACK_ARG)) @@ -713,7 +717,7 @@ void testFormalReturnAndTailEmission(TestContext &Context) { rcc::MachineIRBuilder ReturnBuilder(*ReturnFunction, *Target); if (!ReturnBuilder.setInsertPoint(*ReturnBlock)) return; - auto ReturnLowered = rcc::riscv::callLowering().lowerReturn(ReturnBuilder, Return); + auto ReturnLowered = CallLowering.lowerReturn(ReturnBuilder, Return); rcc::MachineInstr *ReturnPseudo = findInstruction(*ReturnBlock, rcc::riscv::InstructionOpcode::PseudoRET); bool HasReturnRegister = false; @@ -738,7 +742,7 @@ void testFormalReturnAndTailEmission(TestContext &Context) { rcc::MachineIRBuilder TailBuilder(*TailFunction, *Target); if (!TailBuilder.setInsertPoint(*TailBlock)) return; - auto TailLowered = rcc::riscv::callLowering().lowerCall(TailBuilder, Tail); + auto TailLowered = CallLowering.lowerCall(TailBuilder, Tail); Context.expect(TailLowered && instructions(*TailBlock).size() == 1 && TailBlock->firstInstruction()->opcode() == opcode(rcc::riscv::InstructionOpcode::PseudoTAIL) && @@ -767,7 +771,7 @@ void testFormalReturnAndTailEmission(TestContext &Context) { rcc::MachineIRBuilder StackTailBuilder(*StackTailFunction, *Target); if (!StackTailBuilder.setInsertPoint(*StackTailBlock)) return; - auto StackTailLowered = rcc::riscv::callLowering().lowerCall(StackTailBuilder, StackTail); + auto StackTailLowered = CallLowering.lowerCall(StackTailBuilder, StackTail); Context.expect( StackTailLowered && findInstruction(*StackTailBlock, rcc::riscv::InstructionOpcode::PseudoSTORE_STACK_ARG) != @@ -802,7 +806,7 @@ void testFormalStackAlignmentAndPreflight(TestContext &Context) { rcc::MachineIRBuilder StackBuilder(*StackFunction, *Target); if (!StackBuilder.setInsertPoint(*StackBlock)) return; - auto StackLowered = rcc::riscv::callLowering().lowerFormalArguments(StackBuilder, StackRequest); + auto StackLowered = CallLowering.lowerFormalArguments(StackBuilder, StackRequest); bool HasIncomingObject = false; for (const rcc::StackObject &Object : StackFunction->frameInfo().objects()) { const rcc::StackObjectDesc *Description = Object.description(); @@ -840,8 +844,7 @@ void testFormalStackAlignmentAndPreflight(TestContext &Context) { rcc::MachineIRBuilder ConflictBuilder(*ConflictFunction, *Target); if (!ConflictBuilder.setInsertPoint(*ConflictBlock)) return; - auto ConflictLowered = - rcc::riscv::callLowering().lowerFormalArguments(ConflictBuilder, ConflictRequest); + auto ConflictLowered = CallLowering.lowerFormalArguments(ConflictBuilder, ConflictRequest); Context.expect( !ConflictLowered && ConflictBlock->instructionCount() == 0 && ConflictFunction->allInstructions().empty() && @@ -862,7 +865,7 @@ void testPlanningFailureDoesNotMutate(TestContext &Context) { rcc::MachineIRBuilder Builder(*Function, *Target); if (!Builder.setInsertPoint(*Block)) return; - auto Lowered = rcc::riscv::callLowering().lowerCall(Builder, Request); + auto Lowered = CallLowering.lowerCall(Builder, Request); auto UnknownPhysical = rcc::PhysicalRegister::create(999); auto UnknownRequest = UnknownPhysical @@ -870,7 +873,7 @@ void testPlanningFailureDoesNotMutate(TestContext &Context) { 1) : rcc::CallLoweringRequest{}; auto UnknownLowered = UnknownPhysical - ? rcc::riscv::callLowering().lowerCall(Builder, UnknownRequest) + ? CallLowering.lowerCall(Builder, UnknownRequest) : rcc::Expected{std::unexpected(UnknownPhysical.error())}; Context.expect(!Lowered && !UnknownLowered && Block->instructionCount() == 0 && Function->allInstructions().empty() && diff --git a/backend/unittests/Target/RISCVCodeGenTest.cpp b/backend/unittests/Target/RISCVCodeGenTest.cpp index ddacb1b..c5fc10a 100644 --- a/backend/unittests/Target/RISCVCodeGenTest.cpp +++ b/backend/unittests/Target/RISCVCodeGenTest.cpp @@ -1,14 +1,16 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "rcc/Target/RISCV/RISCVCodeGen.h" - +#include "RISCV.h" #include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/InstructionSelector.h" +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" #include "rcc/CodeGen/MachineIRBuilder.h" #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/MachineMemOperand.h" #include "rcc/CodeGen/MachineOperand.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include @@ -22,6 +24,14 @@ namespace { +struct RISCVGISelTestComponents final { + rcc::riscv::RISCVSubtarget Subtarget{rcc::riscv::RISCVSubtargetFeatures::rv64gc()}; + const rcc::LegalizerInfo &Legalizer{*Subtarget.getLegalizerInfo()}; + const rcc::InstructionSelector *Selector{Subtarget.getInstructionSelector()}; +}; + +const RISCVGISelTestComponents GISel; + class TestContext final { public: void expect(bool Condition, std::string_view Message) { @@ -262,32 +272,24 @@ void testClassification(TestContext &Context) { "RISC-V integer test instructions must be constructible"); if (Add32.Instruction == nullptr || Add64.Instruction == nullptr) return; - auto Add32Action = - rcc::riscv::legalizerInfo().classify(*Add32.Instruction, *Add32.Function, *Target); - auto Add64Action = - rcc::riscv::legalizerInfo().classify(*Add64.Instruction, *Add64.Function, *Target); + auto Add32Action = GISel.Legalizer.classify(*Add32.Instruction, *Add32.Function, *Target); + auto Add64Action = GISel.Legalizer.classify(*Add64.Instruction, *Add64.Function, *Target); Context.expect(Add32Action && *Add32Action == rcc::LegalizationAction::Legal, "RV64GC legalizer must accept i32 arithmetic"); Context.expect(Add64Action && *Add64Action == rcc::LegalizationAction::Legal, "RV64GC legalizer must accept i64 arithmetic"); - Context.expect(rcc::riscv::instructionSelector() - .canSelect(*Add32.Instruction, *Add32.Function, *Target) - .has_value(), - "RV64GC selector must pre-match i32 arithmetic"); - Context.expect(rcc::riscv::instructionSelector() - .canSelect(*Add64.Instruction, *Add64.Function, *Target) - .has_value(), - "RV64GC selector must pre-match i64 arithmetic"); - Context.expect(rcc::riscv::instructionSelector() - .select(*Add32.Instruction, *Add32.Function, *Target) - .has_value(), + Context.expect( + (*GISel.Selector).canSelect(*Add32.Instruction, *Add32.Function, *Target).has_value(), + "RV64GC selector must pre-match i32 arithmetic"); + Context.expect( + (*GISel.Selector).canSelect(*Add64.Instruction, *Add64.Function, *Target).has_value(), + "RV64GC selector must pre-match i64 arithmetic"); + Context.expect((*GISel.Selector).select(*Add32.Instruction, *Add32.Function, *Target).has_value(), "RV64GC selector must lower i32 arithmetic"); Context.expect(Add32.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::ADDW)), "i32 G_ADD must select ADDW"); - Context.expect(rcc::riscv::instructionSelector() - .select(*Add64.Instruction, *Add64.Function, *Target) - .has_value(), + Context.expect((*GISel.Selector).select(*Add64.Instruction, *Add64.Function, *Target).has_value(), "RV64GC selector must lower i64 arithmetic"); Context.expect(Add64.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::ADD)), @@ -377,16 +379,14 @@ void testClassification(TestContext &Context) { "floating constant instructions must remain in layout order"); if (F32Instruction == nullptr || F64Instruction == nullptr) return; - auto F32Action = rcc::riscv::legalizerInfo().classify(*F32Instruction, **FloatFunction, *Target); - auto F64Action = rcc::riscv::legalizerInfo().classify(*F64Instruction, **FloatFunction, *Target); + auto F32Action = GISel.Legalizer.classify(*F32Instruction, **FloatFunction, *Target); + auto F64Action = GISel.Legalizer.classify(*F64Instruction, **FloatFunction, *Target); Context.expect(F32Action && *F32Action == rcc::LegalizationAction::Legal, "RV64GC legalizer must accept f32 constants"); Context.expect(F64Action && *F64Action == rcc::LegalizationAction::Legal, "RV64GC legalizer must accept f64 constants"); - auto F32Selection = - rcc::riscv::instructionSelector().select(*F32Instruction, **FloatFunction, *Target); - auto F64Selection = - rcc::riscv::instructionSelector().select(*F64Instruction, **FloatFunction, *Target); + auto F32Selection = (*GISel.Selector).select(*F32Instruction, **FloatFunction, *Target); + auto F64Selection = (*GISel.Selector).select(*F64Instruction, **FloatFunction, *Target); Context.expect(F32Selection.has_value(), "RV64GC selector must lower f32 constants"); Context.expect(F64Selection.has_value(), "RV64GC selector must lower f64 constants"); bool HasFMVWX = false; @@ -425,12 +425,10 @@ void testNarrowIntegerSelection(TestContext &Context) { Context.expect(false, "narrow integer fixture must be constructible"); continue; } - auto Action = - rcc::riscv::legalizerInfo().classify(*Fixture.Instruction, *Fixture.Function, *Target); + auto Action = GISel.Legalizer.classify(*Fixture.Instruction, *Fixture.Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Legal, "i1-i24 scalar operations must be legal on RV64GC GPR containers"); - auto Selection = - rcc::riscv::instructionSelector().select(*Fixture.Instruction, *Fixture.Function, *Target); + auto Selection = (*GISel.Selector).select(*Fixture.Instruction, *Fixture.Function, *Target); Context.expect(Selection.has_value(), "narrow integer operation must select"); bool FoundRequiredOpcode = false; for (const rcc::MachineInstr *Instruction : Fixture.Function->allInstructions()) { @@ -453,18 +451,16 @@ void testNarrowIntegerSelection(TestContext &Context) { rcc::FloatingPredicate::AlwaysFalse); if (Compare && Compare->firstBlock()->firstInstruction() != nullptr) { rcc::MachineInstr *Instruction = Compare->firstBlock()->firstInstruction(); - auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *Compare, *Target); - auto Lowered = Action - ? rcc::riscv::legalizerInfo().lower(*Instruction, *Compare, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); - auto LegalAction = rcc::riscv::legalizerInfo().classify(*Instruction, *Compare, *Target); + auto Action = GISel.Legalizer.classify(*Instruction, *Compare, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(*Instruction, *Compare, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); + auto LegalAction = GISel.Legalizer.classify(*Instruction, *Compare, *Target); bool Selected = Lowered && LegalAction && *LegalAction == rcc::LegalizationAction::Legal && constrainVirtualRegisters(*Compare); const auto Instructions = Compare->allInstructions(); for (rcc::MachineInstr *Candidate : Instructions) { if (Selected && rcc::isGenericOpcode(Candidate->opcode())) - Selected = - rcc::riscv::instructionSelector().select(*Candidate, *Compare, *Target).has_value(); + Selected = (*GISel.Selector).select(*Candidate, *Compare, *Target).has_value(); } Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Selected, "signed i7 comparisons must legalize to RV64 containers and select"); @@ -475,8 +471,8 @@ void testNarrowIntegerSelection(TestContext &Context) { auto Extension = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_SEXT, *I24, *I7); if (Extension && Extension->firstBlock()->firstInstruction() != nullptr) { rcc::MachineInstr *Instruction = Extension->firstBlock()->firstInstruction(); - auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *Extension, *Target); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Extension, *Target); + auto Action = GISel.Legalizer.classify(*Instruction, *Extension, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Extension, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Legal && Selection, "i7-to-i24 sign extension must select through a 32-bit container"); Context.expect(Selection && validatesSelectedFunction(*Target, *Extension), @@ -485,8 +481,8 @@ void testNarrowIntegerSelection(TestContext &Context) { auto UnsupportedMulHigh = createBinary(Context, *Target, rcc::GenericOpcode::G_SMULH, 16); if (UnsupportedMulHigh.Function && UnsupportedMulHigh.Instruction != nullptr) { - auto Action = rcc::riscv::legalizerInfo().classify(*UnsupportedMulHigh.Instruction, - *UnsupportedMulHigh.Function, *Target); + auto Action = GISel.Legalizer.classify(*UnsupportedMulHigh.Instruction, + *UnsupportedMulHigh.Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Unsupported, "non-native multiply-high widths must be explicitly unsupported"); } @@ -519,7 +515,7 @@ void testScalarSelection(TestContext &Context) { if (Entry.first == nullptr) continue; auto *Instruction = Entry.first->firstBlock()->firstInstruction(); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Entry.first, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Entry.first, *Target); const std::string_view Name = Entry.second == rcc::GenericOpcode::G_ANYEXT ? "anyext" : Entry.second == rcc::GenericOpcode::G_SEXT ? "sext" : Entry.second == rcc::GenericOpcode::G_ZEXT ? "zext" @@ -557,45 +553,40 @@ void testScalarSelection(TestContext &Context) { auto Bitcast = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_BITCAST, *I32, *F32); if (FPExt) { auto *Instruction = FPExt->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *FPExt, *Target).has_value(), - "fpext must select"); + Context.expect((*GISel.Selector).select(*Instruction, *FPExt, *Target).has_value(), + "fpext must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FCVT_D_S)), "fpext must select FCVT.D.S"); } if (FPTrunc) { auto *Instruction = FPTrunc->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *FPTrunc, *Target).has_value(), - "fptrunc must select"); + Context.expect((*GISel.Selector).select(*Instruction, *FPTrunc, *Target).has_value(), + "fptrunc must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FCVT_S_D)), "fptrunc must select FCVT.S.D"); } if (FPToSI) { auto *Instruction = FPToSI->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *FPToSI, *Target).has_value(), - "fptosi must select"); + Context.expect((*GISel.Selector).select(*Instruction, *FPToSI, *Target).has_value(), + "fptosi must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FCVT_L_S)), "fptosi must select FCVT.L.S"); } if (SIToFP) { auto *Instruction = SIToFP->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *SIToFP, *Target).has_value(), - "sitofp must select"); + Context.expect((*GISel.Selector).select(*Instruction, *SIToFP, *Target).has_value(), + "sitofp must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FCVT_D_W)), "sitofp must select FCVT.D.W"); } if (Bitcast) { auto *Instruction = Bitcast->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *Bitcast, *Target).has_value(), - "float-to-integer bitcast must select"); + Context.expect((*GISel.Selector).select(*Instruction, *Bitcast, *Target).has_value(), + "float-to-integer bitcast must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FMV_X_W)), "float-to-integer bitcast must select FMV.X.W"); @@ -633,15 +624,14 @@ void testComparisonsAndControl(TestContext &Context) { auto BuiltExtension = Extension->build(); if (!BuiltExtension) return; - auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *Equal, *Target); - auto Lowered = Action - ? rcc::riscv::legalizerInfo().lower(*Instruction, *Equal, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); + auto Action = GISel.Legalizer.classify(*Instruction, *Equal, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(*Instruction, *Equal, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); const rcc::Register *LegalResult = Instruction->operandAt(0)->getIf(); Context.expect(Lowered && (*BuiltExtension)->parent() == nullptr && LegalResult != nullptr && *LegalResult == *WideResult, "RV64 integer comparison legalization must fold a single result extension"); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Equal, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Equal, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered && Selection, "integer equality must legalize its result container and select"); rcc::MachineInstr *FirstSelected = Equal->firstBlock()->firstInstruction(); @@ -659,13 +649,12 @@ void testComparisonsAndControl(TestContext &Context) { } if (LessEqual) { auto *Instruction = LessEqual->firstBlock()->firstInstruction(); - auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *LessEqual, *Target); - auto Lowered = - Action ? rcc::riscv::legalizerInfo().lower(*Instruction, *LessEqual, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); + auto Action = GISel.Legalizer.classify(*Instruction, *LessEqual, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(*Instruction, *LessEqual, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered && constrainVirtualRegisters(*LessEqual) && - rcc::riscv::instructionSelector().select(*Instruction, *LessEqual, *Target), + (*GISel.Selector).select(*Instruction, *LessEqual, *Target), "signed less-equal must legalize its result container and select"); Context.expect(LessEqual->allInstructions().size() == 2, "signed less-equal must emit compare and inversion"); @@ -679,18 +668,15 @@ void testComparisonsAndControl(TestContext &Context) { rcc::IntegerPredicate::Equal, rcc::FloatingPredicate::UnorderedEqual); if (Ordered) { auto *Instruction = Ordered->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *Ordered, *Target).has_value(), - "ordered floating equality must select"); + Context.expect((*GISel.Selector).select(*Instruction, *Ordered, *Target).has_value(), + "ordered floating equality must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FEQ_S)), "ordered floating equality must select FEQ.S"); } if (UnorderedEqual) { auto *Instruction = UnorderedEqual->firstBlock()->firstInstruction(); - Context.expect(rcc::riscv::instructionSelector() - .select(*Instruction, *UnorderedEqual, *Target) - .has_value(), + Context.expect((*GISel.Selector).select(*Instruction, *UnorderedEqual, *Target).has_value(), "unordered floating equality must select"); Context.expect(UnorderedEqual->allInstructions().size() == 4, "unordered floating equality must emit ordered compare sequence"); @@ -699,9 +685,8 @@ void testComparisonsAndControl(TestContext &Context) { auto FMin = createFloatBinaryFunction(Context, *Target, rcc::GenericOpcode::G_FMINNUM, *F32); if (FMin) { auto *Instruction = FMin->firstBlock()->firstInstruction(); - Context.expect( - rcc::riscv::instructionSelector().select(*Instruction, *FMin, *Target).has_value(), - "floating minnum must select"); + Context.expect((*GISel.Selector).select(*Instruction, *FMin, *Target).has_value(), + "floating minnum must select"); Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FMIN_S)), "floating minnum must select FMIN.S"); @@ -728,25 +713,21 @@ void testComparisonsAndControl(TestContext &Context) { !Fence->addImmediate(7) || !Fence->addImmediate(0) || !Fence->build() || !Trap->build()) return; auto *IndirectInstruction = (*ControlFunction)->firstBlock()->firstInstruction(); - Context.expect(rcc::riscv::instructionSelector() - .select(*IndirectInstruction, **ControlFunction, *Target) - .has_value(), - "indirect branch must select"); + Context.expect( + (*GISel.Selector).select(*IndirectInstruction, **ControlFunction, *Target).has_value(), + "indirect branch must select"); Context.expect(IndirectInstruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::JALR)), "indirect branch must select JALR"); auto *FenceInstruction = IndirectInstruction->next(); - Context.expect(rcc::riscv::instructionSelector() - .select(*FenceInstruction, **ControlFunction, *Target) - .has_value(), - "fence must select"); + Context.expect( + (*GISel.Selector).select(*FenceInstruction, **ControlFunction, *Target).has_value(), + "fence must select"); Context.expect(FenceInstruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::FENCE)), "fence must select FENCE"); auto *TrapInstruction = FenceInstruction->next(); - Context.expect(rcc::riscv::instructionSelector() - .select(*TrapInstruction, **ControlFunction, *Target) - .has_value(), + Context.expect((*GISel.Selector).select(*TrapInstruction, **ControlFunction, *Target).has_value(), "trap must select"); Context.expect(TrapInstruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::EBREAK)), @@ -786,10 +767,10 @@ void testCompositeScalarSelection(TestContext &Context) { if (Function == nullptr) continue; rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); - auto Classification = rcc::riscv::legalizerInfo().classify(*Instruction, *Function, *Target); + auto Classification = GISel.Legalizer.classify(*Instruction, *Function, *Target); Context.expect(Classification && *Classification == rcc::LegalizationAction::Legal, "composite scalar operation must be legal on RV64GC"); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Function, *Target); Context.expect(Selection.has_value(), "composite scalar operation must select on RV64GC"); Context.expect(Selection && validatesSelectedFunction(*Target, *Function), "composite scalar selection must satisfy all target descriptors"); @@ -840,7 +821,7 @@ void testOverflowSelection(TestContext &Context) { if (!Function) continue; rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Function, *Target); Context.expect(Selection.has_value(), "RV64GC overflow operation must select"); Context.expect(Selection && validatesSelectedFunction(*Target, *Function), "overflow sequence must satisfy all target descriptors"); @@ -856,7 +837,7 @@ void testOverflowSelection(TestContext &Context) { if (!Function) continue; rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, *Function, *Target); Context.expect(Selection.has_value(), "extended RV64GC overflow operation must select"); Context.expect(Selection && validatesSelectedFunction(*Target, *Function), "extended overflow sequence must satisfy all target descriptors"); @@ -923,10 +904,9 @@ void testAtomicCompareExchangeLegalization(TestContext &Context) { if (!BuiltExtension) return; - auto Action = rcc::riscv::legalizerInfo().classify(**BuiltAtomic, **Function, *Target); - auto Lowered = - Action ? rcc::riscv::legalizerInfo().lower(**BuiltAtomic, **Function, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); + auto Action = GISel.Legalizer.classify(**BuiltAtomic, **Function, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(**BuiltAtomic, **Function, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered.has_value(), "narrow RV64 compare-exchange must use the widening legalizer rule"); if (!Lowered) @@ -940,7 +920,7 @@ void testAtomicCompareExchangeLegalization(TestContext &Context) { ? rcc::Expected>(std::unexpected( rcc::Error(rcc::ErrorCode::InvalidArgument, "missing compare-exchange result"))) : (*Function)->registerInfo().virtualRegisterInfo(*AtomicResult); - auto LegalAction = rcc::riscv::legalizerInfo().classify(**BuiltAtomic, **Function, *Target); + auto LegalAction = GISel.Legalizer.classify(**BuiltAtomic, **Function, *Target); Context.expect(CompareInfo && CompareInfo->get().Type == *I64 && NewValueInfo && NewValueInfo->get().Type == *I64 && AtomicResultInfo && AtomicResultInfo->get().Type == *I64 && @@ -967,11 +947,10 @@ void testAtomicCompareExchangeLegalization(TestContext &Context) { auto BuiltFallback = Fallback->build(); if (!BuiltFallback) return; - auto FallbackAction = rcc::riscv::legalizerInfo().classify(**BuiltFallback, **Function, *Target); + auto FallbackAction = GISel.Legalizer.classify(**BuiltFallback, **Function, *Target); auto FallbackLowered = - FallbackAction - ? rcc::riscv::legalizerInfo().lower(**BuiltFallback, **Function, *Target, *FallbackAction) - : rcc::Expected(std::unexpected(FallbackAction.error())); + FallbackAction ? GISel.Legalizer.lower(**BuiltFallback, **Function, *Target, *FallbackAction) + : rcc::Expected(std::unexpected(FallbackAction.error())); rcc::MachineInstr *Truncation = (*BuiltFallback)->next(); const rcc::Register *WideFallbackResult = (*BuiltFallback)->operandAt(0)->getIf(); const rcc::Register *TruncatedResult = @@ -999,12 +978,11 @@ void testAtomicCompareExchangeLegalization(TestContext &Context) { auto BuiltWithSuccess = WithSuccess->build(); if (!BuiltWithSuccess) return; - auto WithSuccessAction = - rcc::riscv::legalizerInfo().classify(**BuiltWithSuccess, **Function, *Target); - auto WithSuccessLowered = WithSuccessAction - ? rcc::riscv::legalizerInfo().lower(**BuiltWithSuccess, **Function, - *Target, *WithSuccessAction) - : rcc::Expected(std::unexpected(WithSuccessAction.error())); + auto WithSuccessAction = GISel.Legalizer.classify(**BuiltWithSuccess, **Function, *Target); + auto WithSuccessLowered = + WithSuccessAction + ? GISel.Legalizer.lower(**BuiltWithSuccess, **Function, *Target, *WithSuccessAction) + : rcc::Expected(std::unexpected(WithSuccessAction.error())); rcc::MachineInstr *SuccessCompare = (*BuiltWithSuccess)->previous(); rcc::MachineInstr *LoweredAtomic = SuccessCompare == nullptr ? nullptr : SuccessCompare->previous(); @@ -1056,10 +1034,9 @@ void testConstantExtensionPreservesLiveSource(TestContext &Context) { !Copy->build()) return; - auto Action = rcc::riscv::legalizerInfo().classify(**BuiltExtension, **Function, *Target); - auto Lowered = - Action ? rcc::riscv::legalizerInfo().lower(**BuiltExtension, **Function, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); + auto Action = GISel.Legalizer.classify(**BuiltExtension, **Function, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(**BuiltExtension, **Function, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered && (*BuiltExtension)->opcode() == rcc::opcode(rcc::GenericOpcode::G_CONSTANT) && (*BuiltConstant)->parent() != nullptr && @@ -1104,8 +1081,8 @@ void testIntegerCompareReusesConstantExtension(TestContext &Context) { if (!BuiltExtension) return; - auto Action = rcc::riscv::legalizerInfo().classify(*Compare, *Function, *Target); - auto Lowered = Action ? rcc::riscv::legalizerInfo().lower(*Compare, *Function, *Target, *Action) + auto Action = GISel.Legalizer.classify(*Compare, *Function, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(*Compare, *Function, *Target, *Action) : rcc::Expected(std::unexpected(Action.error())); const rcc::Register *LegalRight = Compare->operandAt(3)->getIf(); Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered, @@ -1115,10 +1092,10 @@ void testIntegerCompareReusesConstantExtension(TestContext &Context) { Context.expect(Function->registerInfo().verifyDefUseChains().has_value(), "integer comparison extension reuse must preserve exact def-use chains"); - auto ExtensionAction = rcc::riscv::legalizerInfo().classify(**BuiltExtension, *Function, *Target); - auto Folded = ExtensionAction ? rcc::riscv::legalizerInfo().lower(**BuiltExtension, *Function, - *Target, *ExtensionAction) - : rcc::Expected(std::unexpected(ExtensionAction.error())); + auto ExtensionAction = GISel.Legalizer.classify(**BuiltExtension, *Function, *Target); + auto Folded = ExtensionAction + ? GISel.Legalizer.lower(**BuiltExtension, *Function, *Target, *ExtensionAction) + : rcc::Expected(std::unexpected(ExtensionAction.error())); const rcc::Register *FoldedResult = (*BuiltConstant)->operandAt(0)->getIf(); Context.expect(ExtensionAction && *ExtensionAction == rcc::LegalizationAction::Custom && Folded && (*BuiltExtension)->parent() == nullptr && FoldedResult != nullptr && @@ -1178,10 +1155,9 @@ void testNarrowAnyExtBinaryArtifacts(TestContext &Context) { if (!BuiltMultiply || OuterExtension == nullptr) return; - auto Action = rcc::riscv::legalizerInfo().classify(*OuterExtension, **Function, *Target); - auto Lowered = - Action ? rcc::riscv::legalizerInfo().lower(*OuterExtension, **Function, *Target, *Action) - : rcc::Expected(std::unexpected(Action.error())); + auto Action = GISel.Legalizer.classify(*OuterExtension, **Function, *Target); + auto Lowered = Action ? GISel.Legalizer.lower(*OuterExtension, **Function, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); const rcc::Register *LegalLeft = OuterExtension->operandAt(1)->getIf(); const rcc::Register *LegalRight = OuterExtension->operandAt(2)->getIf(); Context.expect( @@ -1200,10 +1176,12 @@ void testMultiplyLibcallWithoutM(TestContext &Context) { auto Target = rcc::riscv::createTargetDescription(); const std::vector FeatureNames{"+a"}; auto Features = rcc::riscv::RISCVSubtargetFeatures::fromFeatureList(FeatureNames); - auto Legalizer = Features ? rcc::riscv::createLegalizerInfo(*Features) : nullptr; - auto Multiply64 = - Target ? createBinary(Context, *Target, rcc::GenericOpcode::G_MUL, 64) : InstructionFixture{}; - if (!Target || !Features || !Legalizer || Multiply64.Instruction == nullptr) + if (!Target || !Features) + return; + const rcc::riscv::RISCVSubtarget Subtarget(*Features); + const rcc::LegalizerInfo *Legalizer = Subtarget.getLegalizerInfo(); + auto Multiply64 = createBinary(Context, *Target, rcc::GenericOpcode::G_MUL, 64); + if (Multiply64.Instruction == nullptr) return; auto Action = Legalizer->classify(*Multiply64.Instruction, *Multiply64.Function, *Target); @@ -1314,8 +1292,7 @@ void testAtomicSelection(TestContext &Context) { Nand.Instruction != nullptr, "atomic RMW fixtures must be complete"); if (Add.Instruction != nullptr) { - auto Selection = - rcc::riscv::instructionSelector().select(*Add.Instruction, *Add.Function, *Target); + auto Selection = (*GISel.Selector).select(*Add.Instruction, *Add.Function, *Target); Context.expect(Selection.has_value() && Add.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::AMOADD_D)), @@ -1324,21 +1301,18 @@ void testAtomicSelection(TestContext &Context) { "selected AMOADD.D must satisfy its descriptor and MMO contract"); } if (Sub.Instruction != nullptr) { - auto Selection = - rcc::riscv::instructionSelector().select(*Sub.Instruction, *Sub.Function, *Target); + auto Selection = (*GISel.Selector).select(*Sub.Instruction, *Sub.Function, *Target); Context.expect(Selection.has_value() && Sub.Function->allInstructions().size() == 2, "G_ATOMICRMW_SUB must negate its operand before AMOADD.D"); Context.expect(Selection && validatesSelectedFunction(*Target, *Sub.Function), "selected atomic subtraction must satisfy target descriptors"); } if (Nand.Instruction != nullptr) { - auto Classification = - rcc::riscv::legalizerInfo().classify(*Nand.Instruction, *Nand.Function, *Target); + auto Classification = GISel.Legalizer.classify(*Nand.Instruction, *Nand.Function, *Target); Context.expect( Classification && *Classification == rcc::LegalizationAction::Unsupported, "an atomic operation without a single-instruction RV64GC mapping must be explicit"); - auto Match = - rcc::riscv::instructionSelector().canSelect(*Nand.Instruction, *Nand.Function, *Target); + auto Match = (*GISel.Selector).canSelect(*Nand.Instruction, *Nand.Function, *Target); Context.expect(!Match && Match.error().code() == rcc::ErrorCode::Unsupported, "unsupported atomic RMW must fail before mutating MachineIR"); Context.expect(!Match && Match.error().message().find("G_ATOMICRMW_NAND") != std::string::npos, @@ -1391,8 +1365,8 @@ void testAtomicSelection(TestContext &Context) { auto AtomicLoad = MakeAtomicAccess(true, 1303); auto AtomicStore = MakeAtomicAccess(false, 1304); if (AtomicLoad.Instruction != nullptr) { - auto Selection = rcc::riscv::instructionSelector().select(*AtomicLoad.Instruction, - *AtomicLoad.Function, *Target); + auto Selection = + (*GISel.Selector).select(*AtomicLoad.Instruction, *AtomicLoad.Function, *Target); Context.expect(Selection && AtomicLoad.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::LR_D)), @@ -1401,8 +1375,8 @@ void testAtomicSelection(TestContext &Context) { "selected atomic load must satisfy target descriptors"); } if (AtomicStore.Instruction != nullptr) { - auto Selection = rcc::riscv::instructionSelector().select(*AtomicStore.Instruction, - *AtomicStore.Function, *Target); + auto Selection = + (*GISel.Selector).select(*AtomicStore.Instruction, *AtomicStore.Function, *Target); Context.expect(Selection && AtomicStore.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( rcc::riscv::InstructionOpcode::AMOSWAP_D)), @@ -1419,8 +1393,8 @@ void testUnsupportedTypes(TestContext &Context) { return; auto WideInteger = createBinary(Context, *Target, rcc::GenericOpcode::G_ADD, 128); if (WideInteger.Function && WideInteger.Instruction != nullptr) { - auto Selection = rcc::riscv::instructionSelector().canSelect(*WideInteger.Instruction, - *WideInteger.Function, *Target); + auto Selection = + (*GISel.Selector).canSelect(*WideInteger.Instruction, *WideInteger.Function, *Target); Context.expect(!Selection && Selection.error().code() == rcc::ErrorCode::Unsupported, "i128 arithmetic must produce an Unsupported diagnostic"); Context.expect(!Selection && Selection.error().message().find("i128") != std::string::npos, @@ -1449,10 +1423,10 @@ void testUnsupportedTypes(TestContext &Context) { if (!Pending->build()) return; const rcc::MachineInstr &Instruction = **Function->get()->allInstructions().begin(); - auto Action = rcc::riscv::legalizerInfo().classify(Instruction, **Function, *Target); + auto Action = GISel.Legalizer.classify(Instruction, **Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Unsupported, "unsupported floating operations must remain explicitly diagnosed"); - auto Selection = rcc::riscv::instructionSelector().canSelect(Instruction, **Function, *Target); + auto Selection = (*GISel.Selector).canSelect(Instruction, **Function, *Target); Context.expect(!Selection && Selection.error().code() == rcc::ErrorCode::Unsupported, "selector pre-matching must reject unsupported floating operations"); Context.expect(!Selection && Selection.error().message().find("G_FEXP") != std::string::npos, @@ -1492,12 +1466,12 @@ void testVACopyLegalization(TestContext &Context) { auto Built = Pending->build(); if (!Built) return; - auto Action = rcc::riscv::legalizerInfo().classify(**Built, **Function, *Target); + auto Action = GISel.Legalizer.classify(**Built, **Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Custom, "RV64GC must custom-legalize va_copy"); if (!Action) return; - auto Lowered = rcc::riscv::legalizerInfo().lower(**Built, **Function, *Target, *Action); + auto Lowered = GISel.Legalizer.lower(**Built, **Function, *Target, *Action); Context.expect(Lowered.has_value(), "RV64GC va_copy legalization must succeed"); if (!Lowered) return; @@ -1575,12 +1549,12 @@ void testVAArgLegalization(TestContext &Context) { Context.expect(Target->validateInstruction(**Built, (*Function)->registerInfo()).has_value(), "canonical G_VAARG without MMOs must satisfy the generic descriptor"); - auto Action = rcc::riscv::legalizerInfo().classify(**Built, **Function, *Target); + auto Action = GISel.Legalizer.classify(**Built, **Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Custom, "RV64GC must custom-legalize G_VAARG"); if (!Action) return; - auto Lowered = rcc::riscv::legalizerInfo().lower(**Built, **Function, *Target, *Action); + auto Lowered = GISel.Legalizer.lower(**Built, **Function, *Target, *Action); Context.expect(Lowered.has_value(), "RV64GC va_arg legalization must succeed"); if (!Lowered) return; @@ -1666,9 +1640,8 @@ void testFrameIndexLegalization(TestContext &Context) { if (!BuiltUnsupported) return; - auto LegalAction = rcc::riscv::legalizerInfo().classify(**BuiltLegal, **Function, *Target); - auto UnsupportedAction = - rcc::riscv::legalizerInfo().classify(**BuiltUnsupported, **Function, *Target); + auto LegalAction = GISel.Legalizer.classify(**BuiltLegal, **Function, *Target); + auto UnsupportedAction = GISel.Legalizer.classify(**BuiltUnsupported, **Function, *Target); Context.expect(LegalAction && *LegalAction == rcc::LegalizationAction::Legal, "RV64GC must legalize G_FRAME_INDEX for p0"); Context.expect(UnsupportedAction && *UnsupportedAction == rcc::LegalizationAction::Unsupported, @@ -1691,10 +1664,10 @@ void testPointerArithmeticLegalization(TestContext &Context) { {*Pointer}, {*Pointer, *S32}); if (!Legal || !Unsupported) return; - auto LegalAction = rcc::riscv::legalizerInfo().classify(*Legal->firstBlock()->firstInstruction(), - *Legal, *Target); - auto UnsupportedAction = rcc::riscv::legalizerInfo().classify( - *Unsupported->firstBlock()->firstInstruction(), *Unsupported, *Target); + auto LegalAction = + GISel.Legalizer.classify(*Legal->firstBlock()->firstInstruction(), *Legal, *Target); + auto UnsupportedAction = GISel.Legalizer.classify(*Unsupported->firstBlock()->firstInstruction(), + *Unsupported, *Target); Context.expect(LegalAction && *LegalAction == rcc::LegalizationAction::Legal, "RV64GC must legalize G_PTR_ADD for p0 and s64"); Context.expect(UnsupportedAction && *UnsupportedAction == rcc::LegalizationAction::Unsupported, @@ -1745,10 +1718,10 @@ void testMemoryOffsetSelection(TestContext &Context) { Context.expect(Instruction != nullptr, "generic load must remain attached to its block"); if (Instruction == nullptr) return; - auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, **Function, *Target); + auto Action = GISel.Legalizer.classify(*Instruction, **Function, *Target); Context.expect(Action && *Action == rcc::LegalizationAction::Legal, "RV64GC legalizer must accept volatile scalar loads"); - auto Selection = rcc::riscv::instructionSelector().select(*Instruction, **Function, *Target); + auto Selection = (*GISel.Selector).select(*Instruction, **Function, *Target); Context.expect(Selection.has_value(), "RV64GC selector must select a volatile scalar load"); if (!Selection) return; diff --git a/backend/unittests/Target/RISCVFrameLoweringTest.cpp b/backend/unittests/Target/RISCVFrameLoweringTest.cpp index 6a6cd5b..285e962 100644 --- a/backend/unittests/Target/RISCVFrameLoweringTest.cpp +++ b/backend/unittests/Target/RISCVFrameLoweringTest.cpp @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "GISel/RISCVCallLowering.h" +#include "RISCV.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineCFGUpdater.h" @@ -16,8 +18,6 @@ #include "rcc/CodeGen/TargetDescription.h" #include "rcc/CodeGen/TargetFrameLowering.h" #include "rcc/CodeGen/TargetRegisterAllocationInfo.h" -#include "rcc/Target/RISCV/RISCVCallLowering.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/unittests/Target/RISCVLateCodeGenTest.cpp b/backend/unittests/Target/RISCVLateCodeGenTest.cpp index d0bd952..92fbc55 100644 --- a/backend/unittests/Target/RISCVLateCodeGenTest.cpp +++ b/backend/unittests/Target/RISCVLateCodeGenTest.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "RISCVInstrInfo.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" @@ -25,7 +26,6 @@ #include "rcc/CodeGen/MachineTailMerge.h" #include "rcc/CodeGen/PrologueEpilogueInsertion.h" #include "rcc/CodeGen/RegisterMask.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/unittests/Target/RISCVRegisterAllocationTest.cpp b/backend/unittests/Target/RISCVRegisterAllocationTest.cpp index 299308c..ad81615 100644 --- a/backend/unittests/Target/RISCVRegisterAllocationTest.cpp +++ b/backend/unittests/Target/RISCVRegisterAllocationTest.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "rcc/CodeGen/LowLevelType.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFrameInfo.h" @@ -16,7 +17,6 @@ #include "rcc/CodeGen/RegAllocPBQP.h" #include "rcc/CodeGen/TargetDescription.h" #include "rcc/CodeGen/TargetRegisterAllocationInfo.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVInstructions.h" #include "rcc/Target/RISCV/RISCVRegisters.h" #include "rcc/Target/RISCV/RISCVTarget.h" diff --git a/backend/unittests/Target/RISCVSSAExitTest.cpp b/backend/unittests/Target/RISCVSSAExitTest.cpp index 18cbea9..eafb5a0 100644 --- a/backend/unittests/Target/RISCVSSAExitTest.cpp +++ b/backend/unittests/Target/RISCVSSAExitTest.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "RISCV.h" #include "RISCVInstrInfo.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" @@ -12,7 +13,6 @@ #include "rcc/CodeGen/MachinePass.h" #include "rcc/CodeGen/MachineSSAExit.h" #include "rcc/CodeGen/MachineVerifier.h" -#include "rcc/Target/RISCV/RISCVCodeGen.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include -- cgit v1.2.3