diff options
17 files changed, 342 insertions, 36 deletions
diff --git a/backend/include/rcc/CodeGen/CommandFlags.h b/backend/include/rcc/CodeGen/CommandFlags.h index 0ee2f0d..aa7ed0d 100644 --- a/backend/include/rcc/CodeGen/CommandFlags.h +++ b/backend/include/rcc/CodeGen/CommandFlags.h @@ -41,6 +41,12 @@ enum class CodeGenFileType : std::uint8_t { Null, }; +enum class GlobalISelAbortMode : std::uint8_t { + Disable, + Enable, + DisableWithDiagnostic, +}; + struct CodeGenCommandFlags final { std::optional<std::string> Triple; std::optional<std::string> CPU; @@ -63,6 +69,7 @@ struct CodeGenCommandFlags final { [[nodiscard]] Expected<CodeModel> parseCodeModel(std::string_view Value); [[nodiscard]] Expected<OptimizationLevel> parseOptimizationLevel(std::string_view Value); [[nodiscard]] Expected<CodeGenFileType> parseCodeGenFileType(std::string_view Value); +[[nodiscard]] Expected<GlobalISelAbortMode> parseGlobalISelAbortMode(std::string_view Value); [[nodiscard]] Expected<bool> parseCodeGenCommandFlag(std::span<const std::string_view> Arguments, std::size_t &Index, CodeGenCommandFlags &Flags); diff --git a/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h b/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h index 2ba0012..e90d09c 100644 --- a/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h +++ b/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h @@ -15,15 +15,18 @@ namespace rcc { class Legalizer final : public MachineFunctionPass { public: Legalizer(const LegalizerInfo *Info, OptimizationLevel Optimization, - LibcallLoweringInfo Libcalls = {}) - : Info(Info), Optimization(Optimization), Libcalls(Libcalls) {} + LibcallLoweringInfo Libcalls = {}, + GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable) + : Info(Info), Optimization(Optimization), Libcalls(Libcalls), AbortMode(AbortMode) {} Legalizer(const LegalizerInfo &Info, OptimizationLevel Optimization, - LibcallLoweringInfo Libcalls = {}) - : Info(&Info), Optimization(Optimization), Libcalls(Libcalls) {} + LibcallLoweringInfo Libcalls = {}, + GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable) + : Info(&Info), Optimization(Optimization), Libcalls(Libcalls), AbortMode(AbortMode) {} Legalizer(std::unique_ptr<LegalizerInfo> Info, OptimizationLevel Optimization, - LibcallLoweringInfo Libcalls = {}) + LibcallLoweringInfo Libcalls = {}, + GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable) : OwnedInfo(std::move(Info)), Info(OwnedInfo.get()), Optimization(Optimization), - Libcalls(Libcalls) {} + Libcalls(Libcalls), AbortMode(AbortMode) {} [[nodiscard]] const MachinePassContract &contract() const noexcept override; [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, @@ -39,6 +42,7 @@ private: const LegalizerInfo *Info = nullptr; OptimizationLevel Optimization; LibcallLoweringInfo Libcalls; + GlobalISelAbortMode AbortMode; }; } // namespace rcc diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h index 3e14311..aff6443 100644 --- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h +++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h @@ -103,6 +103,8 @@ private: const APInt &Amount, LowLevelType TargetType, LowLevelType AmountType); + [[nodiscard]] Expected<void> fewerElements(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType); struct ShiftParameters final { Register WordShift; diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h index 3433e3a..78e6a32 100644 --- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h +++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h @@ -139,6 +139,7 @@ public: LegalizeRuleSet &custom(); LegalizeRuleSet &widenScalarToNextPow2(std::size_t TypeIndex, std::uint32_t MinimumBitWidth = 0); + LegalizeRuleSet &scalarize(std::size_t TypeIndex); LegalizeRuleSet &minScalar(std::size_t TypeIndex, LowLevelType MinimumType); LegalizeRuleSet &minScalar(bool Enabled, std::size_t TypeIndex, LowLevelType MinimumType); LegalizeRuleSet &maxScalar(std::size_t TypeIndex, LowLevelType MaximumType); diff --git a/backend/include/rcc/Target/RISCV/RISCVPipeline.h b/backend/include/rcc/Target/RISCV/RISCVPipeline.h index 07f3ece..262f7c1 100644 --- a/backend/include/rcc/Target/RISCV/RISCVPipeline.h +++ b/backend/include/rcc/Target/RISCV/RISCVPipeline.h @@ -23,6 +23,7 @@ namespace riscv { struct RISCVPipelineOptions final { OptimizationLevel Optimization = OptimizationLevel::O2; bool UseTargetCopyInstructions = false; + GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable; }; [[nodiscard]] Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline, diff --git a/backend/lib/CodeGen/CommandFlags.cpp b/backend/lib/CodeGen/CommandFlags.cpp index dfbaadc..2a659eb 100644 --- a/backend/lib/CodeGen/CommandFlags.cpp +++ b/backend/lib/CodeGen/CommandFlags.cpp @@ -148,6 +148,16 @@ Expected<CodeGenFileType> parseCodeGenFileType(std::string_view Value) { return makeUnexpected(ErrorCode::InvalidInvocation, "unknown file type"); } +Expected<GlobalISelAbortMode> parseGlobalISelAbortMode(std::string_view Value) { + if (Value == "0") + return GlobalISelAbortMode::Disable; + if (Value == "1") + return GlobalISelAbortMode::Enable; + if (Value == "2") + return GlobalISelAbortMode::DisableWithDiagnostic; + return makeUnexpected(ErrorCode::InvalidInvocation, "-global-isel-abort requires 0, 1, or 2"); +} + Expected<bool> parseCodeGenCommandFlag(std::span<const std::string_view> Arguments, std::size_t &Index, CodeGenCommandFlags &Flags) { const std::string_view Argument = Arguments[Index]; diff --git a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp index c001ac5..09b2b66 100644 --- a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -190,6 +190,12 @@ const MachinePassContract &InstructionSelect::contract() const noexcept { Expected<MachinePassResult> InstructionSelect::run(MachineFunction &Function, const TargetDescription &Target, MachineFunctionAnalysisManager &Analyses) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; if (Info == nullptr) return std::unexpected(passFailure("instruction select has no target behavior interface", ErrorCode::InvalidArgument)); diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp index 877777e..8f3f06f 100644 --- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp +++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -258,6 +258,12 @@ const MachinePassContract &Legalizer::contract() const noexcept { Expected<MachinePassResult> Legalizer::run(MachineFunction &Function, const TargetDescription &Target, MachineFunctionAnalysisManager &Analyses) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; if (Info == nullptr) return std::unexpected( passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument)); @@ -265,6 +271,24 @@ Expected<MachinePassResult> Legalizer::run(MachineFunction &Function, if (!Required) return std::unexpected(Required.error()); + if (AbortMode != GlobalISelAbortMode::Enable) { + if (AbortMode != GlobalISelAbortMode::Disable && + AbortMode != GlobalISelAbortMode::DisableWithDiagnostic) { + return std::unexpected(passFailure("legalizer has an unknown GlobalISel abort mode", + ErrorCode::InvalidArgument)); + } + MachineActionReplay Actions(Analyses); + auto Result = runImplementation(Function, Target, Actions); + if (Result || Result.error().code() != ErrorCode::Unsupported) + return Result; + Function.properties().set(MachineFunctionProperty::FailedISel); + return MachinePassResult{ + .Changed = true, + .Changes = {MachinePassChange::Instructions}, + .Preserved = PreservedAnalyses::none(), + }; + } + auto Trial = cloneMachineFunction( Function, Target, MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index b95f113..69396f4 100644 --- a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -336,6 +336,25 @@ buildFlaggedBinaryInstruction(MachineIRBuilder &Builder, GenericOpcode Generic, } } +[[nodiscard]] Expected<std::string> floatingBinaryLibcallName(GenericOpcode Generic, + std::uint32_t BitWidth) { + if (Generic != GenericOpcode::G_FREM) { + return makeUnexpected(ErrorCode::Unsupported, + "floating libcall has no runtime function mapping"); + } + switch (BitWidth) { + case 32: + return "fmodf"; + case 64: + return "fmod"; + case 128: + return "fmodl"; + default: + return makeUnexpected(ErrorCode::Unsupported, + "floating libcall requires a 32, 64, or 128-bit scalar"); + } +} + [[nodiscard]] Expected<std::string> conversionLibcallName(GenericOpcode Generic, std::uint32_t DestinationWidth, std::uint32_t SourceWidth) { @@ -1698,6 +1717,57 @@ Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size break; } + case GenericOpcode::G_FADD: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMA: + case GenericOpcode::G_FMAD: + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FCANONICALIZE: + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: + case GenericOpcode::G_FMINNUM_IEEE: + case GenericOpcode::G_FMAXNUM_IEEE: + case GenericOpcode::G_FMINIMUM: + case GenericOpcode::G_FMAXIMUM: + case GenericOpcode::G_FMINIMUMNUM: + case GenericOpcode::G_FMAXIMUMNUM: + case GenericOpcode::G_FDIV: + case GenericOpcode::G_FREM: + case GenericOpcode::G_FCEIL: + case GenericOpcode::G_FFLOOR: + case GenericOpcode::G_FCOS: + case GenericOpcode::G_FSIN: + case GenericOpcode::G_FTAN: + case GenericOpcode::G_FACOS: + case GenericOpcode::G_FASIN: + case GenericOpcode::G_FATAN: + case GenericOpcode::G_FATAN2: + case GenericOpcode::G_FCOSH: + case GenericOpcode::G_FSINH: + case GenericOpcode::G_FTANH: + case GenericOpcode::G_FLOG10: + case GenericOpcode::G_FLOG: + case GenericOpcode::G_FLOG2: + case GenericOpcode::G_FRINT: + case GenericOpcode::G_FNEARBYINT: + case GenericOpcode::G_FSQRT: + case GenericOpcode::G_FEXP: + case GenericOpcode::G_FEXP2: + case GenericOpcode::G_FEXP10: + case GenericOpcode::G_FPOW: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "floating operation widening has an invalid type index"); + for (std::size_t OperandIndex = 1; OperandIndex < Instruction.operandCount(); ++OperandIndex) { + if (auto Result = Apply(WidenSource(OperandIndex, GenericOpcode::G_FPEXT)); !Result) + return Result; + } + if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result) + return Result; + break; + case GenericOpcode::G_FPEXT: if (TypeIndex != 1) return makeUnexpected(ErrorCode::Unsupported, @@ -3252,6 +3322,127 @@ Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::siz } } +Expected<void> LegalizerHelper::fewerElements(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target()); + if (!Query) + return std::unexpected(Query.error()); + if (TypeIndex >= Query->Types.size() || TypeIndex >= Query->TypeOperands.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization has an invalid type index"); + } + + const LowLevelType VectorType = Query->Types[TypeIndex]; + const std::optional<LowLevelType> ElementType = VectorType.scalarElementType(); + if (!VectorType.isVector() || VectorType.isScalableVector() || !ElementType || + *ElementType != NarrowType || !NarrowType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "vector legalization requires fixed vectors split into scalar elements"); + } + + std::size_t ExplicitOperandCount = 0; + while (ExplicitOperandCount < Instruction.operandCount() && + !Instruction.operandAt(ExplicitOperandCount)->isImplicit()) { + ++ExplicitOperandCount; + } + const std::vector<std::size_t> &TypedOperands = Query->TypeOperands[TypeIndex]; + if (TypedOperands.size() != ExplicitOperandCount || TypedOperands.empty()) { + return makeUnexpected( + ErrorCode::Unsupported, + "vector legalization requires all explicit operands to share one vector type"); + } + + std::vector<std::size_t> DefinitionIndices; + std::vector<std::size_t> SourceIndices; + DefinitionIndices.reserve(TypedOperands.size()); + SourceIndices.reserve(TypedOperands.size()); + for (const std::size_t OperandIndex : TypedOperands) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *RegisterOperand = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (RegisterOperand == nullptr || !RegisterOperand->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization requires virtual-register operands"); + } + auto OperandType = MRI.getType(*RegisterOperand); + if (!OperandType) + return std::unexpected(OperandType.error()); + if (*OperandType != VectorType) { + return makeUnexpected( + ErrorCode::VerificationError, + "vector legalization type operands do not match the requested vector type"); + } + if (Operand->isDef() && !Operand->isUse()) { + DefinitionIndices.push_back(OperandIndex); + } else if (Operand->isUse() && !Operand->isDef()) { + SourceIndices.push_back(OperandIndex); + } else { + return makeUnexpected(ErrorCode::Unsupported, + "vector legalization does not support tied or non-register operands"); + } + } + if (DefinitionIndices.empty() || SourceIndices.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization requires register definitions and sources"); + } + + const std::size_t ElementCount = VectorType.minimumElementCount(); + std::vector<std::vector<Register>> SourceParts; + SourceParts.reserve(SourceIndices.size()); + for (const std::size_t SourceIndex : SourceIndices) { + const Register Source = *Instruction.operandAt(SourceIndex)->getIf<Register>(); + std::vector<Register> Parts; + auto Extracted = + extractParts(Source, NarrowType, static_cast<int>(ElementCount), Parts, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + SourceParts.push_back(std::move(Parts)); + } + + std::vector<std::vector<Register>> DestinationParts(DefinitionIndices.size()); + for (std::vector<Register> &Parts : DestinationParts) + Parts.reserve(ElementCount); + for (std::size_t ElementIndex = 0; ElementIndex < ElementCount; ++ElementIndex) { + auto ScalarInstruction = MIRBuilder.buildInstr(Instruction.opcode()); + if (!ScalarInstruction) + return std::unexpected(ScalarInstruction.error()); + if (auto SetFlags = ScalarInstruction->setFlags(Instruction.flags()); !SetFlags) + return std::unexpected(SetFlags.error()); + for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size(); + ++DefinitionNumber) { + auto Destination = MRI.createVirtualRegister(NarrowType); + if (!Destination) + return std::unexpected(Destination.error()); + if (auto Added = ScalarInstruction->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + DestinationParts[DefinitionNumber].push_back(*Destination); + } + for (const std::vector<Register> &Parts : SourceParts) { + if (auto Added = ScalarInstruction->addUse(Parts[ElementIndex]); !Added) + return std::unexpected(Added.error()); + } + if (auto Built = ScalarInstruction->build(); !Built) + return std::unexpected(Built.error()); + } + + for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size(); + ++DefinitionNumber) { + const Register Destination = + *Instruction.operandAt(DefinitionIndices[DefinitionNumber])->getIf<Register>(); + if (auto Merged = + MIRBuilder.buildMergeLikeInstr(Destination, DestinationParts[DefinitionNumber]); + !Merged) { + return std::unexpected(Merged.error()); + } + } + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "vector instruction became detached during legalization"); + } + return Block->removeInstruction(Instruction); +} + Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) { if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { @@ -4723,6 +4914,7 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) { Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP; + const bool IsFloatingBinary = Generic == GenericOpcode::G_FREM; if (Instruction.operandCount() < 2 || (IsConversion && Instruction.operandCount() != 2)) { return makeUnexpected(ErrorCode::VerificationError, "libcall has an invalid number of operands"); @@ -4744,18 +4936,19 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) { } Expected<std::string> Name = - IsConversion ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(), - SourceType->scalarBitWidth()) - : integerLibcallName(Generic, DestinationType->scalarBitWidth()); + IsConversion ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(), + SourceType->scalarBitWidth()) + : IsFloatingBinary ? floatingBinaryLibcallName(Generic, DestinationType->scalarBitWidth()) + : integerLibcallName(Generic, DestinationType->scalarBitWidth()); if (!Name) return std::unexpected(Name.error()); const bool DestinationIsFloat = Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || - Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP; + Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP || IsFloatingBinary; const bool SourceIsFloat = Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || - Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI; + Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || IsFloatingBinary; const ABIValueCategory DestinationCategory = DestinationIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer; const ABIValueCategory SourceCategory = @@ -4843,7 +5036,7 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) { } if (!IsConversion && *ArgumentType != *DestinationType) { return makeUnexpected(ErrorCode::VerificationError, - "integer libcall source and destination types must match"); + "binary libcall source and destination types must match"); } const ABIExtensionKind Extension = ExtensionFor(SourceCategory, false, ArgumentType->scalarBitWidth()); @@ -4974,11 +5167,12 @@ Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction, return lower(Instruction, Decision); case LegalizationAction::Libcall: return libcall(Instruction); + case LegalizationAction::FewerElements: + return fewerElements(Instruction, Decision.TypeIndex, Decision.NewType); case LegalizationAction::Legal: return {}; case LegalizationAction::Unsupported: case LegalizationAction::MoreElements: - case LegalizationAction::FewerElements: case LegalizationAction::Bitcast: case LegalizationAction::ChangeElementType: case LegalizationAction::ChangeAddressSpace: diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp index ee7f856..bdc783b 100644 --- a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -256,6 +256,16 @@ LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex, }); } +LegalizeRuleSet &LegalizeRuleSet::scalarize(std::size_t TypeIndex) { + return fewerElementsIf( + [TypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isVector(); + }, + [TypeIndex](const LegalizationQuery &Query) { + return std::pair{TypeIndex, *Query.Types[TypeIndex].scalarElementType()}; + }); +} + LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) { return widenScalarIf( [TypeIndex, MinimumType](const LegalizationQuery &Query) { diff --git a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp index ef29333..a5747c3 100644 --- a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp +++ b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -290,6 +290,12 @@ const MachinePassContract &RegBankSelect::contract() const noexcept { Expected<MachinePassResult> RegBankSelect::run(MachineFunction &Function, const TargetDescription &Target, MachineFunctionAnalysisManager &) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; if (Info == nullptr) return std::unexpected(passFailure("register-bank select has no target behavior interface", ErrorCode::InvalidArgument)); diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp index 14ce8ef..68a28db 100644 --- a/backend/lib/CodeGen/MachineVerifier.cpp +++ b/backend/lib/CodeGen/MachineVerifier.cpp @@ -71,6 +71,8 @@ public: : Function(Function), Target(Target), Options(Options) {} [[nodiscard]] Expected<void> run() { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return {}; if (!Function.stableID().isValid()) return fail("function identity is invalid"); diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp index 050de5b..66b4d87 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp @@ -839,9 +839,14 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std:: return *Extended; } -[[nodiscard]] Expected<void> emitOutgoingRegisterAssignment(MachineIRBuilder &Builder, - const CCAssignment &Assignment, - Register Source) { +struct OutgoingRegisterValue final { + PhysicalRegister Location; + Register Value; +}; + +[[nodiscard]] Expected<OutgoingRegisterValue> +prepareOutgoingRegisterAssignment(MachineIRBuilder &Builder, const CCAssignment &Assignment, + Register Source) { if (!Assignment.LocationRegister) { return makeUnexpected(ErrorCode::InternalError, "RISC-V register assignment has no location register"); @@ -849,9 +854,20 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std:: auto Extended = extendOutgoingRegister(Builder, Source, Assignment); if (!Extended) return std::unexpected(Extended.error()); - auto Copy = Builder.buildCopy(Register::fromPhysical(*Assignment.LocationRegister), *Extended); - if (!Copy) - return std::unexpected(Copy.error()); + return OutgoingRegisterValue{ + .Location = *Assignment.LocationRegister, + .Value = *Extended, + }; +} + +[[nodiscard]] Expected<void> +emitOutgoingRegisterAssignments(MachineIRBuilder &Builder, + std::span<const OutgoingRegisterValue> Assignments) { + for (const OutgoingRegisterValue &Assignment : Assignments) { + auto Copy = Builder.buildCopy(Register::fromPhysical(Assignment.Location), Assignment.Value); + if (!Copy) + return std::unexpected(Copy.error()); + } return {}; } @@ -1348,15 +1364,18 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan, MachineFunction &Function = Builder.function(); MachineFrameInfo &Frame = Function.frameInfo(); const auto Assignments = Plan.argumentAssignments(); + std::vector<OutgoingRegisterValue> RegisterAssignments; + RegisterAssignments.reserve(Assignments.size()); for (std::size_t Index = 0; Index < Assignments.size(); ++Index) { const CCAssignment &Assignment = Assignments[Index]; const auto Replacement = Replacements.find(Index); const Register Source = Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second; if (Assignment.LocationRegister) { - auto Copy = emitOutgoingRegisterAssignment(Builder, Assignment, Source); - if (!Copy) - return std::unexpected(Copy.error()); + auto Prepared = prepareOutgoingRegisterAssignment(Builder, Assignment, Source); + if (!Prepared) + return std::unexpected(Prepared.error()); + RegisterAssignments.push_back(*Prepared); continue; } @@ -1388,7 +1407,8 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan, if (!Store) return std::unexpected(Store.error()); } - return {}; + return emitOutgoingRegisterAssignments( + Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments)); } [[nodiscard]] Expected<void> addCallTarget(MachineInstrBuilder &Instruction, @@ -1505,11 +1525,19 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan, } [[nodiscard]] Expected<void> emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) { + std::vector<OutgoingRegisterValue> RegisterAssignments; + RegisterAssignments.reserve(Plan.returnAssignments().size()); for (const CCAssignment &Assignment : Plan.returnAssignments()) { - auto Copy = emitOutgoingRegisterAssignment(Builder, Assignment, Assignment.OriginalValue); - if (!Copy) - return std::unexpected(Copy.error()); - } + auto Prepared = + prepareOutgoingRegisterAssignment(Builder, Assignment, Assignment.OriginalValue); + if (!Prepared) + return std::unexpected(Prepared.error()); + RegisterAssignments.push_back(*Prepared); + } + auto Copies = emitOutgoingRegisterAssignments( + Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments)); + if (!Copies) + return std::unexpected(Copies.error()); auto Return = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoRET)); if (!Return) diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 3f8f4cb..243c6de 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -576,7 +576,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtar .lowerFor({S32, S64, S128}); getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM)) .libcallFor({S32, S64, S128}) - .minScalar(0, S32); + .minScalar(0, S32) + .scalarize(0); getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN)) .legalFor(HasF, {{S32, S32}}) .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}}) diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp index a8d0a20..f1de092 100644 --- a/backend/lib/Target/RISCV/RISCVPipeline.cpp +++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp @@ -45,7 +45,7 @@ template <typename PassT, typename... ArgumentT> if (!Added) return std::unexpected(Added.error()); Added = addPass<Legalizer>(Pipeline, Subtarget.getLegalizerInfo(), Options.Optimization, - libcallLoweringInfo(Subtarget)); + libcallLoweringInfo(Subtarget), Options.GlobalISelAbort); if (!Added) return std::unexpected(Added.error()); Added = addPass<RegBankSelect>(Pipeline, Subtarget.getRegBankInfo()); @@ -100,7 +100,7 @@ Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline, if (!Added) return std::unexpected(Added.error()); Added = addPass<Legalizer>(Pipeline, Subtarget.getLegalizerInfo(), Options.Optimization, - libcallLoweringInfo(Subtarget)); + libcallLoweringInfo(Subtarget), Options.GlobalISelAbort); if (!Added) return std::unexpected(Added.error()); } diff --git a/backend/lib/Tooling/PipelineDriver.cpp b/backend/lib/Tooling/PipelineDriver.cpp index 7f252a6..42c33e7 100644 --- a/backend/lib/Tooling/PipelineDriver.cpp +++ b/backend/lib/Tooling/PipelineDriver.cpp @@ -98,12 +98,22 @@ struct FunctionRunResult final { [[nodiscard]] Expected<MachineFunctionPassManager::PassList> createPasses(const MachineFunction &Function, const BackendInvocation &Invocation, const riscv::RISCVSubtarget &Subtarget, MachinePassManagerOptions Options) { + const auto GlobalISelAbortOption = std::ranges::find( + Invocation.TargetOptions, std::string_view{"global-isel-abort"}, &BackendOption::Name); + GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable; + if (GlobalISelAbortOption != Invocation.TargetOptions.end()) { + auto Parsed = parseGlobalISelAbortMode(GlobalISelAbortOption->Value); + if (!Parsed) + return std::unexpected(Parsed.error()); + GlobalISelAbort = *Parsed; + } MachineFunctionPassManager Pipeline(Options); if (Invocation.PassPipeline.size() == 1 && Invocation.PassPipeline.front() == "default") { auto Added = riscv::addCanonicalPipeline( Pipeline, Function, Subtarget, riscv::RISCVPipelineOptions{.Optimization = Invocation.Optimization, - .UseTargetCopyInstructions = true}); + .UseTargetCopyInstructions = true, + .GlobalISelAbort = GlobalISelAbort}); if (!Added) return std::unexpected(Added.error()); } else { @@ -119,7 +129,8 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio auto Pass = riscv::createCanonicalPipelinePass( Name, Subtarget, riscv::RISCVPipelineOptions{.Optimization = Invocation.Optimization, - .UseTargetCopyInstructions = UseTargetCopyInstructions}); + .UseTargetCopyInstructions = UseTargetCopyInstructions, + .GlobalISelAbort = GlobalISelAbort}); if (!Pass) return std::unexpected(Pass.error()); auto Added = Pipeline.addPass(std::move(*Pass)); diff --git a/backend/tools/MIRTool/MIRToolMain.cpp b/backend/tools/MIRTool/MIRToolMain.cpp index 5528fbb..d6e7d9e 100644 --- a/backend/tools/MIRTool/MIRToolMain.cpp +++ b/backend/tools/MIRTool/MIRToolMain.cpp @@ -371,10 +371,9 @@ void setBackendOption(std::vector<rcc::tooling::BackendOption> &Options, std::st if (!GlobalISelAbort) return std::unexpected(GlobalISelAbort.error()); if (*GlobalISelAbort) { - if (**GlobalISelAbort != "0" && **GlobalISelAbort != "1" && **GlobalISelAbort != "2") { - return rcc::makeUnexpected(ErrorCode::InvalidInvocation, - "-global-isel-abort requires 0, 1, or 2"); - } + auto Mode = rcc::parseGlobalISelAbortMode(**GlobalISelAbort); + if (!Mode) + return std::unexpected(Mode.error()); setBackendOption(Command.Invocation.TargetOptions, "global-isel-abort", std::string(**GlobalISelAbort)); continue; |