From 495c675771f43cfa12ab7a426a538be72fec1805 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Sun, 9 Aug 2026 17:03:19 +0800 Subject: [CodeGen] Implement machine optimization and scheduling --- .../include/rcc/CodeGen/InstructionDescriptor.h | 1 + .../include/rcc/CodeGen/MachineBlockPlacement.h | 25 + .../rcc/CodeGen/MachineBranchOptimization.h | 39 + .../include/rcc/CodeGen/MachineBranchRelaxation.h | 39 + backend/include/rcc/CodeGen/MachineCSE.h | 35 + backend/include/rcc/CodeGen/MachineCombiner.h | 40 + .../include/rcc/CodeGen/MachineCombinerPattern.h | 82 ++ .../include/rcc/CodeGen/MachineCompilationStatus.h | 1 + .../include/rcc/CodeGen/MachineCopyPropagation.h | 46 + backend/include/rcc/CodeGen/MachineDCE.h | 24 + backend/include/rcc/CodeGen/MachineFallthrough.h | 27 + backend/include/rcc/CodeGen/MachineFinalization.h | 36 + backend/include/rcc/CodeGen/MachineFunction.h | 4 + .../include/rcc/CodeGen/MachineFunctionRecords.h | 3 + backend/include/rcc/CodeGen/MachineIRTypes.h | 4 + backend/include/rcc/CodeGen/MachineLICM.h | 45 + .../include/rcc/CodeGen/MachineLivenessRepair.h | 23 + backend/include/rcc/CodeGen/MachineMemoryFolding.h | 39 + backend/include/rcc/CodeGen/MachineOperand.h | 1 + .../rcc/CodeGen/MachineOptimizationPipeline.h | 43 + .../include/rcc/CodeGen/MachinePseudoExpansion.h | 72 + backend/include/rcc/CodeGen/MachineScheduler.h | 37 + .../include/rcc/CodeGen/MachineShrinkWrapping.h | 36 + backend/include/rcc/CodeGen/MachineSink.h | 38 + .../include/rcc/CodeGen/MachineStackSlotColoring.h | 34 + .../include/rcc/CodeGen/MachineTailDuplication.h | 40 + backend/include/rcc/CodeGen/MachineTailMerge.h | 39 + backend/include/rcc/CodeGen/PostRAScheduler.h | 31 + backend/include/rcc/CodeGen/RegisterPressure.h | 107 ++ backend/include/rcc/CodeGen/ScheduleDAG.h | 110 ++ .../include/rcc/CodeGen/ScheduleHazardRecognizer.h | 68 + backend/include/rcc/CodeGen/TargetBranchInfo.h | 66 + backend/include/rcc/CodeGen/TargetFrameLowering.h | 5 + backend/include/rcc/CodeGen/TargetMemoryFolding.h | 43 + .../include/rcc/CodeGen/TargetPseudoExpansion.h | 40 + .../include/rcc/CodeGen/TargetSchedulingModel.h | 178 +++ backend/include/rcc/Support/Error.h | 5 + backend/include/rcc/Target/RISCV/RISCVCodeGen.h | 12 + backend/lib/CodeGen/CMakeLists.txt | 105 ++ backend/lib/CodeGen/InstructionDescriptor.cpp | 2 +- backend/lib/CodeGen/MachineBlockPlacement.cpp | 242 ++++ backend/lib/CodeGen/MachineBranchOptimization.cpp | 361 +++++ backend/lib/CodeGen/MachineBranchRelaxation.cpp | 194 +++ backend/lib/CodeGen/MachineCSE.cpp | 417 ++++++ backend/lib/CodeGen/MachineCombiner.cpp | 354 +++++ backend/lib/CodeGen/MachineCombinerPattern.cpp | 50 + backend/lib/CodeGen/MachineCompilationStatus.cpp | 6 + backend/lib/CodeGen/MachineCopyPropagation.cpp | 362 +++++ backend/lib/CodeGen/MachineDCE.cpp | 199 +++ backend/lib/CodeGen/MachineFallthrough.cpp | 169 +++ backend/lib/CodeGen/MachineFinalization.cpp | 217 +++ backend/lib/CodeGen/MachineFunction.cpp | 60 + backend/lib/CodeGen/MachineFunctionRecords.cpp | 32 + backend/lib/CodeGen/MachineLICM.cpp | 438 ++++++ backend/lib/CodeGen/MachineMemoryFolding.cpp | 248 ++++ backend/lib/CodeGen/MachineOperand.cpp | 12 + .../lib/CodeGen/MachineOptimizationPipeline.cpp | 156 +++ backend/lib/CodeGen/MachinePseudoExpansion.cpp | 432 ++++++ backend/lib/CodeGen/MachineScheduler.cpp | 441 ++++++ backend/lib/CodeGen/MachineSchedulerInternal.h | 32 + backend/lib/CodeGen/MachineShrinkWrapping.cpp | 227 ++++ backend/lib/CodeGen/MachineSink.cpp | 442 ++++++ backend/lib/CodeGen/MachineStackSlotColoring.cpp | 320 +++++ backend/lib/CodeGen/MachineTailDuplication.cpp | 330 +++++ backend/lib/CodeGen/MachineTailMerge.cpp | 406 ++++++ backend/lib/CodeGen/PostRAScheduler.cpp | 327 +++++ backend/lib/CodeGen/RegisterPressure.cpp | 591 ++++++++ backend/lib/CodeGen/ScheduleDAG.cpp | 514 +++++++ backend/lib/CodeGen/ScheduleHazardRecognizer.cpp | 219 +++ backend/lib/CodeGen/TargetSchedulingModel.cpp | 260 ++++ backend/lib/Support/Error.cpp | 10 + .../Description/TargetDescriptionGenerator.cpp | 7 +- backend/lib/Target/RISCV/CMakeLists.txt | 18 + backend/lib/Target/RISCV/RISCV.rccdesc | 57 +- backend/lib/Target/RISCV/RISCVBranchInfo.cpp | 731 ++++++++++ backend/lib/Target/RISCV/RISCVFrameLowering.cpp | 113 +- backend/lib/Target/RISCV/RISCVMachineCombiner.cpp | 92 ++ .../lib/Target/RISCV/RISCVMachineMemoryFolding.cpp | 48 + .../lib/Target/RISCV/RISCVOptimizationPipeline.cpp | 19 + backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp | 512 +++++++ backend/lib/Target/RISCV/RISCVScheduling.cpp | 136 ++ backend/unittests/CodeGen/CMakeLists.txt | 22 + .../CodeGen/InstructionDescriptorTest.cpp | 10 + backend/unittests/CodeGen/MachineOperandTest.cpp | 43 + .../unittests/CodeGen/MachineOptimizationTest.cpp | 772 +++++++++++ .../unittests/CodeGen/MachineSchedulingTest.cpp | 518 +++++++ backend/unittests/Support/ErrorTest.cpp | 10 + backend/unittests/Target/CMakeLists.txt | 13 + .../unittests/Target/DescriptionGeneratorTest.cpp | 2 +- backend/unittests/Target/RISCVLateCodeGenTest.cpp | 1432 ++++++++++++++++++++ backend/unittests/Target/RISCVTargetTest.cpp | 4 +- 91 files changed, 14212 insertions(+), 40 deletions(-) create mode 100644 backend/include/rcc/CodeGen/MachineBlockPlacement.h create mode 100644 backend/include/rcc/CodeGen/MachineBranchOptimization.h create mode 100644 backend/include/rcc/CodeGen/MachineBranchRelaxation.h create mode 100644 backend/include/rcc/CodeGen/MachineCSE.h create mode 100644 backend/include/rcc/CodeGen/MachineCombiner.h create mode 100644 backend/include/rcc/CodeGen/MachineCombinerPattern.h create mode 100644 backend/include/rcc/CodeGen/MachineCopyPropagation.h create mode 100644 backend/include/rcc/CodeGen/MachineDCE.h create mode 100644 backend/include/rcc/CodeGen/MachineFallthrough.h create mode 100644 backend/include/rcc/CodeGen/MachineFinalization.h create mode 100644 backend/include/rcc/CodeGen/MachineLICM.h create mode 100644 backend/include/rcc/CodeGen/MachineLivenessRepair.h create mode 100644 backend/include/rcc/CodeGen/MachineMemoryFolding.h create mode 100644 backend/include/rcc/CodeGen/MachineOptimizationPipeline.h create mode 100644 backend/include/rcc/CodeGen/MachinePseudoExpansion.h create mode 100644 backend/include/rcc/CodeGen/MachineScheduler.h create mode 100644 backend/include/rcc/CodeGen/MachineShrinkWrapping.h create mode 100644 backend/include/rcc/CodeGen/MachineSink.h create mode 100644 backend/include/rcc/CodeGen/MachineStackSlotColoring.h create mode 100644 backend/include/rcc/CodeGen/MachineTailDuplication.h create mode 100644 backend/include/rcc/CodeGen/MachineTailMerge.h create mode 100644 backend/include/rcc/CodeGen/PostRAScheduler.h create mode 100644 backend/include/rcc/CodeGen/RegisterPressure.h create mode 100644 backend/include/rcc/CodeGen/ScheduleDAG.h create mode 100644 backend/include/rcc/CodeGen/ScheduleHazardRecognizer.h create mode 100644 backend/include/rcc/CodeGen/TargetBranchInfo.h create mode 100644 backend/include/rcc/CodeGen/TargetMemoryFolding.h create mode 100644 backend/include/rcc/CodeGen/TargetPseudoExpansion.h create mode 100644 backend/include/rcc/CodeGen/TargetSchedulingModel.h create mode 100644 backend/lib/CodeGen/MachineBlockPlacement.cpp create mode 100644 backend/lib/CodeGen/MachineBranchOptimization.cpp create mode 100644 backend/lib/CodeGen/MachineBranchRelaxation.cpp create mode 100644 backend/lib/CodeGen/MachineCSE.cpp create mode 100644 backend/lib/CodeGen/MachineCombiner.cpp create mode 100644 backend/lib/CodeGen/MachineCombinerPattern.cpp create mode 100644 backend/lib/CodeGen/MachineCopyPropagation.cpp create mode 100644 backend/lib/CodeGen/MachineDCE.cpp create mode 100644 backend/lib/CodeGen/MachineFallthrough.cpp create mode 100644 backend/lib/CodeGen/MachineFinalization.cpp create mode 100644 backend/lib/CodeGen/MachineLICM.cpp create mode 100644 backend/lib/CodeGen/MachineMemoryFolding.cpp create mode 100644 backend/lib/CodeGen/MachineOptimizationPipeline.cpp create mode 100644 backend/lib/CodeGen/MachinePseudoExpansion.cpp create mode 100644 backend/lib/CodeGen/MachineScheduler.cpp create mode 100644 backend/lib/CodeGen/MachineSchedulerInternal.h create mode 100644 backend/lib/CodeGen/MachineShrinkWrapping.cpp create mode 100644 backend/lib/CodeGen/MachineSink.cpp create mode 100644 backend/lib/CodeGen/MachineStackSlotColoring.cpp create mode 100644 backend/lib/CodeGen/MachineTailDuplication.cpp create mode 100644 backend/lib/CodeGen/MachineTailMerge.cpp create mode 100644 backend/lib/CodeGen/PostRAScheduler.cpp create mode 100644 backend/lib/CodeGen/RegisterPressure.cpp create mode 100644 backend/lib/CodeGen/ScheduleDAG.cpp create mode 100644 backend/lib/CodeGen/ScheduleHazardRecognizer.cpp create mode 100644 backend/lib/CodeGen/TargetSchedulingModel.cpp create mode 100644 backend/lib/Target/RISCV/RISCVBranchInfo.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineCombiner.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp create mode 100644 backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp create mode 100644 backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp create mode 100644 backend/lib/Target/RISCV/RISCVScheduling.cpp create mode 100644 backend/unittests/CodeGen/MachineOptimizationTest.cpp create mode 100644 backend/unittests/CodeGen/MachineSchedulingTest.cpp create mode 100644 backend/unittests/Target/RISCVLateCodeGenTest.cpp diff --git a/backend/include/rcc/CodeGen/InstructionDescriptor.h b/backend/include/rcc/CodeGen/InstructionDescriptor.h index ca2c5cf..ad71213 100644 --- a/backend/include/rcc/CodeGen/InstructionDescriptor.h +++ b/backend/include/rcc/CodeGen/InstructionDescriptor.h @@ -199,6 +199,7 @@ enum class PseudoOwner : std::uint8_t { PostRAExpansion, Finalization, AsmPrinter, + TargetLateExpansion, }; enum class PipelineStage : std::uint8_t { diff --git a/backend/include/rcc/CodeGen/MachineBlockPlacement.h b/backend/include/rcc/CodeGen/MachineBlockPlacement.h new file mode 100644 index 0000000..27a06f4 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineBlockPlacement.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEBLOCKPLACEMENT_H +#define RCC_CODEGEN_MACHINEBLOCKPLACEMENT_H + +#include "rcc/CodeGen/MachinePass.h" + +namespace rcc { + +class MachineBlockPlacementPass final : public MachineFunctionPass { +public: + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineAnalysisManager &Analyses) const; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEBLOCKPLACEMENT_H diff --git a/backend/include/rcc/CodeGen/MachineBranchOptimization.h b/backend/include/rcc/CodeGen/MachineBranchOptimization.h new file mode 100644 index 0000000..a71c20f --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineBranchOptimization.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEBRANCHOPTIMIZATION_H +#define RCC_CODEGEN_MACHINEBRANCHOPTIMIZATION_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetBranchInfo; + +struct MachineBranchOptimizationOptions final { + std::size_t MaximumThreadedEdges = 10'000; +}; + +class MachineBranchOptimizationPass final : public MachineFunctionPass { +public: + explicit MachineBranchOptimizationPass(const TargetBranchInfo &Info, + MachineBranchOptimizationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetBranchInfo *Info; + MachineBranchOptimizationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEBRANCHOPTIMIZATION_H diff --git a/backend/include/rcc/CodeGen/MachineBranchRelaxation.h b/backend/include/rcc/CodeGen/MachineBranchRelaxation.h new file mode 100644 index 0000000..1793a54 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineBranchRelaxation.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEBRANCHRELAXATION_H +#define RCC_CODEGEN_MACHINEBRANCHRELAXATION_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetBranchInfo; + +struct MachineBranchRelaxationOptions final { + std::size_t ConvergenceWatchdog = 1000000; +}; + +class MachineBranchRelaxationPass final : public MachineFunctionPass { +public: + explicit MachineBranchRelaxationPass(const TargetBranchInfo &Info, + MachineBranchRelaxationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetBranchInfo *Info; + MachineBranchRelaxationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEBRANCHRELAXATION_H diff --git a/backend/include/rcc/CodeGen/MachineCSE.h b/backend/include/rcc/CodeGen/MachineCSE.h new file mode 100644 index 0000000..1c2057b --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineCSE.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINECSE_H +#define RCC_CODEGEN_MACHINECSE_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +struct MachineCSEOptions final { + std::size_t MaximumComparisons = 100'000; +}; + +class MachineCSEPass final : public MachineFunctionPass { +public: + explicit MachineCSEPass(MachineCSEOptions Options = {}) : Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses); + + MachineCSEOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINECSE_H diff --git a/backend/include/rcc/CodeGen/MachineCombiner.h b/backend/include/rcc/CodeGen/MachineCombiner.h new file mode 100644 index 0000000..943d6ad --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineCombiner.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINECOMBINER_H +#define RCC_CODEGEN_MACHINECOMBINER_H + +#include "rcc/CodeGen/MachineCombinerPattern.h" +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +struct MachineCombinerOptions final { + std::size_t MaximumRewrites = 4096; + std::size_t MaximumIterations = 8; +}; + +class MachineCombinerPass final : public MachineFunctionPass { +public: + MachineCombinerPass(const TargetMachineCombinerInfo &Info, MachineCombinerStage Stage, + MachineCombinerOptions Options = {}) noexcept + : Info(&Info), Stage(Stage), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetMachineCombinerInfo *Info; + MachineCombinerStage Stage; + MachineCombinerOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINECOMBINER_H diff --git a/backend/include/rcc/CodeGen/MachineCombinerPattern.h b/backend/include/rcc/CodeGen/MachineCombinerPattern.h new file mode 100644 index 0000000..69df70a --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineCombinerPattern.h @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINECOMBINERPATTERN_H +#define RCC_CODEGEN_MACHINECOMBINERPATTERN_H + +#include "rcc/CodeGen/MachineIRTypes.h" +#include "rcc/Support/Error.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc { + +enum class MachineCombinerStage : std::uint8_t { + SelectedSSA, + PreRegisterAllocation, + PostRegisterAllocation, + PostFrame, +}; + +using MachineCombinerStageMask = std::uint8_t; + +[[nodiscard]] constexpr MachineCombinerStageMask +machineCombinerStageMask(MachineCombinerStage Stage) noexcept { + return static_cast(1U << static_cast(Stage)); +} + +enum class MachineCombinerPredicateKind : std::uint8_t { + ImmediateEquals, + RegisterEquals, + PhysicalRegisterEquals, + RegisterIsVirtual, + RegisterIsPhysical, +}; + +struct MachineCombinerPredicate final { + MachineCombinerPredicateKind Kind = MachineCombinerPredicateKind::ImmediateEquals; + std::size_t Operand = 0; + std::size_t OtherOperand = 0; + std::int64_t Value = 0; +}; + +enum class MachineCombinerActionKind : std::uint8_t { + ReplaceDefWithOperandAndErase, +}; + +struct MachineCombinerAction final { + MachineCombinerActionKind Kind = MachineCombinerActionKind::ReplaceDefWithOperandAndErase; + std::size_t DefinitionOperand = 0; + std::size_t SourceOperand = 0; +}; + +struct MachineCombinerPattern final { + std::string Name; + Opcode RootOpcode; + MachineCombinerStageMask Stages = 0; + std::vector Predicates; + MachineCombinerAction Action; +}; + +class TargetMachineCombinerInfo final { +public: + [[nodiscard]] static Expected + create(std::vector Patterns); + + [[nodiscard]] std::span patterns(Opcode RootOpcode) const noexcept; + +private: + explicit TargetMachineCombinerInfo(std::vector Patterns) + : Patterns(std::move(Patterns)) {} + + std::vector Patterns; + std::unordered_map> MatchTable; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINECOMBINERPATTERN_H diff --git a/backend/include/rcc/CodeGen/MachineCompilationStatus.h b/backend/include/rcc/CodeGen/MachineCompilationStatus.h index f889574..9746a4d 100644 --- a/backend/include/rcc/CodeGen/MachineCompilationStatus.h +++ b/backend/include/rcc/CodeGen/MachineCompilationStatus.h @@ -29,6 +29,7 @@ enum class MachineCompilationState : std::uint8_t { FinalizationFailed, AssemblyLoweringFailed, InternalError, + SchedulingFailed, Count, }; diff --git a/backend/include/rcc/CodeGen/MachineCopyPropagation.h b/backend/include/rcc/CodeGen/MachineCopyPropagation.h new file mode 100644 index 0000000..5f84ac6 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineCopyPropagation.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINECOPYPROPAGATION_H +#define RCC_CODEGEN_MACHINECOPYPROPAGATION_H + +#include "rcc/CodeGen/MachinePass.h" + +#include +#include + +namespace rcc { + +class TargetSSAInfo; + +enum class MachineCopyPropagationStage : std::uint8_t { + PreRegisterAllocation, + PostRegisterAllocation, +}; + +struct MachineCopyPropagationOptions final { + std::size_t MaximumPropagatedCopies = 10'000; +}; + +class MachineCopyPropagationPass final : public MachineFunctionPass { +public: + MachineCopyPropagationPass(const TargetSSAInfo &Info, MachineCopyPropagationStage Stage, + MachineCopyPropagationOptions Options = {}) noexcept + : Info(&Info), Stage(Stage), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetSSAInfo *Info; + MachineCopyPropagationStage Stage; + MachineCopyPropagationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINECOPYPROPAGATION_H diff --git a/backend/include/rcc/CodeGen/MachineDCE.h b/backend/include/rcc/CodeGen/MachineDCE.h new file mode 100644 index 0000000..893108d --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineDCE.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEDCE_H +#define RCC_CODEGEN_MACHINEDCE_H + +#include "rcc/CodeGen/MachinePass.h" + +namespace rcc { + +class MachineDCEPass final : public MachineFunctionPass { +public: + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target); +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEDCE_H diff --git a/backend/include/rcc/CodeGen/MachineFallthrough.h b/backend/include/rcc/CodeGen/MachineFallthrough.h new file mode 100644 index 0000000..413efe2 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineFallthrough.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEFALLTHROUGH_H +#define RCC_CODEGEN_MACHINEFALLTHROUGH_H + +#include "rcc/CodeGen/MachinePass.h" + +namespace rcc { + +class TargetBranchInfo; + +class MachineFallthroughPass final : public MachineFunctionPass { +public: + explicit MachineFallthroughPass(const TargetBranchInfo &Info) noexcept : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetBranchInfo *Info; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEFALLTHROUGH_H diff --git a/backend/include/rcc/CodeGen/MachineFinalization.h b/backend/include/rcc/CodeGen/MachineFinalization.h new file mode 100644 index 0000000..147da86 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineFinalization.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEFINALIZATION_H +#define RCC_CODEGEN_MACHINEFINALIZATION_H + +#include "rcc/CodeGen/MachinePass.h" + +namespace rcc { + +class TargetBranchInfo; + +class FinalizeEmissionPseudosPass final : public MachineFunctionPass { +public: + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; +}; + +class FinalEncodabilityVerifierPass final : public MachineFunctionPass { +public: + explicit FinalEncodabilityVerifierPass(const TargetBranchInfo &BranchInfo) noexcept + : BranchInfo(&BranchInfo) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetBranchInfo *BranchInfo; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEFINALIZATION_H diff --git a/backend/include/rcc/CodeGen/MachineFunction.h b/backend/include/rcc/CodeGen/MachineFunction.h index c188d4a..563aa4e 100644 --- a/backend/include/rcc/CodeGen/MachineFunction.h +++ b/backend/include/rcc/CodeGen/MachineFunction.h @@ -157,6 +157,10 @@ public: [[nodiscard]] Expected appendBlock(MachineBasicBlock &Block); [[nodiscard]] Expected insertBlockBefore(MachineBasicBlock &Position, MachineBasicBlock &Block); + [[nodiscard]] Expected moveBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block); + [[nodiscard]] Expected moveBlockAfter(MachineBasicBlock &Position, + MachineBasicBlock &Block); [[nodiscard]] Expected removeBlock(MachineBasicBlock &Block); [[nodiscard]] MachineBasicBlock *firstBlock() noexcept { diff --git a/backend/include/rcc/CodeGen/MachineFunctionRecords.h b/backend/include/rcc/CodeGen/MachineFunctionRecords.h index b792494..645d514 100644 --- a/backend/include/rcc/CodeGen/MachineFunctionRecords.h +++ b/backend/include/rcc/CodeGen/MachineFunctionRecords.h @@ -117,6 +117,9 @@ public: [[nodiscard]] Expected addCalledGlobal(MachineCalledGlobal CalledGlobal); + [[nodiscard]] Expected replaceInstructionReference(MachineInstrStableID OldInstruction, + MachineInstrStableID NewInstruction); + [[nodiscard]] std::span prefetchTargets() const noexcept { return PrefetchTargets; } diff --git a/backend/include/rcc/CodeGen/MachineIRTypes.h b/backend/include/rcc/CodeGen/MachineIRTypes.h index ea65eff..45b5ecf 100644 --- a/backend/include/rcc/CodeGen/MachineIRTypes.h +++ b/backend/include/rcc/CodeGen/MachineIRTypes.h @@ -50,6 +50,8 @@ struct RegisterUnitIDTag; struct RegisterBankIDTag; struct RegisterClassIDTag; struct SchedulingClassIDTag; +struct ProcessorResourceIDTag; +struct RegisterPressureSetIDTag; struct ImmediatePredicateIDTag; struct NamedOperandIDTag; struct TargetOperandTypeIDTag; @@ -76,6 +78,8 @@ using RegisterUnitID = MachineIndex; using RegisterBankID = MachineIndex; using RegisterClassID = MachineIndex; using SchedulingClassID = MachineIndex; +using ProcessorResourceID = MachineIndex; +using RegisterPressureSetID = MachineIndex; using ImmediatePredicateID = MachineIndex; using NamedOperandID = MachineIndex; using TargetOperandTypeID = MachineIndex; diff --git a/backend/include/rcc/CodeGen/MachineLICM.h b/backend/include/rcc/CodeGen/MachineLICM.h new file mode 100644 index 0000000..66b7ef8 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineLICM.h @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINELICM_H +#define RCC_CODEGEN_MACHINELICM_H + +#include "rcc/CodeGen/MachinePass.h" + +#include +#include + +namespace rcc { + +enum class MachineCodeMotionStage : std::uint8_t { + SelectedSSA, + PostRegisterAllocation, +}; + +struct MachineLICMOptions final { + std::size_t MaximumHoistedInstructions = 10'000; + std::size_t MaximumHoistedDefinitionsPerLoop = 256; +}; + +class MachineLICMPass final : public MachineFunctionPass { +public: + explicit MachineLICMPass(MachineLICMOptions Options = {}) : Options(Options) {} + MachineLICMPass(MachineCodeMotionStage Stage, MachineLICMOptions Options = {}) + : Stage(Stage), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses); + + MachineCodeMotionStage Stage = MachineCodeMotionStage::SelectedSSA; + MachineLICMOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINELICM_H diff --git a/backend/include/rcc/CodeGen/MachineLivenessRepair.h b/backend/include/rcc/CodeGen/MachineLivenessRepair.h new file mode 100644 index 0000000..049e166 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineLivenessRepair.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINELIVENESSREPAIR_H +#define RCC_CODEGEN_MACHINELIVENESSREPAIR_H + +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/Support/Error.h" + +#include + +namespace rcc { + +class MachineBasicBlock; +class TargetDescription; + +[[nodiscard]] Expected +repairPostRAMachineBlockLiveness(std::span Blocks, + const TargetDescription &Target, + const LiveVariablesAnalysis::Result &Liveness); + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINELIVENESSREPAIR_H diff --git a/backend/include/rcc/CodeGen/MachineMemoryFolding.h b/backend/include/rcc/CodeGen/MachineMemoryFolding.h new file mode 100644 index 0000000..9a5b47a --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineMemoryFolding.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEMEMORYFOLDING_H +#define RCC_CODEGEN_MACHINEMEMORYFOLDING_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetMemoryFoldingInfo; + +struct MachineMemoryFoldingOptions final { + std::size_t MaximumFolds = 4'096; +}; + +class MachineMemoryFoldingPass final : public MachineFunctionPass { +public: + explicit MachineMemoryFoldingPass(const TargetMemoryFoldingInfo &Info, + MachineMemoryFoldingOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetMemoryFoldingInfo *Info; + MachineMemoryFoldingOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEMEMORYFOLDING_H diff --git a/backend/include/rcc/CodeGen/MachineOperand.h b/backend/include/rcc/CodeGen/MachineOperand.h index 31ad7e4..986ba0e 100644 --- a/backend/include/rcc/CodeGen/MachineOperand.h +++ b/backend/include/rcc/CodeGen/MachineOperand.h @@ -450,6 +450,7 @@ public: [[nodiscard]] Expected setRegisterFlags(RegisterOperandFlags Flags); [[nodiscard]] Expected setSubReg(SubRegIndex SubReg); [[nodiscard]] Expected setBasicBlock(MachineBasicBlock &Block); + [[nodiscard]] Expected setFrameIndex(FrameIndex Index); [[nodiscard]] std::optional tiedOperandIndex() const noexcept { return TiedOperandIndex; diff --git a/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h b/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h new file mode 100644 index 0000000..0953437 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEOPTIMIZATIONPIPELINE_H +#define RCC_CODEGEN_MACHINEOPTIMIZATIONPIPELINE_H + +#include "rcc/CodeGen/MachinePass.h" + +namespace rcc { + +class TargetBranchInfo; +class TargetFrameLowering; +class TargetMachineCombinerInfo; +class TargetMemoryFoldingInfo; +class TargetPseudoExpansionInfo; +class TargetSchedulingModel; +class TargetSSAInfo; + +struct MachineOptimizationPipelineTargetInfo final { + const TargetSSAInfo &SSA; + const TargetMachineCombinerInfo &Combiner; + const TargetMemoryFoldingInfo &MemoryFolding; + const TargetFrameLowering &FrameLowering; + const TargetPseudoExpansionInfo &PseudoExpansion; + const TargetSchedulingModel &SchedulingModel; + const TargetBranchInfo &BranchInfo; +}; + +[[nodiscard]] Expected registerMachineOptimizationAnalyses(MachineAnalysisManager &Analyses); + +[[nodiscard]] Expected +addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo); + +[[nodiscard]] Expected addPreRegisterAllocationOptimizationPipeline( + MachinePassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo); + +[[nodiscard]] Expected +addPostRegisterAllocationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo); + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEOPTIMIZATIONPIPELINE_H diff --git a/backend/include/rcc/CodeGen/MachinePseudoExpansion.h b/backend/include/rcc/CodeGen/MachinePseudoExpansion.h new file mode 100644 index 0000000..1dfb6ba --- /dev/null +++ b/backend/include/rcc/CodeGen/MachinePseudoExpansion.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEPSEUDOEXPANSION_H +#define RCC_CODEGEN_MACHINEPSEUDOEXPANSION_H + +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/TargetPseudoExpansion.h" + +namespace rcc { + +class TargetBranchInfo; +class TargetPseudoExpansionInfo; + +class PreparePostRAPseudosPass final : public MachineFunctionPass { +public: + explicit PreparePostRAPseudosPass(const TargetPseudoExpansionInfo &Info) noexcept : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetPseudoExpansionInfo *Info; +}; + +class PrePEITargetPseudoExpansionPass final : public MachineFunctionPass { +public: + explicit PrePEITargetPseudoExpansionPass(const TargetPseudoExpansionInfo &Info) noexcept + : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetPseudoExpansionInfo *Info; +}; + +class ExpandPostRAPseudosPass final : public MachineFunctionPass { +public: + explicit ExpandPostRAPseudosPass(const TargetPseudoExpansionInfo &Info) noexcept : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetPseudoExpansionInfo *Info; +}; + +class TargetLatePseudoExpansionPass final : public MachineFunctionPass { +public: + TargetLatePseudoExpansionPass(const TargetPseudoExpansionInfo &Info, + const TargetBranchInfo &BranchInfo) noexcept + : Info(&Info), BranchInfo(&BranchInfo) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetPseudoExpansionInfo *Info; + const TargetBranchInfo *BranchInfo; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEPSEUDOEXPANSION_H diff --git a/backend/include/rcc/CodeGen/MachineScheduler.h b/backend/include/rcc/CodeGen/MachineScheduler.h new file mode 100644 index 0000000..2b48fe7 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineScheduler.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINESCHEDULER_H +#define RCC_CODEGEN_MACHINESCHEDULER_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetSchedulingModel; + +struct MachineSchedulerOptions final { + std::size_t MaximumRegionSize = 512; + bool EnableBidirectionalScheduling = true; +}; + +class MachineSchedulerPass final : public MachineFunctionPass { +public: + explicit MachineSchedulerPass(const TargetSchedulingModel &Model, + MachineSchedulerOptions Options = {}) noexcept + : Model(&Model), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetSchedulingModel *Model; + MachineSchedulerOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINESCHEDULER_H diff --git a/backend/include/rcc/CodeGen/MachineShrinkWrapping.h b/backend/include/rcc/CodeGen/MachineShrinkWrapping.h new file mode 100644 index 0000000..ec295cd --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineShrinkWrapping.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINESHRINKWRAPPING_H +#define RCC_CODEGEN_MACHINESHRINKWRAPPING_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetFrameLowering; + +struct MachineShrinkWrappingOptions final { + std::size_t MaximumAnalyzedBlocks = 10'000; +}; + +class MachineShrinkWrappingPass final : public MachineFunctionPass { +public: + explicit MachineShrinkWrappingPass(const TargetFrameLowering &Info, + MachineShrinkWrappingOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetFrameLowering *Info; + MachineShrinkWrappingOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINESHRINKWRAPPING_H diff --git a/backend/include/rcc/CodeGen/MachineSink.h b/backend/include/rcc/CodeGen/MachineSink.h new file mode 100644 index 0000000..92aceba --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineSink.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINESINK_H +#define RCC_CODEGEN_MACHINESINK_H + +#include "rcc/CodeGen/MachineLICM.h" + +#include + +namespace rcc { + +struct MachineSinkOptions final { + std::size_t MaximumSunkInstructions = 10'000; +}; + +class MachineSinkPass final : public MachineFunctionPass { +public: + explicit MachineSinkPass(MachineSinkOptions Options = {}) : Options(Options) {} + MachineSinkPass(MachineCodeMotionStage Stage, MachineSinkOptions Options = {}) + : Stage(Stage), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses); + + MachineCodeMotionStage Stage = MachineCodeMotionStage::SelectedSSA; + MachineSinkOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINESINK_H diff --git a/backend/include/rcc/CodeGen/MachineStackSlotColoring.h b/backend/include/rcc/CodeGen/MachineStackSlotColoring.h new file mode 100644 index 0000000..19c1c70 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineStackSlotColoring.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINESTACKSLOTCOLORING_H +#define RCC_CODEGEN_MACHINESTACKSLOTCOLORING_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +struct MachineStackSlotColoringOptions final { + std::size_t MaximumColoredSlots = 4'096; +}; + +class MachineStackSlotColoringPass final : public MachineFunctionPass { +public: + explicit MachineStackSlotColoringPass(MachineStackSlotColoringOptions Options = {}) noexcept + : Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function) const; + + MachineStackSlotColoringOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINESTACKSLOTCOLORING_H diff --git a/backend/include/rcc/CodeGen/MachineTailDuplication.h b/backend/include/rcc/CodeGen/MachineTailDuplication.h new file mode 100644 index 0000000..cd62509 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineTailDuplication.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINETAILDUPLICATION_H +#define RCC_CODEGEN_MACHINETAILDUPLICATION_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetBranchInfo; + +struct MachineTailDuplicationOptions final { + std::size_t MaximumTailInstructions = 8; + std::size_t MaximumDuplicatedInstructions = 4'096; +}; + +class MachineTailDuplicationPass final : public MachineFunctionPass { +public: + explicit MachineTailDuplicationPass(const TargetBranchInfo &Info, + MachineTailDuplicationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetBranchInfo *Info; + MachineTailDuplicationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINETAILDUPLICATION_H diff --git a/backend/include/rcc/CodeGen/MachineTailMerge.h b/backend/include/rcc/CodeGen/MachineTailMerge.h new file mode 100644 index 0000000..94084ac --- /dev/null +++ b/backend/include/rcc/CodeGen/MachineTailMerge.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINETAILMERGE_H +#define RCC_CODEGEN_MACHINETAILMERGE_H + +#include "rcc/CodeGen/MachinePass.h" + +#include + +namespace rcc { + +class TargetBranchInfo; + +struct MachineTailMergeOptions final { + std::size_t MaximumMergedBlocks = 4'096; +}; + +class MachineTailMergePass final : public MachineFunctionPass { +public: + explicit MachineTailMergePass(const TargetBranchInfo &Info, + MachineTailMergeOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected + runImplementation(MachineFunction &Function, const TargetDescription &Target) const; + + const TargetBranchInfo *Info; + MachineTailMergeOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINETAILMERGE_H diff --git a/backend/include/rcc/CodeGen/PostRAScheduler.h b/backend/include/rcc/CodeGen/PostRAScheduler.h new file mode 100644 index 0000000..13b08df --- /dev/null +++ b/backend/include/rcc/CodeGen/PostRAScheduler.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_POSTRASCHEDULER_H +#define RCC_CODEGEN_POSTRASCHEDULER_H + +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/MachineScheduler.h" + +namespace rcc { + +class TargetSchedulingModel; + +class PostRASchedulerPass final : public MachineFunctionPass { +public: + explicit PostRASchedulerPass(const TargetSchedulingModel &Model, + MachineSchedulerOptions Options = {}) noexcept + : Model(&Model), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetSchedulingModel *Model; + MachineSchedulerOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_POSTRASCHEDULER_H diff --git a/backend/include/rcc/CodeGen/RegisterPressure.h b/backend/include/rcc/CodeGen/RegisterPressure.h new file mode 100644 index 0000000..fa1209a --- /dev/null +++ b/backend/include/rcc/CodeGen/RegisterPressure.h @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_REGISTERPRESSURE_H +#define RCC_CODEGEN_REGISTERPRESSURE_H + +#include "rcc/CodeGen/MachineIRTypes.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/Support/Error.h" + +#include +#include +#include +#include +#include + +namespace rcc { + +class MachineFunction; +class TargetDescription; +class TargetSchedulingModel; +struct MachineBlockLiveness; + +namespace detail { +struct RegisterPressureBuilder; +} + +struct RegisterPressureValue final { + RegisterPressureSetID Set; + std::uint32_t Current = 0; + std::uint32_t Maximum = 0; + std::uint32_t Limit = 0; + + friend constexpr bool operator==(RegisterPressureValue, RegisterPressureValue) noexcept = default; +}; + +struct RegisterPressureCost final { + std::uint64_t CurrentPressure = 0; + std::uint64_t CurrentExcess = 0; + std::uint64_t PeakExcess = 0; + std::uint64_t PeakIncrease = 0; + + friend constexpr bool operator==(RegisterPressureCost, RegisterPressureCost) noexcept = default; +}; + +enum class RegisterPressureDirection : std::uint8_t { + TopDown, + BottomUp, +}; + +class RegisterPressureTracker final { +public: + [[nodiscard]] static Expected + create(const ScheduleDAG &DAG, const MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const MachineBlockLiveness &Liveness, + RegisterPressureDirection Direction = RegisterPressureDirection::TopDown); + + [[nodiscard]] std::span pressure() const noexcept { + return Pressure; + } + + [[nodiscard]] bool wasScheduled(std::size_t UnitID) const noexcept { + return UnitID < Scheduled.size() && Scheduled[UnitID]; + } + + [[nodiscard]] Expected evaluate(std::size_t UnitID) const; + [[nodiscard]] Expected schedule(std::size_t UnitID); + [[nodiscard]] RegisterPressureCost cost() const noexcept; + +private: + using EntityID = std::size_t; + + struct Entity final { + std::vector Sets; + std::size_t RemainingUses = 0; + bool HasDefinition = false; + bool DefinitionPending = false; + bool LiveOut = false; + bool Live = false; + }; + + struct Event final { + std::vector Uses; + std::vector Definitions; + }; + + struct UnitEffects final { + std::vector Events; + }; + + [[nodiscard]] Expected applyEvent(const Event &Value); + [[nodiscard]] Expected makeLive(EntityID Entity); + [[nodiscard]] Expected makeDead(EntityID Entity); + + std::vector Pressure; + std::unordered_map PressureIndexes; + std::vector Entities; + std::vector Effects; + std::vector Scheduled; + RegisterPressureDirection Direction = RegisterPressureDirection::TopDown; + + friend struct detail::RegisterPressureBuilder; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_REGISTERPRESSURE_H diff --git a/backend/include/rcc/CodeGen/ScheduleDAG.h b/backend/include/rcc/CodeGen/ScheduleDAG.h new file mode 100644 index 0000000..644d98a --- /dev/null +++ b/backend/include/rcc/CodeGen/ScheduleDAG.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_SCHEDULEDAG_H +#define RCC_CODEGEN_SCHEDULEDAG_H + +#include "rcc/CodeGen/StableID.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" +#include "rcc/Support/Error.h" + +#include +#include +#include +#include + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class TargetDescription; + +enum class ScheduleDAGMode : std::uint8_t { + PreRegisterAllocation, + PostRegisterAllocation, +}; + +enum class ScheduleDependencyKind : std::uint8_t { + Data, + Anti, + Output, + Order, + Memory, + Barrier, + Artificial, + RegisterUnit, + RegisterMask, +}; + +struct ScheduleDependency final { + std::size_t Source = 0; + std::size_t Destination = 0; + ScheduleDependencyKind Kind = ScheduleDependencyKind::Artificial; + std::uint32_t Latency = 0; + + friend constexpr bool operator==(const ScheduleDependency &, + const ScheduleDependency &) noexcept = default; +}; + +struct ScheduleUnit final { + std::size_t ID = 0; + std::vector Instructions; + std::vector Predecessors; + std::vector Successors; + std::vector Resources; + std::uint32_t MicroOps = 0; + std::uint32_t Latency = 0; + std::uint64_t Depth = 0; + std::uint64_t Height = 0; + MachineInstrStableID StableID; +}; + +class ScheduleDAG final { +public: + [[nodiscard]] static Expected + build(MachineBasicBlock &Block, std::span Instructions, + const MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode); + + [[nodiscard]] MachineBasicBlock &block() const noexcept { + return *Block; + } + + [[nodiscard]] std::span units() const noexcept { + return Units; + } + + [[nodiscard]] std::span units() noexcept { + return Units; + } + + [[nodiscard]] const ScheduleUnit *unit(std::size_t ID) const noexcept { + return ID < Units.size() ? &Units[ID] : nullptr; + } + + [[nodiscard]] Expected addArtificialDependency(std::size_t Source, std::size_t Destination, + std::uint32_t Latency); + +private: + explicit ScheduleDAG(MachineBasicBlock &Block) : Block(&Block) {} + + [[nodiscard]] Expected buildUnits(std::span Instructions, + const TargetDescription &Target, + const TargetSchedulingModel &Model); + [[nodiscard]] Expected buildRegisterDependencies(const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, + ScheduleDAGMode Mode); + [[nodiscard]] Expected buildMemoryDependencies(const TargetDescription &Target); + [[nodiscard]] Expected buildOrderDependencies(const TargetDescription &Target); + void addDependency(std::size_t Source, std::size_t Destination, ScheduleDependencyKind Kind, + std::uint32_t Latency); + void computeCriticalPaths(); + + MachineBasicBlock *Block; + std::vector Units; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_SCHEDULEDAG_H diff --git a/backend/include/rcc/CodeGen/ScheduleHazardRecognizer.h b/backend/include/rcc/CodeGen/ScheduleHazardRecognizer.h new file mode 100644 index 0000000..947db5e --- /dev/null +++ b/backend/include/rcc/CodeGen/ScheduleHazardRecognizer.h @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_SCHEDULEHAZARDRECOGNIZER_H +#define RCC_CODEGEN_SCHEDULEHAZARDRECOGNIZER_H + +#include "rcc/CodeGen/TargetSchedulingModel.h" +#include "rcc/Support/Error.h" + +#include +#include +#include +#include +#include + +namespace rcc { + +struct ScheduleUnit; + +class ScheduleHazardRecognizer final { +public: + [[nodiscard]] static Expected + create(const TargetSchedulingModel &Model, + ScheduleDirection Direction = ScheduleDirection::TopDown); + + ScheduleHazardRecognizer(const ScheduleHazardRecognizer &Other); + ScheduleHazardRecognizer &operator=(const ScheduleHazardRecognizer &Other); + ScheduleHazardRecognizer(ScheduleHazardRecognizer &&) noexcept = default; + ScheduleHazardRecognizer &operator=(ScheduleHazardRecognizer &&) noexcept = default; + ~ScheduleHazardRecognizer() = default; + + [[nodiscard]] std::uint64_t currentCycle() const noexcept { + return CurrentCycle; + } + + [[nodiscard]] Expected canIssue(const ScheduleUnit &Unit, std::uint64_t Cycle) const; + [[nodiscard]] Expected earliestIssueCycle(const ScheduleUnit &Unit, + std::uint64_t Earliest) const; + [[nodiscard]] Expected issue(const ScheduleUnit &Unit, std::uint64_t Cycle); + +private: + struct ResourceState final { + ProcessorResourceID ID; + std::vector AvailableAt; + }; + + ScheduleHazardRecognizer(const TargetSchedulingModel &Model, ScheduleDirection Direction) noexcept + : Model(&Model), Direction(Direction) {} + + [[nodiscard]] Expected reset(); + [[nodiscard]] Expected moveToCycle(std::uint64_t Cycle); + [[nodiscard]] Expected issueImplementation(const ScheduleUnit &Unit, std::uint64_t Cycle); + [[nodiscard]] Expected reserveResource(ProcessorResourceID Resource, std::uint32_t Cycles, + std::uint64_t Cycle); + + const TargetSchedulingModel *Model; + ScheduleDirection Direction = ScheduleDirection::TopDown; + std::unique_ptr TargetHazards; + std::vector Resources; + std::unordered_map ResourceIndexes; + std::uint64_t CurrentCycle = 0; + std::uint64_t DispatchAvailableAt = 0; + std::uint32_t IssuedMicroOps = 0; + std::uint32_t DispatchedUnits = 0; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_SCHEDULEHAZARDRECOGNIZER_H diff --git a/backend/include/rcc/CodeGen/TargetBranchInfo.h b/backend/include/rcc/CodeGen/TargetBranchInfo.h new file mode 100644 index 0000000..39c5122 --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetBranchInfo.h @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETBRANCHINFO_H +#define RCC_CODEGEN_TARGETBRANCHINFO_H + +#include "rcc/Support/Error.h" + +#include +#include + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class TargetDescription; + +struct MachineBranchAnalysis final { + MachineInstr *Conditional = nullptr; + MachineInstr *Unconditional = nullptr; + MachineBasicBlock *TrueTarget = nullptr; + MachineBasicBlock *FalseTarget = nullptr; + bool HasUnanalyzableTerminator = false; +}; + +class TargetBranchInfo { +public: + virtual ~TargetBranchInfo() = default; + + [[nodiscard]] virtual Expected + analyzeBranch(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected + reverseBranchCondition(MachineInstr &Branch, MachineBasicBlock &NewTarget, + MachineFunction &Function, const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected removeBranch(MachineInstr &Branch, MachineFunction &Function, + const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected + rewriteBranchTarget(MachineBasicBlock &Block, MachineBasicBlock &OldTarget, + MachineBasicBlock &NewTarget, MachineFunction &Function, + const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected + insertUnconditionalBranch(MachineBasicBlock &Block, MachineBasicBlock &TargetBlock, + MachineFunction &Function, const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected + instructionSize(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected + branchTarget(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected> + evaluateBranchCondition(const MachineInstr &, const MachineFunction &, + const TargetDescription &) const { + return std::optional{}; + } + [[nodiscard]] virtual bool isBranchOffsetInRange(const MachineInstr &Instruction, + std::int64_t Offset) const noexcept = 0; + [[nodiscard]] virtual Expected relaxBranch(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, + const TargetDescription &Target) const = 0; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETBRANCHINFO_H diff --git a/backend/include/rcc/CodeGen/TargetFrameLowering.h b/backend/include/rcc/CodeGen/TargetFrameLowering.h index 1712616..b63ce22 100644 --- a/backend/include/rcc/CodeGen/TargetFrameLowering.h +++ b/backend/include/rcc/CodeGen/TargetFrameLowering.h @@ -39,6 +39,11 @@ public: [[nodiscard]] virtual Expected finalizeFrame(MachineFunction &Function, const TargetDescription &Target) const = 0; + [[nodiscard]] virtual bool isCalleeSavedRegister(PhysicalRegister, const MachineFunction &, + const TargetDescription &) const noexcept { + return false; + } + [[nodiscard]] virtual bool isCallFramePseudo(const MachineInstr &Instruction) const noexcept = 0; [[nodiscard]] virtual Expected eliminateCallFramePseudo(MachineInstr &Instruction, MachineFunction &Function, diff --git a/backend/include/rcc/CodeGen/TargetMemoryFolding.h b/backend/include/rcc/CodeGen/TargetMemoryFolding.h new file mode 100644 index 0000000..e40fbc0 --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetMemoryFolding.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETMEMORYFOLDING_H +#define RCC_CODEGEN_TARGETMEMORYFOLDING_H + +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/Support/Error.h" + +#include +#include + +namespace rcc { + +class MachineFunction; +class MachineInstr; +class TargetDescription; + +class TargetMemoryFoldingInfo { +public: + virtual ~TargetMemoryFoldingInfo() = default; + + [[nodiscard]] virtual Expected + canFoldMemoryOperand(const MachineInstr &Consumer, std::size_t OperandIndex, + const MachineInstr &MemoryInstruction, const MachineFunction &Function, + const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected + foldMemoryOperand(MachineInstr &Consumer, std::size_t OperandIndex, + MachineInstr &MemoryInstruction, MachineFunction &Function, + const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected + unfoldMemoryOperand(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected> + getFoldedMemorySize(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) const = 0; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETMEMORYFOLDING_H diff --git a/backend/include/rcc/CodeGen/TargetPseudoExpansion.h b/backend/include/rcc/CodeGen/TargetPseudoExpansion.h new file mode 100644 index 0000000..4d9f556 --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetPseudoExpansion.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETPSEUDOEXPANSION_H +#define RCC_CODEGEN_TARGETPSEUDOEXPANSION_H + +#include "rcc/Support/Error.h" + +#include + +namespace rcc { + +class MachineFunction; +class MachineInstr; +class TargetDescription; + +enum class MachinePseudoExpansionPhase : std::uint8_t { + PrePEI, + PostRA, + TargetLate, +}; + +class TargetPseudoExpansionInfo { +public: + virtual ~TargetPseudoExpansionInfo() = default; + + [[nodiscard]] virtual Expected preparePostRAPseudo(MachineInstr &, MachineFunction &, + const TargetDescription &) const { + return false; + } + + [[nodiscard]] virtual Expected expandPseudo(MachineInstr &, MachinePseudoExpansionPhase, + MachineFunction &, + const TargetDescription &) const { + return false; + } +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETPSEUDOEXPANSION_H diff --git a/backend/include/rcc/CodeGen/TargetSchedulingModel.h b/backend/include/rcc/CodeGen/TargetSchedulingModel.h new file mode 100644 index 0000000..4d7b3ce --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetSchedulingModel.h @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETSCHEDULINGMODEL_H +#define RCC_CODEGEN_TARGETSCHEDULINGMODEL_H + +#include "rcc/CodeGen/MachineIRTypes.h" +#include "rcc/Support/Error.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class ScheduleDAG; +class TargetDescription; +struct ScheduleUnit; + +enum class ScheduleDAGMode : std::uint8_t; + +enum class ScheduleDirection : std::uint8_t { + TopDown, + BottomUp, +}; + +enum class ScheduleHazardType : std::uint8_t { + NoHazard, + Hazard, + NoopHazard, +}; + +struct ProcessorResourceDesc final { + ProcessorResourceID ID; + std::string Name; + std::uint32_t Units = 1; + std::optional Parent; +}; + +struct SchedulingResourceUse final { + ProcessorResourceID Resource; + std::uint32_t Cycles = 1; + + friend constexpr bool operator==(SchedulingResourceUse, SchedulingResourceUse) noexcept = default; +}; + +struct SchedulingWriteLatency final { + std::uint32_t OperandIndex = 0; + std::uint32_t Cycles = 0; + + friend constexpr bool operator==(SchedulingWriteLatency, + SchedulingWriteLatency) noexcept = default; +}; + +struct SchedulingReadAdvance final { + std::uint32_t OperandIndex = 0; + std::optional ProducerClass; + std::uint32_t Cycles = 0; + + friend constexpr bool operator==(const SchedulingReadAdvance &, + const SchedulingReadAdvance &) noexcept = default; +}; + +struct SchedulingClassModelDesc final { + SchedulingClassID ID; + std::string Name; + std::uint32_t MicroOps = 1; + std::uint32_t Latency = 1; + std::vector Resources; + std::vector WriteLatencies; + std::vector ReadAdvances; +}; + +struct RegisterPressureSetDesc final { + RegisterPressureSetID ID; + std::string Name; + std::vector RegisterClasses; + std::uint32_t Limit = 1; +}; + +class TargetScheduleHazardRecognizer { +public: + virtual ~TargetScheduleHazardRecognizer() = default; + + [[nodiscard]] virtual std::unique_ptr clone() const = 0; + [[nodiscard]] virtual Expected getHazardType(const ScheduleUnit &Unit) const; + [[nodiscard]] virtual Expected emitInstruction(const ScheduleUnit &Unit); + [[nodiscard]] virtual Expected advanceCycle(); + [[nodiscard]] virtual Expected recedeCycle(); + [[nodiscard]] virtual Expected reset(); +}; + +class TargetSchedulingHooks { +public: + virtual ~TargetSchedulingHooks() = default; + + [[nodiscard]] virtual bool isSchedulingBoundary(const MachineInstr &Instruction, + const TargetDescription &Target) const noexcept; + [[nodiscard]] virtual Expected mutateScheduleDAG(ScheduleDAG &DAG, + const MachineFunction &Function, + const TargetDescription &Target, + ScheduleDAGMode Mode) const; + [[nodiscard]] virtual std::int64_t + candidatePreference(const ScheduleUnit &Unit, ScheduleDirection Direction, + const MachineBasicBlock &Block, std::uint64_t BlockFrequency) const noexcept; + [[nodiscard]] virtual std::unique_ptr + createHazardRecognizer(ScheduleDirection Direction) const; +}; + +struct TargetSchedulingModelDesc final { + std::uint32_t IssueWidth = 1; + std::uint32_t DispatchLimit = 1; + std::vector ProcessorResources; + std::vector SchedulingClasses; + std::vector PressureSets; + const TargetSchedulingHooks *Hooks = nullptr; +}; + +class TargetSchedulingModel final { +public: + [[nodiscard]] static Expected + create(TargetSchedulingModelDesc Description); + + [[nodiscard]] std::uint32_t issueWidth() const noexcept { + return Description.IssueWidth; + } + + [[nodiscard]] std::uint32_t dispatchLimit() const noexcept { + return Description.DispatchLimit; + } + + [[nodiscard]] std::span processorResources() const noexcept { + return Description.ProcessorResources; + } + + [[nodiscard]] std::span schedulingClasses() const noexcept { + return Description.SchedulingClasses; + } + + [[nodiscard]] std::span pressureSets() const noexcept { + return Description.PressureSets; + } + + [[nodiscard]] const ProcessorResourceDesc * + processorResource(ProcessorResourceID ID) const noexcept; + [[nodiscard]] const SchedulingClassModelDesc * + schedulingClass(SchedulingClassID ID) const noexcept; + [[nodiscard]] const RegisterPressureSetDesc *pressureSet(RegisterPressureSetID ID) const noexcept; + [[nodiscard]] std::span + pressureSetsForClass(RegisterClassID Class) const noexcept; + [[nodiscard]] Expected operandLatency(SchedulingClassID ProducerClass, + std::size_t DefinitionOperand, + SchedulingClassID ConsumerClass, + std::size_t UseOperand) const; + [[nodiscard]] const TargetSchedulingHooks &hooks() const noexcept; + +private: + explicit TargetSchedulingModel(TargetSchedulingModelDesc Description) + : Description(std::move(Description)) {} + + TargetSchedulingModelDesc Description; + std::unordered_map ResourceToIndex; + std::unordered_map ClassToIndex; + std::unordered_map PressureSetToIndex; + std::unordered_map> ClassToPressureSets; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETSCHEDULINGMODEL_H diff --git a/backend/include/rcc/Support/Error.h b/backend/include/rcc/Support/Error.h index 44ff967..d125383 100644 --- a/backend/include/rcc/Support/Error.h +++ b/backend/include/rcc/Support/Error.h @@ -20,6 +20,11 @@ enum class ErrorCode : std::uint8_t { IoError, InternalError, FrameLayoutFailed, + SchedulingFailed, + BranchRelaxationFailed, + PseudoExpansionFailed, + FinalEncodabilityFailed, + CFGOptimizationFailed, }; [[nodiscard]] std::string_view getErrorCodeName(ErrorCode Code) noexcept; diff --git a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h index 0cec77d..b6b207c 100644 --- a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h +++ b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h @@ -3,9 +3,15 @@ #ifndef RCC_TARGET_RISCV_RISCVCODEGEN_H #define RCC_TARGET_RISCV_RISCVCODEGEN_H +#include "rcc/CodeGen/MachineCombinerPattern.h" +#include "rcc/CodeGen/MachineOptimizationPipeline.h" +#include "rcc/CodeGen/TargetBranchInfo.h" #include "rcc/CodeGen/TargetCodeGen.h" #include "rcc/CodeGen/TargetFrameLowering.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" +#include "rcc/CodeGen/TargetPseudoExpansion.h" #include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" namespace rcc::riscv { @@ -15,6 +21,12 @@ namespace rcc::riscv { [[nodiscard]] const TargetSSAInfo &ssaInfo() noexcept; [[nodiscard]] const TargetRegisterAllocationInfo ®isterAllocationInfo() noexcept; [[nodiscard]] const TargetFrameLowering &frameLowering() noexcept; +[[nodiscard]] const TargetSchedulingModel &schedulingModel() noexcept; +[[nodiscard]] const TargetMachineCombinerInfo &machineCombinerInfo() noexcept; +[[nodiscard]] const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept; +[[nodiscard]] const TargetBranchInfo &branchInfo() noexcept; +[[nodiscard]] const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept; +[[nodiscard]] MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept; } // namespace rcc::riscv diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index a8cb952..341452a 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -11,11 +11,21 @@ set( MachineBasicBlockAttributes.cpp MachineCFIInfo.cpp MachineCompilationStatus.cpp + MachineCombiner.cpp + MachineCombinerPattern.cpp MachineBasicBlock.cpp + MachineBlockPlacement.cpp + MachineBranchOptimization.cpp + MachineBranchRelaxation.cpp MachineCFGUpdater.cpp MachineConstantPool.cpp + MachineCopyPropagation.cpp + MachineCSE.cpp MachineDebugInfo.cpp + MachineDCE.cpp MachineFrameInfo.cpp + MachineFallthrough.cpp + MachineFinalization.cpp MachineFunction.cpp MachineFunctionAttributes.cpp MachineFunctionCloner.cpp @@ -24,15 +34,29 @@ set( MachineAnalyses.cpp MachineInstr.cpp MachineJumpTableInfo.cpp + MachineLICM.cpp MachineLiveness.cpp + MachineMemoryFolding.cpp MachineMemOperand.cpp MachineMetadata.cpp MachineOperand.cpp + MachineOptimizationPipeline.cpp MachinePass.cpp + MachinePseudoExpansion.cpp MachinePipelinePasses.cpp MachineRegisterInfo.cpp + MachineScheduler.cpp + MachineShrinkWrapping.cpp + MachineSink.cpp + MachineStackSlotColoring.cpp + MachineTailDuplication.cpp + MachineTailMerge.cpp PrologueEpilogueInsertion.cpp + PostRAScheduler.cpp + RegisterPressure.cpp RegisterScavenger.cpp + ScheduleDAG.cpp + ScheduleHazardRecognizer.cpp InlineSpiller.cpp LiveRangeEdit.cpp LiveRegMatrix.cpp @@ -53,6 +77,7 @@ set( TargetFunctionState.cpp TargetMIRTextInfo.cpp TargetRegisterInfo.cpp + TargetSchedulingModel.cpp ) set( RCCBackendCodeGenHeaders @@ -65,11 +90,21 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombinerPattern.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBlockPlacement.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchOptimization.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchRelaxation.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineConstantPool.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCopyPropagation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCSE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDebugInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDCE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFrameInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFallthrough.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFinalization.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunction.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionCloner.h" @@ -78,17 +113,32 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineAnalyses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineInstr.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineJumpTableInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLICM.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLiveness.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLivenessRepair.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemOperand.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePseudoExpansion.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePipelinePasses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PrologueEpilogueInsertion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterPressure.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterScavenger.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleDAG.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleHazardRecognizer.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFrameLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineScheduler.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineShrinkWrapping.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineSink.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineStackSlotColoring.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailDuplication.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailMerge.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PostRAScheduler.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Graph.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Math.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/ReductionRules.h" @@ -111,10 +161,14 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/StableID.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetConstant.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCallLowering.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetBranchInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetDescription.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFunctionState.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMIRTextInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetSchedulingModel.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetPseudoExpansion.h" ) add_library(RCCBackendCodeGen ${RCCBackendCodeGenSources}) @@ -156,9 +210,19 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -167,13 +231,28 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulerInternal.h" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -197,6 +276,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ${RCCBackendCodeGenHeaders} ) rcc_backend_register_tidy_files( @@ -210,9 +290,19 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -221,13 +311,27 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -248,4 +352,5 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ) diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index 5aec744..48d4f10 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -628,7 +628,7 @@ Expected InstructionSet::validateOperand(const OperandDescriptor &Descript } } - if (Descriptor.ImmediatePredicate) { + if (Descriptor.ImmediatePredicate && Operand.kind() == MachineOperand::Kind::Immediate) { const ImmediatePredicate *Predicate = immediatePredicate(*Descriptor.ImmediatePredicate); if (Predicate == nullptr || !Predicate->Predicate(Operand)) { return makeUnexpected(ErrorCode::VerificationError, diff --git a/backend/lib/CodeGen/MachineBlockPlacement.cpp b/backend/lib/CodeGen/MachineBlockPlacement.cpp new file mode 100644 index 0000000..fea1283 --- /dev/null +++ b/backend/lib/CodeGen/MachineBlockPlacement.cpp @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBlockPlacement.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +enum class PlacementClass : std::uint8_t { + Hot, + Normal, + Cold, + Exception, +}; + +[[nodiscard]] const MachinePassContract &placementContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-block-placement", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = + { + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + }, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] PlacementClass classify(const MachineBasicBlock &Block, + std::uint64_t Frequency) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Exception || + Attributes.has(MachineBasicBlockAttribute::LandingPad) || + Attributes.has(MachineBasicBlockAttribute::EHFuncletEntry) || + Attributes.has(MachineBasicBlockAttribute::EHScopeEntry)) { + return PlacementClass::Exception; + } + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Cold) + return PlacementClass::Cold; + if (Frequency >= MachineBlockFrequencyAnalysis::EntryFrequency) + return PlacementClass::Hot; + if (Frequency < MachineBlockFrequencyAnalysis::EntryFrequency / 1024U) + return PlacementClass::Cold; + return PlacementClass::Normal; +} + +[[nodiscard]] std::uint32_t classRank(PlacementClass Class) noexcept { + return static_cast(Class); +} + +struct PlacementFacts final { + MachineBasicBlock *Block = nullptr; + PlacementClass Class = PlacementClass::Normal; + std::uint64_t Frequency = 0; + std::uint32_t LoopDepth = 0; +}; + +[[nodiscard]] bool betterRoot(const PlacementFacts &Left, const PlacementFacts &Right) noexcept { + if (Left.Class != Right.Class) + return classRank(Left.Class) < classRank(Right.Class); + if (Left.Frequency != Right.Frequency) + return Left.Frequency > Right.Frequency; + if (Left.LoopDepth != Right.LoopDepth) + return Left.LoopDepth > Right.LoopDepth; + return Left.Block->stableID().value() < Right.Block->stableID().value(); +} + +} // namespace + +const MachinePassContract &MachineBlockPlacementPass::contract() const noexcept { + return placementContract(); +} + +Expected MachineBlockPlacementPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine block placement requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "block placement setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "block placement diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBlockPlacementPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) const { + auto Probabilities = Analyses.getResult(Function, Target); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + std::vector Facts; + Facts.reserve(Function.blockCount()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const std::uint64_t Frequency = (*Frequencies)->frequency(*Block); + Facts.push_back({Block, classify(*Block, Frequency), Frequency, (*Loops)->loopDepth(*Block)}); + } + if (Facts.size() < 2) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto factsFor = [&Facts](const MachineBasicBlock &Block) -> const PlacementFacts & { + return *std::ranges::find_if( + Facts, [&Block](const PlacementFacts &Value) { return Value.Block == &Block; }); + }; + std::unordered_set Placed; + std::vector Order; + Order.reserve(Facts.size()); + MachineBasicBlock *Root = Function.firstBlock(); + while (Order.size() != Facts.size()) { + if (Root == nullptr || Placed.contains(Root)) { + const auto Best = std::ranges::min_element( + Facts, [&Placed](const PlacementFacts &Left, const PlacementFacts &Right) { + if (Placed.contains(Left.Block)) + return false; + if (Placed.contains(Right.Block)) + return true; + return betterRoot(Left, Right); + }); + if (Best == Facts.end() || Placed.contains(Best->Block)) + return makeUnexpected(ErrorCode::InternalError, "block placement lost an unplaced block"); + Root = Best->Block; + } + + while (Root != nullptr && Placed.insert(Root).second) { + Order.push_back(Root); + MachineBasicBlock *BestSuccessor = nullptr; + BranchProbability BestProbability = BranchProbability::zero(); + for (const MachineSuccessor &Successor : (*Probabilities)->successors(*Root)) { + if (Successor.Block == nullptr || Placed.contains(Successor.Block)) + continue; + if (BestSuccessor == nullptr || BestProbability < Successor.Probability || + (BestProbability == Successor.Probability && + betterRoot(factsFor(*Successor.Block), factsFor(*BestSuccessor)))) { + BestSuccessor = Successor.Block; + BestProbability = Successor.Probability; + } + } + Root = BestSuccessor; + } + Root = nullptr; + } + + const auto &SavePoints = Function.frameInfo().state().SavePoints; + if (!SavePoints.empty() && SavePoints.front().Block != Function.firstBlock()->stableID()) { + const auto SavePosition = std::ranges::find_if(Order, [&](const MachineBasicBlock *Block) { + return Block->stableID() == SavePoints.front().Block; + }); + if (SavePosition == Order.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "block placement cannot find the frame save point"); + } + MachineBasicBlock *SaveBlock = *SavePosition; + std::vector OutsideFrame; + std::vector InsideFrame; + for (MachineBasicBlock *Block : Order) { + ((*Dominators)->dominates(*SaveBlock, *Block) ? InsideFrame : OutsideFrame).push_back(Block); + } + const auto SaveInRegion = std::ranges::find(InsideFrame, SaveBlock); + if (SaveInRegion == InsideFrame.end()) { + return makeUnexpected(ErrorCode::InternalError, "block placement lost the frame save region"); + } + std::rotate(InsideFrame.begin(), SaveInRegion, std::next(SaveInRegion)); + Order.clear(); + Order.insert(Order.end(), OutsideFrame.begin(), OutsideFrame.end()); + Order.insert(Order.end(), InsideFrame.begin(), InsideFrame.end()); + } + + bool Changed = false; + for (std::size_t Index = 1; Index < Order.size(); ++Index) { + Changed |= Order[Index - 1U]->next() != Order[Index]; + auto Moved = Function.moveBlockAfter(*Order[Index - 1U], *Order[Index]); + if (!Moved) + return std::unexpected(Moved.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchOptimization.cpp b/backend/lib/CodeGen/MachineBranchOptimization.cpp new file mode 100644 index 0000000..6c6ac51 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchOptimization.cpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchOptimization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &branchOptimizationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-optimization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasInstructionRecord(const MachineInstr &Instruction, + const MachineFunction &Function) noexcept { + const MachineInstrStableID ID = Instruction.stableID(); + return std::ranges::any_of(Function.records().callSites(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }) || + std::ranges::any_of(Function.records().calledGlobals(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }); +} + +[[nodiscard]] bool hasObservableBlockIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] bool canDeleteBlock(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + if (hasObservableBlockIdentity(Block, Function)) + return false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (hasProtectedAttachment(*Instruction) || hasInstructionRecord(*Instruction, Function)) { + return false; + } + } + return true; +} + +[[nodiscard]] bool hasSuccessor(const MachineBasicBlock &Block, + const MachineBasicBlock &Target) noexcept { + return std::ranges::any_of(Block.successors(), [&Target](const MachineSuccessor &Successor) { + return Successor.Block == &Target; + }); +} + +[[nodiscard]] Expected foldConstantBranches(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + bool Changed = false; + MachineCFGUpdater CFG = Function.cfgUpdater(); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional == nullptr || + hasProtectedAttachment(*Analysis->Conditional)) { + continue; + } + auto Condition = Info.evaluateBranchCondition(*Analysis->Conditional, Function, Target); + if (!Condition) + return std::unexpected(Condition.error()); + if (!Condition->has_value()) + continue; + + MachineBasicBlock *Selected = **Condition ? Analysis->TrueTarget : Analysis->FalseTarget; + MachineBasicBlock *Rejected = **Condition ? Analysis->FalseTarget : Analysis->TrueTarget; + if (Selected == nullptr || !hasSuccessor(*Block, *Selected)) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "constant branch has no selected CFG successor"); + } + if (Analysis->Unconditional != nullptr && hasProtectedAttachment(*Analysis->Unconditional)) { + continue; + } + + auto RemovedConditional = Info.removeBranch(*Analysis->Conditional, Function, Target); + if (!RemovedConditional) + return std::unexpected(RemovedConditional.error()); + if (Analysis->Unconditional != nullptr) { + auto RemovedUnconditional = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!RemovedUnconditional) + return std::unexpected(RemovedUnconditional.error()); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Selected, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + + if (Rejected != nullptr && Rejected != Selected && hasSuccessor(*Block, *Rejected)) { + auto RemovedEdge = CFG.removeSuccessor(*Block, *Rejected); + if (!RemovedEdge) + return std::unexpected(RemovedEdge.error()); + } + auto Probability = CFG.updateBranchProbability(*Block, *Selected, BranchProbability::one()); + if (!Probability) + return std::unexpected(Probability.error()); + Changed = true; + } + return Changed; +} + +[[nodiscard]] std::unordered_set reachableBlocks(MachineFunction &Function) { + std::unordered_set Reachable; + std::vector Worklist; + if (Function.firstBlock() != nullptr) + Worklist.push_back(Function.firstBlock()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!canDeleteBlock(*Block, Function)) + Worklist.push_back(Block); + } + while (!Worklist.empty()) { + MachineBasicBlock *Block = Worklist.back(); + Worklist.pop_back(); + if (Block == nullptr || !Reachable.insert(Block).second) + continue; + for (const MachineSuccessor &Successor : Block->successors()) + Worklist.push_back(Successor.Block); + } + return Reachable; +} + +[[nodiscard]] Expected removeUnreachableBlocks(MachineFunction &Function) { + const std::unordered_set Reachable = reachableBlocks(Function); + MachineCFGUpdater CFG = Function.cfgUpdater(); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;) { + MachineBasicBlock *Next = Block->next(); + if (!Reachable.contains(Block) && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + Changed = true; + } + Block = Next; + } + return Changed; +} + +[[nodiscard]] Expected isForwardingBlock(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + if (&Block == Function.firstBlock() || Block.successors().size() != 1 || + Block.successors().front().Block == nullptr || Block.successors().front().Block == &Block || + hasObservableBlockIdentity(Block, Function)) { + return false; + } + auto Analysis = Info.analyzeBranch(Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional != nullptr) + return false; + MachineBasicBlock *Successor = Block.successors().front().Block; + if (Analysis->Unconditional == nullptr) { + return Block.firstInstruction() == nullptr && Block.next() == Successor; + } + if (Analysis->TrueTarget != Successor || hasProtectedAttachment(*Analysis->Unconditional)) { + return false; + } + return Block.firstInstruction() == Analysis->Unconditional && + Analysis->Unconditional->next() == nullptr; +} + +[[nodiscard]] Expected threadForwardingBlocks(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info, + std::size_t MaximumEdges) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + std::size_t ThreadedEdges = 0; + bool AnyChange = false; + bool LocalChange = true; + while (LocalChange && ThreadedEdges < MaximumEdges) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Forwarding = isForwardingBlock(*Block, Function, Target, Info); + if (!Forwarding) + return std::unexpected(Forwarding.error()); + if (!*Forwarding) + continue; + MachineBasicBlock *Successor = Block->successors().front().Block; + const std::vector Predecessors(Block->predecessors().begin(), + Block->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (ThreadedEdges == MaximumEdges) + break; + if (Predecessor == nullptr || Predecessor->successors().size() != 1 || + Predecessor->successors().front().Block != Block) { + continue; + } + auto PredAnalysis = Info.analyzeBranch(*Predecessor, Function, Target); + if (!PredAnalysis) + return std::unexpected(PredAnalysis.error()); + if (PredAnalysis->HasUnanalyzableTerminator || PredAnalysis->Conditional != nullptr) + continue; + + if (PredAnalysis->Unconditional != nullptr) { + if (PredAnalysis->TrueTarget != Block || + hasProtectedAttachment(*PredAnalysis->Unconditional)) { + continue; + } + auto Rewritten = + Info.rewriteBranchTarget(*Predecessor, *Block, *Successor, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "branch threading did not rewrite an explicit branch"); + } + } else { + if (Predecessor->next() != Block) + continue; + auto Inserted = + Info.insertUnconditionalBranch(*Predecessor, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + auto Redirected = CFG.replaceSuccessor(*Predecessor, *Block, *Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + ++ThreadedEdges; + AnyChange = true; + LocalChange = true; + } + if (Block->predecessors().empty() && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + AnyChange = true; + LocalChange = true; + break; + } + } + } + return AnyChange; +} + +} // namespace + +const MachinePassContract &MachineBranchOptimizationPass::contract() const noexcept { + return branchOptimizationContract(); +} + +Expected MachineBranchOptimizationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch optimization requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "branch optimization transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch optimization diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBranchOptimizationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + auto Folded = foldConstantBranches(Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + auto RemovedBeforeThreading = removeUnreachableBlocks(Function); + if (!RemovedBeforeThreading) + return std::unexpected(RemovedBeforeThreading.error()); + auto Threaded = threadForwardingBlocks(Function, Target, *Info, Options.MaximumThreadedEdges); + if (!Threaded) + return std::unexpected(Threaded.error()); + auto RemovedAfterThreading = removeUnreachableBlocks(Function); + if (!RemovedAfterThreading) + return std::unexpected(RemovedAfterThreading.error()); + + MachinePassResult Result; + Result.Changed = *Folded || *RemovedBeforeThreading || *Threaded || *RemovedAfterThreading; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchRelaxation.cpp b/backend/lib/CodeGen/MachineBranchRelaxation.cpp new file mode 100644 index 0000000..d56f3a0 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchRelaxation.cpp @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchRelaxation.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +struct MachineLayout final { + std::unordered_map BlockOffsets; + std::unordered_map InstructionOffsets; +}; + +[[nodiscard]] const MachinePassContract &relaxationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-relaxation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::VerificationError, "block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits::max() - Mask) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected computeLayout(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + MachineLayout Result; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + Result.BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.InstructionOffsets.emplace(Instruction, Offset); + auto Size = Info.instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + if (Offset > std::numeric_limits::max() - *Size) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout size overflowed"); + Offset += *Size; + } + } + return Result; +} + +[[nodiscard]] Expected displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "forward branch displacement overflowed"); + return static_cast(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "backward branch displacement overflowed"); + return -static_cast(Difference); +} + +} // namespace + +const MachinePassContract &MachineBranchRelaxationPass::contract() const noexcept { + return relaxationContract(); +} + +Expected MachineBranchRelaxationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch relaxation requires finalized frame and tracked liveness"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "branch relaxation setup failed: ")); + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch relaxation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBranchRelaxationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.ConvergenceWatchdog == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "branch relaxation watchdog must be nonzero"); + MachinePassResult Result; + for (std::size_t Iteration = 0; Iteration < Options.ConvergenceWatchdog; ++Iteration) { + auto Layout = computeLayout(Function, Target, *Info); + if (!Layout) + return std::unexpected(Layout.error()); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !Changed; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto BranchTarget = Info->branchTarget(*Instruction, Function, Target); + if (!BranchTarget) + return std::unexpected(BranchTarget.error()); + if (*BranchTarget == nullptr) + continue; + const auto SourcePosition = Layout->InstructionOffsets.find(Instruction); + const auto TargetPosition = Layout->BlockOffsets.find(*BranchTarget); + if (SourcePosition == Layout->InstructionOffsets.end() || + TargetPosition == Layout->BlockOffsets.end()) { + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation lost an instruction or target offset"); + } + auto Offset = displacement(SourcePosition->second, TargetPosition->second); + if (!Offset) + return std::unexpected(Offset.error()); + if (Info->isBranchOffsetInRange(*Instruction, *Offset)) + continue; + auto Relaxed = Info->relaxBranch(*Instruction, *Offset, Function, Target); + if (!Relaxed) + return std::unexpected(Relaxed.error()); + if (!*Relaxed) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "target did not relax an out-of-range branch"); + } + Changed = true; + Result.Changed = true; + break; + } + } + if (Changed) + continue; + if (Result.Changed) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; + } + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation failed to reach a monotonic fixed point"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp new file mode 100644 index 0000000..a144403 --- /dev/null +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCSE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &cseContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-cse", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +template +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf(); + const ValueT *RightValue = Right.getIf(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +[[nodiscard]] bool registerOperandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + const Register *LeftReg = Left.getIf(); + const Register *RightReg = Right.getIf(); + if (LeftReg == nullptr || RightReg == nullptr || Left.subReg() != Right.subReg() || + Left.targetFlags() != Right.targetFlags()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags && (IgnoreDefinitionValue || *LeftReg == *RightReg); +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags()) + return false; + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return registerOperandEquals(Left, Right, IgnoreDefinitionValue); + case MachineOperand::Kind::Immediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals(Left, Right); + case MachineOperand::Kind::RegisterMask: + case MachineOperand::Kind::LiveOutRegisterMask: + return false; + case MachineOperand::Kind::Metadata: + return payloadEquals(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] bool isConstantMemoryBase(MemoryBase::Kind Kind) noexcept { + return Kind == MemoryBase::Kind::ConstantPool || Kind == MemoryBase::Kind::ConstantPoolPseudo; +} + +[[nodiscard]] bool hasSafeInvariantMemory(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().empty()) + return false; + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) + return false; + const MachineMemOperandDesc &Description = Memory->description(); + const bool ExplicitInvariant = Description.Accesses.has(MemoryAccessFlag::Invariant) && + Description.Accesses.has(MemoryAccessFlag::Dereferenceable); + if (!ExplicitInvariant && !isConstantMemoryBase(Description.Base.kind())) + return false; + } + return true; +} + +[[nodiscard]] bool hasFloatingRegister(const MachineInstr &Instruction, + const MachineRegisterInfo &Registers) { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Info = Registers.virtualRegisterInfo(*Reg); + if (Info && Info->get().Type && Info->get().Type->isFloat()) + return true; + } + return false; +} + +[[nodiscard]] Expected isCandidate(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Pseudo) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction)) { + return false; + } + if (Flags.has(InstructionFlag::MayLoad) && !hasSafeInvariantMemory(Instruction)) + return false; + if (!Flags.has(InstructionFlag::MayLoad) && !Instruction.memOperands().empty()) + return false; + if (!Instruction.flags().has(MachineInstrFlag::NoFPExcept) && + hasFloatingRegister(Instruction, Function.registerInfo())) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Reg->isPhysical() || Reg->isNoRegister() || Operand.isDebugUse() || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Operand.isDef()) + HasDefinition = true; + } + return HasDefinition; +} + +[[nodiscard]] bool expressionEquals(const MachineInstr &Left, const MachineInstr &Right) noexcept { + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size()) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + LeftOperand->isDef() != RightOperand->isDef() || + !operandEquals(*LeftOperand, *RightOperand, LeftOperand->isDef())) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory) + return false; + } + return true; +} + +[[nodiscard]] Expected>> +replacementPairs(const MachineInstr &Available, const MachineInstr &Redundant, + const MachineFunction &Function, const TargetDescription &Target) { + std::vector> Result; + std::unordered_set RedundantDefinitions; + const MachineRegisterInfo &Registers = Function.registerInfo(); + for (std::size_t Index = 0; Index < Available.operandCount(); ++Index) { + const MachineOperand *AvailableOperand = Available.operandAt(Index); + const MachineOperand *RedundantOperand = Redundant.operandAt(Index); + if (AvailableOperand == nullptr || RedundantOperand == nullptr || !AvailableOperand->isDef()) { + continue; + } + const Register *AvailableReg = AvailableOperand->getIf(); + const Register *RedundantReg = RedundantOperand->getIf(); + if (AvailableReg == nullptr || RedundantReg == nullptr || !AvailableReg->isVirtual() || + !RedundantReg->isVirtual() || !RedundantDefinitions.insert(*RedundantReg).second) { + return std::vector>{}; + } + auto AvailableInfo = Registers.virtualRegisterInfo(*AvailableReg); + if (!AvailableInfo) + return std::unexpected(AvailableInfo.error()); + auto RedundantInfo = Registers.virtualRegisterInfo(*RedundantReg); + if (!RedundantInfo) + return std::unexpected(RedundantInfo.error()); + const VirtualRegisterInfo &Left = AvailableInfo->get(); + const VirtualRegisterInfo &Right = RedundantInfo->get(); + if (!Left.Type || !Right.Type || Left.Type != Right.Type || + Left.TargetFlags != Right.TargetFlags) + return std::vector>{}; + if (Left.Bank && Right.Bank && + !Target.registers().registerBanksOverlap(*Left.Bank, *Right.Bank)) { + return std::vector>{}; + } + if (Left.RegClass && Right.RegClass && + !Target.registers().registerClassesOverlap(*Left.RegClass, *Right.RegClass)) { + return std::vector>{}; + } + Result.emplace_back(*RedundantReg, *AvailableReg); + } + return Result; +} + +[[nodiscard]] Expected +replaceAndErase(MachineInstr &Redundant, + const std::vector> &Replacements, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Block = Redundant.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine CSE candidate is detached"); + for (const auto &[OldReg, NewReg] : Replacements) { + auto Replaced = Registers.replaceRegWith(OldReg, NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Redundant); + if (!Removed) + return std::unexpected(Removed.error()); + for (const auto &[OldReg, NewReg] : Replacements) { + (void)NewReg; + if (!Registers.useEmpty(OldReg, true) || Registers.getUniqueVRegDef(OldReg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine CSE left references to a redundant result"); + } + auto Erased = Registers.eraseVirtualRegister(OldReg); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineCSEPass::contract() const noexcept { + return cseContract(); +} + +Expected MachineCSEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine CSE transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine CSE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineCSEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + std::unordered_map> Available; + std::size_t Comparisons = 0; + bool BudgetExhausted = false; + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !BudgetExhausted; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Candidate = isCandidate(*Instruction, Function, Target); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!*Candidate) { + Instruction = Next; + continue; + } + + MachineInstr *Match = nullptr; + auto &OpcodeCandidates = Available[Instruction->opcode()]; + for (MachineInstr *Prior : OpcodeCandidates) { + if (Comparisons >= Options.MaximumComparisons) { + BudgetExhausted = true; + break; + } + ++Comparisons; + if (!(*Dominators)->dominates(*Prior, *Instruction) || + !expressionEquals(*Prior, *Instruction)) { + continue; + } + Match = Prior; + break; + } + if (BudgetExhausted) + break; + if (Match == nullptr) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + + auto Replacements = replacementPairs(*Match, *Instruction, Function, Target); + if (!Replacements) + return std::unexpected(Replacements.error()); + if (Replacements->empty()) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + auto Replaced = replaceAndErase(*Instruction, *Replacements, Registers); + if (!Replaced) + return std::unexpected(Replaced.error()); + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombiner.cpp b/backend/lib/CodeGen/MachineCombiner.cpp new file mode 100644 index 0000000..02ea5e3 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombiner.cpp @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombiner.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &combinerContract(MachineCombinerStage Stage) { + static const MachinePassContract SelectedSSA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-selected-ssa", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-pre-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostFrame = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-frame", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + switch (Stage) { + case MachineCombinerStage::SelectedSSA: + return SelectedSSA; + case MachineCombinerStage::PreRegisterAllocation: + return PreRA; + case MachineCombinerStage::PostRegisterAllocation: + return PostRA; + case MachineCombinerStage::PostFrame: + return PostFrame; + } + return SelectedSSA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool matchesPredicate(const MachineInstr &Instruction, + const MachineCombinerPredicate &Predicate) { + const MachineOperand *Operand = Instruction.operandAt(Predicate.Operand); + if (Operand == nullptr) + return false; + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: { + const auto *Immediate = Operand->getIf(); + return Immediate != nullptr && *Immediate == Predicate.Value; + } + case MachineCombinerPredicateKind::RegisterEquals: { + const MachineOperand *Other = Instruction.operandAt(Predicate.OtherOperand); + const Register *LeftReg = Operand->getIf(); + const Register *RightReg = Other == nullptr ? nullptr : Other->getIf(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Operand->subReg() == Other->subReg(); + } + case MachineCombinerPredicateKind::PhysicalRegisterEquals: { + const Register *Reg = Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast(Predicate.Value); + } + case MachineCombinerPredicateKind::RegisterIsVirtual: { + const Register *Reg = Operand->getIf(); + return Reg != nullptr && Reg->isVirtual(); + } + case MachineCombinerPredicateKind::RegisterIsPhysical: { + const Register *Reg = Operand->getIf(); + return Reg != nullptr && Reg->isPhysical(); + } + } + return false; +} + +[[nodiscard]] bool matchesPattern(const MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineCombinerStage Stage) { + if ((Pattern.Stages & machineCombinerStageMask(Stage)) == 0) + return false; + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + if (!matchesPredicate(Instruction, Predicate)) + return false; + } + return true; +} + +[[nodiscard]] bool compatibleReplacement(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || Source.isNoRegister()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + if (!DestinationInfo) + return false; + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + if (Source.isPhysical()) { + const std::optional Physical = Source.physicalRegister(); + if (!Physical) + return false; + if (DestinationValue.RegClass && + !Target.registers().registerClassContains(*DestinationValue.RegClass, *Physical)) { + return false; + } + return !DestinationValue.Bank || + Target.registers().bankContainsRegister(*DestinationValue.Bank, *Physical); + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo) + return false; + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && + (!SourceValue.Bank || + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank))) { + return false; + } + if (DestinationValue.RegClass && + (!SourceValue.RegClass || !Target.registers().registerClassesOverlap( + *DestinationValue.RegClass, *SourceValue.RegClass))) { + return false; + } + return true; +} + +[[nodiscard]] Expected executeReplacement(MachineInstr &Instruction, + const MachineCombinerAction &Action, + MachineFunction &Function, + const TargetDescription &Target) { + MachineOperand *Definition = Instruction.operandAt(Action.DefinitionOperand); + const MachineOperand *SourceOperand = Instruction.operandAt(Action.SourceOperand); + const Register *Destination = Definition == nullptr ? nullptr : Definition->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr || !Definition->isDef() || Definition->isUse() || + !SourceOperand->isUse() || Definition->subReg().value() != 0 || + SourceOperand->subReg().value() != 0 || Destination->isNoRegister() || + Source->isNoRegister()) { + return false; + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine combiner root is detached"); + + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!Destination->isVirtual() || + Function.registerInfo().getUniqueVRegDef(*Destination) != Definition || + !compatibleReplacement(*Destination, *Source, Function, Target)) { + return false; + } + + std::vector Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Destination, true)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(*Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(*Destination, true) || + Function.registerInfo().getUniqueVRegDef(*Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine combiner left references to a replaced result"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(*Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] Expected executePattern(MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "machine combiner root has no descriptor"); + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayLoad) || + Flags.has(InstructionFlag::MayStore) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || !Instruction.memOperands().empty() || + hasProtectedAttachment(Instruction)) { + return false; + } + switch (Pattern.Action.Kind) { + case MachineCombinerActionKind::ReplaceDefWithOperandAndErase: + return executeReplacement(Instruction, Pattern.Action, Function, Target); + } + return false; +} + +} // namespace + +const MachinePassContract &MachineCombinerPass::contract() const noexcept { + return combinerContract(Stage); +} + +Expected MachineCombinerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine combiner input does not satisfy its stage contract"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine combiner transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine combiner diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineCombinerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.MaximumIterations == 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner iteration budget must be nonzero"); + MachinePassResult Result; + std::size_t Rewrites = 0; + for (std::size_t Iteration = 0; Iteration < Options.MaximumIterations; ++Iteration) { + bool IterationChanged = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + for (const MachineCombinerPattern &Pattern : Info->patterns(Instruction->opcode())) { + if (!matchesPattern(*Instruction, Pattern, Stage)) + continue; + if (Rewrites == Options.MaximumRewrites) + break; + auto Executed = executePattern(*Instruction, Pattern, Function, Target); + if (!Executed) + return std::unexpected(Executed.error()); + if (!*Executed) + continue; + ++Rewrites; + IterationChanged = true; + Result.Changed = true; + break; + } + Instruction = Next; + } + } + if (!IterationChanged || Rewrites == Options.MaximumRewrites) + break; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombinerPattern.cpp b/backend/lib/CodeGen/MachineCombinerPattern.cpp new file mode 100644 index 0000000..b1c3c96 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombinerPattern.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombinerPattern.h" + +#include +#include +#include +#include +#include + +namespace rcc { + +Expected +TargetMachineCombinerInfo::create(std::vector Patterns) { + TargetMachineCombinerInfo Result(std::move(Patterns)); + std::unordered_set Names; + for (const MachineCombinerPattern &Pattern : Result.Patterns) { + if (Pattern.Name.empty() || !Pattern.RootOpcode.isValid() || Pattern.Stages == 0 || + !Names.insert(Pattern.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner pattern has an invalid identity or stage mask"); + } + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: + case MachineCombinerPredicateKind::PhysicalRegisterEquals: + case MachineCombinerPredicateKind::RegisterIsVirtual: + case MachineCombinerPredicateKind::RegisterIsPhysical: + break; + case MachineCombinerPredicateKind::RegisterEquals: + if (Predicate.Operand == Predicate.OtherOperand) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register equality pattern compares an operand with itself"); + } + break; + } + } + Result.MatchTable[Pattern.RootOpcode].push_back(Pattern); + } + return Result; +} + +std::span +TargetMachineCombinerInfo::patterns(Opcode RootOpcode) const noexcept { + const auto Position = MatchTable.find(RootOpcode); + return Position == MatchTable.end() ? std::span{} + : std::span(Position->second); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCompilationStatus.cpp b/backend/lib/CodeGen/MachineCompilationStatus.cpp index 765e133..e541d39 100644 --- a/backend/lib/CodeGen/MachineCompilationStatus.cpp +++ b/backend/lib/CodeGen/MachineCompilationStatus.cpp @@ -26,6 +26,7 @@ constexpr std::array StateNames{ std::string_view{"FinalizationFailed"}, std::string_view{"AssemblyLoweringFailed"}, std::string_view{"InternalError"}, + std::string_view{"SchedulingFailed"}, }; static_assert(StateNames.size() == static_cast(MachineCompilationState::Count)); @@ -44,6 +45,11 @@ isKnownDiagnosticSeverity(MachineCompilationDiagnosticSeverity Severity) noexcep case ErrorCode::Unsupported: case ErrorCode::RegisterAllocationFailed: case ErrorCode::FrameLayoutFailed: + case ErrorCode::SchedulingFailed: + case ErrorCode::BranchRelaxationFailed: + case ErrorCode::PseudoExpansionFailed: + case ErrorCode::FinalEncodabilityFailed: + case ErrorCode::CFGOptimizationFailed: case ErrorCode::IoError: case ErrorCode::InternalError: return true; diff --git a/backend/lib/CodeGen/MachineCopyPropagation.cpp b/backend/lib/CodeGen/MachineCopyPropagation.cpp new file mode 100644 index 0000000..e537a06 --- /dev/null +++ b/backend/lib/CodeGen/MachineCopyPropagation.cpp @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCopyPropagation.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetCodeGen.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract & +copyPropagationContract(MachineCopyPropagationStage Stage) { + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "pre-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCopyPropagationStage::PreRegisterAllocation ? PreRA : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct CopyOperands final { + MachineOperand *DestinationOperand = nullptr; + MachineOperand *SourceOperand = nullptr; + Register Destination; + Register Source; +}; + +[[nodiscard]] std::optional copyOperands(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY) || Instruction.operandCount() != 2 || + hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr || !DestinationOperand->isDef() || + !SourceOperand->isUse() || DestinationOperand->isImplicit() || SourceOperand->isImplicit()) { + return std::nullopt; + } + return CopyOperands{DestinationOperand, SourceOperand, *Destination, *Source}; +} + +[[nodiscard]] Expected compatibleVirtualRegisters(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || !Source.isVirtual()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!DestinationInfo || !SourceInfo) + return std::unexpected(!DestinationInfo ? DestinationInfo.error() : SourceInfo.error()); + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && SourceValue.Bank && + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank)) { + return false; + } + if (DestinationValue.RegClass && SourceValue.RegClass && + !Target.registers().isSubclass(*SourceValue.RegClass, *DestinationValue.RegClass)) { + return false; + } + return true; +} + +[[nodiscard]] Expected propagateVirtualCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + if (!Operands.Destination.isVirtual() || !Operands.Source.isVirtual()) + return false; + if (Function.registerInfo().getUniqueVRegDef(Operands.Destination) != + Operands.DestinationOperand) { + return false; + } + auto Compatible = + compatibleVirtualRegisters(Operands.Destination, Operands.Source, Function, Target); + if (!Compatible) + return std::unexpected(Compatible.error()); + if (!*Compatible) + return false; + auto ValidCopy = + Info.validateCopy(Operands.Destination, Operands.DestinationOperand->subReg(), + Operands.Source, Operands.SourceOperand->subReg(), Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + + std::vector Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(Operands.Destination, true)) { + if (&Use == Operands.SourceOperand) + continue; + if (Use.tiedOperandIndex() || Use.isImplicit() || + Use.subReg() != Operands.DestinationOperand->subReg()) { + return false; + } + Uses.push_back(&Use); + } + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation candidate is detached"); + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(Operands.Destination, true) || + Function.registerInfo().getUniqueVRegDef(Operands.Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation left destination references"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Operands.Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] bool shareRegisterClass(PhysicalRegister Left, PhysicalRegister Right, + const TargetRegisterInfo &Registers) noexcept { + return std::ranges::any_of(Registers.registerClasses(), [&](const RegisterClassDesc &Class) { + return Registers.registerClassContains(Class.ID, Left) && + Registers.registerClassContains(Class.ID, Right); + }); +} + +[[nodiscard]] bool clobbersRegister(const MachineInstr &Instruction, PhysicalRegister Reg, + const TargetRegisterInfo &Registers) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (Mask->Mask && Mask->Mask->clobbers(Reg)) + return true; + continue; + } + const Register *OperandReg = Operand.getIf(); + const std::optional Physical = + OperandReg == nullptr ? std::nullopt : OperandReg->physicalRegister(); + if (Physical && Operand.isDef() && Registers.regsOverlap(*Physical, Reg)) + return true; + } + return false; +} + +[[nodiscard]] bool canReplacePhysicalUse(const MachineInstr &Instruction, std::size_t OperandIndex, + PhysicalRegister Source, + const TargetDescription &Target) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Operand == nullptr || Descriptor == nullptr || Operand->isImplicit() || + Operand->tiedOperandIndex() || Operand->subReg().value() != 0 || + OperandIndex >= Descriptor->Operands.size()) { + return false; + } + const OperandDescriptor &Constraint = Descriptor->Operands[OperandIndex]; + if (Constraint.RegClass && + !Target.registers().registerClassContains(*Constraint.RegClass, Source)) { + return false; + } + if (Constraint.RegBank && !Target.registers().bankContainsRegister(*Constraint.RegBank, Source)) { + return false; + } + return true; +} + +[[nodiscard]] Expected propagatePhysicalCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + const std::optional Destination = Operands.Destination.physicalRegister(); + const std::optional Source = Operands.Source.physicalRegister(); + if (!Destination || !Source || Operands.DestinationOperand->subReg().value() != 0 || + Operands.SourceOperand->subReg().value() != 0) { + return false; + } + auto ValidCopy = + Info.validateCopy(Operands.Destination, {}, Operands.Source, {}, Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "physical COPY propagation candidate is detached"); + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!shareRegisterClass(*Destination, *Source, Target.registers())) + return false; + + bool Changed = false; + for (MachineInstr *Instruction = Copy.next(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (clobbersRegister(*Instruction, *Destination, Target.registers()) || + clobbersRegister(*Instruction, *Source, Target.registers())) { + break; + } + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand->isUse() || Operand->isDebugUse() || + !Target.registers().regsOverlap(*Physical, *Destination) || + !canReplacePhysicalUse(*Instruction, Index, *Source, Target)) { + continue; + } + auto Replaced = Operand->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + Changed = true; + } + } + return Changed; +} + +} // namespace + +const MachinePassContract &MachineCopyPropagationPass::contract() const noexcept { + return copyPropagationContract(Stage); +} + +Expected MachineCopyPropagationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine COPY propagation has unmet stage properties"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "COPY propagation transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "COPY propagation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineCopyPropagationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t PropagatedCopies = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && PropagatedCopies < Options.MaximumPropagatedCopies;) { + MachineInstr *Next = Instruction->next(); + const std::optional Operands = copyOperands(*Instruction); + if (!Operands) { + Instruction = Next; + continue; + } + Expected Changed = + Stage == MachineCopyPropagationStage::PreRegisterAllocation + ? propagateVirtualCopy(*Instruction, *Operands, Function, Target, *Info) + : propagatePhysicalCopy(*Instruction, *Operands, Function, Target, *Info); + if (!Changed) + return std::unexpected(Changed.error()); + if (*Changed) { + ++PropagatedCopies; + Result.Changed = true; + } + Instruction = Next; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + if (Stage == MachineCopyPropagationStage::PreRegisterAllocation) + Result.Changes.add(MachinePassChange::Registers); + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineDCE.cpp b/backend/lib/CodeGen/MachineDCE.cpp new file mode 100644 index 0000000..b413499 --- /dev/null +++ b/backend/lib/CodeGen/MachineDCE.cpp @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineDCE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &dceContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-dce", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] Expected>> +deadDefinitions(MachineInstr &Instruction, MachineRegisterInfo &Registers, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + + std::vector VirtualDefinitions; + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Operand.isDef()) + continue; + HasDefinition = true; + if (Operand.isUse() || Operand.tiedOperandIndex()) + return std::nullopt; + if (Reg->isVirtual()) { + if (!Registers.useEmpty(*Reg, false)) + return std::nullopt; + VirtualDefinitions.push_back(*Reg); + continue; + } + if (!Reg->isPhysical() || !Operand.registerFlags().has(RegisterOperandFlag::Dead)) { + return std::nullopt; + } + } + if (!HasDefinition) + return std::nullopt; + return VirtualDefinitions; +} + +[[nodiscard]] Expected clearDebugUses(MachineRegisterInfo &Registers, Register Reg) { + std::vector Uses; + for (MachineOperand &Operand : Registers.useOperands(Reg, true)) { + if (Operand.isDebugUse()) + Uses.push_back(&Operand); + } + for (MachineOperand *Operand : Uses) { + auto Cleared = Operand->setRegister(Register{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineDCEPass::contract() const noexcept { + return dceContract(); +} + +Expected MachineDCEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE requires a selected machine function"); + } + + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine DCE transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine DCE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineDCEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) { + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + + bool LocalChange = true; + while (LocalChange) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Definitions = deadDefinitions(*Instruction, Registers, Target); + if (!Definitions) + return std::unexpected(Definitions.error()); + if (!*Definitions) { + Instruction = Next; + continue; + } + + for (const Register Reg : **Definitions) { + auto Cleared = clearDebugUses(Registers, Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + auto Removed = Block->removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + for (const Register Reg : **Definitions) { + if (!Registers.useEmpty(Reg, true) || Registers.getUniqueVRegDef(Reg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine DCE left references to a removed definition"); + } + auto Erased = Registers.eraseVirtualRegister(Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + LocalChange = true; + Result.Changed = true; + Instruction = Next; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFallthrough.cpp b/backend/lib/CodeGen/MachineFallthrough.cpp new file mode 100644 index 0000000..06d70ef --- /dev/null +++ b/backend/lib/CodeGen/MachineFallthrough.cpp @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFallthrough.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &fallthroughContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-fallthrough-normalization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected> +normalizeFallthroughs(MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info) { + std::vector ChangedBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + MachineBasicBlock *LayoutSuccessor = Block->next(); + if (LayoutSuccessor == nullptr) + continue; + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + continue; + bool Changed = false; + if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + continue; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget != nullptr) { + auto Inserted = + Info.insertUnconditionalBranch(*Block, *Analysis->FalseTarget, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Block->successors().size() == 1 && + Block->successors().front().Block != LayoutSuccessor) { + MachineBasicBlock *Successor = Block->successors().front().Block; + if (Successor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization encountered a null successor"); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } + if (Changed) + ChangedBlocks.push_back(Block); + } + return ChangedBlocks; +} + +} // namespace + +const MachinePassContract &MachineFallthroughPass::contract() const noexcept { + return fallthroughContract(); +} + +Expected MachineFallthroughPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization requires allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "fallthrough setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialChanged = normalizeFallthroughs(*Trial->ClonedFunction, Target, *Info); + if (!TrialChanged) + return std::unexpected(TrialChanged.error()); + auto TrialRepair = repairPostRAMachineBlockLiveness(*TrialChanged, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto ChangedBlocks = normalizeFallthroughs(Function, Target, *Info); + if (!ChangedBlocks) { + return std::unexpected(prefixedError(ChangedBlocks.error(), + "fallthrough diverged after a successful preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(*ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = !ChangedBlocks->empty() || *Repaired; + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFinalization.cpp b/backend/lib/CodeGen/MachineFinalization.cpp new file mode 100644 index 0000000..8523bb8 --- /dev/null +++ b/backend/lib/CodeGen/MachineFinalization.cpp @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFinalization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &finalizePseudosContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "finalize-emission-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {MachineFunctionProperty::NoExecutablePseudos}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &encodabilityContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "final-encodability-verifier", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices, + MachineFunctionProperty::NoExecutablePseudos}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits::max() - Mask) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return static_cast(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return -static_cast(Difference); +} + +} // namespace + +const MachinePassContract &FinalizeEmissionPseudosPass::contract() const noexcept { + return finalizePseudosContract(); +} + +Expected FinalizeEmissionPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization requires finalized allocated MIR"); + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo)) + continue; + const bool NonEmitting = Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::ProducesCode); + const bool Terminal = Descriptor->Flags.has(InstructionFlag::TerminalEmission) && + Descriptor->Owner == PseudoOwner::AsmPrinter; + if (!NonEmitting && !Terminal) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "non-terminal executable pseudo survived all expansion owners"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &FinalEncodabilityVerifierPass::contract() const noexcept { + return encodabilityContract(); +} + +Expected FinalEncodabilityVerifierPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final encodability requires NoExecutablePseudos and finalized MIR"); + + std::unordered_map BlockOffsets; + std::unordered_map InstructionOffsets; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction uses an unknown opcode"); + auto InstructionValidation = + Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!InstructionValidation) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction is not encodable: " + + std::string(InstructionValidation.error().message())); + } + const bool BundledWithPred = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool BundledWithSucc = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if ((BundledWithPred && + (Instruction->previous() == nullptr || + !Instruction->previous()->flags().has(MachineInstrFlag::BundledWithSucc))) || + (BundledWithSucc && + (Instruction->next() == nullptr || + !Instruction->next()->flags().has(MachineInstrFlag::BundledWithPred)))) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction has an inconsistent bundle boundary"); + } + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf(); + if (Reg != nullptr && !Reg->isNoRegister() && !Reg->isPhysical()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a virtual register"); + if (Operand.kind() == MachineOperand::Kind::FrameIndex) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a frame index"); + } + auto Size = BranchInfo->instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + const bool ProducesCode = Descriptor->Flags.has(InstructionFlag::ProducesCode); + if ((ProducesCode && *Size == 0) || + (Descriptor->Flags.has(InstructionFlag::Meta) && *Size != 0)) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction violates its exact size contract"); + } + InstructionOffsets.emplace(Instruction, Offset); + if (Offset > std::numeric_limits::max() - *Size) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine function size overflowed"); + Offset += *Size; + } + } + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto TargetBlock = BranchInfo->branchTarget(*Instruction, Function, Target); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + if (*TargetBlock == nullptr) + continue; + const auto Source = InstructionOffsets.find(Instruction); + const auto Destination = BlockOffsets.find(*TargetBlock); + if (Source == InstructionOffsets.end() || Destination == BlockOffsets.end()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch target has no layout offset"); + auto Distance = displacement(Source->second, Destination->second); + if (!Distance) + return std::unexpected(Distance.error()); + if (!BranchInfo->isBranchOffsetInRange(*Instruction, *Distance)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction contains an out-of-range branch"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunction.cpp b/backend/lib/CodeGen/MachineFunction.cpp index df1ceb5..7f4c3a9 100644 --- a/backend/lib/CodeGen/MachineFunction.cpp +++ b/backend/lib/CodeGen/MachineFunction.cpp @@ -172,6 +172,66 @@ Expected MachineFunction::insertBlockBefore(MachineBasicBlock &Position, return {}; } +Expected MachineFunction::moveBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Block.NextBlock == &Position) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = Position.PreviousBlock; + Block.NextBlock = &Position; + if (Position.PreviousBlock != nullptr) + Position.PreviousBlock->NextBlock = &Block; + else + FirstBlock = &Block; + Position.PreviousBlock = &Block; + renumberBlocks(); + return {}; +} + +Expected MachineFunction::moveBlockAfter(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Position.NextBlock == &Block) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = &Position; + Block.NextBlock = Position.NextBlock; + if (Position.NextBlock != nullptr) + Position.NextBlock->PreviousBlock = &Block; + else + LastBlock = &Block; + Position.NextBlock = &Block; + renumberBlocks(); + return {}; +} + Expected MachineFunction::removeBlock(MachineBasicBlock &Block) { if (Block.OwnerFunction != this || !Block.IsLinked) return makeUnexpected(ErrorCode::InvalidArgument, diff --git a/backend/lib/CodeGen/MachineFunctionRecords.cpp b/backend/lib/CodeGen/MachineFunctionRecords.cpp index 0a21581..18ac596 100644 --- a/backend/lib/CodeGen/MachineFunctionRecords.cpp +++ b/backend/lib/CodeGen/MachineFunctionRecords.cpp @@ -127,6 +127,38 @@ Expected MachineFunctionRecords::addCalledGlobal(MachineCalledGlobal Calle return {}; } +Expected +MachineFunctionRecords::replaceInstructionReference(MachineInstrStableID OldInstruction, + MachineInstrStableID NewInstruction) { + if (!OldInstruction.isValid() || !NewInstruction.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction-record replacement requires two valid stable IDs"); + } + if (OldInstruction == NewInstruction) + return {}; + for (const MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a call-site record"); + } + } + for (const MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a called-global record"); + } + } + for (MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == OldInstruction) + CallSite.CallInstruction = NewInstruction; + } + for (MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == OldInstruction) + CalledGlobal.CallInstruction = NewInstruction; + } + return {}; +} + Expected MachineFunctionRecords::addPrefetchTarget(MachinePrefetchTarget Target) { if (std::ranges::find(PrefetchTargets, Target) != PrefetchTargets.end()) { return makeUnexpected(ErrorCode::InvalidArgument, "prefetch target is already recorded"); diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp new file mode 100644 index 0000000..8c8d732 --- /dev/null +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineLICM.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &licmContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-licm", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-licm", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool containsBlock(const MachineLoop &Loop, const MachineBasicBlock *Block) noexcept { + return std::ranges::find(Loop.Blocks, Block) != Loop.Blocks.end(); +} + +[[nodiscard]] MachineBasicBlock *preheader(const MachineLoop &Loop) noexcept { + if (Loop.Header == nullptr) + return nullptr; + MachineBasicBlock *Result = nullptr; + for (MachineBasicBlock *Predecessor : Loop.Header->predecessors()) { + if (containsBlock(Loop, Predecessor)) + continue; + if (Result != nullptr) + return nullptr; + Result = Predecessor; + } + if (Result == nullptr || Result->successors().size() != 1 || + Result->successors().front().Block != Loop.Header) { + return nullptr; + } + return Result; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected isSafeToHoist(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +[[nodiscard]] std::size_t definitionCount(const MachineInstr &Instruction) noexcept { + return static_cast(std::ranges::count_if( + Instruction.operands(), [](const MachineOperand &Operand) { return Operand.isDef(); })); +} + +[[nodiscard]] bool +operandsAreInvariant(const MachineInstr &Instruction, const MachineLoop &Loop, + const MachineRegisterInfo &Registers, + const std::unordered_set &Hoisted) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Operand.isUse()) + continue; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Registers.getUniqueVRegDef(*Reg); + const MachineInstr *DefiningInstruction = + Definition == nullptr ? nullptr : Definition->parent(); + const MachineBasicBlock *DefiningBlock = + DefiningInstruction == nullptr ? nullptr : DefiningInstruction->parent(); + if (DefiningInstruction == nullptr || DefiningBlock == nullptr) + return false; + if (containsBlock(Loop, DefiningBlock) && !Hoisted.contains(DefiningInstruction)) + return false; + } + return true; +} + +[[nodiscard]] bool postRAOperandsAreInvariant(const MachineInstr &Instruction, + const MachineLoop &Loop, + const MachineFunction &Function, + const TargetDescription &Target) noexcept { + if (Loop.Header == nullptr || Instruction.parent() != Loop.Header) + return false; + + std::vector Uses; + std::vector Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return false; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return false; + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (std::ranges::any_of(Loop.Header->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return false; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return false; + } + } + + const auto ClobbersCandidate = [&](const RegisterMask &Mask) { + return std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }); + }; + for (const MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + return false; + for (const MachineInstr *Candidate = Block->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + if (Candidate == &Instruction) + continue; + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (ClobbersCandidate(*Mask->Mask)) + return false; + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand.isDef()) + continue; + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(*Physical, Use); + }) || + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + } + + for (const MachineInstr *Before = Loop.Header->firstInstruction(); + Before != nullptr && Before != &Instruction; Before = Before->next()) { + for (const MachineOperand &Operand : Before->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return false; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && (Operand.isUse() || Operand.isDef()) && + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + return true; +} + +[[nodiscard]] Expected firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM preheader contains an unknown instruction"); + } + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] Expected moveToPreheader(MachineInstr &Instruction, + MachineBasicBlock &Preheader, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine LICM candidate is detached"); + auto Anchor = firstTerminator(Preheader, Target); + if (!Anchor) + return std::unexpected(Anchor.error()); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (*Anchor != nullptr) + return Preheader.insertInstructionBefore(**Anchor, Instruction); + return Preheader.appendInstruction(Instruction); +} + +} // namespace + +const MachinePassContract &MachineLICMPass::contract() const noexcept { + return licmContract(Stage); +} + +Expected MachineLICMPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine LICM requires selected SSA MachineIR" + : "post-RA machine LICM requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine LICM transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine LICM diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineLICMPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t TotalHoisted = 0; + for (const MachineLoop &Loop : (*Loops)->loops()) { + MachineBasicBlock *Preheader = preheader(Loop); + if (Preheader == nullptr || Loop.Header == nullptr || + (*Frequencies)->frequency(*Preheader) > (*Frequencies)->frequency(*Loop.Header)) { + continue; + } + std::unordered_set Hoisted; + std::size_t HoistedDefinitions = 0; + bool LocalChange = true; + while (LocalChange && TotalHoisted < Options.MaximumHoistedInstructions) { + LocalChange = false; + for (MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + continue; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToHoist(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + const std::size_t Definitions = definitionCount(*Instruction); + const bool Invariant = + Stage == MachineCodeMotionStage::SelectedSSA + ? operandsAreInvariant(*Instruction, Loop, Function.registerInfo(), Hoisted) + : postRAOperandsAreInvariant(*Instruction, Loop, Function, Target); + if (*Safe && Invariant && + Definitions <= + Options.MaximumHoistedDefinitionsPerLoop - + std::min(HoistedDefinitions, Options.MaximumHoistedDefinitionsPerLoop)) { + auto Moved = moveToPreheader(*Instruction, *Preheader, Target); + if (!Moved) + return std::unexpected(Moved.error()); + Hoisted.insert(Instruction); + HoistedDefinitions += Definitions; + ++TotalHoisted; + LocalChange = true; + Result.Changed = true; + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + Instruction = Next; + } + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMemoryFolding.cpp b/backend/lib/CodeGen/MachineMemoryFolding.cpp new file mode 100644 index 0000000..0e15146 --- /dev/null +++ b/backend/lib/CodeGen/MachineMemoryFolding.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMemoryFolding.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &memoryFoldingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-memory-folding", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct FoldCandidate final { + MachineInstr *MemoryInstruction = nullptr; + MachineInstr *Consumer = nullptr; + Register DefinedRegister; + std::size_t ConsumerOperand = 0; +}; + +[[nodiscard]] std::optional findCandidate(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::MayLoad) || + Descriptor->Flags.has(InstructionFlag::MayStore) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Barrier) || + Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects) || + hasProtectedAttachment(Instruction) || Instruction.memOperands().empty()) { + return std::nullopt; + } + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) { + return std::nullopt; + } + } + + MachineOperand *Definition = nullptr; + Register DefinedRegister; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + if (Reg == nullptr || !Operand->isDef()) + continue; + if (Definition != nullptr || !Reg->isVirtual()) + return std::nullopt; + Definition = Operand; + DefinedRegister = *Reg; + } + if (Definition == nullptr || !Function.registerInfo().hasOneUse(DefinedRegister, true)) { + return std::nullopt; + } + MachineOperand *Use = nullptr; + for (MachineOperand &Candidate : Function.registerInfo().useOperands(DefinedRegister, false)) { + Use = &Candidate; + break; + } + MachineInstr *Consumer = Use == nullptr ? nullptr : Use->parent(); + if (Consumer == nullptr || Consumer->parent() != Instruction.parent() || + Instruction.next() != Consumer || hasProtectedAttachment(*Consumer) || Use->isImplicit() || + Use->tiedOperandIndex()) { + return std::nullopt; + } + for (std::size_t Index = 0; Index < Consumer->operandCount(); ++Index) { + if (Consumer->operandAt(Index) == Use) + return FoldCandidate{&Instruction, Consumer, DefinedRegister, Index}; + } + return std::nullopt; +} + +[[nodiscard]] bool containsMemoryDescription(const MachineInstr &Instruction, + const MachineMemOperandDesc &Description) noexcept { + return std::ranges::any_of(Instruction.memOperands(), + [&Description](const MachineMemOperand *Memory) { + return Memory != nullptr && Memory->description() == Description; + }); +} + +[[nodiscard]] Expected applyFold(FoldCandidate Candidate, MachineFunction &Function, + const TargetDescription &Target, + const TargetMemoryFoldingInfo &Info) { + auto CanFold = Info.canFoldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!CanFold) + return std::unexpected(CanFold.error()); + if (!*CanFold) + return false; + std::vector MemoryDescriptions; + for (const MachineMemOperand *Memory : Candidate.MemoryInstruction->memOperands()) { + if (Memory == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding encountered a null memory operand"); + } + MemoryDescriptions.push_back(Memory->description()); + } + auto Folded = Info.foldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!Folded) + return std::unexpected(Folded.error()); + if (*Folded == nullptr || (*Folded)->parent() == nullptr || + Candidate.MemoryInstruction->parent() != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not replace the consumer and erase the producer"); + } + for (const MachineMemOperandDesc &Description : MemoryDescriptions) { + if (!containsMemoryDescription(**Folded, Description)) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not migrate every memory operand"); + } + } + auto FoldedSize = Info.getFoldedMemorySize(**Folded, Function, Target); + if (!FoldedSize) + return std::unexpected(FoldedSize.error()); + if (!FoldedSize->has_value() || !FoldedSize->value().isKnown()) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded instruction has no known folded memory size"); + } + if (MemoryDescriptions.size() == 1 && MemoryDescriptions.front().Size.isKnown() && + FoldedSize->value() != MemoryDescriptions.front().Size) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded memory size disagrees with the migrated MMO"); + } + if (!Function.registerInfo().useEmpty(Candidate.DefinedRegister, true) || + Function.registerInfo().getUniqueVRegDef(Candidate.DefinedRegister) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold retained the load result register"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Candidate.DefinedRegister); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineMemoryFoldingPass::contract() const noexcept { + return memoryFoldingContract(); +} + +Expected MachineMemoryFoldingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "memory folding transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "memory folding diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineMemoryFoldingPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t FoldCount = 0; + bool LocalChange = true; + while (LocalChange && FoldCount < Options.MaximumFolds) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !LocalChange; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + std::optional Candidate = findCandidate(*Instruction, Function, Target); + if (!Candidate) + continue; + auto Folded = applyFold(*Candidate, Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + if (!*Folded) + continue; + ++FoldCount; + Result.Changed = true; + LocalChange = true; + break; + } + } + } + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp index 43a4f91..d1d152e 100644 --- a/backend/lib/CodeGen/MachineOperand.cpp +++ b/backend/lib/CodeGen/MachineOperand.cpp @@ -548,6 +548,18 @@ Expected MachineOperand::setBasicBlock(MachineBasicBlock &Block) { return {}; } +Expected MachineOperand::setFrameIndex(FrameIndex Index) { + if (OperandKind != Kind::FrameIndex) + return makeUnexpected(ErrorCode::InvalidArgument, + "setFrameIndex requires a frame-index operand"); + if (Parent != nullptr && Parent->function().frameInfo().object(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frame-index operand references an unknown stack object"); + } + std::get(Storage).Index = Index; + return {}; +} + MachineOperand MachineOperand::cloneDetached() const { MachineOperand Result(OperandKind, Storage, TargetFlags); Result.RegFlags = RegFlags; diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp new file mode 100644 index 0000000..76dbd44 --- /dev/null +++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOptimizationPipeline.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBlockPlacement.h" +#include "rcc/CodeGen/MachineBranchOptimization.h" +#include "rcc/CodeGen/MachineBranchRelaxation.h" +#include "rcc/CodeGen/MachineCSE.h" +#include "rcc/CodeGen/MachineCombiner.h" +#include "rcc/CodeGen/MachineCopyPropagation.h" +#include "rcc/CodeGen/MachineDCE.h" +#include "rcc/CodeGen/MachineFallthrough.h" +#include "rcc/CodeGen/MachineFinalization.h" +#include "rcc/CodeGen/MachineLICM.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachinePseudoExpansion.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/MachineShrinkWrapping.h" +#include "rcc/CodeGen/MachineSink.h" +#include "rcc/CodeGen/MachineStackSlotColoring.h" +#include "rcc/CodeGen/MachineTailDuplication.h" +#include "rcc/CodeGen/MachineTailMerge.h" +#include "rcc/CodeGen/PostRAScheduler.h" +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" + +#include +#include + +namespace rcc { +namespace { + +template +[[nodiscard]] Expected addPass(MachinePassManager &Pipeline, Arguments &&...Values) { + return Pipeline.addPass(std::make_unique(std::forward(Values)...)); +} + +} // namespace + +Expected registerMachineOptimizationAnalyses(MachineAnalysisManager &Analyses) { + auto Registered = registerStage8MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + return registerStage11MachineAnalyses(Analyses); +} + +Expected +addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.MemoryFolding); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, MachineCodeMotionStage::SelectedSSA); +} + +Expected addPreRegisterAllocationOptimizationPipeline( + MachinePassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, TargetInfo.SchedulingModel); +} + +Expected +addPostRegisterAllocationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = + addPass(Pipeline, TargetInfo.Combiner, MachineCombinerStage::PostFrame); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SchedulingModel); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion, + TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, TargetInfo.BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePseudoExpansion.cpp b/backend/lib/CodeGen/MachinePseudoExpansion.cpp new file mode 100644 index 0000000..e654d97 --- /dev/null +++ b/backend/lib/CodeGen/MachinePseudoExpansion.cpp @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePseudoExpansion.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +using CFGEdge = std::tuple; + +struct ResourceSnapshot final { + std::size_t BlockCount = 0; + std::size_t VirtualRegisterCount = 0; + std::size_t FrameObjectCount = 0; + bool FrameFrozen = false; + std::vector Edges; +}; + +using BlockSizeSnapshot = std::vector>; + +[[nodiscard]] Expected blockSize(const MachineBasicBlock &Block, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + std::uint64_t Size = 0; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto InstructionSize = BranchInfo.instructionSize(*Instruction, Function, Target); + if (!InstructionSize) + return std::unexpected(InstructionSize.error()); + if (Size > std::numeric_limits::max() - *InstructionSize) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, "target-late pseudo size overflowed"); + } + Size += *InstructionSize; + } + return Size; +} + +[[nodiscard]] Expected snapshotBlockSizes(const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + BlockSizeSnapshot Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Size = blockSize(*Block, Function, Target, BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + Result.emplace_back(Block->stableID(), *Size); + } + return Result; +} + +[[nodiscard]] ResourceSnapshot snapshotResources(const MachineFunction &Function) { + ResourceSnapshot Result; + Result.BlockCount = Function.blockCount(); + Result.VirtualRegisterCount = Function.registerInfo().virtualRegisters().size(); + Result.FrameObjectCount = Function.frameInfo().objects().size(); + Result.FrameFrozen = Function.frameInfo().isFrozen(); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineSuccessor &Successor : Block->successors()) { + Result.Edges.emplace_back(Block->stableID().value(), Successor.Block->stableID().value(), + Successor.Probability.rawValue()); + } + } + return Result; +} + +[[nodiscard]] Expected verifyResourceContract(const ResourceSnapshot &Before, + const MachineFunction &Function, + MachinePseudoExpansionPhase Phase) { + const ResourceSnapshot After = snapshotResources(Function); + if (Before.BlockCount != After.BlockCount || Before.Edges != After.Edges) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion changed CFG resources outside its owner contract"); + } + if (Before.VirtualRegisterCount != After.VirtualRegisterCount) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-selection pseudo expansion created a virtual register"); + } + if (Phase != MachinePseudoExpansionPhase::PrePEI && + (Before.FrameObjectCount != After.FrameObjectCount || + Before.FrameFrozen != After.FrameFrozen)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-PEI pseudo expansion changed frozen frame resources"); + } + return {}; +} + +[[nodiscard]] PseudoOwner ownerFor(MachinePseudoExpansionPhase Phase) noexcept { + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PseudoOwner::PrePEIExpansion; + case MachinePseudoExpansionPhase::PostRA: + return PseudoOwner::PostRAExpansion; + case MachinePseudoExpansionPhase::TargetLate: + return PseudoOwner::TargetLateExpansion; + } + return PseudoOwner::None; +} + +[[nodiscard]] const MachinePassContract &preparationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "prepare-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &phaseContract(MachinePseudoExpansionPhase Phase) { + static const MachinePassContract PrePEI = [] { + MachinePassContract Value{ + .Name = "pre-pei-target-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Frame}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "expand-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + static const MachinePassContract TargetLate = [] { + MachinePassContract Value{ + .Name = "target-late-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PrePEI; + case MachinePseudoExpansionPhase::PostRA: + return PostRA; + case MachinePseudoExpansionPhase::TargetLate: + return TargetLate; + } + return TargetLate; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected +prepareImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + bool Changed = false; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo preparation encountered an unknown opcode"); + if (Descriptor->Owner != PseudoOwner::PostRAExpansion) + continue; + auto Prepared = Info.preparePostRAPseudo(*Instruction, Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + Changed |= *Prepared; + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected +expandImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + const ResourceSnapshot Before = snapshotResources(Function); + std::optional SizesBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + if (BranchInfo == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target-late pseudo expansion requires branch sizing"); + } + auto Snapshot = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!Snapshot) + return std::unexpected(Snapshot.error()); + SizesBefore = std::move(*Snapshot); + } + std::vector Worklist; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction != nullptr && Instruction->parent() != nullptr) + Worklist.push_back(Instruction); + } + bool Changed = false; + for (MachineInstr *Instruction : Worklist) { + if (Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo expansion encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo) || Descriptor->Owner != ownerFor(Phase)) { + continue; + } + MachineBasicBlock *ExpansionBlock = Instruction->parent(); + std::optional ExpansionSizeBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + auto Size = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + ExpansionSizeBefore = *Size; + } + auto Expanded = Info.expandPseudo(*Instruction, Phase, Function, Target); + if (!Expanded) + return std::unexpected(Expanded.error()); + if (!*Expanded) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target has no expansion for an owned executable pseudo"); + } + if (ExpansionSizeBefore.has_value()) { + auto SizeAfter = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!SizeAfter) + return std::unexpected(SizeAfter.error()); + if (*SizeAfter > *ExpansionSizeBefore) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "one target-late pseudo expansion increased its encoded block size"); + } + } + Changed = true; + } + + auto Resources = verifyResourceContract(Before, Function, Phase); + if (!Resources) + return std::unexpected(Resources.error()); + if (SizesBefore.has_value()) { + auto SizesAfter = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!SizesAfter) + return std::unexpected(SizesAfter.error()); + if (SizesAfter->size() != SizesBefore->size()) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion changed block layout"); + } + for (std::size_t Index = 0; Index < SizesAfter->size(); ++Index) { + if ((*SizesAfter)[Index].first != (*SizesBefore)[Index].first || + (*SizesAfter)[Index].second > (*SizesBefore)[Index].second) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion increased an encoded block size"); + } + } + } + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Pseudo) && + Descriptor->Owner == ownerFor(Phase)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion left an instruction owned by the completed phase"); + } + } + if (Changed && Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected +runPreparationTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo preparation setup failed: ")); + auto TrialResult = prepareImplementation(*Trial->ClonedFunction, Target, Info); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = prepareImplementation(Function, Target, Info); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo preparation diverged after successful preflight: ")); + } + return Result; +} + +[[nodiscard]] Expected +runExpansionTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo expansion setup failed: ")); + auto TrialResult = expandImplementation(*Trial->ClonedFunction, Target, Info, Phase, BranchInfo); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = expandImplementation(Function, Target, Info, Phase, BranchInfo); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo expansion diverged after successful preflight: ")); + } + return Result; +} + +} // namespace + +const MachinePassContract &PreparePostRAPseudosPass::contract() const noexcept { + return preparationContract(); +} + +Expected PreparePostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo preparation requires selected MachineIR"); + return runPreparationTransaction(Function, Target, *Info); +} + +const MachinePassContract &PrePEITargetPseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PrePEI); +} + +Expected PrePEITargetPseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "pre-PEI pseudo expansion requires allocated tracked-liveness MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PrePEI, + nullptr); +} + +const MachinePassContract &ExpandPostRAPseudosPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PostRA); +} + +Expected ExpandPostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo expansion requires finalized frame and liveness"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PostRA, + nullptr); +} + +const MachinePassContract &TargetLatePseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::TargetLate); +} + +Expected TargetLatePseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "target-late pseudo expansion requires finalized allocated MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::TargetLate, + BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineScheduler.cpp b/backend/lib/CodeGen/MachineScheduler.cpp new file mode 100644 index 0000000..f861be2 --- /dev/null +++ b/backend/lib/CodeGen/MachineScheduler.cpp @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/RegisterPressure.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &schedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] Expected isSchedulingBoundary(const MachineInstr &Instruction, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Model.hooks().isSchedulingBoundary(Instruction, Target)) + return true; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi)) { + return true; + } + if (Model.schedulingClass(Descriptor->SchedulingClass) == nullptr) { + if (Flags.has(InstructionFlag::Pseudo) || !Flags.has(InstructionFlag::ProducesCode)) + return true; + return makeUnexpected(ErrorCode::VerificationError, + "executable instruction has no scheduling class"); + } + return false; +} + +struct ScheduleCandidate final { + std::size_t Unit = 0; + bool Bottom = false; + std::uint64_t Cycle = 0; + RegisterPressureCost Pressure; + std::uint64_t CriticalPath = 0; + std::uint32_t Latency = 0; + std::int64_t TargetPreference = 0; + bool OptimizeLatency = false; + std::uint64_t StableID = 0; +}; + +[[nodiscard]] bool isBetterCandidate(const ScheduleCandidate &Left, + const ScheduleCandidate &Right) noexcept { + if (Left.Cycle != Right.Cycle) + return Left.Cycle < Right.Cycle; + if (Left.TargetPreference != Right.TargetPreference) + return Left.TargetPreference > Right.TargetPreference; + if (Left.OptimizeLatency && Right.OptimizeLatency) { + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + } + if (Left.Pressure.PeakIncrease != Right.Pressure.PeakIncrease) + return Left.Pressure.PeakIncrease < Right.Pressure.PeakIncrease; + if (Left.Pressure.PeakExcess != Right.Pressure.PeakExcess) + return Left.Pressure.PeakExcess < Right.Pressure.PeakExcess; + if (Left.Pressure.CurrentExcess != Right.Pressure.CurrentExcess) + return Left.Pressure.CurrentExcess < Right.Pressure.CurrentExcess; + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + if (Left.StableID != Right.StableID) + return Left.StableID < Right.StableID; + return !Left.Bottom && Right.Bottom; +} + +[[nodiscard]] Expected> +scheduleDAG(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + std::uint64_t BlockFrequency, bool EnableBidirectionalScheduling) { + auto TopPressure = RegisterPressureTracker::create(DAG, Function, Target, Model, Mode, Liveness, + RegisterPressureDirection::TopDown); + if (!TopPressure) + return std::unexpected(TopPressure.error()); + auto BottomPressure = RegisterPressureTracker::create( + DAG, Function, Target, Model, Mode, Liveness, RegisterPressureDirection::BottomUp); + if (!BottomPressure) + return std::unexpected(BottomPressure.error()); + auto TopHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::TopDown); + if (!TopHazards) + return std::unexpected(TopHazards.error()); + auto BottomHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::BottomUp); + if (!BottomHazards) + return std::unexpected(BottomHazards.error()); + + const std::size_t UnitCount = DAG.units().size(); + std::vector TopPredecessors(UnitCount); + std::vector BottomSuccessors(UnitCount); + std::vector TopReadyCycle(UnitCount, 0); + std::vector BottomReadyCycle(UnitCount, 0); + std::vector Scheduled(UnitCount, false); + for (const ScheduleUnit &Unit : DAG.units()) { + TopPredecessors[Unit.ID] = Unit.Predecessors.size(); + BottomSuccessors[Unit.ID] = Unit.Successors.size(); + } + + std::vector TopOrder; + std::vector BottomOrder; + TopOrder.reserve(UnitCount); + BottomOrder.reserve(UnitCount); + for (std::size_t ScheduledCount = 0; ScheduledCount < UnitCount; ++ScheduledCount) { + std::optional Best; + for (const ScheduleUnit &Unit : DAG.units()) { + if (Scheduled[Unit.ID]) + continue; + if (TopPredecessors[Unit.ID] == 0) { + auto Cycle = TopHazards->earliestIssueCycle(Unit, TopReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = TopPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + false, + *Cycle, + *Pressure, + Unit.Height, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::TopDown, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + if (EnableBidirectionalScheduling && BottomSuccessors[Unit.ID] == 0) { + auto Cycle = BottomHazards->earliestIssueCycle(Unit, BottomReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = BottomPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + true, + *Cycle, + *Pressure, + Unit.Depth, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::BottomUp, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + } + if (!Best) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "schedule DAG has no legal top or bottom ready unit"); + } + + const ScheduleUnit &Selected = DAG.units()[Best->Unit]; + Scheduled[Best->Unit] = true; + if (Best->Bottom) { + auto Issued = BottomHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = BottomPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + BottomOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Predecessors) { + if (BottomSuccessors[Dependency.Source] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "bottom scheduler successor count underflowed"); + } + --BottomSuccessors[Dependency.Source]; + BottomReadyCycle[Dependency.Source] = std::max( + BottomReadyCycle[Dependency.Source], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } else { + auto Issued = TopHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = TopPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + TopOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Successors) { + if (TopPredecessors[Dependency.Destination] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "top scheduler predecessor count underflowed"); + } + --TopPredecessors[Dependency.Destination]; + TopReadyCycle[Dependency.Destination] = std::max( + TopReadyCycle[Dependency.Destination], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } + } + + TopOrder.insert(TopOrder.end(), BottomOrder.rbegin(), BottomOrder.rend()); + if (TopOrder.size() != UnitCount) + return makeUnexpected(ErrorCode::InternalError, "scheduler lost a schedule unit"); + return TopOrder; +} + +[[nodiscard]] Expected +scheduleRegion(MachineBasicBlock &Block, std::span Instructions, + MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const MachineBlockLiveness &Liveness, std::uint64_t BlockFrequency, + MachineSchedulerOptions Options) { + if (Instructions.size() < 2 || Instructions.size() > Options.MaximumRegionSize) + return false; + auto DAG = ScheduleDAG::build(Block, Instructions, Function, Target, Model, Mode); + if (!DAG) + return std::unexpected(DAG.error()); + if (DAG->units().size() < 2) + return false; + auto UnitOrder = scheduleDAG(*DAG, Function, Target, Model, Mode, Liveness, BlockFrequency, + Options.EnableBidirectionalScheduling); + if (!UnitOrder) + return std::unexpected(UnitOrder.error()); + + std::vector NewOrder; + NewOrder.reserve(Instructions.size()); + for (const std::size_t UnitID : *UnitOrder) { + const ScheduleUnit *Unit = DAG->unit(UnitID); + if (Unit == nullptr) + return makeUnexpected(ErrorCode::InternalError, "scheduler produced an unknown unit ID"); + NewOrder.insert(NewOrder.end(), Unit->Instructions.begin(), Unit->Instructions.end()); + } + if (std::ranges::equal(NewOrder, Instructions)) + return false; + + MachineInstr *Anchor = Instructions.back()->next(); + for (MachineInstr *Instruction : Instructions) { + auto Removed = Block.removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + } + for (MachineInstr *Instruction : NewOrder) { + Expected Inserted = Anchor == nullptr + ? Block.appendInstruction(*Instruction) + : Block.insertInstructionBefore(*Anchor, *Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + return true; +} + +} // namespace + +Expected runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options) { + if (Options.MaximumRegionSize == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine scheduler region budget must be nonzero"); + } + MachineSchedulingRunResult Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const MachineBlockLiveness *BlockLiveness = Liveness.block(*Block); + if (BlockLiveness == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine scheduler has no block liveness boundary"); + } + bool BlockChanged = false; + std::vector Region; + auto FlushRegion = [&]() -> Expected { + auto Changed = scheduleRegion(*Block, Region, Function, Target, Model, Mode, *BlockLiveness, + Frequencies.frequency(*Block), Options); + if (!Changed) + return std::unexpected(Changed.error()); + BlockChanged |= *Changed; + Region.clear(); + return {}; + }; + + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a broken bundle header"); + } + std::vector Group; + bool Boundary = false; + for (;;) { + Group.push_back(Instruction); + auto IsBoundary = isSchedulingBoundary(*Instruction, Target, Model); + if (!IsBoundary) + return std::unexpected(IsBoundary.error()); + Boundary |= *IsBoundary; + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!HasSuccessor) + break; + MachineInstr *Next = Instruction->next(); + if (Next == nullptr || !Next->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a truncated bundle"); + } + Instruction = Next; + } + MachineInstr *NextGroup = Instruction->next(); + if (Boundary) { + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + } else { + Region.insert(Region.end(), Group.begin(), Group.end()); + } + Instruction = NextGroup; + } + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + if (BlockChanged) { + Result.Changed = true; + Result.ChangedBlocks.push_back(Block); + } + } + return Result; +} + +const MachinePassContract &MachineSchedulerPass::contract() const noexcept { + return schedulerContract(); +} + +Expected MachineSchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA machine scheduler requires selected, SSA-exited MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialResult = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PreRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PreRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "machine scheduler diverged after a successful transaction preflight: ")); + } + + MachinePassResult Result; + Result.Changed = Run->Changed; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSchedulerInternal.h b/backend/lib/CodeGen/MachineSchedulerInternal.h new file mode 100644 index 0000000..a612548 --- /dev/null +++ b/backend/lib/CodeGen/MachineSchedulerInternal.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H +#define RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/Support/Error.h" + +#include + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class TargetDescription; +class TargetSchedulingModel; +struct MachineSchedulingRunResult final { + bool Changed = false; + std::vector ChangedBlocks; +}; + +[[nodiscard]] Expected runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H diff --git a/backend/lib/CodeGen/MachineShrinkWrapping.cpp b/backend/lib/CodeGen/MachineShrinkWrapping.cpp new file mode 100644 index 0000000..aee3323 --- /dev/null +++ b/backend/lib/CodeGen/MachineShrinkWrapping.cpp @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineShrinkWrapping.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &shrinkWrappingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-shrink-wrapping", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool requiresEntryFrame(const MachineFrameState &State) noexcept { + return State.IsFrameAddressTaken || State.IsReturnAddressTaken || State.HasStackMap || + State.HasPatchPoint || State.HasOpaqueSPAdjustment || State.HasVAStart || + State.HasMustTailInVarArgFunc || State.HasTailCall; +} + +[[nodiscard]] bool containsEpilogueTerminator(const MachineBasicBlock &Block, + const TargetFrameLowering &Info, + const TargetDescription &Target) noexcept { + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Info.isEpilogueTerminator(*Instruction, Target)) + return true; + } + return false; +} + +} // namespace + +const MachinePassContract &MachineShrinkWrappingPass::contract() const noexcept { + return shrinkWrappingContract(); +} + +Expected MachineShrinkWrappingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping requires allocated tracked-liveness MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "shrink wrapping must run before frame layout freeze"); + } + if (Function.blockCount() > Options.MaximumAnalyzedBlocks) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineFrameState &OriginalState = Function.frameInfo().state(); + if (!OriginalState.SavePoints.empty() || !OriginalState.RestorePoints.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr || requiresEntryFrame(OriginalState)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + + std::vector RequiredBlocks; + std::unordered_set ReferencedObjects; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + bool Required = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping encountered an unknown opcode"); + } + Required |= Descriptor->Flags.has(InstructionFlag::Call); + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Frame = Operand.getIf()) { + ReferencedObjects.insert(Frame->Index); + Required = true; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && Operand.isDef() && + Info->isCalleeSavedRegister(*Physical, Function, Target)) { + Required = true; + } + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + const std::optional Frame = + Memory == nullptr ? std::nullopt : Memory->description().Base.frameIndex(); + if (Frame) { + ReferencedObjects.insert(*Frame); + Required = true; + } + } + } + if (Required) + RequiredBlocks.push_back(Block); + } + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || Description->IsDead) + continue; + if (!ReferencedObjects.contains(Object.index())) { + RequiredBlocks.push_back(Entry); + break; + } + } + if (RequiredBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineBasicBlock *Save = RequiredBlocks.front(); + if (!(*Dominators)->isReachable(*Save)) + Save = Entry; + for (MachineBasicBlock *Block : RequiredBlocks) { + if (!(*Dominators)->isReachable(*Block)) { + Save = Entry; + break; + } + const MachineBasicBlock *Common = (*Dominators)->nearestCommonDominator(*Save, *Block); + if (Common == nullptr) { + Save = Entry; + break; + } + Save = Common; + } + if (Save == Entry || Save->predecessors().size() != 1 || (*Loops)->loopDepth(*Save) != 0 || + (*Loops)->isIrreducible(*Save)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + std::vector RestoreBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!(*Dominators)->dominates(*Save, *Block)) + continue; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || !(*Dominators)->dominates(*Save, *Successor.Block)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + } + if (!Block->successors().empty()) + continue; + if (!containsEpilogueTerminator(*Block, *Info, Target)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + RestoreBlocks.push_back(Block); + } + if (RestoreBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + MachineFrameState Planned = OriginalState; + Planned.SavePoints.push_back({Save->stableID(), {}}); + std::ranges::sort(RestoreBlocks, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + for (const MachineBasicBlock *Block : RestoreBlocks) + Planned.RestorePoints.push_back({Block->stableID(), {}}); + auto Updated = Function.frameInfo().setState(std::move(Planned)); + if (!Updated) + return std::unexpected(Updated.error()); + + MachinePassResult Result; + Result.Changed = true; + Result.Changes = {MachinePassChange::Frame}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp new file mode 100644 index 0000000..9e73cea --- /dev/null +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineSink.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &sinkContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-sink", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-sink", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected isSafeToSink(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine sink encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +struct SinkDestination final { + MachineBasicBlock *Block = nullptr; + MachineInstr *FirstUse = nullptr; +}; + +[[nodiscard]] std::optional findDestination(MachineInstr &Instruction, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return std::nullopt; + std::unordered_set Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg != nullptr && Operand.isDef()) + Definitions.insert(*Reg); + } + if (Definitions.empty()) + return std::nullopt; + + MachineBasicBlock *Destination = nullptr; + for (const Register Reg : Definitions) { + for (MachineOperand &Use : Registers.useOperands(Reg, false)) { + MachineInstr *UseInstruction = Use.parent(); + MachineBasicBlock *UseBlock = UseInstruction == nullptr ? nullptr : UseInstruction->parent(); + if (UseBlock == nullptr || UseBlock == Source) + return std::nullopt; + if (Destination == nullptr) + Destination = UseBlock; + else if (Destination != UseBlock) + return std::nullopt; + } + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source || + std::ranges::find_if(Source->successors(), [Destination](const MachineSuccessor &Successor) { + return Successor.Block == Destination; + }) == Source->successors().end()) { + return std::nullopt; + } + + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + const bool UsesDefinition = + std::ranges::any_of(Candidate->operands(), [&Definitions](const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + return Reg != nullptr && Operand.isUse() && !Operand.isDebugUse() && + Definitions.contains(*Reg); + }); + if (UsesDefinition) { + FirstUse = Candidate; + break; + } + } + return FirstUse == nullptr ? std::nullopt + : std::optional{{Destination, FirstUse}}; +} + +[[nodiscard]] std::optional +findPostRADestination(MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Source->successors().empty()) + return std::nullopt; + + std::vector Uses; + std::vector Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return std::nullopt; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return std::nullopt; + MachineBasicBlock *Destination = nullptr; + for (const MachineSuccessor &Successor : Source->successors()) { + if (Successor.Block == nullptr) + return std::nullopt; + const bool CarriesAny = std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + const bool CarriesAll = std::ranges::all_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + if (CarriesAny != CarriesAll || (CarriesAll && Destination != nullptr)) + return std::nullopt; + if (CarriesAll) + Destination = Successor.Block; + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source) { + return std::nullopt; + } + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (!std::ranges::any_of(Destination->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return std::nullopt; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return std::nullopt; + } + } + + const auto OverlapsAny = [&](PhysicalRegister Reg, std::span Values) { + return std::ranges::any_of( + Values, [&](PhysicalRegister Value) { return Target.registers().regsOverlap(Reg, Value); }); + }; + for (MachineInstr *After = Instruction.next(); After != nullptr; After = After->next()) { + for (const MachineOperand &Operand : After->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if ((Operand.isUse() || Operand.isDef()) && OverlapsAny(*Physical, Definitions)) { + return std::nullopt; + } + if (Operand.isDef() && OverlapsAny(*Physical, Uses)) + return std::nullopt; + } + } + + std::vector SeenUse(Definitions.size()); + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!SeenUse[Index] && Mask->Mask->clobbers(Definitions[Index])) + return std::nullopt; + } + if (FirstUse == nullptr && std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { + return Mask->Mask->clobbers(Reg); + })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!Target.registers().regsOverlap(*Physical, Definitions[Index])) + continue; + if (Operand.isUse() && !Operand.isDebugUse()) { + SeenUse[Index] = true; + if (FirstUse == nullptr) + FirstUse = Candidate; + } + if (Operand.isDef() && !SeenUse[Index]) + return std::nullopt; + } + if (FirstUse == nullptr && Operand.isDef() && OverlapsAny(*Physical, Uses)) { + return std::nullopt; + } + } + } + if (FirstUse == nullptr || !std::ranges::all_of(SeenUse, [](bool Value) { return Value; })) + return std::nullopt; + return SinkDestination{Destination, FirstUse}; +} + +[[nodiscard]] Expected moveBefore(MachineInstr &Instruction, + const SinkDestination &Destination) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Destination.Block == nullptr || Destination.FirstUse == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine sink has an invalid move plan"); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return Destination.Block->insertInstructionBefore(*Destination.FirstUse, Instruction); +} + +} // namespace + +const MachinePassContract &MachineSinkPass::contract() const noexcept { + return sinkContract(Stage); +} + +Expected MachineSinkPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine sink requires selected SSA MachineIR" + : "post-RA machine sink requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine sink transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine sink diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineSinkPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t Sunk = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); + Block != nullptr && Sunk < Options.MaximumSunkInstructions; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && Sunk < Options.MaximumSunkInstructions;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToSink(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + if (!*Safe) { + Instruction = Next; + continue; + } + const std::optional Destination = + Stage == MachineCodeMotionStage::SelectedSSA + ? findDestination(*Instruction, Function.registerInfo()) + : findPostRADestination(*Instruction, Function, Target); + if (!Destination || !(*Dominators)->dominates(*Block, *Destination->Block) || + (*Frequencies)->frequency(*Destination->Block) > (*Frequencies)->frequency(*Block)) { + Instruction = Next; + continue; + } + auto Moved = moveBefore(*Instruction, *Destination); + if (!Moved) + return std::unexpected(Moved.error()); + ++Sunk; + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineStackSlotColoring.cpp b/backend/lib/CodeGen/MachineStackSlotColoring.cpp new file mode 100644 index 0000000..b3b9af5 --- /dev/null +++ b/backend/lib/CodeGen/MachineStackSlotColoring.cpp @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineStackSlotColoring.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &stackSlotColoringContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-stack-slot-coloring", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands, MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct SlotInterval final { + MachineBasicBlock *Block = nullptr; + std::size_t First = 0; + std::size_t Last = 0; + bool FirstStores = false; + bool LastLoads = false; + bool IsValid = true; +}; + +struct InstructionReference final { + MachineInstr *Instruction = nullptr; + std::size_t Position = 0; + bool Loads = false; + bool Stores = false; + bool HasMemoryReference = false; + bool HasFrameOperand = false; +}; + +struct SlotUseInfo final { + FrameIndex Slot; + SlotInterval Interval; + std::vector References; +}; + +[[nodiscard]] InstructionReference &getReference(SlotUseInfo &Info, MachineInstr &Instruction, + std::size_t Position) { + const auto Existing = + std::ranges::find_if(Info.References, [&Instruction](const InstructionReference &Reference) { + return Reference.Instruction == &Instruction; + }); + if (Existing != Info.References.end()) + return *Existing; + Info.References.push_back({.Instruction = &Instruction, .Position = Position}); + return Info.References.back(); +} + +[[nodiscard]] std::unordered_map +collectSlotUses(MachineFunction &Function) { + std::unordered_map Result; + std::unordered_map> Locations; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::size_t Position = 0; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next(), ++Position) { + Locations.emplace(Instruction->stableID(), std::pair{Block, Position}); + for (const MachineOperand &Operand : Instruction->operands()) { + const FrameIndexOperand *Frame = Operand.getIf(); + if (Frame == nullptr) + continue; + SlotUseInfo &Info = + Result + .try_emplace(Frame->Index, + SlotUseInfo{.Slot = Frame->Index, .Interval = {}, .References = {}}) + .first->second; + getReference(Info, *Instruction, Position).HasFrameOperand = true; + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + if (Memory == nullptr) + continue; + const std::optional Frame = Memory->description().Base.frameIndex(); + if (!Frame) + continue; + SlotUseInfo &Info = + Result + .try_emplace(*Frame, SlotUseInfo{.Slot = *Frame, .Interval = {}, .References = {}}) + .first->second; + InstructionReference &Reference = getReference(Info, *Instruction, Position); + Reference.HasMemoryReference = true; + Reference.Loads |= Memory->isLoad(); + Reference.Stores |= Memory->isStore(); + } + } + } + + for (auto &[Slot, Info] : Result) { + (void)Slot; + if (Info.References.empty()) { + Info.Interval.IsValid = false; + continue; + } + Info.Interval.Block = Info.References.front().Instruction->parent(); + Info.Interval.First = Info.References.front().Position; + Info.Interval.Last = Info.References.back().Position; + Info.Interval.FirstStores = Info.References.front().Stores; + Info.Interval.LastLoads = Info.References.back().Loads; + for (const InstructionReference &Reference : Info.References) { + if (Reference.Instruction->parent() != Info.Interval.Block || !Reference.HasMemoryReference || + (Reference.HasFrameOperand && !Reference.HasMemoryReference)) { + Info.Interval.IsValid = false; + } + } + Info.Interval.IsValid &= + Info.Interval.Block != nullptr && Info.Interval.FirstStores && Info.Interval.LastLoads; + const StackObject *Object = Function.frameInfo().object(Info.Slot); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr) { + Info.Interval.IsValid = false; + continue; + } + for (const StackObjectLifetimeRange &Range : Description->Lifetime) { + const auto Begin = Locations.find(Range.Begin); + const auto End = Locations.find(Range.End); + if (Begin == Locations.end() || End == Locations.end() || + Begin->second.first != Info.Interval.Block || End->second.first != Info.Interval.Block || + Begin->second.second > End->second.second) { + Info.Interval.IsValid = false; + break; + } + Info.Interval.First = std::min(Info.Interval.First, Begin->second.second); + Info.Interval.Last = std::max(Info.Interval.Last, End->second.second); + } + } + return Result; +} + +[[nodiscard]] bool isEligibleSlot(const StackObjectDesc &Description) noexcept { + return Description.Kind == StackObjectKind::SpillSlot && !Description.IsDead && + !Description.IsImmutable && !Description.IsAliased && !Description.AddressTaken && + !Description.AllocaName && !Description.Offset && !Description.LocalOffset && + !Description.CalleeSavedRegister && !Description.DebugInfo && + Description.TargetFlags == 0 && Description.Size.has_value(); +} + +[[nodiscard]] bool compatibleSlots(const StackObjectDesc &Left, + const StackObjectDesc &Right) noexcept { + return isEligibleSlot(Left) && isEligibleSlot(Right) && Left.Kind == Right.Kind && + Left.Size == Right.Size && Left.Align == Right.Align && Left.Stack == Right.Stack; +} + +[[nodiscard]] bool intervalsOverlap(const SlotInterval &Left, const SlotInterval &Right) noexcept { + if (Left.Block != Right.Block) + return true; + return !(Left.Last < Right.First || Right.Last < Left.First); +} + +struct StackColor final { + FrameIndex Canonical; + const StackObjectDesc *Description = nullptr; + std::vector Intervals; +}; + +[[nodiscard]] Expected rewriteSlot(MachineFunction &Function, FrameIndex OldSlot, + FrameIndex NewSlot) { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const FrameIndexOperand *Frame = + Operand == nullptr ? nullptr : Operand->getIf(); + if (Frame == nullptr || Frame->Index != OldSlot) + continue; + auto Rewritten = Operand->setFrameIndex(NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + for (std::size_t Index = 0; Index < Instruction->memOperands().size(); ++Index) { + const MachineMemOperand *Memory = Instruction->memOperands()[Index]; + if (Memory == nullptr || Memory->description().Base.frameIndex() != OldSlot) + continue; + MachineMemOperandDesc Description = Memory->description(); + Description.Base = MemoryBase::frameIndex(NewSlot); + auto Replacement = Function.createMemOperand(std::move(Description)); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Replaced = Instruction->replaceMemOperand(Index, **Replacement); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineStackSlotColoringPass::contract() const noexcept { + return stackSlotColoringContract(); +} + +Expected MachineStackSlotColoringPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "stack-slot coloring requires allocated PHI-free MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "stack-slot coloring must run before frame layout is frozen"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "stack-slot coloring transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "stack-slot coloring diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineStackSlotColoringPass::runImplementation(MachineFunction &Function) const { + std::unordered_map Uses = collectSlotUses(Function); + std::vector Colors; + std::vector> Rewrites; + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + const auto Use = Uses.find(Object.index()); + if (Description == nullptr || Use == Uses.end() || !Use->second.Interval.IsValid || + !isEligibleSlot(*Description)) { + continue; + } + auto Color = std::ranges::find_if(Colors, [&](const StackColor &Candidate) { + return Candidate.Description != nullptr && + compatibleSlots(*Candidate.Description, *Description) && + std::ranges::none_of(Candidate.Intervals, [&](const SlotInterval &Interval) { + return intervalsOverlap(Interval, Use->second.Interval); + }); + }); + if (Color == Colors.end()) { + Colors.push_back({Object.index(), Description, {Use->second.Interval}}); + continue; + } + if (Rewrites.size() == Options.MaximumColoredSlots) + break; + Rewrites.emplace_back(Object.index(), Color->Canonical); + Color->Intervals.push_back(Use->second.Interval); + } + + for (const auto &[OldSlot, NewSlot] : Rewrites) { + const StackObject *OldObject = Function.frameInfo().object(OldSlot); + const StackObject *NewObject = Function.frameInfo().object(NewSlot); + if (OldObject == nullptr || NewObject == nullptr || OldObject->description() == nullptr || + NewObject->description() == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "stack-slot coloring lost a planned stack object"); + } + auto Rewritten = rewriteSlot(Function, OldSlot, NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + StackObjectDesc Combined = *NewObject->description(); + Combined.Lifetime.insert(Combined.Lifetime.end(), OldObject->description()->Lifetime.begin(), + OldObject->description()->Lifetime.end()); + auto Updated = Function.frameInfo().updateObject(NewSlot, std::move(Combined)); + if (!Updated) + return std::unexpected(Updated.error()); + auto Dead = Function.frameInfo().markObjectDead(OldSlot); + if (!Dead) + return std::unexpected(Dead.error()); + } + + MachinePassResult Result; + Result.Changed = !Rewrites.empty(); + if (Result.Changed) { + Result.Changes = {MachinePassChange::Operands, MachinePassChange::Frame}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailDuplication.cpp b/backend/lib/CodeGen/MachineTailDuplication.cpp new file mode 100644 index 0000000..a130680 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailDuplication.cpp @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailDuplication.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailDuplicationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-duplication", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] Expected isDuplicableInstruction(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::NotDuplicable) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Instruction.flags().has(MachineInstrFlag::NoMerge) || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc) || + hasProtectedAttachment(Instruction)) { + return false; + } + return std::ranges::none_of(Instruction.operands(), [](const MachineOperand &Operand) { + return Operand.kind() == MachineOperand::Kind::BasicBlock || + Operand.kind() == MachineOperand::Kind::DebugInstrRef; + }); +} + +[[nodiscard]] Expected +cloneInstructionBefore(MachineInstr &Source, MachineBasicBlock &Destination, MachineInstr *Before) { + MachineFunction &Function = Destination.function(); + auto Created = Function.createInstruction(Source.opcode()); + if (!Created) + return std::unexpected(Created.error()); + MachineInstr *Clone = *Created; + Clone->reserveOperands(Source.operandCount()); + for (const MachineOperand &Operand : Source.operands()) { + auto Added = Clone->addOperand(Operand.cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + const std::optional Tied = + Operand == nullptr ? std::nullopt : Operand->tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = Operand->isDef() ? Index : *Tied; + const std::size_t UseIndex = Operand->isDef() ? *Tied : Index; + auto TiedResult = Clone->tieOperands(DefIndex, UseIndex); + if (!TiedResult) + return std::unexpected(TiedResult.error()); + } + for (const MachineMemOperand *Memory : Source.memOperands()) { + if (Memory == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered a null memory operand"); + auto Added = Clone->addMemOperand(*Memory); + if (!Added) + return std::unexpected(Added.error()); + } + Clone->setFlags(Source.flags()); + if (Source.sourceLocation()) + Clone->setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Clone->setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Clone->setOrigin(*Source.origin()); + Expected Inserted = Before == nullptr + ? Destination.appendInstruction(*Clone) + : Destination.insertInstructionBefore(*Before, *Clone); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Clone; +} + +struct TailCandidate final { + MachineBasicBlock *Successor = nullptr; + MachineInstr *Branch = nullptr; + std::vector Body; +}; + +[[nodiscard]] Expected> +analyzeTail(MachineBasicBlock &Tail, MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info, std::size_t MaximumInstructions) { + if (&Tail == Function.firstBlock() || Tail.predecessors().size() < 2 || + Tail.successors().size() != 1 || Tail.successors().front().Block == nullptr || + hasProtectedIdentity(Tail, Function)) { + return std::nullopt; + } + auto Branch = Info.analyzeBranch(Tail, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + MachineBasicBlock *Successor = Tail.successors().front().Block; + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr || + Branch->Unconditional == nullptr || Branch->TrueTarget != Successor || + Branch->Unconditional != Tail.lastInstruction() || + hasProtectedAttachment(*Branch->Unconditional)) { + return std::nullopt; + } + + TailCandidate Candidate{.Successor = Successor, .Branch = Branch->Unconditional, .Body = {}}; + for (MachineInstr *Instruction = Tail.firstInstruction(); Instruction != Candidate.Branch; + Instruction = Instruction->next()) { + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail branch is not reachable from its block body"); + auto Duplicable = isDuplicableInstruction(*Instruction, Target); + if (!Duplicable) + return std::unexpected(Duplicable.error()); + if (!*Duplicable) + return std::nullopt; + Candidate.Body.push_back(Instruction); + if (Candidate.Body.size() > MaximumInstructions) + return std::nullopt; + } + return Candidate; +} + +[[nodiscard]] Expected +duplicateTailIntoPredecessor(const TailCandidate &Candidate, MachineBasicBlock &Tail, + MachineBasicBlock &Predecessor, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + if (Predecessor.successors().size() != 1 || Predecessor.successors().front().Block != &Tail) { + return false; + } + auto Branch = Info.analyzeBranch(Predecessor, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr) + return false; + MachineInstr *OldBranch = Branch->Unconditional; + if (OldBranch != nullptr) { + if (Branch->TrueTarget != &Tail || OldBranch != Predecessor.lastInstruction() || + hasProtectedAttachment(*OldBranch)) { + return false; + } + } else if (Predecessor.next() != &Tail) { + return false; + } + + for (MachineInstr *Instruction : Candidate.Body) { + auto Cloned = cloneInstructionBefore(*Instruction, Predecessor, OldBranch); + if (!Cloned) + return std::unexpected(Cloned.error()); + } + if (OldBranch != nullptr) { + auto Removed = Info.removeBranch(*OldBranch, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + } + auto NewBranch = + Info.insertUnconditionalBranch(Predecessor, *Candidate.Successor, Function, Target); + if (!NewBranch) + return std::unexpected(NewBranch.error()); + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Tail, *Candidate.Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailDuplicationPass::contract() const noexcept { + return tailDuplicationContract(); +} + +Expected MachineTailDuplicationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "tail duplication transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail duplication diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineTailDuplicationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t DuplicatedInstructions = 0; + for (MachineBasicBlock *Tail = Function.firstBlock(); Tail != nullptr;) { + MachineBasicBlock *Next = Tail->next(); + auto Candidate = analyzeTail(*Tail, Function, Target, *Info, Options.MaximumTailInstructions); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!Candidate->has_value()) { + Tail = Next; + continue; + } + const std::size_t Cost = (*Candidate)->Body.size() + 1U; + const std::vector Predecessors(Tail->predecessors().begin(), + Tail->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || + DuplicatedInstructions > Options.MaximumDuplicatedInstructions || + Cost > Options.MaximumDuplicatedInstructions - DuplicatedInstructions) { + break; + } + auto Duplicated = + duplicateTailIntoPredecessor(**Candidate, *Tail, *Predecessor, Function, Target, *Info); + if (!Duplicated) + return std::unexpected(Duplicated.error()); + if (!*Duplicated) + continue; + DuplicatedInstructions += Cost; + Result.Changed = true; + } + if (Tail->predecessors().empty()) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Tail); + if (!Deleted) + return std::unexpected(Deleted.error()); + Result.Changed = true; + } + Tail = Next; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailMerge.cpp b/backend/lib/CodeGen/MachineTailMerge.cpp new file mode 100644 index 0000000..b754978 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailMerge.cpp @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailMerge.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailMergeContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-merge", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +template +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf(); + const ValueT *RightValue = Right.getIf(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +template +[[nodiscard]] bool maskEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const MaskOperandT *LeftValue = Left.getIf(); + const MaskOperandT *RightValue = Right.getIf(); + if (LeftValue == nullptr || RightValue == nullptr) + return false; + if (!LeftValue->Mask || !RightValue->Mask) + return LeftValue->Mask == RightValue->Mask; + return *LeftValue->Mask == *RightValue->Mask; +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() || + Left.tiedOperandIndex() != Right.tiedOperandIndex()) { + return false; + } + if (Left.kind() == MachineOperand::Kind::Register) { + const Register *LeftReg = Left.getIf(); + const Register *RightReg = Right.getIf(); + if (LeftReg == nullptr || RightReg == nullptr || *LeftReg != *RightReg || + Left.subReg() != Right.subReg()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags; + } + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return false; + case MachineOperand::Kind::Immediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals(Left, Right); + case MachineOperand::Kind::RegisterMask: + return maskEquals(Left, Right); + case MachineOperand::Kind::LiveOutRegisterMask: + return maskEquals(Left, Right); + case MachineOperand::Kind::Metadata: + return payloadEquals(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] Expected instructionEquals(const MachineInstr &Left, const MachineInstr &Right, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Left.opcode()); + if (Descriptor == nullptr || Target.instructionDescriptor(Right.opcode()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge encountered an unknown instruction"); + } + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size() || + Left.sourceLocation() != Right.sourceLocation() || + Left.debugLocation() != Right.debugLocation() || Left.origin() != Right.origin() || + hasProtectedAttachment(Left) || hasProtectedAttachment(Right) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Convergent) || + Descriptor->Flags.has(InstructionFlag::NotDuplicable) || + Left.flags().has(MachineInstrFlag::NoMerge) || + Left.flags().has(MachineInstrFlag::BundledWithPred) || + Left.flags().has(MachineInstrFlag::BundledWithSucc)) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + !operandEquals(*LeftOperand, *RightOperand)) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || + LeftMemory->description() != RightMemory->description()) { + return false; + } + } + return true; +} + +[[nodiscard]] Expected blockEquals(const MachineBasicBlock &Left, + const MachineBasicBlock &Right, + const MachineFunction &Function, + const TargetDescription &Target) { + if (&Left == &Right || &Left == Function.firstBlock() || &Right == Function.firstBlock() || + hasProtectedIdentity(Left, Function) || hasProtectedIdentity(Right, Function) || + Left.attributes().alignment() != Right.attributes().alignment() || + Left.attributes().sectionID() != Right.attributes().sectionID() || + Left.attributes().callFrameSize() != Right.attributes().callFrameSize() || + !std::ranges::equal(Left.liveIns(), Right.liveIns()) || + !std::ranges::equal(Left.successors(), Right.successors()) || + Left.instructionCount() != Right.instructionCount()) { + return false; + } + const MachineInstr *LeftInstruction = Left.firstInstruction(); + const MachineInstr *RightInstruction = Right.firstInstruction(); + while (LeftInstruction != nullptr && RightInstruction != nullptr) { + auto Equal = instructionEquals(*LeftInstruction, *RightInstruction, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + return false; + LeftInstruction = LeftInstruction->next(); + RightInstruction = RightInstruction->next(); + } + return LeftInstruction == nullptr && RightInstruction == nullptr; +} + +[[nodiscard]] Expected canRedirectPredecessor(MachineBasicBlock &Predecessor, + MachineBasicBlock &Duplicate, + MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + return false; + if ((Analysis->Conditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Conditional)) || + (Analysis->Unconditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Unconditional))) { + return false; + } + if (Analysis->TrueTarget == &Duplicate || Analysis->FalseTarget == &Duplicate) + return true; + return Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate; +} + +[[nodiscard]] Expected +redirectPredecessor(MachineBasicBlock &Predecessor, MachineBasicBlock &Duplicate, + MachineBasicBlock &Canonical, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + auto Rewritten = Info.rewriteBranchTarget(Predecessor, Duplicate, Canonical, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + const bool IsImplicitEdge = + (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate) || + (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == &Duplicate); + if (!IsImplicitEdge) + return false; + auto Inserted = Info.insertUnconditionalBranch(Predecessor, Canonical, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Duplicate, Canonical); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailMergePass::contract() const noexcept { + return tailMergeContract(); +} + +Expected MachineTailMergePass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "tail merge transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail merge diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineTailMergePass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t MergedBlocks = 0; + bool LocalChange = true; + while (LocalChange && MergedBlocks < Options.MaximumMergedBlocks) { + LocalChange = false; + std::vector Blocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Blocks.push_back(Block); + } + for (std::size_t DuplicateIndex = 1; DuplicateIndex < Blocks.size(); ++DuplicateIndex) { + MachineBasicBlock *Duplicate = Blocks[DuplicateIndex]; + if (Duplicate == nullptr || !Duplicate->isLinked()) + continue; + for (std::size_t CanonicalIndex = 1; CanonicalIndex < DuplicateIndex; ++CanonicalIndex) { + MachineBasicBlock *Canonical = Blocks[CanonicalIndex]; + if (Canonical == nullptr || !Canonical->isLinked()) + continue; + auto Equal = blockEquals(*Canonical, *Duplicate, Function, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + continue; + + const std::vector Predecessors(Duplicate->predecessors().begin(), + Duplicate->predecessors().end()); + bool RedirectedAll = true; + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || Predecessor == Canonical || Predecessor == Duplicate) { + RedirectedAll = false; + break; + } + auto CanRedirect = + canRedirectPredecessor(*Predecessor, *Duplicate, Function, Target, *Info); + if (!CanRedirect) + return std::unexpected(CanRedirect.error()); + if (!*CanRedirect) { + RedirectedAll = false; + break; + } + } + if (!RedirectedAll) + continue; + for (MachineBasicBlock *Predecessor : Predecessors) { + auto Redirected = + redirectPredecessor(*Predecessor, *Duplicate, *Canonical, Function, Target, *Info); + if (!Redirected) + return std::unexpected(Redirected.error()); + if (!*Redirected) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail merge predecessor changed after redirect preflight"); + } + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Duplicate); + if (!Deleted) + return std::unexpected(Deleted.error()); + ++MergedBlocks; + Result.Changed = true; + LocalChange = true; + break; + } + if (LocalChange) + break; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PostRAScheduler.cpp b/backend/lib/CodeGen/PostRAScheduler.cpp new file mode 100644 index 0000000..9aff5ac --- /dev/null +++ b/backend/lib/CodeGen/PostRAScheduler.cpp @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PostRAScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &postRASchedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct PhysicalUnitAccess final { + RegisterUnitID Unit; + LaneMask Lanes; +}; + +struct OperandFlagUpdate final { + MachineOperand *Operand = nullptr; + RegisterOperandFlags Flags; +}; + +[[nodiscard]] LaneMask normalizeUnitLanes(LaneMask UnitViewLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +[[nodiscard]] Expected> +physicalOperandUnits(const MachineOperand &Operand, const TargetDescription &Target) { + const Register *Reg = Operand.getIf(); + std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a virtual register"); + } + if (Operand.subReg().value() != 0) { + Physical = Target.registers().subRegister(*Physical, Operand.subReg()); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered an invalid subregister"); + } + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr || Description->Units.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a unitless register"); + } + std::vector Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) { + if (Unit.Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered empty register-unit lanes"); + } + Result.push_back({Unit.Unit, normalizeUnitLanes(Unit.Lanes)}); + } + return Result; +} + +void addPhysicalRegister(RegisterUnitLaneSet &Live, const PhysicalRegisterDesc &Register) { + for (const RegisterUnitLane &Unit : Register.Units) + Live.add(Unit.Unit, normalizeUnitLanes(Unit.Lanes)); +} + +[[nodiscard]] bool intersects(const RegisterUnitLaneSet &Live, + const std::vector &Units) noexcept { + return std::ranges::any_of(Units, [&Live](const PhysicalUnitAccess &Unit) { + return Live.intersects(Unit.Unit, Unit.Lanes); + }); +} + +void addUnits(RegisterUnitLaneSet &Live, const std::vector &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.add(Unit.Unit, Unit.Lanes); +} + +void removeUnits(RegisterUnitLaneSet &Live, const std::vector &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.remove(Unit.Unit, Unit.Lanes); +} + +[[nodiscard]] OperandFlagUpdate &getFlagUpdate(std::vector &Updates, + MachineOperand &Operand) { + const auto Position = std::ranges::find_if( + Updates, [&Operand](const OperandFlagUpdate &Update) { return Update.Operand == &Operand; }); + if (Position != Updates.end()) + return *Position; + Updates.push_back({&Operand, Operand.registerFlags()}); + return Updates.back(); +} + +[[nodiscard]] Expected> +buildBlockLivenessRepairPlan(MachineBasicBlock &Block, const TargetDescription &Target, + const MachineBlockLiveness &Boundary) { + RegisterUnitLaneSet Live = Boundary.PhysicalLiveOut; + std::vector Updates; + for (MachineInstr *Instruction = Block.lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *Mask = Operand.getIf(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null live-out mask"); + } + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->preserves(Register.Reg)) + addPhysicalRegister(Live, Register); + } + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Dead); + if (!Operand->isDef()) + continue; + const Register *Reg = Operand->getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Dead); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + removeUnits(Live, *Units); + continue; + } + const auto *Mask = Operand.getIf(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null register mask"); + } + RegisterUnitLaneSet Clobbers; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->clobbers(Register.Reg)) + addPhysicalRegister(Clobbers, Register); + } + Live.subtract(Clobbers); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Kill); + if (!Operand->isUse() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand->getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Kill); + addUnits(Live, *Units); + } + } + if (Live != Boundary.PhysicalLiveIn) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA local liveness repair changed the block boundary"); + } + return Updates; +} + +} // namespace + +Expected repairPostRAMachineBlockLiveness(std::span Blocks, + const TargetDescription &Target, + const LiveVariablesAnalysis::Result &Liveness) { + std::vector Updates; + for (MachineBasicBlock *Block : Blocks) { + const MachineBlockLiveness *Boundary = Liveness.block(*Block); + if (Boundary == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA scheduler lost a block liveness boundary"); + } + auto BlockUpdates = buildBlockLivenessRepairPlan(*Block, Target, *Boundary); + if (!BlockUpdates) + return std::unexpected(BlockUpdates.error()); + Updates.insert(Updates.end(), BlockUpdates->begin(), BlockUpdates->end()); + } + bool Changed = false; + for (const OperandFlagUpdate &Update : Updates) { + Changed |= Update.Operand->registerFlags() != Update.Flags; + auto Applied = Update.Operand->setRegisterFlags(Update.Flags); + if (!Applied) + return std::unexpected(Applied.error()); + } + return Changed; +} + +const MachinePassContract &PostRASchedulerPass::contract() const noexcept { + return postRASchedulerContract(); +} + +Expected PostRASchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduler requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "post-RA scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialRun = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PostRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialRepair = + repairPostRAMachineBlockLiveness(TrialRun->ChangedBlocks, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PostRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "post-RA scheduler diverged after a successful transaction preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(Run->ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = Run->Changed || *Repaired; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterPressure.cpp b/backend/lib/CodeGen/RegisterPressure.cpp new file mode 100644 index 0000000..523a443 --- /dev/null +++ b/backend/lib/CodeGen/RegisterPressure.cpp @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterPressure.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +using EntityID = std::size_t; + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +template void sortAndUnique(std::vector &Values) { + std::ranges::sort(Values); + const auto [FirstDuplicate, End] = std::ranges::unique(Values); + Values.erase(FirstDuplicate, End); +} + +} // namespace + +namespace detail { + +struct RegisterPressureBuilder final { + RegisterPressureTracker Result; + const ScheduleDAG *DAG = nullptr; + const MachineFunction *Function = nullptr; + const TargetDescription *Target = nullptr; + const TargetSchedulingModel *Model = nullptr; + ScheduleDAGMode Mode = ScheduleDAGMode::PreRegisterAllocation; + std::unordered_map CurrentVirtualEntities; + std::unordered_map CurrentPhysicalEntities; + std::unordered_map> PhysicalSets; + + [[nodiscard]] Expected initializePressureSets() { + Result.Pressure.reserve(Model->pressureSets().size()); + for (const RegisterPressureSetDesc &Set : Model->pressureSets()) { + Result.PressureIndexes.emplace(Set.ID, Result.Pressure.size()); + Result.Pressure.push_back({Set.ID, 0, 0, Set.Limit}); + for (const RegisterClassID ClassID : Set.RegisterClasses) { + const RegisterClassDesc *Class = Target->registers().registerClass(ClassID); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure set references an unknown target register class"); + } + for (const PhysicalRegister Physical : Class->Registers) { + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "register class references an unknown physical register"); + } + for (const RegisterUnitLane &Unit : Description->Units) + PhysicalSets[Unit.Unit].push_back(Set.ID); + } + } + } + for (auto &[Unit, Sets] : PhysicalSets) { + (void)Unit; + sortAndUnique(Sets); + } + return {}; + } + + [[nodiscard]] EntityID createEntity(std::vector Sets) { + sortAndUnique(Sets); + const EntityID ID = Result.Entities.size(); + Result.Entities.push_back({std::move(Sets), 0, false, false, false, false}); + return ID; + } + + [[nodiscard]] Expected> + virtualPressureSets(Register Reg) const { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pressure tracking encountered a virtual register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().RegClass) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA pressure tracking requires constrained virtual registers"); + } + const std::span Sets = + Model->pressureSetsForClass(*Info->get().RegClass); + return std::vector{Sets.begin(), Sets.end()}; + } + + [[nodiscard]] Expected boundaryVirtualEntity(Register Reg) { + if (const auto Position = CurrentVirtualEntities.find(Reg); + Position != CurrentVirtualEntities.end()) { + return Position->second; + } + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.emplace(Reg, ID); + return ID; + } + + [[nodiscard]] Expected defineVirtualEntity(Register Reg) { + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.insert_or_assign(Reg, ID); + return ID; + } + + [[nodiscard]] EntityID boundaryPhysicalEntity(RegisterUnitID Unit) { + if (const auto Position = CurrentPhysicalEntities.find(Unit); + Position != CurrentPhysicalEntities.end()) { + return Position->second; + } + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector{} : Sets->second); + CurrentPhysicalEntities.emplace(Unit, ID); + return ID; + } + + [[nodiscard]] EntityID definePhysicalEntity(RegisterUnitID Unit) { + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector{} : Sets->second); + CurrentPhysicalEntities.insert_or_assign(Unit, ID); + return ID; + } + + [[nodiscard]] Expected> physicalUnits(Register Reg, + SubRegIndex SubReg) const { + std::optional Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical register"); + } + if (SubReg.value() != 0) { + Physical = Target->registers().subRegister(*Physical, SubReg); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical subregister"); + } + } + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(*Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an unknown physical register"); + } + std::vector Units; + Units.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Units.push_back(Unit.Unit); + sortAndUnique(Units); + return Units; + } + + [[nodiscard]] Expected appendRegisterUse(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = boundaryVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Uses.push_back(*Entity); + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit)); + return {}; + } + + [[nodiscard]] Expected appendRegisterDefinition(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = defineVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Definitions.push_back(*Entity); + Result.Entities[*Entity].HasDefinition = true; + Result.Entities[*Entity].DefinitionPending = true; + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + const EntityID Entity = definePhysicalEntity(Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + return {}; + } + + [[nodiscard]] Expected appendMaskDefinition(RegisterPressureTracker::Event &Event, + const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) { + const EntityID Entity = definePhysicalEntity(Unit.Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + } + return {}; + } + + void appendLiveOutMaskUse(RegisterPressureTracker::Event &Event, const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit.Unit)); + } + } + + [[nodiscard]] Expected + buildEvent(const MachineInstr &Instruction) { + RegisterPressureTracker::Event Event; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null live-out register mask"); + } + appendLiveOutMaskUse(Event, *Mask->Mask); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + if (Operand.isUse()) { + auto Added = appendRegisterUse(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null register mask"); + } + auto Added = appendMaskDefinition(Event, *Mask->Mask); + if (!Added) + return std::unexpected(Added.error()); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse()) + continue; + if (Operand.isDef()) { + auto Added = appendRegisterDefinition(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + sortAndUnique(Event.Uses); + sortAndUnique(Event.Definitions); + return Event; + } + + [[nodiscard]] Expected buildEffects() { + Result.Effects.resize(DAG->units().size()); + Result.Scheduled.assign(DAG->units().size(), false); + for (const ScheduleUnit &Unit : DAG->units()) { + RegisterPressureTracker::UnitEffects &Effects = Result.Effects[Unit.ID]; + Effects.Events.reserve(Unit.Instructions.size()); + for (const MachineInstr *Instruction : Unit.Instructions) { + auto Event = buildEvent(*Instruction); + if (!Event) + return std::unexpected(Event.error()); + for (const EntityID Entity : Event->Uses) + ++Result.Entities[Entity].RemainingUses; + Effects.Events.push_back(std::move(*Event)); + } + } + return {}; + } + + void markEntityLiveOut(EntityID Entity) { + RegisterPressureTracker::Entity &Value = Result.Entities[Entity]; + if (Value.LiveOut) + return; + Value.LiveOut = true; + ++Value.RemainingUses; + } + + [[nodiscard]] Expected markBlockLiveOut(const MachineBlockLiveness &Liveness) { + for (const auto &[Reg, Lanes] : Liveness.VirtualLiveOut.entries()) { + (void)Lanes; + auto Entity = boundaryVirtualEntity(Reg); + if (!Entity) + return std::unexpected(Entity.error()); + markEntityLiveOut(*Entity); + } + for (const auto &[Unit, Lanes] : Liveness.PhysicalLiveOut.entries()) { + (void)Lanes; + markEntityLiveOut(boundaryPhysicalEntity(Unit)); + } + return {}; + } + + [[nodiscard]] Expected markSuffixUses() { + if (DAG->units().empty()) + return {}; + const MachineInstr *Instruction = DAG->units().back().Instructions.back()->next(); + std::unordered_set KilledVirtual; + std::unordered_set KilledPhysical; + for (; Instruction != nullptr; Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure suffix scan encountered a null register mask"); + } + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask->Mask->clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + KilledPhysical.insert(Unit.Unit); + } + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isVirtual()) { + if (Operand.isUse() && !KilledVirtual.contains(*Reg)) { + const auto Position = CurrentVirtualEntities.find(*Reg); + if (Position != CurrentVirtualEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledVirtual.insert(*Reg); + continue; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + if (Operand.isUse() && !KilledPhysical.contains(Unit)) { + const auto Position = CurrentPhysicalEntities.find(Unit); + if (Position != CurrentPhysicalEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledPhysical.insert(Unit); + } + } + } + return {}; + } + + [[nodiscard]] Expected initializeLivePressure() { + for (EntityID ID = 0; ID < Result.Entities.size(); ++ID) { + RegisterPressureTracker::Entity &Entity = Result.Entities[ID]; + const bool InitiallyLive = Result.Direction == RegisterPressureDirection::TopDown + ? !Entity.HasDefinition && Entity.RemainingUses != 0 + : Entity.LiveOut; + if (!InitiallyLive) + continue; + auto Added = Result.makeLive(ID); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; + } +}; + +} // namespace detail + +Expected +RegisterPressureTracker::create(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + RegisterPressureDirection Direction) { + detail::RegisterPressureBuilder Builder; + Builder.DAG = &DAG; + Builder.Function = &Function; + Builder.Target = &Target; + Builder.Model = &Model; + Builder.Mode = Mode; + Builder.Result.Direction = Direction; + auto Sets = Builder.initializePressureSets(); + if (!Sets) + return std::unexpected(Sets.error()); + auto Effects = Builder.buildEffects(); + if (!Effects) + return std::unexpected(Effects.error()); + auto Suffix = Builder.markSuffixUses(); + if (!Suffix) + return std::unexpected(Suffix.error()); + auto LiveOut = Builder.markBlockLiveOut(Liveness); + if (!LiveOut) + return std::unexpected(LiveOut.error()); + auto Initial = Builder.initializeLivePressure(); + if (!Initial) + return std::unexpected(Initial.error()); + return std::move(Builder.Result); +} + +Expected RegisterPressureTracker::evaluate(std::size_t UnitID) const { + RegisterPressureTracker Trial = *this; + const RegisterPressureCost Before = cost(); + auto ScheduledUnit = Trial.schedule(UnitID); + if (!ScheduledUnit) + return std::unexpected(ScheduledUnit.error()); + RegisterPressureCost After = Trial.cost(); + After.PeakIncrease = + After.PeakExcess > Before.PeakExcess ? After.PeakExcess - Before.PeakExcess : 0; + return After; +} + +Expected RegisterPressureTracker::schedule(std::size_t UnitID) { + if (UnitID >= Effects.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "pressure tracker unit ID is out of range"); + if (Scheduled[UnitID]) + return makeUnexpected(ErrorCode::InvalidArgument, "schedule unit was already issued"); + RegisterPressureTracker Trial = *this; + if (Trial.Direction == RegisterPressureDirection::TopDown) { + for (const Event &Value : Trial.Effects[UnitID].Events) { + auto Applied = Trial.applyEvent(Value); + if (!Applied) + return std::unexpected(Applied.error()); + } + } else { + for (auto Position = Trial.Effects[UnitID].Events.rbegin(); + Position != Trial.Effects[UnitID].Events.rend(); ++Position) { + auto Applied = Trial.applyEvent(*Position); + if (!Applied) + return std::unexpected(Applied.error()); + } + } + Trial.Scheduled[UnitID] = true; + *this = std::move(Trial); + return {}; +} + +RegisterPressureCost RegisterPressureTracker::cost() const noexcept { + RegisterPressureCost Result; + for (const RegisterPressureValue &Value : Pressure) { + Result.CurrentPressure = saturatingAdd(Result.CurrentPressure, Value.Current); + if (Value.Current > Value.Limit) { + Result.CurrentExcess = saturatingAdd(Result.CurrentExcess, Value.Current - Value.Limit); + } + if (Value.Maximum > Value.Limit) + Result.PeakExcess = saturatingAdd(Result.PeakExcess, Value.Maximum - Value.Limit); + } + return Result; +} + +Expected RegisterPressureTracker::applyEvent(const Event &Value) { + if (Direction == RegisterPressureDirection::BottomUp) { + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size() || !Entities[Entity].DefinitionPending) { + return makeUnexpected(ErrorCode::VerificationError, + "bottom-up scheduler crossed an unavailable definition"); + } + Entities[Entity].DefinitionPending = false; + if (Entities[Entity].Live) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown use entity"); + } + if (!Entities[Entity].Live) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size() || !Entities[Entity].Live || + Entities[Entity].RemainingUses == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler selected a use before its pressure value became live"); + } + --Entities[Entity].RemainingUses; + if (Entities[Entity].RemainingUses == 0) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown definition entity"); + } + if (Entities[Entity].Live) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler redefined a pressure value while it was live"); + } + if (Entities[Entity].RemainingUses != 0) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; +} + +Expected RegisterPressureTracker::makeLive(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became live"); + Entity &Value = Entities[EntityIDValue]; + if (Value.Live) + return {}; + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure entity references an unknown pressure set"); + } + RegisterPressureValue &SetPressure = Pressure[Position->second]; + if (SetPressure.Current == std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, "register pressure counter overflowed"); + } + ++SetPressure.Current; + SetPressure.Maximum = std::max(SetPressure.Maximum, SetPressure.Current); + } + Value.Live = true; + return {}; +} + +Expected RegisterPressureTracker::makeDead(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became dead"); + Entity &Value = Entities[EntityIDValue]; + if (!Value.Live) + return makeUnexpected(ErrorCode::InternalError, "dead pressure entity was not live"); + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end() || Pressure[Position->second].Current == 0) { + return makeUnexpected(ErrorCode::InternalError, "register pressure counter underflowed"); + } + --Pressure[Position->second].Current; + } + Value.Live = false; + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleDAG.cpp b/backend/lib/CodeGen/ScheduleDAG.cpp new file mode 100644 index 0000000..8696f2e --- /dev/null +++ b/backend/lib/CodeGen/ScheduleDAG.cpp @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleDAG.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +struct RegisterDependencyKey final { + bool IsUnit = false; + std::uint32_t Value = 0; + + friend constexpr bool operator==(RegisterDependencyKey, RegisterDependencyKey) noexcept = default; +}; + +struct RegisterDependencyKeyHash final { + [[nodiscard]] std::size_t operator()(RegisterDependencyKey Key) const noexcept { + const std::uint64_t Value = (static_cast(Key.IsUnit) << 32U) | Key.Value; + return std::hash{}(Value); + } +}; + +struct RegisterDependencyState final { + struct Definition final { + std::size_t Unit = 0; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; + }; + + std::optional LastDefinition; + std::vector LastUses; +}; + +struct RegisterAccess final { + RegisterDependencyKey Key; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; +}; + +[[nodiscard]] bool keyLess(RegisterDependencyKey Left, RegisterDependencyKey Right) noexcept { + return Left.IsUnit == Right.IsUnit ? Left.Value < Right.Value : Left.IsUnit < Right.IsUnit; +} + +[[nodiscard]] std::uint32_t saturatingAdd(std::uint32_t Left, std::uint32_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] Expected> +operandKeys(const MachineOperand &Operand, const MachineFunction &Function, + const TargetDescription &Target, ScheduleDAGMode Mode) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return std::vector{}; + if (Reg->isVirtual()) { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduling region contains a virtual register"); + } + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown virtual register"); + } + return std::vector{{false, Reg->opaqueValue()}}; + } + + std::optional Physical = Reg->physicalRegister(); + if (!Physical) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical register"); + if (Operand.subReg().value() != 0) { + const std::optional SubReg = + Target.registers().subRegister(*Physical, Operand.subReg()); + if (!SubReg) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical subregister"); + } + Physical = SubReg; + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown physical register"); + std::vector Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Result.push_back({true, Unit.Unit.value()}); + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] Expected> +maskKeys(const RegisterMask &Mask, const TargetDescription &Target) { + std::vector Result; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Result.push_back({true, Unit.Unit.value()}); + } + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] bool mayAlias(const MachineInstr &Earlier, const MachineInstr &Later, + const MachineAliasAnalysis::Result &Aliases) noexcept { + if (Earlier.memOperands().empty() || Later.memOperands().empty()) + return true; + for (const MachineMemOperand *Left : Earlier.memOperands()) { + if (Left == nullptr) + return true; + for (const MachineMemOperand *Right : Later.memOperands()) { + if (Right == nullptr || Aliases.alias(*Left, *Right) != MachineAliasResult::NoAlias) + return true; + } + } + return false; +} + +[[nodiscard]] bool hasOrderedMemory(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isAtomic() || Memory->isVolatile(); + }); +} + +[[nodiscard]] bool hasLoad(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isLoad(); + }); +} + +[[nodiscard]] bool hasStore(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isStore(); + }); +} + +} // namespace + +Expected ScheduleDAG::build(MachineBasicBlock &Block, + std::span Instructions, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode) { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule DAG block belongs to another function"); + ScheduleDAG Result(Block); + auto Units = Result.buildUnits(Instructions, Target, Model); + if (!Units) + return std::unexpected(Units.error()); + auto Registers = Result.buildRegisterDependencies(Function, Target, Model, Mode); + if (!Registers) + return std::unexpected(Registers.error()); + auto Memory = Result.buildMemoryDependencies(Target); + if (!Memory) + return std::unexpected(Memory.error()); + auto Order = Result.buildOrderDependencies(Target); + if (!Order) + return std::unexpected(Order.error()); + auto Mutated = Model.hooks().mutateScheduleDAG(Result, Function, Target, Mode); + if (!Mutated) + return std::unexpected(Mutated.error()); + Result.computeCriticalPaths(); + return Result; +} + +Expected ScheduleDAG::buildUnits(std::span Instructions, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + MachineInstr *ExpectedInstruction = Instructions.empty() ? nullptr : Instructions.front(); + for (std::size_t Index = 0; Index < Instructions.size();) { + MachineInstr *Instruction = Instructions[Index]; + if (Instruction == nullptr || Instruction->parent() != Block) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region contains a detached or foreign instruction"); + if (Instruction != ExpectedInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region is not a contiguous instruction sequence"); + } + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region begins in the middle of an instruction bundle"); + } + + ScheduleUnit Unit; + Unit.ID = Units.size(); + Unit.StableID = Instruction->stableID(); + std::unordered_map ResourceCycles; + for (;;) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const SchedulingClassModelDesc *Class = Model.schedulingClass(Descriptor->SchedulingClass); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction references an unknown scheduling class"); + } + Unit.MicroOps = saturatingAdd(Unit.MicroOps, Class->MicroOps); + Unit.Latency = std::max(Unit.Latency, Class->Latency); + for (const SchedulingResourceUse &Use : Class->Resources) { + ResourceCycles[Use.Resource] = saturatingAdd(ResourceCycles[Use.Resource], Use.Cycles); + } + Unit.Instructions.push_back(Instruction); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + ++Index; + if (!HasSuccessor) + break; + if (Index >= Instructions.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region ends in the middle of an instruction bundle"); + } + MachineInstr *Next = Instructions[Index]; + if (Next == nullptr || Next->parent() != Block || + !Next->flags().has(MachineInstrFlag::BundledWithPred) || Instruction->next() != Next) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction bundle links are not reciprocal and contiguous"); + } + Instruction = Next; + } + Unit.Resources.reserve(ResourceCycles.size()); + for (const auto &[Resource, Cycles] : ResourceCycles) + Unit.Resources.push_back({Resource, Cycles}); + std::ranges::sort(Unit.Resources, {}, &SchedulingResourceUse::Resource); + ExpectedInstruction = Unit.Instructions.back()->next(); + Units.push_back(std::move(Unit)); + } + return {}; +} + +Expected ScheduleDAG::buildRegisterDependencies(const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, + ScheduleDAGMode Mode) { + std::unordered_map + States; + for (ScheduleUnit &Unit : Units) { + std::vector Uses; + std::vector Definitions; + for (MachineInstr *Instruction : Unit.Instructions) { + for (std::size_t OperandIndex = 0; OperandIndex < Instruction->operandCount(); + ++OperandIndex) { + const MachineOperand *Operand = Instruction->operandAt(OperandIndex); + if (Operand == nullptr) + continue; + if (const auto *Mask = Operand->getIf()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null register mask"); + auto Keys = maskKeys(*Mask->Mask, Target); + if (!Keys) + return std::unexpected(Keys.error()); + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + continue; + } + if (const auto *Mask = Operand->getIf()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null live-out register mask"); + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask->Mask->preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &RegisterUnit : Register.Units) + Uses.push_back({{true, RegisterUnit.Unit.value()}, Instruction, OperandIndex}); + } + continue; + } + if (!Operand->isRegister() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + auto Keys = operandKeys(*Operand, Function, Target, Mode); + if (!Keys) + return std::unexpected(Keys.error()); + if (Operand->isUse()) { + for (const RegisterDependencyKey Key : *Keys) + Uses.push_back({Key, Instruction, OperandIndex}); + } + if (Operand->isDef()) { + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + } + } + } + std::ranges::stable_sort(Uses, [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + std::ranges::stable_sort(Definitions, + [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + + for (std::size_t Begin = 0; Begin < Uses.size();) { + std::size_t End = Begin + 1; + while (End < Uses.size() && Uses[End].Key == Uses[Begin].Key) + ++End; + RegisterDependencyState &State = States[Uses[Begin].Key]; + if (State.LastDefinition) { + std::uint32_t Latency = 0; + for (std::size_t Index = Begin; Index < End; ++Index) { + const InstructionDescriptor *Producer = + Target.instructionDescriptor(State.LastDefinition->Instruction->opcode()); + const InstructionDescriptor *Consumer = + Target.instructionDescriptor(Uses[Index].Instruction->opcode()); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler latency lookup encountered an unknown instruction"); + } + auto OperandLatency = + Model.operandLatency(Producer->SchedulingClass, State.LastDefinition->OperandIndex, + Consumer->SchedulingClass, Uses[Index].OperandIndex); + if (!OperandLatency) + return std::unexpected(OperandLatency.error()); + Latency = std::max(Latency, *OperandLatency); + } + addDependency(State.LastDefinition->Unit, Unit.ID, + Uses[Begin].Key.IsUnit ? ScheduleDependencyKind::RegisterUnit + : ScheduleDependencyKind::Data, + Latency); + } + if (State.LastUses.empty() || State.LastUses.back() != Unit.ID) + State.LastUses.push_back(Unit.ID); + Begin = End; + } + + for (std::size_t Begin = 0; Begin < Definitions.size();) { + std::size_t End = Begin + 1; + while (End < Definitions.size() && Definitions[End].Key == Definitions[Begin].Key) { + ++End; + } + RegisterDependencyState &State = States[Definitions[Begin].Key]; + if (State.LastDefinition) { + addDependency(State.LastDefinition->Unit, Unit.ID, ScheduleDependencyKind::Output, 0); + } + for (const std::size_t Use : State.LastUses) + addDependency(Use, Unit.ID, ScheduleDependencyKind::Anti, 0); + State.LastUses.clear(); + const RegisterAccess &Last = Definitions[End - 1]; + State.LastDefinition = + RegisterDependencyState::Definition{Unit.ID, Last.Instruction, Last.OperandIndex}; + Begin = End; + } + } + return {}; +} + +Expected ScheduleDAG::addArtificialDependency(std::size_t Source, std::size_t Destination, + std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source >= Destination) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "target schedule mutation requires a forward dependency between known units"); + } + addDependency(Source, Destination, ScheduleDependencyKind::Artificial, Latency); + return {}; +} + +Expected ScheduleDAG::buildMemoryDependencies(const TargetDescription &Target) { + MachineAliasAnalysis::Result Aliases; + for (std::size_t LaterIndex = 0; LaterIndex < Units.size(); ++LaterIndex) { + for (std::size_t EarlierIndex = 0; EarlierIndex < LaterIndex; ++EarlierIndex) { + bool NeedsEdge = false; + for (MachineInstr *Earlier : Units[EarlierIndex].Instructions) { + const InstructionDescriptor *EarlierDescriptor = + Target.instructionDescriptor(Earlier->opcode()); + if (EarlierDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool EarlierMemory = EarlierDescriptor->Flags.has(InstructionFlag::MayLoad) || + EarlierDescriptor->Flags.has(InstructionFlag::MayStore) || + !Earlier->memOperands().empty(); + if (!EarlierMemory) + continue; + for (MachineInstr *Later : Units[LaterIndex].Instructions) { + const InstructionDescriptor *LaterDescriptor = + Target.instructionDescriptor(Later->opcode()); + if (LaterDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool LaterMemory = LaterDescriptor->Flags.has(InstructionFlag::MayLoad) || + LaterDescriptor->Flags.has(InstructionFlag::MayStore) || + !Later->memOperands().empty(); + if (!LaterMemory) + continue; + if (hasOrderedMemory(*Earlier) || hasOrderedMemory(*Later) || + ((hasStore(*Earlier) || hasStore(*Later)) && mayAlias(*Earlier, *Later, Aliases)) || + (Earlier->memOperands().empty() || Later->memOperands().empty()) || + (!hasLoad(*Earlier) && !hasStore(*Earlier)) || + (!hasLoad(*Later) && !hasStore(*Later))) { + NeedsEdge = true; + break; + } + } + if (NeedsEdge) + break; + } + if (NeedsEdge) + addDependency(EarlierIndex, LaterIndex, ScheduleDependencyKind::Memory, 0); + } + } + return {}; +} + +Expected ScheduleDAG::buildOrderDependencies(const TargetDescription &Target) { + for (std::size_t Barrier = 0; Barrier < Units.size(); ++Barrier) { + bool IsBarrier = false; + for (const MachineInstr *Instruction : Units[Barrier].Instructions) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + IsBarrier |= Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || + Flags.has(InstructionFlag::Convergent) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::Meta); + } + if (!IsBarrier) + continue; + for (std::size_t Earlier = 0; Earlier < Barrier; ++Earlier) + addDependency(Earlier, Barrier, ScheduleDependencyKind::Barrier, 0); + for (std::size_t Later = Barrier + 1; Later < Units.size(); ++Later) + addDependency(Barrier, Later, ScheduleDependencyKind::Barrier, 0); + } + return {}; +} + +void ScheduleDAG::addDependency(std::size_t Source, std::size_t Destination, + ScheduleDependencyKind Kind, std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source == Destination) + return; + const auto Existing = std::ranges::find_if( + Units[Source].Successors, [Destination, Kind](const ScheduleDependency &Dependency) { + return Dependency.Destination == Destination && Dependency.Kind == Kind; + }); + if (Existing != Units[Source].Successors.end()) { + Existing->Latency = std::max(Existing->Latency, Latency); + const auto Predecessor = std::ranges::find_if( + Units[Destination].Predecessors, [Source, Kind](const ScheduleDependency &Dependency) { + return Dependency.Source == Source && Dependency.Kind == Kind; + }); + if (Predecessor != Units[Destination].Predecessors.end()) + Predecessor->Latency = Existing->Latency; + return; + } + const ScheduleDependency Dependency{Source, Destination, Kind, Latency}; + Units[Source].Successors.push_back(Dependency); + Units[Destination].Predecessors.push_back(Dependency); +} + +void ScheduleDAG::computeCriticalPaths() { + for (ScheduleUnit &Unit : Units) { + Unit.Depth = 0; + for (const ScheduleDependency &Dependency : Unit.Predecessors) { + Unit.Depth = + std::max(Unit.Depth, saturatingAdd(Units[Dependency.Source].Depth, + static_cast(Dependency.Latency))); + } + } + for (auto Position = Units.rbegin(); Position != Units.rend(); ++Position) { + Position->Height = Position->Latency; + for (const ScheduleDependency &Dependency : Position->Successors) { + Position->Height = + std::max(Position->Height, saturatingAdd(Units[Dependency.Destination].Height, + static_cast(Dependency.Latency))); + } + } + for (ScheduleUnit &Unit : Units) { + std::ranges::sort(Unit.Predecessors, {}, &ScheduleDependency::Source); + std::ranges::sort(Unit.Successors, {}, &ScheduleDependency::Destination); + } +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp new file mode 100644 index 0000000..28c032d --- /dev/null +++ b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" + +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +} // namespace + +Expected +ScheduleHazardRecognizer::create(const TargetSchedulingModel &Model, ScheduleDirection Direction) { + ScheduleHazardRecognizer Result(Model, Direction); + Result.TargetHazards = Model.hooks().createHazardRecognizer(Direction); + auto Reset = Result.reset(); + if (!Reset) + return std::unexpected(Reset.error()); + return Result; +} + +ScheduleHazardRecognizer::ScheduleHazardRecognizer(const ScheduleHazardRecognizer &Other) + : Model(Other.Model), Direction(Other.Direction), + TargetHazards(Other.TargetHazards ? Other.TargetHazards->clone() : nullptr), + Resources(Other.Resources), ResourceIndexes(Other.ResourceIndexes), + CurrentCycle(Other.CurrentCycle), DispatchAvailableAt(Other.DispatchAvailableAt), + IssuedMicroOps(Other.IssuedMicroOps), DispatchedUnits(Other.DispatchedUnits) {} + +ScheduleHazardRecognizer & +ScheduleHazardRecognizer::operator=(const ScheduleHazardRecognizer &Other) { + if (this == &Other) + return *this; + Model = Other.Model; + Direction = Other.Direction; + TargetHazards = Other.TargetHazards ? Other.TargetHazards->clone() : nullptr; + Resources = Other.Resources; + ResourceIndexes = Other.ResourceIndexes; + CurrentCycle = Other.CurrentCycle; + DispatchAvailableAt = Other.DispatchAvailableAt; + IssuedMicroOps = Other.IssuedMicroOps; + DispatchedUnits = Other.DispatchedUnits; + return *this; +} + +Expected ScheduleHazardRecognizer::reset() { + Resources.clear(); + ResourceIndexes.clear(); + Resources.reserve(Model->processorResources().size()); + for (const ProcessorResourceDesc &Resource : Model->processorResources()) { + ResourceIndexes.emplace(Resource.ID, Resources.size()); + Resources.push_back({Resource.ID, std::vector(Resource.Units, 0)}); + } + CurrentCycle = 0; + DispatchAvailableAt = 0; + IssuedMicroOps = 0; + DispatchedUnits = 0; + if (TargetHazards != nullptr) { + auto Reset = TargetHazards->reset(); + if (!Reset) + return std::unexpected(Reset.error()); + } + return {}; +} + +Expected ScheduleHazardRecognizer::canIssue(const ScheduleUnit &Unit, + std::uint64_t Cycle) const { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) { + if (Issued.error().code() == ErrorCode::SchedulingFailed) + return false; + return std::unexpected(Issued.error()); + } + return true; +} + +Expected ScheduleHazardRecognizer::earliestIssueCycle(const ScheduleUnit &Unit, + std::uint64_t Earliest) const { + std::uint64_t Candidate = std::max(Earliest, CurrentCycle); + for (;;) { + auto Available = canIssue(Unit, Candidate); + if (!Available) + return std::unexpected(Available.error()); + if (*Available) + return Candidate; + if (Candidate == std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler resource timeline is exhausted"); + } + ++Candidate; + } +} + +Expected ScheduleHazardRecognizer::issue(const ScheduleUnit &Unit, std::uint64_t Cycle) { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + *this = std::move(Trial); + return {}; +} + +Expected ScheduleHazardRecognizer::issueImplementation(const ScheduleUnit &Unit, + std::uint64_t Cycle) { + if (Cycle < CurrentCycle) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduler cannot issue into an earlier cycle"); + } + auto Moved = moveToCycle(Cycle); + if (!Moved) + return std::unexpected(Moved.error()); + if (Cycle < DispatchAvailableAt) { + return makeUnexpected(ErrorCode::SchedulingFailed, "scheduler dispatch pipeline is occupied"); + } + if (TargetHazards != nullptr) { + auto Hazard = TargetHazards->getHazardType(Unit); + if (!Hazard) + return std::unexpected(Hazard.error()); + if (*Hazard != ScheduleHazardType::NoHazard) { + return makeUnexpected(ErrorCode::SchedulingFailed, + *Hazard == ScheduleHazardType::NoopHazard + ? "target hazard requires an unavailable scheduling noop" + : "target dynamic scheduling hazard prevents issue"); + } + } + + const std::uint32_t Width = Model->issueWidth(); + if (Unit.MicroOps <= Width) { + if (Unit.MicroOps > Width - IssuedMicroOps || DispatchedUnits == Model->dispatchLimit()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler issue or dispatch width is exhausted"); + } + IssuedMicroOps += Unit.MicroOps; + ++DispatchedUnits; + } else { + if (IssuedMicroOps != 0 || DispatchedUnits != 0) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "multi-cycle dispatch requires an empty issue cycle"); + } + const std::uint64_t DispatchCycles = + (static_cast(Unit.MicroOps) + Width - 1U) / Width; + DispatchAvailableAt = saturatingAdd(Cycle, DispatchCycles); + IssuedMicroOps = Width; + DispatchedUnits = 1; + } + + for (const SchedulingResourceUse &Use : Unit.Resources) { + auto Reserved = reserveResource(Use.Resource, Use.Cycles, Cycle); + if (!Reserved) + return std::unexpected(Reserved.error()); + } + if (TargetHazards != nullptr) { + auto Emitted = TargetHazards->emitInstruction(Unit); + if (!Emitted) + return std::unexpected(Emitted.error()); + } + return {}; +} + +Expected ScheduleHazardRecognizer::moveToCycle(std::uint64_t Cycle) { + while (CurrentCycle < Cycle) { + if (TargetHazards != nullptr) { + Expected Advanced = Direction == ScheduleDirection::TopDown + ? TargetHazards->advanceCycle() + : TargetHazards->recedeCycle(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + ++CurrentCycle; + IssuedMicroOps = 0; + DispatchedUnits = 0; + } + return {}; +} + +Expected ScheduleHazardRecognizer::reserveResource(ProcessorResourceID Resource, + std::uint32_t Cycles, + std::uint64_t Cycle) { + const auto Position = ResourceIndexes.find(Resource); + const ProcessorResourceDesc *Description = Model->processorResource(Resource); + if (Position == ResourceIndexes.end() || Description == nullptr || Cycles == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule unit references an invalid processor resource"); + } + ResourceState &State = Resources[Position->second]; + std::optional Selected; + for (std::size_t Index = 0; Index < State.AvailableAt.size(); ++Index) { + if (State.AvailableAt[Index] <= Cycle && + (!Selected || State.AvailableAt[Index] < State.AvailableAt[*Selected])) { + Selected = Index; + } + } + if (!Selected) { + return makeUnexpected(ErrorCode::SchedulingFailed, "processor resource has no available unit"); + } + State.AvailableAt[*Selected] = saturatingAdd(Cycle, Cycles); + if (Description->Parent) { + auto Parent = reserveResource(*Description->Parent, Cycles, Cycle); + if (!Parent) + return std::unexpected(Parent.error()); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetSchedulingModel.cpp b/backend/lib/CodeGen/TargetSchedulingModel.cpp new file mode 100644 index 0000000..97abecd --- /dev/null +++ b/backend/lib/CodeGen/TargetSchedulingModel.cpp @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +template [[nodiscard]] bool isValidID(IDT ID) noexcept { + return ID.value() != 0; +} + +} // namespace + +Expected +TargetScheduleHazardRecognizer::getHazardType(const ScheduleUnit &) const { + return ScheduleHazardType::NoHazard; +} + +Expected TargetScheduleHazardRecognizer::emitInstruction(const ScheduleUnit &) { + return {}; +} + +Expected TargetScheduleHazardRecognizer::advanceCycle() { + return {}; +} + +Expected TargetScheduleHazardRecognizer::recedeCycle() { + return {}; +} + +Expected TargetScheduleHazardRecognizer::reset() { + return {}; +} + +bool TargetSchedulingHooks::isSchedulingBoundary(const MachineInstr &, + const TargetDescription &) const noexcept { + return false; +} + +Expected TargetSchedulingHooks::mutateScheduleDAG(ScheduleDAG &, const MachineFunction &, + const TargetDescription &, + ScheduleDAGMode) const { + return {}; +} + +std::int64_t TargetSchedulingHooks::candidatePreference(const ScheduleUnit &, ScheduleDirection, + const MachineBasicBlock &, + std::uint64_t) const noexcept { + return 0; +} + +std::unique_ptr +TargetSchedulingHooks::createHazardRecognizer(ScheduleDirection) const { + return nullptr; +} + +Expected +TargetSchedulingModel::create(TargetSchedulingModelDesc Description) { + if (Description.IssueWidth == 0 || Description.DispatchLimit == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model requires nonzero issue and dispatch widths"); + } + + TargetSchedulingModel Result(std::move(Description)); + std::unordered_set Names; + for (std::size_t Index = 0; Index < Result.Description.ProcessorResources.size(); ++Index) { + const ProcessorResourceDesc &Resource = Result.Description.ProcessorResources[Index]; + if (!isValidID(Resource.ID) || Resource.Name.empty() || Resource.Units == 0 || + !Result.ResourceToIndex.emplace(Resource.ID, Index).second || + !Names.insert(Resource.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid processor resource"); + } + } + for (const ProcessorResourceDesc &Resource : Result.Description.ProcessorResources) { + if (Resource.Parent && + (Resource.Parent == Resource.ID || !Result.ResourceToIndex.contains(*Resource.Parent))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource has an invalid parent resource"); + } + std::unordered_set Path; + const ProcessorResourceDesc *Current = &Resource; + while (Current->Parent) { + if (!Path.insert(Current->ID).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource hierarchy contains a cycle"); + } + Current = &Result.Description.ProcessorResources[Result.ResourceToIndex.at(*Current->Parent)]; + } + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.SchedulingClasses.size(); ++Index) { + SchedulingClassModelDesc &Class = Result.Description.SchedulingClasses[Index]; + if (!isValidID(Class.ID) || Class.Name.empty() || Class.MicroOps == 0 || Class.Latency == 0 || + !Result.ClassToIndex.emplace(Class.ID, Index).second || !Names.insert(Class.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid scheduling class"); + } + } + for (SchedulingClassModelDesc &Class : Result.Description.SchedulingClasses) { + std::unordered_set UsedResources; + for (const SchedulingResourceUse &Use : Class.Resources) { + if (!isValidID(Use.Resource) || Use.Cycles == 0 || + !Result.ResourceToIndex.contains(Use.Resource) || + !UsedResources.insert(Use.Resource).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has an invalid resource use"); + } + } + std::ranges::sort(Class.Resources, {}, + [](const SchedulingResourceUse &Use) { return Use.Resource.value(); }); + + std::unordered_set WriteOperands; + for (const SchedulingWriteLatency &Latency : Class.WriteLatencies) { + if (!WriteOperands.insert(Latency.OperandIndex).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate write latencies"); + } + } + std::ranges::sort(Class.WriteLatencies, {}, &SchedulingWriteLatency::OperandIndex); + + std::unordered_set ReadAdvanceKeys; + for (const SchedulingReadAdvance &Advance : Class.ReadAdvances) { + if (Advance.ProducerClass && !Result.ClassToIndex.contains(*Advance.ProducerClass)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "read advance references an unknown producer class"); + } + const std::uint64_t Producer = Advance.ProducerClass ? Advance.ProducerClass->value() : 0; + const std::uint64_t Key = (Producer << 32U) | Advance.OperandIndex; + if (!ReadAdvanceKeys.insert(Key).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate read advances"); + } + } + std::ranges::sort(Class.ReadAdvances, + [](const SchedulingReadAdvance &Left, const SchedulingReadAdvance &Right) { + if (Left.OperandIndex != Right.OperandIndex) + return Left.OperandIndex < Right.OperandIndex; + return Left.ProducerClass.value_or(SchedulingClassID{}).value() < + Right.ProducerClass.value_or(SchedulingClassID{}).value(); + }); + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.PressureSets.size(); ++Index) { + RegisterPressureSetDesc &Set = Result.Description.PressureSets[Index]; + if (!isValidID(Set.ID) || Set.Name.empty() || Set.Limit == 0 || Set.RegisterClasses.empty() || + !Result.PressureSetToIndex.emplace(Set.ID, Index).second || + !Names.insert(Set.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid register pressure set"); + } + std::ranges::sort(Set.RegisterClasses, [](RegisterClassID Left, RegisterClassID Right) { + return Left.value() < Right.value(); + }); + if (std::ranges::adjacent_find(Set.RegisterClasses) != Set.RegisterClasses.end() || + std::ranges::any_of(Set.RegisterClasses, + [](RegisterClassID Class) { return !isValidID(Class); })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register pressure set has duplicate or invalid register classes"); + } + for (const RegisterClassID Class : Set.RegisterClasses) + Result.ClassToPressureSets[Class].push_back(Set.ID); + } + for (auto &[Class, Sets] : Result.ClassToPressureSets) { + (void)Class; + std::ranges::sort(Sets, [](RegisterPressureSetID Left, RegisterPressureSetID Right) { + return Left.value() < Right.value(); + }); + } + return Result; +} + +const ProcessorResourceDesc * +TargetSchedulingModel::processorResource(ProcessorResourceID ID) const noexcept { + const auto Position = ResourceToIndex.find(ID); + return Position == ResourceToIndex.end() ? nullptr + : &Description.ProcessorResources[Position->second]; +} + +const SchedulingClassModelDesc * +TargetSchedulingModel::schedulingClass(SchedulingClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + return Position == ClassToIndex.end() ? nullptr + : &Description.SchedulingClasses[Position->second]; +} + +const RegisterPressureSetDesc * +TargetSchedulingModel::pressureSet(RegisterPressureSetID ID) const noexcept { + const auto Position = PressureSetToIndex.find(ID); + return Position == PressureSetToIndex.end() ? nullptr + : &Description.PressureSets[Position->second]; +} + +std::span +TargetSchedulingModel::pressureSetsForClass(RegisterClassID Class) const noexcept { + const auto Position = ClassToPressureSets.find(Class); + return Position == ClassToPressureSets.end() + ? std::span{} + : std::span(Position->second); +} + +Expected TargetSchedulingModel::operandLatency(SchedulingClassID ProducerClass, + std::size_t DefinitionOperand, + SchedulingClassID ConsumerClass, + std::size_t UseOperand) const { + if (DefinitionOperand > std::numeric_limits::max() || + UseOperand > std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduling operand index exceeds the target model range"); + } + const SchedulingClassModelDesc *Producer = schedulingClass(ProducerClass); + const SchedulingClassModelDesc *Consumer = schedulingClass(ConsumerClass); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand latency references an unknown scheduling class"); + } + + const std::uint32_t DefinitionIndex = static_cast(DefinitionOperand); + const std::uint32_t UseIndex = static_cast(UseOperand); + std::uint32_t Latency = Producer->Latency; + if (const auto Position = std::ranges::find(Producer->WriteLatencies, DefinitionIndex, + &SchedulingWriteLatency::OperandIndex); + Position != Producer->WriteLatencies.end()) { + Latency = Position->Cycles; + } + + const SchedulingReadAdvance *Wildcard = nullptr; + const SchedulingReadAdvance *Exact = nullptr; + for (const SchedulingReadAdvance &Advance : Consumer->ReadAdvances) { + if (Advance.OperandIndex != UseIndex) + continue; + if (!Advance.ProducerClass) + Wildcard = &Advance; + else if (*Advance.ProducerClass == ProducerClass) + Exact = &Advance; + } + const SchedulingReadAdvance *Advance = Exact != nullptr ? Exact : Wildcard; + return Advance == nullptr || Advance->Cycles >= Latency ? (Advance == nullptr ? Latency : 0U) + : Latency - Advance->Cycles; +} + +const TargetSchedulingHooks &TargetSchedulingModel::hooks() const noexcept { + static const TargetSchedulingHooks DefaultHooks; + return Description.Hooks == nullptr ? DefaultHooks : *Description.Hooks; +} + +} // namespace rcc diff --git a/backend/lib/Support/Error.cpp b/backend/lib/Support/Error.cpp index f18cc45..88e16c9 100644 --- a/backend/lib/Support/Error.cpp +++ b/backend/lib/Support/Error.cpp @@ -18,6 +18,16 @@ std::string_view getErrorCodeName(ErrorCode Code) noexcept { return "register allocation failed"; case ErrorCode::FrameLayoutFailed: return "frame layout failed"; + case ErrorCode::SchedulingFailed: + return "scheduling failed"; + case ErrorCode::BranchRelaxationFailed: + return "branch relaxation failed"; + case ErrorCode::PseudoExpansionFailed: + return "pseudo expansion failed"; + case ErrorCode::FinalEncodabilityFailed: + return "final encodability failed"; + case ErrorCode::CFGOptimizationFailed: + return "CFG optimization failed"; case ErrorCode::IoError: return "I/O error"; case ErrorCode::InternalError: diff --git a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp index 9d6b8e1..cd5ba97 100644 --- a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp +++ b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp @@ -1683,7 +1683,7 @@ instructionFlagEnumerator(std::string_view Value) noexcept { [[nodiscard]] std::optional pseudoOwnerEnumerator(std::string_view Value) noexcept { - static constexpr std::array, 11> Owners = {{ + static constexpr std::array, 12> Owners = {{ {"none", "None"}, {"legalizer", "Legalizer"}, {"register_bank_select", "RegisterBankSelect"}, @@ -1695,6 +1695,7 @@ pseudoOwnerEnumerator(std::string_view Value) noexcept { {"prologue_epilogue_insertion", "PrologueEpilogueInsertion"}, {"post_ra_expansion", "PostRAExpansion"}, {"finalization", "Finalization"}, + {"target_late_expansion", "TargetLateExpansion"}, }}; if (Value == "asm_printer") return "AsmPrinter"; @@ -2328,8 +2329,8 @@ private: } if (InputOperand.ImmediatePredicateSymbol) { const auto Position = ImmediatePredicateBySymbol.find(*InputOperand.ImmediatePredicateSymbol); - if (Position == ImmediatePredicateBySymbol.end() || Result.Kinds.size() != 1 || - Result.Kinds.front() != "Immediate") { + if (Position == ImmediatePredicateBySymbol.end() || + std::ranges::find(Result.Kinds, "Immediate") == Result.Kinds.end()) { fail(InputOperand.Range, "instruction immediate predicates must name a declaration and constrain an immediate"); return false; diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index 442e75e..a29315a 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -47,10 +47,16 @@ set_source_files_properties( add_library( RCCBackendRISCV + RISCVBranchInfo.cpp RISCVCodeGen.cpp RISCVCallLowering.cpp RISCVFrameLowering.cpp + RISCVMachineCombiner.cpp + RISCVMachineMemoryFolding.cpp + RISCVOptimizationPipeline.cpp + RISCVPseudoExpansion.cpp RISCVRegisterAllocation.cpp + RISCVScheduling.cpp RISCVTarget.cpp "${RCCBackendRISCVGeneratedRegisterHeader}" "${RCCBackendRISCVGeneratedInstructionHeader}" @@ -93,19 +99,31 @@ install( ) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTarget.h" ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" ) diff --git a/backend/lib/Target/RISCV/RISCV.rccdesc b/backend/lib/Target/RISCV/RISCV.rccdesc index e248bc7..dce984c 100644 --- a/backend/lib/Target/RISCV/RISCV.rccdesc +++ b/backend/lib/Target/RISCV/RISCV.rccdesc @@ -245,6 +245,10 @@ mir_operand_flags: direct_mask: 31 direct: - { name: riscv-call, value: 1 } + - { name: riscv-pcrel-hi, value: 2 } + - { name: riscv-pcrel-lo, value: 4 } + - { name: riscv-copy-f32, value: 8 } + - { name: riscv-copy-f64, value: 16 } bitmask: [] calling_conventions: @@ -386,7 +390,7 @@ instructions: assembly_mnemonic: auipc operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - - { kinds: [immediate], immediate_predicate: SIMM20, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM20, allowed_target_flags: 3, name: IMM } num_explicit_defs: 1 flags: [produces_code] scheduling_class: ALU @@ -871,7 +875,7 @@ instructions: operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } - - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } num_explicit_defs: 1 flags: [terminator, branch, indirect_branch, produces_code] scheduling_class: BRANCH @@ -3195,3 +3199,52 @@ instructions: flags: [pseudo, produces_code, allowed_pre_legalizer, terminator, return, barrier] owner: post_ra_expansion deadline: post_ra_finalized + - + id: 1225 + symbol: JALR_CALL + name: JALR_CALL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [call, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1226 + symbol: JALR_TAIL + name: JALR_TAIL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, branch, indirect_branch, call, barrier, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1227 + symbol: JALR_RET + name: JALR_RET + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, return, barrier, produces_code] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM diff --git a/backend/lib/Target/RISCV/RISCVBranchInfo.cpp b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp new file mode 100644 index 0000000..18812d8 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] bool isConditionalBranch(Opcode Value) noexcept { + return Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BNE) || + Value == targetOpcode(InstructionOpcode::BLT) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BLTU) || + Value == targetOpcode(InstructionOpcode::BGEU) || + Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ); +} + +[[nodiscard]] std::optional reversedOpcode(Opcode Value) noexcept { + constexpr std::array, 4> Pairs = {{ + {InstructionOpcode::BEQ, InstructionOpcode::BNE}, + {InstructionOpcode::BLT, InstructionOpcode::BGE}, + {InstructionOpcode::BLTU, InstructionOpcode::BGEU}, + {InstructionOpcode::C_BEQZ, InstructionOpcode::C_BNEZ}, + }}; + for (const auto &[Left, Right] : Pairs) { + if (Value == targetOpcode(Left)) + return targetOpcode(Right); + if (Value == targetOpcode(Right)) + return targetOpcode(Left); + } + return std::nullopt; +} + +[[nodiscard]] bool isZeroRegister(const MachineOperand *Operand) noexcept { + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] bool sameRegister(const MachineOperand *Left, const MachineOperand *Right) noexcept { + const Register *LeftReg = Left == nullptr ? nullptr : Left->getIf(); + const Register *RightReg = Right == nullptr ? nullptr : Right->getIf(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Left->subReg() == Right->subReg(); +} + +[[nodiscard]] bool isUnconditionalBranch(const MachineInstr &Instruction) noexcept { + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) + return true; + return Instruction.opcode() == targetOpcode(InstructionOpcode::JAL) && + isZeroRegister(Instruction.operandAt(0)); +} + +[[nodiscard]] std::size_t branchTargetIndex(Opcode Value) noexcept { + if (Value == targetOpcode(InstructionOpcode::C_J)) + return 0; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return 1; + } + if (Value == targetOpcode(InstructionOpcode::JAL)) + return 1; + return 2; +} + +[[nodiscard]] Expected directTarget(MachineInstr &Instruction) { + if (!isConditionalBranch(Instruction.opcode()) && + Instruction.opcode() != targetOpcode(InstructionOpcode::JAL) && + Instruction.opcode() != targetOpcode(InstructionOpcode::C_J)) { + return static_cast(nullptr); + } + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + const BasicBlockOperand *Block = + Operand == nullptr ? nullptr : Operand->getIf(); + if (Block == nullptr || Block->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no basic-block target"); + } + return Block->Block; +} + +[[nodiscard]] Expected setDirectTarget(MachineInstr &Instruction, + MachineBasicBlock &TargetBlock) { + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + if (Operand == nullptr || Operand->kind() != MachineOperand::Kind::BasicBlock) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no mutable basic-block target"); + } + return Operand->setBasicBlock(TargetBlock); +} + +[[nodiscard]] Expected removeInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch instruction is not linked to a block"); + } + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] Expected +buildJAL(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, + MachineInstr *Before, PhysicalRegister Destination, MachineBasicBlock &TargetBlock) { + MachineIRBuilder Builder(Function, Target); + Expected Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(Destination)); + if (!Def) + return std::unexpected(Def.error()); + auto TargetOperand = Pending->addBasicBlock(TargetBlock); + if (!TargetOperand) + return std::unexpected(TargetOperand.error()); + return Pending->build(); +} + +[[nodiscard]] Expected copySourceIdentity(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyTrailingAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.debugInstrNumber() != 0) { + auto DebugNumber = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!DebugNumber) + return std::unexpected(DebugNumber.error()); + } + if (Source.postInstrSymbol()) { + auto Symbol = Destination.setPostInstrSymbol(std::string(*Source.postInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + if (Source.heapAllocMarker()) { + auto Marker = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Marker) + return std::unexpected(Marker.error()); + } + if (Source.pcSections()) { + auto Sections = Destination.setPCSections(*Source.pcSections()); + if (!Sections) + return std::unexpected(Sections.error()); + } + if (Source.mmra()) { + auto Metadata = Destination.setMMRA(*Source.mmra()); + if (!Metadata) + return std::unexpected(Metadata.error()); + } + if (Source.cfiType()) { + auto Type = Destination.setCFIType(*Source.cfiType()); + if (!Type) + return std::unexpected(Type.error()); + } + if (Source.deactivationSymbol()) { + auto Symbol = Destination.setDeactivationSymbol(std::string(*Source.deactivationSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return {}; +} + +[[nodiscard]] Expected copySingleInstructionAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + Destination.setFlags(Source.flags()); + auto Identity = copySourceIdentity(Source, Destination); + if (!Identity) + return std::unexpected(Identity.error()); + if (Source.preInstrSymbol()) { + auto Symbol = Destination.setPreInstrSymbol(std::string(*Source.preInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return copyTrailingAnnotations(Source, Destination); +} + +[[nodiscard]] bool isZeroPhysicalRegister(PhysicalRegister Reg) noexcept { + return Reg.number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] Expected +selectLongBranchScratch(const MachineInstr &Instruction, const MachineBasicBlock &TargetBlock, + const MachineFunction &Function, const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const std::optional DestinationPhysical = + Destination == nullptr ? std::nullopt : Destination->physicalRegister(); + if (!DestinationPhysical) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL has no physical destination"); + } + if (!isZeroPhysicalRegister(*DestinationPhysical)) + return *DestinationPhysical; + + const auto GPRClass = + RegisterClassID::fromValue(static_cast(RegisterClass::GPR_NX0)); + const RegisterClassDesc *Candidates = Target.registers().registerClass(GPRClass); + if (Candidates == nullptr) { + return makeUnexpected(ErrorCode::InternalError, "RISC-V target has no nonzero GPR class"); + } + for (const PhysicalRegister Candidate : Candidates->AllocationOrder) { + if (Target.registers().isReservedRegister(Candidate) || + Function.registerInfo().isReserved(Candidate)) { + continue; + } + const bool LiveAtTarget = + std::ranges::any_of(TargetBlock.liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Candidate, LiveIn.PhysReg); + }); + if (!LiveAtTarget) + return Candidate; + } + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V long branch has no dead physical scratch register"); +} + +[[nodiscard]] Expected expandCompressedBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "compressed branch is not linked"); + auto TargetBlock = directTarget(Instruction); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) { + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Replacement = buildJAL(Function, Target, *Block, &Instruction, *Zero, **TargetBlock); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + + const MachineOperand *SourceOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "compressed conditional branch has no source register"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + const InstructionOpcode Expanded = Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) + ? InstructionOpcode::BEQ + : InstructionOpcode::BNE; + auto Pending = Builder.buildInstr(targetOpcode(Expanded)); + if (!Pending) + return std::unexpected(Pending.error()); + auto First = Pending->addUse(*Source); + if (!First) + return std::unexpected(First.error()); + auto Second = Pending->addUse(Register::fromPhysical(*Zero)); + if (!Second) + return std::unexpected(Second.error()); + auto Destination = Pending->addBasicBlock(**TargetBlock); + if (!Destination) + return std::unexpected(Destination.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected relaxConditionalBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "conditional branch is not linked"); + auto TrueTarget = directTarget(Instruction); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + MachineBasicBlock *FalseTarget = Block->next(); + if (FalseTarget == nullptr || FalseTarget == *TrueTarget || + std::ranges::none_of(Block->successors(), [FalseTarget](const MachineSuccessor &Successor) { + return Successor.Block == FalseTarget; + })) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V conditional relaxation requires its false edge as fallthrough"); + } + + auto Reversed = reversedOpcode(Instruction.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V conditional branch cannot be inverted"); + auto SetOpcode = Instruction.setOpcode(*Reversed); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetTarget = setDirectTarget(Instruction, *FalseTarget); + if (!SetTarget) + return std::unexpected(SetTarget.error()); + + MachineInstr *Following = Instruction.next(); + if (Following != nullptr && isUnconditionalBranch(*Following)) { + auto Retargeted = setDirectTarget(*Following, **TrueTarget); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + return true; + } + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Destination = Pending->addBasicBlock(**TrueTarget); + if (!Destination) + return std::unexpected(Destination.error()); + auto LongJump = Pending->build(); + if (!LongJump) + return std::unexpected(LongJump.error()); + auto Identity = copySourceIdentity(Instruction, **LongJump); + if (!Identity) + return std::unexpected(Identity.error()); + MachineInstrFlags OriginalFlags = Instruction.flags(); + const bool HadBundledSuccessor = OriginalFlags.has(MachineInstrFlag::BundledWithSucc); + OriginalFlags.set(MachineInstrFlag::BundledWithSucc); + Instruction.setFlags(OriginalFlags); + MachineInstrFlags LongFlags; + LongFlags.set(MachineInstrFlag::BundledWithPred); + if (HadBundledSuccessor) + LongFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + LongFlags.set(MachineInstrFlag::NoMerge); + (*LongJump)->setFlags(LongFlags); + return true; +} + +[[nodiscard]] Expected relaxJAL(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V JAL is not linked"); + if (Offset < std::numeric_limits::min() || + Offset > std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V PC-relative long branch exceeds the signed 32-bit range"); + } + auto DestinationBlock = directTarget(Instruction); + if (!DestinationBlock) + return std::unexpected(DestinationBlock.error()); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || !Destination->isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL requires a physical destination register"); + } + auto Scratch = selectLongBranchScratch(Instruction, **DestinationBlock, Function, Target); + if (!Scratch) + return std::unexpected(Scratch.error()); + const std::string Anchor = Instruction.preInstrSymbol() + ? std::string(*Instruction.preInstrSymbol()) + : ".Lrcc_long_" + std::to_string(Function.stableID().value()) + + "_" + std::to_string(Instruction.stableID().value()); + const MachineInstrFlags OriginalFlags = Instruction.flags(); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Register::fromPhysical(*Scratch)); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = HighPending->addBasicBlock(**DestinationBlock, PCRelativeHighFlag); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto HighSymbol = (*High)->setPreInstrSymbol(Anchor); + if (!HighSymbol) + return std::unexpected(HighSymbol.error()); + auto HighIdentity = copySourceIdentity(Instruction, **High); + if (!HighIdentity) + return std::unexpected(HighIdentity.error()); + + auto LowPending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR)); + if (!LowPending) + return std::unexpected(LowPending.error()); + auto LowDef = LowPending->addDef(*Destination); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Register::fromPhysical(*Scratch)); + if (!LowBase) + return std::unexpected(LowBase.error()); + auto LowSymbol = LowPending->addMCSymbol(Anchor, 0, PCRelativeLowFlag); + if (!LowSymbol) + return std::unexpected(LowSymbol.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto LowIdentity = copySourceIdentity(Instruction, **Low); + if (!LowIdentity) + return std::unexpected(LowIdentity.error()); + auto LowAnnotations = copyTrailingAnnotations(Instruction, **Low); + if (!LowAnnotations) + return std::unexpected(LowAnnotations.error()); + + MachineInstrFlags HighFlags; + if (OriginalFlags.has(MachineInstrFlag::BundledWithPred)) + HighFlags.set(MachineInstrFlag::BundledWithPred); + HighFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + HighFlags.set(MachineInstrFlag::NoMerge); + (*High)->setFlags(HighFlags); + MachineInstrFlags LowFlags = OriginalFlags; + LowFlags.clear(MachineInstrFlag::BundledWithPred); + LowFlags.set(MachineInstrFlag::BundledWithPred); + (*Low)->setFlags(LowFlags); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVBranchInfo final : public TargetBranchInfo { +public: + [[nodiscard]] Expected + analyzeBranch(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch analysis references another function"); + std::vector Terminators; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V branch analysis encountered an unknown opcode"); + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + Terminators.push_back(Instruction); + } + + MachineBranchAnalysis Result; + if (Terminators.empty()) + return Result; + if (Terminators.size() > 2) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + MachineInstr *First = Terminators.front(); + if (isConditionalBranch(First->opcode())) { + auto TrueTarget = directTarget(*First); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + Result.Conditional = First; + Result.TrueTarget = *TrueTarget; + if (Terminators.size() == 2) { + MachineInstr *Second = Terminators.back(); + if (!isUnconditionalBranch(*Second)) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + auto FalseTarget = directTarget(*Second); + if (!FalseTarget) + return std::unexpected(FalseTarget.error()); + Result.Unconditional = Second; + Result.FalseTarget = *FalseTarget; + } else { + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block == nullptr || Successor.Block == Result.TrueTarget) + continue; + if (Result.FalseTarget != nullptr) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + Result.FalseTarget = Successor.Block; + } + if (Result.FalseTarget == nullptr) + Result.FalseTarget = Block.next(); + } + return Result; + } + if (Terminators.size() == 1 && isUnconditionalBranch(*First)) { + auto TargetBlock = directTarget(*First); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + Result.Unconditional = First; + Result.TrueTarget = *TargetBlock; + return Result; + } + Result.HasUnanalyzableTerminator = true; + return Result; + } + + [[nodiscard]] Expected reverseBranchCondition(MachineInstr &Branch, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function || &NewTarget.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch reversal references another function"); + const std::optional Reversed = reversedOpcode(Branch.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch opcode has no inverse condition"); + auto OpcodeResult = Branch.setOpcode(*Reversed); + if (!OpcodeResult) + return std::unexpected(OpcodeResult.error()); + return setDirectTarget(Branch, NewTarget); + } + + [[nodiscard]] Expected removeBranch(MachineInstr &Branch, MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch removal references another function"); + return removeInstruction(Branch); + } + + [[nodiscard]] Expected rewriteBranchTarget(MachineBasicBlock &Block, + MachineBasicBlock &OldTarget, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Block.function() != &Function || &OldTarget.function() != &Function || + &NewTarget.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch rewrite references another function"); + } + bool Changed = false; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isConditionalBranch(Instruction->opcode()) && !isUnconditionalBranch(*Instruction)) { + continue; + } + auto CurrentTarget = directTarget(*Instruction); + if (!CurrentTarget) + return std::unexpected(CurrentTarget.error()); + if (*CurrentTarget != &OldTarget) + continue; + auto Rewritten = setDirectTarget(*Instruction, NewTarget); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + Changed = true; + } + return Changed; + } + + [[nodiscard]] Expected + insertUnconditionalBranch(MachineBasicBlock &Block, MachineBasicBlock &TargetBlock, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function || &TargetBlock.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch insertion references another function"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + return buildJAL(Function, Target, Block, nullptr, *Zero, TargetBlock); + } + + [[nodiscard]] Expected + instructionSize(const MachineInstr &Instruction, const MachineFunction &, + const TargetDescription &Target) const override { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction-size query encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::ProducesCode)) + return 0; + const auto Category = + std::ranges::find_if(encodingCategories(), [Descriptor](const EncodingCategoryInfo &Info) { + return Info.ID == Descriptor->EncodingCategory; + }); + if (Category == encodingCategories().end() || Category->SizeInBytes == 0) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V producing instruction has no exact encoding size"); + return Category->SizeInBytes; + } + + [[nodiscard]] Expected + branchTarget(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch target query references another function"); + return directTarget(Instruction); + } + + [[nodiscard]] Expected> + evaluateBranchCondition(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch evaluation references another function"); + } + const Opcode Value = Instruction.opcode(); + if (!isConditionalBranch(Value)) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + if (!isZeroRegister(Instruction.operandAt(0))) + return std::optional{}; + return std::optional{Value == targetOpcode(InstructionOpcode::C_BEQZ)}; + } + if (!sameRegister(Instruction.operandAt(0), Instruction.operandAt(1))) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BGEU)) { + return std::optional{true}; + } + return std::optional{false}; + } + + [[nodiscard]] bool isBranchOffsetInRange(const MachineInstr &Instruction, + std::int64_t Offset) const noexcept override { + if ((Offset & 1) != 0) + return false; + const Opcode Value = Instruction.opcode(); + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return Offset >= -256 && Offset <= 254; + } + if (Value == targetOpcode(InstructionOpcode::C_J)) + return Offset >= -2048 && Offset <= 2046; + if (isConditionalBranch(Value)) + return Offset >= -4096 && Offset <= 4094; + if (Value == targetOpcode(InstructionOpcode::JAL)) + return Offset >= -1048576 && Offset <= 1048574; + return true; + } + + [[nodiscard]] Expected relaxBranch(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch relaxation references another function"); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BNEZ)) { + return expandCompressedBranch(Instruction, Function, Target); + } + if (isConditionalBranch(Instruction.opcode())) + return relaxConditionalBranch(Instruction, Function, Target); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::JAL)) + return relaxJAL(Instruction, Offset, Function, Target); + return false; + } +}; + +} // namespace + +const TargetBranchInfo &branchInfo() noexcept { + static const RISCVBranchInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp index 0e6fec9..e47a9c8 100644 --- a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -1124,8 +1124,43 @@ calleeSaveOperations(MachineBasicBlock &Block, const MachineFunction &Function, return Result; } +[[nodiscard]] MachineBasicBlock *blockWithID(MachineFunction &Function, + MachineBlockStableID ID) noexcept { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (Block->stableID() == ID) + return Block; + } + return nullptr; +} + +[[nodiscard]] MachineBasicBlock *frameSaveBlock(MachineFunction &Function) noexcept { + const auto &SavePoints = Function.frameInfo().state().SavePoints; + return SavePoints.empty() ? Function.firstBlock() + : blockWithID(Function, SavePoints.front().Block); +} + +[[nodiscard]] bool isFrameRestoreBlock(const MachineFunction &Function, + const MachineBasicBlock &Block) noexcept { + const MachineFrameState &State = Function.frameInfo().state(); + if (State.RestorePoints.empty()) + return State.SavePoints.empty(); + return std::ranges::any_of(State.RestorePoints, + [&Block](const MachineFrameSaveRestorePoint &Point) { + return Point.Block == Block.stableID(); + }); +} + class RISCVFrameLowering final : public TargetFrameLowering { public: + [[nodiscard]] bool + isCalleeSavedRegister(PhysicalRegister Reg, const MachineFunction &, + const TargetDescription &Target) const noexcept override { + return std::ranges::any_of(saveOrder(), [&](PhysicalRegisterID ID) { + const auto Candidate = physical(ID); + return Candidate && Target.registers().regsOverlap(Reg, *Candidate); + }); + } + [[nodiscard]] Expected prepareFrame(MachineFunction &Function, const TargetDescription &Target) const override { MachineFrameInfo &Frame = Function.frameInfo(); @@ -1133,6 +1168,10 @@ public: return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V frame layout is already frozen"); MachineFrameState State = Frame.state(); + const std::vector RequestedSavePoints = State.SavePoints; + const std::vector RequestedRestorePoints = State.RestorePoints; + const bool HasRequestedPlacement = + !RequestedSavePoints.empty() || !RequestedRestorePoints.empty(); for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; @@ -1249,25 +1288,32 @@ public: State.IsCalleeSavedInfoValid = true; State.SavePoints.clear(); State.RestorePoints.clear(); - if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { - State.SavePoints.push_back( - MachineFrameSaveRestorePoint{Function.firstBlock()->stableID(), *SavedRegisters}); - } - for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; - Block = Block->next()) { - bool HasEpilogue = false; - for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; - Instruction = Instruction->next()) { - HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + std::vector RestoredRegisters; + std::ranges::copy_if( + *SavedRegisters, std::back_inserter(RestoredRegisters), + [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); + if (HasRequestedPlacement) { + if (RequestedSavePoints.size() != 1 || RequestedRestorePoints.empty()) { + return makeUnexpected( + ErrorCode::VerificationError, + "RISC-V shrink-wrapped frame requires one save point and at least one restore point"); } - if (HasEpilogue && !SavedRegisters->empty()) { - std::vector Restored; - std::ranges::copy_if( - *SavedRegisters, std::back_inserter(Restored), - [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); - if (!Restored.empty()) { - State.RestorePoints.push_back( - MachineFrameSaveRestorePoint{Block->stableID(), std::move(Restored)}); + State.SavePoints.push_back({RequestedSavePoints.front().Block, *SavedRegisters}); + for (const MachineFrameSaveRestorePoint &Point : RequestedRestorePoints) + State.RestorePoints.push_back({Point.Block, RestoredRegisters}); + } else { + if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { + State.SavePoints.push_back({Function.firstBlock()->stableID(), *SavedRegisters}); + } + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + bool HasEpilogue = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + } + if (HasEpilogue && !RestoredRegisters.empty()) { + State.RestorePoints.push_back({Block->stableID(), RestoredRegisters}); } } } @@ -1290,10 +1336,10 @@ public: const std::span Saved = Function.records().calleeSavedRegisters(); if (Saved.empty()) return {}; - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) { + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) { return makeUnexpected(ErrorCode::VerificationError, - "RISC-V callee-save frame has no entry block"); + "RISC-V callee-save frame has no valid save block"); } auto FP = physical(PhysicalRegisterID::X8); auto BP = physical(PhysicalRegisterID::X9); @@ -1303,9 +1349,9 @@ public: const bool Realign = requiresStackRealignment(Function); MachineIRBuilder SaveBuilder(Function, Target); - auto SavePoint = Entry->firstInstruction() != nullptr - ? SaveBuilder.setInsertPointBefore(*Entry->firstInstruction()) - : SaveBuilder.setInsertPoint(*Entry); + auto SavePoint = SaveBlock->firstInstruction() != nullptr + ? SaveBuilder.setInsertPointBefore(*SaveBlock->firstInstruction()) + : SaveBuilder.setInsertPoint(*SaveBlock); if (!SavePoint) return std::unexpected(SavePoint.error()); for (const PhysicalRegister Reg : Saved) { @@ -1322,6 +1368,8 @@ public: for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!isFrameRestoreBlock(Function, *Block)) + continue; std::vector Terminators; for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; Instruction = Instruction->next()) { @@ -1433,21 +1481,22 @@ public: [[nodiscard]] Expected emitPrologue(MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) return Function.frameInfo().state().StackSize == 0 ? Expected{} : makeUnexpected(ErrorCode::VerificationError, - "RISC-V nonempty frame has no entry block"); - const std::vector Saves = calleeSaveOperations(*Entry, Function, false); + "RISC-V nonempty frame has no save block"); + const std::vector Saves = + calleeSaveOperations(*SaveBlock, Function, false); if (Saves.size() != Function.records().calleeSavedRegisters().size()) { return makeUnexpected(ErrorCode::InternalError, "RISC-V prologue has an incomplete callee-save sequence"); } - MachineInstr *Anchor = Entry->firstInstruction(); + MachineInstr *Anchor = SaveBlock->firstInstruction(); MachineIRBuilder Builder(Function, Target); - auto Point = - Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) : Builder.setInsertPoint(*Entry); + auto Point = Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) + : Builder.setInsertPoint(*SaveBlock); if (!Point) return std::unexpected(Point.error()); @@ -1637,6 +1686,8 @@ public: MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { + if (!isFrameRestoreBlock(Function, Block)) + return {}; auto FP = physical(PhysicalRegisterID::X8); auto SP = physical(PhysicalRegisterID::X2); auto BP = physical(PhysicalRegisterID::X9); diff --git a/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp new file mode 100644 index 0000000..5c22dde --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr MachineCombinerStageMask allOptimizationStages() noexcept { + return machineCombinerStageMask(MachineCombinerStage::SelectedSSA) | + machineCombinerStageMask(MachineCombinerStage::PreRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostFrame); +} + +[[nodiscard]] MachineCombinerPattern immediateIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::int64_t Immediate) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::ImmediateEquals, 2, 0, Immediate}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] MachineCombinerPattern zeroRegisterIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::size_t ZeroOperand, + std::size_t SourceOperand) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::PhysicalRegisterEquals, ZeroOperand, 0, + static_cast(PhysicalRegisterID::X0)}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, SourceOperand}, + }; +} + +[[nodiscard]] MachineCombinerPattern equalSourceIdentity(InstructionOpcode OpcodeValue, + std::string Name) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::RegisterEquals, 1, 2, 0}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] std::vector createPatterns() { + std::vector Patterns; + Patterns.push_back(immediateIdentity(InstructionOpcode::ADDI, "addi-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::XORI, "xori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ORI, "ori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ANDI, "andi-all-ones", -1)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SLLI, "slli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRLI, "srli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRAI, "srai-zero", 0)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::SUB, "sub-zero-right", 2, 1)); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_S, "fsgnj-s-copy")); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_D, "fsgnj-d-copy")); + return Patterns; +} + +} // namespace + +const TargetMachineCombinerInfo &machineCombinerInfo() noexcept { + static const Expected Info = + TargetMachineCombinerInfo::create(createPatterns()); + if (!Info) + std::abort(); + return *Info; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp new file mode 100644 index 0000000..2cc5f24 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +#include +#include + +namespace rcc::riscv { +namespace { + +class RISCVMachineMemoryFoldingInfo final : public TargetMemoryFoldingInfo { +public: + [[nodiscard]] Expected canFoldMemoryOperand(const MachineInstr &, std::size_t, + const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected + foldMemoryOperand(MachineInstr &, std::size_t, MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC has no register instruction form with a folded memory operand"); + } + + [[nodiscard]] Expected unfoldMemoryOperand(MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected> + getFoldedMemorySize(const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return std::optional{}; + } +}; + +} // namespace + +const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept { + static const RISCVMachineMemoryFoldingInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp new file mode 100644 index 0000000..2f8c5d1 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +namespace rcc::riscv { + +MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept { + return { + .SSA = ssaInfo(), + .Combiner = machineCombinerInfo(), + .MemoryFolding = memoryFoldingInfo(), + .FrameLowering = frameLowering(), + .PseudoExpansion = pseudoExpansionInfo(), + .SchedulingModel = schedulingModel(), + .BranchInfo = branchInfo(), + }; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp new file mode 100644 index 0000000..ed73794 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; +constexpr std::uint32_t CopyF32Flag = 8; +constexpr std::uint32_t CopyF64Flag = 16; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] std::optional +registerBank(Register Reg, const MachineFunction &Function, const TargetDescription &Target) { + if (Reg.isVirtual()) { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Bank : std::nullopt; + } + const std::optional Physical = Reg.physicalRegister(); + if (!Physical) + return std::nullopt; + for (const RegisterBankDesc &Bank : Target.registers().registerBanks()) { + if (Target.registers().bankContainsRegister(Bank.ID, *Physical)) + return Bank.ID; + } + return std::nullopt; +} + +[[nodiscard]] std::optional registerType(Register Reg, const MachineFunction &Function) { + if (!Reg.isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Type : std::nullopt; +} + +[[nodiscard]] Expected copyLocation(const MachineInstr &Source, MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyCommonDecoration(const MachineInstr &Source, + MachineInstr &Destination, bool CopyPreSymbol, + bool CopyPostSymbol, bool CopyDebugNumber) { + auto Location = copyLocation(Source, Destination); + if (!Location) + return std::unexpected(Location.error()); + Destination.setFlags(Source.flags()); + if (CopyDebugNumber && Source.debugInstrNumber() != 0) { + auto Number = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!Number) + return std::unexpected(Number.error()); + } + if (CopyPreSymbol) { + if (const auto Symbol = Source.preInstrSymbol()) { + auto Set = Destination.setPreInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (CopyPostSymbol) { + if (const auto Symbol = Source.postInstrSymbol()) { + auto Set = Destination.setPostInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (Source.heapAllocMarker()) { + auto Set = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.pcSections()) { + auto Set = Destination.setPCSections(*Source.pcSections()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.mmra()) { + auto Set = Destination.setMMRA(*Source.mmra()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.cfiType()) { + auto Set = Destination.setCFIType(*Source.cfiType()); + if (!Set) + return std::unexpected(Set.error()); + } + if (const auto Symbol = Source.deactivationSymbol()) { + auto Set = Destination.setDeactivationSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + return {}; +} + +[[nodiscard]] Expected removeOldInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "pseudo instruction is not linked"); + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected> +copyRegisters(const MachineInstr &Instruction) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA copy pseudo requires two register operands"); + return std::pair{*Destination, *Source}; +} + +[[nodiscard]] Expected prepareCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY)) + return false; + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const auto DestinationBank = registerBank(Registers->first, Function, Target); + const auto SourceBank = registerBank(Registers->second, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V copy preparation requires constrained register banks"); + if (*DestinationBank == *SourceBank) + return false; + + const auto DestinationType = registerType(Registers->first, Function); + const auto SourceType = registerType(Registers->second, Function); + const std::optional Type = DestinationType ? DestinationType : SourceType; + if (!Type || Type->isVector() || (Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V cross-bank COPY requires a scalar 32-bit or 64-bit type"); + } + if (DestinationType && SourceType && + DestinationType->scalarBitWidth() != SourceType->scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V cross-bank COPY has mismatched value widths"); + } + const std::uint32_t Flag = Type->scalarBitWidth() == 32 ? CopyF32Flag : CopyF64Flag; + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const bool Changed = + DestinationOperand->targetFlags() != Flag || SourceOperand->targetFlags() != Flag; + DestinationOperand->setTargetFlags(Flag); + SourceOperand->setTargetFlags(Flag); + return Changed; +} + +[[nodiscard]] Expected expandCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const Register Destination = Registers->first; + const Register Source = Registers->second; + if (!Destination.isPhysical() || !Source.isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY retains a virtual register"); + } + if (Destination == Source) { + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + const auto DestinationBank = registerBank(Destination, Function, Target); + const auto SourceBank = registerBank(Source, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY references an unknown register bank"); + const RegisterBankID GPR = + RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); + const RegisterBankID FPR = + RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); + + InstructionOpcode Selected = InstructionOpcode::Invalid; + bool DuplicateSource = false; + bool AddZeroImmediate = false; + if (*DestinationBank == GPR && *SourceBank == GPR) { + Selected = InstructionOpcode::ADDI; + AddZeroImmediate = true; + } else if (*DestinationBank == FPR && *SourceBank == FPR) { + Selected = InstructionOpcode::FSGNJ_D; + DuplicateSource = true; + } else { + const std::uint32_t DestinationFlags = Instruction.operandAt(0)->targetFlags(); + const std::uint32_t SourceFlags = Instruction.operandAt(1)->targetFlags(); + if (DestinationFlags != SourceFlags || + (DestinationFlags != CopyF32Flag && DestinationFlags != CopyF64Flag)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "RISC-V cross-bank COPY has no prepared transfer width"); + } + if (*DestinationBank == GPR && *SourceBank == FPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; + } else if (*DestinationBank == FPR && *SourceBank == GPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + } + } + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V COPY crosses unsupported banks"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Selected)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Source); + if (!Use) + return std::unexpected(Use.error()); + if (DuplicateSource) { + auto SecondUse = Pending->addUse(Source); + if (!SecondUse) + return std::unexpected(SecondUse.error()); + } + if (AddZeroImmediate) { + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + } + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandSubregisterCopy(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const Opcode Value = Instruction.opcode(); + if (Value == opcode(GenericOpcode::EXTRACT_SUBREG) || + Value == opcode(GenericOpcode::SUBREG_TO_REG)) { + const MachineOperand *Index = Instruction.operandAt(2); + const auto *Immediate = Index == nullptr ? nullptr : Index->getIf(); + if (Immediate == nullptr || *Immediate != 0) + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC does not support nonzero subregister indices"); + } else if (Value == opcode(GenericOpcode::COPY_LANEMASK)) { + const MachineOperand *MaskOperand = Instruction.operandAt(2); + const auto *Mask = MaskOperand == nullptr ? nullptr : MaskOperand->getIf(); + if (Mask == nullptr || Mask->Mask.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V lane copy requires a nonempty lane mask"); + } else { + return false; + } + return expandCopy(Instruction, Function, Target); +} + +[[nodiscard]] Expected addDirectTarget(MachineInstrBuilder &Builder, + const MachineOperand &Callee) { + if (const auto *Global = Callee.getIf()) + return Builder.addGlobalSymbol(Global->Name, Global->Offset, PCRelativeHighFlag); + if (const auto *External = Callee.getIf()) + return Builder.addExternalSymbol(External->Name, External->Offset, PCRelativeHighFlag); + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct call pseudo has no symbol target"); +} + +[[nodiscard]] bool isRegister(const MachineOperand &Operand, PhysicalRegister Reg) noexcept { + const Register *Value = Operand.getIf(); + return Value != nullptr && Value->physicalRegister() == Reg; +} + +[[nodiscard]] Expected copyExtraImplicitOperands( + const MachineInstr &Source, std::size_t FirstImplicit, MachineInstrBuilder &Destination, + std::optional ExplicitDef, std::optional ExplicitUse) { + for (std::size_t Index = FirstImplicit; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + if (Operand == nullptr || !Operand->isImplicit()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed implicit operands"); + if ((ExplicitDef && Operand->isDef() && isRegister(*Operand, *ExplicitDef)) || + (ExplicitUse && Operand->isUse() && isRegister(*Operand, *ExplicitUse))) { + continue; + } + auto Added = Destination.addImplicitOperand(Operand->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] Expected expandCall(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, bool IsTail) { + MachineBasicBlock *Block = Instruction.parent(); + const MachineOperand *Callee = Instruction.operandAt(0); + const MachineOperand *CallSite = Instruction.operandAt(1); + const MachineOperand *Mask = Instruction.operandAt(2); + if (Block == nullptr || Callee == nullptr || CallSite == nullptr || Mask == nullptr || + Mask->kind() != MachineOperand::Kind::RegisterMask) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed explicit operands"); + } + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + auto Scratch = physical(PhysicalRegisterID::X6); + if (!Zero || !Link || !Scratch) + return makeUnexpected(ErrorCode::InternalError, "RISC-V call registers are unavailable"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + Register Base; + std::optional Anchor; + const bool Direct = Callee->kind() == MachineOperand::Kind::GlobalSymbol || + Callee->kind() == MachineOperand::Kind::ExternalSymbol; + if (Direct) { + const PhysicalRegister HighRegister = IsTail ? *Scratch : *Link; + Base = Register::fromPhysical(HighRegister); + Anchor = Instruction.preInstrSymbol() + ? std::optional{std::string(*Instruction.preInstrSymbol())} + : std::optional{".Lrcc_call_" + + std::to_string(Function.stableID().value()) + "_" + + std::to_string(Instruction.stableID().value())}; + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Base); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = addDirectTarget(*HighPending, *Callee); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto Symbol = (*High)->setPreInstrSymbol(*Anchor); + if (!Symbol) + return std::unexpected(Symbol.error()); + auto Decoration = copyLocation(Instruction, **High); + if (!Decoration) + return std::unexpected(Decoration.error()); + } else { + const Register *CalleeRegister = Callee->getIf(); + if (CalleeRegister == nullptr || !CalleeRegister->isPhysical()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V indirect call retains a virtual callee register"); + Base = *CalleeRegister; + } + + const InstructionOpcode LowOpcode = + IsTail ? InstructionOpcode::JALR_TAIL : InstructionOpcode::JALR_CALL; + auto LowPending = Builder.buildInstr(targetOpcode(LowOpcode)); + if (!LowPending) + return std::unexpected(LowPending.error()); + const PhysicalRegister DestinationRegister = IsTail ? *Zero : *Link; + auto LowDef = LowPending->addDef(Register::fromPhysical(DestinationRegister)); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Base); + if (!LowBase) + return std::unexpected(LowBase.error()); + Expected LowOffset = + Anchor ? LowPending->addMCSymbol(*Anchor, 0, PCRelativeLowFlag) : LowPending->addImmediate(0); + if (!LowOffset) + return std::unexpected(LowOffset.error()); + auto LowCallSite = LowPending->addOperand(CallSite->cloneDetached()); + if (!LowCallSite) + return std::unexpected(LowCallSite.error()); + auto LowMask = LowPending->addOperand(Mask->cloneDetached()); + if (!LowMask) + return std::unexpected(LowMask.error()); + auto Implicit = copyExtraImplicitOperands( + Instruction, 3, *LowPending, IsTail ? std::nullopt : std::optional{*Link}, + std::nullopt); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto Decoration = copyCommonDecoration(Instruction, **Low, !Direct, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Records = + Function.records().replaceInstructionReference(Instruction.stableID(), (*Low)->stableID()); + if (!Records) + return std::unexpected(Records.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandReturn(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + if (!Zero || !Link) + return makeUnexpected(ErrorCode::InternalError, "RISC-V return registers are unavailable"); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR_RET)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Register::fromPhysical(*Link)); + if (!Use) + return std::unexpected(Use.error()); + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto Implicit = copyExtraImplicitOperands(Instruction, 0, *Pending, std::nullopt, *Link); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVPseudoExpansionInfo final : public TargetPseudoExpansionInfo { +public: + [[nodiscard]] Expected preparePostRAPseudo(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const override { + return prepareCopy(Instruction, Function, Target); + } + + [[nodiscard]] Expected expandPseudo(MachineInstr &Instruction, + MachinePseudoExpansionPhase Phase, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (Phase != MachinePseudoExpansionPhase::PostRA) + return false; + if (Instruction.opcode() == opcode(GenericOpcode::COPY)) + return expandCopy(Instruction, Function, Target); + auto Subregister = expandSubregisterCopy(Instruction, Function, Target); + if (!Subregister || *Subregister) + return Subregister; + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoCALL)) + return expandCall(Instruction, Function, Target, false); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoTAIL)) + return expandCall(Instruction, Function, Target, true); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoRET)) + return expandReturn(Instruction, Function, Target); + return false; + } +}; + +} // namespace + +const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept { + static const RISCVPseudoExpansionInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVScheduling.cpp b/backend/lib/Target/RISCV/RISCVScheduling.cpp new file mode 100644 index 0000000..8ef8d91 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVScheduling.cpp @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +enum class Resource : std::uint32_t { + Core = 1, + IntegerALU = 2, + Memory = 3, + Branch = 4, + MultiplyDivide = 5, + FloatingPoint = 6, + System = 7, +}; + +enum class PressureSet : std::uint32_t { + GPR = 1, + FPR = 2, +}; + +[[nodiscard]] constexpr ProcessorResourceID resource(Resource Value) noexcept { + return ProcessorResourceID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterPressureSetID pressureSet(PressureSet Value) noexcept { + return RegisterPressureSetID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterClassID registerClass(RegisterClass Value) noexcept { + return RegisterClassID::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::vector resourceUses(SchedulingClass Class) { + switch (Class) { + case SchedulingClass::ALU: + case SchedulingClass::COMPRESSED: + return {{resource(Resource::IntegerALU), 1}}; + case SchedulingClass::LOAD: + case SchedulingClass::STORE: + return {{resource(Resource::Memory), 1}}; + case SchedulingClass::BRANCH: + return {{resource(Resource::Branch), 1}}; + case SchedulingClass::SYSTEM: + case SchedulingClass::FENCE: + return {{resource(Resource::System), 1}}; + case SchedulingClass::MUL: + return {{resource(Resource::MultiplyDivide), 3}}; + case SchedulingClass::DIV: + return {{resource(Resource::MultiplyDivide), 20}}; + case SchedulingClass::ATOMIC: + return {{resource(Resource::Memory), 4}, {resource(Resource::System), 1}}; + case SchedulingClass::FP_ADD: + case SchedulingClass::FP_MUL: + case SchedulingClass::FP_CONVERT: + return {{resource(Resource::FloatingPoint), 4}}; + case SchedulingClass::FP_DIV: + case SchedulingClass::FP_SQRT: + return {{resource(Resource::FloatingPoint), 20}}; + case SchedulingClass::None: + return {}; + } + return {}; +} + +[[nodiscard]] TargetSchedulingModelDesc createDescription() { + TargetSchedulingModelDesc Description; + Description.IssueWidth = 2; + Description.DispatchLimit = 2; + Description.ProcessorResources = { + {resource(Resource::Core), "core", 2, std::nullopt}, + {resource(Resource::IntegerALU), "integer-alu", 2, resource(Resource::Core)}, + {resource(Resource::Memory), "memory", 1, resource(Resource::Core)}, + {resource(Resource::Branch), "branch", 1, resource(Resource::Core)}, + {resource(Resource::MultiplyDivide), "multiply-divide", 1, resource(Resource::Core)}, + {resource(Resource::FloatingPoint), "floating-point", 1, resource(Resource::Core)}, + {resource(Resource::System), "system", 1, resource(Resource::Core)}, + }; + for (const SchedulingClassInfo &Class : schedulingClasses()) { + SchedulingClassModelDesc Model{ + .ID = Class.ID, + .Name = std::string(Class.Name), + .MicroOps = Class.MicroOps, + .Latency = Class.Latency, + .Resources = resourceUses(static_cast(Class.ID.value())), + .WriteLatencies = {}, + .ReadAdvances = {}, + }; + Model.WriteLatencies.push_back({0, Class.Latency}); + switch (static_cast(Class.ID.value())) { + case SchedulingClass::ALU: + Model.ReadAdvances = { + {1, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + {2, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + }; + break; + default: + break; + } + Description.SchedulingClasses.push_back(std::move(Model)); + } + Description.PressureSets = { + {pressureSet(PressureSet::GPR), + "gpr", + {registerClass(RegisterClass::GPR), registerClass(RegisterClass::GPRC), + registerClass(RegisterClass::GPR_NX0), registerClass(RegisterClass::GPR_NX0_NX2)}, + 27}, + {pressureSet(PressureSet::FPR), + "fpr", + {registerClass(RegisterClass::FPR64), registerClass(RegisterClass::FPR64C), + registerClass(RegisterClass::FPR32), registerClass(RegisterClass::FPR32C)}, + 32}, + }; + return Description; +} + +} // namespace + +const TargetSchedulingModel &schedulingModel() noexcept { + static const Expected Model = + TargetSchedulingModel::create(createDescription()); + if (!Model) + std::abort(); + return *Model; +} + +} // namespace rcc::riscv diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt index 65d9168..1b66194 100644 --- a/backend/unittests/CodeGen/CMakeLists.txt +++ b/backend/unittests/CodeGen/CMakeLists.txt @@ -20,8 +20,10 @@ set( MachineLivenessTest.cpp MachineMetadataTest.cpp MachineOperandTest.cpp + MachineOptimizationTest.cpp MachinePassTest.cpp MachinePipelinePassesTest.cpp + MachineSchedulingTest.cpp PBQPTest.cpp RegisterAllocationTest.cpp MachineSSAExitTest.cpp @@ -355,6 +357,22 @@ target_link_libraries( add_test(NAME RCCBackend.CodeGen.MachineAnalyses COMMAND RCCBackendCodeGenMachineAnalysesTests) +add_executable(RCCBackendCodeGenMachineOptimizationTests MachineOptimizationTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenMachineOptimizationTests) +target_link_libraries( + RCCBackendCodeGenMachineOptimizationTests + PRIVATE RCCBackend::CodeGen RCCBackend::RISCV +) +add_test(NAME RCCBackend.CodeGen.MachineOptimization COMMAND RCCBackendCodeGenMachineOptimizationTests) + +add_executable(RCCBackendCodeGenMachineSchedulingTests MachineSchedulingTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenMachineSchedulingTests) +target_link_libraries( + RCCBackendCodeGenMachineSchedulingTests + PRIVATE RCCBackend::CodeGen RCCBackend::RISCV +) +add_test(NAME RCCBackend.CodeGen.MachineScheduling COMMAND RCCBackendCodeGenMachineSchedulingTests) + add_executable(RCCBackendCodeGenMachineSSAExitTests MachineSSAExitTest.cpp) rcc_backend_configure_target(RCCBackendCodeGenMachineSSAExitTests) target_link_libraries( @@ -439,8 +457,10 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineLivenessTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulingTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp" @@ -479,8 +499,10 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineLivenessTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulingTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp" diff --git a/backend/unittests/CodeGen/InstructionDescriptorTest.cpp b/backend/unittests/CodeGen/InstructionDescriptorTest.cpp index 424d2e4..687e723 100644 --- a/backend/unittests/CodeGen/InstructionDescriptorTest.cpp +++ b/backend/unittests/CodeGen/InstructionDescriptorTest.cpp @@ -86,6 +86,7 @@ private: RegisterOrImmediate.Kinds = { rcc::MachineOperand::Kind::Register, rcc::MachineOperand::Kind::Immediate, + rcc::MachineOperand::Kind::GlobalSymbol, }; RegisterOrImmediate.RegisterRole = rcc::RegisterOperandRole::Use; RegisterOrImmediate.ImmediatePredicate = rcc::ImmediatePredicateID::fromValue(1); @@ -218,6 +219,15 @@ void testOperandValidation(TestContext &Context) { .has_value(), "matching roles, constraints, flags, and predicate must validate"); + std::vector SymbolOperands; + SymbolOperands.push_back(*registerOperand(*Destination, {rcc::RegisterOperandFlag::Define})); + SymbolOperands.push_back(*registerOperand(*Source, {rcc::RegisterOperandFlag::Use})); + SymbolOperands.push_back(*rcc::MachineOperand::globalSymbol("object", 0, 0x1)); + Context.expect( + InstructionSet->validateExplicitOperands(rcc::Opcode::fromValue(1), SymbolOperands, RegInfo) + .has_value(), + "an immediate predicate must not reject another allowed symbolic operand kind"); + std::vector WrongRole; WrongRole.push_back(*registerOperand(*Destination, {rcc::RegisterOperandFlag::Use})); WrongRole.push_back(*registerOperand(*Source, {rcc::RegisterOperandFlag::Use})); diff --git a/backend/unittests/CodeGen/MachineOperandTest.cpp b/backend/unittests/CodeGen/MachineOperandTest.cpp index 1b246ca..3cc7060 100644 --- a/backend/unittests/CodeGen/MachineOperandTest.cpp +++ b/backend/unittests/CodeGen/MachineOperandTest.cpp @@ -264,6 +264,48 @@ void testOperandFactories(TestContext &Context) { "detached operand cloning must preserve the new payload kinds"); } +void testFrameIndexRewrite(TestContext &Context) { + auto Function = createFunction(Context, 5, "frame-index-rewrite"); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + auto Align = rcc::Alignment::create(8); + if (Block == nullptr || !Align) + return; + + rcc::StackObjectDesc Description; + Description.Kind = rcc::StackObjectKind::SpillSlot; + Description.Size = 8; + Description.Align = *Align; + auto First = Function->frameInfo().createObject(Description); + auto Second = Function->frameInfo().createObject(Description); + Context.expect(First.has_value() && Second.has_value(), + "frame-index rewrite objects must be created"); + if (!First || !Second) + return; + + auto Instruction = Function->createInstruction(rcc::Opcode::fromValue(OperandOpcode)); + Context.expect(Instruction.has_value() && + (*Instruction)->addOperand(rcc::MachineOperand::frameIndex(*First, 4)) && + Block->appendInstruction(**Instruction), + "frame-index rewrite operand must be linked to its function"); + if (!Instruction || (*Instruction)->operandCount() == 0) + return; + rcc::MachineOperand *Operand = (*Instruction)->operandAt(0); + Context.expect(Operand != nullptr && Operand->setFrameIndex(*Second).has_value() && + Operand->getIf()->Index == *Second && + Operand->getIf()->Offset == 4, + "setFrameIndex must rewrite only the object identity"); + + const rcc::FrameIndex Unknown = rcc::FrameIndex::fromValue(99); + Context.expect(Operand != nullptr && !Operand->setFrameIndex(Unknown) && + Operand->getIf()->Index == *Second, + "setFrameIndex must reject an unknown attached object atomically"); + auto Immediate = rcc::MachineOperand::immediate(0); + Context.expect(!Immediate.setFrameIndex(*First), + "setFrameIndex must reject non-frame-index operands"); +} + void testCFIInfo(TestContext &Context) { const auto R1 = rcc::DwarfRegisterNumber::fromValue(1); const auto R2 = rcc::DwarfRegisterNumber::fromValue(2); @@ -476,6 +518,7 @@ int main() { TestContext Context; testIntegerImmediate(Context); testOperandFactories(Context); + testFrameIndexRewrite(Context); testCFIInfo(Context); testBuilderVerifierAndClone(Context); return Context.result(); diff --git a/backend/unittests/CodeGen/MachineOptimizationTest.cpp b/backend/unittests/CodeGen/MachineOptimizationTest.cpp new file mode 100644 index 0000000..8d01a65 --- /dev/null +++ b/backend/unittests/CodeGen/MachineOptimizationTest.cpp @@ -0,0 +1,772 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineCSE.h" +#include "rcc/CodeGen/MachineCombiner.h" +#include "rcc/CodeGen/MachineDCE.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLICM.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineSink.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +struct Fixture final { + std::unique_ptr Function; + rcc::MachineBasicBlock *Entry = nullptr; +}; + +[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept; + +class TestMemoryFoldingInfo final : public rcc::TargetMemoryFoldingInfo { +public: + explicit TestMemoryFoldingInfo(bool DropMemory = false) noexcept : DropMemory(DropMemory) {} + + [[nodiscard]] rcc::Expected + canFoldMemoryOperand(const rcc::MachineInstr &Consumer, std::size_t OperandIndex, + const rcc::MachineInstr &MemoryInstruction, const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return Consumer.opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) && OperandIndex == 1 && + MemoryInstruction.opcode() == opcode(rcc::riscv::InstructionOpcode::LD); + } + + [[nodiscard]] rcc::Expected + foldMemoryOperand(rcc::MachineInstr &Consumer, std::size_t, rcc::MachineInstr &MemoryInstruction, + rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) const override { + const rcc::MachineOperand *DestinationOperand = Consumer.operandAt(0); + const rcc::MachineOperand *BaseOperand = MemoryInstruction.operandAt(1); + const rcc::MachineOperand *OffsetOperand = MemoryInstruction.operandAt(2); + const rcc::Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const rcc::Register *Base = + BaseOperand == nullptr ? nullptr : BaseOperand->getIf(); + const std::int64_t *Offset = + OffsetOperand == nullptr ? nullptr : OffsetOperand->getIf(); + rcc::MachineBasicBlock *Block = Consumer.parent(); + if (Destination == nullptr || Base == nullptr || Offset == nullptr || Block == nullptr || + MemoryInstruction.parent() != Block) { + return rcc::makeUnexpected(rcc::ErrorCode::VerificationError, + "test memory fold received an invalid candidate"); + } + + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Consumer); !Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::LD)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(*Base); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(*Offset); !Added) + return std::unexpected(Added.error()); + for (const rcc::MachineMemOperand *Memory : MemoryInstruction.memOperands()) { + if (Memory == nullptr) + return rcc::makeUnexpected(rcc::ErrorCode::VerificationError, + "test memory fold received a null MMO"); + if (auto Added = Pending->addMemOperand(*Memory); !Added) + return std::unexpected(Added.error()); + } + auto Folded = Pending->build(); + if (!Folded) + return std::unexpected(Folded.error()); + if (DropMemory) { + while (!(*Folded)->memOperands().empty()) { + if (auto Erased = (*Folded)->eraseMemOperand(0); !Erased) + return std::unexpected(Erased.error()); + } + } + if (auto Removed = Block->removeInstruction(MemoryInstruction); !Removed) + return std::unexpected(Removed.error()); + if (auto Removed = Block->removeInstruction(Consumer); !Removed) + return std::unexpected(Removed.error()); + return *Folded; + } + + [[nodiscard]] rcc::Expected + unfoldMemoryOperand(rcc::MachineInstr &, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return false; + } + + [[nodiscard]] rcc::Expected> + getFoldedMemorySize(const rcc::MachineInstr &, const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + auto Size = rcc::MemorySize::fixed(8); + if (!Size) + return std::unexpected(Size.error()); + return std::optional(*Size); + } + +private: + bool DropMemory; +}; + +[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept { + return rcc::Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] Fixture makeFixture(TestContext &Context, std::string Name, std::uint64_t ID) { + auto Function = + rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID)); + Context.expect(Function.has_value(), "optimization fixture function creation must succeed"); + if (!Function) + return {}; + auto Entry = (*Function)->createBasicBlock(); + Context.expect(Entry.has_value(), "optimization fixture block creation must succeed"); + if (!Entry || !(*Function)->appendBlock(**Entry)) + return {}; + (*Function)->properties().set(rcc::MachineFunctionProperty::IsSSA); + (*Function)->properties().set(rcc::MachineFunctionProperty::Legalized); + (*Function)->properties().set(rcc::MachineFunctionProperty::RegBankSelected); + (*Function)->properties().set(rcc::MachineFunctionProperty::Selected); + return {std::move(*Function), *Entry}; +} + +void setPostRAProperties(rcc::MachineFunction &Function) { + Function.properties().clear(rcc::MachineFunctionProperty::IsSSA); + Function.properties().set(rcc::MachineFunctionProperty::NoPHIs); + Function.properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten); + Function.properties().set(rcc::MachineFunctionProperty::NoVRegs); + Function.properties().set(rcc::MachineFunctionProperty::TracksLiveness); +} + +[[nodiscard]] rcc::Register makeGPR(TestContext &Context, rcc::MachineFunction &Function) { + auto Type = rcc::LLT::makeInteger(64); + auto Reg = Type ? Function.registerInfo().createVirtualRegister(*Type) + : rcc::Expected(std::unexpected( + rcc::Error(rcc::ErrorCode::InternalError, "test LLT creation failed"))); + Context.expect(Reg.has_value(), "optimization virtual register creation must succeed"); + if (!Reg) + return {}; + const auto Bank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + const auto Class = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::GPR)); + Context.expect(Function.registerInfo().setRegisterBank(*Reg, Bank).has_value() && + Function.registerInfo().setRegisterClass(*Reg, Class).has_value(), + "optimization register constraints must be assigned"); + return *Reg; +} + +[[nodiscard]] rcc::PhysicalRegister physical(TestContext &Context, + rcc::riscv::PhysicalRegisterID ID) { + auto Reg = rcc::riscv::physicalRegister(ID); + Context.expect(Reg.has_value(), "optimization physical register lookup must succeed"); + return Reg.value_or(rcc::PhysicalRegister{}); +} + +[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Source, + std::int64_t Immediate) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) || + !Pending->addImmediate(Immediate)) { + Context.expect(false, "ADDI construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "ADDI insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr *buildADD(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Left, + rcc::Register Right) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADD)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Left) || + !Pending->addUse(Right)) { + Context.expect(false, "ADD construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "ADD insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] const rcc::MachineMemOperand * +makeMemory(TestContext &Context, rcc::MachineFunction &Function, bool Store, bool Invariant) { + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + Context.expect(Size.has_value() && Align.has_value(), "memory shape creation must succeed"); + if (!Size || !Align) + return nullptr; + rcc::MemoryAccessFlags Flags{Store ? rcc::MemoryAccessFlag::Store : rcc::MemoryAccessFlag::Load}; + if (Invariant) { + Flags.set(rcc::MemoryAccessFlag::Invariant); + Flags.set(rcc::MemoryAccessFlag::Dereferenceable); + } + rcc::MachineMemOperandDesc Description; + Description.Accesses = Flags; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = Invariant ? rcc::MemoryBase::constantPoolPseudo() : rcc::MemoryBase::stack(); + auto Memory = Function.createMemOperand(std::move(Description)); + Context.expect(Memory.has_value(), "memory operand creation must succeed"); + return Memory.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr *buildStore(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Value, rcc::PhysicalRegister StackPointer, + const rcc::MachineMemOperand &Memory) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::SD)); + if (!Pending || !Pending->addUse(Value) || + !Pending->addUse(rcc::Register::fromPhysical(StackPointer)) || !Pending->addImmediate(0) || + !Pending->addMemOperand(Memory)) { + Context.expect(false, "store construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "store insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr *buildLoad(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Address, + const rcc::MachineMemOperand &Memory) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::LD)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Address) || + !Pending->addImmediate(0) || !Pending->addMemOperand(Memory)) { + Context.expect(false, "load construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "load insertion must succeed"); + return Built.value_or(nullptr); +} + +void testDCEFixedPointAndDebugUses(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "dce", 1); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "DCE insert point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2); + + const rcc::Register LiveRoot = makeGPR(Context, *Fixture.Function); + const rcc::Register LiveValue = makeGPR(Context, *Fixture.Function); + const rcc::Register DeadRoot = makeGPR(Context, *Fixture.Function); + const rcc::Register DeadValue = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, LiveRoot, rcc::Register::fromPhysical(X0), 1); + (void)buildADDI(Context, Builder, LiveValue, LiveRoot, 2); + (void)buildADDI(Context, Builder, DeadRoot, rcc::Register::fromPhysical(X0), 3); + (void)buildADDI(Context, Builder, DeadValue, DeadRoot, 4); + + auto Debug = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::DBG_VALUE)); + Context.expect(Debug.has_value() && + Debug->addUse(DeadValue, {rcc::RegisterOperandFlag::Debug}).has_value() && + Debug->build().has_value(), + "debug-only use construction must succeed"); + const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, true, false); + if (Memory == nullptr) + return; + rcc::MachineInstr *Store = buildStore(Context, Builder, LiveValue, X2, *Memory); + const std::size_t Before = Fixture.Entry->instructionCount(); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineDCEPass Pass; + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine DCE failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "machine DCE must remove the debug-only dead chain"); + Context.expect(Fixture.Entry->instructionCount() + 2 == Before, + "machine DCE must reach a fixed point across the dead chain"); + Context.expect(Store != nullptr && Store->isLinked(), "machine DCE must preserve a store"); + Context.expect(!Fixture.Function->registerInfo().isKnownVirtualRegister(DeadRoot) && + !Fixture.Function->registerInfo().isKnownVirtualRegister(DeadValue), + "machine DCE must retire removed virtual registers"); + rcc::MachineInstr *DebugInstruction = nullptr; + for (rcc::MachineInstr *Instruction = Fixture.Entry->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->opcode() == rcc::opcode(rcc::GenericOpcode::DBG_VALUE)) + DebugInstruction = Instruction; + } + const rcc::MachineOperand *DebugOperand = + DebugInstruction == nullptr ? nullptr : DebugInstruction->operandAt(0); + Context.expect(DebugOperand != nullptr && DebugOperand->registerValue().has_value() && + DebugOperand->registerValue()->isNoRegister(), + "machine DCE must clear a dangling debug register use"); +} + +void testMachineCombinerIdentityRules(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "combiner", 6); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "combiner insert point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2); + const rcc::Register Seed = makeGPR(Context, *Fixture.Function); + const rcc::Register Identity = makeGPR(Context, *Fixture.Function); + const rcc::Register Consumer = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 7); + rcc::MachineInstr *IdentityInstruction = buildADDI(Context, Builder, Identity, Seed, 0); + rcc::MachineInstr *ConsumerInstruction = buildADD(Context, Builder, Consumer, Identity, Seed); + const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, true, false); + if (Memory == nullptr) + return; + (void)buildStore(Context, Builder, Consumer, X2, *Memory); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineCombinerPass Pass(rcc::riscv::machineCombinerInfo(), + rcc::MachineCombinerStage::SelectedSSA); + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine combiner failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "machine combiner must apply a registered RV64GC identity rule"); + Context.expect(IdentityInstruction != nullptr && !IdentityInstruction->isLinked() && + !Fixture.Function->registerInfo().isKnownVirtualRegister(Identity), + "machine combiner must erase the identity and retire its result"); + const rcc::MachineOperand *ConsumerUse = + ConsumerInstruction == nullptr ? nullptr : ConsumerInstruction->operandAt(1); + Context.expect(ConsumerUse != nullptr && ConsumerUse->registerValue() == Seed, + "machine combiner executor must rewrite all result uses"); +} + +void testPostRAMachineCombinerIdentity(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "post_ra_combiner", 7); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + Fixture.Function->properties().clear(rcc::MachineFunctionProperty::IsSSA); + Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoPHIs); + Fixture.Function->properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten); + Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoVRegs); + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "post-RA combiner insert point must be established"); + const rcc::PhysicalRegister X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10); + rcc::MachineInstr *Identity = buildADDI(Context, Builder, rcc::Register::fromPhysical(X10), + rcc::Register::fromPhysical(X10), 0); + rcc::MachineAnalysisManager Analyses; + rcc::MachineCombinerPass Pass(rcc::riscv::machineCombinerInfo(), + rcc::MachineCombinerStage::PostRegisterAllocation); + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + Context.expect( + Result.has_value() && Result->Changed && Identity != nullptr && !Identity->isLinked(), + "post-RA combiner must erase a physical-register no-op only when identities match"); +} + +void testCSERegisterAndMemoryRules(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "cse", 2); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "CSE insert point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + + const rcc::Register Seed = makeGPR(Context, *Fixture.Function); + const rcc::Register First = makeGPR(Context, *Fixture.Function); + const rcc::Register Redundant = makeGPR(Context, *Fixture.Function); + const rcc::Register Consumer = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 9); + rcc::MachineInstr *FirstInstruction = buildADDI(Context, Builder, First, Seed, 7); + rcc::MachineInstr *RedundantInstruction = buildADDI(Context, Builder, Redundant, Seed, 7); + rcc::MachineInstr *ConsumerInstruction = buildADD(Context, Builder, Consumer, Redundant, Seed); + + const rcc::Register MutableLoadOne = makeGPR(Context, *Fixture.Function); + const rcc::Register MutableLoadTwo = makeGPR(Context, *Fixture.Function); + const rcc::Register InvariantLoadOne = makeGPR(Context, *Fixture.Function); + const rcc::Register InvariantLoadTwo = makeGPR(Context, *Fixture.Function); + const rcc::MachineMemOperand *MutableMemory = + makeMemory(Context, *Fixture.Function, false, false); + const rcc::MachineMemOperand *InvariantMemory = + makeMemory(Context, *Fixture.Function, false, true); + if (MutableMemory == nullptr || InvariantMemory == nullptr) + return; + rcc::MachineInstr *MutableOne = buildLoad(Context, Builder, MutableLoadOne, Seed, *MutableMemory); + rcc::MachineInstr *MutableTwo = buildLoad(Context, Builder, MutableLoadTwo, Seed, *MutableMemory); + rcc::MachineInstr *InvariantOne = + buildLoad(Context, Builder, InvariantLoadOne, Seed, *InvariantMemory); + rcc::MachineInstr *InvariantTwo = + buildLoad(Context, Builder, InvariantLoadTwo, Seed, *InvariantMemory); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineCSEPass Pass; + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine CSE failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "machine CSE must eliminate safe redundant expressions"); + Context.expect(FirstInstruction != nullptr && FirstInstruction->isLinked() && + RedundantInstruction != nullptr && !RedundantInstruction->isLinked(), + "machine CSE must keep the dominating expression only"); + const rcc::MachineOperand *ConsumerUse = + ConsumerInstruction == nullptr ? nullptr : ConsumerInstruction->operandAt(1); + Context.expect(ConsumerUse != nullptr && ConsumerUse->registerValue() == First, + "machine CSE must rewrite uses to the dominating result"); + Context.expect(MutableOne != nullptr && MutableOne->isLinked() && MutableTwo != nullptr && + MutableTwo->isLinked(), + "machine CSE must not merge mutable loads"); + Context.expect(InvariantOne != nullptr && InvariantOne->isLinked() && InvariantTwo != nullptr && + !InvariantTwo->isLinked(), + "machine CSE must merge invariant dereferenceable loads"); + Context.expect(!Fixture.Function->registerInfo().isKnownVirtualRegister(Redundant) && + !Fixture.Function->registerInfo().isKnownVirtualRegister(InvariantLoadTwo), + "machine CSE must retire redundant result registers"); +} + +void testCSEBudgetPreservesLegalMIR(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "cse_budget", 3); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "CSE budget insert point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::Register Seed = makeGPR(Context, *Fixture.Function); + const rcc::Register First = makeGPR(Context, *Fixture.Function); + const rcc::Register Second = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1); + (void)buildADDI(Context, Builder, First, Seed, 2); + rcc::MachineInstr *SecondInstruction = buildADDI(Context, Builder, Second, Seed, 2); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineCSEPass Pass({.MaximumComparisons = 0}); + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine CSE budget failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && !Result->Changed && SecondInstruction != nullptr && + SecondInstruction->isLinked(), + "machine CSE budget exhaustion must stop without rejecting or mutating MIR"); +} + +void testLICMHoistsOnlyInvariantInstructions(TestContext &Context, + const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "licm", 4); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + auto Header = Fixture.Function->createBasicBlock(); + auto Body = Fixture.Function->createBasicBlock(); + auto Exit = Fixture.Function->createBasicBlock(); + Context.expect(Header.has_value() && Body.has_value() && Exit.has_value() && + Fixture.Function->appendBlock(**Header).has_value() && + Fixture.Function->appendBlock(**Body).has_value() && + Fixture.Function->appendBlock(**Exit).has_value(), + "LICM loop blocks must be created"); + if (!Header || !Body || !Exit) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + Context.expect(Half.has_value(), "LICM branch probability must be created"); + if (!Half) + return; + auto Updater = Fixture.Function->cfgUpdater(); + Context.expect( + Updater.addSuccessor(*Fixture.Entry, **Header, rcc::BranchProbability::one()).has_value() && + Updater.addSuccessor(**Header, **Body, *Half).has_value() && + Updater.addSuccessor(**Header, **Exit, *Half).has_value() && + Updater.addSuccessor(**Body, **Header, rcc::BranchProbability::one()).has_value(), + "LICM loop CFG must be connected"); + + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "LICM preheader insertion point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::Register Seed = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1); + + Context.expect(Builder.setInsertPoint(**Body).has_value(), + "LICM body insertion point must be established"); + const rcc::Register HoistedValue = makeGPR(Context, *Fixture.Function); + const rcc::Register DependentValue = makeGPR(Context, *Fixture.Function); + rcc::MachineInstr *Hoisted = buildADDI(Context, Builder, HoistedValue, Seed, 8); + rcc::MachineInstr *Dependent = buildADDI(Context, Builder, DependentValue, HoistedValue, 9); + const rcc::Register MutableLoad = makeGPR(Context, *Fixture.Function); + const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, false, false); + rcc::MachineInstr *Load = + Memory == nullptr ? nullptr : buildLoad(Context, Builder, MutableLoad, Seed, *Memory); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineLICMPass Pass; + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine LICM failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "machine LICM must hoist loop-invariant instructions"); + Context.expect(Hoisted != nullptr && Hoisted->parent() == Fixture.Entry && Dependent != nullptr && + Dependent->parent() == Fixture.Entry, + "machine LICM must reach a fixed point for dependent invariants"); + Context.expect(Load != nullptr && Load->parent() == *Body, + "machine LICM must keep mutable loads in the loop"); +} + +void testMachineSinkUsesColdUniqueSuccessor(TestContext &Context, + const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "sink", 5); + if (!Fixture.Function || Fixture.Entry == nullptr) + return; + auto Cold = Fixture.Function->createBasicBlock(); + auto Hot = Fixture.Function->createBasicBlock(); + Context.expect(Cold.has_value() && Hot.has_value() && + Fixture.Function->appendBlock(**Cold).has_value() && + Fixture.Function->appendBlock(**Hot).has_value(), + "machine sink successor blocks must be created"); + if (!Cold || !Hot) + return; + auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10); + auto HotProbability = rcc::BranchProbability::fromFraction(9, 10); + Context.expect(ColdProbability.has_value() && HotProbability.has_value(), + "machine sink probabilities must be created"); + if (!ColdProbability || !HotProbability) + return; + auto Updater = Fixture.Function->cfgUpdater(); + Context.expect(Updater.addSuccessor(*Fixture.Entry, **Cold, *ColdProbability).has_value() && + Updater.addSuccessor(*Fixture.Entry, **Hot, *HotProbability).has_value(), + "machine sink CFG must be connected"); + + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "machine sink source insertion point must be established"); + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::Register Seed = makeGPR(Context, *Fixture.Function); + const rcc::Register SunkValue = makeGPR(Context, *Fixture.Function); + (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1); + rcc::MachineInstr *Sunk = buildADDI(Context, Builder, SunkValue, Seed, 4); + + Context.expect(Builder.setInsertPoint(**Cold).has_value(), + "machine sink destination insertion point must be established"); + const rcc::Register Consumer = makeGPR(Context, *Fixture.Function); + rcc::MachineInstr *Use = buildADD(Context, Builder, Consumer, SunkValue, Seed); + + rcc::MachineAnalysisManager Analyses; + rcc::MachineSinkPass Pass; + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "machine sink failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "machine sink must move a cold-path-only expression"); + Context.expect(Sunk != nullptr && Sunk->parent() == *Cold && Use != nullptr && + Sunk->next() == Use, + "machine sink must place the definition immediately before its first use"); +} + +void testPostRACodeMotion(TestContext &Context, const rcc::TargetDescription &Target) { + auto LICMFixture = makeFixture(Context, "post-ra-licm", 10); + if (!LICMFixture.Function || LICMFixture.Entry == nullptr) + return; + auto Header = LICMFixture.Function->createBasicBlock(); + auto Body = LICMFixture.Function->createBasicBlock(); + auto Exit = LICMFixture.Function->createBasicBlock(); + if (!Header || !Body || !Exit || !LICMFixture.Function->appendBlock(**Header) || + !LICMFixture.Function->appendBlock(**Body) || !LICMFixture.Function->appendBlock(**Exit)) { + Context.expect(false, "post-RA LICM blocks must be created"); + return; + } + auto Half = rcc::BranchProbability::fromFraction(1, 2); + if (!Half) + return; + auto LICMUpdater = LICMFixture.Function->cfgUpdater(); + if (!LICMUpdater.addSuccessor(*LICMFixture.Entry, **Header, rcc::BranchProbability::one()) || + !LICMUpdater.addSuccessor(**Header, **Body, *Half) || + !LICMUpdater.addSuccessor(**Header, **Exit, *Half) || + !LICMUpdater.addSuccessor(**Body, **Header, rcc::BranchProbability::one())) { + Context.expect(false, "post-RA LICM CFG must be connected"); + return; + } + rcc::MachineIRBuilder LICMBuilder(*LICMFixture.Function, Target); + if (!LICMBuilder.setInsertPoint(*LICMFixture.Entry)) + return; + const rcc::Register X0 = + rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X0)); + const rcc::Register X2 = + rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X2)); + const rcc::Register X10 = + rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X10)); + const rcc::Register X12 = + rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X12)); + const rcc::Register X13 = + rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X13)); + (void)buildADDI(Context, LICMBuilder, X10, X0, 1); + if (!LICMBuilder.setInsertPoint(**Header)) + return; + rcc::MachineInstr *Hoisted = buildADDI(Context, LICMBuilder, X12, X10, 4); + if (Hoisted == nullptr) + return; + Hoisted->setFlags({rcc::MachineInstrFlag::NoFPExcept}); + if (!LICMBuilder.setInsertPoint(**Body)) + return; + (void)buildADDI(Context, LICMBuilder, X13, X12, 1); + if (!LICMBuilder.setInsertPoint(**Exit)) + return; + const rcc::MachineMemOperand *LICMMemory = + makeMemory(Context, *LICMFixture.Function, true, false); + if (LICMMemory == nullptr) + return; + (void)buildStore(Context, LICMBuilder, X12, *X2.physicalRegister(), *LICMMemory); + setPostRAProperties(*LICMFixture.Function); + Context.expect(rcc::rebuildMachineLiveness(*LICMFixture.Function, Target).has_value(), + "post-RA LICM fixture liveness must be built"); + rcc::MachineAnalysisManager LICMAnalyses; + rcc::MachineLICMPass PostRALICM(rcc::MachineCodeMotionStage::PostRegisterAllocation); + auto LICMResult = PostRALICM.run(*LICMFixture.Function, Target, LICMAnalyses); + Context.expect(LICMResult && LICMResult->Changed && Hoisted->parent() == LICMFixture.Entry && + rcc::verifyTrackedMachineLiveness(*LICMFixture.Function, Target).has_value(), + "post-RA LICM must hoist a safe physical-register invariant and repair liveness"); + + auto SinkFixture = makeFixture(Context, "post-ra-sink", 11); + if (!SinkFixture.Function || SinkFixture.Entry == nullptr) + return; + auto Cold = SinkFixture.Function->createBasicBlock(); + auto Hot = SinkFixture.Function->createBasicBlock(); + if (!Cold || !Hot || !SinkFixture.Function->appendBlock(**Cold) || + !SinkFixture.Function->appendBlock(**Hot)) { + return; + } + auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10); + auto HotProbability = rcc::BranchProbability::fromFraction(9, 10); + if (!ColdProbability || !HotProbability) + return; + auto SinkUpdater = SinkFixture.Function->cfgUpdater(); + if (!SinkUpdater.addSuccessor(*SinkFixture.Entry, **Cold, *ColdProbability) || + !SinkUpdater.addSuccessor(*SinkFixture.Entry, **Hot, *HotProbability)) { + return; + } + rcc::MachineIRBuilder SinkBuilder(*SinkFixture.Function, Target); + if (!SinkBuilder.setInsertPoint(*SinkFixture.Entry)) + return; + (void)buildADDI(Context, SinkBuilder, X10, X0, 2); + rcc::MachineInstr *Sunk = buildADDI(Context, SinkBuilder, X12, X10, 8); + if (Sunk == nullptr) + return; + Sunk->setFlags({rcc::MachineInstrFlag::NoFPExcept}); + if (!SinkBuilder.setInsertPoint(**Cold)) + return; + rcc::MachineInstr *ColdUse = buildADDI(Context, SinkBuilder, X13, X12, 1); + setPostRAProperties(*SinkFixture.Function); + Context.expect(rcc::rebuildMachineLiveness(*SinkFixture.Function, Target).has_value(), + "post-RA sink fixture liveness must be built"); + rcc::MachineAnalysisManager SinkAnalyses; + rcc::MachineSinkPass PostRASink(rcc::MachineCodeMotionStage::PostRegisterAllocation); + auto SinkResult = PostRASink.run(*SinkFixture.Function, Target, SinkAnalyses); + Context.expect(SinkResult && SinkResult->Changed && Sunk->parent() == *Cold && + Sunk->next() == ColdUse && + rcc::verifyTrackedMachineLiveness(*SinkFixture.Function, Target).has_value(), + "post-RA sinking must move a physical definition to its sole cold use"); +} + +void testMemoryFoldingContract(TestContext &Context, const rcc::TargetDescription &Target) { + auto BuildCandidate = [&](std::string Name, std::uint64_t ID) { + Fixture Candidate = makeFixture(Context, std::move(Name), ID); + if (!Candidate.Function || Candidate.Entry == nullptr) + return Candidate; + rcc::MachineIRBuilder Builder(*Candidate.Function, Target); + if (!Builder.setInsertPoint(*Candidate.Entry)) + return Candidate; + const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2); + const rcc::Register Address = makeGPR(Context, *Candidate.Function); + const rcc::Register Loaded = makeGPR(Context, *Candidate.Function); + const rcc::Register FoldedResult = makeGPR(Context, *Candidate.Function); + (void)buildADDI(Context, Builder, Address, rcc::Register::fromPhysical(X0), 0); + const rcc::MachineMemOperand *LoadMemory = + makeMemory(Context, *Candidate.Function, false, false); + const rcc::MachineMemOperand *StoreMemory = + makeMemory(Context, *Candidate.Function, true, false); + if (LoadMemory == nullptr || StoreMemory == nullptr) + return Candidate; + (void)buildLoad(Context, Builder, Loaded, Address, *LoadMemory); + (void)buildADDI(Context, Builder, FoldedResult, Loaded, 0); + (void)buildStore(Context, Builder, FoldedResult, X2, *StoreMemory); + return Candidate; + }; + + Fixture Folded = BuildCandidate("memory-fold", 8); + if (!Folded.Function || Folded.Entry == nullptr) + return; + TestMemoryFoldingInfo GoodInfo; + rcc::MachineAnalysisManager Analyses; + rcc::MachineMemoryFoldingPass Pass(GoodInfo); + auto Result = Pass.run(*Folded.Function, Target, Analyses); + if (!Result) + std::cerr << "machine memory folding failure: " << Result.error().message() << '\n'; + std::size_t LoadCount = 0; + rcc::MachineInstr *FoldedLoad = nullptr; + for (rcc::MachineInstr *Instruction = Folded.Entry->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::LD)) { + ++LoadCount; + FoldedLoad = Instruction; + } + } + Context.expect(Result && Result->Changed && LoadCount == 1 && FoldedLoad != nullptr && + FoldedLoad->memOperands().size() == 1, + "memory folding must replace an adjacent load-use pair and migrate its MMO"); + + Fixture Rejected = BuildCandidate("memory-fold-missing-mmo", 9); + if (!Rejected.Function || Rejected.Entry == nullptr) + return; + rcc::MachineInstr *OriginalLoad = Rejected.Entry->firstInstruction(); + OriginalLoad = OriginalLoad == nullptr ? nullptr : OriginalLoad->next(); + rcc::MachineInstr *OriginalConsumer = OriginalLoad == nullptr ? nullptr : OriginalLoad->next(); + const std::size_t OriginalCount = Rejected.Entry->instructionCount(); + TestMemoryFoldingInfo BadInfo(true); + rcc::MachineMemoryFoldingPass BadPass(BadInfo); + auto Failure = BadPass.run(*Rejected.Function, Target, Analyses); + Context.expect(!Failure && Failure.error().code() == rcc::ErrorCode::VerificationError && + Rejected.Entry->instructionCount() == OriginalCount && + OriginalLoad != nullptr && OriginalLoad->isLinked() && + OriginalConsumer != nullptr && OriginalConsumer->isLinked(), + "memory folding must reject missing MMO migration without mutating the function"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "optimization target construction must succeed"); + if (!Target) + return Context.result(); + testDCEFixedPointAndDebugUses(Context, *Target); + testMachineCombinerIdentityRules(Context, *Target); + testPostRAMachineCombinerIdentity(Context, *Target); + testCSERegisterAndMemoryRules(Context, *Target); + testCSEBudgetPreservesLegalMIR(Context, *Target); + testLICMHoistsOnlyInvariantInstructions(Context, *Target); + testMachineSinkUsesColdUniqueSuccessor(Context, *Target); + testPostRACodeMotion(Context, *Target); + testMemoryFoldingContract(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/MachineSchedulingTest.cpp b/backend/unittests/CodeGen/MachineSchedulingTest.cpp new file mode 100644 index 0000000..a485146 --- /dev/null +++ b/backend/unittests/CodeGen/MachineSchedulingTest.cpp @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/PostRAScheduler.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +struct Fixture final { + std::unique_ptr Function; + rcc::MachineBasicBlock *Block = nullptr; +}; + +class OnePerCycleHazard final : public rcc::TargetScheduleHazardRecognizer { +public: + [[nodiscard]] std::unique_ptr clone() const override { + return std::make_unique(*this); + } + + [[nodiscard]] rcc::Expected + getHazardType(const rcc::ScheduleUnit &) const override { + return Emitted ? rcc::ScheduleHazardType::Hazard : rcc::ScheduleHazardType::NoHazard; + } + + [[nodiscard]] rcc::Expected emitInstruction(const rcc::ScheduleUnit &) override { + Emitted = true; + return {}; + } + + [[nodiscard]] rcc::Expected advanceCycle() override { + Emitted = false; + return {}; + } + + [[nodiscard]] rcc::Expected recedeCycle() override { + Emitted = false; + return {}; + } + + [[nodiscard]] rcc::Expected reset() override { + Emitted = false; + return {}; + } + +private: + bool Emitted = false; +}; + +class TestSchedulingHooks final : public rcc::TargetSchedulingHooks { +public: + bool MakeADDIBoundary = false; + bool AddMutation = false; + bool PreferSecondUnit = false; + bool EnableDynamicHazards = false; + + [[nodiscard]] bool isSchedulingBoundary(const rcc::MachineInstr &Instruction, + const rcc::TargetDescription &) const noexcept override { + return MakeADDIBoundary && + Instruction.opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::ADDI)); + } + + [[nodiscard]] rcc::Expected mutateScheduleDAG(rcc::ScheduleDAG &DAG, + const rcc::MachineFunction &, + const rcc::TargetDescription &, + rcc::ScheduleDAGMode) const override { + if (!AddMutation || DAG.units().size() < 2) + return {}; + return DAG.addArtificialDependency(0, 1, 2); + } + + [[nodiscard]] std::int64_t candidatePreference(const rcc::ScheduleUnit &Unit, + rcc::ScheduleDirection, + const rcc::MachineBasicBlock &, + std::uint64_t) const noexcept override { + return PreferSecondUnit && Unit.ID == 1 ? 100 : 0; + } + + [[nodiscard]] std::unique_ptr + createHazardRecognizer(rcc::ScheduleDirection) const override { + return EnableDynamicHazards ? std::make_unique() : nullptr; + } +}; + +[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept { + return rcc::Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] rcc::Expected +makeSchedulingModel(const rcc::TargetSchedulingHooks *Hooks = nullptr) { + rcc::TargetSchedulingModelDesc Description; + Description.IssueWidth = 2; + Description.DispatchLimit = 2; + for (const rcc::riscv::SchedulingClassInfo &Class : rcc::riscv::schedulingClasses()) { + Description.SchedulingClasses.push_back({ + .ID = Class.ID, + .Name = std::string(Class.Name), + .MicroOps = Class.MicroOps, + .Latency = Class.Latency, + .Resources = {}, + .WriteLatencies = {{0, Class.Latency}}, + .ReadAdvances = {}, + }); + } + Description.PressureSets.push_back({ + .ID = rcc::RegisterPressureSetID::fromValue(1), + .Name = "gpr", + .RegisterClasses = + { + rcc::RegisterClassID::fromValue( + static_cast(rcc::riscv::RegisterClass::GPR)), + }, + .Limit = 27, + }); + Description.Hooks = Hooks; + return rcc::TargetSchedulingModel::create(std::move(Description)); +} + +[[nodiscard]] rcc::Register physical(TestContext &Context, rcc::riscv::PhysicalRegisterID ID) { + auto Value = rcc::riscv::physicalRegister(ID); + Context.expect(Value.has_value(), "physical register creation must succeed"); + return Value ? rcc::Register::fromPhysical(*Value) : rcc::Register{}; +} + +[[nodiscard]] Fixture makeFixture(TestContext &Context, std::string Name, std::uint64_t ID) { + auto Function = + rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID)); + Context.expect(Function.has_value(), "scheduler function creation must succeed"); + if (!Function) + return {}; + auto Block = (*Function)->createBasicBlock(); + Context.expect(Block.has_value() && (*Function)->appendBlock(**Block).has_value(), + "scheduler block creation must succeed"); + if (!Block) + return {}; + (*Function)->properties().set(rcc::MachineFunctionProperty::Legalized); + (*Function)->properties().set(rcc::MachineFunctionProperty::RegBankSelected); + (*Function)->properties().set(rcc::MachineFunctionProperty::Selected); + (*Function)->properties().set(rcc::MachineFunctionProperty::NoPHIs); + (*Function)->properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten); + return {std::move(*Function), *Block}; +} + +[[nodiscard]] rcc::Register makeGPR(TestContext &Context, rcc::MachineFunction &Function) { + auto Type = rcc::LLT::makeInteger(64); + auto Reg = Type ? Function.registerInfo().createVirtualRegister(*Type) + : rcc::Expected(std::unexpected( + rcc::Error(rcc::ErrorCode::InternalError, "test LLT creation failed"))); + Context.expect(Reg.has_value(), "scheduler virtual register creation must succeed"); + if (!Reg) + return {}; + const auto Bank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + const auto Class = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::GPR)); + Context.expect(Function.registerInfo().setRegisterBank(*Reg, Bank).has_value() && + Function.registerInfo().setRegisterClass(*Reg, Class).has_value(), + "scheduler virtual register constraints must be assigned"); + return *Reg; +} + +[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Source, + std::int64_t Immediate) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) || + !Pending->addImmediate(Immediate)) { + Context.expect(false, "scheduler ADDI construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "scheduler ADDI insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr *buildBinary(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::riscv::InstructionOpcode Opcode, + rcc::Register Destination, rcc::Register Left, + rcc::Register Right) { + auto Pending = Builder.buildInstr(opcode(Opcode)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Left) || + !Pending->addUse(Right)) { + Context.expect(false, "scheduler binary instruction construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "scheduler binary instruction insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] bool hasDependency(const rcc::ScheduleUnit &Unit, std::size_t Destination, + rcc::ScheduleDependencyKind Kind) { + for (const rcc::ScheduleDependency &Dependency : Unit.Successors) { + if (Dependency.Destination == Destination && Dependency.Kind == Kind) + return true; + } + return false; +} + +void testDAGRegisterDependencies(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "schedule_dag", 1); + if (!Fixture.Function || Fixture.Block == nullptr) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(), + "schedule DAG insertion point must be established"); + const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::Register X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10); + const rcc::Register Value = makeGPR(Context, *Fixture.Function); + const rcc::Register Consumer = makeGPR(Context, *Fixture.Function); + rcc::MachineInstr *Definition = buildADDI(Context, Builder, Value, X0, 1); + rcc::MachineInstr *Use = + buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, Consumer, Value, X10); + rcc::MachineInstr *Redefinition = buildADDI(Context, Builder, Value, X0, 2); + std::array Instructions{Definition, Use, Redefinition}; + auto DAG = rcc::ScheduleDAG::build(*Fixture.Block, Instructions, *Fixture.Function, Target, + rcc::riscv::schedulingModel(), + rcc::ScheduleDAGMode::PreRegisterAllocation); + if (!DAG) + std::cerr << "schedule DAG failure: " << DAG.error().message() << '\n'; + Context.expect(DAG.has_value() && DAG->units().size() == 3, + "schedule DAG must create one unit per instruction"); + if (!DAG) + return; + Context.expect(hasDependency(DAG->units()[0], 1, rcc::ScheduleDependencyKind::Data), + "schedule DAG must model virtual-register data dependencies"); + Context.expect(hasDependency(DAG->units()[0], 2, rcc::ScheduleDependencyKind::Output), + "schedule DAG must model virtual-register output dependencies"); + Context.expect(hasDependency(DAG->units()[1], 2, rcc::ScheduleDependencyKind::Anti), + "schedule DAG must model virtual-register anti dependencies"); + Context.expect(DAG->units()[0].Latency != 0 && DAG->units()[0].MicroOps != 0, + "schedule DAG must attach target latency and resource facts"); +} + +void testHazardScoreboard(TestContext &Context) { + auto Hazards = rcc::ScheduleHazardRecognizer::create(rcc::riscv::schedulingModel()); + Context.expect(Hazards.has_value(), "hazard recognizer construction must succeed"); + if (!Hazards) + return; + const auto Memory = rcc::ProcessorResourceID::fromValue(3); + rcc::ScheduleUnit Load; + Load.MicroOps = 1; + Load.Resources = {{Memory, 1}}; + Context.expect(Hazards->issue(Load, 0).has_value(), + "first memory operation must issue in cycle zero"); + auto SecondCycle = Hazards->earliestIssueCycle(Load, 0); + Context.expect(SecondCycle.has_value() && *SecondCycle == 1, + "single-unit memory resource must serialize operations"); + + rcc::ScheduleUnit Wide; + Wide.MicroOps = 5; + auto Fresh = rcc::ScheduleHazardRecognizer::create(rcc::riscv::schedulingModel()); + Context.expect(Fresh.has_value() && Fresh->issue(Wide, 0).has_value(), + "multi-cycle dispatch must accept instructions wider than issue width"); + auto AfterWide = Fresh ? Fresh->earliestIssueCycle(Load, 0) + : rcc::Expected(std::unexpected( + rcc::Error(rcc::ErrorCode::InternalError, "no recognizer"))); + Context.expect(AfterWide.has_value() && *AfterWide == 3, + "multi-cycle dispatch must block the complete dispatch interval"); +} + +void testOperandLatencyAndModelValidation(TestContext &Context) { + const rcc::SchedulingClassID ALU = rcc::SchedulingClassID::fromValue( + static_cast(rcc::riscv::SchedulingClass::ALU)); + const rcc::SchedulingClassID Load = rcc::SchedulingClassID::fromValue( + static_cast(rcc::riscv::SchedulingClass::LOAD)); + auto Forwarded = rcc::riscv::schedulingModel().operandLatency(Load, 0, ALU, 1); + auto Unforwarded = rcc::riscv::schedulingModel().operandLatency(Load, 0, ALU, 0); + auto ALUChain = rcc::riscv::schedulingModel().operandLatency(ALU, 0, ALU, 1); + Context.expect(Forwarded.has_value() && *Forwarded == 2 && Unforwarded.has_value() && + *Unforwarded == 3 && ALUChain.has_value() && *ALUChain == 1, + "operand latency must combine write latency with exact read advance"); + + rcc::TargetSchedulingModelDesc Invalid; + Invalid.SchedulingClasses.push_back({ + .ID = ALU, + .Name = "invalid", + .MicroOps = 1, + .Latency = 1, + .Resources = {}, + .WriteLatencies = {{0, 1}, {0, 2}}, + .ReadAdvances = {}, + }); + auto Rejected = rcc::TargetSchedulingModel::create(std::move(Invalid)); + Context.expect(!Rejected && Rejected.error().code() == rcc::ErrorCode::InvalidArgument, + "scheduling model must reject duplicate operand latency facts"); +} + +void testDynamicTargetHazards(TestContext &Context) { + TestSchedulingHooks Hooks; + Hooks.EnableDynamicHazards = true; + auto Model = makeSchedulingModel(&Hooks); + Context.expect(Model.has_value(), "dynamic-hazard scheduling model construction must succeed"); + if (!Model) + return; + + rcc::ScheduleUnit First; + First.ID = 1; + First.MicroOps = 1; + rcc::ScheduleUnit Second; + Second.ID = 2; + Second.MicroOps = 1; + auto Top = rcc::ScheduleHazardRecognizer::create(*Model, rcc::ScheduleDirection::TopDown); + Context.expect(Top.has_value() && Top->issue(First, 0).has_value(), + "target dynamic hazard must accept the first top-down issue"); + auto TopCycle = Top ? Top->earliestIssueCycle(Second, 0) + : rcc::Expected(std::unexpected( + rcc::Error(rcc::ErrorCode::InternalError, "no top-down recognizer"))); + Context.expect(TopCycle.has_value() && *TopCycle == 1, + "target dynamic hazard must advance the top-down cycle"); + + auto Bottom = rcc::ScheduleHazardRecognizer::create(*Model, rcc::ScheduleDirection::BottomUp); + Context.expect(Bottom.has_value() && Bottom->issue(First, 0).has_value(), + "target dynamic hazard must accept the first bottom-up issue"); + auto BottomCycle = Bottom ? Bottom->earliestIssueCycle(Second, 0) + : rcc::Expected(std::unexpected(rcc::Error( + rcc::ErrorCode::InternalError, "no bottom-up recognizer"))); + Context.expect(BottomCycle.has_value() && *BottomCycle == 1, + "target dynamic hazard must recede the bottom-up cycle"); +} + +void testTargetSchedulingHooks(TestContext &Context, const rcc::TargetDescription &Target) { + TestSchedulingHooks MutationHooks; + MutationHooks.AddMutation = true; + auto MutationModel = makeSchedulingModel(&MutationHooks); + auto Fixture = makeFixture(Context, "schedule_mutation", 4); + if (!MutationModel || !Fixture.Function || Fixture.Block == nullptr) { + Context.expect(false, "target schedule mutation fixture must be created"); + return; + } + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(), + "target schedule mutation insertion point must be established"); + const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + rcc::MachineInstr *First = + buildADDI(Context, Builder, makeGPR(Context, *Fixture.Function), X0, 1); + rcc::MachineInstr *Second = + buildADDI(Context, Builder, makeGPR(Context, *Fixture.Function), X0, 2); + std::array Instructions{First, Second}; + auto DAG = rcc::ScheduleDAG::build(*Fixture.Block, Instructions, *Fixture.Function, Target, + *MutationModel, rcc::ScheduleDAGMode::PreRegisterAllocation); + Context.expect(DAG.has_value() && + hasDependency(DAG->units()[0], 1, rcc::ScheduleDependencyKind::Artificial), + "target schedule mutation must add a checked artificial dependency"); + + TestSchedulingHooks PreferenceHooks; + PreferenceHooks.PreferSecondUnit = true; + auto PreferenceModel = makeSchedulingModel(&PreferenceHooks); + if (!PreferenceModel) { + Context.expect(false, "target scheduling preference model must be created"); + return; + } + rcc::MachineAnalysisManager Analyses; + rcc::MachineSchedulerPass PreferencePass( + *PreferenceModel, {.MaximumRegionSize = 8, .EnableBidirectionalScheduling = false}); + auto Preferred = PreferencePass.run(*Fixture.Function, Target, Analyses); + Context.expect(Preferred.has_value() && Preferred->Changed && + Fixture.Block->firstInstruction() == Second, + "target candidate preference must affect the production scheduler"); + + auto BoundaryFixture = makeFixture(Context, "schedule_boundary", 5); + TestSchedulingHooks BoundaryHooks; + BoundaryHooks.MakeADDIBoundary = true; + auto BoundaryModel = makeSchedulingModel(&BoundaryHooks); + if (!BoundaryFixture.Function || BoundaryFixture.Block == nullptr || !BoundaryModel) { + Context.expect(false, "target schedule boundary fixture must be created"); + return; + } + rcc::MachineIRBuilder BoundaryBuilder(*BoundaryFixture.Function, Target); + Context.expect(BoundaryBuilder.setInsertPoint(*BoundaryFixture.Block).has_value(), + "target schedule boundary insertion point must be established"); + rcc::MachineInstr *BoundaryFirst = + buildADDI(Context, BoundaryBuilder, makeGPR(Context, *BoundaryFixture.Function), X0, 1); + (void)buildADDI(Context, BoundaryBuilder, makeGPR(Context, *BoundaryFixture.Function), X0, 2); + rcc::MachineAnalysisManager BoundaryAnalyses; + rcc::MachineSchedulerPass BoundaryPass( + *BoundaryModel, {.MaximumRegionSize = 8, .EnableBidirectionalScheduling = false}); + auto BoundaryResult = BoundaryPass.run(*BoundaryFixture.Function, Target, BoundaryAnalyses); + Context.expect(BoundaryResult.has_value() && !BoundaryResult->Changed && + BoundaryFixture.Block->firstInstruction() == BoundaryFirst, + "target scheduling boundary must split production schedule regions"); +} + +void buildSchedulingChain(TestContext &Context, Fixture &Fixture, + const rcc::TargetDescription &Target, bool PhysicalRegisters, + rcc::MachineInstr *&Short, rcc::MachineInstr *&Long) { + rcc::MachineIRBuilder Builder(*Fixture.Function, Target); + Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(), + "scheduler chain insertion point must be established"); + const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0); + const rcc::Register X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10); + const rcc::Register X11 = physical(Context, rcc::riscv::PhysicalRegisterID::X11); + const rcc::Register ShortValue = PhysicalRegisters + ? physical(Context, rcc::riscv::PhysicalRegisterID::X12) + : makeGPR(Context, *Fixture.Function); + const rcc::Register LongValue = PhysicalRegisters + ? physical(Context, rcc::riscv::PhysicalRegisterID::X13) + : makeGPR(Context, *Fixture.Function); + const rcc::Register LongResult = PhysicalRegisters + ? physical(Context, rcc::riscv::PhysicalRegisterID::X14) + : makeGPR(Context, *Fixture.Function); + const rcc::Register ShortResult = PhysicalRegisters + ? physical(Context, rcc::riscv::PhysicalRegisterID::X15) + : makeGPR(Context, *Fixture.Function); + Short = buildADDI(Context, Builder, ShortValue, X0, 1); + Long = buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::DIV, LongValue, X10, X11); + (void)buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, LongResult, LongValue, + X10); + (void)buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, ShortResult, ShortValue, + X11); +} + +void testPreRAScheduler(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "pre_ra_schedule", 2); + if (!Fixture.Function || Fixture.Block == nullptr) + return; + rcc::MachineInstr *Short = nullptr; + rcc::MachineInstr *Long = nullptr; + buildSchedulingChain(Context, Fixture, Target, false, Short, Long); + Context.expect(Fixture.Block->firstInstruction() == Short, + "pre-RA fixture must begin in intentionally suboptimal order"); + rcc::MachineAnalysisManager Analyses; + rcc::MachineSchedulerPass Pass(rcc::riscv::schedulingModel(), + {.MaximumRegionSize = 64, .EnableBidirectionalScheduling = false}); + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "pre-RA scheduler failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "pre-RA scheduler must reorder the latency-critical chain"); + Context.expect(Fixture.Block->firstInstruction() == Long, + "pre-RA scheduler must prioritize the long critical path"); +} + +void testPostRASchedulerAndLivenessRepair(TestContext &Context, + const rcc::TargetDescription &Target) { + auto Fixture = makeFixture(Context, "post_ra_schedule", 3); + if (!Fixture.Function || Fixture.Block == nullptr) + return; + Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoVRegs); + Fixture.Function->properties().set(rcc::MachineFunctionProperty::TracksLiveness); + rcc::MachineInstr *Short = nullptr; + rcc::MachineInstr *Long = nullptr; + buildSchedulingChain(Context, Fixture, Target, true, Short, Long); + auto Rebuilt = rcc::rebuildMachineLiveness(*Fixture.Function, Target); + Context.expect(Rebuilt.has_value(), "post-RA fixture liveness rebuild must succeed"); + if (!Rebuilt) + return; + + rcc::MachineAnalysisManager Analyses; + rcc::PostRASchedulerPass Pass(rcc::riscv::schedulingModel(), + {.MaximumRegionSize = 64, .EnableBidirectionalScheduling = false}); + auto Result = Pass.run(*Fixture.Function, Target, Analyses); + if (!Result) + std::cerr << "post-RA scheduler failure: " << Result.error().message() << '\n'; + Context.expect(Result.has_value() && Result->Changed, + "post-RA scheduler must reorder the latency-critical chain"); + Context.expect(Fixture.Block->firstInstruction() == Long, + "post-RA scheduler must prioritize the long critical path"); + auto Verified = rcc::verifyTrackedMachineLiveness(*Fixture.Function, Target); + if (!Verified) + std::cerr << "post-RA liveness verification failure: " << Verified.error().message() << '\n'; + Context.expect(Verified.has_value(), + "post-RA scheduler must locally repair exact kill/dead state"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "scheduler target creation must succeed"); + if (!Target) + return Context.result(); + testDAGRegisterDependencies(Context, *Target); + testHazardScoreboard(Context); + testOperandLatencyAndModelValidation(Context); + testDynamicTargetHazards(Context); + testTargetSchedulingHooks(Context, *Target); + testPreRAScheduler(Context, *Target); + testPostRASchedulerAndLivenessRepair(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/Support/ErrorTest.cpp b/backend/unittests/Support/ErrorTest.cpp index af2ece9..de75aa3 100644 --- a/backend/unittests/Support/ErrorTest.cpp +++ b/backend/unittests/Support/ErrorTest.cpp @@ -30,6 +30,16 @@ int main() { return fail("Error code must have a stable name"); if (rcc::getErrorCodeName(rcc::ErrorCode::FrameLayoutFailed) != "frame layout failed") return fail("frame-layout errors must have a stable name"); + if (rcc::getErrorCodeName(rcc::ErrorCode::SchedulingFailed) != "scheduling failed") + return fail("scheduling errors must have a stable name"); + if (rcc::getErrorCodeName(rcc::ErrorCode::BranchRelaxationFailed) != "branch relaxation failed") + return fail("branch-relaxation errors must have a stable name"); + if (rcc::getErrorCodeName(rcc::ErrorCode::PseudoExpansionFailed) != "pseudo expansion failed") + return fail("pseudo-expansion errors must have a stable name"); + if (rcc::getErrorCodeName(rcc::ErrorCode::FinalEncodabilityFailed) != "final encodability failed") + return fail("final-encodability errors must have a stable name"); + if (rcc::getErrorCodeName(rcc::ErrorCode::CFGOptimizationFailed) != "CFG optimization failed") + return fail("CFG-optimization errors must have a stable name"); return 0; } diff --git a/backend/unittests/Target/CMakeLists.txt b/backend/unittests/Target/CMakeLists.txt index a114c7f..c1bef1d 100644 --- a/backend/unittests/Target/CMakeLists.txt +++ b/backend/unittests/Target/CMakeLists.txt @@ -60,6 +60,17 @@ add_test( COMMAND RCCBackendRISCVFrameLoweringTests ) +add_executable(RCCBackendRISCVLateCodeGenTests RISCVLateCodeGenTest.cpp) +rcc_backend_configure_target(RCCBackendRISCVLateCodeGenTests) +target_link_libraries( + RCCBackendRISCVLateCodeGenTests + PRIVATE RCCBackend::RISCV RCCBackend::CodeGen +) +add_test( + NAME RCCBackendRISCVLateCodeGenTests + COMMAND RCCBackendRISCVLateCodeGenTests +) + rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/DescriptionGeneratorTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLoweringTest.cpp" @@ -68,6 +79,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLoweringTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVLateCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.h" ) @@ -79,5 +91,6 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLoweringTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVLateCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" ) diff --git a/backend/unittests/Target/DescriptionGeneratorTest.cpp b/backend/unittests/Target/DescriptionGeneratorTest.cpp index 0f8d1e9..ea073bf 100644 --- a/backend/unittests/Target/DescriptionGeneratorTest.cpp +++ b/backend/unittests/Target/DescriptionGeneratorTest.cpp @@ -65,7 +65,7 @@ encoding_categories: assembly_formats: - { id: 1, symbol: RD_IMM, name: rd-imm } instructions: - - { id: 1024, symbol: LOAD_IMM, name: LOAD_IMM, assembly_mnemonic: load.imm, operands: [{ kinds: [register], role: def, register_class: GPR64, register_bank: GPR, name: RD }, { kinds: [immediate], immediate_predicate: SIMM8, name: IMM, allowed_target_flags: 33 }], num_explicit_defs: 1, flags: [produces_code], scheduling_class: ALU, encoding_category: I, assembly_format: RD_IMM } + - { id: 1024, symbol: LOAD_IMM, name: LOAD_IMM, assembly_mnemonic: load.imm, operands: [{ kinds: [register], role: def, register_class: GPR64, register_bank: GPR, name: RD }, { kinds: [immediate, global_symbol], immediate_predicate: SIMM8, name: IMM, allowed_target_flags: 33 }], num_explicit_defs: 1, flags: [produces_code], scheduling_class: ALU, encoding_category: I, assembly_format: RD_IMM } )"; } diff --git a/backend/unittests/Target/RISCVLateCodeGenTest.cpp b/backend/unittests/Target/RISCVLateCodeGenTest.cpp new file mode 100644 index 0000000..7ab3a59 --- /dev/null +++ b/backend/unittests/Target/RISCVLateCodeGenTest.cpp @@ -0,0 +1,1432 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBlockPlacement.h" +#include "rcc/CodeGen/MachineBranchOptimization.h" +#include "rcc/CodeGen/MachineBranchRelaxation.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineCopyPropagation.h" +#include "rcc/CodeGen/MachineFallthrough.h" +#include "rcc/CodeGen/MachineFinalization.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineOptimizationPipeline.h" +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/MachinePseudoExpansion.h" +#include "rcc/CodeGen/MachineShrinkWrapping.h" +#include "rcc/CodeGen/MachineStackSlotColoring.h" +#include "rcc/CodeGen/MachineTailDuplication.h" +#include "rcc/CodeGen/MachineTailMerge.h" +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +[[nodiscard]] constexpr rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept { + return rcc::Opcode::fromValue(static_cast(Value)); +} + +constexpr rcc::Opcode GrowingLatePseudoOpcode = rcc::Opcode::fromValue(10'000); +constexpr rcc::Opcode LateExpansionRealOpcode = rcc::Opcode::fromValue(10'001); +constexpr rcc::Opcode ShrinkingLatePseudoOpcode = rcc::Opcode::fromValue(10'002); + +class GrowingLatePseudoInfo final : public rcc::TargetPseudoExpansionInfo { +public: + [[nodiscard]] rcc::Expected + expandPseudo(rcc::MachineInstr &Instruction, rcc::MachinePseudoExpansionPhase Phase, + rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) const override { + if (Phase != rcc::MachinePseudoExpansionPhase::TargetLate || + (Instruction.opcode() != GrowingLatePseudoOpcode && + Instruction.opcode() != ShrinkingLatePseudoOpcode)) { + return false; + } + rcc::MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return rcc::makeUnexpected(rcc::ErrorCode::VerificationError, + "late expansion test pseudo is detached"); + } + if (Instruction.opcode() == GrowingLatePseudoOpcode) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + for (std::size_t Index = 0; Index < 2; ++Index) { + auto Pending = Builder.buildInstr(LateExpansionRealOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + } + } + if (auto Removed = Block->removeInstruction(Instruction); !Removed) + return std::unexpected(Removed.error()); + return true; + } +}; + +class FixedSizeBranchInfo final : public rcc::TargetBranchInfo { +public: + [[nodiscard]] rcc::Expected + analyzeBranch(rcc::MachineBasicBlock &, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return rcc::MachineBranchAnalysis{}; + } + + [[nodiscard]] rcc::Expected + reverseBranchCondition(rcc::MachineInstr &, rcc::MachineBasicBlock &, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches"); + } + + [[nodiscard]] rcc::Expected removeBranch(rcc::MachineInstr &, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches"); + } + + [[nodiscard]] rcc::Expected + rewriteBranchTarget(rcc::MachineBasicBlock &, rcc::MachineBasicBlock &, rcc::MachineBasicBlock &, + rcc::MachineFunction &, const rcc::TargetDescription &) const override { + return false; + } + + [[nodiscard]] rcc::Expected + insertUnconditionalBranch(rcc::MachineBasicBlock &, rcc::MachineBasicBlock &, + rcc::MachineFunction &, const rcc::TargetDescription &) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches"); + } + + [[nodiscard]] rcc::Expected + instructionSize(const rcc::MachineInstr &, const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return 4; + } + + [[nodiscard]] rcc::Expected + branchTarget(rcc::MachineInstr &, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return static_cast(nullptr); + } + + [[nodiscard]] bool isBranchOffsetInRange(const rcc::MachineInstr &, + std::int64_t) const noexcept override { + return true; + } + + [[nodiscard]] rcc::Expected relaxBranch(rcc::MachineInstr &, std::int64_t, + rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return false; + } +}; + +[[nodiscard]] rcc::Expected createLateExpansionSizeTarget() { + rcc::InstructionDescriptor Pseudo; + Pseudo.InstructionOpcode = GrowingLatePseudoOpcode; + Pseudo.Name = "TEST_GROWING_LATE_PSEUDO"; + Pseudo.Flags = {rcc::InstructionFlag::Pseudo, rcc::InstructionFlag::ProducesCode}; + Pseudo.Owner = rcc::PseudoOwner::TargetLateExpansion; + Pseudo.Deadline = rcc::PipelineStage::Emission; + + rcc::InstructionDescriptor ShrinkingPseudo; + ShrinkingPseudo.InstructionOpcode = ShrinkingLatePseudoOpcode; + ShrinkingPseudo.Name = "TEST_SHRINKING_LATE_PSEUDO"; + ShrinkingPseudo.Flags = {rcc::InstructionFlag::Pseudo, rcc::InstructionFlag::ProducesCode}; + ShrinkingPseudo.Owner = rcc::PseudoOwner::TargetLateExpansion; + ShrinkingPseudo.Deadline = rcc::PipelineStage::Emission; + + rcc::InstructionDescriptor Real; + Real.InstructionOpcode = LateExpansionRealOpcode; + Real.Name = "TEST_LATE_REAL"; + Real.Flags = {rcc::InstructionFlag::ProducesCode}; + + std::vector Descriptors; + Descriptors.push_back(std::move(Pseudo)); + Descriptors.push_back(std::move(Real)); + Descriptors.push_back(std::move(ShrinkingPseudo)); + auto Instructions = rcc::InstructionSet::create(std::move(Descriptors)); + if (!Instructions) + return std::unexpected(Instructions.error()); + auto Registers = rcc::riscv::createRegisterInfo(); + if (!Registers) + return std::unexpected(Registers.error()); + return rcc::TargetDescription::create(std::move(*Instructions), std::move(*Registers)); +} + +[[nodiscard]] rcc::Register physical(rcc::riscv::PhysicalRegisterID ID) { + return rcc::Register::fromPhysical(*rcc::riscv::physicalRegister(ID)); +} + +[[nodiscard]] std::unique_ptr +makeFunction(TestContext &Context, std::string Name, std::uint64_t StableID) { + auto Function = rcc::MachineFunction::create(std::move(Name), + rcc::MachineFunctionStableID::fromValue(StableID)); + Context.expect(Function.has_value(), "late-codegen function creation must succeed"); + return Function ? std::move(*Function) : nullptr; +} + +[[nodiscard]] rcc::MachineBasicBlock *appendBlock(TestContext &Context, + rcc::MachineFunction &Function) { + auto Block = Function.createBasicBlock(); + Context.expect(Block.has_value() && Function.appendBlock(**Block).has_value(), + "late-codegen block creation must succeed"); + return Block.value_or(nullptr); +} + +void setSelectedProperties(rcc::MachineFunction &Function) { + for (const rcc::MachineFunctionProperty Property : { + rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected, + rcc::MachineFunctionProperty::Selected, + rcc::MachineFunctionProperty::NoPHIs, + rcc::MachineFunctionProperty::TiedOpsRewritten, + }) { + Function.properties().set(Property); + } +} + +void setPostRAProperties(rcc::MachineFunction &Function) { + setSelectedProperties(Function); + Function.properties().set(rcc::MachineFunctionProperty::TracksLiveness); + Function.properties().set(rcc::MachineFunctionProperty::NoVRegs); +} + +void setRelaxationProperties(TestContext &Context, rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) { + setPostRAProperties(Function); + auto StackAlignment = rcc::Alignment::create(16); + Context.expect(StackAlignment && Function.frameInfo().freeze(0, *StackAlignment).has_value(), + "relaxation fixture frame must be frozen"); + Function.properties().set(rcc::MachineFunctionProperty::FrameFinalized); + Function.properties().set(rcc::MachineFunctionProperty::NoFrameIndices); + Function.properties().set(rcc::MachineFunctionProperty::NoExecutablePseudos); + auto Liveness = rcc::rebuildMachineLiveness(Function, Target); + Context.expect(Liveness.has_value(), "relaxation fixture liveness must be rebuilt"); +} + +[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Source, + std::int64_t Immediate) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) || + !Pending->addImmediate(Immediate)) { + Context.expect(false, "late-codegen ADDI construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "late-codegen ADDI insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] bool constrainGPR64(TestContext &Context, rcc::MachineFunction &Function, + rcc::Register Reg) { + const auto Bank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + const auto Class = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::GPR)); + const bool Valid = Function.registerInfo().setRegisterBank(Reg, Bank).has_value() && + Function.registerInfo().setRegisterClass(Reg, Class).has_value(); + Context.expect(Valid, "late-codegen GPR64 constraint must succeed"); + return Valid; +} + +[[nodiscard]] rcc::Expected +createStackMemory(rcc::MachineFunction &Function, rcc::FrameIndex Slot, + rcc::MemoryAccessFlag Access) { + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + if (!Size) + return std::unexpected(Size.error()); + if (!Align) + return std::unexpected(Align.error()); + rcc::MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::frameIndex(Slot); + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] rcc::MachineInstr *buildStackAccess(TestContext &Context, + rcc::MachineIRBuilder &Builder, + rcc::riscv::InstructionOpcode Opcode, + rcc::Register Reg, rcc::FrameIndex Slot, + rcc::MemoryAccessFlag Access) { + auto Memory = createStackMemory(Builder.function(), Slot, Access); + auto Pending = Builder.buildInstr(opcode(Opcode)); + const bool IsLoad = Access == rcc::MemoryAccessFlag::Load; + if (!Memory || !Pending || + !(IsLoad ? Pending->addDef(Reg).has_value() : Pending->addUse(Reg).has_value()) || + !Pending->addFrameIndex(Slot) || !Pending->addImmediate(0) || + !Pending->addMemOperand(**Memory)) { + Context.expect(false, "late-codegen stack access construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "late-codegen stack access insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr *buildConditional(TestContext &Context, + rcc::MachineIRBuilder &Builder, + rcc::riscv::InstructionOpcode Opcode, + rcc::MachineBasicBlock &Target) { + auto Pending = Builder.buildInstr(opcode(Opcode)); + if (!Pending || !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) || + !Pending->addBasicBlock(Target)) { + Context.expect(false, "late-codegen conditional branch construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "late-codegen conditional branch insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineInstr * +buildJAL(TestContext &Context, rcc::MachineIRBuilder &Builder, rcc::MachineBasicBlock &Target, + rcc::riscv::PhysicalRegisterID Destination = rcc::riscv::PhysicalRegisterID::X0) { + auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::JAL)); + if (!Pending || !Pending->addDef(physical(Destination)) || !Pending->addBasicBlock(Target)) { + Context.expect(false, "late-codegen JAL construction must succeed"); + return nullptr; + } + auto Built = Pending->build(); + Context.expect(Built.has_value(), "late-codegen JAL insertion must succeed"); + return Built.value_or(nullptr); +} + +[[nodiscard]] rcc::MachineBasicBlock *branchTarget(rcc::MachineInstr &Instruction) { + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + rcc::MachineOperand *Operand = Instruction.operandAt(Index); + const auto *Block = Operand == nullptr ? nullptr : Operand->getIf(); + if (Block != nullptr) + return Block->Block; + } + return nullptr; +} + +void testBlockPlacement(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "block_placement", 1); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Cold = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Hot = appendBlock(Context, *Function); + if (Entry == nullptr || Cold == nullptr || Hot == nullptr) + return; + auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10); + auto HotProbability = rcc::BranchProbability::fromFraction(9, 10); + Context.expect(ColdProbability && HotProbability, + "placement probabilities must be representable"); + if (!ColdProbability || !HotProbability) + return; + auto CFG = Function->cfgUpdater(); + Context.expect(CFG.addSuccessor(*Entry, *Cold, *ColdProbability).has_value() && + CFG.addSuccessor(*Entry, *Hot, *HotProbability).has_value(), + "placement CFG construction must succeed"); + setSelectedProperties(*Function); + rcc::MachineAnalysisManager Analyses; + rcc::MachineBlockPlacementPass Pass; + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "block placement failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && Entry->next() == Hot && Hot->next() == Cold, + "block placement must put the highest-probability successor on the trace"); +} + +void testFallthroughNormalization(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "fallthrough", 2); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *False = appendBlock(Context, *Function); + rcc::MachineBasicBlock *True = appendBlock(Context, *Function); + if (Entry == nullptr || False == nullptr || True == nullptr) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto CFG = Function->cfgUpdater(); + Context.expect(Half && CFG.addSuccessor(*Entry, *True, *Half).has_value() && + CFG.addSuccessor(*Entry, *False, *Half).has_value(), + "fallthrough CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + Context.expect(Builder.setInsertPoint(*Entry).has_value(), + "fallthrough insertion point must be established"); + rcc::MachineInstr *Conditional = + buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BEQ, *True); + rcc::MachineInstr *Jump = buildJAL(Context, Builder, *False); + if (Conditional == nullptr || Jump == nullptr) + return; + setPostRAProperties(*Function); + auto Liveness = rcc::rebuildMachineLiveness(*Function, Target); + Context.expect(Liveness.has_value(), "fallthrough fixture liveness must be rebuilt"); + rcc::MachineAnalysisManager Analyses; + rcc::MachineFallthroughPass Pass(rcc::riscv::branchInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "fallthrough normalization failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && Conditional->next() == nullptr && + Jump->parent() == nullptr, + "fallthrough normalization must remove a jump to the next block"); +} + +void testFallthroughMaterialization(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "fallthrough_materialization", 8); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Padding = appendBlock(Context, *Function); + rcc::MachineBasicBlock *True = appendBlock(Context, *Function); + rcc::MachineBasicBlock *False = appendBlock(Context, *Function); + if (Entry == nullptr || Padding == nullptr || True == nullptr || False == nullptr) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto CFG = Function->cfgUpdater(); + Context.expect(Half && CFG.addSuccessor(*Entry, *True, *Half).has_value() && + CFG.addSuccessor(*Entry, *False, *Half).has_value(), + "materialized fallthrough CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + Context.expect(Builder.setInsertPoint(*Entry).has_value(), + "materialized fallthrough insertion point must be established"); + rcc::MachineInstr *Conditional = + buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *True); + if (Conditional == nullptr) + return; + setPostRAProperties(*Function); + Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(), + "materialized fallthrough liveness must be rebuilt"); + rcc::MachineAnalysisManager Analyses; + rcc::MachineFallthroughPass Pass(rcc::riscv::branchInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "fallthrough materialization failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && Conditional->next() != nullptr && + Conditional->next()->opcode() == opcode(rcc::riscv::InstructionOpcode::JAL) && + branchTarget(*Conditional->next()) == False, + "fallthrough normalization must materialize a displaced false edge"); +} + +void testBranchOptimization(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "branch_optimization", 9); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Forward = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Dead = appendBlock(Context, *Function); + rcc::MachineBasicBlock *AddressTaken = appendBlock(Context, *Function); + if (Entry == nullptr || Forward == nullptr || Exit == nullptr || Dead == nullptr || + AddressTaken == nullptr) { + return; + } + AddressTaken->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken); + auto CFG = Function->cfgUpdater(); + Context.expect(CFG.addSuccessor(*Entry, *Forward, rcc::BranchProbability::one()).has_value() && + CFG.addSuccessor(*Forward, *Exit, rcc::BranchProbability::one()).has_value(), + "branch-optimization CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Entry) || buildJAL(Context, Builder, *Forward) == nullptr || + !Builder.setInsertPoint(*Forward) || buildJAL(Context, Builder, *Exit) == nullptr || + !Builder.setInsertPoint(*Dead) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12), + physical(rcc::riscv::PhysicalRegisterID::X11), 1) == nullptr || + !Builder.setInsertPoint(*AddressTaken) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X13), + physical(rcc::riscv::PhysicalRegisterID::X11), 2) == nullptr) { + return; + } + setSelectedProperties(*Function); + rcc::MachineAnalysisManager Analyses; + rcc::MachineBranchOptimizationPass Pass(rcc::riscv::branchInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "branch optimization failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && !Forward->isLinked() && !Dead->isLinked() && + AddressTaken->isLinked() && Entry->successors().size() == 1 && + Entry->successors().front().Block == Exit && + branchTarget(*Entry->lastInstruction()) == Exit, + "branch optimization must thread empty blocks and delete unreachable code"); + + auto ConstantFunction = makeFunction(Context, "constant_branch", 10); + if (!ConstantFunction) + return; + rcc::MachineBasicBlock *ConstantEntry = appendBlock(Context, *ConstantFunction); + rcc::MachineBasicBlock *Taken = appendBlock(Context, *ConstantFunction); + rcc::MachineBasicBlock *NotTaken = appendBlock(Context, *ConstantFunction); + if (ConstantEntry == nullptr || Taken == nullptr || NotTaken == nullptr) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto ConstantCFG = ConstantFunction->cfgUpdater(); + Context.expect(Half && ConstantCFG.addSuccessor(*ConstantEntry, *Taken, *Half).has_value() && + ConstantCFG.addSuccessor(*ConstantEntry, *NotTaken, *Half).has_value(), + "constant-branch CFG construction must succeed"); + rcc::MachineIRBuilder ConstantBuilder(*ConstantFunction, Target); + if (!ConstantBuilder.setInsertPoint(*ConstantEntry)) + return; + auto Pending = ConstantBuilder.buildInstr(opcode(rcc::riscv::InstructionOpcode::BEQ)); + if (!Pending || !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) || + !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) || + !Pending->addBasicBlock(*Taken) || !Pending->build() || + buildJAL(Context, ConstantBuilder, *NotTaken) == nullptr) { + Context.expect(false, "constant branch construction must succeed"); + return; + } + setSelectedProperties(*ConstantFunction); + rcc::MachineAnalysisManager ConstantAnalyses; + auto ConstantResult = Pass.run(*ConstantFunction, Target, ConstantAnalyses); + if (!ConstantResult) + std::cerr << "constant branch folding failure: " << ConstantResult.error().message() << '\n'; + Context.expect( + ConstantResult && ConstantResult->Changed && ConstantEntry->successors().size() == 1 && + ConstantEntry->successors().front().Block == Taken && + branchTarget(*ConstantEntry->lastInstruction()) == Taken && !NotTaken->isLinked(), + "branch optimization must fold an always-true RV64GC branch"); +} + +void testTailDuplication(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "tail_duplication", 11); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Left = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Right = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Tail = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function); + if (Entry == nullptr || Left == nullptr || Right == nullptr || Tail == nullptr || + Exit == nullptr) { + return; + } + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto CFG = Function->cfgUpdater(); + Context.expect(Half && CFG.addSuccessor(*Entry, *Left, *Half).has_value() && + CFG.addSuccessor(*Entry, *Right, *Half).has_value() && + CFG.addSuccessor(*Left, *Tail, rcc::BranchProbability::one()).has_value() && + CFG.addSuccessor(*Right, *Tail, rcc::BranchProbability::one()).has_value() && + CFG.addSuccessor(*Tail, *Exit, rcc::BranchProbability::one()).has_value(), + "tail-duplication CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Entry) || + buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *Left) == nullptr || + buildJAL(Context, Builder, *Right) == nullptr || !Builder.setInsertPoint(*Left) || + buildJAL(Context, Builder, *Tail) == nullptr || !Builder.setInsertPoint(*Right) || + buildJAL(Context, Builder, *Tail) == nullptr || !Builder.setInsertPoint(*Tail) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12), + physical(rcc::riscv::PhysicalRegisterID::X11), 1) == nullptr || + buildJAL(Context, Builder, *Exit) == nullptr) { + return; + } + setRelaxationProperties(Context, *Function, Target); + rcc::MachineAnalysisManager Analyses; + rcc::MachineTailDuplicationPass Pass(rcc::riscv::branchInfo()); + Tail->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken); + auto Protected = Pass.run(*Function, Target, Analyses); + Context.expect(Protected && !Protected->Changed && Tail->isLinked(), + "tail duplication must preserve an address-taken tail"); + Tail->attributes().clear(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "tail duplication failure: " << Result.error().message() << '\n'; + Context.expect( + Result && Result->Changed && !Tail->isLinked() && Left->instructionCount() == 2 && + Right->instructionCount() == 2 && + Left->firstInstruction()->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) && + Right->firstInstruction()->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) && + branchTarget(*Left->lastInstruction()) == Exit && + branchTarget(*Right->lastInstruction()) == Exit && + rcc::verifyTrackedMachineLiveness(*Function, Target).has_value(), + "tail duplication must clone a small shared tail and repair liveness"); +} + +void testTailMerge(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "tail_merge", 12); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *First = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Second = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function); + if (Entry == nullptr || First == nullptr || Second == nullptr || Exit == nullptr) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto CFG = Function->cfgUpdater(); + Context.expect(Half && CFG.addSuccessor(*Entry, *First, *Half).has_value() && + CFG.addSuccessor(*Entry, *Second, *Half).has_value() && + CFG.addSuccessor(*First, *Exit, rcc::BranchProbability::one()).has_value() && + CFG.addSuccessor(*Second, *Exit, rcc::BranchProbability::one()).has_value(), + "tail-merge CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Entry) || + buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *First) == nullptr || + buildJAL(Context, Builder, *Second) == nullptr || !Builder.setInsertPoint(*First) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12), + physical(rcc::riscv::PhysicalRegisterID::X11), 7) == nullptr || + buildJAL(Context, Builder, *Exit) == nullptr || !Builder.setInsertPoint(*Second) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12), + physical(rcc::riscv::PhysicalRegisterID::X11), 7) == nullptr || + buildJAL(Context, Builder, *Exit) == nullptr) { + return; + } + setRelaxationProperties(Context, *Function, Target); + rcc::MachineAnalysisManager Analyses; + rcc::MachineTailMergePass Pass(rcc::riscv::branchInfo()); + Second->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken); + auto Protected = Pass.run(*Function, Target, Analyses); + Context.expect(Protected && !Protected->Changed && Second->isLinked(), + "tail merge must preserve an address-taken duplicate block"); + Second->attributes().clear(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "tail merge failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && !Second->isLinked() && + Entry->successors().size() == 1 && + Entry->successors().front().Block == First && + branchTarget(*Entry->firstInstruction()) == First && + branchTarget(*Entry->lastInstruction()) == First && + rcc::verifyTrackedMachineLiveness(*Function, Target).has_value(), + "tail merge must redirect identical tails and repair liveness"); +} + +void testCopyPropagation(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "pre_ra_copy_propagation", 13); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + auto I64 = rcc::LLT::makeInteger(64); + if (Block == nullptr || !I64) + return; + auto Source = Function->registerInfo().createVirtualRegister(*I64); + auto CopyResult = Function->registerInfo().createVirtualRegister(*I64); + auto ResultReg = Function->registerInfo().createVirtualRegister(*I64); + if (!Source || !CopyResult || !ResultReg || !constrainGPR64(Context, *Function, *Source) || + !constrainGPR64(Context, *Function, *CopyResult) || + !constrainGPR64(Context, *Function, *ResultReg)) { + return; + } + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Block)) + return; + auto Copy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY)); + if (!Copy || !Copy->addDef(*CopyResult) || !Copy->addUse(*Source) || !Copy->build()) + return; + rcc::MachineInstr *Add = buildADDI(Context, Builder, *ResultReg, *CopyResult, 3); + if (Add == nullptr) + return; + setSelectedProperties(*Function); + rcc::MachineAnalysisManager Analyses; + rcc::MachineCopyPropagationPass PreRA(rcc::riscv::ssaInfo(), + rcc::MachineCopyPropagationStage::PreRegisterAllocation); + auto PreResult = PreRA.run(*Function, Target, Analyses); + if (!PreResult) + std::cerr << "pre-RA COPY propagation failure: " << PreResult.error().message() << '\n'; + Context.expect(PreResult && PreResult->Changed && Block->instructionCount() == 1 && + Add->operandAt(1)->registerValue() == *Source && + !Function->registerInfo().isKnownVirtualRegister(*CopyResult), + "pre-RA COPY propagation must replace uses and retire the COPY result"); + + auto PostFunction = makeFunction(Context, "post_ra_copy_propagation", 14); + if (!PostFunction) + return; + rcc::MachineBasicBlock *PostBlock = appendBlock(Context, *PostFunction); + if (PostBlock == nullptr) + return; + rcc::MachineIRBuilder PostBuilder(*PostFunction, Target); + if (!PostBuilder.setInsertPoint(*PostBlock)) + return; + auto PhysicalCopy = PostBuilder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY)); + if (!PhysicalCopy || !PhysicalCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::X11)) || + !PhysicalCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !PhysicalCopy->build()) { + return; + } + rcc::MachineInstr *PhysicalAdd = + buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X12), + physical(rcc::riscv::PhysicalRegisterID::X11), 5); + rcc::MachineInstr *SourceClobber = + buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X10), + physical(rcc::riscv::PhysicalRegisterID::X0), 9); + rcc::MachineInstr *AfterClobber = + buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X13), + physical(rcc::riscv::PhysicalRegisterID::X11), 6); + if (PhysicalAdd == nullptr || SourceClobber == nullptr || AfterClobber == nullptr) + return; + setPostRAProperties(*PostFunction); + Context.expect(rcc::rebuildMachineLiveness(*PostFunction, Target).has_value(), + "post-RA COPY propagation liveness must be rebuilt"); + rcc::MachineAnalysisManager PostAnalyses; + rcc::MachineCopyPropagationPass PostRA(rcc::riscv::ssaInfo(), + rcc::MachineCopyPropagationStage::PostRegisterAllocation); + auto PostResult = PostRA.run(*PostFunction, Target, PostAnalyses); + if (!PostResult) + std::cerr << "post-RA COPY propagation failure: " << PostResult.error().message() << '\n'; + Context.expect(PostResult && PostResult->Changed && + PhysicalAdd->operandAt(1)->registerValue() == + physical(rcc::riscv::PhysicalRegisterID::X10) && + AfterClobber->operandAt(1)->registerValue() == + physical(rcc::riscv::PhysicalRegisterID::X11) && + rcc::verifyTrackedMachineLiveness(*PostFunction, Target).has_value(), + "post-RA COPY propagation must stop at clobbers and repair liveness"); +} + +void testStackSlotColoring(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "stack_slot_coloring", 15); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + auto Align = rcc::Alignment::create(8); + if (Block == nullptr || !Align) + return; + rcc::StackObjectDesc Description; + Description.Kind = rcc::StackObjectKind::SpillSlot; + Description.Size = 8; + Description.Align = *Align; + auto FirstSlot = Function->frameInfo().createObject(Description); + auto SecondSlot = Function->frameInfo().createObject(Description); + if (!FirstSlot || !SecondSlot) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Block) || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X10), *FirstSlot, + rcc::MemoryAccessFlag::Store) == nullptr || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD, + physical(rcc::riscv::PhysicalRegisterID::X11), *FirstSlot, + rcc::MemoryAccessFlag::Load) == nullptr || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X12), *SecondSlot, + rcc::MemoryAccessFlag::Store) == nullptr || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD, + physical(rcc::riscv::PhysicalRegisterID::X13), *SecondSlot, + rcc::MemoryAccessFlag::Load) == nullptr) { + return; + } + setPostRAProperties(*Function); + Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(), + "stack-slot coloring liveness must be rebuilt"); + rcc::MachineAnalysisManager Analyses; + rcc::MachineStackSlotColoringPass Pass; + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "stack-slot coloring failure: " << Result.error().message() << '\n'; + const rcc::StackObject *Colored = Function->frameInfo().object(*SecondSlot); + rcc::MachineInstr *SecondStore = Block->firstInstruction(); + SecondStore = SecondStore == nullptr ? nullptr : SecondStore->next(); + SecondStore = SecondStore == nullptr ? nullptr : SecondStore->next(); + const auto *Frame = + SecondStore == nullptr ? nullptr : SecondStore->operandAt(1)->getIf(); + Context.expect( + Result && Result->Changed && Colored != nullptr && Colored->description() != nullptr && + Colored->description()->IsDead && Frame != nullptr && Frame->Index == *FirstSlot && + SecondStore->memOperands().front()->description().Base.frameIndex() == *FirstSlot, + "stack-slot coloring must coalesce disjoint spills and rewrite MMOs"); + + auto OverlapFunction = makeFunction(Context, "stack_slot_overlap", 17); + if (!OverlapFunction) + return; + rcc::MachineBasicBlock *OverlapBlock = appendBlock(Context, *OverlapFunction); + auto Left = OverlapFunction->frameInfo().createObject(Description); + auto Right = OverlapFunction->frameInfo().createObject(Description); + if (OverlapBlock == nullptr || !Left || !Right) + return; + rcc::MachineIRBuilder OverlapBuilder(*OverlapFunction, Target); + if (!OverlapBuilder.setInsertPoint(*OverlapBlock) || + buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X10), *Left, + rcc::MemoryAccessFlag::Store) == nullptr || + buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X11), *Right, + rcc::MemoryAccessFlag::Store) == nullptr || + buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::LD, + physical(rcc::riscv::PhysicalRegisterID::X12), *Left, + rcc::MemoryAccessFlag::Load) == nullptr || + buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::LD, + physical(rcc::riscv::PhysicalRegisterID::X13), *Right, + rcc::MemoryAccessFlag::Load) == nullptr) { + return; + } + setPostRAProperties(*OverlapFunction); + Context.expect(rcc::rebuildMachineLiveness(*OverlapFunction, Target).has_value(), + "overlapping stack-slot fixture liveness must be rebuilt"); + auto OverlapResult = Pass.run(*OverlapFunction, Target, Analyses); + const rcc::StackObject *RightObject = OverlapFunction->frameInfo().object(*Right); + Context.expect(OverlapResult && !OverlapResult->Changed && RightObject != nullptr && + RightObject->description() != nullptr && !RightObject->description()->IsDead, + "stack-slot coloring must preserve overlapping spill lifetimes"); +} + +void testRISCVMemoryFoldingBoundary(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "riscv_memory_folding", 16); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + auto I64 = rcc::LLT::makeInteger(64); + auto Align = rcc::Alignment::create(8); + if (Block == nullptr || !I64 || !Align) + return; + auto Loaded = Function->registerInfo().createVirtualRegister(*I64); + auto ResultReg = Function->registerInfo().createVirtualRegister(*I64); + if (!Loaded || !ResultReg || !constrainGPR64(Context, *Function, *Loaded) || + !constrainGPR64(Context, *Function, *ResultReg)) { + return; + } + rcc::StackObjectDesc Description; + Description.Size = 8; + Description.Align = *Align; + auto Slot = Function->frameInfo().createObject(std::move(Description)); + if (!Slot) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + rcc::MachineInstr *Load = nullptr; + rcc::MachineInstr *Consumer = nullptr; + if (Builder.setInsertPoint(*Block)) { + Load = buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD, *Loaded, *Slot, + rcc::MemoryAccessFlag::Load); + Consumer = buildADDI(Context, Builder, *ResultReg, *Loaded, 1); + } + if (Load == nullptr || Consumer == nullptr) + return; + setSelectedProperties(*Function); + Function->properties().set(rcc::MachineFunctionProperty::IsSSA); + rcc::MachineAnalysisManager Analyses; + rcc::MachineMemoryFoldingPass Pass(rcc::riscv::memoryFoldingInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "RISC-V memory folding boundary failure: " << Result.error().message() << '\n'; + Context.expect(Result && !Result->Changed && Load->parent() == Block && + Load->memOperands().size() == 1 && Consumer->parent() == Block, + "RV64GC must explicitly decline unsupported folded-memory ALU forms"); +} + +void testShrinkWrappedFrameRegion(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "shrink_wrapped_frame", 20); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Cold = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Hot = appendBlock(Context, *Function); + auto Align = rcc::Alignment::create(8); + auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10); + auto HotProbability = rcc::BranchProbability::fromFraction(9, 10); + if (Entry == nullptr || Cold == nullptr || Hot == nullptr || !Align || !ColdProbability || + !HotProbability) { + return; + } + auto CFG = Function->cfgUpdater(); + if (!CFG.addSuccessor(*Entry, *Cold, *ColdProbability) || + !CFG.addSuccessor(*Entry, *Hot, *HotProbability)) { + return; + } + rcc::StackObjectDesc SpillDescription; + SpillDescription.Kind = rcc::StackObjectKind::SpillSlot; + SpillDescription.Size = 8; + SpillDescription.Align = *Align; + auto Spill = Function->frameInfo().createObject(SpillDescription); + if (!Spill) + return; + + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Cold) || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X10), *Spill, + rcc::MemoryAccessFlag::Store) == nullptr) { + return; + } + auto ColdReturn = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET)); + if (!ColdReturn || !ColdReturn->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !ColdReturn->build() || !Builder.setInsertPoint(*Hot)) { + return; + } + auto HotReturn = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET)); + if (!HotReturn || !HotReturn->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !HotReturn->build()) { + return; + } + + setPostRAProperties(*Function); + Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(), + "shrink-wrapping fixture liveness must be rebuilt"); + rcc::MachineAnalysisManager Analyses; + rcc::MachineShrinkWrappingPass Shrink{rcc::riscv::frameLowering()}; + auto Shrunk = Shrink.run(*Function, Target, Analyses); + Context.expect(Shrunk && Shrunk->Changed && + Function->frameInfo().state().SavePoints.size() == 1 && + Function->frameInfo().state().SavePoints.front().Block == Cold->stableID() && + Function->frameInfo().state().RestorePoints.size() == 1 && + Function->frameInfo().state().RestorePoints.front().Block == Cold->stableID(), + "shrink wrapping must select the cold frame region only"); + if (!Shrunk) + return; + + rcc::MachinePassManager FramePipeline; + Context.expect(FramePipeline + .addPass(std::make_unique( + rcc::riscv::frameLowering())) + .has_value(), + "shrink-wrapped PEI pass must register"); + auto Lowered = FramePipeline.run(*Function, Target, Analyses); + if (!Lowered) + std::cerr << "shrink-wrapped PEI failure: " << Lowered.error().message() << '\n'; + const bool EntryHasFrameCode = std::ranges::any_of( + Function->allInstructions(), [Entry](const rcc::MachineInstr *Instruction) { + return Instruction != nullptr && Instruction->parent() == Entry && + (Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup) || + Instruction->flags().has(rcc::MachineInstrFlag::FrameDestroy)); + }); + const bool ColdHasSetup = std::ranges::any_of( + Function->allInstructions(), [Cold](const rcc::MachineInstr *Instruction) { + return Instruction != nullptr && Instruction->parent() == Cold && + Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup); + }); + const bool ColdHasDestroy = std::ranges::any_of( + Function->allInstructions(), [Cold](const rcc::MachineInstr *Instruction) { + return Instruction != nullptr && Instruction->parent() == Cold && + Instruction->flags().has(rcc::MachineInstrFlag::FrameDestroy); + }); + Context.expect( + Lowered && !EntryHasFrameCode && ColdHasSetup && ColdHasDestroy && + Function->properties().has(rcc::MachineFunctionProperty::FrameFinalized), + "PEI must materialize a shrink-wrapped prologue and epilogue only in the frame region"); + + rcc::MachineBlockPlacementPass Placement; + auto Placed = Placement.run(*Function, Target, Analyses); + Context.expect(Placed && Cold->next() == nullptr, + "block placement must keep a shrink-wrapped frame region as a layout suffix"); +} + +void testCompressedAndConditionalRelaxation(TestContext &Context, + const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "branch_relaxation", 3); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Padding = appendBlock(Context, *Function); + rcc::MachineBasicBlock *Far = appendBlock(Context, *Function); + if (Entry == nullptr || Padding == nullptr || Far == nullptr) + return; + auto Half = rcc::BranchProbability::fromFraction(1, 2); + auto CFG = Function->cfgUpdater(); + Context.expect(Half && CFG.addSuccessor(*Entry, *Far, *Half).has_value() && + CFG.addSuccessor(*Entry, *Padding, *Half).has_value() && + CFG.addSuccessor(*Padding, *Far, rcc::BranchProbability::one()).has_value(), + "relaxation CFG construction must succeed"); + rcc::MachineIRBuilder Builder(*Function, Target); + Context.expect(Builder.setInsertPoint(*Entry).has_value(), + "relaxation entry insertion point must be established"); + rcc::MachineInstr *Conditional = + buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BEQ, *Far); + Context.expect(Builder.setInsertPoint(*Padding).has_value(), + "relaxation padding insertion point must be established"); + for (std::size_t Index = 0; Index < 1100; ++Index) { + if (buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X0), + physical(rcc::riscv::PhysicalRegisterID::X0), 0) == nullptr) { + return; + } + } + (void)buildJAL(Context, Builder, *Far); + if (Conditional == nullptr) + return; + setRelaxationProperties(Context, *Function, Target); + rcc::MachineAnalysisManager Analyses; + rcc::MachineBranchRelaxationPass Pass(rcc::riscv::branchInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "conditional relaxation failure: " << Result.error().message() << '\n'; + Context.expect(Result && Result->Changed && + Conditional->opcode() == opcode(rcc::riscv::InstructionOpcode::BNE) && + branchTarget(*Conditional) == Padding, + "out-of-range conditional branch must invert onto the local fallthrough"); + rcc::MachineInstr *LongJump = Conditional->next(); + Context.expect(LongJump != nullptr && + LongJump->opcode() == opcode(rcc::riscv::InstructionOpcode::JAL) && + branchTarget(*LongJump) == Far, + "conditional relaxation must insert a reachable long jump to the old target"); + + auto CompressedFunction = makeFunction(Context, "compressed_relaxation", 4); + if (!CompressedFunction) + return; + rcc::MachineBasicBlock *CompressedEntry = appendBlock(Context, *CompressedFunction); + rcc::MachineBasicBlock *CompressedTarget = appendBlock(Context, *CompressedFunction); + rcc::MachineIRBuilder CompressedBuilder(*CompressedFunction, Target); + Context.expect(CompressedEntry != nullptr && CompressedTarget != nullptr && + CompressedBuilder.setInsertPoint(*CompressedEntry).has_value(), + "compressed relaxation fixture must be valid"); + if (CompressedEntry == nullptr || CompressedTarget == nullptr) + return; + auto Pending = CompressedBuilder.buildInstr(opcode(rcc::riscv::InstructionOpcode::C_J)); + if (!Pending || !Pending->addBasicBlock(*CompressedTarget)) { + Context.expect(false, "compressed jump construction must succeed"); + return; + } + auto Compressed = Pending->build(); + Context.expect(Compressed.has_value(), "compressed jump insertion must succeed"); + if (!Compressed) + return; + auto Expanded = + rcc::riscv::branchInfo().relaxBranch(**Compressed, 4096, *CompressedFunction, Target); + Context.expect(Expanded && *Expanded && CompressedEntry->firstInstruction() != nullptr && + CompressedEntry->firstInstruction()->opcode() == + opcode(rcc::riscv::InstructionOpcode::JAL) && + (*Compressed)->parent() == nullptr, + "compressed jump relaxation must monotonically expand to JAL"); +} + +void testLongJALRelaxation(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "long_jal", 5); + if (!Function) + return; + rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function); + rcc::MachineBasicBlock *TargetBlock = appendBlock(Context, *Function); + if (Entry == nullptr || TargetBlock == nullptr) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + Context.expect(Builder.setInsertPoint(*Entry).has_value(), + "long JAL insertion point must be established"); + rcc::MachineInstr *JAL = buildJAL(Context, Builder, *TargetBlock); + if (JAL == nullptr) + return; + auto X5 = rcc::riscv::physicalRegister(rcc::riscv::PhysicalRegisterID::X5); + auto X6 = rcc::riscv::physicalRegister(rcc::riscv::PhysicalRegisterID::X6); + Context.expect(X5 && X6 && TargetBlock->addLiveIn(*X5).has_value() && + TargetBlock->addLiveIn(*X6).has_value() && + JAL->setPreInstrSymbol(".Lpreserved_long_branch").has_value() && + JAL->setPostInstrSymbol(".Lafter_long_branch").has_value(), + "long JAL live-in and annotation fixture must be valid"); + if (!X5 || !X6) + return; + auto TooFar = rcc::riscv::branchInfo().relaxBranch(*JAL, std::numeric_limits::max(), + *Function, Target); + Context.expect(!TooFar && TooFar.error().code() == rcc::ErrorCode::BranchRelaxationFailed && + JAL->parent() == Entry, + "long JAL relaxation must reject an unencodable PC-relative distance atomically"); + auto Relaxed = rcc::riscv::branchInfo().relaxBranch(*JAL, 2 * 1024 * 1024, *Function, Target); + if (!Relaxed) + std::cerr << "long JAL relaxation failure: " << Relaxed.error().message() << '\n'; + rcc::MachineInstr *High = Entry->firstInstruction(); + rcc::MachineInstr *Low = High == nullptr ? nullptr : High->next(); + const rcc::Register *Scratch = High == nullptr || High->operandAt(0) == nullptr + ? nullptr + : High->operandAt(0)->getIf(); + const std::optional ScratchPhysical = + Scratch == nullptr ? std::nullopt : Scratch->physicalRegister(); + Context.expect(Relaxed && *Relaxed && High != nullptr && Low != nullptr && + High->opcode() == opcode(rcc::riscv::InstructionOpcode::AUIPC) && + Low->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR) && + ScratchPhysical && *ScratchPhysical != *X5 && *ScratchPhysical != *X6 && + High->operandAt(1)->targetFlags() == 2 && + Low->operandAt(2)->targetFlags() == 4 && + High->flags().has(rcc::MachineInstrFlag::BundledWithSucc) && + Low->flags().has(rcc::MachineInstrFlag::BundledWithPred) && + High->preInstrSymbol() == ".Lpreserved_long_branch" && + Low->postInstrSymbol() == ".Lafter_long_branch" && JAL->parent() == nullptr, + "out-of-range JAL must use a dead scratch and preserve its paired annotations"); +} + +void testPseudoPreparation(TestContext &Context, const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "pseudo_preparation", 6); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + auto I64 = rcc::LLT::makeInteger(64); + auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + if (Block == nullptr || !I64 || !F64) + return; + auto Source = Function->registerInfo().createVirtualRegister(*I64); + auto Destination = Function->registerInfo().createVirtualRegister(*F64); + const auto GPRBank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + const auto FPRBank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::FPR)); + const auto GPRClass = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::GPR)); + const auto FPRClass = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::FPR64)); + Context.expect(Source && Destination && + Function->registerInfo().setRegisterBank(*Source, GPRBank).has_value() && + Function->registerInfo().setRegisterClass(*Source, GPRClass).has_value() && + Function->registerInfo().setRegisterBank(*Destination, FPRBank).has_value() && + Function->registerInfo().setRegisterClass(*Destination, FPRClass).has_value(), + "pseudo preparation virtual-register constraints must be valid"); + if (!Source || !Destination) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Block)) + return; + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY)); + if (!Pending || !Pending->addDef(*Destination) || !Pending->addUse(*Source)) + return; + auto Copy = Pending->build(); + if (!Copy) + return; + setSelectedProperties(*Function); + rcc::MachineAnalysisManager Analyses; + rcc::PreparePostRAPseudosPass Pass(rcc::riscv::pseudoExpansionInfo()); + auto Result = Pass.run(*Function, Target, Analyses); + Context.expect(Result && Result->Changed && (*Copy)->operandAt(0)->targetFlags() == 16 && + (*Copy)->operandAt(1)->targetFlags() == 16, + "pre-RA pseudo preparation must retain a 64-bit cross-bank COPY width"); +} + +void testPostRAPseudoExpansionAndFinalization(TestContext &Context, + const rcc::TargetDescription &Target) { + auto Function = makeFunction(Context, "post_ra_pseudos", 7); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + if (Block == nullptr) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Block)) + return; + + auto GPRCopy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY)); + if (!GPRCopy || !GPRCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::X11)) || + !GPRCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || !GPRCopy->build()) { + Context.expect(false, "post-RA GPR copy construction must succeed"); + return; + } + auto CrossCopy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY)); + if (!CrossCopy || !CrossCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::F10), {}, {}, 16) || + !CrossCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10), {}, {}, 16) || + !CrossCopy->build()) { + Context.expect(false, "post-RA cross-bank copy construction must succeed"); + return; + } + auto Call = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoCALL)); + auto Preserved = rcc::riscv::createCallPreservedMask(); + if (!Call || !Call->addExternalSymbol("callee", 0, 1) || !Call->addImmediate(11) || + !Call->addRegisterMask(Preserved) || + !Call->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || !Call->build()) { + Context.expect(false, "post-RA call pseudo construction must succeed"); + return; + } + auto Return = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET)); + if (!Return || !Return->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !Return->build()) { + Context.expect(false, "post-RA return pseudo construction must succeed"); + return; + } + + setRelaxationProperties(Context, *Function, Target); + Function->properties().clear(rcc::MachineFunctionProperty::NoExecutablePseudos); + rcc::MachineAnalysisManager Analyses; + rcc::ExpandPostRAPseudosPass Expansion(rcc::riscv::pseudoExpansionInfo()); + auto Expanded = Expansion.run(*Function, Target, Analyses); + if (!Expanded) + std::cerr << "post-RA pseudo expansion failure: " << Expanded.error().message() << '\n'; + std::size_t ADDICount = 0; + std::size_t CrossMoveCount = 0; + std::size_t CallCount = 0; + std::size_t ReturnCount = 0; + for (rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + ADDICount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI); + CrossMoveCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::FMV_D_X); + CallCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_CALL); + ReturnCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_RET); + } + Context.expect(Expanded && Expanded->Changed && ADDICount == 1 && CrossMoveCount == 1 && + CallCount == 1 && ReturnCount == 1, + "post-RA expansion must lower copies, calls, and returns to real RV64GC code"); + if (!Expanded) + return; + + rcc::MachinePassManager Pipeline; + Context.expect(Pipeline.addPass(std::make_unique()) && + Pipeline.addPass(std::make_unique( + rcc::riscv::branchInfo())), + "finalization passes must register"); + auto Finalized = Pipeline.run(*Function, Target, Analyses); + if (!Finalized) + std::cerr << "final encodability failure: " << Finalized.error().message() << '\n'; + Context.expect(Finalized && + Function->properties().has(rcc::MachineFunctionProperty::NoExecutablePseudos), + "emission finalization must exclusively publish NoExecutablePseudos"); +} + +void testCanonicalOptimizationPipelines(TestContext &Context, + const rcc::TargetDescription &Target) { + const rcc::MachineOptimizationPipelineTargetInfo TargetInfo = + rcc::riscv::optimizationPipelineInfo(); + rcc::MachinePassManager SelectedPipeline; + rcc::MachinePassManager PreRAPipeline; + Context.expect(rcc::addSelectedSSAOptimizationPipeline(SelectedPipeline, TargetInfo) && + SelectedPipeline.size() == 6, + "canonical selected-SSA optimization pipeline must register completely"); + Context.expect(rcc::addPreRegisterAllocationOptimizationPipeline(PreRAPipeline, TargetInfo) && + PreRAPipeline.size() == 5, + "canonical pre-RA optimization pipeline must register completely"); + + auto Function = makeFunction(Context, "canonical_post_ra_pipeline", 21); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + if (Block == nullptr) + return; + auto SlotAlignment = rcc::Alignment::create(8); + if (!SlotAlignment) + return; + rcc::StackObjectDesc SpillDescription; + SpillDescription.Kind = rcc::StackObjectKind::SpillSlot; + SpillDescription.Size = 8; + SpillDescription.Align = *SlotAlignment; + auto Spill = Function->frameInfo().createObject(SpillDescription); + if (!Spill) + return; + rcc::MachineIRBuilder Builder(*Function, Target); + if (!Builder.setInsertPoint(*Block) || + buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X10), + physical(rcc::riscv::PhysicalRegisterID::X0), 42) == nullptr || + buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD, + physical(rcc::riscv::PhysicalRegisterID::X10), *Spill, + rcc::MemoryAccessFlag::Store) == nullptr) { + return; + } + auto Return = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET)); + if (!Return || !Return->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || + !Return->build()) { + Context.expect(false, "canonical post-RA return construction must succeed"); + return; + } + setPostRAProperties(*Function); + if (!rcc::rebuildMachineLiveness(*Function, Target)) { + Context.expect(false, "canonical post-RA input liveness must be valid"); + return; + } + + rcc::MachinePassManager Pipeline; + auto Added = rcc::addPostRegisterAllocationPipeline(Pipeline, TargetInfo); + Context.expect(Added && Pipeline.size() == 20, + "canonical post-RA pipeline must register every required pass"); + if (!Added) + return; + rcc::MachineAnalysisManager Analyses; + auto Registered = rcc::registerMachineOptimizationAnalyses(Analyses); + Context.expect(Registered.has_value(), + "canonical optimization analyses must register completely"); + if (!Registered) + return; + auto Result = Pipeline.run(*Function, Target, Analyses); + if (!Result) + std::cerr << "canonical post-RA pipeline failure: " << Result.error().message() << '\n'; + const bool HasReturn = + std::ranges::any_of(Function->allInstructions(), [](const rcc::MachineInstr *Instruction) { + return Instruction != nullptr && + Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_RET); + }); + const bool HasFrameSetup = + std::ranges::any_of(Function->allInstructions(), [](const rcc::MachineInstr *Instruction) { + return Instruction != nullptr && + Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup); + }); + Context.expect(Result && HasReturn && HasFrameSetup && + Function->properties().has(rcc::MachineFunctionProperty::FrameFinalized) && + Function->properties().has(rcc::MachineFunctionProperty::NoFrameIndices) && + Function->properties().has(rcc::MachineFunctionProperty::NoExecutablePseudos), + "canonical post-RA pipeline must reach final encodable RV64GC MachineIR"); +} + +void testTargetLateExpansionRejectsSizeGrowth(TestContext &Context) { + auto Target = createLateExpansionSizeTarget(); + Context.expect(Target.has_value(), "target-late size-contract target construction must succeed"); + if (!Target) + return; + auto Function = makeFunction(Context, "late_expansion_size_growth", 22); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + if (Block == nullptr) + return; + rcc::MachineIRBuilder Builder(*Function, *Target); + if (!Builder.setInsertPoint(*Block)) + return; + auto Pending = Builder.buildInstr(GrowingLatePseudoOpcode); + if (!Pending) + return; + auto Pseudo = Pending->build(); + Context.expect(Pseudo.has_value(), "target-late growth pseudo construction must succeed"); + if (!Pseudo) + return; + + setPostRAProperties(*Function); + auto StackAlignment = rcc::Alignment::create(16); + if (!StackAlignment || !Function->frameInfo().freeze(0, *StackAlignment)) + return; + Function->properties().set(rcc::MachineFunctionProperty::FrameFinalized); + Function->properties().set(rcc::MachineFunctionProperty::NoFrameIndices); + if (!rcc::rebuildMachineLiveness(*Function, *Target)) + return; + + GrowingLatePseudoInfo ExpansionInfo; + FixedSizeBranchInfo BranchInfo; + rcc::TargetLatePseudoExpansionPass Pass(ExpansionInfo, BranchInfo); + rcc::MachineAnalysisManager Analyses; + auto Result = Pass.run(*Function, *Target, Analyses); + Context.expect(!Result && Result.error().code() == rcc::ErrorCode::PseudoExpansionFailed && + (*Pseudo)->parent() == Block && Block->firstInstruction() == *Pseudo && + Block->lastInstruction() == *Pseudo, + "target-late expansion must reject block-size growth atomically"); +} + +void testTargetLateExpansionRejectsCompensatedGrowth(TestContext &Context) { + auto Target = createLateExpansionSizeTarget(); + Context.expect(Target.has_value(), "target-late compensation target construction must succeed"); + if (!Target) + return; + auto Function = makeFunction(Context, "late_expansion_compensated_growth", 23); + if (!Function) + return; + rcc::MachineBasicBlock *Block = appendBlock(Context, *Function); + if (Block == nullptr) + return; + rcc::MachineIRBuilder Builder(*Function, *Target); + if (!Builder.setInsertPoint(*Block)) + return; + auto Shrinking = Builder.buildInstr(ShrinkingLatePseudoOpcode); + auto ShrinkingPseudo = + Shrinking ? Shrinking->build() + : rcc::Expected(std::unexpected( + rcc::Error(rcc::ErrorCode::InternalError, "no shrinking pseudo builder"))); + auto Growing = Builder.buildInstr(GrowingLatePseudoOpcode); + auto GrowingPseudo = Growing ? Growing->build() + : rcc::Expected(std::unexpected(rcc::Error( + rcc::ErrorCode::InternalError, "no growing pseudo builder"))); + Context.expect(ShrinkingPseudo.has_value() && GrowingPseudo.has_value(), + "compensated target-late pseudos must be constructed"); + if (!ShrinkingPseudo || !GrowingPseudo) + return; + + setPostRAProperties(*Function); + auto StackAlignment = rcc::Alignment::create(16); + if (!StackAlignment || !Function->frameInfo().freeze(0, *StackAlignment)) + return; + Function->properties().set(rcc::MachineFunctionProperty::FrameFinalized); + Function->properties().set(rcc::MachineFunctionProperty::NoFrameIndices); + if (!rcc::rebuildMachineLiveness(*Function, *Target)) + return; + + GrowingLatePseudoInfo ExpansionInfo; + FixedSizeBranchInfo BranchInfo; + rcc::TargetLatePseudoExpansionPass Pass(ExpansionInfo, BranchInfo); + rcc::MachineAnalysisManager Analyses; + auto Result = Pass.run(*Function, *Target, Analyses); + Context.expect( + !Result && Result.error().code() == rcc::ErrorCode::PseudoExpansionFailed && + (*ShrinkingPseudo)->parent() == Block && (*GrowingPseudo)->parent() == Block && + Block->firstInstruction() == *ShrinkingPseudo && + Block->lastInstruction() == *GrowingPseudo, + "one growing target-late expansion must not be hidden by another pseudo shrinking"); +} + +void testFinalEncodabilityRejectsInvalidInstruction(TestContext &Context, + const rcc::TargetDescription &Target) { + auto ImmediateFunction = makeFunction(Context, "final_invalid_immediate", 18); + if (!ImmediateFunction) + return; + rcc::MachineBasicBlock *ImmediateBlock = appendBlock(Context, *ImmediateFunction); + if (ImmediateBlock == nullptr) + return; + rcc::MachineIRBuilder ImmediateBuilder(*ImmediateFunction, Target); + if (!ImmediateBuilder.setInsertPoint(*ImmediateBlock)) + return; + rcc::MachineInstr *InvalidImmediate = + buildADDI(Context, ImmediateBuilder, physical(rcc::riscv::PhysicalRegisterID::X10), + physical(rcc::riscv::PhysicalRegisterID::X0), 0); + if (InvalidImmediate == nullptr || !InvalidImmediate->eraseOperand(2) || + !InvalidImmediate->insertOperand(2, rcc::MachineOperand::immediate(4096))) { + Context.expect(false, "invalid final immediate fixture must be constructed"); + return; + } + setRelaxationProperties(Context, *ImmediateFunction, Target); + rcc::MachineAnalysisManager Analyses; + rcc::FinalEncodabilityVerifierPass Verifier(rcc::riscv::branchInfo()); + auto ImmediateResult = Verifier.run(*ImmediateFunction, Target, Analyses); + Context.expect(!ImmediateResult && + ImmediateResult.error().code() == rcc::ErrorCode::FinalEncodabilityFailed, + "final encodability must reject an out-of-range target immediate"); + + auto BundleFunction = makeFunction(Context, "final_invalid_bundle", 19); + if (!BundleFunction) + return; + rcc::MachineBasicBlock *BundleBlock = appendBlock(Context, *BundleFunction); + if (BundleBlock == nullptr) + return; + rcc::MachineIRBuilder BundleBuilder(*BundleFunction, Target); + if (!BundleBuilder.setInsertPoint(*BundleBlock)) + return; + rcc::MachineInstr *InvalidBundle = + buildADDI(Context, BundleBuilder, physical(rcc::riscv::PhysicalRegisterID::X10), + physical(rcc::riscv::PhysicalRegisterID::X0), 0); + if (InvalidBundle == nullptr) + return; + InvalidBundle->setFlags({rcc::MachineInstrFlag::BundledWithSucc}); + setRelaxationProperties(Context, *BundleFunction, Target); + auto BundleResult = Verifier.run(*BundleFunction, Target, Analyses); + Context.expect(!BundleResult && + BundleResult.error().code() == rcc::ErrorCode::FinalEncodabilityFailed, + "final encodability must reject an inconsistent bundle boundary"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target construction must succeed"); + if (!Target) + return Context.result(); + testBlockPlacement(Context, *Target); + testFallthroughNormalization(Context, *Target); + testFallthroughMaterialization(Context, *Target); + testBranchOptimization(Context, *Target); + testTailDuplication(Context, *Target); + testTailMerge(Context, *Target); + testCopyPropagation(Context, *Target); + testStackSlotColoring(Context, *Target); + testRISCVMemoryFoldingBoundary(Context, *Target); + testShrinkWrappedFrameRegion(Context, *Target); + testCompressedAndConditionalRelaxation(Context, *Target); + testLongJALRelaxation(Context, *Target); + testPseudoPreparation(Context, *Target); + testPostRAPseudoExpansionAndFinalization(Context, *Target); + testCanonicalOptimizationPipelines(Context, *Target); + testTargetLateExpansionRejectsSizeGrowth(Context); + testTargetLateExpansionRejectsCompensatedGrowth(Context); + testFinalEncodabilityRejectsInvalidInstruction(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/Target/RISCVTargetTest.cpp b/backend/unittests/Target/RISCVTargetTest.cpp index 98525e9..6203461 100644 --- a/backend/unittests/Target/RISCVTargetTest.cpp +++ b/backend/unittests/Target/RISCVTargetTest.cpp @@ -149,7 +149,7 @@ void testInstructionFacts(TestContext &Context) { rcc::Opcode::fromValue(static_cast(rcc::riscv::InstructionOpcode::FENCE_I))); Context.expect( Target->instructions().descriptors().size() == - 201 + rcc::genericInstructionDescriptors().size() && + 204 + rcc::genericInstructionDescriptors().size() && Addi != nullptr && Addi->Name == "ADDI" && Addi->NumExplicitDefs == 1 && Addi->Operands.size() == 3 && Addi->Operands.back().ImmediatePredicate && Load != nullptr && Load->Flags.has(rcc::InstructionFlag::MayLoad) && @@ -173,7 +173,7 @@ void testInstructionFacts(TestContext &Context) { const auto Encodings = rcc::riscv::encodingCategories(); const auto Formats = rcc::riscv::assemblyFormats(); Context.expect(Schedules.size() == 15 && Encodings.size() == 10 && Formats.size() == 34 && - rcc::riscv::assemblyInstructions().size() == 192 && FenceAssembly != nullptr && + rcc::riscv::assemblyInstructions().size() == 195 && FenceAssembly != nullptr && FenceAssembly->Mnemonic == "fence.i" && std::ranges::any_of(Schedules, [](const rcc::riscv::SchedulingClassInfo &Class) { -- cgit v1.2.3