From 495c675771f43cfa12ab7a426a538be72fec1805 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Sun, 9 Aug 2026 17:03:19 +0800 Subject: [CodeGen] Implement machine optimization and scheduling --- backend/lib/CodeGen/CMakeLists.txt | 105 +++ backend/lib/CodeGen/InstructionDescriptor.cpp | 2 +- backend/lib/CodeGen/MachineBlockPlacement.cpp | 242 +++++++ backend/lib/CodeGen/MachineBranchOptimization.cpp | 361 ++++++++++ backend/lib/CodeGen/MachineBranchRelaxation.cpp | 194 ++++++ backend/lib/CodeGen/MachineCSE.cpp | 417 ++++++++++++ backend/lib/CodeGen/MachineCombiner.cpp | 354 ++++++++++ backend/lib/CodeGen/MachineCombinerPattern.cpp | 50 ++ backend/lib/CodeGen/MachineCompilationStatus.cpp | 6 + backend/lib/CodeGen/MachineCopyPropagation.cpp | 362 ++++++++++ backend/lib/CodeGen/MachineDCE.cpp | 199 ++++++ backend/lib/CodeGen/MachineFallthrough.cpp | 169 +++++ backend/lib/CodeGen/MachineFinalization.cpp | 217 ++++++ backend/lib/CodeGen/MachineFunction.cpp | 60 ++ backend/lib/CodeGen/MachineFunctionRecords.cpp | 32 + backend/lib/CodeGen/MachineLICM.cpp | 438 ++++++++++++ backend/lib/CodeGen/MachineMemoryFolding.cpp | 248 +++++++ backend/lib/CodeGen/MachineOperand.cpp | 12 + .../lib/CodeGen/MachineOptimizationPipeline.cpp | 156 +++++ backend/lib/CodeGen/MachinePseudoExpansion.cpp | 432 ++++++++++++ backend/lib/CodeGen/MachineScheduler.cpp | 441 +++++++++++++ backend/lib/CodeGen/MachineSchedulerInternal.h | 32 + backend/lib/CodeGen/MachineShrinkWrapping.cpp | 227 +++++++ backend/lib/CodeGen/MachineSink.cpp | 442 +++++++++++++ backend/lib/CodeGen/MachineStackSlotColoring.cpp | 320 +++++++++ backend/lib/CodeGen/MachineTailDuplication.cpp | 330 ++++++++++ backend/lib/CodeGen/MachineTailMerge.cpp | 406 ++++++++++++ backend/lib/CodeGen/PostRAScheduler.cpp | 327 +++++++++ backend/lib/CodeGen/RegisterPressure.cpp | 591 +++++++++++++++++ backend/lib/CodeGen/ScheduleDAG.cpp | 514 +++++++++++++++ backend/lib/CodeGen/ScheduleHazardRecognizer.cpp | 219 ++++++ backend/lib/CodeGen/TargetSchedulingModel.cpp | 260 ++++++++ backend/lib/Support/Error.cpp | 10 + .../Description/TargetDescriptionGenerator.cpp | 7 +- backend/lib/Target/RISCV/CMakeLists.txt | 18 + backend/lib/Target/RISCV/RISCV.rccdesc | 57 +- backend/lib/Target/RISCV/RISCVBranchInfo.cpp | 731 +++++++++++++++++++++ backend/lib/Target/RISCV/RISCVFrameLowering.cpp | 113 +++- backend/lib/Target/RISCV/RISCVMachineCombiner.cpp | 92 +++ .../lib/Target/RISCV/RISCVMachineMemoryFolding.cpp | 48 ++ .../lib/Target/RISCV/RISCVOptimizationPipeline.cpp | 19 + backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp | 512 +++++++++++++++ backend/lib/Target/RISCV/RISCVScheduling.cpp | 136 ++++ 43 files changed, 9871 insertions(+), 37 deletions(-) create mode 100644 backend/lib/CodeGen/MachineBlockPlacement.cpp create mode 100644 backend/lib/CodeGen/MachineBranchOptimization.cpp create mode 100644 backend/lib/CodeGen/MachineBranchRelaxation.cpp create mode 100644 backend/lib/CodeGen/MachineCSE.cpp create mode 100644 backend/lib/CodeGen/MachineCombiner.cpp create mode 100644 backend/lib/CodeGen/MachineCombinerPattern.cpp create mode 100644 backend/lib/CodeGen/MachineCopyPropagation.cpp create mode 100644 backend/lib/CodeGen/MachineDCE.cpp create mode 100644 backend/lib/CodeGen/MachineFallthrough.cpp create mode 100644 backend/lib/CodeGen/MachineFinalization.cpp create mode 100644 backend/lib/CodeGen/MachineLICM.cpp create mode 100644 backend/lib/CodeGen/MachineMemoryFolding.cpp create mode 100644 backend/lib/CodeGen/MachineOptimizationPipeline.cpp create mode 100644 backend/lib/CodeGen/MachinePseudoExpansion.cpp create mode 100644 backend/lib/CodeGen/MachineScheduler.cpp create mode 100644 backend/lib/CodeGen/MachineSchedulerInternal.h create mode 100644 backend/lib/CodeGen/MachineShrinkWrapping.cpp create mode 100644 backend/lib/CodeGen/MachineSink.cpp create mode 100644 backend/lib/CodeGen/MachineStackSlotColoring.cpp create mode 100644 backend/lib/CodeGen/MachineTailDuplication.cpp create mode 100644 backend/lib/CodeGen/MachineTailMerge.cpp create mode 100644 backend/lib/CodeGen/PostRAScheduler.cpp create mode 100644 backend/lib/CodeGen/RegisterPressure.cpp create mode 100644 backend/lib/CodeGen/ScheduleDAG.cpp create mode 100644 backend/lib/CodeGen/ScheduleHazardRecognizer.cpp create mode 100644 backend/lib/CodeGen/TargetSchedulingModel.cpp create mode 100644 backend/lib/Target/RISCV/RISCVBranchInfo.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineCombiner.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp create mode 100644 backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp create mode 100644 backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp create mode 100644 backend/lib/Target/RISCV/RISCVScheduling.cpp (limited to 'backend/lib') diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index a8cb952..341452a 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -11,11 +11,21 @@ set( MachineBasicBlockAttributes.cpp MachineCFIInfo.cpp MachineCompilationStatus.cpp + MachineCombiner.cpp + MachineCombinerPattern.cpp MachineBasicBlock.cpp + MachineBlockPlacement.cpp + MachineBranchOptimization.cpp + MachineBranchRelaxation.cpp MachineCFGUpdater.cpp MachineConstantPool.cpp + MachineCopyPropagation.cpp + MachineCSE.cpp MachineDebugInfo.cpp + MachineDCE.cpp MachineFrameInfo.cpp + MachineFallthrough.cpp + MachineFinalization.cpp MachineFunction.cpp MachineFunctionAttributes.cpp MachineFunctionCloner.cpp @@ -24,15 +34,29 @@ set( MachineAnalyses.cpp MachineInstr.cpp MachineJumpTableInfo.cpp + MachineLICM.cpp MachineLiveness.cpp + MachineMemoryFolding.cpp MachineMemOperand.cpp MachineMetadata.cpp MachineOperand.cpp + MachineOptimizationPipeline.cpp MachinePass.cpp + MachinePseudoExpansion.cpp MachinePipelinePasses.cpp MachineRegisterInfo.cpp + MachineScheduler.cpp + MachineShrinkWrapping.cpp + MachineSink.cpp + MachineStackSlotColoring.cpp + MachineTailDuplication.cpp + MachineTailMerge.cpp PrologueEpilogueInsertion.cpp + PostRAScheduler.cpp + RegisterPressure.cpp RegisterScavenger.cpp + ScheduleDAG.cpp + ScheduleHazardRecognizer.cpp InlineSpiller.cpp LiveRangeEdit.cpp LiveRegMatrix.cpp @@ -53,6 +77,7 @@ set( TargetFunctionState.cpp TargetMIRTextInfo.cpp TargetRegisterInfo.cpp + TargetSchedulingModel.cpp ) set( RCCBackendCodeGenHeaders @@ -65,11 +90,21 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombinerPattern.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBlockPlacement.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchOptimization.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchRelaxation.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineConstantPool.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCopyPropagation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCSE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDebugInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDCE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFrameInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFallthrough.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFinalization.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunction.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionCloner.h" @@ -78,17 +113,32 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineAnalyses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineInstr.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineJumpTableInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLICM.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLiveness.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLivenessRepair.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemOperand.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePseudoExpansion.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePipelinePasses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PrologueEpilogueInsertion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterPressure.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterScavenger.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleDAG.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleHazardRecognizer.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFrameLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineScheduler.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineShrinkWrapping.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineSink.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineStackSlotColoring.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailDuplication.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailMerge.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PostRAScheduler.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Graph.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Math.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/ReductionRules.h" @@ -111,10 +161,14 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/StableID.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetConstant.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCallLowering.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetBranchInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetDescription.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFunctionState.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMIRTextInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetSchedulingModel.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetPseudoExpansion.h" ) add_library(RCCBackendCodeGen ${RCCBackendCodeGenSources}) @@ -156,9 +210,19 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -167,13 +231,28 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulerInternal.h" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -197,6 +276,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ${RCCBackendCodeGenHeaders} ) rcc_backend_register_tidy_files( @@ -210,9 +290,19 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -221,13 +311,27 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -248,4 +352,5 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ) diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index 5aec744..48d4f10 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -628,7 +628,7 @@ Expected InstructionSet::validateOperand(const OperandDescriptor &Descript } } - if (Descriptor.ImmediatePredicate) { + if (Descriptor.ImmediatePredicate && Operand.kind() == MachineOperand::Kind::Immediate) { const ImmediatePredicate *Predicate = immediatePredicate(*Descriptor.ImmediatePredicate); if (Predicate == nullptr || !Predicate->Predicate(Operand)) { return makeUnexpected(ErrorCode::VerificationError, diff --git a/backend/lib/CodeGen/MachineBlockPlacement.cpp b/backend/lib/CodeGen/MachineBlockPlacement.cpp new file mode 100644 index 0000000..fea1283 --- /dev/null +++ b/backend/lib/CodeGen/MachineBlockPlacement.cpp @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBlockPlacement.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +enum class PlacementClass : std::uint8_t { + Hot, + Normal, + Cold, + Exception, +}; + +[[nodiscard]] const MachinePassContract &placementContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-block-placement", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = + { + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + }, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] PlacementClass classify(const MachineBasicBlock &Block, + std::uint64_t Frequency) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Exception || + Attributes.has(MachineBasicBlockAttribute::LandingPad) || + Attributes.has(MachineBasicBlockAttribute::EHFuncletEntry) || + Attributes.has(MachineBasicBlockAttribute::EHScopeEntry)) { + return PlacementClass::Exception; + } + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Cold) + return PlacementClass::Cold; + if (Frequency >= MachineBlockFrequencyAnalysis::EntryFrequency) + return PlacementClass::Hot; + if (Frequency < MachineBlockFrequencyAnalysis::EntryFrequency / 1024U) + return PlacementClass::Cold; + return PlacementClass::Normal; +} + +[[nodiscard]] std::uint32_t classRank(PlacementClass Class) noexcept { + return static_cast(Class); +} + +struct PlacementFacts final { + MachineBasicBlock *Block = nullptr; + PlacementClass Class = PlacementClass::Normal; + std::uint64_t Frequency = 0; + std::uint32_t LoopDepth = 0; +}; + +[[nodiscard]] bool betterRoot(const PlacementFacts &Left, const PlacementFacts &Right) noexcept { + if (Left.Class != Right.Class) + return classRank(Left.Class) < classRank(Right.Class); + if (Left.Frequency != Right.Frequency) + return Left.Frequency > Right.Frequency; + if (Left.LoopDepth != Right.LoopDepth) + return Left.LoopDepth > Right.LoopDepth; + return Left.Block->stableID().value() < Right.Block->stableID().value(); +} + +} // namespace + +const MachinePassContract &MachineBlockPlacementPass::contract() const noexcept { + return placementContract(); +} + +Expected MachineBlockPlacementPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine block placement requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "block placement setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "block placement diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBlockPlacementPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) const { + auto Probabilities = Analyses.getResult(Function, Target); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + std::vector Facts; + Facts.reserve(Function.blockCount()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const std::uint64_t Frequency = (*Frequencies)->frequency(*Block); + Facts.push_back({Block, classify(*Block, Frequency), Frequency, (*Loops)->loopDepth(*Block)}); + } + if (Facts.size() < 2) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto factsFor = [&Facts](const MachineBasicBlock &Block) -> const PlacementFacts & { + return *std::ranges::find_if( + Facts, [&Block](const PlacementFacts &Value) { return Value.Block == &Block; }); + }; + std::unordered_set Placed; + std::vector Order; + Order.reserve(Facts.size()); + MachineBasicBlock *Root = Function.firstBlock(); + while (Order.size() != Facts.size()) { + if (Root == nullptr || Placed.contains(Root)) { + const auto Best = std::ranges::min_element( + Facts, [&Placed](const PlacementFacts &Left, const PlacementFacts &Right) { + if (Placed.contains(Left.Block)) + return false; + if (Placed.contains(Right.Block)) + return true; + return betterRoot(Left, Right); + }); + if (Best == Facts.end() || Placed.contains(Best->Block)) + return makeUnexpected(ErrorCode::InternalError, "block placement lost an unplaced block"); + Root = Best->Block; + } + + while (Root != nullptr && Placed.insert(Root).second) { + Order.push_back(Root); + MachineBasicBlock *BestSuccessor = nullptr; + BranchProbability BestProbability = BranchProbability::zero(); + for (const MachineSuccessor &Successor : (*Probabilities)->successors(*Root)) { + if (Successor.Block == nullptr || Placed.contains(Successor.Block)) + continue; + if (BestSuccessor == nullptr || BestProbability < Successor.Probability || + (BestProbability == Successor.Probability && + betterRoot(factsFor(*Successor.Block), factsFor(*BestSuccessor)))) { + BestSuccessor = Successor.Block; + BestProbability = Successor.Probability; + } + } + Root = BestSuccessor; + } + Root = nullptr; + } + + const auto &SavePoints = Function.frameInfo().state().SavePoints; + if (!SavePoints.empty() && SavePoints.front().Block != Function.firstBlock()->stableID()) { + const auto SavePosition = std::ranges::find_if(Order, [&](const MachineBasicBlock *Block) { + return Block->stableID() == SavePoints.front().Block; + }); + if (SavePosition == Order.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "block placement cannot find the frame save point"); + } + MachineBasicBlock *SaveBlock = *SavePosition; + std::vector OutsideFrame; + std::vector InsideFrame; + for (MachineBasicBlock *Block : Order) { + ((*Dominators)->dominates(*SaveBlock, *Block) ? InsideFrame : OutsideFrame).push_back(Block); + } + const auto SaveInRegion = std::ranges::find(InsideFrame, SaveBlock); + if (SaveInRegion == InsideFrame.end()) { + return makeUnexpected(ErrorCode::InternalError, "block placement lost the frame save region"); + } + std::rotate(InsideFrame.begin(), SaveInRegion, std::next(SaveInRegion)); + Order.clear(); + Order.insert(Order.end(), OutsideFrame.begin(), OutsideFrame.end()); + Order.insert(Order.end(), InsideFrame.begin(), InsideFrame.end()); + } + + bool Changed = false; + for (std::size_t Index = 1; Index < Order.size(); ++Index) { + Changed |= Order[Index - 1U]->next() != Order[Index]; + auto Moved = Function.moveBlockAfter(*Order[Index - 1U], *Order[Index]); + if (!Moved) + return std::unexpected(Moved.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchOptimization.cpp b/backend/lib/CodeGen/MachineBranchOptimization.cpp new file mode 100644 index 0000000..6c6ac51 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchOptimization.cpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchOptimization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &branchOptimizationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-optimization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasInstructionRecord(const MachineInstr &Instruction, + const MachineFunction &Function) noexcept { + const MachineInstrStableID ID = Instruction.stableID(); + return std::ranges::any_of(Function.records().callSites(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }) || + std::ranges::any_of(Function.records().calledGlobals(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }); +} + +[[nodiscard]] bool hasObservableBlockIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] bool canDeleteBlock(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + if (hasObservableBlockIdentity(Block, Function)) + return false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (hasProtectedAttachment(*Instruction) || hasInstructionRecord(*Instruction, Function)) { + return false; + } + } + return true; +} + +[[nodiscard]] bool hasSuccessor(const MachineBasicBlock &Block, + const MachineBasicBlock &Target) noexcept { + return std::ranges::any_of(Block.successors(), [&Target](const MachineSuccessor &Successor) { + return Successor.Block == &Target; + }); +} + +[[nodiscard]] Expected foldConstantBranches(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + bool Changed = false; + MachineCFGUpdater CFG = Function.cfgUpdater(); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional == nullptr || + hasProtectedAttachment(*Analysis->Conditional)) { + continue; + } + auto Condition = Info.evaluateBranchCondition(*Analysis->Conditional, Function, Target); + if (!Condition) + return std::unexpected(Condition.error()); + if (!Condition->has_value()) + continue; + + MachineBasicBlock *Selected = **Condition ? Analysis->TrueTarget : Analysis->FalseTarget; + MachineBasicBlock *Rejected = **Condition ? Analysis->FalseTarget : Analysis->TrueTarget; + if (Selected == nullptr || !hasSuccessor(*Block, *Selected)) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "constant branch has no selected CFG successor"); + } + if (Analysis->Unconditional != nullptr && hasProtectedAttachment(*Analysis->Unconditional)) { + continue; + } + + auto RemovedConditional = Info.removeBranch(*Analysis->Conditional, Function, Target); + if (!RemovedConditional) + return std::unexpected(RemovedConditional.error()); + if (Analysis->Unconditional != nullptr) { + auto RemovedUnconditional = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!RemovedUnconditional) + return std::unexpected(RemovedUnconditional.error()); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Selected, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + + if (Rejected != nullptr && Rejected != Selected && hasSuccessor(*Block, *Rejected)) { + auto RemovedEdge = CFG.removeSuccessor(*Block, *Rejected); + if (!RemovedEdge) + return std::unexpected(RemovedEdge.error()); + } + auto Probability = CFG.updateBranchProbability(*Block, *Selected, BranchProbability::one()); + if (!Probability) + return std::unexpected(Probability.error()); + Changed = true; + } + return Changed; +} + +[[nodiscard]] std::unordered_set reachableBlocks(MachineFunction &Function) { + std::unordered_set Reachable; + std::vector Worklist; + if (Function.firstBlock() != nullptr) + Worklist.push_back(Function.firstBlock()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!canDeleteBlock(*Block, Function)) + Worklist.push_back(Block); + } + while (!Worklist.empty()) { + MachineBasicBlock *Block = Worklist.back(); + Worklist.pop_back(); + if (Block == nullptr || !Reachable.insert(Block).second) + continue; + for (const MachineSuccessor &Successor : Block->successors()) + Worklist.push_back(Successor.Block); + } + return Reachable; +} + +[[nodiscard]] Expected removeUnreachableBlocks(MachineFunction &Function) { + const std::unordered_set Reachable = reachableBlocks(Function); + MachineCFGUpdater CFG = Function.cfgUpdater(); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;) { + MachineBasicBlock *Next = Block->next(); + if (!Reachable.contains(Block) && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + Changed = true; + } + Block = Next; + } + return Changed; +} + +[[nodiscard]] Expected isForwardingBlock(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + if (&Block == Function.firstBlock() || Block.successors().size() != 1 || + Block.successors().front().Block == nullptr || Block.successors().front().Block == &Block || + hasObservableBlockIdentity(Block, Function)) { + return false; + } + auto Analysis = Info.analyzeBranch(Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional != nullptr) + return false; + MachineBasicBlock *Successor = Block.successors().front().Block; + if (Analysis->Unconditional == nullptr) { + return Block.firstInstruction() == nullptr && Block.next() == Successor; + } + if (Analysis->TrueTarget != Successor || hasProtectedAttachment(*Analysis->Unconditional)) { + return false; + } + return Block.firstInstruction() == Analysis->Unconditional && + Analysis->Unconditional->next() == nullptr; +} + +[[nodiscard]] Expected threadForwardingBlocks(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info, + std::size_t MaximumEdges) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + std::size_t ThreadedEdges = 0; + bool AnyChange = false; + bool LocalChange = true; + while (LocalChange && ThreadedEdges < MaximumEdges) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Forwarding = isForwardingBlock(*Block, Function, Target, Info); + if (!Forwarding) + return std::unexpected(Forwarding.error()); + if (!*Forwarding) + continue; + MachineBasicBlock *Successor = Block->successors().front().Block; + const std::vector Predecessors(Block->predecessors().begin(), + Block->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (ThreadedEdges == MaximumEdges) + break; + if (Predecessor == nullptr || Predecessor->successors().size() != 1 || + Predecessor->successors().front().Block != Block) { + continue; + } + auto PredAnalysis = Info.analyzeBranch(*Predecessor, Function, Target); + if (!PredAnalysis) + return std::unexpected(PredAnalysis.error()); + if (PredAnalysis->HasUnanalyzableTerminator || PredAnalysis->Conditional != nullptr) + continue; + + if (PredAnalysis->Unconditional != nullptr) { + if (PredAnalysis->TrueTarget != Block || + hasProtectedAttachment(*PredAnalysis->Unconditional)) { + continue; + } + auto Rewritten = + Info.rewriteBranchTarget(*Predecessor, *Block, *Successor, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "branch threading did not rewrite an explicit branch"); + } + } else { + if (Predecessor->next() != Block) + continue; + auto Inserted = + Info.insertUnconditionalBranch(*Predecessor, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + auto Redirected = CFG.replaceSuccessor(*Predecessor, *Block, *Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + ++ThreadedEdges; + AnyChange = true; + LocalChange = true; + } + if (Block->predecessors().empty() && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + AnyChange = true; + LocalChange = true; + break; + } + } + } + return AnyChange; +} + +} // namespace + +const MachinePassContract &MachineBranchOptimizationPass::contract() const noexcept { + return branchOptimizationContract(); +} + +Expected MachineBranchOptimizationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch optimization requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "branch optimization transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch optimization diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBranchOptimizationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + auto Folded = foldConstantBranches(Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + auto RemovedBeforeThreading = removeUnreachableBlocks(Function); + if (!RemovedBeforeThreading) + return std::unexpected(RemovedBeforeThreading.error()); + auto Threaded = threadForwardingBlocks(Function, Target, *Info, Options.MaximumThreadedEdges); + if (!Threaded) + return std::unexpected(Threaded.error()); + auto RemovedAfterThreading = removeUnreachableBlocks(Function); + if (!RemovedAfterThreading) + return std::unexpected(RemovedAfterThreading.error()); + + MachinePassResult Result; + Result.Changed = *Folded || *RemovedBeforeThreading || *Threaded || *RemovedAfterThreading; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchRelaxation.cpp b/backend/lib/CodeGen/MachineBranchRelaxation.cpp new file mode 100644 index 0000000..d56f3a0 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchRelaxation.cpp @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchRelaxation.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +struct MachineLayout final { + std::unordered_map BlockOffsets; + std::unordered_map InstructionOffsets; +}; + +[[nodiscard]] const MachinePassContract &relaxationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-relaxation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::VerificationError, "block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits::max() - Mask) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected computeLayout(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + MachineLayout Result; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + Result.BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.InstructionOffsets.emplace(Instruction, Offset); + auto Size = Info.instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + if (Offset > std::numeric_limits::max() - *Size) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout size overflowed"); + Offset += *Size; + } + } + return Result; +} + +[[nodiscard]] Expected displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "forward branch displacement overflowed"); + return static_cast(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "backward branch displacement overflowed"); + return -static_cast(Difference); +} + +} // namespace + +const MachinePassContract &MachineBranchRelaxationPass::contract() const noexcept { + return relaxationContract(); +} + +Expected MachineBranchRelaxationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch relaxation requires finalized frame and tracked liveness"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "branch relaxation setup failed: ")); + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch relaxation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineBranchRelaxationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.ConvergenceWatchdog == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "branch relaxation watchdog must be nonzero"); + MachinePassResult Result; + for (std::size_t Iteration = 0; Iteration < Options.ConvergenceWatchdog; ++Iteration) { + auto Layout = computeLayout(Function, Target, *Info); + if (!Layout) + return std::unexpected(Layout.error()); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !Changed; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto BranchTarget = Info->branchTarget(*Instruction, Function, Target); + if (!BranchTarget) + return std::unexpected(BranchTarget.error()); + if (*BranchTarget == nullptr) + continue; + const auto SourcePosition = Layout->InstructionOffsets.find(Instruction); + const auto TargetPosition = Layout->BlockOffsets.find(*BranchTarget); + if (SourcePosition == Layout->InstructionOffsets.end() || + TargetPosition == Layout->BlockOffsets.end()) { + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation lost an instruction or target offset"); + } + auto Offset = displacement(SourcePosition->second, TargetPosition->second); + if (!Offset) + return std::unexpected(Offset.error()); + if (Info->isBranchOffsetInRange(*Instruction, *Offset)) + continue; + auto Relaxed = Info->relaxBranch(*Instruction, *Offset, Function, Target); + if (!Relaxed) + return std::unexpected(Relaxed.error()); + if (!*Relaxed) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "target did not relax an out-of-range branch"); + } + Changed = true; + Result.Changed = true; + break; + } + } + if (Changed) + continue; + if (Result.Changed) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; + } + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation failed to reach a monotonic fixed point"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp new file mode 100644 index 0000000..a144403 --- /dev/null +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCSE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &cseContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-cse", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +template +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf(); + const ValueT *RightValue = Right.getIf(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +[[nodiscard]] bool registerOperandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + const Register *LeftReg = Left.getIf(); + const Register *RightReg = Right.getIf(); + if (LeftReg == nullptr || RightReg == nullptr || Left.subReg() != Right.subReg() || + Left.targetFlags() != Right.targetFlags()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags && (IgnoreDefinitionValue || *LeftReg == *RightReg); +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags()) + return false; + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return registerOperandEquals(Left, Right, IgnoreDefinitionValue); + case MachineOperand::Kind::Immediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals(Left, Right); + case MachineOperand::Kind::RegisterMask: + case MachineOperand::Kind::LiveOutRegisterMask: + return false; + case MachineOperand::Kind::Metadata: + return payloadEquals(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] bool isConstantMemoryBase(MemoryBase::Kind Kind) noexcept { + return Kind == MemoryBase::Kind::ConstantPool || Kind == MemoryBase::Kind::ConstantPoolPseudo; +} + +[[nodiscard]] bool hasSafeInvariantMemory(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().empty()) + return false; + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) + return false; + const MachineMemOperandDesc &Description = Memory->description(); + const bool ExplicitInvariant = Description.Accesses.has(MemoryAccessFlag::Invariant) && + Description.Accesses.has(MemoryAccessFlag::Dereferenceable); + if (!ExplicitInvariant && !isConstantMemoryBase(Description.Base.kind())) + return false; + } + return true; +} + +[[nodiscard]] bool hasFloatingRegister(const MachineInstr &Instruction, + const MachineRegisterInfo &Registers) { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Info = Registers.virtualRegisterInfo(*Reg); + if (Info && Info->get().Type && Info->get().Type->isFloat()) + return true; + } + return false; +} + +[[nodiscard]] Expected isCandidate(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Pseudo) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction)) { + return false; + } + if (Flags.has(InstructionFlag::MayLoad) && !hasSafeInvariantMemory(Instruction)) + return false; + if (!Flags.has(InstructionFlag::MayLoad) && !Instruction.memOperands().empty()) + return false; + if (!Instruction.flags().has(MachineInstrFlag::NoFPExcept) && + hasFloatingRegister(Instruction, Function.registerInfo())) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Reg->isPhysical() || Reg->isNoRegister() || Operand.isDebugUse() || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Operand.isDef()) + HasDefinition = true; + } + return HasDefinition; +} + +[[nodiscard]] bool expressionEquals(const MachineInstr &Left, const MachineInstr &Right) noexcept { + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size()) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + LeftOperand->isDef() != RightOperand->isDef() || + !operandEquals(*LeftOperand, *RightOperand, LeftOperand->isDef())) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory) + return false; + } + return true; +} + +[[nodiscard]] Expected>> +replacementPairs(const MachineInstr &Available, const MachineInstr &Redundant, + const MachineFunction &Function, const TargetDescription &Target) { + std::vector> Result; + std::unordered_set RedundantDefinitions; + const MachineRegisterInfo &Registers = Function.registerInfo(); + for (std::size_t Index = 0; Index < Available.operandCount(); ++Index) { + const MachineOperand *AvailableOperand = Available.operandAt(Index); + const MachineOperand *RedundantOperand = Redundant.operandAt(Index); + if (AvailableOperand == nullptr || RedundantOperand == nullptr || !AvailableOperand->isDef()) { + continue; + } + const Register *AvailableReg = AvailableOperand->getIf(); + const Register *RedundantReg = RedundantOperand->getIf(); + if (AvailableReg == nullptr || RedundantReg == nullptr || !AvailableReg->isVirtual() || + !RedundantReg->isVirtual() || !RedundantDefinitions.insert(*RedundantReg).second) { + return std::vector>{}; + } + auto AvailableInfo = Registers.virtualRegisterInfo(*AvailableReg); + if (!AvailableInfo) + return std::unexpected(AvailableInfo.error()); + auto RedundantInfo = Registers.virtualRegisterInfo(*RedundantReg); + if (!RedundantInfo) + return std::unexpected(RedundantInfo.error()); + const VirtualRegisterInfo &Left = AvailableInfo->get(); + const VirtualRegisterInfo &Right = RedundantInfo->get(); + if (!Left.Type || !Right.Type || Left.Type != Right.Type || + Left.TargetFlags != Right.TargetFlags) + return std::vector>{}; + if (Left.Bank && Right.Bank && + !Target.registers().registerBanksOverlap(*Left.Bank, *Right.Bank)) { + return std::vector>{}; + } + if (Left.RegClass && Right.RegClass && + !Target.registers().registerClassesOverlap(*Left.RegClass, *Right.RegClass)) { + return std::vector>{}; + } + Result.emplace_back(*RedundantReg, *AvailableReg); + } + return Result; +} + +[[nodiscard]] Expected +replaceAndErase(MachineInstr &Redundant, + const std::vector> &Replacements, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Block = Redundant.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine CSE candidate is detached"); + for (const auto &[OldReg, NewReg] : Replacements) { + auto Replaced = Registers.replaceRegWith(OldReg, NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Redundant); + if (!Removed) + return std::unexpected(Removed.error()); + for (const auto &[OldReg, NewReg] : Replacements) { + (void)NewReg; + if (!Registers.useEmpty(OldReg, true) || Registers.getUniqueVRegDef(OldReg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine CSE left references to a redundant result"); + } + auto Erased = Registers.eraseVirtualRegister(OldReg); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineCSEPass::contract() const noexcept { + return cseContract(); +} + +Expected MachineCSEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine CSE transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine CSE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineCSEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + std::unordered_map> Available; + std::size_t Comparisons = 0; + bool BudgetExhausted = false; + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !BudgetExhausted; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Candidate = isCandidate(*Instruction, Function, Target); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!*Candidate) { + Instruction = Next; + continue; + } + + MachineInstr *Match = nullptr; + auto &OpcodeCandidates = Available[Instruction->opcode()]; + for (MachineInstr *Prior : OpcodeCandidates) { + if (Comparisons >= Options.MaximumComparisons) { + BudgetExhausted = true; + break; + } + ++Comparisons; + if (!(*Dominators)->dominates(*Prior, *Instruction) || + !expressionEquals(*Prior, *Instruction)) { + continue; + } + Match = Prior; + break; + } + if (BudgetExhausted) + break; + if (Match == nullptr) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + + auto Replacements = replacementPairs(*Match, *Instruction, Function, Target); + if (!Replacements) + return std::unexpected(Replacements.error()); + if (Replacements->empty()) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + auto Replaced = replaceAndErase(*Instruction, *Replacements, Registers); + if (!Replaced) + return std::unexpected(Replaced.error()); + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombiner.cpp b/backend/lib/CodeGen/MachineCombiner.cpp new file mode 100644 index 0000000..02ea5e3 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombiner.cpp @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombiner.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &combinerContract(MachineCombinerStage Stage) { + static const MachinePassContract SelectedSSA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-selected-ssa", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-pre-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostFrame = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-frame", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + switch (Stage) { + case MachineCombinerStage::SelectedSSA: + return SelectedSSA; + case MachineCombinerStage::PreRegisterAllocation: + return PreRA; + case MachineCombinerStage::PostRegisterAllocation: + return PostRA; + case MachineCombinerStage::PostFrame: + return PostFrame; + } + return SelectedSSA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool matchesPredicate(const MachineInstr &Instruction, + const MachineCombinerPredicate &Predicate) { + const MachineOperand *Operand = Instruction.operandAt(Predicate.Operand); + if (Operand == nullptr) + return false; + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: { + const auto *Immediate = Operand->getIf(); + return Immediate != nullptr && *Immediate == Predicate.Value; + } + case MachineCombinerPredicateKind::RegisterEquals: { + const MachineOperand *Other = Instruction.operandAt(Predicate.OtherOperand); + const Register *LeftReg = Operand->getIf(); + const Register *RightReg = Other == nullptr ? nullptr : Other->getIf(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Operand->subReg() == Other->subReg(); + } + case MachineCombinerPredicateKind::PhysicalRegisterEquals: { + const Register *Reg = Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast(Predicate.Value); + } + case MachineCombinerPredicateKind::RegisterIsVirtual: { + const Register *Reg = Operand->getIf(); + return Reg != nullptr && Reg->isVirtual(); + } + case MachineCombinerPredicateKind::RegisterIsPhysical: { + const Register *Reg = Operand->getIf(); + return Reg != nullptr && Reg->isPhysical(); + } + } + return false; +} + +[[nodiscard]] bool matchesPattern(const MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineCombinerStage Stage) { + if ((Pattern.Stages & machineCombinerStageMask(Stage)) == 0) + return false; + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + if (!matchesPredicate(Instruction, Predicate)) + return false; + } + return true; +} + +[[nodiscard]] bool compatibleReplacement(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || Source.isNoRegister()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + if (!DestinationInfo) + return false; + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + if (Source.isPhysical()) { + const std::optional Physical = Source.physicalRegister(); + if (!Physical) + return false; + if (DestinationValue.RegClass && + !Target.registers().registerClassContains(*DestinationValue.RegClass, *Physical)) { + return false; + } + return !DestinationValue.Bank || + Target.registers().bankContainsRegister(*DestinationValue.Bank, *Physical); + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo) + return false; + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && + (!SourceValue.Bank || + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank))) { + return false; + } + if (DestinationValue.RegClass && + (!SourceValue.RegClass || !Target.registers().registerClassesOverlap( + *DestinationValue.RegClass, *SourceValue.RegClass))) { + return false; + } + return true; +} + +[[nodiscard]] Expected executeReplacement(MachineInstr &Instruction, + const MachineCombinerAction &Action, + MachineFunction &Function, + const TargetDescription &Target) { + MachineOperand *Definition = Instruction.operandAt(Action.DefinitionOperand); + const MachineOperand *SourceOperand = Instruction.operandAt(Action.SourceOperand); + const Register *Destination = Definition == nullptr ? nullptr : Definition->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr || !Definition->isDef() || Definition->isUse() || + !SourceOperand->isUse() || Definition->subReg().value() != 0 || + SourceOperand->subReg().value() != 0 || Destination->isNoRegister() || + Source->isNoRegister()) { + return false; + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine combiner root is detached"); + + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!Destination->isVirtual() || + Function.registerInfo().getUniqueVRegDef(*Destination) != Definition || + !compatibleReplacement(*Destination, *Source, Function, Target)) { + return false; + } + + std::vector Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Destination, true)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(*Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(*Destination, true) || + Function.registerInfo().getUniqueVRegDef(*Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine combiner left references to a replaced result"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(*Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] Expected executePattern(MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "machine combiner root has no descriptor"); + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayLoad) || + Flags.has(InstructionFlag::MayStore) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || !Instruction.memOperands().empty() || + hasProtectedAttachment(Instruction)) { + return false; + } + switch (Pattern.Action.Kind) { + case MachineCombinerActionKind::ReplaceDefWithOperandAndErase: + return executeReplacement(Instruction, Pattern.Action, Function, Target); + } + return false; +} + +} // namespace + +const MachinePassContract &MachineCombinerPass::contract() const noexcept { + return combinerContract(Stage); +} + +Expected MachineCombinerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine combiner input does not satisfy its stage contract"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine combiner transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine combiner diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineCombinerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.MaximumIterations == 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner iteration budget must be nonzero"); + MachinePassResult Result; + std::size_t Rewrites = 0; + for (std::size_t Iteration = 0; Iteration < Options.MaximumIterations; ++Iteration) { + bool IterationChanged = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + for (const MachineCombinerPattern &Pattern : Info->patterns(Instruction->opcode())) { + if (!matchesPattern(*Instruction, Pattern, Stage)) + continue; + if (Rewrites == Options.MaximumRewrites) + break; + auto Executed = executePattern(*Instruction, Pattern, Function, Target); + if (!Executed) + return std::unexpected(Executed.error()); + if (!*Executed) + continue; + ++Rewrites; + IterationChanged = true; + Result.Changed = true; + break; + } + Instruction = Next; + } + } + if (!IterationChanged || Rewrites == Options.MaximumRewrites) + break; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombinerPattern.cpp b/backend/lib/CodeGen/MachineCombinerPattern.cpp new file mode 100644 index 0000000..b1c3c96 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombinerPattern.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombinerPattern.h" + +#include +#include +#include +#include +#include + +namespace rcc { + +Expected +TargetMachineCombinerInfo::create(std::vector Patterns) { + TargetMachineCombinerInfo Result(std::move(Patterns)); + std::unordered_set Names; + for (const MachineCombinerPattern &Pattern : Result.Patterns) { + if (Pattern.Name.empty() || !Pattern.RootOpcode.isValid() || Pattern.Stages == 0 || + !Names.insert(Pattern.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner pattern has an invalid identity or stage mask"); + } + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: + case MachineCombinerPredicateKind::PhysicalRegisterEquals: + case MachineCombinerPredicateKind::RegisterIsVirtual: + case MachineCombinerPredicateKind::RegisterIsPhysical: + break; + case MachineCombinerPredicateKind::RegisterEquals: + if (Predicate.Operand == Predicate.OtherOperand) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register equality pattern compares an operand with itself"); + } + break; + } + } + Result.MatchTable[Pattern.RootOpcode].push_back(Pattern); + } + return Result; +} + +std::span +TargetMachineCombinerInfo::patterns(Opcode RootOpcode) const noexcept { + const auto Position = MatchTable.find(RootOpcode); + return Position == MatchTable.end() ? std::span{} + : std::span(Position->second); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCompilationStatus.cpp b/backend/lib/CodeGen/MachineCompilationStatus.cpp index 765e133..e541d39 100644 --- a/backend/lib/CodeGen/MachineCompilationStatus.cpp +++ b/backend/lib/CodeGen/MachineCompilationStatus.cpp @@ -26,6 +26,7 @@ constexpr std::array StateNames{ std::string_view{"FinalizationFailed"}, std::string_view{"AssemblyLoweringFailed"}, std::string_view{"InternalError"}, + std::string_view{"SchedulingFailed"}, }; static_assert(StateNames.size() == static_cast(MachineCompilationState::Count)); @@ -44,6 +45,11 @@ isKnownDiagnosticSeverity(MachineCompilationDiagnosticSeverity Severity) noexcep case ErrorCode::Unsupported: case ErrorCode::RegisterAllocationFailed: case ErrorCode::FrameLayoutFailed: + case ErrorCode::SchedulingFailed: + case ErrorCode::BranchRelaxationFailed: + case ErrorCode::PseudoExpansionFailed: + case ErrorCode::FinalEncodabilityFailed: + case ErrorCode::CFGOptimizationFailed: case ErrorCode::IoError: case ErrorCode::InternalError: return true; diff --git a/backend/lib/CodeGen/MachineCopyPropagation.cpp b/backend/lib/CodeGen/MachineCopyPropagation.cpp new file mode 100644 index 0000000..e537a06 --- /dev/null +++ b/backend/lib/CodeGen/MachineCopyPropagation.cpp @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCopyPropagation.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetCodeGen.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract & +copyPropagationContract(MachineCopyPropagationStage Stage) { + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "pre-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCopyPropagationStage::PreRegisterAllocation ? PreRA : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct CopyOperands final { + MachineOperand *DestinationOperand = nullptr; + MachineOperand *SourceOperand = nullptr; + Register Destination; + Register Source; +}; + +[[nodiscard]] std::optional copyOperands(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY) || Instruction.operandCount() != 2 || + hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr || !DestinationOperand->isDef() || + !SourceOperand->isUse() || DestinationOperand->isImplicit() || SourceOperand->isImplicit()) { + return std::nullopt; + } + return CopyOperands{DestinationOperand, SourceOperand, *Destination, *Source}; +} + +[[nodiscard]] Expected compatibleVirtualRegisters(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || !Source.isVirtual()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!DestinationInfo || !SourceInfo) + return std::unexpected(!DestinationInfo ? DestinationInfo.error() : SourceInfo.error()); + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && SourceValue.Bank && + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank)) { + return false; + } + if (DestinationValue.RegClass && SourceValue.RegClass && + !Target.registers().isSubclass(*SourceValue.RegClass, *DestinationValue.RegClass)) { + return false; + } + return true; +} + +[[nodiscard]] Expected propagateVirtualCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + if (!Operands.Destination.isVirtual() || !Operands.Source.isVirtual()) + return false; + if (Function.registerInfo().getUniqueVRegDef(Operands.Destination) != + Operands.DestinationOperand) { + return false; + } + auto Compatible = + compatibleVirtualRegisters(Operands.Destination, Operands.Source, Function, Target); + if (!Compatible) + return std::unexpected(Compatible.error()); + if (!*Compatible) + return false; + auto ValidCopy = + Info.validateCopy(Operands.Destination, Operands.DestinationOperand->subReg(), + Operands.Source, Operands.SourceOperand->subReg(), Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + + std::vector Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(Operands.Destination, true)) { + if (&Use == Operands.SourceOperand) + continue; + if (Use.tiedOperandIndex() || Use.isImplicit() || + Use.subReg() != Operands.DestinationOperand->subReg()) { + return false; + } + Uses.push_back(&Use); + } + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation candidate is detached"); + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(Operands.Destination, true) || + Function.registerInfo().getUniqueVRegDef(Operands.Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation left destination references"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Operands.Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] bool shareRegisterClass(PhysicalRegister Left, PhysicalRegister Right, + const TargetRegisterInfo &Registers) noexcept { + return std::ranges::any_of(Registers.registerClasses(), [&](const RegisterClassDesc &Class) { + return Registers.registerClassContains(Class.ID, Left) && + Registers.registerClassContains(Class.ID, Right); + }); +} + +[[nodiscard]] bool clobbersRegister(const MachineInstr &Instruction, PhysicalRegister Reg, + const TargetRegisterInfo &Registers) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (Mask->Mask && Mask->Mask->clobbers(Reg)) + return true; + continue; + } + const Register *OperandReg = Operand.getIf(); + const std::optional Physical = + OperandReg == nullptr ? std::nullopt : OperandReg->physicalRegister(); + if (Physical && Operand.isDef() && Registers.regsOverlap(*Physical, Reg)) + return true; + } + return false; +} + +[[nodiscard]] bool canReplacePhysicalUse(const MachineInstr &Instruction, std::size_t OperandIndex, + PhysicalRegister Source, + const TargetDescription &Target) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Operand == nullptr || Descriptor == nullptr || Operand->isImplicit() || + Operand->tiedOperandIndex() || Operand->subReg().value() != 0 || + OperandIndex >= Descriptor->Operands.size()) { + return false; + } + const OperandDescriptor &Constraint = Descriptor->Operands[OperandIndex]; + if (Constraint.RegClass && + !Target.registers().registerClassContains(*Constraint.RegClass, Source)) { + return false; + } + if (Constraint.RegBank && !Target.registers().bankContainsRegister(*Constraint.RegBank, Source)) { + return false; + } + return true; +} + +[[nodiscard]] Expected propagatePhysicalCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + const std::optional Destination = Operands.Destination.physicalRegister(); + const std::optional Source = Operands.Source.physicalRegister(); + if (!Destination || !Source || Operands.DestinationOperand->subReg().value() != 0 || + Operands.SourceOperand->subReg().value() != 0) { + return false; + } + auto ValidCopy = + Info.validateCopy(Operands.Destination, {}, Operands.Source, {}, Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "physical COPY propagation candidate is detached"); + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!shareRegisterClass(*Destination, *Source, Target.registers())) + return false; + + bool Changed = false; + for (MachineInstr *Instruction = Copy.next(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (clobbersRegister(*Instruction, *Destination, Target.registers()) || + clobbersRegister(*Instruction, *Source, Target.registers())) { + break; + } + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand->isUse() || Operand->isDebugUse() || + !Target.registers().regsOverlap(*Physical, *Destination) || + !canReplacePhysicalUse(*Instruction, Index, *Source, Target)) { + continue; + } + auto Replaced = Operand->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + Changed = true; + } + } + return Changed; +} + +} // namespace + +const MachinePassContract &MachineCopyPropagationPass::contract() const noexcept { + return copyPropagationContract(Stage); +} + +Expected MachineCopyPropagationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine COPY propagation has unmet stage properties"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "COPY propagation transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "COPY propagation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineCopyPropagationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t PropagatedCopies = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && PropagatedCopies < Options.MaximumPropagatedCopies;) { + MachineInstr *Next = Instruction->next(); + const std::optional Operands = copyOperands(*Instruction); + if (!Operands) { + Instruction = Next; + continue; + } + Expected Changed = + Stage == MachineCopyPropagationStage::PreRegisterAllocation + ? propagateVirtualCopy(*Instruction, *Operands, Function, Target, *Info) + : propagatePhysicalCopy(*Instruction, *Operands, Function, Target, *Info); + if (!Changed) + return std::unexpected(Changed.error()); + if (*Changed) { + ++PropagatedCopies; + Result.Changed = true; + } + Instruction = Next; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + if (Stage == MachineCopyPropagationStage::PreRegisterAllocation) + Result.Changes.add(MachinePassChange::Registers); + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineDCE.cpp b/backend/lib/CodeGen/MachineDCE.cpp new file mode 100644 index 0000000..b413499 --- /dev/null +++ b/backend/lib/CodeGen/MachineDCE.cpp @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineDCE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &dceContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-dce", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] Expected>> +deadDefinitions(MachineInstr &Instruction, MachineRegisterInfo &Registers, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + + std::vector VirtualDefinitions; + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Operand.isDef()) + continue; + HasDefinition = true; + if (Operand.isUse() || Operand.tiedOperandIndex()) + return std::nullopt; + if (Reg->isVirtual()) { + if (!Registers.useEmpty(*Reg, false)) + return std::nullopt; + VirtualDefinitions.push_back(*Reg); + continue; + } + if (!Reg->isPhysical() || !Operand.registerFlags().has(RegisterOperandFlag::Dead)) { + return std::nullopt; + } + } + if (!HasDefinition) + return std::nullopt; + return VirtualDefinitions; +} + +[[nodiscard]] Expected clearDebugUses(MachineRegisterInfo &Registers, Register Reg) { + std::vector Uses; + for (MachineOperand &Operand : Registers.useOperands(Reg, true)) { + if (Operand.isDebugUse()) + Uses.push_back(&Operand); + } + for (MachineOperand *Operand : Uses) { + auto Cleared = Operand->setRegister(Register{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineDCEPass::contract() const noexcept { + return dceContract(); +} + +Expected MachineDCEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE requires a selected machine function"); + } + + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine DCE transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine DCE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineDCEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) { + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + + bool LocalChange = true; + while (LocalChange) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Definitions = deadDefinitions(*Instruction, Registers, Target); + if (!Definitions) + return std::unexpected(Definitions.error()); + if (!*Definitions) { + Instruction = Next; + continue; + } + + for (const Register Reg : **Definitions) { + auto Cleared = clearDebugUses(Registers, Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + auto Removed = Block->removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + for (const Register Reg : **Definitions) { + if (!Registers.useEmpty(Reg, true) || Registers.getUniqueVRegDef(Reg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine DCE left references to a removed definition"); + } + auto Erased = Registers.eraseVirtualRegister(Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + LocalChange = true; + Result.Changed = true; + Instruction = Next; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFallthrough.cpp b/backend/lib/CodeGen/MachineFallthrough.cpp new file mode 100644 index 0000000..06d70ef --- /dev/null +++ b/backend/lib/CodeGen/MachineFallthrough.cpp @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFallthrough.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &fallthroughContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-fallthrough-normalization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected> +normalizeFallthroughs(MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info) { + std::vector ChangedBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + MachineBasicBlock *LayoutSuccessor = Block->next(); + if (LayoutSuccessor == nullptr) + continue; + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + continue; + bool Changed = false; + if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + continue; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget != nullptr) { + auto Inserted = + Info.insertUnconditionalBranch(*Block, *Analysis->FalseTarget, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Block->successors().size() == 1 && + Block->successors().front().Block != LayoutSuccessor) { + MachineBasicBlock *Successor = Block->successors().front().Block; + if (Successor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization encountered a null successor"); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } + if (Changed) + ChangedBlocks.push_back(Block); + } + return ChangedBlocks; +} + +} // namespace + +const MachinePassContract &MachineFallthroughPass::contract() const noexcept { + return fallthroughContract(); +} + +Expected MachineFallthroughPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization requires allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "fallthrough setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialChanged = normalizeFallthroughs(*Trial->ClonedFunction, Target, *Info); + if (!TrialChanged) + return std::unexpected(TrialChanged.error()); + auto TrialRepair = repairPostRAMachineBlockLiveness(*TrialChanged, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto ChangedBlocks = normalizeFallthroughs(Function, Target, *Info); + if (!ChangedBlocks) { + return std::unexpected(prefixedError(ChangedBlocks.error(), + "fallthrough diverged after a successful preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(*ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = !ChangedBlocks->empty() || *Repaired; + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFinalization.cpp b/backend/lib/CodeGen/MachineFinalization.cpp new file mode 100644 index 0000000..8523bb8 --- /dev/null +++ b/backend/lib/CodeGen/MachineFinalization.cpp @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFinalization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &finalizePseudosContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "finalize-emission-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {MachineFunctionProperty::NoExecutablePseudos}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &encodabilityContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "final-encodability-verifier", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices, + MachineFunctionProperty::NoExecutablePseudos}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Expected alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits::max() - Mask) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return static_cast(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast(std::numeric_limits::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return -static_cast(Difference); +} + +} // namespace + +const MachinePassContract &FinalizeEmissionPseudosPass::contract() const noexcept { + return finalizePseudosContract(); +} + +Expected FinalizeEmissionPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization requires finalized allocated MIR"); + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo)) + continue; + const bool NonEmitting = Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::ProducesCode); + const bool Terminal = Descriptor->Flags.has(InstructionFlag::TerminalEmission) && + Descriptor->Owner == PseudoOwner::AsmPrinter; + if (!NonEmitting && !Terminal) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "non-terminal executable pseudo survived all expansion owners"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &FinalEncodabilityVerifierPass::contract() const noexcept { + return encodabilityContract(); +} + +Expected FinalEncodabilityVerifierPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final encodability requires NoExecutablePseudos and finalized MIR"); + + std::unordered_map BlockOffsets; + std::unordered_map InstructionOffsets; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction uses an unknown opcode"); + auto InstructionValidation = + Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!InstructionValidation) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction is not encodable: " + + std::string(InstructionValidation.error().message())); + } + const bool BundledWithPred = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool BundledWithSucc = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if ((BundledWithPred && + (Instruction->previous() == nullptr || + !Instruction->previous()->flags().has(MachineInstrFlag::BundledWithSucc))) || + (BundledWithSucc && + (Instruction->next() == nullptr || + !Instruction->next()->flags().has(MachineInstrFlag::BundledWithPred)))) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction has an inconsistent bundle boundary"); + } + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf(); + if (Reg != nullptr && !Reg->isNoRegister() && !Reg->isPhysical()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a virtual register"); + if (Operand.kind() == MachineOperand::Kind::FrameIndex) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a frame index"); + } + auto Size = BranchInfo->instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + const bool ProducesCode = Descriptor->Flags.has(InstructionFlag::ProducesCode); + if ((ProducesCode && *Size == 0) || + (Descriptor->Flags.has(InstructionFlag::Meta) && *Size != 0)) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction violates its exact size contract"); + } + InstructionOffsets.emplace(Instruction, Offset); + if (Offset > std::numeric_limits::max() - *Size) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine function size overflowed"); + Offset += *Size; + } + } + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto TargetBlock = BranchInfo->branchTarget(*Instruction, Function, Target); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + if (*TargetBlock == nullptr) + continue; + const auto Source = InstructionOffsets.find(Instruction); + const auto Destination = BlockOffsets.find(*TargetBlock); + if (Source == InstructionOffsets.end() || Destination == BlockOffsets.end()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch target has no layout offset"); + auto Distance = displacement(Source->second, Destination->second); + if (!Distance) + return std::unexpected(Distance.error()); + if (!BranchInfo->isBranchOffsetInRange(*Instruction, *Distance)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction contains an out-of-range branch"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunction.cpp b/backend/lib/CodeGen/MachineFunction.cpp index df1ceb5..7f4c3a9 100644 --- a/backend/lib/CodeGen/MachineFunction.cpp +++ b/backend/lib/CodeGen/MachineFunction.cpp @@ -172,6 +172,66 @@ Expected MachineFunction::insertBlockBefore(MachineBasicBlock &Position, return {}; } +Expected MachineFunction::moveBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Block.NextBlock == &Position) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = Position.PreviousBlock; + Block.NextBlock = &Position; + if (Position.PreviousBlock != nullptr) + Position.PreviousBlock->NextBlock = &Block; + else + FirstBlock = &Block; + Position.PreviousBlock = &Block; + renumberBlocks(); + return {}; +} + +Expected MachineFunction::moveBlockAfter(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Position.NextBlock == &Block) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = &Position; + Block.NextBlock = Position.NextBlock; + if (Position.NextBlock != nullptr) + Position.NextBlock->PreviousBlock = &Block; + else + LastBlock = &Block; + Position.NextBlock = &Block; + renumberBlocks(); + return {}; +} + Expected MachineFunction::removeBlock(MachineBasicBlock &Block) { if (Block.OwnerFunction != this || !Block.IsLinked) return makeUnexpected(ErrorCode::InvalidArgument, diff --git a/backend/lib/CodeGen/MachineFunctionRecords.cpp b/backend/lib/CodeGen/MachineFunctionRecords.cpp index 0a21581..18ac596 100644 --- a/backend/lib/CodeGen/MachineFunctionRecords.cpp +++ b/backend/lib/CodeGen/MachineFunctionRecords.cpp @@ -127,6 +127,38 @@ Expected MachineFunctionRecords::addCalledGlobal(MachineCalledGlobal Calle return {}; } +Expected +MachineFunctionRecords::replaceInstructionReference(MachineInstrStableID OldInstruction, + MachineInstrStableID NewInstruction) { + if (!OldInstruction.isValid() || !NewInstruction.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction-record replacement requires two valid stable IDs"); + } + if (OldInstruction == NewInstruction) + return {}; + for (const MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a call-site record"); + } + } + for (const MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a called-global record"); + } + } + for (MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == OldInstruction) + CallSite.CallInstruction = NewInstruction; + } + for (MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == OldInstruction) + CalledGlobal.CallInstruction = NewInstruction; + } + return {}; +} + Expected MachineFunctionRecords::addPrefetchTarget(MachinePrefetchTarget Target) { if (std::ranges::find(PrefetchTargets, Target) != PrefetchTargets.end()) { return makeUnexpected(ErrorCode::InvalidArgument, "prefetch target is already recorded"); diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp new file mode 100644 index 0000000..8c8d732 --- /dev/null +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineLICM.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &licmContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-licm", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-licm", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool containsBlock(const MachineLoop &Loop, const MachineBasicBlock *Block) noexcept { + return std::ranges::find(Loop.Blocks, Block) != Loop.Blocks.end(); +} + +[[nodiscard]] MachineBasicBlock *preheader(const MachineLoop &Loop) noexcept { + if (Loop.Header == nullptr) + return nullptr; + MachineBasicBlock *Result = nullptr; + for (MachineBasicBlock *Predecessor : Loop.Header->predecessors()) { + if (containsBlock(Loop, Predecessor)) + continue; + if (Result != nullptr) + return nullptr; + Result = Predecessor; + } + if (Result == nullptr || Result->successors().size() != 1 || + Result->successors().front().Block != Loop.Header) { + return nullptr; + } + return Result; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected isSafeToHoist(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +[[nodiscard]] std::size_t definitionCount(const MachineInstr &Instruction) noexcept { + return static_cast(std::ranges::count_if( + Instruction.operands(), [](const MachineOperand &Operand) { return Operand.isDef(); })); +} + +[[nodiscard]] bool +operandsAreInvariant(const MachineInstr &Instruction, const MachineLoop &Loop, + const MachineRegisterInfo &Registers, + const std::unordered_set &Hoisted) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Operand.isUse()) + continue; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Registers.getUniqueVRegDef(*Reg); + const MachineInstr *DefiningInstruction = + Definition == nullptr ? nullptr : Definition->parent(); + const MachineBasicBlock *DefiningBlock = + DefiningInstruction == nullptr ? nullptr : DefiningInstruction->parent(); + if (DefiningInstruction == nullptr || DefiningBlock == nullptr) + return false; + if (containsBlock(Loop, DefiningBlock) && !Hoisted.contains(DefiningInstruction)) + return false; + } + return true; +} + +[[nodiscard]] bool postRAOperandsAreInvariant(const MachineInstr &Instruction, + const MachineLoop &Loop, + const MachineFunction &Function, + const TargetDescription &Target) noexcept { + if (Loop.Header == nullptr || Instruction.parent() != Loop.Header) + return false; + + std::vector Uses; + std::vector Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return false; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return false; + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (std::ranges::any_of(Loop.Header->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return false; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return false; + } + } + + const auto ClobbersCandidate = [&](const RegisterMask &Mask) { + return std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }); + }; + for (const MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + return false; + for (const MachineInstr *Candidate = Block->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + if (Candidate == &Instruction) + continue; + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (ClobbersCandidate(*Mask->Mask)) + return false; + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand.isDef()) + continue; + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(*Physical, Use); + }) || + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + } + + for (const MachineInstr *Before = Loop.Header->firstInstruction(); + Before != nullptr && Before != &Instruction; Before = Before->next()) { + for (const MachineOperand &Operand : Before->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return false; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && (Operand.isUse() || Operand.isDef()) && + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + return true; +} + +[[nodiscard]] Expected firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM preheader contains an unknown instruction"); + } + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] Expected moveToPreheader(MachineInstr &Instruction, + MachineBasicBlock &Preheader, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine LICM candidate is detached"); + auto Anchor = firstTerminator(Preheader, Target); + if (!Anchor) + return std::unexpected(Anchor.error()); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (*Anchor != nullptr) + return Preheader.insertInstructionBefore(**Anchor, Instruction); + return Preheader.appendInstruction(Instruction); +} + +} // namespace + +const MachinePassContract &MachineLICMPass::contract() const noexcept { + return licmContract(Stage); +} + +Expected MachineLICMPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine LICM requires selected SSA MachineIR" + : "post-RA machine LICM requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine LICM transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine LICM diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineLICMPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t TotalHoisted = 0; + for (const MachineLoop &Loop : (*Loops)->loops()) { + MachineBasicBlock *Preheader = preheader(Loop); + if (Preheader == nullptr || Loop.Header == nullptr || + (*Frequencies)->frequency(*Preheader) > (*Frequencies)->frequency(*Loop.Header)) { + continue; + } + std::unordered_set Hoisted; + std::size_t HoistedDefinitions = 0; + bool LocalChange = true; + while (LocalChange && TotalHoisted < Options.MaximumHoistedInstructions) { + LocalChange = false; + for (MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + continue; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToHoist(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + const std::size_t Definitions = definitionCount(*Instruction); + const bool Invariant = + Stage == MachineCodeMotionStage::SelectedSSA + ? operandsAreInvariant(*Instruction, Loop, Function.registerInfo(), Hoisted) + : postRAOperandsAreInvariant(*Instruction, Loop, Function, Target); + if (*Safe && Invariant && + Definitions <= + Options.MaximumHoistedDefinitionsPerLoop - + std::min(HoistedDefinitions, Options.MaximumHoistedDefinitionsPerLoop)) { + auto Moved = moveToPreheader(*Instruction, *Preheader, Target); + if (!Moved) + return std::unexpected(Moved.error()); + Hoisted.insert(Instruction); + HoistedDefinitions += Definitions; + ++TotalHoisted; + LocalChange = true; + Result.Changed = true; + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + Instruction = Next; + } + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMemoryFolding.cpp b/backend/lib/CodeGen/MachineMemoryFolding.cpp new file mode 100644 index 0000000..0e15146 --- /dev/null +++ b/backend/lib/CodeGen/MachineMemoryFolding.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMemoryFolding.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &memoryFoldingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-memory-folding", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct FoldCandidate final { + MachineInstr *MemoryInstruction = nullptr; + MachineInstr *Consumer = nullptr; + Register DefinedRegister; + std::size_t ConsumerOperand = 0; +}; + +[[nodiscard]] std::optional findCandidate(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::MayLoad) || + Descriptor->Flags.has(InstructionFlag::MayStore) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Barrier) || + Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects) || + hasProtectedAttachment(Instruction) || Instruction.memOperands().empty()) { + return std::nullopt; + } + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) { + return std::nullopt; + } + } + + MachineOperand *Definition = nullptr; + Register DefinedRegister; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + if (Reg == nullptr || !Operand->isDef()) + continue; + if (Definition != nullptr || !Reg->isVirtual()) + return std::nullopt; + Definition = Operand; + DefinedRegister = *Reg; + } + if (Definition == nullptr || !Function.registerInfo().hasOneUse(DefinedRegister, true)) { + return std::nullopt; + } + MachineOperand *Use = nullptr; + for (MachineOperand &Candidate : Function.registerInfo().useOperands(DefinedRegister, false)) { + Use = &Candidate; + break; + } + MachineInstr *Consumer = Use == nullptr ? nullptr : Use->parent(); + if (Consumer == nullptr || Consumer->parent() != Instruction.parent() || + Instruction.next() != Consumer || hasProtectedAttachment(*Consumer) || Use->isImplicit() || + Use->tiedOperandIndex()) { + return std::nullopt; + } + for (std::size_t Index = 0; Index < Consumer->operandCount(); ++Index) { + if (Consumer->operandAt(Index) == Use) + return FoldCandidate{&Instruction, Consumer, DefinedRegister, Index}; + } + return std::nullopt; +} + +[[nodiscard]] bool containsMemoryDescription(const MachineInstr &Instruction, + const MachineMemOperandDesc &Description) noexcept { + return std::ranges::any_of(Instruction.memOperands(), + [&Description](const MachineMemOperand *Memory) { + return Memory != nullptr && Memory->description() == Description; + }); +} + +[[nodiscard]] Expected applyFold(FoldCandidate Candidate, MachineFunction &Function, + const TargetDescription &Target, + const TargetMemoryFoldingInfo &Info) { + auto CanFold = Info.canFoldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!CanFold) + return std::unexpected(CanFold.error()); + if (!*CanFold) + return false; + std::vector MemoryDescriptions; + for (const MachineMemOperand *Memory : Candidate.MemoryInstruction->memOperands()) { + if (Memory == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding encountered a null memory operand"); + } + MemoryDescriptions.push_back(Memory->description()); + } + auto Folded = Info.foldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!Folded) + return std::unexpected(Folded.error()); + if (*Folded == nullptr || (*Folded)->parent() == nullptr || + Candidate.MemoryInstruction->parent() != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not replace the consumer and erase the producer"); + } + for (const MachineMemOperandDesc &Description : MemoryDescriptions) { + if (!containsMemoryDescription(**Folded, Description)) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not migrate every memory operand"); + } + } + auto FoldedSize = Info.getFoldedMemorySize(**Folded, Function, Target); + if (!FoldedSize) + return std::unexpected(FoldedSize.error()); + if (!FoldedSize->has_value() || !FoldedSize->value().isKnown()) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded instruction has no known folded memory size"); + } + if (MemoryDescriptions.size() == 1 && MemoryDescriptions.front().Size.isKnown() && + FoldedSize->value() != MemoryDescriptions.front().Size) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded memory size disagrees with the migrated MMO"); + } + if (!Function.registerInfo().useEmpty(Candidate.DefinedRegister, true) || + Function.registerInfo().getUniqueVRegDef(Candidate.DefinedRegister) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold retained the load result register"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Candidate.DefinedRegister); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineMemoryFoldingPass::contract() const noexcept { + return memoryFoldingContract(); +} + +Expected MachineMemoryFoldingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "memory folding transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "memory folding diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineMemoryFoldingPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t FoldCount = 0; + bool LocalChange = true; + while (LocalChange && FoldCount < Options.MaximumFolds) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !LocalChange; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + std::optional Candidate = findCandidate(*Instruction, Function, Target); + if (!Candidate) + continue; + auto Folded = applyFold(*Candidate, Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + if (!*Folded) + continue; + ++FoldCount; + Result.Changed = true; + LocalChange = true; + break; + } + } + } + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp index 43a4f91..d1d152e 100644 --- a/backend/lib/CodeGen/MachineOperand.cpp +++ b/backend/lib/CodeGen/MachineOperand.cpp @@ -548,6 +548,18 @@ Expected MachineOperand::setBasicBlock(MachineBasicBlock &Block) { return {}; } +Expected MachineOperand::setFrameIndex(FrameIndex Index) { + if (OperandKind != Kind::FrameIndex) + return makeUnexpected(ErrorCode::InvalidArgument, + "setFrameIndex requires a frame-index operand"); + if (Parent != nullptr && Parent->function().frameInfo().object(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frame-index operand references an unknown stack object"); + } + std::get(Storage).Index = Index; + return {}; +} + MachineOperand MachineOperand::cloneDetached() const { MachineOperand Result(OperandKind, Storage, TargetFlags); Result.RegFlags = RegFlags; diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp new file mode 100644 index 0000000..76dbd44 --- /dev/null +++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOptimizationPipeline.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBlockPlacement.h" +#include "rcc/CodeGen/MachineBranchOptimization.h" +#include "rcc/CodeGen/MachineBranchRelaxation.h" +#include "rcc/CodeGen/MachineCSE.h" +#include "rcc/CodeGen/MachineCombiner.h" +#include "rcc/CodeGen/MachineCopyPropagation.h" +#include "rcc/CodeGen/MachineDCE.h" +#include "rcc/CodeGen/MachineFallthrough.h" +#include "rcc/CodeGen/MachineFinalization.h" +#include "rcc/CodeGen/MachineLICM.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachinePseudoExpansion.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/MachineShrinkWrapping.h" +#include "rcc/CodeGen/MachineSink.h" +#include "rcc/CodeGen/MachineStackSlotColoring.h" +#include "rcc/CodeGen/MachineTailDuplication.h" +#include "rcc/CodeGen/MachineTailMerge.h" +#include "rcc/CodeGen/PostRAScheduler.h" +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" + +#include +#include + +namespace rcc { +namespace { + +template +[[nodiscard]] Expected addPass(MachinePassManager &Pipeline, Arguments &&...Values) { + return Pipeline.addPass(std::make_unique(std::forward(Values)...)); +} + +} // namespace + +Expected registerMachineOptimizationAnalyses(MachineAnalysisManager &Analyses) { + auto Registered = registerStage8MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + return registerStage11MachineAnalyses(Analyses); +} + +Expected +addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.MemoryFolding); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, MachineCodeMotionStage::SelectedSSA); +} + +Expected addPreRegisterAllocationOptimizationPipeline( + MachinePassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, TargetInfo.SchedulingModel); +} + +Expected +addPostRegisterAllocationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = + addPass(Pipeline, TargetInfo.Combiner, MachineCombinerStage::PostFrame); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.SchedulingModel); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline, TargetInfo.PseudoExpansion, + TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass(Pipeline, TargetInfo.BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePseudoExpansion.cpp b/backend/lib/CodeGen/MachinePseudoExpansion.cpp new file mode 100644 index 0000000..e654d97 --- /dev/null +++ b/backend/lib/CodeGen/MachinePseudoExpansion.cpp @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePseudoExpansion.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +using CFGEdge = std::tuple; + +struct ResourceSnapshot final { + std::size_t BlockCount = 0; + std::size_t VirtualRegisterCount = 0; + std::size_t FrameObjectCount = 0; + bool FrameFrozen = false; + std::vector Edges; +}; + +using BlockSizeSnapshot = std::vector>; + +[[nodiscard]] Expected blockSize(const MachineBasicBlock &Block, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + std::uint64_t Size = 0; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto InstructionSize = BranchInfo.instructionSize(*Instruction, Function, Target); + if (!InstructionSize) + return std::unexpected(InstructionSize.error()); + if (Size > std::numeric_limits::max() - *InstructionSize) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, "target-late pseudo size overflowed"); + } + Size += *InstructionSize; + } + return Size; +} + +[[nodiscard]] Expected snapshotBlockSizes(const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + BlockSizeSnapshot Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Size = blockSize(*Block, Function, Target, BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + Result.emplace_back(Block->stableID(), *Size); + } + return Result; +} + +[[nodiscard]] ResourceSnapshot snapshotResources(const MachineFunction &Function) { + ResourceSnapshot Result; + Result.BlockCount = Function.blockCount(); + Result.VirtualRegisterCount = Function.registerInfo().virtualRegisters().size(); + Result.FrameObjectCount = Function.frameInfo().objects().size(); + Result.FrameFrozen = Function.frameInfo().isFrozen(); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineSuccessor &Successor : Block->successors()) { + Result.Edges.emplace_back(Block->stableID().value(), Successor.Block->stableID().value(), + Successor.Probability.rawValue()); + } + } + return Result; +} + +[[nodiscard]] Expected verifyResourceContract(const ResourceSnapshot &Before, + const MachineFunction &Function, + MachinePseudoExpansionPhase Phase) { + const ResourceSnapshot After = snapshotResources(Function); + if (Before.BlockCount != After.BlockCount || Before.Edges != After.Edges) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion changed CFG resources outside its owner contract"); + } + if (Before.VirtualRegisterCount != After.VirtualRegisterCount) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-selection pseudo expansion created a virtual register"); + } + if (Phase != MachinePseudoExpansionPhase::PrePEI && + (Before.FrameObjectCount != After.FrameObjectCount || + Before.FrameFrozen != After.FrameFrozen)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-PEI pseudo expansion changed frozen frame resources"); + } + return {}; +} + +[[nodiscard]] PseudoOwner ownerFor(MachinePseudoExpansionPhase Phase) noexcept { + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PseudoOwner::PrePEIExpansion; + case MachinePseudoExpansionPhase::PostRA: + return PseudoOwner::PostRAExpansion; + case MachinePseudoExpansionPhase::TargetLate: + return PseudoOwner::TargetLateExpansion; + } + return PseudoOwner::None; +} + +[[nodiscard]] const MachinePassContract &preparationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "prepare-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &phaseContract(MachinePseudoExpansionPhase Phase) { + static const MachinePassContract PrePEI = [] { + MachinePassContract Value{ + .Name = "pre-pei-target-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Frame}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "expand-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + static const MachinePassContract TargetLate = [] { + MachinePassContract Value{ + .Name = "target-late-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PrePEI; + case MachinePseudoExpansionPhase::PostRA: + return PostRA; + case MachinePseudoExpansionPhase::TargetLate: + return TargetLate; + } + return TargetLate; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected +prepareImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + bool Changed = false; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo preparation encountered an unknown opcode"); + if (Descriptor->Owner != PseudoOwner::PostRAExpansion) + continue; + auto Prepared = Info.preparePostRAPseudo(*Instruction, Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + Changed |= *Prepared; + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected +expandImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + const ResourceSnapshot Before = snapshotResources(Function); + std::optional SizesBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + if (BranchInfo == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target-late pseudo expansion requires branch sizing"); + } + auto Snapshot = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!Snapshot) + return std::unexpected(Snapshot.error()); + SizesBefore = std::move(*Snapshot); + } + std::vector Worklist; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction != nullptr && Instruction->parent() != nullptr) + Worklist.push_back(Instruction); + } + bool Changed = false; + for (MachineInstr *Instruction : Worklist) { + if (Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo expansion encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo) || Descriptor->Owner != ownerFor(Phase)) { + continue; + } + MachineBasicBlock *ExpansionBlock = Instruction->parent(); + std::optional ExpansionSizeBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + auto Size = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + ExpansionSizeBefore = *Size; + } + auto Expanded = Info.expandPseudo(*Instruction, Phase, Function, Target); + if (!Expanded) + return std::unexpected(Expanded.error()); + if (!*Expanded) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target has no expansion for an owned executable pseudo"); + } + if (ExpansionSizeBefore.has_value()) { + auto SizeAfter = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!SizeAfter) + return std::unexpected(SizeAfter.error()); + if (*SizeAfter > *ExpansionSizeBefore) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "one target-late pseudo expansion increased its encoded block size"); + } + } + Changed = true; + } + + auto Resources = verifyResourceContract(Before, Function, Phase); + if (!Resources) + return std::unexpected(Resources.error()); + if (SizesBefore.has_value()) { + auto SizesAfter = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!SizesAfter) + return std::unexpected(SizesAfter.error()); + if (SizesAfter->size() != SizesBefore->size()) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion changed block layout"); + } + for (std::size_t Index = 0; Index < SizesAfter->size(); ++Index) { + if ((*SizesAfter)[Index].first != (*SizesBefore)[Index].first || + (*SizesAfter)[Index].second > (*SizesBefore)[Index].second) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion increased an encoded block size"); + } + } + } + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Pseudo) && + Descriptor->Owner == ownerFor(Phase)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion left an instruction owned by the completed phase"); + } + } + if (Changed && Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected +runPreparationTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo preparation setup failed: ")); + auto TrialResult = prepareImplementation(*Trial->ClonedFunction, Target, Info); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = prepareImplementation(Function, Target, Info); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo preparation diverged after successful preflight: ")); + } + return Result; +} + +[[nodiscard]] Expected +runExpansionTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo expansion setup failed: ")); + auto TrialResult = expandImplementation(*Trial->ClonedFunction, Target, Info, Phase, BranchInfo); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = expandImplementation(Function, Target, Info, Phase, BranchInfo); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo expansion diverged after successful preflight: ")); + } + return Result; +} + +} // namespace + +const MachinePassContract &PreparePostRAPseudosPass::contract() const noexcept { + return preparationContract(); +} + +Expected PreparePostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo preparation requires selected MachineIR"); + return runPreparationTransaction(Function, Target, *Info); +} + +const MachinePassContract &PrePEITargetPseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PrePEI); +} + +Expected PrePEITargetPseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "pre-PEI pseudo expansion requires allocated tracked-liveness MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PrePEI, + nullptr); +} + +const MachinePassContract &ExpandPostRAPseudosPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PostRA); +} + +Expected ExpandPostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo expansion requires finalized frame and liveness"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PostRA, + nullptr); +} + +const MachinePassContract &TargetLatePseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::TargetLate); +} + +Expected TargetLatePseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "target-late pseudo expansion requires finalized allocated MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::TargetLate, + BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineScheduler.cpp b/backend/lib/CodeGen/MachineScheduler.cpp new file mode 100644 index 0000000..f861be2 --- /dev/null +++ b/backend/lib/CodeGen/MachineScheduler.cpp @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/RegisterPressure.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &schedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] Expected isSchedulingBoundary(const MachineInstr &Instruction, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Model.hooks().isSchedulingBoundary(Instruction, Target)) + return true; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi)) { + return true; + } + if (Model.schedulingClass(Descriptor->SchedulingClass) == nullptr) { + if (Flags.has(InstructionFlag::Pseudo) || !Flags.has(InstructionFlag::ProducesCode)) + return true; + return makeUnexpected(ErrorCode::VerificationError, + "executable instruction has no scheduling class"); + } + return false; +} + +struct ScheduleCandidate final { + std::size_t Unit = 0; + bool Bottom = false; + std::uint64_t Cycle = 0; + RegisterPressureCost Pressure; + std::uint64_t CriticalPath = 0; + std::uint32_t Latency = 0; + std::int64_t TargetPreference = 0; + bool OptimizeLatency = false; + std::uint64_t StableID = 0; +}; + +[[nodiscard]] bool isBetterCandidate(const ScheduleCandidate &Left, + const ScheduleCandidate &Right) noexcept { + if (Left.Cycle != Right.Cycle) + return Left.Cycle < Right.Cycle; + if (Left.TargetPreference != Right.TargetPreference) + return Left.TargetPreference > Right.TargetPreference; + if (Left.OptimizeLatency && Right.OptimizeLatency) { + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + } + if (Left.Pressure.PeakIncrease != Right.Pressure.PeakIncrease) + return Left.Pressure.PeakIncrease < Right.Pressure.PeakIncrease; + if (Left.Pressure.PeakExcess != Right.Pressure.PeakExcess) + return Left.Pressure.PeakExcess < Right.Pressure.PeakExcess; + if (Left.Pressure.CurrentExcess != Right.Pressure.CurrentExcess) + return Left.Pressure.CurrentExcess < Right.Pressure.CurrentExcess; + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + if (Left.StableID != Right.StableID) + return Left.StableID < Right.StableID; + return !Left.Bottom && Right.Bottom; +} + +[[nodiscard]] Expected> +scheduleDAG(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + std::uint64_t BlockFrequency, bool EnableBidirectionalScheduling) { + auto TopPressure = RegisterPressureTracker::create(DAG, Function, Target, Model, Mode, Liveness, + RegisterPressureDirection::TopDown); + if (!TopPressure) + return std::unexpected(TopPressure.error()); + auto BottomPressure = RegisterPressureTracker::create( + DAG, Function, Target, Model, Mode, Liveness, RegisterPressureDirection::BottomUp); + if (!BottomPressure) + return std::unexpected(BottomPressure.error()); + auto TopHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::TopDown); + if (!TopHazards) + return std::unexpected(TopHazards.error()); + auto BottomHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::BottomUp); + if (!BottomHazards) + return std::unexpected(BottomHazards.error()); + + const std::size_t UnitCount = DAG.units().size(); + std::vector TopPredecessors(UnitCount); + std::vector BottomSuccessors(UnitCount); + std::vector TopReadyCycle(UnitCount, 0); + std::vector BottomReadyCycle(UnitCount, 0); + std::vector Scheduled(UnitCount, false); + for (const ScheduleUnit &Unit : DAG.units()) { + TopPredecessors[Unit.ID] = Unit.Predecessors.size(); + BottomSuccessors[Unit.ID] = Unit.Successors.size(); + } + + std::vector TopOrder; + std::vector BottomOrder; + TopOrder.reserve(UnitCount); + BottomOrder.reserve(UnitCount); + for (std::size_t ScheduledCount = 0; ScheduledCount < UnitCount; ++ScheduledCount) { + std::optional Best; + for (const ScheduleUnit &Unit : DAG.units()) { + if (Scheduled[Unit.ID]) + continue; + if (TopPredecessors[Unit.ID] == 0) { + auto Cycle = TopHazards->earliestIssueCycle(Unit, TopReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = TopPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + false, + *Cycle, + *Pressure, + Unit.Height, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::TopDown, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + if (EnableBidirectionalScheduling && BottomSuccessors[Unit.ID] == 0) { + auto Cycle = BottomHazards->earliestIssueCycle(Unit, BottomReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = BottomPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + true, + *Cycle, + *Pressure, + Unit.Depth, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::BottomUp, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + } + if (!Best) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "schedule DAG has no legal top or bottom ready unit"); + } + + const ScheduleUnit &Selected = DAG.units()[Best->Unit]; + Scheduled[Best->Unit] = true; + if (Best->Bottom) { + auto Issued = BottomHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = BottomPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + BottomOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Predecessors) { + if (BottomSuccessors[Dependency.Source] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "bottom scheduler successor count underflowed"); + } + --BottomSuccessors[Dependency.Source]; + BottomReadyCycle[Dependency.Source] = std::max( + BottomReadyCycle[Dependency.Source], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } else { + auto Issued = TopHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = TopPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + TopOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Successors) { + if (TopPredecessors[Dependency.Destination] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "top scheduler predecessor count underflowed"); + } + --TopPredecessors[Dependency.Destination]; + TopReadyCycle[Dependency.Destination] = std::max( + TopReadyCycle[Dependency.Destination], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } + } + + TopOrder.insert(TopOrder.end(), BottomOrder.rbegin(), BottomOrder.rend()); + if (TopOrder.size() != UnitCount) + return makeUnexpected(ErrorCode::InternalError, "scheduler lost a schedule unit"); + return TopOrder; +} + +[[nodiscard]] Expected +scheduleRegion(MachineBasicBlock &Block, std::span Instructions, + MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const MachineBlockLiveness &Liveness, std::uint64_t BlockFrequency, + MachineSchedulerOptions Options) { + if (Instructions.size() < 2 || Instructions.size() > Options.MaximumRegionSize) + return false; + auto DAG = ScheduleDAG::build(Block, Instructions, Function, Target, Model, Mode); + if (!DAG) + return std::unexpected(DAG.error()); + if (DAG->units().size() < 2) + return false; + auto UnitOrder = scheduleDAG(*DAG, Function, Target, Model, Mode, Liveness, BlockFrequency, + Options.EnableBidirectionalScheduling); + if (!UnitOrder) + return std::unexpected(UnitOrder.error()); + + std::vector NewOrder; + NewOrder.reserve(Instructions.size()); + for (const std::size_t UnitID : *UnitOrder) { + const ScheduleUnit *Unit = DAG->unit(UnitID); + if (Unit == nullptr) + return makeUnexpected(ErrorCode::InternalError, "scheduler produced an unknown unit ID"); + NewOrder.insert(NewOrder.end(), Unit->Instructions.begin(), Unit->Instructions.end()); + } + if (std::ranges::equal(NewOrder, Instructions)) + return false; + + MachineInstr *Anchor = Instructions.back()->next(); + for (MachineInstr *Instruction : Instructions) { + auto Removed = Block.removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + } + for (MachineInstr *Instruction : NewOrder) { + Expected Inserted = Anchor == nullptr + ? Block.appendInstruction(*Instruction) + : Block.insertInstructionBefore(*Anchor, *Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + return true; +} + +} // namespace + +Expected runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options) { + if (Options.MaximumRegionSize == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine scheduler region budget must be nonzero"); + } + MachineSchedulingRunResult Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const MachineBlockLiveness *BlockLiveness = Liveness.block(*Block); + if (BlockLiveness == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine scheduler has no block liveness boundary"); + } + bool BlockChanged = false; + std::vector Region; + auto FlushRegion = [&]() -> Expected { + auto Changed = scheduleRegion(*Block, Region, Function, Target, Model, Mode, *BlockLiveness, + Frequencies.frequency(*Block), Options); + if (!Changed) + return std::unexpected(Changed.error()); + BlockChanged |= *Changed; + Region.clear(); + return {}; + }; + + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a broken bundle header"); + } + std::vector Group; + bool Boundary = false; + for (;;) { + Group.push_back(Instruction); + auto IsBoundary = isSchedulingBoundary(*Instruction, Target, Model); + if (!IsBoundary) + return std::unexpected(IsBoundary.error()); + Boundary |= *IsBoundary; + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!HasSuccessor) + break; + MachineInstr *Next = Instruction->next(); + if (Next == nullptr || !Next->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a truncated bundle"); + } + Instruction = Next; + } + MachineInstr *NextGroup = Instruction->next(); + if (Boundary) { + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + } else { + Region.insert(Region.end(), Group.begin(), Group.end()); + } + Instruction = NextGroup; + } + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + if (BlockChanged) { + Result.Changed = true; + Result.ChangedBlocks.push_back(Block); + } + } + return Result; +} + +const MachinePassContract &MachineSchedulerPass::contract() const noexcept { + return schedulerContract(); +} + +Expected MachineSchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA machine scheduler requires selected, SSA-exited MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialResult = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PreRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PreRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "machine scheduler diverged after a successful transaction preflight: ")); + } + + MachinePassResult Result; + Result.Changed = Run->Changed; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSchedulerInternal.h b/backend/lib/CodeGen/MachineSchedulerInternal.h new file mode 100644 index 0000000..a612548 --- /dev/null +++ b/backend/lib/CodeGen/MachineSchedulerInternal.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H +#define RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/Support/Error.h" + +#include + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class TargetDescription; +class TargetSchedulingModel; +struct MachineSchedulingRunResult final { + bool Changed = false; + std::vector ChangedBlocks; +}; + +[[nodiscard]] Expected runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H diff --git a/backend/lib/CodeGen/MachineShrinkWrapping.cpp b/backend/lib/CodeGen/MachineShrinkWrapping.cpp new file mode 100644 index 0000000..aee3323 --- /dev/null +++ b/backend/lib/CodeGen/MachineShrinkWrapping.cpp @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineShrinkWrapping.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &shrinkWrappingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-shrink-wrapping", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool requiresEntryFrame(const MachineFrameState &State) noexcept { + return State.IsFrameAddressTaken || State.IsReturnAddressTaken || State.HasStackMap || + State.HasPatchPoint || State.HasOpaqueSPAdjustment || State.HasVAStart || + State.HasMustTailInVarArgFunc || State.HasTailCall; +} + +[[nodiscard]] bool containsEpilogueTerminator(const MachineBasicBlock &Block, + const TargetFrameLowering &Info, + const TargetDescription &Target) noexcept { + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Info.isEpilogueTerminator(*Instruction, Target)) + return true; + } + return false; +} + +} // namespace + +const MachinePassContract &MachineShrinkWrappingPass::contract() const noexcept { + return shrinkWrappingContract(); +} + +Expected MachineShrinkWrappingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping requires allocated tracked-liveness MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "shrink wrapping must run before frame layout freeze"); + } + if (Function.blockCount() > Options.MaximumAnalyzedBlocks) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineFrameState &OriginalState = Function.frameInfo().state(); + if (!OriginalState.SavePoints.empty() || !OriginalState.RestorePoints.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr || requiresEntryFrame(OriginalState)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Loops = Analyses.getResult(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + + std::vector RequiredBlocks; + std::unordered_set ReferencedObjects; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + bool Required = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping encountered an unknown opcode"); + } + Required |= Descriptor->Flags.has(InstructionFlag::Call); + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Frame = Operand.getIf()) { + ReferencedObjects.insert(Frame->Index); + Required = true; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && Operand.isDef() && + Info->isCalleeSavedRegister(*Physical, Function, Target)) { + Required = true; + } + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + const std::optional Frame = + Memory == nullptr ? std::nullopt : Memory->description().Base.frameIndex(); + if (Frame) { + ReferencedObjects.insert(*Frame); + Required = true; + } + } + } + if (Required) + RequiredBlocks.push_back(Block); + } + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || Description->IsDead) + continue; + if (!ReferencedObjects.contains(Object.index())) { + RequiredBlocks.push_back(Entry); + break; + } + } + if (RequiredBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineBasicBlock *Save = RequiredBlocks.front(); + if (!(*Dominators)->isReachable(*Save)) + Save = Entry; + for (MachineBasicBlock *Block : RequiredBlocks) { + if (!(*Dominators)->isReachable(*Block)) { + Save = Entry; + break; + } + const MachineBasicBlock *Common = (*Dominators)->nearestCommonDominator(*Save, *Block); + if (Common == nullptr) { + Save = Entry; + break; + } + Save = Common; + } + if (Save == Entry || Save->predecessors().size() != 1 || (*Loops)->loopDepth(*Save) != 0 || + (*Loops)->isIrreducible(*Save)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + std::vector RestoreBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!(*Dominators)->dominates(*Save, *Block)) + continue; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || !(*Dominators)->dominates(*Save, *Successor.Block)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + } + if (!Block->successors().empty()) + continue; + if (!containsEpilogueTerminator(*Block, *Info, Target)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + RestoreBlocks.push_back(Block); + } + if (RestoreBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + MachineFrameState Planned = OriginalState; + Planned.SavePoints.push_back({Save->stableID(), {}}); + std::ranges::sort(RestoreBlocks, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + for (const MachineBasicBlock *Block : RestoreBlocks) + Planned.RestorePoints.push_back({Block->stableID(), {}}); + auto Updated = Function.frameInfo().setState(std::move(Planned)); + if (!Updated) + return std::unexpected(Updated.error()); + + MachinePassResult Result; + Result.Changed = true; + Result.Changes = {MachinePassChange::Frame}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp new file mode 100644 index 0000000..9e73cea --- /dev/null +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineSink.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &sinkContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-sink", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-sink", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id(), + MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected isSafeToSink(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine sink encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +struct SinkDestination final { + MachineBasicBlock *Block = nullptr; + MachineInstr *FirstUse = nullptr; +}; + +[[nodiscard]] std::optional findDestination(MachineInstr &Instruction, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return std::nullopt; + std::unordered_set Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg != nullptr && Operand.isDef()) + Definitions.insert(*Reg); + } + if (Definitions.empty()) + return std::nullopt; + + MachineBasicBlock *Destination = nullptr; + for (const Register Reg : Definitions) { + for (MachineOperand &Use : Registers.useOperands(Reg, false)) { + MachineInstr *UseInstruction = Use.parent(); + MachineBasicBlock *UseBlock = UseInstruction == nullptr ? nullptr : UseInstruction->parent(); + if (UseBlock == nullptr || UseBlock == Source) + return std::nullopt; + if (Destination == nullptr) + Destination = UseBlock; + else if (Destination != UseBlock) + return std::nullopt; + } + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source || + std::ranges::find_if(Source->successors(), [Destination](const MachineSuccessor &Successor) { + return Successor.Block == Destination; + }) == Source->successors().end()) { + return std::nullopt; + } + + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + const bool UsesDefinition = + std::ranges::any_of(Candidate->operands(), [&Definitions](const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + return Reg != nullptr && Operand.isUse() && !Operand.isDebugUse() && + Definitions.contains(*Reg); + }); + if (UsesDefinition) { + FirstUse = Candidate; + break; + } + } + return FirstUse == nullptr ? std::nullopt + : std::optional{{Destination, FirstUse}}; +} + +[[nodiscard]] std::optional +findPostRADestination(MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Source->successors().empty()) + return std::nullopt; + + std::vector Uses; + std::vector Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return std::nullopt; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return std::nullopt; + MachineBasicBlock *Destination = nullptr; + for (const MachineSuccessor &Successor : Source->successors()) { + if (Successor.Block == nullptr) + return std::nullopt; + const bool CarriesAny = std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + const bool CarriesAll = std::ranges::all_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + if (CarriesAny != CarriesAll || (CarriesAll && Destination != nullptr)) + return std::nullopt; + if (CarriesAll) + Destination = Successor.Block; + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source) { + return std::nullopt; + } + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (!std::ranges::any_of(Destination->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return std::nullopt; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return std::nullopt; + } + } + + const auto OverlapsAny = [&](PhysicalRegister Reg, std::span Values) { + return std::ranges::any_of( + Values, [&](PhysicalRegister Value) { return Target.registers().regsOverlap(Reg, Value); }); + }; + for (MachineInstr *After = Instruction.next(); After != nullptr; After = After->next()) { + for (const MachineOperand &Operand : After->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if ((Operand.isUse() || Operand.isDef()) && OverlapsAny(*Physical, Definitions)) { + return std::nullopt; + } + if (Operand.isDef() && OverlapsAny(*Physical, Uses)) + return std::nullopt; + } + } + + std::vector SeenUse(Definitions.size()); + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf(); + Mask != nullptr && Mask->Mask != nullptr) { + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!SeenUse[Index] && Mask->Mask->clobbers(Definitions[Index])) + return std::nullopt; + } + if (FirstUse == nullptr && std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { + return Mask->Mask->clobbers(Reg); + })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!Target.registers().regsOverlap(*Physical, Definitions[Index])) + continue; + if (Operand.isUse() && !Operand.isDebugUse()) { + SeenUse[Index] = true; + if (FirstUse == nullptr) + FirstUse = Candidate; + } + if (Operand.isDef() && !SeenUse[Index]) + return std::nullopt; + } + if (FirstUse == nullptr && Operand.isDef() && OverlapsAny(*Physical, Uses)) { + return std::nullopt; + } + } + } + if (FirstUse == nullptr || !std::ranges::all_of(SeenUse, [](bool Value) { return Value; })) + return std::nullopt; + return SinkDestination{Destination, FirstUse}; +} + +[[nodiscard]] Expected moveBefore(MachineInstr &Instruction, + const SinkDestination &Destination) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Destination.Block == nullptr || Destination.FirstUse == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine sink has an invalid move plan"); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return Destination.Block->insertInstructionBefore(*Destination.FirstUse, Instruction); +} + +} // namespace + +const MachinePassContract &MachineSinkPass::contract() const noexcept { + return sinkContract(Stage); +} + +Expected MachineSinkPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine sink requires selected SSA MachineIR" + : "post-RA machine sink requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine sink transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine sink diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected MachineSinkPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t Sunk = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); + Block != nullptr && Sunk < Options.MaximumSunkInstructions; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && Sunk < Options.MaximumSunkInstructions;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToSink(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + if (!*Safe) { + Instruction = Next; + continue; + } + const std::optional Destination = + Stage == MachineCodeMotionStage::SelectedSSA + ? findDestination(*Instruction, Function.registerInfo()) + : findPostRADestination(*Instruction, Function, Target); + if (!Destination || !(*Dominators)->dominates(*Block, *Destination->Block) || + (*Frequencies)->frequency(*Destination->Block) > (*Frequencies)->frequency(*Block)) { + Instruction = Next; + continue; + } + auto Moved = moveBefore(*Instruction, *Destination); + if (!Moved) + return std::unexpected(Moved.error()); + ++Sunk; + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve(); + Result.Preserved.preserve(); + Result.Preserved.preserve(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineStackSlotColoring.cpp b/backend/lib/CodeGen/MachineStackSlotColoring.cpp new file mode 100644 index 0000000..b3b9af5 --- /dev/null +++ b/backend/lib/CodeGen/MachineStackSlotColoring.cpp @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineStackSlotColoring.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &stackSlotColoringContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-stack-slot-coloring", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands, MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct SlotInterval final { + MachineBasicBlock *Block = nullptr; + std::size_t First = 0; + std::size_t Last = 0; + bool FirstStores = false; + bool LastLoads = false; + bool IsValid = true; +}; + +struct InstructionReference final { + MachineInstr *Instruction = nullptr; + std::size_t Position = 0; + bool Loads = false; + bool Stores = false; + bool HasMemoryReference = false; + bool HasFrameOperand = false; +}; + +struct SlotUseInfo final { + FrameIndex Slot; + SlotInterval Interval; + std::vector References; +}; + +[[nodiscard]] InstructionReference &getReference(SlotUseInfo &Info, MachineInstr &Instruction, + std::size_t Position) { + const auto Existing = + std::ranges::find_if(Info.References, [&Instruction](const InstructionReference &Reference) { + return Reference.Instruction == &Instruction; + }); + if (Existing != Info.References.end()) + return *Existing; + Info.References.push_back({.Instruction = &Instruction, .Position = Position}); + return Info.References.back(); +} + +[[nodiscard]] std::unordered_map +collectSlotUses(MachineFunction &Function) { + std::unordered_map Result; + std::unordered_map> Locations; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::size_t Position = 0; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next(), ++Position) { + Locations.emplace(Instruction->stableID(), std::pair{Block, Position}); + for (const MachineOperand &Operand : Instruction->operands()) { + const FrameIndexOperand *Frame = Operand.getIf(); + if (Frame == nullptr) + continue; + SlotUseInfo &Info = + Result + .try_emplace(Frame->Index, + SlotUseInfo{.Slot = Frame->Index, .Interval = {}, .References = {}}) + .first->second; + getReference(Info, *Instruction, Position).HasFrameOperand = true; + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + if (Memory == nullptr) + continue; + const std::optional Frame = Memory->description().Base.frameIndex(); + if (!Frame) + continue; + SlotUseInfo &Info = + Result + .try_emplace(*Frame, SlotUseInfo{.Slot = *Frame, .Interval = {}, .References = {}}) + .first->second; + InstructionReference &Reference = getReference(Info, *Instruction, Position); + Reference.HasMemoryReference = true; + Reference.Loads |= Memory->isLoad(); + Reference.Stores |= Memory->isStore(); + } + } + } + + for (auto &[Slot, Info] : Result) { + (void)Slot; + if (Info.References.empty()) { + Info.Interval.IsValid = false; + continue; + } + Info.Interval.Block = Info.References.front().Instruction->parent(); + Info.Interval.First = Info.References.front().Position; + Info.Interval.Last = Info.References.back().Position; + Info.Interval.FirstStores = Info.References.front().Stores; + Info.Interval.LastLoads = Info.References.back().Loads; + for (const InstructionReference &Reference : Info.References) { + if (Reference.Instruction->parent() != Info.Interval.Block || !Reference.HasMemoryReference || + (Reference.HasFrameOperand && !Reference.HasMemoryReference)) { + Info.Interval.IsValid = false; + } + } + Info.Interval.IsValid &= + Info.Interval.Block != nullptr && Info.Interval.FirstStores && Info.Interval.LastLoads; + const StackObject *Object = Function.frameInfo().object(Info.Slot); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr) { + Info.Interval.IsValid = false; + continue; + } + for (const StackObjectLifetimeRange &Range : Description->Lifetime) { + const auto Begin = Locations.find(Range.Begin); + const auto End = Locations.find(Range.End); + if (Begin == Locations.end() || End == Locations.end() || + Begin->second.first != Info.Interval.Block || End->second.first != Info.Interval.Block || + Begin->second.second > End->second.second) { + Info.Interval.IsValid = false; + break; + } + Info.Interval.First = std::min(Info.Interval.First, Begin->second.second); + Info.Interval.Last = std::max(Info.Interval.Last, End->second.second); + } + } + return Result; +} + +[[nodiscard]] bool isEligibleSlot(const StackObjectDesc &Description) noexcept { + return Description.Kind == StackObjectKind::SpillSlot && !Description.IsDead && + !Description.IsImmutable && !Description.IsAliased && !Description.AddressTaken && + !Description.AllocaName && !Description.Offset && !Description.LocalOffset && + !Description.CalleeSavedRegister && !Description.DebugInfo && + Description.TargetFlags == 0 && Description.Size.has_value(); +} + +[[nodiscard]] bool compatibleSlots(const StackObjectDesc &Left, + const StackObjectDesc &Right) noexcept { + return isEligibleSlot(Left) && isEligibleSlot(Right) && Left.Kind == Right.Kind && + Left.Size == Right.Size && Left.Align == Right.Align && Left.Stack == Right.Stack; +} + +[[nodiscard]] bool intervalsOverlap(const SlotInterval &Left, const SlotInterval &Right) noexcept { + if (Left.Block != Right.Block) + return true; + return !(Left.Last < Right.First || Right.Last < Left.First); +} + +struct StackColor final { + FrameIndex Canonical; + const StackObjectDesc *Description = nullptr; + std::vector Intervals; +}; + +[[nodiscard]] Expected rewriteSlot(MachineFunction &Function, FrameIndex OldSlot, + FrameIndex NewSlot) { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const FrameIndexOperand *Frame = + Operand == nullptr ? nullptr : Operand->getIf(); + if (Frame == nullptr || Frame->Index != OldSlot) + continue; + auto Rewritten = Operand->setFrameIndex(NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + for (std::size_t Index = 0; Index < Instruction->memOperands().size(); ++Index) { + const MachineMemOperand *Memory = Instruction->memOperands()[Index]; + if (Memory == nullptr || Memory->description().Base.frameIndex() != OldSlot) + continue; + MachineMemOperandDesc Description = Memory->description(); + Description.Base = MemoryBase::frameIndex(NewSlot); + auto Replacement = Function.createMemOperand(std::move(Description)); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Replaced = Instruction->replaceMemOperand(Index, **Replacement); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineStackSlotColoringPass::contract() const noexcept { + return stackSlotColoringContract(); +} + +Expected MachineStackSlotColoringPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "stack-slot coloring requires allocated PHI-free MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "stack-slot coloring must run before frame layout is frozen"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "stack-slot coloring transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "stack-slot coloring diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineStackSlotColoringPass::runImplementation(MachineFunction &Function) const { + std::unordered_map Uses = collectSlotUses(Function); + std::vector Colors; + std::vector> Rewrites; + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + const auto Use = Uses.find(Object.index()); + if (Description == nullptr || Use == Uses.end() || !Use->second.Interval.IsValid || + !isEligibleSlot(*Description)) { + continue; + } + auto Color = std::ranges::find_if(Colors, [&](const StackColor &Candidate) { + return Candidate.Description != nullptr && + compatibleSlots(*Candidate.Description, *Description) && + std::ranges::none_of(Candidate.Intervals, [&](const SlotInterval &Interval) { + return intervalsOverlap(Interval, Use->second.Interval); + }); + }); + if (Color == Colors.end()) { + Colors.push_back({Object.index(), Description, {Use->second.Interval}}); + continue; + } + if (Rewrites.size() == Options.MaximumColoredSlots) + break; + Rewrites.emplace_back(Object.index(), Color->Canonical); + Color->Intervals.push_back(Use->second.Interval); + } + + for (const auto &[OldSlot, NewSlot] : Rewrites) { + const StackObject *OldObject = Function.frameInfo().object(OldSlot); + const StackObject *NewObject = Function.frameInfo().object(NewSlot); + if (OldObject == nullptr || NewObject == nullptr || OldObject->description() == nullptr || + NewObject->description() == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "stack-slot coloring lost a planned stack object"); + } + auto Rewritten = rewriteSlot(Function, OldSlot, NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + StackObjectDesc Combined = *NewObject->description(); + Combined.Lifetime.insert(Combined.Lifetime.end(), OldObject->description()->Lifetime.begin(), + OldObject->description()->Lifetime.end()); + auto Updated = Function.frameInfo().updateObject(NewSlot, std::move(Combined)); + if (!Updated) + return std::unexpected(Updated.error()); + auto Dead = Function.frameInfo().markObjectDead(OldSlot); + if (!Dead) + return std::unexpected(Dead.error()); + } + + MachinePassResult Result; + Result.Changed = !Rewrites.empty(); + if (Result.Changed) { + Result.Changes = {MachinePassChange::Operands, MachinePassChange::Frame}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailDuplication.cpp b/backend/lib/CodeGen/MachineTailDuplication.cpp new file mode 100644 index 0000000..a130680 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailDuplication.cpp @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailDuplication.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailDuplicationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-duplication", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] Expected isDuplicableInstruction(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::NotDuplicable) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Instruction.flags().has(MachineInstrFlag::NoMerge) || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc) || + hasProtectedAttachment(Instruction)) { + return false; + } + return std::ranges::none_of(Instruction.operands(), [](const MachineOperand &Operand) { + return Operand.kind() == MachineOperand::Kind::BasicBlock || + Operand.kind() == MachineOperand::Kind::DebugInstrRef; + }); +} + +[[nodiscard]] Expected +cloneInstructionBefore(MachineInstr &Source, MachineBasicBlock &Destination, MachineInstr *Before) { + MachineFunction &Function = Destination.function(); + auto Created = Function.createInstruction(Source.opcode()); + if (!Created) + return std::unexpected(Created.error()); + MachineInstr *Clone = *Created; + Clone->reserveOperands(Source.operandCount()); + for (const MachineOperand &Operand : Source.operands()) { + auto Added = Clone->addOperand(Operand.cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + const std::optional Tied = + Operand == nullptr ? std::nullopt : Operand->tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = Operand->isDef() ? Index : *Tied; + const std::size_t UseIndex = Operand->isDef() ? *Tied : Index; + auto TiedResult = Clone->tieOperands(DefIndex, UseIndex); + if (!TiedResult) + return std::unexpected(TiedResult.error()); + } + for (const MachineMemOperand *Memory : Source.memOperands()) { + if (Memory == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered a null memory operand"); + auto Added = Clone->addMemOperand(*Memory); + if (!Added) + return std::unexpected(Added.error()); + } + Clone->setFlags(Source.flags()); + if (Source.sourceLocation()) + Clone->setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Clone->setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Clone->setOrigin(*Source.origin()); + Expected Inserted = Before == nullptr + ? Destination.appendInstruction(*Clone) + : Destination.insertInstructionBefore(*Before, *Clone); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Clone; +} + +struct TailCandidate final { + MachineBasicBlock *Successor = nullptr; + MachineInstr *Branch = nullptr; + std::vector Body; +}; + +[[nodiscard]] Expected> +analyzeTail(MachineBasicBlock &Tail, MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info, std::size_t MaximumInstructions) { + if (&Tail == Function.firstBlock() || Tail.predecessors().size() < 2 || + Tail.successors().size() != 1 || Tail.successors().front().Block == nullptr || + hasProtectedIdentity(Tail, Function)) { + return std::nullopt; + } + auto Branch = Info.analyzeBranch(Tail, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + MachineBasicBlock *Successor = Tail.successors().front().Block; + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr || + Branch->Unconditional == nullptr || Branch->TrueTarget != Successor || + Branch->Unconditional != Tail.lastInstruction() || + hasProtectedAttachment(*Branch->Unconditional)) { + return std::nullopt; + } + + TailCandidate Candidate{.Successor = Successor, .Branch = Branch->Unconditional, .Body = {}}; + for (MachineInstr *Instruction = Tail.firstInstruction(); Instruction != Candidate.Branch; + Instruction = Instruction->next()) { + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail branch is not reachable from its block body"); + auto Duplicable = isDuplicableInstruction(*Instruction, Target); + if (!Duplicable) + return std::unexpected(Duplicable.error()); + if (!*Duplicable) + return std::nullopt; + Candidate.Body.push_back(Instruction); + if (Candidate.Body.size() > MaximumInstructions) + return std::nullopt; + } + return Candidate; +} + +[[nodiscard]] Expected +duplicateTailIntoPredecessor(const TailCandidate &Candidate, MachineBasicBlock &Tail, + MachineBasicBlock &Predecessor, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + if (Predecessor.successors().size() != 1 || Predecessor.successors().front().Block != &Tail) { + return false; + } + auto Branch = Info.analyzeBranch(Predecessor, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr) + return false; + MachineInstr *OldBranch = Branch->Unconditional; + if (OldBranch != nullptr) { + if (Branch->TrueTarget != &Tail || OldBranch != Predecessor.lastInstruction() || + hasProtectedAttachment(*OldBranch)) { + return false; + } + } else if (Predecessor.next() != &Tail) { + return false; + } + + for (MachineInstr *Instruction : Candidate.Body) { + auto Cloned = cloneInstructionBefore(*Instruction, Predecessor, OldBranch); + if (!Cloned) + return std::unexpected(Cloned.error()); + } + if (OldBranch != nullptr) { + auto Removed = Info.removeBranch(*OldBranch, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + } + auto NewBranch = + Info.insertUnconditionalBranch(Predecessor, *Candidate.Successor, Function, Target); + if (!NewBranch) + return std::unexpected(NewBranch.error()); + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Tail, *Candidate.Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailDuplicationPass::contract() const noexcept { + return tailDuplicationContract(); +} + +Expected MachineTailDuplicationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "tail duplication transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail duplication diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineTailDuplicationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t DuplicatedInstructions = 0; + for (MachineBasicBlock *Tail = Function.firstBlock(); Tail != nullptr;) { + MachineBasicBlock *Next = Tail->next(); + auto Candidate = analyzeTail(*Tail, Function, Target, *Info, Options.MaximumTailInstructions); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!Candidate->has_value()) { + Tail = Next; + continue; + } + const std::size_t Cost = (*Candidate)->Body.size() + 1U; + const std::vector Predecessors(Tail->predecessors().begin(), + Tail->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || + DuplicatedInstructions > Options.MaximumDuplicatedInstructions || + Cost > Options.MaximumDuplicatedInstructions - DuplicatedInstructions) { + break; + } + auto Duplicated = + duplicateTailIntoPredecessor(**Candidate, *Tail, *Predecessor, Function, Target, *Info); + if (!Duplicated) + return std::unexpected(Duplicated.error()); + if (!*Duplicated) + continue; + DuplicatedInstructions += Cost; + Result.Changed = true; + } + if (Tail->predecessors().empty()) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Tail); + if (!Deleted) + return std::unexpected(Deleted.error()); + Result.Changed = true; + } + Tail = Next; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailMerge.cpp b/backend/lib/CodeGen/MachineTailMerge.cpp new file mode 100644 index 0000000..b754978 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailMerge.cpp @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailMerge.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailMergeContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-merge", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +template +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf(); + const ValueT *RightValue = Right.getIf(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +template +[[nodiscard]] bool maskEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const MaskOperandT *LeftValue = Left.getIf(); + const MaskOperandT *RightValue = Right.getIf(); + if (LeftValue == nullptr || RightValue == nullptr) + return false; + if (!LeftValue->Mask || !RightValue->Mask) + return LeftValue->Mask == RightValue->Mask; + return *LeftValue->Mask == *RightValue->Mask; +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() || + Left.tiedOperandIndex() != Right.tiedOperandIndex()) { + return false; + } + if (Left.kind() == MachineOperand::Kind::Register) { + const Register *LeftReg = Left.getIf(); + const Register *RightReg = Right.getIf(); + if (LeftReg == nullptr || RightReg == nullptr || *LeftReg != *RightReg || + Left.subReg() != Right.subReg()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags; + } + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return false; + case MachineOperand::Kind::Immediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals(Left, Right); + case MachineOperand::Kind::RegisterMask: + return maskEquals(Left, Right); + case MachineOperand::Kind::LiveOutRegisterMask: + return maskEquals(Left, Right); + case MachineOperand::Kind::Metadata: + return payloadEquals(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] Expected instructionEquals(const MachineInstr &Left, const MachineInstr &Right, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Left.opcode()); + if (Descriptor == nullptr || Target.instructionDescriptor(Right.opcode()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge encountered an unknown instruction"); + } + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size() || + Left.sourceLocation() != Right.sourceLocation() || + Left.debugLocation() != Right.debugLocation() || Left.origin() != Right.origin() || + hasProtectedAttachment(Left) || hasProtectedAttachment(Right) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Convergent) || + Descriptor->Flags.has(InstructionFlag::NotDuplicable) || + Left.flags().has(MachineInstrFlag::NoMerge) || + Left.flags().has(MachineInstrFlag::BundledWithPred) || + Left.flags().has(MachineInstrFlag::BundledWithSucc)) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + !operandEquals(*LeftOperand, *RightOperand)) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || + LeftMemory->description() != RightMemory->description()) { + return false; + } + } + return true; +} + +[[nodiscard]] Expected blockEquals(const MachineBasicBlock &Left, + const MachineBasicBlock &Right, + const MachineFunction &Function, + const TargetDescription &Target) { + if (&Left == &Right || &Left == Function.firstBlock() || &Right == Function.firstBlock() || + hasProtectedIdentity(Left, Function) || hasProtectedIdentity(Right, Function) || + Left.attributes().alignment() != Right.attributes().alignment() || + Left.attributes().sectionID() != Right.attributes().sectionID() || + Left.attributes().callFrameSize() != Right.attributes().callFrameSize() || + !std::ranges::equal(Left.liveIns(), Right.liveIns()) || + !std::ranges::equal(Left.successors(), Right.successors()) || + Left.instructionCount() != Right.instructionCount()) { + return false; + } + const MachineInstr *LeftInstruction = Left.firstInstruction(); + const MachineInstr *RightInstruction = Right.firstInstruction(); + while (LeftInstruction != nullptr && RightInstruction != nullptr) { + auto Equal = instructionEquals(*LeftInstruction, *RightInstruction, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + return false; + LeftInstruction = LeftInstruction->next(); + RightInstruction = RightInstruction->next(); + } + return LeftInstruction == nullptr && RightInstruction == nullptr; +} + +[[nodiscard]] Expected canRedirectPredecessor(MachineBasicBlock &Predecessor, + MachineBasicBlock &Duplicate, + MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + return false; + if ((Analysis->Conditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Conditional)) || + (Analysis->Unconditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Unconditional))) { + return false; + } + if (Analysis->TrueTarget == &Duplicate || Analysis->FalseTarget == &Duplicate) + return true; + return Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate; +} + +[[nodiscard]] Expected +redirectPredecessor(MachineBasicBlock &Predecessor, MachineBasicBlock &Duplicate, + MachineBasicBlock &Canonical, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + auto Rewritten = Info.rewriteBranchTarget(Predecessor, Duplicate, Canonical, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + const bool IsImplicitEdge = + (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate) || + (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == &Duplicate); + if (!IsImplicitEdge) + return false; + auto Inserted = Info.insertUnconditionalBranch(Predecessor, Canonical, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Duplicate, Canonical); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailMergePass::contract() const noexcept { + return tailMergeContract(); +} + +Expected MachineTailMergePass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "tail merge transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail merge diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +MachineTailMergePass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t MergedBlocks = 0; + bool LocalChange = true; + while (LocalChange && MergedBlocks < Options.MaximumMergedBlocks) { + LocalChange = false; + std::vector Blocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Blocks.push_back(Block); + } + for (std::size_t DuplicateIndex = 1; DuplicateIndex < Blocks.size(); ++DuplicateIndex) { + MachineBasicBlock *Duplicate = Blocks[DuplicateIndex]; + if (Duplicate == nullptr || !Duplicate->isLinked()) + continue; + for (std::size_t CanonicalIndex = 1; CanonicalIndex < DuplicateIndex; ++CanonicalIndex) { + MachineBasicBlock *Canonical = Blocks[CanonicalIndex]; + if (Canonical == nullptr || !Canonical->isLinked()) + continue; + auto Equal = blockEquals(*Canonical, *Duplicate, Function, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + continue; + + const std::vector Predecessors(Duplicate->predecessors().begin(), + Duplicate->predecessors().end()); + bool RedirectedAll = true; + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || Predecessor == Canonical || Predecessor == Duplicate) { + RedirectedAll = false; + break; + } + auto CanRedirect = + canRedirectPredecessor(*Predecessor, *Duplicate, Function, Target, *Info); + if (!CanRedirect) + return std::unexpected(CanRedirect.error()); + if (!*CanRedirect) { + RedirectedAll = false; + break; + } + } + if (!RedirectedAll) + continue; + for (MachineBasicBlock *Predecessor : Predecessors) { + auto Redirected = + redirectPredecessor(*Predecessor, *Duplicate, *Canonical, Function, Target, *Info); + if (!Redirected) + return std::unexpected(Redirected.error()); + if (!*Redirected) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail merge predecessor changed after redirect preflight"); + } + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Duplicate); + if (!Deleted) + return std::unexpected(Deleted.error()); + ++MergedBlocks; + Result.Changed = true; + LocalChange = true; + break; + } + if (LocalChange) + break; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PostRAScheduler.cpp b/backend/lib/CodeGen/PostRAScheduler.cpp new file mode 100644 index 0000000..9aff5ac --- /dev/null +++ b/backend/lib/CodeGen/PostRAScheduler.cpp @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PostRAScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &postRASchedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct PhysicalUnitAccess final { + RegisterUnitID Unit; + LaneMask Lanes; +}; + +struct OperandFlagUpdate final { + MachineOperand *Operand = nullptr; + RegisterOperandFlags Flags; +}; + +[[nodiscard]] LaneMask normalizeUnitLanes(LaneMask UnitViewLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +[[nodiscard]] Expected> +physicalOperandUnits(const MachineOperand &Operand, const TargetDescription &Target) { + const Register *Reg = Operand.getIf(); + std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a virtual register"); + } + if (Operand.subReg().value() != 0) { + Physical = Target.registers().subRegister(*Physical, Operand.subReg()); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered an invalid subregister"); + } + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr || Description->Units.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a unitless register"); + } + std::vector Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) { + if (Unit.Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered empty register-unit lanes"); + } + Result.push_back({Unit.Unit, normalizeUnitLanes(Unit.Lanes)}); + } + return Result; +} + +void addPhysicalRegister(RegisterUnitLaneSet &Live, const PhysicalRegisterDesc &Register) { + for (const RegisterUnitLane &Unit : Register.Units) + Live.add(Unit.Unit, normalizeUnitLanes(Unit.Lanes)); +} + +[[nodiscard]] bool intersects(const RegisterUnitLaneSet &Live, + const std::vector &Units) noexcept { + return std::ranges::any_of(Units, [&Live](const PhysicalUnitAccess &Unit) { + return Live.intersects(Unit.Unit, Unit.Lanes); + }); +} + +void addUnits(RegisterUnitLaneSet &Live, const std::vector &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.add(Unit.Unit, Unit.Lanes); +} + +void removeUnits(RegisterUnitLaneSet &Live, const std::vector &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.remove(Unit.Unit, Unit.Lanes); +} + +[[nodiscard]] OperandFlagUpdate &getFlagUpdate(std::vector &Updates, + MachineOperand &Operand) { + const auto Position = std::ranges::find_if( + Updates, [&Operand](const OperandFlagUpdate &Update) { return Update.Operand == &Operand; }); + if (Position != Updates.end()) + return *Position; + Updates.push_back({&Operand, Operand.registerFlags()}); + return Updates.back(); +} + +[[nodiscard]] Expected> +buildBlockLivenessRepairPlan(MachineBasicBlock &Block, const TargetDescription &Target, + const MachineBlockLiveness &Boundary) { + RegisterUnitLaneSet Live = Boundary.PhysicalLiveOut; + std::vector Updates; + for (MachineInstr *Instruction = Block.lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *Mask = Operand.getIf(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null live-out mask"); + } + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->preserves(Register.Reg)) + addPhysicalRegister(Live, Register); + } + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Dead); + if (!Operand->isDef()) + continue; + const Register *Reg = Operand->getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Dead); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + removeUnits(Live, *Units); + continue; + } + const auto *Mask = Operand.getIf(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null register mask"); + } + RegisterUnitLaneSet Clobbers; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->clobbers(Register.Reg)) + addPhysicalRegister(Clobbers, Register); + } + Live.subtract(Clobbers); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Kill); + if (!Operand->isUse() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand->getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Kill); + addUnits(Live, *Units); + } + } + if (Live != Boundary.PhysicalLiveIn) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA local liveness repair changed the block boundary"); + } + return Updates; +} + +} // namespace + +Expected repairPostRAMachineBlockLiveness(std::span Blocks, + const TargetDescription &Target, + const LiveVariablesAnalysis::Result &Liveness) { + std::vector Updates; + for (MachineBasicBlock *Block : Blocks) { + const MachineBlockLiveness *Boundary = Liveness.block(*Block); + if (Boundary == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA scheduler lost a block liveness boundary"); + } + auto BlockUpdates = buildBlockLivenessRepairPlan(*Block, Target, *Boundary); + if (!BlockUpdates) + return std::unexpected(BlockUpdates.error()); + Updates.insert(Updates.end(), BlockUpdates->begin(), BlockUpdates->end()); + } + bool Changed = false; + for (const OperandFlagUpdate &Update : Updates) { + Changed |= Update.Operand->registerFlags() != Update.Flags; + auto Applied = Update.Operand->setRegisterFlags(Update.Flags); + if (!Applied) + return std::unexpected(Applied.error()); + } + return Changed; +} + +const MachinePassContract &PostRASchedulerPass::contract() const noexcept { + return postRASchedulerContract(); +} + +Expected PostRASchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduler requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "post-RA scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialRun = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PostRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialRepair = + repairPostRAMachineBlockLiveness(TrialRun->ChangedBlocks, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PostRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "post-RA scheduler diverged after a successful transaction preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(Run->ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = Run->Changed || *Repaired; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterPressure.cpp b/backend/lib/CodeGen/RegisterPressure.cpp new file mode 100644 index 0000000..523a443 --- /dev/null +++ b/backend/lib/CodeGen/RegisterPressure.cpp @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterPressure.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +using EntityID = std::size_t; + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +template void sortAndUnique(std::vector &Values) { + std::ranges::sort(Values); + const auto [FirstDuplicate, End] = std::ranges::unique(Values); + Values.erase(FirstDuplicate, End); +} + +} // namespace + +namespace detail { + +struct RegisterPressureBuilder final { + RegisterPressureTracker Result; + const ScheduleDAG *DAG = nullptr; + const MachineFunction *Function = nullptr; + const TargetDescription *Target = nullptr; + const TargetSchedulingModel *Model = nullptr; + ScheduleDAGMode Mode = ScheduleDAGMode::PreRegisterAllocation; + std::unordered_map CurrentVirtualEntities; + std::unordered_map CurrentPhysicalEntities; + std::unordered_map> PhysicalSets; + + [[nodiscard]] Expected initializePressureSets() { + Result.Pressure.reserve(Model->pressureSets().size()); + for (const RegisterPressureSetDesc &Set : Model->pressureSets()) { + Result.PressureIndexes.emplace(Set.ID, Result.Pressure.size()); + Result.Pressure.push_back({Set.ID, 0, 0, Set.Limit}); + for (const RegisterClassID ClassID : Set.RegisterClasses) { + const RegisterClassDesc *Class = Target->registers().registerClass(ClassID); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure set references an unknown target register class"); + } + for (const PhysicalRegister Physical : Class->Registers) { + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "register class references an unknown physical register"); + } + for (const RegisterUnitLane &Unit : Description->Units) + PhysicalSets[Unit.Unit].push_back(Set.ID); + } + } + } + for (auto &[Unit, Sets] : PhysicalSets) { + (void)Unit; + sortAndUnique(Sets); + } + return {}; + } + + [[nodiscard]] EntityID createEntity(std::vector Sets) { + sortAndUnique(Sets); + const EntityID ID = Result.Entities.size(); + Result.Entities.push_back({std::move(Sets), 0, false, false, false, false}); + return ID; + } + + [[nodiscard]] Expected> + virtualPressureSets(Register Reg) const { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pressure tracking encountered a virtual register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().RegClass) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA pressure tracking requires constrained virtual registers"); + } + const std::span Sets = + Model->pressureSetsForClass(*Info->get().RegClass); + return std::vector{Sets.begin(), Sets.end()}; + } + + [[nodiscard]] Expected boundaryVirtualEntity(Register Reg) { + if (const auto Position = CurrentVirtualEntities.find(Reg); + Position != CurrentVirtualEntities.end()) { + return Position->second; + } + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.emplace(Reg, ID); + return ID; + } + + [[nodiscard]] Expected defineVirtualEntity(Register Reg) { + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.insert_or_assign(Reg, ID); + return ID; + } + + [[nodiscard]] EntityID boundaryPhysicalEntity(RegisterUnitID Unit) { + if (const auto Position = CurrentPhysicalEntities.find(Unit); + Position != CurrentPhysicalEntities.end()) { + return Position->second; + } + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector{} : Sets->second); + CurrentPhysicalEntities.emplace(Unit, ID); + return ID; + } + + [[nodiscard]] EntityID definePhysicalEntity(RegisterUnitID Unit) { + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector{} : Sets->second); + CurrentPhysicalEntities.insert_or_assign(Unit, ID); + return ID; + } + + [[nodiscard]] Expected> physicalUnits(Register Reg, + SubRegIndex SubReg) const { + std::optional Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical register"); + } + if (SubReg.value() != 0) { + Physical = Target->registers().subRegister(*Physical, SubReg); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical subregister"); + } + } + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(*Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an unknown physical register"); + } + std::vector Units; + Units.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Units.push_back(Unit.Unit); + sortAndUnique(Units); + return Units; + } + + [[nodiscard]] Expected appendRegisterUse(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = boundaryVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Uses.push_back(*Entity); + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit)); + return {}; + } + + [[nodiscard]] Expected appendRegisterDefinition(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = defineVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Definitions.push_back(*Entity); + Result.Entities[*Entity].HasDefinition = true; + Result.Entities[*Entity].DefinitionPending = true; + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + const EntityID Entity = definePhysicalEntity(Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + return {}; + } + + [[nodiscard]] Expected appendMaskDefinition(RegisterPressureTracker::Event &Event, + const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) { + const EntityID Entity = definePhysicalEntity(Unit.Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + } + return {}; + } + + void appendLiveOutMaskUse(RegisterPressureTracker::Event &Event, const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit.Unit)); + } + } + + [[nodiscard]] Expected + buildEvent(const MachineInstr &Instruction) { + RegisterPressureTracker::Event Event; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null live-out register mask"); + } + appendLiveOutMaskUse(Event, *Mask->Mask); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + if (Operand.isUse()) { + auto Added = appendRegisterUse(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null register mask"); + } + auto Added = appendMaskDefinition(Event, *Mask->Mask); + if (!Added) + return std::unexpected(Added.error()); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse()) + continue; + if (Operand.isDef()) { + auto Added = appendRegisterDefinition(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + sortAndUnique(Event.Uses); + sortAndUnique(Event.Definitions); + return Event; + } + + [[nodiscard]] Expected buildEffects() { + Result.Effects.resize(DAG->units().size()); + Result.Scheduled.assign(DAG->units().size(), false); + for (const ScheduleUnit &Unit : DAG->units()) { + RegisterPressureTracker::UnitEffects &Effects = Result.Effects[Unit.ID]; + Effects.Events.reserve(Unit.Instructions.size()); + for (const MachineInstr *Instruction : Unit.Instructions) { + auto Event = buildEvent(*Instruction); + if (!Event) + return std::unexpected(Event.error()); + for (const EntityID Entity : Event->Uses) + ++Result.Entities[Entity].RemainingUses; + Effects.Events.push_back(std::move(*Event)); + } + } + return {}; + } + + void markEntityLiveOut(EntityID Entity) { + RegisterPressureTracker::Entity &Value = Result.Entities[Entity]; + if (Value.LiveOut) + return; + Value.LiveOut = true; + ++Value.RemainingUses; + } + + [[nodiscard]] Expected markBlockLiveOut(const MachineBlockLiveness &Liveness) { + for (const auto &[Reg, Lanes] : Liveness.VirtualLiveOut.entries()) { + (void)Lanes; + auto Entity = boundaryVirtualEntity(Reg); + if (!Entity) + return std::unexpected(Entity.error()); + markEntityLiveOut(*Entity); + } + for (const auto &[Unit, Lanes] : Liveness.PhysicalLiveOut.entries()) { + (void)Lanes; + markEntityLiveOut(boundaryPhysicalEntity(Unit)); + } + return {}; + } + + [[nodiscard]] Expected markSuffixUses() { + if (DAG->units().empty()) + return {}; + const MachineInstr *Instruction = DAG->units().back().Instructions.back()->next(); + std::unordered_set KilledVirtual; + std::unordered_set KilledPhysical; + for (; Instruction != nullptr; Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Mask = Operand.getIf()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure suffix scan encountered a null register mask"); + } + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask->Mask->clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + KilledPhysical.insert(Unit.Unit); + } + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isVirtual()) { + if (Operand.isUse() && !KilledVirtual.contains(*Reg)) { + const auto Position = CurrentVirtualEntities.find(*Reg); + if (Position != CurrentVirtualEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledVirtual.insert(*Reg); + continue; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + if (Operand.isUse() && !KilledPhysical.contains(Unit)) { + const auto Position = CurrentPhysicalEntities.find(Unit); + if (Position != CurrentPhysicalEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledPhysical.insert(Unit); + } + } + } + return {}; + } + + [[nodiscard]] Expected initializeLivePressure() { + for (EntityID ID = 0; ID < Result.Entities.size(); ++ID) { + RegisterPressureTracker::Entity &Entity = Result.Entities[ID]; + const bool InitiallyLive = Result.Direction == RegisterPressureDirection::TopDown + ? !Entity.HasDefinition && Entity.RemainingUses != 0 + : Entity.LiveOut; + if (!InitiallyLive) + continue; + auto Added = Result.makeLive(ID); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; + } +}; + +} // namespace detail + +Expected +RegisterPressureTracker::create(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + RegisterPressureDirection Direction) { + detail::RegisterPressureBuilder Builder; + Builder.DAG = &DAG; + Builder.Function = &Function; + Builder.Target = &Target; + Builder.Model = &Model; + Builder.Mode = Mode; + Builder.Result.Direction = Direction; + auto Sets = Builder.initializePressureSets(); + if (!Sets) + return std::unexpected(Sets.error()); + auto Effects = Builder.buildEffects(); + if (!Effects) + return std::unexpected(Effects.error()); + auto Suffix = Builder.markSuffixUses(); + if (!Suffix) + return std::unexpected(Suffix.error()); + auto LiveOut = Builder.markBlockLiveOut(Liveness); + if (!LiveOut) + return std::unexpected(LiveOut.error()); + auto Initial = Builder.initializeLivePressure(); + if (!Initial) + return std::unexpected(Initial.error()); + return std::move(Builder.Result); +} + +Expected RegisterPressureTracker::evaluate(std::size_t UnitID) const { + RegisterPressureTracker Trial = *this; + const RegisterPressureCost Before = cost(); + auto ScheduledUnit = Trial.schedule(UnitID); + if (!ScheduledUnit) + return std::unexpected(ScheduledUnit.error()); + RegisterPressureCost After = Trial.cost(); + After.PeakIncrease = + After.PeakExcess > Before.PeakExcess ? After.PeakExcess - Before.PeakExcess : 0; + return After; +} + +Expected RegisterPressureTracker::schedule(std::size_t UnitID) { + if (UnitID >= Effects.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "pressure tracker unit ID is out of range"); + if (Scheduled[UnitID]) + return makeUnexpected(ErrorCode::InvalidArgument, "schedule unit was already issued"); + RegisterPressureTracker Trial = *this; + if (Trial.Direction == RegisterPressureDirection::TopDown) { + for (const Event &Value : Trial.Effects[UnitID].Events) { + auto Applied = Trial.applyEvent(Value); + if (!Applied) + return std::unexpected(Applied.error()); + } + } else { + for (auto Position = Trial.Effects[UnitID].Events.rbegin(); + Position != Trial.Effects[UnitID].Events.rend(); ++Position) { + auto Applied = Trial.applyEvent(*Position); + if (!Applied) + return std::unexpected(Applied.error()); + } + } + Trial.Scheduled[UnitID] = true; + *this = std::move(Trial); + return {}; +} + +RegisterPressureCost RegisterPressureTracker::cost() const noexcept { + RegisterPressureCost Result; + for (const RegisterPressureValue &Value : Pressure) { + Result.CurrentPressure = saturatingAdd(Result.CurrentPressure, Value.Current); + if (Value.Current > Value.Limit) { + Result.CurrentExcess = saturatingAdd(Result.CurrentExcess, Value.Current - Value.Limit); + } + if (Value.Maximum > Value.Limit) + Result.PeakExcess = saturatingAdd(Result.PeakExcess, Value.Maximum - Value.Limit); + } + return Result; +} + +Expected RegisterPressureTracker::applyEvent(const Event &Value) { + if (Direction == RegisterPressureDirection::BottomUp) { + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size() || !Entities[Entity].DefinitionPending) { + return makeUnexpected(ErrorCode::VerificationError, + "bottom-up scheduler crossed an unavailable definition"); + } + Entities[Entity].DefinitionPending = false; + if (Entities[Entity].Live) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown use entity"); + } + if (!Entities[Entity].Live) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size() || !Entities[Entity].Live || + Entities[Entity].RemainingUses == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler selected a use before its pressure value became live"); + } + --Entities[Entity].RemainingUses; + if (Entities[Entity].RemainingUses == 0) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown definition entity"); + } + if (Entities[Entity].Live) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler redefined a pressure value while it was live"); + } + if (Entities[Entity].RemainingUses != 0) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; +} + +Expected RegisterPressureTracker::makeLive(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became live"); + Entity &Value = Entities[EntityIDValue]; + if (Value.Live) + return {}; + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure entity references an unknown pressure set"); + } + RegisterPressureValue &SetPressure = Pressure[Position->second]; + if (SetPressure.Current == std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, "register pressure counter overflowed"); + } + ++SetPressure.Current; + SetPressure.Maximum = std::max(SetPressure.Maximum, SetPressure.Current); + } + Value.Live = true; + return {}; +} + +Expected RegisterPressureTracker::makeDead(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became dead"); + Entity &Value = Entities[EntityIDValue]; + if (!Value.Live) + return makeUnexpected(ErrorCode::InternalError, "dead pressure entity was not live"); + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end() || Pressure[Position->second].Current == 0) { + return makeUnexpected(ErrorCode::InternalError, "register pressure counter underflowed"); + } + --Pressure[Position->second].Current; + } + Value.Live = false; + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleDAG.cpp b/backend/lib/CodeGen/ScheduleDAG.cpp new file mode 100644 index 0000000..8696f2e --- /dev/null +++ b/backend/lib/CodeGen/ScheduleDAG.cpp @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleDAG.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +struct RegisterDependencyKey final { + bool IsUnit = false; + std::uint32_t Value = 0; + + friend constexpr bool operator==(RegisterDependencyKey, RegisterDependencyKey) noexcept = default; +}; + +struct RegisterDependencyKeyHash final { + [[nodiscard]] std::size_t operator()(RegisterDependencyKey Key) const noexcept { + const std::uint64_t Value = (static_cast(Key.IsUnit) << 32U) | Key.Value; + return std::hash{}(Value); + } +}; + +struct RegisterDependencyState final { + struct Definition final { + std::size_t Unit = 0; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; + }; + + std::optional LastDefinition; + std::vector LastUses; +}; + +struct RegisterAccess final { + RegisterDependencyKey Key; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; +}; + +[[nodiscard]] bool keyLess(RegisterDependencyKey Left, RegisterDependencyKey Right) noexcept { + return Left.IsUnit == Right.IsUnit ? Left.Value < Right.Value : Left.IsUnit < Right.IsUnit; +} + +[[nodiscard]] std::uint32_t saturatingAdd(std::uint32_t Left, std::uint32_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +[[nodiscard]] Expected> +operandKeys(const MachineOperand &Operand, const MachineFunction &Function, + const TargetDescription &Target, ScheduleDAGMode Mode) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || Reg->isNoRegister()) + return std::vector{}; + if (Reg->isVirtual()) { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduling region contains a virtual register"); + } + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown virtual register"); + } + return std::vector{{false, Reg->opaqueValue()}}; + } + + std::optional Physical = Reg->physicalRegister(); + if (!Physical) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical register"); + if (Operand.subReg().value() != 0) { + const std::optional SubReg = + Target.registers().subRegister(*Physical, Operand.subReg()); + if (!SubReg) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical subregister"); + } + Physical = SubReg; + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown physical register"); + std::vector Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Result.push_back({true, Unit.Unit.value()}); + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] Expected> +maskKeys(const RegisterMask &Mask, const TargetDescription &Target) { + std::vector Result; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Result.push_back({true, Unit.Unit.value()}); + } + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] bool mayAlias(const MachineInstr &Earlier, const MachineInstr &Later, + const MachineAliasAnalysis::Result &Aliases) noexcept { + if (Earlier.memOperands().empty() || Later.memOperands().empty()) + return true; + for (const MachineMemOperand *Left : Earlier.memOperands()) { + if (Left == nullptr) + return true; + for (const MachineMemOperand *Right : Later.memOperands()) { + if (Right == nullptr || Aliases.alias(*Left, *Right) != MachineAliasResult::NoAlias) + return true; + } + } + return false; +} + +[[nodiscard]] bool hasOrderedMemory(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isAtomic() || Memory->isVolatile(); + }); +} + +[[nodiscard]] bool hasLoad(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isLoad(); + }); +} + +[[nodiscard]] bool hasStore(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isStore(); + }); +} + +} // namespace + +Expected ScheduleDAG::build(MachineBasicBlock &Block, + std::span Instructions, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode) { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule DAG block belongs to another function"); + ScheduleDAG Result(Block); + auto Units = Result.buildUnits(Instructions, Target, Model); + if (!Units) + return std::unexpected(Units.error()); + auto Registers = Result.buildRegisterDependencies(Function, Target, Model, Mode); + if (!Registers) + return std::unexpected(Registers.error()); + auto Memory = Result.buildMemoryDependencies(Target); + if (!Memory) + return std::unexpected(Memory.error()); + auto Order = Result.buildOrderDependencies(Target); + if (!Order) + return std::unexpected(Order.error()); + auto Mutated = Model.hooks().mutateScheduleDAG(Result, Function, Target, Mode); + if (!Mutated) + return std::unexpected(Mutated.error()); + Result.computeCriticalPaths(); + return Result; +} + +Expected ScheduleDAG::buildUnits(std::span Instructions, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + MachineInstr *ExpectedInstruction = Instructions.empty() ? nullptr : Instructions.front(); + for (std::size_t Index = 0; Index < Instructions.size();) { + MachineInstr *Instruction = Instructions[Index]; + if (Instruction == nullptr || Instruction->parent() != Block) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region contains a detached or foreign instruction"); + if (Instruction != ExpectedInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region is not a contiguous instruction sequence"); + } + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region begins in the middle of an instruction bundle"); + } + + ScheduleUnit Unit; + Unit.ID = Units.size(); + Unit.StableID = Instruction->stableID(); + std::unordered_map ResourceCycles; + for (;;) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const SchedulingClassModelDesc *Class = Model.schedulingClass(Descriptor->SchedulingClass); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction references an unknown scheduling class"); + } + Unit.MicroOps = saturatingAdd(Unit.MicroOps, Class->MicroOps); + Unit.Latency = std::max(Unit.Latency, Class->Latency); + for (const SchedulingResourceUse &Use : Class->Resources) { + ResourceCycles[Use.Resource] = saturatingAdd(ResourceCycles[Use.Resource], Use.Cycles); + } + Unit.Instructions.push_back(Instruction); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + ++Index; + if (!HasSuccessor) + break; + if (Index >= Instructions.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region ends in the middle of an instruction bundle"); + } + MachineInstr *Next = Instructions[Index]; + if (Next == nullptr || Next->parent() != Block || + !Next->flags().has(MachineInstrFlag::BundledWithPred) || Instruction->next() != Next) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction bundle links are not reciprocal and contiguous"); + } + Instruction = Next; + } + Unit.Resources.reserve(ResourceCycles.size()); + for (const auto &[Resource, Cycles] : ResourceCycles) + Unit.Resources.push_back({Resource, Cycles}); + std::ranges::sort(Unit.Resources, {}, &SchedulingResourceUse::Resource); + ExpectedInstruction = Unit.Instructions.back()->next(); + Units.push_back(std::move(Unit)); + } + return {}; +} + +Expected ScheduleDAG::buildRegisterDependencies(const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, + ScheduleDAGMode Mode) { + std::unordered_map + States; + for (ScheduleUnit &Unit : Units) { + std::vector Uses; + std::vector Definitions; + for (MachineInstr *Instruction : Unit.Instructions) { + for (std::size_t OperandIndex = 0; OperandIndex < Instruction->operandCount(); + ++OperandIndex) { + const MachineOperand *Operand = Instruction->operandAt(OperandIndex); + if (Operand == nullptr) + continue; + if (const auto *Mask = Operand->getIf()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null register mask"); + auto Keys = maskKeys(*Mask->Mask, Target); + if (!Keys) + return std::unexpected(Keys.error()); + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + continue; + } + if (const auto *Mask = Operand->getIf()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null live-out register mask"); + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask->Mask->preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &RegisterUnit : Register.Units) + Uses.push_back({{true, RegisterUnit.Unit.value()}, Instruction, OperandIndex}); + } + continue; + } + if (!Operand->isRegister() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + auto Keys = operandKeys(*Operand, Function, Target, Mode); + if (!Keys) + return std::unexpected(Keys.error()); + if (Operand->isUse()) { + for (const RegisterDependencyKey Key : *Keys) + Uses.push_back({Key, Instruction, OperandIndex}); + } + if (Operand->isDef()) { + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + } + } + } + std::ranges::stable_sort(Uses, [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + std::ranges::stable_sort(Definitions, + [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + + for (std::size_t Begin = 0; Begin < Uses.size();) { + std::size_t End = Begin + 1; + while (End < Uses.size() && Uses[End].Key == Uses[Begin].Key) + ++End; + RegisterDependencyState &State = States[Uses[Begin].Key]; + if (State.LastDefinition) { + std::uint32_t Latency = 0; + for (std::size_t Index = Begin; Index < End; ++Index) { + const InstructionDescriptor *Producer = + Target.instructionDescriptor(State.LastDefinition->Instruction->opcode()); + const InstructionDescriptor *Consumer = + Target.instructionDescriptor(Uses[Index].Instruction->opcode()); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler latency lookup encountered an unknown instruction"); + } + auto OperandLatency = + Model.operandLatency(Producer->SchedulingClass, State.LastDefinition->OperandIndex, + Consumer->SchedulingClass, Uses[Index].OperandIndex); + if (!OperandLatency) + return std::unexpected(OperandLatency.error()); + Latency = std::max(Latency, *OperandLatency); + } + addDependency(State.LastDefinition->Unit, Unit.ID, + Uses[Begin].Key.IsUnit ? ScheduleDependencyKind::RegisterUnit + : ScheduleDependencyKind::Data, + Latency); + } + if (State.LastUses.empty() || State.LastUses.back() != Unit.ID) + State.LastUses.push_back(Unit.ID); + Begin = End; + } + + for (std::size_t Begin = 0; Begin < Definitions.size();) { + std::size_t End = Begin + 1; + while (End < Definitions.size() && Definitions[End].Key == Definitions[Begin].Key) { + ++End; + } + RegisterDependencyState &State = States[Definitions[Begin].Key]; + if (State.LastDefinition) { + addDependency(State.LastDefinition->Unit, Unit.ID, ScheduleDependencyKind::Output, 0); + } + for (const std::size_t Use : State.LastUses) + addDependency(Use, Unit.ID, ScheduleDependencyKind::Anti, 0); + State.LastUses.clear(); + const RegisterAccess &Last = Definitions[End - 1]; + State.LastDefinition = + RegisterDependencyState::Definition{Unit.ID, Last.Instruction, Last.OperandIndex}; + Begin = End; + } + } + return {}; +} + +Expected ScheduleDAG::addArtificialDependency(std::size_t Source, std::size_t Destination, + std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source >= Destination) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "target schedule mutation requires a forward dependency between known units"); + } + addDependency(Source, Destination, ScheduleDependencyKind::Artificial, Latency); + return {}; +} + +Expected ScheduleDAG::buildMemoryDependencies(const TargetDescription &Target) { + MachineAliasAnalysis::Result Aliases; + for (std::size_t LaterIndex = 0; LaterIndex < Units.size(); ++LaterIndex) { + for (std::size_t EarlierIndex = 0; EarlierIndex < LaterIndex; ++EarlierIndex) { + bool NeedsEdge = false; + for (MachineInstr *Earlier : Units[EarlierIndex].Instructions) { + const InstructionDescriptor *EarlierDescriptor = + Target.instructionDescriptor(Earlier->opcode()); + if (EarlierDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool EarlierMemory = EarlierDescriptor->Flags.has(InstructionFlag::MayLoad) || + EarlierDescriptor->Flags.has(InstructionFlag::MayStore) || + !Earlier->memOperands().empty(); + if (!EarlierMemory) + continue; + for (MachineInstr *Later : Units[LaterIndex].Instructions) { + const InstructionDescriptor *LaterDescriptor = + Target.instructionDescriptor(Later->opcode()); + if (LaterDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool LaterMemory = LaterDescriptor->Flags.has(InstructionFlag::MayLoad) || + LaterDescriptor->Flags.has(InstructionFlag::MayStore) || + !Later->memOperands().empty(); + if (!LaterMemory) + continue; + if (hasOrderedMemory(*Earlier) || hasOrderedMemory(*Later) || + ((hasStore(*Earlier) || hasStore(*Later)) && mayAlias(*Earlier, *Later, Aliases)) || + (Earlier->memOperands().empty() || Later->memOperands().empty()) || + (!hasLoad(*Earlier) && !hasStore(*Earlier)) || + (!hasLoad(*Later) && !hasStore(*Later))) { + NeedsEdge = true; + break; + } + } + if (NeedsEdge) + break; + } + if (NeedsEdge) + addDependency(EarlierIndex, LaterIndex, ScheduleDependencyKind::Memory, 0); + } + } + return {}; +} + +Expected ScheduleDAG::buildOrderDependencies(const TargetDescription &Target) { + for (std::size_t Barrier = 0; Barrier < Units.size(); ++Barrier) { + bool IsBarrier = false; + for (const MachineInstr *Instruction : Units[Barrier].Instructions) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + IsBarrier |= Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || + Flags.has(InstructionFlag::Convergent) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::Meta); + } + if (!IsBarrier) + continue; + for (std::size_t Earlier = 0; Earlier < Barrier; ++Earlier) + addDependency(Earlier, Barrier, ScheduleDependencyKind::Barrier, 0); + for (std::size_t Later = Barrier + 1; Later < Units.size(); ++Later) + addDependency(Barrier, Later, ScheduleDependencyKind::Barrier, 0); + } + return {}; +} + +void ScheduleDAG::addDependency(std::size_t Source, std::size_t Destination, + ScheduleDependencyKind Kind, std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source == Destination) + return; + const auto Existing = std::ranges::find_if( + Units[Source].Successors, [Destination, Kind](const ScheduleDependency &Dependency) { + return Dependency.Destination == Destination && Dependency.Kind == Kind; + }); + if (Existing != Units[Source].Successors.end()) { + Existing->Latency = std::max(Existing->Latency, Latency); + const auto Predecessor = std::ranges::find_if( + Units[Destination].Predecessors, [Source, Kind](const ScheduleDependency &Dependency) { + return Dependency.Source == Source && Dependency.Kind == Kind; + }); + if (Predecessor != Units[Destination].Predecessors.end()) + Predecessor->Latency = Existing->Latency; + return; + } + const ScheduleDependency Dependency{Source, Destination, Kind, Latency}; + Units[Source].Successors.push_back(Dependency); + Units[Destination].Predecessors.push_back(Dependency); +} + +void ScheduleDAG::computeCriticalPaths() { + for (ScheduleUnit &Unit : Units) { + Unit.Depth = 0; + for (const ScheduleDependency &Dependency : Unit.Predecessors) { + Unit.Depth = + std::max(Unit.Depth, saturatingAdd(Units[Dependency.Source].Depth, + static_cast(Dependency.Latency))); + } + } + for (auto Position = Units.rbegin(); Position != Units.rend(); ++Position) { + Position->Height = Position->Latency; + for (const ScheduleDependency &Dependency : Position->Successors) { + Position->Height = + std::max(Position->Height, saturatingAdd(Units[Dependency.Destination].Height, + static_cast(Dependency.Latency))); + } + } + for (ScheduleUnit &Unit : Units) { + std::ranges::sort(Unit.Predecessors, {}, &ScheduleDependency::Source); + std::ranges::sort(Unit.Successors, {}, &ScheduleDependency::Destination); + } +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp new file mode 100644 index 0000000..28c032d --- /dev/null +++ b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" + +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits::max() - Left < Right + ? std::numeric_limits::max() + : Left + Right; +} + +} // namespace + +Expected +ScheduleHazardRecognizer::create(const TargetSchedulingModel &Model, ScheduleDirection Direction) { + ScheduleHazardRecognizer Result(Model, Direction); + Result.TargetHazards = Model.hooks().createHazardRecognizer(Direction); + auto Reset = Result.reset(); + if (!Reset) + return std::unexpected(Reset.error()); + return Result; +} + +ScheduleHazardRecognizer::ScheduleHazardRecognizer(const ScheduleHazardRecognizer &Other) + : Model(Other.Model), Direction(Other.Direction), + TargetHazards(Other.TargetHazards ? Other.TargetHazards->clone() : nullptr), + Resources(Other.Resources), ResourceIndexes(Other.ResourceIndexes), + CurrentCycle(Other.CurrentCycle), DispatchAvailableAt(Other.DispatchAvailableAt), + IssuedMicroOps(Other.IssuedMicroOps), DispatchedUnits(Other.DispatchedUnits) {} + +ScheduleHazardRecognizer & +ScheduleHazardRecognizer::operator=(const ScheduleHazardRecognizer &Other) { + if (this == &Other) + return *this; + Model = Other.Model; + Direction = Other.Direction; + TargetHazards = Other.TargetHazards ? Other.TargetHazards->clone() : nullptr; + Resources = Other.Resources; + ResourceIndexes = Other.ResourceIndexes; + CurrentCycle = Other.CurrentCycle; + DispatchAvailableAt = Other.DispatchAvailableAt; + IssuedMicroOps = Other.IssuedMicroOps; + DispatchedUnits = Other.DispatchedUnits; + return *this; +} + +Expected ScheduleHazardRecognizer::reset() { + Resources.clear(); + ResourceIndexes.clear(); + Resources.reserve(Model->processorResources().size()); + for (const ProcessorResourceDesc &Resource : Model->processorResources()) { + ResourceIndexes.emplace(Resource.ID, Resources.size()); + Resources.push_back({Resource.ID, std::vector(Resource.Units, 0)}); + } + CurrentCycle = 0; + DispatchAvailableAt = 0; + IssuedMicroOps = 0; + DispatchedUnits = 0; + if (TargetHazards != nullptr) { + auto Reset = TargetHazards->reset(); + if (!Reset) + return std::unexpected(Reset.error()); + } + return {}; +} + +Expected ScheduleHazardRecognizer::canIssue(const ScheduleUnit &Unit, + std::uint64_t Cycle) const { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) { + if (Issued.error().code() == ErrorCode::SchedulingFailed) + return false; + return std::unexpected(Issued.error()); + } + return true; +} + +Expected ScheduleHazardRecognizer::earliestIssueCycle(const ScheduleUnit &Unit, + std::uint64_t Earliest) const { + std::uint64_t Candidate = std::max(Earliest, CurrentCycle); + for (;;) { + auto Available = canIssue(Unit, Candidate); + if (!Available) + return std::unexpected(Available.error()); + if (*Available) + return Candidate; + if (Candidate == std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler resource timeline is exhausted"); + } + ++Candidate; + } +} + +Expected ScheduleHazardRecognizer::issue(const ScheduleUnit &Unit, std::uint64_t Cycle) { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + *this = std::move(Trial); + return {}; +} + +Expected ScheduleHazardRecognizer::issueImplementation(const ScheduleUnit &Unit, + std::uint64_t Cycle) { + if (Cycle < CurrentCycle) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduler cannot issue into an earlier cycle"); + } + auto Moved = moveToCycle(Cycle); + if (!Moved) + return std::unexpected(Moved.error()); + if (Cycle < DispatchAvailableAt) { + return makeUnexpected(ErrorCode::SchedulingFailed, "scheduler dispatch pipeline is occupied"); + } + if (TargetHazards != nullptr) { + auto Hazard = TargetHazards->getHazardType(Unit); + if (!Hazard) + return std::unexpected(Hazard.error()); + if (*Hazard != ScheduleHazardType::NoHazard) { + return makeUnexpected(ErrorCode::SchedulingFailed, + *Hazard == ScheduleHazardType::NoopHazard + ? "target hazard requires an unavailable scheduling noop" + : "target dynamic scheduling hazard prevents issue"); + } + } + + const std::uint32_t Width = Model->issueWidth(); + if (Unit.MicroOps <= Width) { + if (Unit.MicroOps > Width - IssuedMicroOps || DispatchedUnits == Model->dispatchLimit()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler issue or dispatch width is exhausted"); + } + IssuedMicroOps += Unit.MicroOps; + ++DispatchedUnits; + } else { + if (IssuedMicroOps != 0 || DispatchedUnits != 0) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "multi-cycle dispatch requires an empty issue cycle"); + } + const std::uint64_t DispatchCycles = + (static_cast(Unit.MicroOps) + Width - 1U) / Width; + DispatchAvailableAt = saturatingAdd(Cycle, DispatchCycles); + IssuedMicroOps = Width; + DispatchedUnits = 1; + } + + for (const SchedulingResourceUse &Use : Unit.Resources) { + auto Reserved = reserveResource(Use.Resource, Use.Cycles, Cycle); + if (!Reserved) + return std::unexpected(Reserved.error()); + } + if (TargetHazards != nullptr) { + auto Emitted = TargetHazards->emitInstruction(Unit); + if (!Emitted) + return std::unexpected(Emitted.error()); + } + return {}; +} + +Expected ScheduleHazardRecognizer::moveToCycle(std::uint64_t Cycle) { + while (CurrentCycle < Cycle) { + if (TargetHazards != nullptr) { + Expected Advanced = Direction == ScheduleDirection::TopDown + ? TargetHazards->advanceCycle() + : TargetHazards->recedeCycle(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + ++CurrentCycle; + IssuedMicroOps = 0; + DispatchedUnits = 0; + } + return {}; +} + +Expected ScheduleHazardRecognizer::reserveResource(ProcessorResourceID Resource, + std::uint32_t Cycles, + std::uint64_t Cycle) { + const auto Position = ResourceIndexes.find(Resource); + const ProcessorResourceDesc *Description = Model->processorResource(Resource); + if (Position == ResourceIndexes.end() || Description == nullptr || Cycles == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule unit references an invalid processor resource"); + } + ResourceState &State = Resources[Position->second]; + std::optional Selected; + for (std::size_t Index = 0; Index < State.AvailableAt.size(); ++Index) { + if (State.AvailableAt[Index] <= Cycle && + (!Selected || State.AvailableAt[Index] < State.AvailableAt[*Selected])) { + Selected = Index; + } + } + if (!Selected) { + return makeUnexpected(ErrorCode::SchedulingFailed, "processor resource has no available unit"); + } + State.AvailableAt[*Selected] = saturatingAdd(Cycle, Cycles); + if (Description->Parent) { + auto Parent = reserveResource(*Description->Parent, Cycles, Cycle); + if (!Parent) + return std::unexpected(Parent.error()); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetSchedulingModel.cpp b/backend/lib/CodeGen/TargetSchedulingModel.cpp new file mode 100644 index 0000000..97abecd --- /dev/null +++ b/backend/lib/CodeGen/TargetSchedulingModel.cpp @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +template [[nodiscard]] bool isValidID(IDT ID) noexcept { + return ID.value() != 0; +} + +} // namespace + +Expected +TargetScheduleHazardRecognizer::getHazardType(const ScheduleUnit &) const { + return ScheduleHazardType::NoHazard; +} + +Expected TargetScheduleHazardRecognizer::emitInstruction(const ScheduleUnit &) { + return {}; +} + +Expected TargetScheduleHazardRecognizer::advanceCycle() { + return {}; +} + +Expected TargetScheduleHazardRecognizer::recedeCycle() { + return {}; +} + +Expected TargetScheduleHazardRecognizer::reset() { + return {}; +} + +bool TargetSchedulingHooks::isSchedulingBoundary(const MachineInstr &, + const TargetDescription &) const noexcept { + return false; +} + +Expected TargetSchedulingHooks::mutateScheduleDAG(ScheduleDAG &, const MachineFunction &, + const TargetDescription &, + ScheduleDAGMode) const { + return {}; +} + +std::int64_t TargetSchedulingHooks::candidatePreference(const ScheduleUnit &, ScheduleDirection, + const MachineBasicBlock &, + std::uint64_t) const noexcept { + return 0; +} + +std::unique_ptr +TargetSchedulingHooks::createHazardRecognizer(ScheduleDirection) const { + return nullptr; +} + +Expected +TargetSchedulingModel::create(TargetSchedulingModelDesc Description) { + if (Description.IssueWidth == 0 || Description.DispatchLimit == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model requires nonzero issue and dispatch widths"); + } + + TargetSchedulingModel Result(std::move(Description)); + std::unordered_set Names; + for (std::size_t Index = 0; Index < Result.Description.ProcessorResources.size(); ++Index) { + const ProcessorResourceDesc &Resource = Result.Description.ProcessorResources[Index]; + if (!isValidID(Resource.ID) || Resource.Name.empty() || Resource.Units == 0 || + !Result.ResourceToIndex.emplace(Resource.ID, Index).second || + !Names.insert(Resource.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid processor resource"); + } + } + for (const ProcessorResourceDesc &Resource : Result.Description.ProcessorResources) { + if (Resource.Parent && + (Resource.Parent == Resource.ID || !Result.ResourceToIndex.contains(*Resource.Parent))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource has an invalid parent resource"); + } + std::unordered_set Path; + const ProcessorResourceDesc *Current = &Resource; + while (Current->Parent) { + if (!Path.insert(Current->ID).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource hierarchy contains a cycle"); + } + Current = &Result.Description.ProcessorResources[Result.ResourceToIndex.at(*Current->Parent)]; + } + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.SchedulingClasses.size(); ++Index) { + SchedulingClassModelDesc &Class = Result.Description.SchedulingClasses[Index]; + if (!isValidID(Class.ID) || Class.Name.empty() || Class.MicroOps == 0 || Class.Latency == 0 || + !Result.ClassToIndex.emplace(Class.ID, Index).second || !Names.insert(Class.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid scheduling class"); + } + } + for (SchedulingClassModelDesc &Class : Result.Description.SchedulingClasses) { + std::unordered_set UsedResources; + for (const SchedulingResourceUse &Use : Class.Resources) { + if (!isValidID(Use.Resource) || Use.Cycles == 0 || + !Result.ResourceToIndex.contains(Use.Resource) || + !UsedResources.insert(Use.Resource).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has an invalid resource use"); + } + } + std::ranges::sort(Class.Resources, {}, + [](const SchedulingResourceUse &Use) { return Use.Resource.value(); }); + + std::unordered_set WriteOperands; + for (const SchedulingWriteLatency &Latency : Class.WriteLatencies) { + if (!WriteOperands.insert(Latency.OperandIndex).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate write latencies"); + } + } + std::ranges::sort(Class.WriteLatencies, {}, &SchedulingWriteLatency::OperandIndex); + + std::unordered_set ReadAdvanceKeys; + for (const SchedulingReadAdvance &Advance : Class.ReadAdvances) { + if (Advance.ProducerClass && !Result.ClassToIndex.contains(*Advance.ProducerClass)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "read advance references an unknown producer class"); + } + const std::uint64_t Producer = Advance.ProducerClass ? Advance.ProducerClass->value() : 0; + const std::uint64_t Key = (Producer << 32U) | Advance.OperandIndex; + if (!ReadAdvanceKeys.insert(Key).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate read advances"); + } + } + std::ranges::sort(Class.ReadAdvances, + [](const SchedulingReadAdvance &Left, const SchedulingReadAdvance &Right) { + if (Left.OperandIndex != Right.OperandIndex) + return Left.OperandIndex < Right.OperandIndex; + return Left.ProducerClass.value_or(SchedulingClassID{}).value() < + Right.ProducerClass.value_or(SchedulingClassID{}).value(); + }); + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.PressureSets.size(); ++Index) { + RegisterPressureSetDesc &Set = Result.Description.PressureSets[Index]; + if (!isValidID(Set.ID) || Set.Name.empty() || Set.Limit == 0 || Set.RegisterClasses.empty() || + !Result.PressureSetToIndex.emplace(Set.ID, Index).second || + !Names.insert(Set.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid register pressure set"); + } + std::ranges::sort(Set.RegisterClasses, [](RegisterClassID Left, RegisterClassID Right) { + return Left.value() < Right.value(); + }); + if (std::ranges::adjacent_find(Set.RegisterClasses) != Set.RegisterClasses.end() || + std::ranges::any_of(Set.RegisterClasses, + [](RegisterClassID Class) { return !isValidID(Class); })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register pressure set has duplicate or invalid register classes"); + } + for (const RegisterClassID Class : Set.RegisterClasses) + Result.ClassToPressureSets[Class].push_back(Set.ID); + } + for (auto &[Class, Sets] : Result.ClassToPressureSets) { + (void)Class; + std::ranges::sort(Sets, [](RegisterPressureSetID Left, RegisterPressureSetID Right) { + return Left.value() < Right.value(); + }); + } + return Result; +} + +const ProcessorResourceDesc * +TargetSchedulingModel::processorResource(ProcessorResourceID ID) const noexcept { + const auto Position = ResourceToIndex.find(ID); + return Position == ResourceToIndex.end() ? nullptr + : &Description.ProcessorResources[Position->second]; +} + +const SchedulingClassModelDesc * +TargetSchedulingModel::schedulingClass(SchedulingClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + return Position == ClassToIndex.end() ? nullptr + : &Description.SchedulingClasses[Position->second]; +} + +const RegisterPressureSetDesc * +TargetSchedulingModel::pressureSet(RegisterPressureSetID ID) const noexcept { + const auto Position = PressureSetToIndex.find(ID); + return Position == PressureSetToIndex.end() ? nullptr + : &Description.PressureSets[Position->second]; +} + +std::span +TargetSchedulingModel::pressureSetsForClass(RegisterClassID Class) const noexcept { + const auto Position = ClassToPressureSets.find(Class); + return Position == ClassToPressureSets.end() + ? std::span{} + : std::span(Position->second); +} + +Expected TargetSchedulingModel::operandLatency(SchedulingClassID ProducerClass, + std::size_t DefinitionOperand, + SchedulingClassID ConsumerClass, + std::size_t UseOperand) const { + if (DefinitionOperand > std::numeric_limits::max() || + UseOperand > std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduling operand index exceeds the target model range"); + } + const SchedulingClassModelDesc *Producer = schedulingClass(ProducerClass); + const SchedulingClassModelDesc *Consumer = schedulingClass(ConsumerClass); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand latency references an unknown scheduling class"); + } + + const std::uint32_t DefinitionIndex = static_cast(DefinitionOperand); + const std::uint32_t UseIndex = static_cast(UseOperand); + std::uint32_t Latency = Producer->Latency; + if (const auto Position = std::ranges::find(Producer->WriteLatencies, DefinitionIndex, + &SchedulingWriteLatency::OperandIndex); + Position != Producer->WriteLatencies.end()) { + Latency = Position->Cycles; + } + + const SchedulingReadAdvance *Wildcard = nullptr; + const SchedulingReadAdvance *Exact = nullptr; + for (const SchedulingReadAdvance &Advance : Consumer->ReadAdvances) { + if (Advance.OperandIndex != UseIndex) + continue; + if (!Advance.ProducerClass) + Wildcard = &Advance; + else if (*Advance.ProducerClass == ProducerClass) + Exact = &Advance; + } + const SchedulingReadAdvance *Advance = Exact != nullptr ? Exact : Wildcard; + return Advance == nullptr || Advance->Cycles >= Latency ? (Advance == nullptr ? Latency : 0U) + : Latency - Advance->Cycles; +} + +const TargetSchedulingHooks &TargetSchedulingModel::hooks() const noexcept { + static const TargetSchedulingHooks DefaultHooks; + return Description.Hooks == nullptr ? DefaultHooks : *Description.Hooks; +} + +} // namespace rcc diff --git a/backend/lib/Support/Error.cpp b/backend/lib/Support/Error.cpp index f18cc45..88e16c9 100644 --- a/backend/lib/Support/Error.cpp +++ b/backend/lib/Support/Error.cpp @@ -18,6 +18,16 @@ std::string_view getErrorCodeName(ErrorCode Code) noexcept { return "register allocation failed"; case ErrorCode::FrameLayoutFailed: return "frame layout failed"; + case ErrorCode::SchedulingFailed: + return "scheduling failed"; + case ErrorCode::BranchRelaxationFailed: + return "branch relaxation failed"; + case ErrorCode::PseudoExpansionFailed: + return "pseudo expansion failed"; + case ErrorCode::FinalEncodabilityFailed: + return "final encodability failed"; + case ErrorCode::CFGOptimizationFailed: + return "CFG optimization failed"; case ErrorCode::IoError: return "I/O error"; case ErrorCode::InternalError: diff --git a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp index 9d6b8e1..cd5ba97 100644 --- a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp +++ b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp @@ -1683,7 +1683,7 @@ instructionFlagEnumerator(std::string_view Value) noexcept { [[nodiscard]] std::optional pseudoOwnerEnumerator(std::string_view Value) noexcept { - static constexpr std::array, 11> Owners = {{ + static constexpr std::array, 12> Owners = {{ {"none", "None"}, {"legalizer", "Legalizer"}, {"register_bank_select", "RegisterBankSelect"}, @@ -1695,6 +1695,7 @@ pseudoOwnerEnumerator(std::string_view Value) noexcept { {"prologue_epilogue_insertion", "PrologueEpilogueInsertion"}, {"post_ra_expansion", "PostRAExpansion"}, {"finalization", "Finalization"}, + {"target_late_expansion", "TargetLateExpansion"}, }}; if (Value == "asm_printer") return "AsmPrinter"; @@ -2328,8 +2329,8 @@ private: } if (InputOperand.ImmediatePredicateSymbol) { const auto Position = ImmediatePredicateBySymbol.find(*InputOperand.ImmediatePredicateSymbol); - if (Position == ImmediatePredicateBySymbol.end() || Result.Kinds.size() != 1 || - Result.Kinds.front() != "Immediate") { + if (Position == ImmediatePredicateBySymbol.end() || + std::ranges::find(Result.Kinds, "Immediate") == Result.Kinds.end()) { fail(InputOperand.Range, "instruction immediate predicates must name a declaration and constrain an immediate"); return false; diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index 442e75e..a29315a 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -47,10 +47,16 @@ set_source_files_properties( add_library( RCCBackendRISCV + RISCVBranchInfo.cpp RISCVCodeGen.cpp RISCVCallLowering.cpp RISCVFrameLowering.cpp + RISCVMachineCombiner.cpp + RISCVMachineMemoryFolding.cpp + RISCVOptimizationPipeline.cpp + RISCVPseudoExpansion.cpp RISCVRegisterAllocation.cpp + RISCVScheduling.cpp RISCVTarget.cpp "${RCCBackendRISCVGeneratedRegisterHeader}" "${RCCBackendRISCVGeneratedInstructionHeader}" @@ -93,19 +99,31 @@ install( ) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTarget.h" ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" ) diff --git a/backend/lib/Target/RISCV/RISCV.rccdesc b/backend/lib/Target/RISCV/RISCV.rccdesc index e248bc7..dce984c 100644 --- a/backend/lib/Target/RISCV/RISCV.rccdesc +++ b/backend/lib/Target/RISCV/RISCV.rccdesc @@ -245,6 +245,10 @@ mir_operand_flags: direct_mask: 31 direct: - { name: riscv-call, value: 1 } + - { name: riscv-pcrel-hi, value: 2 } + - { name: riscv-pcrel-lo, value: 4 } + - { name: riscv-copy-f32, value: 8 } + - { name: riscv-copy-f64, value: 16 } bitmask: [] calling_conventions: @@ -386,7 +390,7 @@ instructions: assembly_mnemonic: auipc operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - - { kinds: [immediate], immediate_predicate: SIMM20, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM20, allowed_target_flags: 3, name: IMM } num_explicit_defs: 1 flags: [produces_code] scheduling_class: ALU @@ -871,7 +875,7 @@ instructions: operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } - - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } num_explicit_defs: 1 flags: [terminator, branch, indirect_branch, produces_code] scheduling_class: BRANCH @@ -3195,3 +3199,52 @@ instructions: flags: [pseudo, produces_code, allowed_pre_legalizer, terminator, return, barrier] owner: post_ra_expansion deadline: post_ra_finalized + - + id: 1225 + symbol: JALR_CALL + name: JALR_CALL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [call, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1226 + symbol: JALR_TAIL + name: JALR_TAIL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, branch, indirect_branch, call, barrier, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1227 + symbol: JALR_RET + name: JALR_RET + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, return, barrier, produces_code] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM diff --git a/backend/lib/Target/RISCV/RISCVBranchInfo.cpp b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp new file mode 100644 index 0000000..18812d8 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] bool isConditionalBranch(Opcode Value) noexcept { + return Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BNE) || + Value == targetOpcode(InstructionOpcode::BLT) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BLTU) || + Value == targetOpcode(InstructionOpcode::BGEU) || + Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ); +} + +[[nodiscard]] std::optional reversedOpcode(Opcode Value) noexcept { + constexpr std::array, 4> Pairs = {{ + {InstructionOpcode::BEQ, InstructionOpcode::BNE}, + {InstructionOpcode::BLT, InstructionOpcode::BGE}, + {InstructionOpcode::BLTU, InstructionOpcode::BGEU}, + {InstructionOpcode::C_BEQZ, InstructionOpcode::C_BNEZ}, + }}; + for (const auto &[Left, Right] : Pairs) { + if (Value == targetOpcode(Left)) + return targetOpcode(Right); + if (Value == targetOpcode(Right)) + return targetOpcode(Left); + } + return std::nullopt; +} + +[[nodiscard]] bool isZeroRegister(const MachineOperand *Operand) noexcept { + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] bool sameRegister(const MachineOperand *Left, const MachineOperand *Right) noexcept { + const Register *LeftReg = Left == nullptr ? nullptr : Left->getIf(); + const Register *RightReg = Right == nullptr ? nullptr : Right->getIf(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Left->subReg() == Right->subReg(); +} + +[[nodiscard]] bool isUnconditionalBranch(const MachineInstr &Instruction) noexcept { + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) + return true; + return Instruction.opcode() == targetOpcode(InstructionOpcode::JAL) && + isZeroRegister(Instruction.operandAt(0)); +} + +[[nodiscard]] std::size_t branchTargetIndex(Opcode Value) noexcept { + if (Value == targetOpcode(InstructionOpcode::C_J)) + return 0; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return 1; + } + if (Value == targetOpcode(InstructionOpcode::JAL)) + return 1; + return 2; +} + +[[nodiscard]] Expected directTarget(MachineInstr &Instruction) { + if (!isConditionalBranch(Instruction.opcode()) && + Instruction.opcode() != targetOpcode(InstructionOpcode::JAL) && + Instruction.opcode() != targetOpcode(InstructionOpcode::C_J)) { + return static_cast(nullptr); + } + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + const BasicBlockOperand *Block = + Operand == nullptr ? nullptr : Operand->getIf(); + if (Block == nullptr || Block->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no basic-block target"); + } + return Block->Block; +} + +[[nodiscard]] Expected setDirectTarget(MachineInstr &Instruction, + MachineBasicBlock &TargetBlock) { + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + if (Operand == nullptr || Operand->kind() != MachineOperand::Kind::BasicBlock) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no mutable basic-block target"); + } + return Operand->setBasicBlock(TargetBlock); +} + +[[nodiscard]] Expected removeInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch instruction is not linked to a block"); + } + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] Expected +buildJAL(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, + MachineInstr *Before, PhysicalRegister Destination, MachineBasicBlock &TargetBlock) { + MachineIRBuilder Builder(Function, Target); + Expected Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(Destination)); + if (!Def) + return std::unexpected(Def.error()); + auto TargetOperand = Pending->addBasicBlock(TargetBlock); + if (!TargetOperand) + return std::unexpected(TargetOperand.error()); + return Pending->build(); +} + +[[nodiscard]] Expected copySourceIdentity(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyTrailingAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.debugInstrNumber() != 0) { + auto DebugNumber = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!DebugNumber) + return std::unexpected(DebugNumber.error()); + } + if (Source.postInstrSymbol()) { + auto Symbol = Destination.setPostInstrSymbol(std::string(*Source.postInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + if (Source.heapAllocMarker()) { + auto Marker = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Marker) + return std::unexpected(Marker.error()); + } + if (Source.pcSections()) { + auto Sections = Destination.setPCSections(*Source.pcSections()); + if (!Sections) + return std::unexpected(Sections.error()); + } + if (Source.mmra()) { + auto Metadata = Destination.setMMRA(*Source.mmra()); + if (!Metadata) + return std::unexpected(Metadata.error()); + } + if (Source.cfiType()) { + auto Type = Destination.setCFIType(*Source.cfiType()); + if (!Type) + return std::unexpected(Type.error()); + } + if (Source.deactivationSymbol()) { + auto Symbol = Destination.setDeactivationSymbol(std::string(*Source.deactivationSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return {}; +} + +[[nodiscard]] Expected copySingleInstructionAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + Destination.setFlags(Source.flags()); + auto Identity = copySourceIdentity(Source, Destination); + if (!Identity) + return std::unexpected(Identity.error()); + if (Source.preInstrSymbol()) { + auto Symbol = Destination.setPreInstrSymbol(std::string(*Source.preInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return copyTrailingAnnotations(Source, Destination); +} + +[[nodiscard]] bool isZeroPhysicalRegister(PhysicalRegister Reg) noexcept { + return Reg.number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] Expected +selectLongBranchScratch(const MachineInstr &Instruction, const MachineBasicBlock &TargetBlock, + const MachineFunction &Function, const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const std::optional DestinationPhysical = + Destination == nullptr ? std::nullopt : Destination->physicalRegister(); + if (!DestinationPhysical) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL has no physical destination"); + } + if (!isZeroPhysicalRegister(*DestinationPhysical)) + return *DestinationPhysical; + + const auto GPRClass = + RegisterClassID::fromValue(static_cast(RegisterClass::GPR_NX0)); + const RegisterClassDesc *Candidates = Target.registers().registerClass(GPRClass); + if (Candidates == nullptr) { + return makeUnexpected(ErrorCode::InternalError, "RISC-V target has no nonzero GPR class"); + } + for (const PhysicalRegister Candidate : Candidates->AllocationOrder) { + if (Target.registers().isReservedRegister(Candidate) || + Function.registerInfo().isReserved(Candidate)) { + continue; + } + const bool LiveAtTarget = + std::ranges::any_of(TargetBlock.liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Candidate, LiveIn.PhysReg); + }); + if (!LiveAtTarget) + return Candidate; + } + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V long branch has no dead physical scratch register"); +} + +[[nodiscard]] Expected expandCompressedBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "compressed branch is not linked"); + auto TargetBlock = directTarget(Instruction); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) { + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Replacement = buildJAL(Function, Target, *Block, &Instruction, *Zero, **TargetBlock); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + + const MachineOperand *SourceOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "compressed conditional branch has no source register"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + const InstructionOpcode Expanded = Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) + ? InstructionOpcode::BEQ + : InstructionOpcode::BNE; + auto Pending = Builder.buildInstr(targetOpcode(Expanded)); + if (!Pending) + return std::unexpected(Pending.error()); + auto First = Pending->addUse(*Source); + if (!First) + return std::unexpected(First.error()); + auto Second = Pending->addUse(Register::fromPhysical(*Zero)); + if (!Second) + return std::unexpected(Second.error()); + auto Destination = Pending->addBasicBlock(**TargetBlock); + if (!Destination) + return std::unexpected(Destination.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected relaxConditionalBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "conditional branch is not linked"); + auto TrueTarget = directTarget(Instruction); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + MachineBasicBlock *FalseTarget = Block->next(); + if (FalseTarget == nullptr || FalseTarget == *TrueTarget || + std::ranges::none_of(Block->successors(), [FalseTarget](const MachineSuccessor &Successor) { + return Successor.Block == FalseTarget; + })) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V conditional relaxation requires its false edge as fallthrough"); + } + + auto Reversed = reversedOpcode(Instruction.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V conditional branch cannot be inverted"); + auto SetOpcode = Instruction.setOpcode(*Reversed); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetTarget = setDirectTarget(Instruction, *FalseTarget); + if (!SetTarget) + return std::unexpected(SetTarget.error()); + + MachineInstr *Following = Instruction.next(); + if (Following != nullptr && isUnconditionalBranch(*Following)) { + auto Retargeted = setDirectTarget(*Following, **TrueTarget); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + return true; + } + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Destination = Pending->addBasicBlock(**TrueTarget); + if (!Destination) + return std::unexpected(Destination.error()); + auto LongJump = Pending->build(); + if (!LongJump) + return std::unexpected(LongJump.error()); + auto Identity = copySourceIdentity(Instruction, **LongJump); + if (!Identity) + return std::unexpected(Identity.error()); + MachineInstrFlags OriginalFlags = Instruction.flags(); + const bool HadBundledSuccessor = OriginalFlags.has(MachineInstrFlag::BundledWithSucc); + OriginalFlags.set(MachineInstrFlag::BundledWithSucc); + Instruction.setFlags(OriginalFlags); + MachineInstrFlags LongFlags; + LongFlags.set(MachineInstrFlag::BundledWithPred); + if (HadBundledSuccessor) + LongFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + LongFlags.set(MachineInstrFlag::NoMerge); + (*LongJump)->setFlags(LongFlags); + return true; +} + +[[nodiscard]] Expected relaxJAL(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V JAL is not linked"); + if (Offset < std::numeric_limits::min() || + Offset > std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V PC-relative long branch exceeds the signed 32-bit range"); + } + auto DestinationBlock = directTarget(Instruction); + if (!DestinationBlock) + return std::unexpected(DestinationBlock.error()); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || !Destination->isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL requires a physical destination register"); + } + auto Scratch = selectLongBranchScratch(Instruction, **DestinationBlock, Function, Target); + if (!Scratch) + return std::unexpected(Scratch.error()); + const std::string Anchor = Instruction.preInstrSymbol() + ? std::string(*Instruction.preInstrSymbol()) + : ".Lrcc_long_" + std::to_string(Function.stableID().value()) + + "_" + std::to_string(Instruction.stableID().value()); + const MachineInstrFlags OriginalFlags = Instruction.flags(); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Register::fromPhysical(*Scratch)); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = HighPending->addBasicBlock(**DestinationBlock, PCRelativeHighFlag); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto HighSymbol = (*High)->setPreInstrSymbol(Anchor); + if (!HighSymbol) + return std::unexpected(HighSymbol.error()); + auto HighIdentity = copySourceIdentity(Instruction, **High); + if (!HighIdentity) + return std::unexpected(HighIdentity.error()); + + auto LowPending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR)); + if (!LowPending) + return std::unexpected(LowPending.error()); + auto LowDef = LowPending->addDef(*Destination); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Register::fromPhysical(*Scratch)); + if (!LowBase) + return std::unexpected(LowBase.error()); + auto LowSymbol = LowPending->addMCSymbol(Anchor, 0, PCRelativeLowFlag); + if (!LowSymbol) + return std::unexpected(LowSymbol.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto LowIdentity = copySourceIdentity(Instruction, **Low); + if (!LowIdentity) + return std::unexpected(LowIdentity.error()); + auto LowAnnotations = copyTrailingAnnotations(Instruction, **Low); + if (!LowAnnotations) + return std::unexpected(LowAnnotations.error()); + + MachineInstrFlags HighFlags; + if (OriginalFlags.has(MachineInstrFlag::BundledWithPred)) + HighFlags.set(MachineInstrFlag::BundledWithPred); + HighFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + HighFlags.set(MachineInstrFlag::NoMerge); + (*High)->setFlags(HighFlags); + MachineInstrFlags LowFlags = OriginalFlags; + LowFlags.clear(MachineInstrFlag::BundledWithPred); + LowFlags.set(MachineInstrFlag::BundledWithPred); + (*Low)->setFlags(LowFlags); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVBranchInfo final : public TargetBranchInfo { +public: + [[nodiscard]] Expected + analyzeBranch(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch analysis references another function"); + std::vector Terminators; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V branch analysis encountered an unknown opcode"); + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + Terminators.push_back(Instruction); + } + + MachineBranchAnalysis Result; + if (Terminators.empty()) + return Result; + if (Terminators.size() > 2) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + MachineInstr *First = Terminators.front(); + if (isConditionalBranch(First->opcode())) { + auto TrueTarget = directTarget(*First); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + Result.Conditional = First; + Result.TrueTarget = *TrueTarget; + if (Terminators.size() == 2) { + MachineInstr *Second = Terminators.back(); + if (!isUnconditionalBranch(*Second)) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + auto FalseTarget = directTarget(*Second); + if (!FalseTarget) + return std::unexpected(FalseTarget.error()); + Result.Unconditional = Second; + Result.FalseTarget = *FalseTarget; + } else { + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block == nullptr || Successor.Block == Result.TrueTarget) + continue; + if (Result.FalseTarget != nullptr) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + Result.FalseTarget = Successor.Block; + } + if (Result.FalseTarget == nullptr) + Result.FalseTarget = Block.next(); + } + return Result; + } + if (Terminators.size() == 1 && isUnconditionalBranch(*First)) { + auto TargetBlock = directTarget(*First); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + Result.Unconditional = First; + Result.TrueTarget = *TargetBlock; + return Result; + } + Result.HasUnanalyzableTerminator = true; + return Result; + } + + [[nodiscard]] Expected reverseBranchCondition(MachineInstr &Branch, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function || &NewTarget.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch reversal references another function"); + const std::optional Reversed = reversedOpcode(Branch.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch opcode has no inverse condition"); + auto OpcodeResult = Branch.setOpcode(*Reversed); + if (!OpcodeResult) + return std::unexpected(OpcodeResult.error()); + return setDirectTarget(Branch, NewTarget); + } + + [[nodiscard]] Expected removeBranch(MachineInstr &Branch, MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch removal references another function"); + return removeInstruction(Branch); + } + + [[nodiscard]] Expected rewriteBranchTarget(MachineBasicBlock &Block, + MachineBasicBlock &OldTarget, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Block.function() != &Function || &OldTarget.function() != &Function || + &NewTarget.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch rewrite references another function"); + } + bool Changed = false; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isConditionalBranch(Instruction->opcode()) && !isUnconditionalBranch(*Instruction)) { + continue; + } + auto CurrentTarget = directTarget(*Instruction); + if (!CurrentTarget) + return std::unexpected(CurrentTarget.error()); + if (*CurrentTarget != &OldTarget) + continue; + auto Rewritten = setDirectTarget(*Instruction, NewTarget); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + Changed = true; + } + return Changed; + } + + [[nodiscard]] Expected + insertUnconditionalBranch(MachineBasicBlock &Block, MachineBasicBlock &TargetBlock, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function || &TargetBlock.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch insertion references another function"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + return buildJAL(Function, Target, Block, nullptr, *Zero, TargetBlock); + } + + [[nodiscard]] Expected + instructionSize(const MachineInstr &Instruction, const MachineFunction &, + const TargetDescription &Target) const override { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction-size query encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::ProducesCode)) + return 0; + const auto Category = + std::ranges::find_if(encodingCategories(), [Descriptor](const EncodingCategoryInfo &Info) { + return Info.ID == Descriptor->EncodingCategory; + }); + if (Category == encodingCategories().end() || Category->SizeInBytes == 0) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V producing instruction has no exact encoding size"); + return Category->SizeInBytes; + } + + [[nodiscard]] Expected + branchTarget(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch target query references another function"); + return directTarget(Instruction); + } + + [[nodiscard]] Expected> + evaluateBranchCondition(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch evaluation references another function"); + } + const Opcode Value = Instruction.opcode(); + if (!isConditionalBranch(Value)) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + if (!isZeroRegister(Instruction.operandAt(0))) + return std::optional{}; + return std::optional{Value == targetOpcode(InstructionOpcode::C_BEQZ)}; + } + if (!sameRegister(Instruction.operandAt(0), Instruction.operandAt(1))) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BGEU)) { + return std::optional{true}; + } + return std::optional{false}; + } + + [[nodiscard]] bool isBranchOffsetInRange(const MachineInstr &Instruction, + std::int64_t Offset) const noexcept override { + if ((Offset & 1) != 0) + return false; + const Opcode Value = Instruction.opcode(); + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return Offset >= -256 && Offset <= 254; + } + if (Value == targetOpcode(InstructionOpcode::C_J)) + return Offset >= -2048 && Offset <= 2046; + if (isConditionalBranch(Value)) + return Offset >= -4096 && Offset <= 4094; + if (Value == targetOpcode(InstructionOpcode::JAL)) + return Offset >= -1048576 && Offset <= 1048574; + return true; + } + + [[nodiscard]] Expected relaxBranch(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch relaxation references another function"); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BNEZ)) { + return expandCompressedBranch(Instruction, Function, Target); + } + if (isConditionalBranch(Instruction.opcode())) + return relaxConditionalBranch(Instruction, Function, Target); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::JAL)) + return relaxJAL(Instruction, Offset, Function, Target); + return false; + } +}; + +} // namespace + +const TargetBranchInfo &branchInfo() noexcept { + static const RISCVBranchInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp index 0e6fec9..e47a9c8 100644 --- a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -1124,8 +1124,43 @@ calleeSaveOperations(MachineBasicBlock &Block, const MachineFunction &Function, return Result; } +[[nodiscard]] MachineBasicBlock *blockWithID(MachineFunction &Function, + MachineBlockStableID ID) noexcept { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (Block->stableID() == ID) + return Block; + } + return nullptr; +} + +[[nodiscard]] MachineBasicBlock *frameSaveBlock(MachineFunction &Function) noexcept { + const auto &SavePoints = Function.frameInfo().state().SavePoints; + return SavePoints.empty() ? Function.firstBlock() + : blockWithID(Function, SavePoints.front().Block); +} + +[[nodiscard]] bool isFrameRestoreBlock(const MachineFunction &Function, + const MachineBasicBlock &Block) noexcept { + const MachineFrameState &State = Function.frameInfo().state(); + if (State.RestorePoints.empty()) + return State.SavePoints.empty(); + return std::ranges::any_of(State.RestorePoints, + [&Block](const MachineFrameSaveRestorePoint &Point) { + return Point.Block == Block.stableID(); + }); +} + class RISCVFrameLowering final : public TargetFrameLowering { public: + [[nodiscard]] bool + isCalleeSavedRegister(PhysicalRegister Reg, const MachineFunction &, + const TargetDescription &Target) const noexcept override { + return std::ranges::any_of(saveOrder(), [&](PhysicalRegisterID ID) { + const auto Candidate = physical(ID); + return Candidate && Target.registers().regsOverlap(Reg, *Candidate); + }); + } + [[nodiscard]] Expected prepareFrame(MachineFunction &Function, const TargetDescription &Target) const override { MachineFrameInfo &Frame = Function.frameInfo(); @@ -1133,6 +1168,10 @@ public: return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V frame layout is already frozen"); MachineFrameState State = Frame.state(); + const std::vector RequestedSavePoints = State.SavePoints; + const std::vector RequestedRestorePoints = State.RestorePoints; + const bool HasRequestedPlacement = + !RequestedSavePoints.empty() || !RequestedRestorePoints.empty(); for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; @@ -1249,25 +1288,32 @@ public: State.IsCalleeSavedInfoValid = true; State.SavePoints.clear(); State.RestorePoints.clear(); - if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { - State.SavePoints.push_back( - MachineFrameSaveRestorePoint{Function.firstBlock()->stableID(), *SavedRegisters}); - } - for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; - Block = Block->next()) { - bool HasEpilogue = false; - for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; - Instruction = Instruction->next()) { - HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + std::vector RestoredRegisters; + std::ranges::copy_if( + *SavedRegisters, std::back_inserter(RestoredRegisters), + [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); + if (HasRequestedPlacement) { + if (RequestedSavePoints.size() != 1 || RequestedRestorePoints.empty()) { + return makeUnexpected( + ErrorCode::VerificationError, + "RISC-V shrink-wrapped frame requires one save point and at least one restore point"); } - if (HasEpilogue && !SavedRegisters->empty()) { - std::vector Restored; - std::ranges::copy_if( - *SavedRegisters, std::back_inserter(Restored), - [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); - if (!Restored.empty()) { - State.RestorePoints.push_back( - MachineFrameSaveRestorePoint{Block->stableID(), std::move(Restored)}); + State.SavePoints.push_back({RequestedSavePoints.front().Block, *SavedRegisters}); + for (const MachineFrameSaveRestorePoint &Point : RequestedRestorePoints) + State.RestorePoints.push_back({Point.Block, RestoredRegisters}); + } else { + if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { + State.SavePoints.push_back({Function.firstBlock()->stableID(), *SavedRegisters}); + } + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + bool HasEpilogue = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + } + if (HasEpilogue && !RestoredRegisters.empty()) { + State.RestorePoints.push_back({Block->stableID(), RestoredRegisters}); } } } @@ -1290,10 +1336,10 @@ public: const std::span Saved = Function.records().calleeSavedRegisters(); if (Saved.empty()) return {}; - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) { + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) { return makeUnexpected(ErrorCode::VerificationError, - "RISC-V callee-save frame has no entry block"); + "RISC-V callee-save frame has no valid save block"); } auto FP = physical(PhysicalRegisterID::X8); auto BP = physical(PhysicalRegisterID::X9); @@ -1303,9 +1349,9 @@ public: const bool Realign = requiresStackRealignment(Function); MachineIRBuilder SaveBuilder(Function, Target); - auto SavePoint = Entry->firstInstruction() != nullptr - ? SaveBuilder.setInsertPointBefore(*Entry->firstInstruction()) - : SaveBuilder.setInsertPoint(*Entry); + auto SavePoint = SaveBlock->firstInstruction() != nullptr + ? SaveBuilder.setInsertPointBefore(*SaveBlock->firstInstruction()) + : SaveBuilder.setInsertPoint(*SaveBlock); if (!SavePoint) return std::unexpected(SavePoint.error()); for (const PhysicalRegister Reg : Saved) { @@ -1322,6 +1368,8 @@ public: for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!isFrameRestoreBlock(Function, *Block)) + continue; std::vector Terminators; for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; Instruction = Instruction->next()) { @@ -1433,21 +1481,22 @@ public: [[nodiscard]] Expected emitPrologue(MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) return Function.frameInfo().state().StackSize == 0 ? Expected{} : makeUnexpected(ErrorCode::VerificationError, - "RISC-V nonempty frame has no entry block"); - const std::vector Saves = calleeSaveOperations(*Entry, Function, false); + "RISC-V nonempty frame has no save block"); + const std::vector Saves = + calleeSaveOperations(*SaveBlock, Function, false); if (Saves.size() != Function.records().calleeSavedRegisters().size()) { return makeUnexpected(ErrorCode::InternalError, "RISC-V prologue has an incomplete callee-save sequence"); } - MachineInstr *Anchor = Entry->firstInstruction(); + MachineInstr *Anchor = SaveBlock->firstInstruction(); MachineIRBuilder Builder(Function, Target); - auto Point = - Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) : Builder.setInsertPoint(*Entry); + auto Point = Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) + : Builder.setInsertPoint(*SaveBlock); if (!Point) return std::unexpected(Point.error()); @@ -1637,6 +1686,8 @@ public: MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { + if (!isFrameRestoreBlock(Function, Block)) + return {}; auto FP = physical(PhysicalRegisterID::X8); auto SP = physical(PhysicalRegisterID::X2); auto BP = physical(PhysicalRegisterID::X9); diff --git a/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp new file mode 100644 index 0000000..5c22dde --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr MachineCombinerStageMask allOptimizationStages() noexcept { + return machineCombinerStageMask(MachineCombinerStage::SelectedSSA) | + machineCombinerStageMask(MachineCombinerStage::PreRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostFrame); +} + +[[nodiscard]] MachineCombinerPattern immediateIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::int64_t Immediate) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::ImmediateEquals, 2, 0, Immediate}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] MachineCombinerPattern zeroRegisterIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::size_t ZeroOperand, + std::size_t SourceOperand) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::PhysicalRegisterEquals, ZeroOperand, 0, + static_cast(PhysicalRegisterID::X0)}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, SourceOperand}, + }; +} + +[[nodiscard]] MachineCombinerPattern equalSourceIdentity(InstructionOpcode OpcodeValue, + std::string Name) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::RegisterEquals, 1, 2, 0}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] std::vector createPatterns() { + std::vector Patterns; + Patterns.push_back(immediateIdentity(InstructionOpcode::ADDI, "addi-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::XORI, "xori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ORI, "ori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ANDI, "andi-all-ones", -1)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SLLI, "slli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRLI, "srli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRAI, "srai-zero", 0)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::SUB, "sub-zero-right", 2, 1)); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_S, "fsgnj-s-copy")); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_D, "fsgnj-d-copy")); + return Patterns; +} + +} // namespace + +const TargetMachineCombinerInfo &machineCombinerInfo() noexcept { + static const Expected Info = + TargetMachineCombinerInfo::create(createPatterns()); + if (!Info) + std::abort(); + return *Info; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp new file mode 100644 index 0000000..2cc5f24 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +#include +#include + +namespace rcc::riscv { +namespace { + +class RISCVMachineMemoryFoldingInfo final : public TargetMemoryFoldingInfo { +public: + [[nodiscard]] Expected canFoldMemoryOperand(const MachineInstr &, std::size_t, + const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected + foldMemoryOperand(MachineInstr &, std::size_t, MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC has no register instruction form with a folded memory operand"); + } + + [[nodiscard]] Expected unfoldMemoryOperand(MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected> + getFoldedMemorySize(const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return std::optional{}; + } +}; + +} // namespace + +const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept { + static const RISCVMachineMemoryFoldingInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp new file mode 100644 index 0000000..2f8c5d1 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +namespace rcc::riscv { + +MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept { + return { + .SSA = ssaInfo(), + .Combiner = machineCombinerInfo(), + .MemoryFolding = memoryFoldingInfo(), + .FrameLowering = frameLowering(), + .PseudoExpansion = pseudoExpansionInfo(), + .SchedulingModel = schedulingModel(), + .BranchInfo = branchInfo(), + }; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp new file mode 100644 index 0000000..ed73794 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; +constexpr std::uint32_t CopyF32Flag = 8; +constexpr std::uint32_t CopyF64Flag = 16; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] std::optional +registerBank(Register Reg, const MachineFunction &Function, const TargetDescription &Target) { + if (Reg.isVirtual()) { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Bank : std::nullopt; + } + const std::optional Physical = Reg.physicalRegister(); + if (!Physical) + return std::nullopt; + for (const RegisterBankDesc &Bank : Target.registers().registerBanks()) { + if (Target.registers().bankContainsRegister(Bank.ID, *Physical)) + return Bank.ID; + } + return std::nullopt; +} + +[[nodiscard]] std::optional registerType(Register Reg, const MachineFunction &Function) { + if (!Reg.isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Type : std::nullopt; +} + +[[nodiscard]] Expected copyLocation(const MachineInstr &Source, MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyCommonDecoration(const MachineInstr &Source, + MachineInstr &Destination, bool CopyPreSymbol, + bool CopyPostSymbol, bool CopyDebugNumber) { + auto Location = copyLocation(Source, Destination); + if (!Location) + return std::unexpected(Location.error()); + Destination.setFlags(Source.flags()); + if (CopyDebugNumber && Source.debugInstrNumber() != 0) { + auto Number = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!Number) + return std::unexpected(Number.error()); + } + if (CopyPreSymbol) { + if (const auto Symbol = Source.preInstrSymbol()) { + auto Set = Destination.setPreInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (CopyPostSymbol) { + if (const auto Symbol = Source.postInstrSymbol()) { + auto Set = Destination.setPostInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (Source.heapAllocMarker()) { + auto Set = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.pcSections()) { + auto Set = Destination.setPCSections(*Source.pcSections()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.mmra()) { + auto Set = Destination.setMMRA(*Source.mmra()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.cfiType()) { + auto Set = Destination.setCFIType(*Source.cfiType()); + if (!Set) + return std::unexpected(Set.error()); + } + if (const auto Symbol = Source.deactivationSymbol()) { + auto Set = Destination.setDeactivationSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + return {}; +} + +[[nodiscard]] Expected removeOldInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "pseudo instruction is not linked"); + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected> +copyRegisters(const MachineInstr &Instruction) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA copy pseudo requires two register operands"); + return std::pair{*Destination, *Source}; +} + +[[nodiscard]] Expected prepareCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY)) + return false; + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const auto DestinationBank = registerBank(Registers->first, Function, Target); + const auto SourceBank = registerBank(Registers->second, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V copy preparation requires constrained register banks"); + if (*DestinationBank == *SourceBank) + return false; + + const auto DestinationType = registerType(Registers->first, Function); + const auto SourceType = registerType(Registers->second, Function); + const std::optional Type = DestinationType ? DestinationType : SourceType; + if (!Type || Type->isVector() || (Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V cross-bank COPY requires a scalar 32-bit or 64-bit type"); + } + if (DestinationType && SourceType && + DestinationType->scalarBitWidth() != SourceType->scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V cross-bank COPY has mismatched value widths"); + } + const std::uint32_t Flag = Type->scalarBitWidth() == 32 ? CopyF32Flag : CopyF64Flag; + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const bool Changed = + DestinationOperand->targetFlags() != Flag || SourceOperand->targetFlags() != Flag; + DestinationOperand->setTargetFlags(Flag); + SourceOperand->setTargetFlags(Flag); + return Changed; +} + +[[nodiscard]] Expected expandCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const Register Destination = Registers->first; + const Register Source = Registers->second; + if (!Destination.isPhysical() || !Source.isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY retains a virtual register"); + } + if (Destination == Source) { + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + const auto DestinationBank = registerBank(Destination, Function, Target); + const auto SourceBank = registerBank(Source, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY references an unknown register bank"); + const RegisterBankID GPR = + RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); + const RegisterBankID FPR = + RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); + + InstructionOpcode Selected = InstructionOpcode::Invalid; + bool DuplicateSource = false; + bool AddZeroImmediate = false; + if (*DestinationBank == GPR && *SourceBank == GPR) { + Selected = InstructionOpcode::ADDI; + AddZeroImmediate = true; + } else if (*DestinationBank == FPR && *SourceBank == FPR) { + Selected = InstructionOpcode::FSGNJ_D; + DuplicateSource = true; + } else { + const std::uint32_t DestinationFlags = Instruction.operandAt(0)->targetFlags(); + const std::uint32_t SourceFlags = Instruction.operandAt(1)->targetFlags(); + if (DestinationFlags != SourceFlags || + (DestinationFlags != CopyF32Flag && DestinationFlags != CopyF64Flag)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "RISC-V cross-bank COPY has no prepared transfer width"); + } + if (*DestinationBank == GPR && *SourceBank == FPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; + } else if (*DestinationBank == FPR && *SourceBank == GPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + } + } + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V COPY crosses unsupported banks"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Selected)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Source); + if (!Use) + return std::unexpected(Use.error()); + if (DuplicateSource) { + auto SecondUse = Pending->addUse(Source); + if (!SecondUse) + return std::unexpected(SecondUse.error()); + } + if (AddZeroImmediate) { + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + } + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandSubregisterCopy(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const Opcode Value = Instruction.opcode(); + if (Value == opcode(GenericOpcode::EXTRACT_SUBREG) || + Value == opcode(GenericOpcode::SUBREG_TO_REG)) { + const MachineOperand *Index = Instruction.operandAt(2); + const auto *Immediate = Index == nullptr ? nullptr : Index->getIf(); + if (Immediate == nullptr || *Immediate != 0) + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC does not support nonzero subregister indices"); + } else if (Value == opcode(GenericOpcode::COPY_LANEMASK)) { + const MachineOperand *MaskOperand = Instruction.operandAt(2); + const auto *Mask = MaskOperand == nullptr ? nullptr : MaskOperand->getIf(); + if (Mask == nullptr || Mask->Mask.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V lane copy requires a nonempty lane mask"); + } else { + return false; + } + return expandCopy(Instruction, Function, Target); +} + +[[nodiscard]] Expected addDirectTarget(MachineInstrBuilder &Builder, + const MachineOperand &Callee) { + if (const auto *Global = Callee.getIf()) + return Builder.addGlobalSymbol(Global->Name, Global->Offset, PCRelativeHighFlag); + if (const auto *External = Callee.getIf()) + return Builder.addExternalSymbol(External->Name, External->Offset, PCRelativeHighFlag); + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct call pseudo has no symbol target"); +} + +[[nodiscard]] bool isRegister(const MachineOperand &Operand, PhysicalRegister Reg) noexcept { + const Register *Value = Operand.getIf(); + return Value != nullptr && Value->physicalRegister() == Reg; +} + +[[nodiscard]] Expected copyExtraImplicitOperands( + const MachineInstr &Source, std::size_t FirstImplicit, MachineInstrBuilder &Destination, + std::optional ExplicitDef, std::optional ExplicitUse) { + for (std::size_t Index = FirstImplicit; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + if (Operand == nullptr || !Operand->isImplicit()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed implicit operands"); + if ((ExplicitDef && Operand->isDef() && isRegister(*Operand, *ExplicitDef)) || + (ExplicitUse && Operand->isUse() && isRegister(*Operand, *ExplicitUse))) { + continue; + } + auto Added = Destination.addImplicitOperand(Operand->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] Expected expandCall(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, bool IsTail) { + MachineBasicBlock *Block = Instruction.parent(); + const MachineOperand *Callee = Instruction.operandAt(0); + const MachineOperand *CallSite = Instruction.operandAt(1); + const MachineOperand *Mask = Instruction.operandAt(2); + if (Block == nullptr || Callee == nullptr || CallSite == nullptr || Mask == nullptr || + Mask->kind() != MachineOperand::Kind::RegisterMask) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed explicit operands"); + } + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + auto Scratch = physical(PhysicalRegisterID::X6); + if (!Zero || !Link || !Scratch) + return makeUnexpected(ErrorCode::InternalError, "RISC-V call registers are unavailable"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + Register Base; + std::optional Anchor; + const bool Direct = Callee->kind() == MachineOperand::Kind::GlobalSymbol || + Callee->kind() == MachineOperand::Kind::ExternalSymbol; + if (Direct) { + const PhysicalRegister HighRegister = IsTail ? *Scratch : *Link; + Base = Register::fromPhysical(HighRegister); + Anchor = Instruction.preInstrSymbol() + ? std::optional{std::string(*Instruction.preInstrSymbol())} + : std::optional{".Lrcc_call_" + + std::to_string(Function.stableID().value()) + "_" + + std::to_string(Instruction.stableID().value())}; + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Base); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = addDirectTarget(*HighPending, *Callee); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto Symbol = (*High)->setPreInstrSymbol(*Anchor); + if (!Symbol) + return std::unexpected(Symbol.error()); + auto Decoration = copyLocation(Instruction, **High); + if (!Decoration) + return std::unexpected(Decoration.error()); + } else { + const Register *CalleeRegister = Callee->getIf(); + if (CalleeRegister == nullptr || !CalleeRegister->isPhysical()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V indirect call retains a virtual callee register"); + Base = *CalleeRegister; + } + + const InstructionOpcode LowOpcode = + IsTail ? InstructionOpcode::JALR_TAIL : InstructionOpcode::JALR_CALL; + auto LowPending = Builder.buildInstr(targetOpcode(LowOpcode)); + if (!LowPending) + return std::unexpected(LowPending.error()); + const PhysicalRegister DestinationRegister = IsTail ? *Zero : *Link; + auto LowDef = LowPending->addDef(Register::fromPhysical(DestinationRegister)); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Base); + if (!LowBase) + return std::unexpected(LowBase.error()); + Expected LowOffset = + Anchor ? LowPending->addMCSymbol(*Anchor, 0, PCRelativeLowFlag) : LowPending->addImmediate(0); + if (!LowOffset) + return std::unexpected(LowOffset.error()); + auto LowCallSite = LowPending->addOperand(CallSite->cloneDetached()); + if (!LowCallSite) + return std::unexpected(LowCallSite.error()); + auto LowMask = LowPending->addOperand(Mask->cloneDetached()); + if (!LowMask) + return std::unexpected(LowMask.error()); + auto Implicit = copyExtraImplicitOperands( + Instruction, 3, *LowPending, IsTail ? std::nullopt : std::optional{*Link}, + std::nullopt); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto Decoration = copyCommonDecoration(Instruction, **Low, !Direct, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Records = + Function.records().replaceInstructionReference(Instruction.stableID(), (*Low)->stableID()); + if (!Records) + return std::unexpected(Records.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandReturn(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + if (!Zero || !Link) + return makeUnexpected(ErrorCode::InternalError, "RISC-V return registers are unavailable"); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR_RET)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Register::fromPhysical(*Link)); + if (!Use) + return std::unexpected(Use.error()); + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto Implicit = copyExtraImplicitOperands(Instruction, 0, *Pending, std::nullopt, *Link); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVPseudoExpansionInfo final : public TargetPseudoExpansionInfo { +public: + [[nodiscard]] Expected preparePostRAPseudo(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const override { + return prepareCopy(Instruction, Function, Target); + } + + [[nodiscard]] Expected expandPseudo(MachineInstr &Instruction, + MachinePseudoExpansionPhase Phase, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (Phase != MachinePseudoExpansionPhase::PostRA) + return false; + if (Instruction.opcode() == opcode(GenericOpcode::COPY)) + return expandCopy(Instruction, Function, Target); + auto Subregister = expandSubregisterCopy(Instruction, Function, Target); + if (!Subregister || *Subregister) + return Subregister; + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoCALL)) + return expandCall(Instruction, Function, Target, false); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoTAIL)) + return expandCall(Instruction, Function, Target, true); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoRET)) + return expandReturn(Instruction, Function, Target); + return false; + } +}; + +} // namespace + +const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept { + static const RISCVPseudoExpansionInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVScheduling.cpp b/backend/lib/Target/RISCV/RISCVScheduling.cpp new file mode 100644 index 0000000..8ef8d91 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVScheduling.cpp @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +enum class Resource : std::uint32_t { + Core = 1, + IntegerALU = 2, + Memory = 3, + Branch = 4, + MultiplyDivide = 5, + FloatingPoint = 6, + System = 7, +}; + +enum class PressureSet : std::uint32_t { + GPR = 1, + FPR = 2, +}; + +[[nodiscard]] constexpr ProcessorResourceID resource(Resource Value) noexcept { + return ProcessorResourceID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterPressureSetID pressureSet(PressureSet Value) noexcept { + return RegisterPressureSetID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterClassID registerClass(RegisterClass Value) noexcept { + return RegisterClassID::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::vector resourceUses(SchedulingClass Class) { + switch (Class) { + case SchedulingClass::ALU: + case SchedulingClass::COMPRESSED: + return {{resource(Resource::IntegerALU), 1}}; + case SchedulingClass::LOAD: + case SchedulingClass::STORE: + return {{resource(Resource::Memory), 1}}; + case SchedulingClass::BRANCH: + return {{resource(Resource::Branch), 1}}; + case SchedulingClass::SYSTEM: + case SchedulingClass::FENCE: + return {{resource(Resource::System), 1}}; + case SchedulingClass::MUL: + return {{resource(Resource::MultiplyDivide), 3}}; + case SchedulingClass::DIV: + return {{resource(Resource::MultiplyDivide), 20}}; + case SchedulingClass::ATOMIC: + return {{resource(Resource::Memory), 4}, {resource(Resource::System), 1}}; + case SchedulingClass::FP_ADD: + case SchedulingClass::FP_MUL: + case SchedulingClass::FP_CONVERT: + return {{resource(Resource::FloatingPoint), 4}}; + case SchedulingClass::FP_DIV: + case SchedulingClass::FP_SQRT: + return {{resource(Resource::FloatingPoint), 20}}; + case SchedulingClass::None: + return {}; + } + return {}; +} + +[[nodiscard]] TargetSchedulingModelDesc createDescription() { + TargetSchedulingModelDesc Description; + Description.IssueWidth = 2; + Description.DispatchLimit = 2; + Description.ProcessorResources = { + {resource(Resource::Core), "core", 2, std::nullopt}, + {resource(Resource::IntegerALU), "integer-alu", 2, resource(Resource::Core)}, + {resource(Resource::Memory), "memory", 1, resource(Resource::Core)}, + {resource(Resource::Branch), "branch", 1, resource(Resource::Core)}, + {resource(Resource::MultiplyDivide), "multiply-divide", 1, resource(Resource::Core)}, + {resource(Resource::FloatingPoint), "floating-point", 1, resource(Resource::Core)}, + {resource(Resource::System), "system", 1, resource(Resource::Core)}, + }; + for (const SchedulingClassInfo &Class : schedulingClasses()) { + SchedulingClassModelDesc Model{ + .ID = Class.ID, + .Name = std::string(Class.Name), + .MicroOps = Class.MicroOps, + .Latency = Class.Latency, + .Resources = resourceUses(static_cast(Class.ID.value())), + .WriteLatencies = {}, + .ReadAdvances = {}, + }; + Model.WriteLatencies.push_back({0, Class.Latency}); + switch (static_cast(Class.ID.value())) { + case SchedulingClass::ALU: + Model.ReadAdvances = { + {1, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + {2, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + }; + break; + default: + break; + } + Description.SchedulingClasses.push_back(std::move(Model)); + } + Description.PressureSets = { + {pressureSet(PressureSet::GPR), + "gpr", + {registerClass(RegisterClass::GPR), registerClass(RegisterClass::GPRC), + registerClass(RegisterClass::GPR_NX0), registerClass(RegisterClass::GPR_NX0_NX2)}, + 27}, + {pressureSet(PressureSet::FPR), + "fpr", + {registerClass(RegisterClass::FPR64), registerClass(RegisterClass::FPR64C), + registerClass(RegisterClass::FPR32), registerClass(RegisterClass::FPR32C)}, + 32}, + }; + return Description; +} + +} // namespace + +const TargetSchedulingModel &schedulingModel() noexcept { + static const Expected Model = + TargetSchedulingModel::create(createDescription()); + if (!Model) + std::abort(); + return *Model; +} + +} // namespace rcc::riscv -- cgit v1.2.3