diff options
Diffstat (limited to 'backend/lib/CodeGen')
32 files changed, 8164 insertions, 1 deletions
diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index a8cb952..341452a 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -11,11 +11,21 @@ set( MachineBasicBlockAttributes.cpp MachineCFIInfo.cpp MachineCompilationStatus.cpp + MachineCombiner.cpp + MachineCombinerPattern.cpp MachineBasicBlock.cpp + MachineBlockPlacement.cpp + MachineBranchOptimization.cpp + MachineBranchRelaxation.cpp MachineCFGUpdater.cpp MachineConstantPool.cpp + MachineCopyPropagation.cpp + MachineCSE.cpp MachineDebugInfo.cpp + MachineDCE.cpp MachineFrameInfo.cpp + MachineFallthrough.cpp + MachineFinalization.cpp MachineFunction.cpp MachineFunctionAttributes.cpp MachineFunctionCloner.cpp @@ -24,15 +34,29 @@ set( MachineAnalyses.cpp MachineInstr.cpp MachineJumpTableInfo.cpp + MachineLICM.cpp MachineLiveness.cpp + MachineMemoryFolding.cpp MachineMemOperand.cpp MachineMetadata.cpp MachineOperand.cpp + MachineOptimizationPipeline.cpp MachinePass.cpp + MachinePseudoExpansion.cpp MachinePipelinePasses.cpp MachineRegisterInfo.cpp + MachineScheduler.cpp + MachineShrinkWrapping.cpp + MachineSink.cpp + MachineStackSlotColoring.cpp + MachineTailDuplication.cpp + MachineTailMerge.cpp PrologueEpilogueInsertion.cpp + PostRAScheduler.cpp + RegisterPressure.cpp RegisterScavenger.cpp + ScheduleDAG.cpp + ScheduleHazardRecognizer.cpp InlineSpiller.cpp LiveRangeEdit.cpp LiveRegMatrix.cpp @@ -53,6 +77,7 @@ set( TargetFunctionState.cpp TargetMIRTextInfo.cpp TargetRegisterInfo.cpp + TargetSchedulingModel.cpp ) set( RCCBackendCodeGenHeaders @@ -65,11 +90,21 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombinerPattern.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBlockPlacement.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchOptimization.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchRelaxation.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineConstantPool.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCopyPropagation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCSE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDebugInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDCE.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFrameInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFallthrough.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFinalization.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunction.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAttributes.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionCloner.h" @@ -78,17 +113,32 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineAnalyses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineInstr.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineJumpTableInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLICM.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLiveness.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLivenessRepair.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemOperand.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePseudoExpansion.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePipelinePasses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PrologueEpilogueInsertion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterPressure.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterScavenger.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleDAG.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleHazardRecognizer.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFrameLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineScheduler.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineShrinkWrapping.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineSink.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineStackSlotColoring.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailDuplication.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailMerge.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PostRAScheduler.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Graph.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Math.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/ReductionRules.h" @@ -111,10 +161,14 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/StableID.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetConstant.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCallLowering.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetBranchInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetDescription.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFunctionState.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMIRTextInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMemoryFolding.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetSchedulingModel.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetPseudoExpansion.h" ) add_library(RCCBackendCodeGen ${RCCBackendCodeGenSources}) @@ -156,9 +210,19 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -167,13 +231,28 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulerInternal.h" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -197,6 +276,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ${RCCBackendCodeGenHeaders} ) rcc_backend_register_tidy_files( @@ -210,9 +290,19 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBlockPlacement.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchOptimization.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineBranchRelaxation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCombinerPattern.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineConstantPool.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCopyPropagation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineCSE.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineDCE.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineDebugInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFrameInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFallthrough.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineFinalization.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunction.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionAttributes.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionCloner.cpp" @@ -221,13 +311,27 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineAnalyses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineInstr.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineLICM.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineLiveness.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemoryFolding.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMemOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineShrinkWrapping.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSink.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineStackSlotColoring.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailDuplication.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachineTailMerge.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PostRAScheduler.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterPressure.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleDAG.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/ScheduleHazardRecognizer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" @@ -248,4 +352,5 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionState.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/TargetSchedulingModel.cpp" ) diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index 5aec744..48d4f10 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -628,7 +628,7 @@ Expected<void> InstructionSet::validateOperand(const OperandDescriptor &Descript } } - if (Descriptor.ImmediatePredicate) { + if (Descriptor.ImmediatePredicate && Operand.kind() == MachineOperand::Kind::Immediate) { const ImmediatePredicate *Predicate = immediatePredicate(*Descriptor.ImmediatePredicate); if (Predicate == nullptr || !Predicate->Predicate(Operand)) { return makeUnexpected(ErrorCode::VerificationError, diff --git a/backend/lib/CodeGen/MachineBlockPlacement.cpp b/backend/lib/CodeGen/MachineBlockPlacement.cpp new file mode 100644 index 0000000..fea1283 --- /dev/null +++ b/backend/lib/CodeGen/MachineBlockPlacement.cpp @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBlockPlacement.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +enum class PlacementClass : std::uint8_t { + Hot, + Normal, + Cold, + Exception, +}; + +[[nodiscard]] const MachinePassContract &placementContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-block-placement", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = + { + MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineBranchProbabilityAnalysis>(), + MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>(), + MachineAnalysisManager::id<MachineLoopAnalysis>(), + }, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] PlacementClass classify(const MachineBasicBlock &Block, + std::uint64_t Frequency) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Exception || + Attributes.has(MachineBasicBlockAttribute::LandingPad) || + Attributes.has(MachineBasicBlockAttribute::EHFuncletEntry) || + Attributes.has(MachineBasicBlockAttribute::EHScopeEntry)) { + return PlacementClass::Exception; + } + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Cold) + return PlacementClass::Cold; + if (Frequency >= MachineBlockFrequencyAnalysis::EntryFrequency) + return PlacementClass::Hot; + if (Frequency < MachineBlockFrequencyAnalysis::EntryFrequency / 1024U) + return PlacementClass::Cold; + return PlacementClass::Normal; +} + +[[nodiscard]] std::uint32_t classRank(PlacementClass Class) noexcept { + return static_cast<std::uint32_t>(Class); +} + +struct PlacementFacts final { + MachineBasicBlock *Block = nullptr; + PlacementClass Class = PlacementClass::Normal; + std::uint64_t Frequency = 0; + std::uint32_t LoopDepth = 0; +}; + +[[nodiscard]] bool betterRoot(const PlacementFacts &Left, const PlacementFacts &Right) noexcept { + if (Left.Class != Right.Class) + return classRank(Left.Class) < classRank(Right.Class); + if (Left.Frequency != Right.Frequency) + return Left.Frequency > Right.Frequency; + if (Left.LoopDepth != Right.LoopDepth) + return Left.LoopDepth > Right.LoopDepth; + return Left.Block->stableID().value() < Right.Block->stableID().value(); +} + +} // namespace + +const MachinePassContract &MachineBlockPlacementPass::contract() const noexcept { + return placementContract(); +} + +Expected<MachinePassResult> MachineBlockPlacementPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine block placement requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "block placement setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "block placement diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineBlockPlacementPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) const { + auto Probabilities = Analyses.getResult<MachineBranchProbabilityAnalysis>(Function, Target); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + std::vector<PlacementFacts> Facts; + Facts.reserve(Function.blockCount()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const std::uint64_t Frequency = (*Frequencies)->frequency(*Block); + Facts.push_back({Block, classify(*Block, Frequency), Frequency, (*Loops)->loopDepth(*Block)}); + } + if (Facts.size() < 2) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto factsFor = [&Facts](const MachineBasicBlock &Block) -> const PlacementFacts & { + return *std::ranges::find_if( + Facts, [&Block](const PlacementFacts &Value) { return Value.Block == &Block; }); + }; + std::unordered_set<MachineBasicBlock *> Placed; + std::vector<MachineBasicBlock *> Order; + Order.reserve(Facts.size()); + MachineBasicBlock *Root = Function.firstBlock(); + while (Order.size() != Facts.size()) { + if (Root == nullptr || Placed.contains(Root)) { + const auto Best = std::ranges::min_element( + Facts, [&Placed](const PlacementFacts &Left, const PlacementFacts &Right) { + if (Placed.contains(Left.Block)) + return false; + if (Placed.contains(Right.Block)) + return true; + return betterRoot(Left, Right); + }); + if (Best == Facts.end() || Placed.contains(Best->Block)) + return makeUnexpected(ErrorCode::InternalError, "block placement lost an unplaced block"); + Root = Best->Block; + } + + while (Root != nullptr && Placed.insert(Root).second) { + Order.push_back(Root); + MachineBasicBlock *BestSuccessor = nullptr; + BranchProbability BestProbability = BranchProbability::zero(); + for (const MachineSuccessor &Successor : (*Probabilities)->successors(*Root)) { + if (Successor.Block == nullptr || Placed.contains(Successor.Block)) + continue; + if (BestSuccessor == nullptr || BestProbability < Successor.Probability || + (BestProbability == Successor.Probability && + betterRoot(factsFor(*Successor.Block), factsFor(*BestSuccessor)))) { + BestSuccessor = Successor.Block; + BestProbability = Successor.Probability; + } + } + Root = BestSuccessor; + } + Root = nullptr; + } + + const auto &SavePoints = Function.frameInfo().state().SavePoints; + if (!SavePoints.empty() && SavePoints.front().Block != Function.firstBlock()->stableID()) { + const auto SavePosition = std::ranges::find_if(Order, [&](const MachineBasicBlock *Block) { + return Block->stableID() == SavePoints.front().Block; + }); + if (SavePosition == Order.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "block placement cannot find the frame save point"); + } + MachineBasicBlock *SaveBlock = *SavePosition; + std::vector<MachineBasicBlock *> OutsideFrame; + std::vector<MachineBasicBlock *> InsideFrame; + for (MachineBasicBlock *Block : Order) { + ((*Dominators)->dominates(*SaveBlock, *Block) ? InsideFrame : OutsideFrame).push_back(Block); + } + const auto SaveInRegion = std::ranges::find(InsideFrame, SaveBlock); + if (SaveInRegion == InsideFrame.end()) { + return makeUnexpected(ErrorCode::InternalError, "block placement lost the frame save region"); + } + std::rotate(InsideFrame.begin(), SaveInRegion, std::next(SaveInRegion)); + Order.clear(); + Order.insert(Order.end(), OutsideFrame.begin(), OutsideFrame.end()); + Order.insert(Order.end(), InsideFrame.begin(), InsideFrame.end()); + } + + bool Changed = false; + for (std::size_t Index = 1; Index < Order.size(); ++Index) { + Changed |= Order[Index - 1U]->next() != Order[Index]; + auto Moved = Function.moveBlockAfter(*Order[Index - 1U], *Order[Index]); + if (!Moved) + return std::unexpected(Moved.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchOptimization.cpp b/backend/lib/CodeGen/MachineBranchOptimization.cpp new file mode 100644 index 0000000..6c6ac51 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchOptimization.cpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchOptimization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &branchOptimizationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-optimization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasInstructionRecord(const MachineInstr &Instruction, + const MachineFunction &Function) noexcept { + const MachineInstrStableID ID = Instruction.stableID(); + return std::ranges::any_of(Function.records().callSites(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }) || + std::ranges::any_of(Function.records().calledGlobals(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }); +} + +[[nodiscard]] bool hasObservableBlockIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] bool canDeleteBlock(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + if (hasObservableBlockIdentity(Block, Function)) + return false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (hasProtectedAttachment(*Instruction) || hasInstructionRecord(*Instruction, Function)) { + return false; + } + } + return true; +} + +[[nodiscard]] bool hasSuccessor(const MachineBasicBlock &Block, + const MachineBasicBlock &Target) noexcept { + return std::ranges::any_of(Block.successors(), [&Target](const MachineSuccessor &Successor) { + return Successor.Block == &Target; + }); +} + +[[nodiscard]] Expected<bool> foldConstantBranches(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + bool Changed = false; + MachineCFGUpdater CFG = Function.cfgUpdater(); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional == nullptr || + hasProtectedAttachment(*Analysis->Conditional)) { + continue; + } + auto Condition = Info.evaluateBranchCondition(*Analysis->Conditional, Function, Target); + if (!Condition) + return std::unexpected(Condition.error()); + if (!Condition->has_value()) + continue; + + MachineBasicBlock *Selected = **Condition ? Analysis->TrueTarget : Analysis->FalseTarget; + MachineBasicBlock *Rejected = **Condition ? Analysis->FalseTarget : Analysis->TrueTarget; + if (Selected == nullptr || !hasSuccessor(*Block, *Selected)) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "constant branch has no selected CFG successor"); + } + if (Analysis->Unconditional != nullptr && hasProtectedAttachment(*Analysis->Unconditional)) { + continue; + } + + auto RemovedConditional = Info.removeBranch(*Analysis->Conditional, Function, Target); + if (!RemovedConditional) + return std::unexpected(RemovedConditional.error()); + if (Analysis->Unconditional != nullptr) { + auto RemovedUnconditional = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!RemovedUnconditional) + return std::unexpected(RemovedUnconditional.error()); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Selected, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + + if (Rejected != nullptr && Rejected != Selected && hasSuccessor(*Block, *Rejected)) { + auto RemovedEdge = CFG.removeSuccessor(*Block, *Rejected); + if (!RemovedEdge) + return std::unexpected(RemovedEdge.error()); + } + auto Probability = CFG.updateBranchProbability(*Block, *Selected, BranchProbability::one()); + if (!Probability) + return std::unexpected(Probability.error()); + Changed = true; + } + return Changed; +} + +[[nodiscard]] std::unordered_set<MachineBasicBlock *> reachableBlocks(MachineFunction &Function) { + std::unordered_set<MachineBasicBlock *> Reachable; + std::vector<MachineBasicBlock *> Worklist; + if (Function.firstBlock() != nullptr) + Worklist.push_back(Function.firstBlock()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!canDeleteBlock(*Block, Function)) + Worklist.push_back(Block); + } + while (!Worklist.empty()) { + MachineBasicBlock *Block = Worklist.back(); + Worklist.pop_back(); + if (Block == nullptr || !Reachable.insert(Block).second) + continue; + for (const MachineSuccessor &Successor : Block->successors()) + Worklist.push_back(Successor.Block); + } + return Reachable; +} + +[[nodiscard]] Expected<bool> removeUnreachableBlocks(MachineFunction &Function) { + const std::unordered_set<MachineBasicBlock *> Reachable = reachableBlocks(Function); + MachineCFGUpdater CFG = Function.cfgUpdater(); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;) { + MachineBasicBlock *Next = Block->next(); + if (!Reachable.contains(Block) && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + Changed = true; + } + Block = Next; + } + return Changed; +} + +[[nodiscard]] Expected<bool> isForwardingBlock(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + if (&Block == Function.firstBlock() || Block.successors().size() != 1 || + Block.successors().front().Block == nullptr || Block.successors().front().Block == &Block || + hasObservableBlockIdentity(Block, Function)) { + return false; + } + auto Analysis = Info.analyzeBranch(Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional != nullptr) + return false; + MachineBasicBlock *Successor = Block.successors().front().Block; + if (Analysis->Unconditional == nullptr) { + return Block.firstInstruction() == nullptr && Block.next() == Successor; + } + if (Analysis->TrueTarget != Successor || hasProtectedAttachment(*Analysis->Unconditional)) { + return false; + } + return Block.firstInstruction() == Analysis->Unconditional && + Analysis->Unconditional->next() == nullptr; +} + +[[nodiscard]] Expected<bool> threadForwardingBlocks(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info, + std::size_t MaximumEdges) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + std::size_t ThreadedEdges = 0; + bool AnyChange = false; + bool LocalChange = true; + while (LocalChange && ThreadedEdges < MaximumEdges) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Forwarding = isForwardingBlock(*Block, Function, Target, Info); + if (!Forwarding) + return std::unexpected(Forwarding.error()); + if (!*Forwarding) + continue; + MachineBasicBlock *Successor = Block->successors().front().Block; + const std::vector<MachineBasicBlock *> Predecessors(Block->predecessors().begin(), + Block->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (ThreadedEdges == MaximumEdges) + break; + if (Predecessor == nullptr || Predecessor->successors().size() != 1 || + Predecessor->successors().front().Block != Block) { + continue; + } + auto PredAnalysis = Info.analyzeBranch(*Predecessor, Function, Target); + if (!PredAnalysis) + return std::unexpected(PredAnalysis.error()); + if (PredAnalysis->HasUnanalyzableTerminator || PredAnalysis->Conditional != nullptr) + continue; + + if (PredAnalysis->Unconditional != nullptr) { + if (PredAnalysis->TrueTarget != Block || + hasProtectedAttachment(*PredAnalysis->Unconditional)) { + continue; + } + auto Rewritten = + Info.rewriteBranchTarget(*Predecessor, *Block, *Successor, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "branch threading did not rewrite an explicit branch"); + } + } else { + if (Predecessor->next() != Block) + continue; + auto Inserted = + Info.insertUnconditionalBranch(*Predecessor, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + auto Redirected = CFG.replaceSuccessor(*Predecessor, *Block, *Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + ++ThreadedEdges; + AnyChange = true; + LocalChange = true; + } + if (Block->predecessors().empty() && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + AnyChange = true; + LocalChange = true; + break; + } + } + } + return AnyChange; +} + +} // namespace + +const MachinePassContract &MachineBranchOptimizationPass::contract() const noexcept { + return branchOptimizationContract(); +} + +Expected<MachinePassResult> MachineBranchOptimizationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch optimization requires selected PHI-free MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "branch optimization transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch optimization diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineBranchOptimizationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + auto Folded = foldConstantBranches(Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + auto RemovedBeforeThreading = removeUnreachableBlocks(Function); + if (!RemovedBeforeThreading) + return std::unexpected(RemovedBeforeThreading.error()); + auto Threaded = threadForwardingBlocks(Function, Target, *Info, Options.MaximumThreadedEdges); + if (!Threaded) + return std::unexpected(Threaded.error()); + auto RemovedAfterThreading = removeUnreachableBlocks(Function); + if (!RemovedAfterThreading) + return std::unexpected(RemovedAfterThreading.error()); + + MachinePassResult Result; + Result.Changed = *Folded || *RemovedBeforeThreading || *Threaded || *RemovedAfterThreading; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchRelaxation.cpp b/backend/lib/CodeGen/MachineBranchRelaxation.cpp new file mode 100644 index 0000000..d56f3a0 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchRelaxation.cpp @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchRelaxation.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <unordered_map> +#include <utility> + +namespace rcc { +namespace { + +struct MachineLayout final { + std::unordered_map<const MachineBasicBlock *, std::uint64_t> BlockOffsets; + std::unordered_map<const MachineInstr *, std::uint64_t> InstructionOffsets; +}; + +[[nodiscard]] const MachinePassContract &relaxationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-relaxation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::VerificationError, "block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits<std::uint64_t>::max() - Mask) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected<MachineLayout> computeLayout(MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + MachineLayout Result; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + Result.BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.InstructionOffsets.emplace(Instruction, Offset); + auto Size = Info.instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + if (Offset > std::numeric_limits<std::uint64_t>::max() - *Size) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout size overflowed"); + Offset += *Size; + } + } + return Result; +} + +[[nodiscard]] Expected<std::int64_t> displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "forward branch displacement overflowed"); + return static_cast<std::int64_t>(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "backward branch displacement overflowed"); + return -static_cast<std::int64_t>(Difference); +} + +} // namespace + +const MachinePassContract &MachineBranchRelaxationPass::contract() const noexcept { + return relaxationContract(); +} + +Expected<MachinePassResult> MachineBranchRelaxationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch relaxation requires finalized frame and tracked liveness"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "branch relaxation setup failed: ")); + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "branch relaxation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineBranchRelaxationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.ConvergenceWatchdog == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "branch relaxation watchdog must be nonzero"); + MachinePassResult Result; + for (std::size_t Iteration = 0; Iteration < Options.ConvergenceWatchdog; ++Iteration) { + auto Layout = computeLayout(Function, Target, *Info); + if (!Layout) + return std::unexpected(Layout.error()); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !Changed; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto BranchTarget = Info->branchTarget(*Instruction, Function, Target); + if (!BranchTarget) + return std::unexpected(BranchTarget.error()); + if (*BranchTarget == nullptr) + continue; + const auto SourcePosition = Layout->InstructionOffsets.find(Instruction); + const auto TargetPosition = Layout->BlockOffsets.find(*BranchTarget); + if (SourcePosition == Layout->InstructionOffsets.end() || + TargetPosition == Layout->BlockOffsets.end()) { + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation lost an instruction or target offset"); + } + auto Offset = displacement(SourcePosition->second, TargetPosition->second); + if (!Offset) + return std::unexpected(Offset.error()); + if (Info->isBranchOffsetInRange(*Instruction, *Offset)) + continue; + auto Relaxed = Info->relaxBranch(*Instruction, *Offset, Function, Target); + if (!Relaxed) + return std::unexpected(Relaxed.error()); + if (!*Relaxed) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "target did not relax an out-of-range branch"); + } + Changed = true; + Result.Changed = true; + break; + } + } + if (Changed) + continue; + if (Result.Changed) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; + } + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation failed to reach a monotonic fixed point"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp new file mode 100644 index 0000000..a144403 --- /dev/null +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCSE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <cstdint> +#include <optional> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &cseContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-cse", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +template <typename ValueT> +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf<ValueT>(); + const ValueT *RightValue = Right.getIf<ValueT>(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +[[nodiscard]] bool registerOperandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + const Register *LeftReg = Left.getIf<Register>(); + const Register *RightReg = Right.getIf<Register>(); + if (LeftReg == nullptr || RightReg == nullptr || Left.subReg() != Right.subReg() || + Left.targetFlags() != Right.targetFlags()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags && (IgnoreDefinitionValue || *LeftReg == *RightReg); +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags()) + return false; + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return registerOperandEquals(Left, Right, IgnoreDefinitionValue); + case MachineOperand::Kind::Immediate: + return payloadEquals<std::int64_t>(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals<IntegerImmediateOperand>(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals<FloatingImmediateOperand>(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals<BasicBlockOperand>(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals<FrameIndexOperand>(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals<ConstantPoolOperand>(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals<JumpTableOperand>(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals<GlobalSymbolOperand>(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals<ExternalSymbolOperand>(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals<BlockAddressOperand>(Left, Right); + case MachineOperand::Kind::RegisterMask: + case MachineOperand::Kind::LiveOutRegisterMask: + return false; + case MachineOperand::Kind::Metadata: + return payloadEquals<MetadataOperand>(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals<TargetIndexOperand>(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals<MCSymbolOperand>(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals<CFIIndexOperand>(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals<IntrinsicOperand>(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals<PredicateOperand>(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals<ShuffleMaskOperand>(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals<DebugInstrRefOperand>(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals<LaneMaskOperand>(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] bool isConstantMemoryBase(MemoryBase::Kind Kind) noexcept { + return Kind == MemoryBase::Kind::ConstantPool || Kind == MemoryBase::Kind::ConstantPoolPseudo; +} + +[[nodiscard]] bool hasSafeInvariantMemory(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().empty()) + return false; + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) + return false; + const MachineMemOperandDesc &Description = Memory->description(); + const bool ExplicitInvariant = Description.Accesses.has(MemoryAccessFlag::Invariant) && + Description.Accesses.has(MemoryAccessFlag::Dereferenceable); + if (!ExplicitInvariant && !isConstantMemoryBase(Description.Base.kind())) + return false; + } + return true; +} + +[[nodiscard]] bool hasFloatingRegister(const MachineInstr &Instruction, + const MachineRegisterInfo &Registers) { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Info = Registers.virtualRegisterInfo(*Reg); + if (Info && Info->get().Type && Info->get().Type->isFloat()) + return true; + } + return false; +} + +[[nodiscard]] Expected<bool> isCandidate(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Pseudo) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction)) { + return false; + } + if (Flags.has(InstructionFlag::MayLoad) && !hasSafeInvariantMemory(Instruction)) + return false; + if (!Flags.has(InstructionFlag::MayLoad) && !Instruction.memOperands().empty()) + return false; + if (!Instruction.flags().has(MachineInstrFlag::NoFPExcept) && + hasFloatingRegister(Instruction, Function.registerInfo())) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Reg->isPhysical() || Reg->isNoRegister() || Operand.isDebugUse() || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Operand.isDef()) + HasDefinition = true; + } + return HasDefinition; +} + +[[nodiscard]] bool expressionEquals(const MachineInstr &Left, const MachineInstr &Right) noexcept { + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size()) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + LeftOperand->isDef() != RightOperand->isDef() || + !operandEquals(*LeftOperand, *RightOperand, LeftOperand->isDef())) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory) + return false; + } + return true; +} + +[[nodiscard]] Expected<std::vector<std::pair<Register, Register>>> +replacementPairs(const MachineInstr &Available, const MachineInstr &Redundant, + const MachineFunction &Function, const TargetDescription &Target) { + std::vector<std::pair<Register, Register>> Result; + std::unordered_set<Register> RedundantDefinitions; + const MachineRegisterInfo &Registers = Function.registerInfo(); + for (std::size_t Index = 0; Index < Available.operandCount(); ++Index) { + const MachineOperand *AvailableOperand = Available.operandAt(Index); + const MachineOperand *RedundantOperand = Redundant.operandAt(Index); + if (AvailableOperand == nullptr || RedundantOperand == nullptr || !AvailableOperand->isDef()) { + continue; + } + const Register *AvailableReg = AvailableOperand->getIf<Register>(); + const Register *RedundantReg = RedundantOperand->getIf<Register>(); + if (AvailableReg == nullptr || RedundantReg == nullptr || !AvailableReg->isVirtual() || + !RedundantReg->isVirtual() || !RedundantDefinitions.insert(*RedundantReg).second) { + return std::vector<std::pair<Register, Register>>{}; + } + auto AvailableInfo = Registers.virtualRegisterInfo(*AvailableReg); + if (!AvailableInfo) + return std::unexpected(AvailableInfo.error()); + auto RedundantInfo = Registers.virtualRegisterInfo(*RedundantReg); + if (!RedundantInfo) + return std::unexpected(RedundantInfo.error()); + const VirtualRegisterInfo &Left = AvailableInfo->get(); + const VirtualRegisterInfo &Right = RedundantInfo->get(); + if (!Left.Type || !Right.Type || Left.Type != Right.Type || + Left.TargetFlags != Right.TargetFlags) + return std::vector<std::pair<Register, Register>>{}; + if (Left.Bank && Right.Bank && + !Target.registers().registerBanksOverlap(*Left.Bank, *Right.Bank)) { + return std::vector<std::pair<Register, Register>>{}; + } + if (Left.RegClass && Right.RegClass && + !Target.registers().registerClassesOverlap(*Left.RegClass, *Right.RegClass)) { + return std::vector<std::pair<Register, Register>>{}; + } + Result.emplace_back(*RedundantReg, *AvailableReg); + } + return Result; +} + +[[nodiscard]] Expected<void> +replaceAndErase(MachineInstr &Redundant, + const std::vector<std::pair<Register, Register>> &Replacements, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Block = Redundant.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine CSE candidate is detached"); + for (const auto &[OldReg, NewReg] : Replacements) { + auto Replaced = Registers.replaceRegWith(OldReg, NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Redundant); + if (!Removed) + return std::unexpected(Removed.error()); + for (const auto &[OldReg, NewReg] : Replacements) { + (void)NewReg; + if (!Registers.useEmpty(OldReg, true) || Registers.getUniqueVRegDef(OldReg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine CSE left references to a redundant result"); + } + auto Erased = Registers.eraseVirtualRegister(OldReg); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineCSEPass::contract() const noexcept { + return cseContract(); +} + +Expected<MachinePassResult> MachineCSEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine CSE transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine CSE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> MachineCSEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + std::unordered_map<Opcode, std::vector<MachineInstr *>> Available; + std::size_t Comparisons = 0; + bool BudgetExhausted = false; + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !BudgetExhausted; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Candidate = isCandidate(*Instruction, Function, Target); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!*Candidate) { + Instruction = Next; + continue; + } + + MachineInstr *Match = nullptr; + auto &OpcodeCandidates = Available[Instruction->opcode()]; + for (MachineInstr *Prior : OpcodeCandidates) { + if (Comparisons >= Options.MaximumComparisons) { + BudgetExhausted = true; + break; + } + ++Comparisons; + if (!(*Dominators)->dominates(*Prior, *Instruction) || + !expressionEquals(*Prior, *Instruction)) { + continue; + } + Match = Prior; + break; + } + if (BudgetExhausted) + break; + if (Match == nullptr) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + + auto Replacements = replacementPairs(*Match, *Instruction, Function, Target); + if (!Replacements) + return std::unexpected(Replacements.error()); + if (Replacements->empty()) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + auto Replaced = replaceAndErase(*Instruction, *Replacements, Registers); + if (!Replaced) + return std::unexpected(Replaced.error()); + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombiner.cpp b/backend/lib/CodeGen/MachineCombiner.cpp new file mode 100644 index 0000000..02ea5e3 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombiner.cpp @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombiner.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <cstddef> +#include <cstdint> +#include <optional> +#include <span> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &combinerContract(MachineCombinerStage Stage) { + static const MachinePassContract SelectedSSA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-selected-ssa", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-pre-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-ra", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostFrame = [] { + MachinePassContract Value{ + .Name = "machine-combiner-post-frame", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + switch (Stage) { + case MachineCombinerStage::SelectedSSA: + return SelectedSSA; + case MachineCombinerStage::PreRegisterAllocation: + return PreRA; + case MachineCombinerStage::PostRegisterAllocation: + return PostRA; + case MachineCombinerStage::PostFrame: + return PostFrame; + } + return SelectedSSA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool matchesPredicate(const MachineInstr &Instruction, + const MachineCombinerPredicate &Predicate) { + const MachineOperand *Operand = Instruction.operandAt(Predicate.Operand); + if (Operand == nullptr) + return false; + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: { + const auto *Immediate = Operand->getIf<std::int64_t>(); + return Immediate != nullptr && *Immediate == Predicate.Value; + } + case MachineCombinerPredicateKind::RegisterEquals: { + const MachineOperand *Other = Instruction.operandAt(Predicate.OtherOperand); + const Register *LeftReg = Operand->getIf<Register>(); + const Register *RightReg = Other == nullptr ? nullptr : Other->getIf<Register>(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Operand->subReg() == Other->subReg(); + } + case MachineCombinerPredicateKind::PhysicalRegisterEquals: { + const Register *Reg = Operand->getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast<std::uint32_t>(Predicate.Value); + } + case MachineCombinerPredicateKind::RegisterIsVirtual: { + const Register *Reg = Operand->getIf<Register>(); + return Reg != nullptr && Reg->isVirtual(); + } + case MachineCombinerPredicateKind::RegisterIsPhysical: { + const Register *Reg = Operand->getIf<Register>(); + return Reg != nullptr && Reg->isPhysical(); + } + } + return false; +} + +[[nodiscard]] bool matchesPattern(const MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineCombinerStage Stage) { + if ((Pattern.Stages & machineCombinerStageMask(Stage)) == 0) + return false; + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + if (!matchesPredicate(Instruction, Predicate)) + return false; + } + return true; +} + +[[nodiscard]] bool compatibleReplacement(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || Source.isNoRegister()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + if (!DestinationInfo) + return false; + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + if (Source.isPhysical()) { + const std::optional<PhysicalRegister> Physical = Source.physicalRegister(); + if (!Physical) + return false; + if (DestinationValue.RegClass && + !Target.registers().registerClassContains(*DestinationValue.RegClass, *Physical)) { + return false; + } + return !DestinationValue.Bank || + Target.registers().bankContainsRegister(*DestinationValue.Bank, *Physical); + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo) + return false; + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && + (!SourceValue.Bank || + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank))) { + return false; + } + if (DestinationValue.RegClass && + (!SourceValue.RegClass || !Target.registers().registerClassesOverlap( + *DestinationValue.RegClass, *SourceValue.RegClass))) { + return false; + } + return true; +} + +[[nodiscard]] Expected<bool> executeReplacement(MachineInstr &Instruction, + const MachineCombinerAction &Action, + MachineFunction &Function, + const TargetDescription &Target) { + MachineOperand *Definition = Instruction.operandAt(Action.DefinitionOperand); + const MachineOperand *SourceOperand = Instruction.operandAt(Action.SourceOperand); + const Register *Destination = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !Definition->isDef() || Definition->isUse() || + !SourceOperand->isUse() || Definition->subReg().value() != 0 || + SourceOperand->subReg().value() != 0 || Destination->isNoRegister() || + Source->isNoRegister()) { + return false; + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine combiner root is detached"); + + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!Destination->isVirtual() || + Function.registerInfo().getUniqueVRegDef(*Destination) != Definition || + !compatibleReplacement(*Destination, *Source, Function, Target)) { + return false; + } + + std::vector<MachineOperand *> Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Destination, true)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(*Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Block->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(*Destination, true) || + Function.registerInfo().getUniqueVRegDef(*Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine combiner left references to a replaced result"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(*Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] Expected<bool> executePattern(MachineInstr &Instruction, + const MachineCombinerPattern &Pattern, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "machine combiner root has no descriptor"); + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayLoad) || + Flags.has(InstructionFlag::MayStore) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || !Instruction.memOperands().empty() || + hasProtectedAttachment(Instruction)) { + return false; + } + switch (Pattern.Action.Kind) { + case MachineCombinerActionKind::ReplaceDefWithOperandAndErase: + return executeReplacement(Instruction, Pattern.Action, Function, Target); + } + return false; +} + +} // namespace + +const MachinePassContract &MachineCombinerPass::contract() const noexcept { + return combinerContract(Stage); +} + +Expected<MachinePassResult> MachineCombinerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine combiner input does not satisfy its stage contract"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine combiner transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine combiner diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineCombinerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + if (Options.MaximumIterations == 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner iteration budget must be nonzero"); + MachinePassResult Result; + std::size_t Rewrites = 0; + for (std::size_t Iteration = 0; Iteration < Options.MaximumIterations; ++Iteration) { + bool IterationChanged = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + for (const MachineCombinerPattern &Pattern : Info->patterns(Instruction->opcode())) { + if (!matchesPattern(*Instruction, Pattern, Stage)) + continue; + if (Rewrites == Options.MaximumRewrites) + break; + auto Executed = executePattern(*Instruction, Pattern, Function, Target); + if (!Executed) + return std::unexpected(Executed.error()); + if (!*Executed) + continue; + ++Rewrites; + IterationChanged = true; + Result.Changed = true; + break; + } + Instruction = Next; + } + } + if (!IterationChanged || Rewrites == Options.MaximumRewrites) + break; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombinerPattern.cpp b/backend/lib/CodeGen/MachineCombinerPattern.cpp new file mode 100644 index 0000000..b1c3c96 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombinerPattern.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombinerPattern.h" + +#include <cstddef> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> + +namespace rcc { + +Expected<TargetMachineCombinerInfo> +TargetMachineCombinerInfo::create(std::vector<MachineCombinerPattern> Patterns) { + TargetMachineCombinerInfo Result(std::move(Patterns)); + std::unordered_set<std::string> Names; + for (const MachineCombinerPattern &Pattern : Result.Patterns) { + if (Pattern.Name.empty() || !Pattern.RootOpcode.isValid() || Pattern.Stages == 0 || + !Names.insert(Pattern.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner pattern has an invalid identity or stage mask"); + } + for (const MachineCombinerPredicate &Predicate : Pattern.Predicates) { + switch (Predicate.Kind) { + case MachineCombinerPredicateKind::ImmediateEquals: + case MachineCombinerPredicateKind::PhysicalRegisterEquals: + case MachineCombinerPredicateKind::RegisterIsVirtual: + case MachineCombinerPredicateKind::RegisterIsPhysical: + break; + case MachineCombinerPredicateKind::RegisterEquals: + if (Predicate.Operand == Predicate.OtherOperand) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register equality pattern compares an operand with itself"); + } + break; + } + } + Result.MatchTable[Pattern.RootOpcode].push_back(Pattern); + } + return Result; +} + +std::span<const MachineCombinerPattern> +TargetMachineCombinerInfo::patterns(Opcode RootOpcode) const noexcept { + const auto Position = MatchTable.find(RootOpcode); + return Position == MatchTable.end() ? std::span<const MachineCombinerPattern>{} + : std::span<const MachineCombinerPattern>(Position->second); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCompilationStatus.cpp b/backend/lib/CodeGen/MachineCompilationStatus.cpp index 765e133..e541d39 100644 --- a/backend/lib/CodeGen/MachineCompilationStatus.cpp +++ b/backend/lib/CodeGen/MachineCompilationStatus.cpp @@ -26,6 +26,7 @@ constexpr std::array StateNames{ std::string_view{"FinalizationFailed"}, std::string_view{"AssemblyLoweringFailed"}, std::string_view{"InternalError"}, + std::string_view{"SchedulingFailed"}, }; static_assert(StateNames.size() == static_cast<std::size_t>(MachineCompilationState::Count)); @@ -44,6 +45,11 @@ isKnownDiagnosticSeverity(MachineCompilationDiagnosticSeverity Severity) noexcep case ErrorCode::Unsupported: case ErrorCode::RegisterAllocationFailed: case ErrorCode::FrameLayoutFailed: + case ErrorCode::SchedulingFailed: + case ErrorCode::BranchRelaxationFailed: + case ErrorCode::PseudoExpansionFailed: + case ErrorCode::FinalEncodabilityFailed: + case ErrorCode::CFGOptimizationFailed: case ErrorCode::IoError: case ErrorCode::InternalError: return true; diff --git a/backend/lib/CodeGen/MachineCopyPropagation.cpp b/backend/lib/CodeGen/MachineCopyPropagation.cpp new file mode 100644 index 0000000..e537a06 --- /dev/null +++ b/backend/lib/CodeGen/MachineCopyPropagation.cpp @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCopyPropagation.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetCodeGen.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract & +copyPropagationContract(MachineCopyPropagationStage Stage) { + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "pre-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-copy-propagation", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCopyPropagationStage::PreRegisterAllocation ? PreRA : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct CopyOperands final { + MachineOperand *DestinationOperand = nullptr; + MachineOperand *SourceOperand = nullptr; + Register Destination; + Register Source; +}; + +[[nodiscard]] std::optional<CopyOperands> copyOperands(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY) || Instruction.operandCount() != 2 || + hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !DestinationOperand->isDef() || + !SourceOperand->isUse() || DestinationOperand->isImplicit() || SourceOperand->isImplicit()) { + return std::nullopt; + } + return CopyOperands{DestinationOperand, SourceOperand, *Destination, *Source}; +} + +[[nodiscard]] Expected<bool> compatibleVirtualRegisters(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || !Source.isVirtual()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!DestinationInfo || !SourceInfo) + return std::unexpected(!DestinationInfo ? DestinationInfo.error() : SourceInfo.error()); + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && SourceValue.Bank && + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank)) { + return false; + } + if (DestinationValue.RegClass && SourceValue.RegClass && + !Target.registers().isSubclass(*SourceValue.RegClass, *DestinationValue.RegClass)) { + return false; + } + return true; +} + +[[nodiscard]] Expected<bool> propagateVirtualCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + if (!Operands.Destination.isVirtual() || !Operands.Source.isVirtual()) + return false; + if (Function.registerInfo().getUniqueVRegDef(Operands.Destination) != + Operands.DestinationOperand) { + return false; + } + auto Compatible = + compatibleVirtualRegisters(Operands.Destination, Operands.Source, Function, Target); + if (!Compatible) + return std::unexpected(Compatible.error()); + if (!*Compatible) + return false; + auto ValidCopy = + Info.validateCopy(Operands.Destination, Operands.DestinationOperand->subReg(), + Operands.Source, Operands.SourceOperand->subReg(), Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + + std::vector<MachineOperand *> Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(Operands.Destination, true)) { + if (&Use == Operands.SourceOperand) + continue; + if (Use.tiedOperandIndex() || Use.isImplicit() || + Use.subReg() != Operands.DestinationOperand->subReg()) { + return false; + } + Uses.push_back(&Use); + } + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation candidate is detached"); + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + if (!Function.registerInfo().useEmpty(Operands.Destination, true) || + Function.registerInfo().getUniqueVRegDef(Operands.Destination) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "virtual COPY propagation left destination references"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Operands.Destination); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +[[nodiscard]] bool shareRegisterClass(PhysicalRegister Left, PhysicalRegister Right, + const TargetRegisterInfo &Registers) noexcept { + return std::ranges::any_of(Registers.registerClasses(), [&](const RegisterClassDesc &Class) { + return Registers.registerClassContains(Class.ID, Left) && + Registers.registerClassContains(Class.ID, Right); + }); +} + +[[nodiscard]] bool clobbersRegister(const MachineInstr &Instruction, PhysicalRegister Reg, + const TargetRegisterInfo &Registers) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (Mask->Mask && Mask->Mask->clobbers(Reg)) + return true; + continue; + } + const Register *OperandReg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + OperandReg == nullptr ? std::nullopt : OperandReg->physicalRegister(); + if (Physical && Operand.isDef() && Registers.regsOverlap(*Physical, Reg)) + return true; + } + return false; +} + +[[nodiscard]] bool canReplacePhysicalUse(const MachineInstr &Instruction, std::size_t OperandIndex, + PhysicalRegister Source, + const TargetDescription &Target) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Operand == nullptr || Descriptor == nullptr || Operand->isImplicit() || + Operand->tiedOperandIndex() || Operand->subReg().value() != 0 || + OperandIndex >= Descriptor->Operands.size()) { + return false; + } + const OperandDescriptor &Constraint = Descriptor->Operands[OperandIndex]; + if (Constraint.RegClass && + !Target.registers().registerClassContains(*Constraint.RegClass, Source)) { + return false; + } + if (Constraint.RegBank && !Target.registers().bankContainsRegister(*Constraint.RegBank, Source)) { + return false; + } + return true; +} + +[[nodiscard]] Expected<bool> propagatePhysicalCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetSSAInfo &Info) { + const std::optional<PhysicalRegister> Destination = Operands.Destination.physicalRegister(); + const std::optional<PhysicalRegister> Source = Operands.Source.physicalRegister(); + if (!Destination || !Source || Operands.DestinationOperand->subReg().value() != 0 || + Operands.SourceOperand->subReg().value() != 0) { + return false; + } + auto ValidCopy = + Info.validateCopy(Operands.Destination, {}, Operands.Source, {}, Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "physical COPY propagation candidate is detached"); + if (*Destination == *Source) { + auto Removed = Block->removeInstruction(Copy); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!shareRegisterClass(*Destination, *Source, Target.registers())) + return false; + + bool Changed = false; + for (MachineInstr *Instruction = Copy.next(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (clobbersRegister(*Instruction, *Destination, Target.registers()) || + clobbersRegister(*Instruction, *Source, Target.registers())) { + break; + } + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand->isUse() || Operand->isDebugUse() || + !Target.registers().regsOverlap(*Physical, *Destination) || + !canReplacePhysicalUse(*Instruction, Index, *Source, Target)) { + continue; + } + auto Replaced = Operand->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + Changed = true; + } + } + return Changed; +} + +} // namespace + +const MachinePassContract &MachineCopyPropagationPass::contract() const noexcept { + return copyPropagationContract(Stage); +} + +Expected<MachinePassResult> MachineCopyPropagationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine COPY propagation has unmet stage properties"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "COPY propagation transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "COPY propagation diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineCopyPropagationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t PropagatedCopies = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && PropagatedCopies < Options.MaximumPropagatedCopies;) { + MachineInstr *Next = Instruction->next(); + const std::optional<CopyOperands> Operands = copyOperands(*Instruction); + if (!Operands) { + Instruction = Next; + continue; + } + Expected<bool> Changed = + Stage == MachineCopyPropagationStage::PreRegisterAllocation + ? propagateVirtualCopy(*Instruction, *Operands, Function, Target, *Info) + : propagatePhysicalCopy(*Instruction, *Operands, Function, Target, *Info); + if (!Changed) + return std::unexpected(Changed.error()); + if (*Changed) { + ++PropagatedCopies; + Result.Changed = true; + } + Instruction = Next; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + if (Stage == MachineCopyPropagationStage::PreRegisterAllocation) + Result.Changes.add(MachinePassChange::Registers); + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineDCE.cpp b/backend/lib/CodeGen/MachineDCE.cpp new file mode 100644 index 0000000..b413499 --- /dev/null +++ b/backend/lib/CodeGen/MachineDCE.cpp @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineDCE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <optional> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &dceContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-dce", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] Expected<std::optional<std::vector<Register>>> +deadDefinitions(MachineInstr &Instruction, MachineRegisterInfo &Registers, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + + std::vector<Register> VirtualDefinitions; + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Operand.isDef()) + continue; + HasDefinition = true; + if (Operand.isUse() || Operand.tiedOperandIndex()) + return std::nullopt; + if (Reg->isVirtual()) { + if (!Registers.useEmpty(*Reg, false)) + return std::nullopt; + VirtualDefinitions.push_back(*Reg); + continue; + } + if (!Reg->isPhysical() || !Operand.registerFlags().has(RegisterOperandFlag::Dead)) { + return std::nullopt; + } + } + if (!HasDefinition) + return std::nullopt; + return VirtualDefinitions; +} + +[[nodiscard]] Expected<void> clearDebugUses(MachineRegisterInfo &Registers, Register Reg) { + std::vector<MachineOperand *> Uses; + for (MachineOperand &Operand : Registers.useOperands(Reg, true)) { + if (Operand.isDebugUse()) + Uses.push_back(&Operand); + } + for (MachineOperand *Operand : Uses) { + auto Cleared = Operand->setRegister(Register{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineDCEPass::contract() const noexcept { + return dceContract(); +} + +Expected<MachinePassResult> MachineDCEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE requires a selected machine function"); + } + + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine DCE transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine DCE diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> MachineDCEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) { + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + + bool LocalChange = true; + while (LocalChange) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Definitions = deadDefinitions(*Instruction, Registers, Target); + if (!Definitions) + return std::unexpected(Definitions.error()); + if (!*Definitions) { + Instruction = Next; + continue; + } + + for (const Register Reg : **Definitions) { + auto Cleared = clearDebugUses(Registers, Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + auto Removed = Block->removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + for (const Register Reg : **Definitions) { + if (!Registers.useEmpty(Reg, true) || Registers.getUniqueVRegDef(Reg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine DCE left references to a removed definition"); + } + auto Erased = Registers.eraseVirtualRegister(Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + LocalChange = true; + Result.Changed = true; + Instruction = Next; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFallthrough.cpp b/backend/lib/CodeGen/MachineFallthrough.cpp new file mode 100644 index 0000000..06d70ef --- /dev/null +++ b/backend/lib/CodeGen/MachineFallthrough.cpp @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFallthrough.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <optional> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &fallthroughContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-fallthrough-normalization", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected<std::vector<MachineBasicBlock *>> +normalizeFallthroughs(MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info) { + std::vector<MachineBasicBlock *> ChangedBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + MachineBasicBlock *LayoutSuccessor = Block->next(); + if (LayoutSuccessor == nullptr) + continue; + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + continue; + bool Changed = false; + if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == LayoutSuccessor) { + continue; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->TrueTarget == LayoutSuccessor && Analysis->FalseTarget != nullptr) { + auto Reversed = Info.reverseBranchCondition(*Analysis->Conditional, *Analysis->FalseTarget, + Function, Target); + if (!Reversed) + return std::unexpected(Reversed.error()); + Changed = true; + } else if (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget != nullptr) { + auto Inserted = + Info.insertUnconditionalBranch(*Block, *Analysis->FalseTarget, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional != nullptr && + Analysis->TrueTarget == LayoutSuccessor) { + auto Removed = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } else if (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Block->successors().size() == 1 && + Block->successors().front().Block != LayoutSuccessor) { + MachineBasicBlock *Successor = Block->successors().front().Block; + if (Successor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization encountered a null successor"); + } + auto Inserted = Info.insertUnconditionalBranch(*Block, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Changed = true; + } + if (Changed) + ChangedBlocks.push_back(Block); + } + return ChangedBlocks; +} + +} // namespace + +const MachinePassContract &MachineFallthroughPass::contract() const noexcept { + return fallthroughContract(); +} + +Expected<MachinePassResult> MachineFallthroughPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "fallthrough normalization requires allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "fallthrough setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult<LiveVariablesAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialChanged = normalizeFallthroughs(*Trial->ClonedFunction, Target, *Info); + if (!TrialChanged) + return std::unexpected(TrialChanged.error()); + auto TrialRepair = repairPostRAMachineBlockLiveness(*TrialChanged, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto ChangedBlocks = normalizeFallthroughs(Function, Target, *Info); + if (!ChangedBlocks) { + return std::unexpected(prefixedError(ChangedBlocks.error(), + "fallthrough diverged after a successful preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(*ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = !ChangedBlocks->empty() || *Repaired; + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve<LiveVariablesAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFinalization.cpp b/backend/lib/CodeGen/MachineFinalization.cpp new file mode 100644 index 0000000..8523bb8 --- /dev/null +++ b/backend/lib/CodeGen/MachineFinalization.cpp @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFinalization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstdint> +#include <limits> +#include <string> +#include <unordered_map> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &finalizePseudosContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "finalize-emission-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {MachineFunctionProperty::NoExecutablePseudos}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &encodabilityContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "final-encodability-verifier", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices, + MachineFunctionProperty::NoExecutablePseudos}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits<std::uint64_t>::max() - Mask) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected<std::int64_t> displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return static_cast<std::int64_t>(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return -static_cast<std::int64_t>(Difference); +} + +} // namespace + +const MachinePassContract &FinalizeEmissionPseudosPass::contract() const noexcept { + return finalizePseudosContract(); +} + +Expected<MachinePassResult> FinalizeEmissionPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization requires finalized allocated MIR"); + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo)) + continue; + const bool NonEmitting = Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::ProducesCode); + const bool Terminal = Descriptor->Flags.has(InstructionFlag::TerminalEmission) && + Descriptor->Owner == PseudoOwner::AsmPrinter; + if (!NonEmitting && !Terminal) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "non-terminal executable pseudo survived all expansion owners"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &FinalEncodabilityVerifierPass::contract() const noexcept { + return encodabilityContract(); +} + +Expected<MachinePassResult> FinalEncodabilityVerifierPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final encodability requires NoExecutablePseudos and finalized MIR"); + + std::unordered_map<const MachineBasicBlock *, std::uint64_t> BlockOffsets; + std::unordered_map<const MachineInstr *, std::uint64_t> InstructionOffsets; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction uses an unknown opcode"); + auto InstructionValidation = + Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!InstructionValidation) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction is not encodable: " + + std::string(InstructionValidation.error().message())); + } + const bool BundledWithPred = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool BundledWithSucc = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if ((BundledWithPred && + (Instruction->previous() == nullptr || + !Instruction->previous()->flags().has(MachineInstrFlag::BundledWithSucc))) || + (BundledWithSucc && + (Instruction->next() == nullptr || + !Instruction->next()->flags().has(MachineInstrFlag::BundledWithPred)))) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction has an inconsistent bundle boundary"); + } + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && !Reg->isNoRegister() && !Reg->isPhysical()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a virtual register"); + if (Operand.kind() == MachineOperand::Kind::FrameIndex) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a frame index"); + } + auto Size = BranchInfo->instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + const bool ProducesCode = Descriptor->Flags.has(InstructionFlag::ProducesCode); + if ((ProducesCode && *Size == 0) || + (Descriptor->Flags.has(InstructionFlag::Meta) && *Size != 0)) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction violates its exact size contract"); + } + InstructionOffsets.emplace(Instruction, Offset); + if (Offset > std::numeric_limits<std::uint64_t>::max() - *Size) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine function size overflowed"); + Offset += *Size; + } + } + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto TargetBlock = BranchInfo->branchTarget(*Instruction, Function, Target); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + if (*TargetBlock == nullptr) + continue; + const auto Source = InstructionOffsets.find(Instruction); + const auto Destination = BlockOffsets.find(*TargetBlock); + if (Source == InstructionOffsets.end() || Destination == BlockOffsets.end()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch target has no layout offset"); + auto Distance = displacement(Source->second, Destination->second); + if (!Distance) + return std::unexpected(Distance.error()); + if (!BranchInfo->isBranchOffsetInRange(*Instruction, *Distance)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction contains an out-of-range branch"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunction.cpp b/backend/lib/CodeGen/MachineFunction.cpp index df1ceb5..7f4c3a9 100644 --- a/backend/lib/CodeGen/MachineFunction.cpp +++ b/backend/lib/CodeGen/MachineFunction.cpp @@ -172,6 +172,66 @@ Expected<void> MachineFunction::insertBlockBefore(MachineBasicBlock &Position, return {}; } +Expected<void> MachineFunction::moveBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Block.NextBlock == &Position) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = Position.PreviousBlock; + Block.NextBlock = &Position; + if (Position.PreviousBlock != nullptr) + Position.PreviousBlock->NextBlock = &Block; + else + FirstBlock = &Block; + Position.PreviousBlock = &Block; + renumberBlocks(); + return {}; +} + +Expected<void> MachineFunction::moveBlockAfter(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Position.NextBlock == &Block) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = &Position; + Block.NextBlock = Position.NextBlock; + if (Position.NextBlock != nullptr) + Position.NextBlock->PreviousBlock = &Block; + else + LastBlock = &Block; + Position.NextBlock = &Block; + renumberBlocks(); + return {}; +} + Expected<void> MachineFunction::removeBlock(MachineBasicBlock &Block) { if (Block.OwnerFunction != this || !Block.IsLinked) return makeUnexpected(ErrorCode::InvalidArgument, diff --git a/backend/lib/CodeGen/MachineFunctionRecords.cpp b/backend/lib/CodeGen/MachineFunctionRecords.cpp index 0a21581..18ac596 100644 --- a/backend/lib/CodeGen/MachineFunctionRecords.cpp +++ b/backend/lib/CodeGen/MachineFunctionRecords.cpp @@ -127,6 +127,38 @@ Expected<void> MachineFunctionRecords::addCalledGlobal(MachineCalledGlobal Calle return {}; } +Expected<void> +MachineFunctionRecords::replaceInstructionReference(MachineInstrStableID OldInstruction, + MachineInstrStableID NewInstruction) { + if (!OldInstruction.isValid() || !NewInstruction.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction-record replacement requires two valid stable IDs"); + } + if (OldInstruction == NewInstruction) + return {}; + for (const MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a call-site record"); + } + } + for (const MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a called-global record"); + } + } + for (MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == OldInstruction) + CallSite.CallInstruction = NewInstruction; + } + for (MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == OldInstruction) + CalledGlobal.CallInstruction = NewInstruction; + } + return {}; +} + Expected<void> MachineFunctionRecords::addPrefetchTarget(MachinePrefetchTarget Target) { if (std::ranges::find(PrefetchTargets, Target) != PrefetchTargets.end()) { return makeUnexpected(ErrorCode::InvalidArgument, "prefetch target is already recorded"); diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp new file mode 100644 index 0000000..8c8d732 --- /dev/null +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineLICM.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &licmContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-licm", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineLoopAnalysis>(), + MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-licm", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineLoopAnalysis>(), + MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool containsBlock(const MachineLoop &Loop, const MachineBasicBlock *Block) noexcept { + return std::ranges::find(Loop.Blocks, Block) != Loop.Blocks.end(); +} + +[[nodiscard]] MachineBasicBlock *preheader(const MachineLoop &Loop) noexcept { + if (Loop.Header == nullptr) + return nullptr; + MachineBasicBlock *Result = nullptr; + for (MachineBasicBlock *Predecessor : Loop.Header->predecessors()) { + if (containsBlock(Loop, Predecessor)) + continue; + if (Result != nullptr) + return nullptr; + Result = Predecessor; + } + if (Result == nullptr || Result->successors().size() != 1 || + Result->successors().front().Block != Loop.Header) { + return nullptr; + } + return Result; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected<bool> isSafeToHoist(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +[[nodiscard]] std::size_t definitionCount(const MachineInstr &Instruction) noexcept { + return static_cast<std::size_t>(std::ranges::count_if( + Instruction.operands(), [](const MachineOperand &Operand) { return Operand.isDef(); })); +} + +[[nodiscard]] bool +operandsAreInvariant(const MachineInstr &Instruction, const MachineLoop &Loop, + const MachineRegisterInfo &Registers, + const std::unordered_set<const MachineInstr *> &Hoisted) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Operand.isUse()) + continue; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Registers.getUniqueVRegDef(*Reg); + const MachineInstr *DefiningInstruction = + Definition == nullptr ? nullptr : Definition->parent(); + const MachineBasicBlock *DefiningBlock = + DefiningInstruction == nullptr ? nullptr : DefiningInstruction->parent(); + if (DefiningInstruction == nullptr || DefiningBlock == nullptr) + return false; + if (containsBlock(Loop, DefiningBlock) && !Hoisted.contains(DefiningInstruction)) + return false; + } + return true; +} + +[[nodiscard]] bool postRAOperandsAreInvariant(const MachineInstr &Instruction, + const MachineLoop &Loop, + const MachineFunction &Function, + const TargetDescription &Target) noexcept { + if (Loop.Header == nullptr || Instruction.parent() != Loop.Header) + return false; + + std::vector<PhysicalRegister> Uses; + std::vector<PhysicalRegister> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return false; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return false; + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (std::ranges::any_of(Loop.Header->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return false; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return false; + } + } + + const auto ClobbersCandidate = [&](const RegisterMask &Mask) { + return std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }); + }; + for (const MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + return false; + for (const MachineInstr *Candidate = Block->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + if (Candidate == &Instruction) + continue; + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (ClobbersCandidate(*Mask->Mask)) + return false; + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand.isDef()) + continue; + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(*Physical, Use); + }) || + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + } + + for (const MachineInstr *Before = Loop.Header->firstInstruction(); + Before != nullptr && Before != &Instruction; Before = Before->next()) { + for (const MachineOperand &Operand : Before->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return false; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && (Operand.isUse() || Operand.isDef()) && + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + return true; +} + +[[nodiscard]] Expected<MachineInstr *> firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM preheader contains an unknown instruction"); + } + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] Expected<void> moveToPreheader(MachineInstr &Instruction, + MachineBasicBlock &Preheader, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine LICM candidate is detached"); + auto Anchor = firstTerminator(Preheader, Target); + if (!Anchor) + return std::unexpected(Anchor.error()); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + if (*Anchor != nullptr) + return Preheader.insertInstructionBefore(**Anchor, Instruction); + return Preheader.appendInstruction(Instruction); +} + +} // namespace + +const MachinePassContract &MachineLICMPass::contract() const noexcept { + return licmContract(Stage); +} + +Expected<MachinePassResult> MachineLICMPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine LICM requires selected SSA MachineIR" + : "post-RA machine LICM requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine LICM transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine LICM diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> MachineLICMPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t TotalHoisted = 0; + for (const MachineLoop &Loop : (*Loops)->loops()) { + MachineBasicBlock *Preheader = preheader(Loop); + if (Preheader == nullptr || Loop.Header == nullptr || + (*Frequencies)->frequency(*Preheader) > (*Frequencies)->frequency(*Loop.Header)) { + continue; + } + std::unordered_set<const MachineInstr *> Hoisted; + std::size_t HoistedDefinitions = 0; + bool LocalChange = true; + while (LocalChange && TotalHoisted < Options.MaximumHoistedInstructions) { + LocalChange = false; + for (MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + continue; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToHoist(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + const std::size_t Definitions = definitionCount(*Instruction); + const bool Invariant = + Stage == MachineCodeMotionStage::SelectedSSA + ? operandsAreInvariant(*Instruction, Loop, Function.registerInfo(), Hoisted) + : postRAOperandsAreInvariant(*Instruction, Loop, Function, Target); + if (*Safe && Invariant && + Definitions <= + Options.MaximumHoistedDefinitionsPerLoop - + std::min(HoistedDefinitions, Options.MaximumHoistedDefinitionsPerLoop)) { + auto Moved = moveToPreheader(*Instruction, *Preheader, Target); + if (!Moved) + return std::unexpected(Moved.error()); + Hoisted.insert(Instruction); + HoistedDefinitions += Definitions; + ++TotalHoisted; + LocalChange = true; + Result.Changed = true; + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + Instruction = Next; + } + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineLoopAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMemoryFolding.cpp b/backend/lib/CodeGen/MachineMemoryFolding.cpp new file mode 100644 index 0000000..0e15146 --- /dev/null +++ b/backend/lib/CodeGen/MachineMemoryFolding.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMemoryFolding.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &memoryFoldingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-memory-folding", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct FoldCandidate final { + MachineInstr *MemoryInstruction = nullptr; + MachineInstr *Consumer = nullptr; + Register DefinedRegister; + std::size_t ConsumerOperand = 0; +}; + +[[nodiscard]] std::optional<FoldCandidate> findCandidate(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::MayLoad) || + Descriptor->Flags.has(InstructionFlag::MayStore) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Barrier) || + Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects) || + hasProtectedAttachment(Instruction) || Instruction.memOperands().empty()) { + return std::nullopt; + } + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) { + return std::nullopt; + } + } + + MachineOperand *Definition = nullptr; + Register DefinedRegister; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Operand->isDef()) + continue; + if (Definition != nullptr || !Reg->isVirtual()) + return std::nullopt; + Definition = Operand; + DefinedRegister = *Reg; + } + if (Definition == nullptr || !Function.registerInfo().hasOneUse(DefinedRegister, true)) { + return std::nullopt; + } + MachineOperand *Use = nullptr; + for (MachineOperand &Candidate : Function.registerInfo().useOperands(DefinedRegister, false)) { + Use = &Candidate; + break; + } + MachineInstr *Consumer = Use == nullptr ? nullptr : Use->parent(); + if (Consumer == nullptr || Consumer->parent() != Instruction.parent() || + Instruction.next() != Consumer || hasProtectedAttachment(*Consumer) || Use->isImplicit() || + Use->tiedOperandIndex()) { + return std::nullopt; + } + for (std::size_t Index = 0; Index < Consumer->operandCount(); ++Index) { + if (Consumer->operandAt(Index) == Use) + return FoldCandidate{&Instruction, Consumer, DefinedRegister, Index}; + } + return std::nullopt; +} + +[[nodiscard]] bool containsMemoryDescription(const MachineInstr &Instruction, + const MachineMemOperandDesc &Description) noexcept { + return std::ranges::any_of(Instruction.memOperands(), + [&Description](const MachineMemOperand *Memory) { + return Memory != nullptr && Memory->description() == Description; + }); +} + +[[nodiscard]] Expected<bool> applyFold(FoldCandidate Candidate, MachineFunction &Function, + const TargetDescription &Target, + const TargetMemoryFoldingInfo &Info) { + auto CanFold = Info.canFoldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!CanFold) + return std::unexpected(CanFold.error()); + if (!*CanFold) + return false; + std::vector<MachineMemOperandDesc> MemoryDescriptions; + for (const MachineMemOperand *Memory : Candidate.MemoryInstruction->memOperands()) { + if (Memory == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding encountered a null memory operand"); + } + MemoryDescriptions.push_back(Memory->description()); + } + auto Folded = Info.foldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!Folded) + return std::unexpected(Folded.error()); + if (*Folded == nullptr || (*Folded)->parent() == nullptr || + Candidate.MemoryInstruction->parent() != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not replace the consumer and erase the producer"); + } + for (const MachineMemOperandDesc &Description : MemoryDescriptions) { + if (!containsMemoryDescription(**Folded, Description)) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not migrate every memory operand"); + } + } + auto FoldedSize = Info.getFoldedMemorySize(**Folded, Function, Target); + if (!FoldedSize) + return std::unexpected(FoldedSize.error()); + if (!FoldedSize->has_value() || !FoldedSize->value().isKnown()) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded instruction has no known folded memory size"); + } + if (MemoryDescriptions.size() == 1 && MemoryDescriptions.front().Size.isKnown() && + FoldedSize->value() != MemoryDescriptions.front().Size) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded memory size disagrees with the migrated MMO"); + } + if (!Function.registerInfo().useEmpty(Candidate.DefinedRegister, true) || + Function.registerInfo().getUniqueVRegDef(Candidate.DefinedRegister) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold retained the load result register"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Candidate.DefinedRegister); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineMemoryFoldingPass::contract() const noexcept { + return memoryFoldingContract(); +} + +Expected<MachinePassResult> MachineMemoryFoldingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding requires selected SSA MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "memory folding transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "memory folding diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineMemoryFoldingPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t FoldCount = 0; + bool LocalChange = true; + while (LocalChange && FoldCount < Options.MaximumFolds) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !LocalChange; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + std::optional<FoldCandidate> Candidate = findCandidate(*Instruction, Function, Target); + if (!Candidate) + continue; + auto Folded = applyFold(*Candidate, Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + if (!*Folded) + continue; + ++FoldCount; + Result.Changed = true; + LocalChange = true; + break; + } + } + } + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp index 43a4f91..d1d152e 100644 --- a/backend/lib/CodeGen/MachineOperand.cpp +++ b/backend/lib/CodeGen/MachineOperand.cpp @@ -548,6 +548,18 @@ Expected<void> MachineOperand::setBasicBlock(MachineBasicBlock &Block) { return {}; } +Expected<void> MachineOperand::setFrameIndex(FrameIndex Index) { + if (OperandKind != Kind::FrameIndex) + return makeUnexpected(ErrorCode::InvalidArgument, + "setFrameIndex requires a frame-index operand"); + if (Parent != nullptr && Parent->function().frameInfo().object(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frame-index operand references an unknown stack object"); + } + std::get<FrameIndexOperand>(Storage).Index = Index; + return {}; +} + MachineOperand MachineOperand::cloneDetached() const { MachineOperand Result(OperandKind, Storage, TargetFlags); Result.RegFlags = RegFlags; diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp new file mode 100644 index 0000000..76dbd44 --- /dev/null +++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOptimizationPipeline.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBlockPlacement.h" +#include "rcc/CodeGen/MachineBranchOptimization.h" +#include "rcc/CodeGen/MachineBranchRelaxation.h" +#include "rcc/CodeGen/MachineCSE.h" +#include "rcc/CodeGen/MachineCombiner.h" +#include "rcc/CodeGen/MachineCopyPropagation.h" +#include "rcc/CodeGen/MachineDCE.h" +#include "rcc/CodeGen/MachineFallthrough.h" +#include "rcc/CodeGen/MachineFinalization.h" +#include "rcc/CodeGen/MachineLICM.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachinePseudoExpansion.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/MachineShrinkWrapping.h" +#include "rcc/CodeGen/MachineSink.h" +#include "rcc/CodeGen/MachineStackSlotColoring.h" +#include "rcc/CodeGen/MachineTailDuplication.h" +#include "rcc/CodeGen/MachineTailMerge.h" +#include "rcc/CodeGen/PostRAScheduler.h" +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" + +#include <memory> +#include <utility> + +namespace rcc { +namespace { + +template <typename Pass, typename... Arguments> +[[nodiscard]] Expected<void> addPass(MachinePassManager &Pipeline, Arguments &&...Values) { + return Pipeline.addPass(std::make_unique<Pass>(std::forward<Arguments>(Values)...)); +} + +} // namespace + +Expected<void> registerMachineOptimizationAnalyses(MachineAnalysisManager &Analyses) { + auto Registered = registerStage8MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + return registerStage11MachineAnalyses(Analyses); +} + +Expected<void> +addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass<MachineCombinerPass>(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineMemoryFoldingPass>(Pipeline, TargetInfo.MemoryFolding); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCSEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineDCEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineLICMPass>(Pipeline, MachineCodeMotionStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + return addPass<MachineSinkPass>(Pipeline, MachineCodeMotionStage::SelectedSSA); +} + +Expected<void> addPreRegisterAllocationOptimizationPipeline( + MachinePassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass<MachineBranchOptimizationPass>(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCopyPropagationPass>(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCombinerPass>(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PreRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineDCEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass<MachineSchedulerPass>(Pipeline, TargetInfo.SchedulingModel); +} + +Expected<void> +addPostRegisterAllocationPipeline(MachinePassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass<MachineStackSlotColoringPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCopyPropagationPass>(Pipeline, TargetInfo.SSA, + MachineCopyPropagationStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCombinerPass>(Pipeline, TargetInfo.Combiner, + MachineCombinerStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineLICMPass>(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineSinkPass>(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineShrinkWrappingPass>(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<PrePEITargetPseudoExpansionPass>(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<PrologueEpilogueInsertionPass>(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = + addPass<MachineCombinerPass>(Pipeline, TargetInfo.Combiner, MachineCombinerStage::PostFrame); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineDCEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<ExpandPostRAPseudosPass>(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<PostRASchedulerPass>(Pipeline, TargetInfo.SchedulingModel); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineTailDuplicationPass>(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineTailMergePass>(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineBlockPlacementPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineFallthroughPass>(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineBranchRelaxationPass>(Pipeline, TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<TargetLatePseudoExpansionPass>(Pipeline, TargetInfo.PseudoExpansion, + TargetInfo.BranchInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<FinalizeEmissionPseudosPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass<FinalEncodabilityVerifierPass>(Pipeline, TargetInfo.BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePseudoExpansion.cpp b/backend/lib/CodeGen/MachinePseudoExpansion.cpp new file mode 100644 index 0000000..e654d97 --- /dev/null +++ b/backend/lib/CodeGen/MachinePseudoExpansion.cpp @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePseudoExpansion.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using CFGEdge = std::tuple<std::uint32_t, std::uint32_t, std::uint32_t>; + +struct ResourceSnapshot final { + std::size_t BlockCount = 0; + std::size_t VirtualRegisterCount = 0; + std::size_t FrameObjectCount = 0; + bool FrameFrozen = false; + std::vector<CFGEdge> Edges; +}; + +using BlockSizeSnapshot = std::vector<std::pair<MachineBlockStableID, std::uint64_t>>; + +[[nodiscard]] Expected<std::uint64_t> blockSize(const MachineBasicBlock &Block, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + std::uint64_t Size = 0; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto InstructionSize = BranchInfo.instructionSize(*Instruction, Function, Target); + if (!InstructionSize) + return std::unexpected(InstructionSize.error()); + if (Size > std::numeric_limits<std::uint64_t>::max() - *InstructionSize) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, "target-late pseudo size overflowed"); + } + Size += *InstructionSize; + } + return Size; +} + +[[nodiscard]] Expected<BlockSizeSnapshot> snapshotBlockSizes(const MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &BranchInfo) { + BlockSizeSnapshot Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Size = blockSize(*Block, Function, Target, BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + Result.emplace_back(Block->stableID(), *Size); + } + return Result; +} + +[[nodiscard]] ResourceSnapshot snapshotResources(const MachineFunction &Function) { + ResourceSnapshot Result; + Result.BlockCount = Function.blockCount(); + Result.VirtualRegisterCount = Function.registerInfo().virtualRegisters().size(); + Result.FrameObjectCount = Function.frameInfo().objects().size(); + Result.FrameFrozen = Function.frameInfo().isFrozen(); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineSuccessor &Successor : Block->successors()) { + Result.Edges.emplace_back(Block->stableID().value(), Successor.Block->stableID().value(), + Successor.Probability.rawValue()); + } + } + return Result; +} + +[[nodiscard]] Expected<void> verifyResourceContract(const ResourceSnapshot &Before, + const MachineFunction &Function, + MachinePseudoExpansionPhase Phase) { + const ResourceSnapshot After = snapshotResources(Function); + if (Before.BlockCount != After.BlockCount || Before.Edges != After.Edges) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion changed CFG resources outside its owner contract"); + } + if (Before.VirtualRegisterCount != After.VirtualRegisterCount) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-selection pseudo expansion created a virtual register"); + } + if (Phase != MachinePseudoExpansionPhase::PrePEI && + (Before.FrameObjectCount != After.FrameObjectCount || + Before.FrameFrozen != After.FrameFrozen)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-PEI pseudo expansion changed frozen frame resources"); + } + return {}; +} + +[[nodiscard]] PseudoOwner ownerFor(MachinePseudoExpansionPhase Phase) noexcept { + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PseudoOwner::PrePEIExpansion; + case MachinePseudoExpansionPhase::PostRA: + return PseudoOwner::PostRAExpansion; + case MachinePseudoExpansionPhase::TargetLate: + return PseudoOwner::TargetLateExpansion; + } + return PseudoOwner::None; +} + +[[nodiscard]] const MachinePassContract &preparationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "prepare-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &phaseContract(MachinePseudoExpansionPhase Phase) { + static const MachinePassContract PrePEI = [] { + MachinePassContract Value{ + .Name = "pre-pei-target-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Frame}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "expand-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + static const MachinePassContract TargetLate = [] { + MachinePassContract Value{ + .Name = "target-late-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + switch (Phase) { + case MachinePseudoExpansionPhase::PrePEI: + return PrePEI; + case MachinePseudoExpansionPhase::PostRA: + return PostRA; + case MachinePseudoExpansionPhase::TargetLate: + return TargetLate; + } + return TargetLate; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] Expected<MachinePassResult> +prepareImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + bool Changed = false; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo preparation encountered an unknown opcode"); + if (Descriptor->Owner != PseudoOwner::PostRAExpansion) + continue; + auto Prepared = Info.preparePostRAPseudo(*Instruction, Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + Changed |= *Prepared; + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected<MachinePassResult> +expandImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + const ResourceSnapshot Before = snapshotResources(Function); + std::optional<BlockSizeSnapshot> SizesBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + if (BranchInfo == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target-late pseudo expansion requires branch sizing"); + } + auto Snapshot = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!Snapshot) + return std::unexpected(Snapshot.error()); + SizesBefore = std::move(*Snapshot); + } + std::vector<MachineInstr *> Worklist; + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction != nullptr && Instruction->parent() != nullptr) + Worklist.push_back(Instruction); + } + bool Changed = false; + for (MachineInstr *Instruction : Worklist) { + if (Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo expansion encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo) || Descriptor->Owner != ownerFor(Phase)) { + continue; + } + MachineBasicBlock *ExpansionBlock = Instruction->parent(); + std::optional<std::uint64_t> ExpansionSizeBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + auto Size = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + ExpansionSizeBefore = *Size; + } + auto Expanded = Info.expandPseudo(*Instruction, Phase, Function, Target); + if (!Expanded) + return std::unexpected(Expanded.error()); + if (!*Expanded) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target has no expansion for an owned executable pseudo"); + } + if (ExpansionSizeBefore.has_value()) { + auto SizeAfter = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!SizeAfter) + return std::unexpected(SizeAfter.error()); + if (*SizeAfter > *ExpansionSizeBefore) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "one target-late pseudo expansion increased its encoded block size"); + } + } + Changed = true; + } + + auto Resources = verifyResourceContract(Before, Function, Phase); + if (!Resources) + return std::unexpected(Resources.error()); + if (SizesBefore.has_value()) { + auto SizesAfter = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!SizesAfter) + return std::unexpected(SizesAfter.error()); + if (SizesAfter->size() != SizesBefore->size()) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion changed block layout"); + } + for (std::size_t Index = 0; Index < SizesAfter->size(); ++Index) { + if ((*SizesAfter)[Index].first != (*SizesBefore)[Index].first || + (*SizesAfter)[Index].second > (*SizesBefore)[Index].second) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion increased an encoded block size"); + } + } + } + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Pseudo) && + Descriptor->Owner == ownerFor(Phase)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion left an instruction owned by the completed phase"); + } + } + if (Changed && Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected<MachinePassResult> +runPreparationTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo preparation setup failed: ")); + auto TrialResult = prepareImplementation(*Trial->ClonedFunction, Target, Info); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = prepareImplementation(Function, Target, Info); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo preparation diverged after successful preflight: ")); + } + return Result; +} + +[[nodiscard]] Expected<MachinePassResult> +runExpansionTransaction(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetBranchInfo *BranchInfo) { + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "pseudo expansion setup failed: ")); + auto TrialResult = expandImplementation(*Trial->ClonedFunction, Target, Info, Phase, BranchInfo); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = expandImplementation(Function, Target, Info, Phase, BranchInfo); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), "pseudo expansion diverged after successful preflight: ")); + } + return Result; +} + +} // namespace + +const MachinePassContract &PreparePostRAPseudosPass::contract() const noexcept { + return preparationContract(); +} + +Expected<MachinePassResult> PreparePostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo preparation requires selected MachineIR"); + return runPreparationTransaction(Function, Target, *Info); +} + +const MachinePassContract &PrePEITargetPseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PrePEI); +} + +Expected<MachinePassResult> PrePEITargetPseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "pre-PEI pseudo expansion requires allocated tracked-liveness MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PrePEI, + nullptr); +} + +const MachinePassContract &ExpandPostRAPseudosPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PostRA); +} + +Expected<MachinePassResult> ExpandPostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pseudo expansion requires finalized frame and liveness"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::PostRA, + nullptr); +} + +const MachinePassContract &TargetLatePseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::TargetLate); +} + +Expected<MachinePassResult> TargetLatePseudoExpansionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) + return makeUnexpected(ErrorCode::VerificationError, + "target-late pseudo expansion requires finalized allocated MIR"); + return runExpansionTransaction(Function, Target, *Info, MachinePseudoExpansionPhase::TargetLate, + BranchInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineScheduler.cpp b/backend/lib/CodeGen/MachineScheduler.cpp new file mode 100644 index 0000000..f861be2 --- /dev/null +++ b/backend/lib/CodeGen/MachineScheduler.cpp @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/RegisterPressure.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &schedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +[[nodiscard]] Expected<bool> isSchedulingBoundary(const MachineInstr &Instruction, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Model.hooks().isSchedulingBoundary(Instruction, Target)) + return true; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi)) { + return true; + } + if (Model.schedulingClass(Descriptor->SchedulingClass) == nullptr) { + if (Flags.has(InstructionFlag::Pseudo) || !Flags.has(InstructionFlag::ProducesCode)) + return true; + return makeUnexpected(ErrorCode::VerificationError, + "executable instruction has no scheduling class"); + } + return false; +} + +struct ScheduleCandidate final { + std::size_t Unit = 0; + bool Bottom = false; + std::uint64_t Cycle = 0; + RegisterPressureCost Pressure; + std::uint64_t CriticalPath = 0; + std::uint32_t Latency = 0; + std::int64_t TargetPreference = 0; + bool OptimizeLatency = false; + std::uint64_t StableID = 0; +}; + +[[nodiscard]] bool isBetterCandidate(const ScheduleCandidate &Left, + const ScheduleCandidate &Right) noexcept { + if (Left.Cycle != Right.Cycle) + return Left.Cycle < Right.Cycle; + if (Left.TargetPreference != Right.TargetPreference) + return Left.TargetPreference > Right.TargetPreference; + if (Left.OptimizeLatency && Right.OptimizeLatency) { + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + } + if (Left.Pressure.PeakIncrease != Right.Pressure.PeakIncrease) + return Left.Pressure.PeakIncrease < Right.Pressure.PeakIncrease; + if (Left.Pressure.PeakExcess != Right.Pressure.PeakExcess) + return Left.Pressure.PeakExcess < Right.Pressure.PeakExcess; + if (Left.Pressure.CurrentExcess != Right.Pressure.CurrentExcess) + return Left.Pressure.CurrentExcess < Right.Pressure.CurrentExcess; + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + if (Left.StableID != Right.StableID) + return Left.StableID < Right.StableID; + return !Left.Bottom && Right.Bottom; +} + +[[nodiscard]] Expected<std::vector<std::size_t>> +scheduleDAG(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + std::uint64_t BlockFrequency, bool EnableBidirectionalScheduling) { + auto TopPressure = RegisterPressureTracker::create(DAG, Function, Target, Model, Mode, Liveness, + RegisterPressureDirection::TopDown); + if (!TopPressure) + return std::unexpected(TopPressure.error()); + auto BottomPressure = RegisterPressureTracker::create( + DAG, Function, Target, Model, Mode, Liveness, RegisterPressureDirection::BottomUp); + if (!BottomPressure) + return std::unexpected(BottomPressure.error()); + auto TopHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::TopDown); + if (!TopHazards) + return std::unexpected(TopHazards.error()); + auto BottomHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::BottomUp); + if (!BottomHazards) + return std::unexpected(BottomHazards.error()); + + const std::size_t UnitCount = DAG.units().size(); + std::vector<std::size_t> TopPredecessors(UnitCount); + std::vector<std::size_t> BottomSuccessors(UnitCount); + std::vector<std::uint64_t> TopReadyCycle(UnitCount, 0); + std::vector<std::uint64_t> BottomReadyCycle(UnitCount, 0); + std::vector<bool> Scheduled(UnitCount, false); + for (const ScheduleUnit &Unit : DAG.units()) { + TopPredecessors[Unit.ID] = Unit.Predecessors.size(); + BottomSuccessors[Unit.ID] = Unit.Successors.size(); + } + + std::vector<std::size_t> TopOrder; + std::vector<std::size_t> BottomOrder; + TopOrder.reserve(UnitCount); + BottomOrder.reserve(UnitCount); + for (std::size_t ScheduledCount = 0; ScheduledCount < UnitCount; ++ScheduledCount) { + std::optional<ScheduleCandidate> Best; + for (const ScheduleUnit &Unit : DAG.units()) { + if (Scheduled[Unit.ID]) + continue; + if (TopPredecessors[Unit.ID] == 0) { + auto Cycle = TopHazards->earliestIssueCycle(Unit, TopReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = TopPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + false, + *Cycle, + *Pressure, + Unit.Height, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::TopDown, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + if (EnableBidirectionalScheduling && BottomSuccessors[Unit.ID] == 0) { + auto Cycle = BottomHazards->earliestIssueCycle(Unit, BottomReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = BottomPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + true, + *Cycle, + *Pressure, + Unit.Depth, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::BottomUp, DAG.block(), + BlockFrequency), + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.StableID.value()}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + } + if (!Best) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "schedule DAG has no legal top or bottom ready unit"); + } + + const ScheduleUnit &Selected = DAG.units()[Best->Unit]; + Scheduled[Best->Unit] = true; + if (Best->Bottom) { + auto Issued = BottomHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = BottomPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + BottomOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Predecessors) { + if (BottomSuccessors[Dependency.Source] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "bottom scheduler successor count underflowed"); + } + --BottomSuccessors[Dependency.Source]; + BottomReadyCycle[Dependency.Source] = std::max( + BottomReadyCycle[Dependency.Source], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } else { + auto Issued = TopHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = TopPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + TopOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Successors) { + if (TopPredecessors[Dependency.Destination] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "top scheduler predecessor count underflowed"); + } + --TopPredecessors[Dependency.Destination]; + TopReadyCycle[Dependency.Destination] = std::max( + TopReadyCycle[Dependency.Destination], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } + } + + TopOrder.insert(TopOrder.end(), BottomOrder.rbegin(), BottomOrder.rend()); + if (TopOrder.size() != UnitCount) + return makeUnexpected(ErrorCode::InternalError, "scheduler lost a schedule unit"); + return TopOrder; +} + +[[nodiscard]] Expected<bool> +scheduleRegion(MachineBasicBlock &Block, std::span<MachineInstr *const> Instructions, + MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const MachineBlockLiveness &Liveness, std::uint64_t BlockFrequency, + MachineSchedulerOptions Options) { + if (Instructions.size() < 2 || Instructions.size() > Options.MaximumRegionSize) + return false; + auto DAG = ScheduleDAG::build(Block, Instructions, Function, Target, Model, Mode); + if (!DAG) + return std::unexpected(DAG.error()); + if (DAG->units().size() < 2) + return false; + auto UnitOrder = scheduleDAG(*DAG, Function, Target, Model, Mode, Liveness, BlockFrequency, + Options.EnableBidirectionalScheduling); + if (!UnitOrder) + return std::unexpected(UnitOrder.error()); + + std::vector<MachineInstr *> NewOrder; + NewOrder.reserve(Instructions.size()); + for (const std::size_t UnitID : *UnitOrder) { + const ScheduleUnit *Unit = DAG->unit(UnitID); + if (Unit == nullptr) + return makeUnexpected(ErrorCode::InternalError, "scheduler produced an unknown unit ID"); + NewOrder.insert(NewOrder.end(), Unit->Instructions.begin(), Unit->Instructions.end()); + } + if (std::ranges::equal(NewOrder, Instructions)) + return false; + + MachineInstr *Anchor = Instructions.back()->next(); + for (MachineInstr *Instruction : Instructions) { + auto Removed = Block.removeInstruction(*Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + } + for (MachineInstr *Instruction : NewOrder) { + Expected<void> Inserted = Anchor == nullptr + ? Block.appendInstruction(*Instruction) + : Block.insertInstructionBefore(*Anchor, *Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + return true; +} + +} // namespace + +Expected<MachineSchedulingRunResult> runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options) { + if (Options.MaximumRegionSize == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine scheduler region budget must be nonzero"); + } + MachineSchedulingRunResult Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const MachineBlockLiveness *BlockLiveness = Liveness.block(*Block); + if (BlockLiveness == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine scheduler has no block liveness boundary"); + } + bool BlockChanged = false; + std::vector<MachineInstr *> Region; + auto FlushRegion = [&]() -> Expected<void> { + auto Changed = scheduleRegion(*Block, Region, Function, Target, Model, Mode, *BlockLiveness, + Frequencies.frequency(*Block), Options); + if (!Changed) + return std::unexpected(Changed.error()); + BlockChanged |= *Changed; + Region.clear(); + return {}; + }; + + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a broken bundle header"); + } + std::vector<MachineInstr *> Group; + bool Boundary = false; + for (;;) { + Group.push_back(Instruction); + auto IsBoundary = isSchedulingBoundary(*Instruction, Target, Model); + if (!IsBoundary) + return std::unexpected(IsBoundary.error()); + Boundary |= *IsBoundary; + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!HasSuccessor) + break; + MachineInstr *Next = Instruction->next(); + if (Next == nullptr || !Next->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a truncated bundle"); + } + Instruction = Next; + } + MachineInstr *NextGroup = Instruction->next(); + if (Boundary) { + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + } else { + Region.insert(Region.end(), Group.begin(), Group.end()); + } + Instruction = NextGroup; + } + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + if (BlockChanged) { + Result.Changed = true; + Result.ChangedBlocks.push_back(Block); + } + } + return Result; +} + +const MachinePassContract &MachineSchedulerPass::contract() const noexcept { + return schedulerContract(); +} + +Expected<MachinePassResult> MachineSchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA machine scheduler requires selected, SSA-exited MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "machine scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult<LiveVariablesAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult<MachineBlockFrequencyAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialResult = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PreRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PreRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "machine scheduler diverged after a successful transaction preflight: ")); + } + + MachinePassResult Result; + Result.Changed = Run->Changed; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSchedulerInternal.h b/backend/lib/CodeGen/MachineSchedulerInternal.h new file mode 100644 index 0000000..a612548 --- /dev/null +++ b/backend/lib/CodeGen/MachineSchedulerInternal.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H +#define RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/Support/Error.h" + +#include <vector> + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class TargetDescription; +class TargetSchedulingModel; +struct MachineSchedulingRunResult final { + bool Changed = false; + std::vector<MachineBasicBlock *> ChangedBlocks; +}; + +[[nodiscard]] Expected<MachineSchedulingRunResult> runMachineScheduler( + MachineFunction &Function, const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, MachineSchedulerOptions Options); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H diff --git a/backend/lib/CodeGen/MachineShrinkWrapping.cpp b/backend/lib/CodeGen/MachineShrinkWrapping.cpp new file mode 100644 index 0000000..aee3323 --- /dev/null +++ b/backend/lib/CodeGen/MachineShrinkWrapping.cpp @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineShrinkWrapping.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <unordered_set> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &shrinkWrappingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-shrink-wrapping", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineLoopAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool requiresEntryFrame(const MachineFrameState &State) noexcept { + return State.IsFrameAddressTaken || State.IsReturnAddressTaken || State.HasStackMap || + State.HasPatchPoint || State.HasOpaqueSPAdjustment || State.HasVAStart || + State.HasMustTailInVarArgFunc || State.HasTailCall; +} + +[[nodiscard]] bool containsEpilogueTerminator(const MachineBasicBlock &Block, + const TargetFrameLowering &Info, + const TargetDescription &Target) noexcept { + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Info.isEpilogueTerminator(*Instruction, Target)) + return true; + } + return false; +} + +} // namespace + +const MachinePassContract &MachineShrinkWrappingPass::contract() const noexcept { + return shrinkWrappingContract(); +} + +Expected<MachinePassResult> MachineShrinkWrappingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping requires allocated tracked-liveness MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "shrink wrapping must run before frame layout freeze"); + } + if (Function.blockCount() > Options.MaximumAnalyzedBlocks) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineFrameState &OriginalState = Function.frameInfo().state(); + if (!OriginalState.SavePoints.empty() || !OriginalState.RestorePoints.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr || requiresEntryFrame(OriginalState)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + + std::vector<MachineBasicBlock *> RequiredBlocks; + std::unordered_set<FrameIndex> ReferencedObjects; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + bool Required = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping encountered an unknown opcode"); + } + Required |= Descriptor->Flags.has(InstructionFlag::Call); + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Frame = Operand.getIf<FrameIndexOperand>()) { + ReferencedObjects.insert(Frame->Index); + Required = true; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && Operand.isDef() && + Info->isCalleeSavedRegister(*Physical, Function, Target)) { + Required = true; + } + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + const std::optional<FrameIndex> Frame = + Memory == nullptr ? std::nullopt : Memory->description().Base.frameIndex(); + if (Frame) { + ReferencedObjects.insert(*Frame); + Required = true; + } + } + } + if (Required) + RequiredBlocks.push_back(Block); + } + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || Description->IsDead) + continue; + if (!ReferencedObjects.contains(Object.index())) { + RequiredBlocks.push_back(Entry); + break; + } + } + if (RequiredBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineBasicBlock *Save = RequiredBlocks.front(); + if (!(*Dominators)->isReachable(*Save)) + Save = Entry; + for (MachineBasicBlock *Block : RequiredBlocks) { + if (!(*Dominators)->isReachable(*Block)) { + Save = Entry; + break; + } + const MachineBasicBlock *Common = (*Dominators)->nearestCommonDominator(*Save, *Block); + if (Common == nullptr) { + Save = Entry; + break; + } + Save = Common; + } + if (Save == Entry || Save->predecessors().size() != 1 || (*Loops)->loopDepth(*Save) != 0 || + (*Loops)->isIrreducible(*Save)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + std::vector<MachineBasicBlock *> RestoreBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!(*Dominators)->dominates(*Save, *Block)) + continue; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || !(*Dominators)->dominates(*Save, *Successor.Block)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + } + if (!Block->successors().empty()) + continue; + if (!containsEpilogueTerminator(*Block, *Info, Target)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + RestoreBlocks.push_back(Block); + } + if (RestoreBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + MachineFrameState Planned = OriginalState; + Planned.SavePoints.push_back({Save->stableID(), {}}); + std::ranges::sort(RestoreBlocks, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + for (const MachineBasicBlock *Block : RestoreBlocks) + Planned.RestorePoints.push_back({Block->stableID(), {}}); + auto Updated = Function.frameInfo().setState(std::move(Planned)); + if (!Updated) + return std::unexpected(Updated.error()); + + MachinePassResult Result; + Result.Changed = true; + Result.Changes = {MachinePassChange::Frame}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineLoopAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp new file mode 100644 index 0000000..9e73cea --- /dev/null +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineSink.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &sinkContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-sink", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-sink", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected<bool> isSafeToSink(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine sink encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().Type && Info->get().Type->isFloat() && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + return HasDefinition; +} + +struct SinkDestination final { + MachineBasicBlock *Block = nullptr; + MachineInstr *FirstUse = nullptr; +}; + +[[nodiscard]] std::optional<SinkDestination> findDestination(MachineInstr &Instruction, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return std::nullopt; + std::unordered_set<Register> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Operand.isDef()) + Definitions.insert(*Reg); + } + if (Definitions.empty()) + return std::nullopt; + + MachineBasicBlock *Destination = nullptr; + for (const Register Reg : Definitions) { + for (MachineOperand &Use : Registers.useOperands(Reg, false)) { + MachineInstr *UseInstruction = Use.parent(); + MachineBasicBlock *UseBlock = UseInstruction == nullptr ? nullptr : UseInstruction->parent(); + if (UseBlock == nullptr || UseBlock == Source) + return std::nullopt; + if (Destination == nullptr) + Destination = UseBlock; + else if (Destination != UseBlock) + return std::nullopt; + } + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source || + std::ranges::find_if(Source->successors(), [Destination](const MachineSuccessor &Successor) { + return Successor.Block == Destination; + }) == Source->successors().end()) { + return std::nullopt; + } + + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + const bool UsesDefinition = + std::ranges::any_of(Candidate->operands(), [&Definitions](const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + return Reg != nullptr && Operand.isUse() && !Operand.isDebugUse() && + Definitions.contains(*Reg); + }); + if (UsesDefinition) { + FirstUse = Candidate; + break; + } + } + return FirstUse == nullptr ? std::nullopt + : std::optional<SinkDestination>{{Destination, FirstUse}}; +} + +[[nodiscard]] std::optional<SinkDestination> +findPostRADestination(MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Source->successors().empty()) + return std::nullopt; + + std::vector<PhysicalRegister> Uses; + std::vector<PhysicalRegister> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return std::nullopt; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return std::nullopt; + MachineBasicBlock *Destination = nullptr; + for (const MachineSuccessor &Successor : Source->successors()) { + if (Successor.Block == nullptr) + return std::nullopt; + const bool CarriesAny = std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + const bool CarriesAll = std::ranges::all_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + if (CarriesAny != CarriesAll || (CarriesAll && Destination != nullptr)) + return std::nullopt; + if (CarriesAll) + Destination = Successor.Block; + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source) { + return std::nullopt; + } + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (!std::ranges::any_of(Destination->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return std::nullopt; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return std::nullopt; + } + } + + const auto OverlapsAny = [&](PhysicalRegister Reg, std::span<const PhysicalRegister> Values) { + return std::ranges::any_of( + Values, [&](PhysicalRegister Value) { return Target.registers().regsOverlap(Reg, Value); }); + }; + for (MachineInstr *After = Instruction.next(); After != nullptr; After = After->next()) { + for (const MachineOperand &Operand : After->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if ((Operand.isUse() || Operand.isDef()) && OverlapsAny(*Physical, Definitions)) { + return std::nullopt; + } + if (Operand.isDef() && OverlapsAny(*Physical, Uses)) + return std::nullopt; + } + } + + std::vector<bool> SeenUse(Definitions.size()); + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!SeenUse[Index] && Mask->Mask->clobbers(Definitions[Index])) + return std::nullopt; + } + if (FirstUse == nullptr && std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { + return Mask->Mask->clobbers(Reg); + })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!Target.registers().regsOverlap(*Physical, Definitions[Index])) + continue; + if (Operand.isUse() && !Operand.isDebugUse()) { + SeenUse[Index] = true; + if (FirstUse == nullptr) + FirstUse = Candidate; + } + if (Operand.isDef() && !SeenUse[Index]) + return std::nullopt; + } + if (FirstUse == nullptr && Operand.isDef() && OverlapsAny(*Physical, Uses)) { + return std::nullopt; + } + } + } + if (FirstUse == nullptr || !std::ranges::all_of(SeenUse, [](bool Value) { return Value; })) + return std::nullopt; + return SinkDestination{Destination, FirstUse}; +} + +[[nodiscard]] Expected<void> moveBefore(MachineInstr &Instruction, + const SinkDestination &Destination) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Destination.Block == nullptr || Destination.FirstUse == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine sink has an invalid move plan"); + auto Removed = Source->removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return Destination.Block->insertInstructionBefore(*Destination.FirstUse, Instruction); +} + +} // namespace + +const MachinePassContract &MachineSinkPass::contract() const noexcept { + return sinkContract(Stage); +} + +Expected<MachinePassResult> MachineSinkPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected( + ErrorCode::VerificationError, + Stage == MachineCodeMotionStage::SelectedSSA + ? "machine sink requires selected SSA MachineIR" + : "post-RA machine sink requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "machine sink transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "machine sink diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> MachineSinkPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t Sunk = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); + Block != nullptr && Sunk < Options.MaximumSunkInstructions; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && Sunk < Options.MaximumSunkInstructions;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToSink(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + if (!*Safe) { + Instruction = Next; + continue; + } + const std::optional<SinkDestination> Destination = + Stage == MachineCodeMotionStage::SelectedSSA + ? findDestination(*Instruction, Function.registerInfo()) + : findPostRADestination(*Instruction, Function, Target); + if (!Destination || !(*Dominators)->dominates(*Block, *Destination->Block) || + (*Frequencies)->frequency(*Destination->Block) > (*Frequencies)->frequency(*Block)) { + Instruction = Next; + continue; + } + auto Moved = moveBefore(*Instruction, *Destination); + if (!Moved) + return std::unexpected(Moved.error()); + ++Sunk; + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineStackSlotColoring.cpp b/backend/lib/CodeGen/MachineStackSlotColoring.cpp new file mode 100644 index 0000000..b3b9af5 --- /dev/null +++ b/backend/lib/CodeGen/MachineStackSlotColoring.cpp @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineStackSlotColoring.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &stackSlotColoringContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-stack-slot-coloring", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands, MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct SlotInterval final { + MachineBasicBlock *Block = nullptr; + std::size_t First = 0; + std::size_t Last = 0; + bool FirstStores = false; + bool LastLoads = false; + bool IsValid = true; +}; + +struct InstructionReference final { + MachineInstr *Instruction = nullptr; + std::size_t Position = 0; + bool Loads = false; + bool Stores = false; + bool HasMemoryReference = false; + bool HasFrameOperand = false; +}; + +struct SlotUseInfo final { + FrameIndex Slot; + SlotInterval Interval; + std::vector<InstructionReference> References; +}; + +[[nodiscard]] InstructionReference &getReference(SlotUseInfo &Info, MachineInstr &Instruction, + std::size_t Position) { + const auto Existing = + std::ranges::find_if(Info.References, [&Instruction](const InstructionReference &Reference) { + return Reference.Instruction == &Instruction; + }); + if (Existing != Info.References.end()) + return *Existing; + Info.References.push_back({.Instruction = &Instruction, .Position = Position}); + return Info.References.back(); +} + +[[nodiscard]] std::unordered_map<FrameIndex, SlotUseInfo> +collectSlotUses(MachineFunction &Function) { + std::unordered_map<FrameIndex, SlotUseInfo> Result; + std::unordered_map<MachineInstrStableID, std::pair<MachineBasicBlock *, std::size_t>> Locations; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::size_t Position = 0; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next(), ++Position) { + Locations.emplace(Instruction->stableID(), std::pair{Block, Position}); + for (const MachineOperand &Operand : Instruction->operands()) { + const FrameIndexOperand *Frame = Operand.getIf<FrameIndexOperand>(); + if (Frame == nullptr) + continue; + SlotUseInfo &Info = + Result + .try_emplace(Frame->Index, + SlotUseInfo{.Slot = Frame->Index, .Interval = {}, .References = {}}) + .first->second; + getReference(Info, *Instruction, Position).HasFrameOperand = true; + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + if (Memory == nullptr) + continue; + const std::optional<FrameIndex> Frame = Memory->description().Base.frameIndex(); + if (!Frame) + continue; + SlotUseInfo &Info = + Result + .try_emplace(*Frame, SlotUseInfo{.Slot = *Frame, .Interval = {}, .References = {}}) + .first->second; + InstructionReference &Reference = getReference(Info, *Instruction, Position); + Reference.HasMemoryReference = true; + Reference.Loads |= Memory->isLoad(); + Reference.Stores |= Memory->isStore(); + } + } + } + + for (auto &[Slot, Info] : Result) { + (void)Slot; + if (Info.References.empty()) { + Info.Interval.IsValid = false; + continue; + } + Info.Interval.Block = Info.References.front().Instruction->parent(); + Info.Interval.First = Info.References.front().Position; + Info.Interval.Last = Info.References.back().Position; + Info.Interval.FirstStores = Info.References.front().Stores; + Info.Interval.LastLoads = Info.References.back().Loads; + for (const InstructionReference &Reference : Info.References) { + if (Reference.Instruction->parent() != Info.Interval.Block || !Reference.HasMemoryReference || + (Reference.HasFrameOperand && !Reference.HasMemoryReference)) { + Info.Interval.IsValid = false; + } + } + Info.Interval.IsValid &= + Info.Interval.Block != nullptr && Info.Interval.FirstStores && Info.Interval.LastLoads; + const StackObject *Object = Function.frameInfo().object(Info.Slot); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr) { + Info.Interval.IsValid = false; + continue; + } + for (const StackObjectLifetimeRange &Range : Description->Lifetime) { + const auto Begin = Locations.find(Range.Begin); + const auto End = Locations.find(Range.End); + if (Begin == Locations.end() || End == Locations.end() || + Begin->second.first != Info.Interval.Block || End->second.first != Info.Interval.Block || + Begin->second.second > End->second.second) { + Info.Interval.IsValid = false; + break; + } + Info.Interval.First = std::min(Info.Interval.First, Begin->second.second); + Info.Interval.Last = std::max(Info.Interval.Last, End->second.second); + } + } + return Result; +} + +[[nodiscard]] bool isEligibleSlot(const StackObjectDesc &Description) noexcept { + return Description.Kind == StackObjectKind::SpillSlot && !Description.IsDead && + !Description.IsImmutable && !Description.IsAliased && !Description.AddressTaken && + !Description.AllocaName && !Description.Offset && !Description.LocalOffset && + !Description.CalleeSavedRegister && !Description.DebugInfo && + Description.TargetFlags == 0 && Description.Size.has_value(); +} + +[[nodiscard]] bool compatibleSlots(const StackObjectDesc &Left, + const StackObjectDesc &Right) noexcept { + return isEligibleSlot(Left) && isEligibleSlot(Right) && Left.Kind == Right.Kind && + Left.Size == Right.Size && Left.Align == Right.Align && Left.Stack == Right.Stack; +} + +[[nodiscard]] bool intervalsOverlap(const SlotInterval &Left, const SlotInterval &Right) noexcept { + if (Left.Block != Right.Block) + return true; + return !(Left.Last < Right.First || Right.Last < Left.First); +} + +struct StackColor final { + FrameIndex Canonical; + const StackObjectDesc *Description = nullptr; + std::vector<SlotInterval> Intervals; +}; + +[[nodiscard]] Expected<void> rewriteSlot(MachineFunction &Function, FrameIndex OldSlot, + FrameIndex NewSlot) { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const FrameIndexOperand *Frame = + Operand == nullptr ? nullptr : Operand->getIf<FrameIndexOperand>(); + if (Frame == nullptr || Frame->Index != OldSlot) + continue; + auto Rewritten = Operand->setFrameIndex(NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + for (std::size_t Index = 0; Index < Instruction->memOperands().size(); ++Index) { + const MachineMemOperand *Memory = Instruction->memOperands()[Index]; + if (Memory == nullptr || Memory->description().Base.frameIndex() != OldSlot) + continue; + MachineMemOperandDesc Description = Memory->description(); + Description.Base = MemoryBase::frameIndex(NewSlot); + auto Replacement = Function.createMemOperand(std::move(Description)); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Replaced = Instruction->replaceMemOperand(Index, **Replacement); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineStackSlotColoringPass::contract() const noexcept { + return stackSlotColoringContract(); +} + +Expected<MachinePassResult> MachineStackSlotColoringPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "stack-slot coloring requires allocated PHI-free MachineIR"); + } + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "stack-slot coloring must run before frame layout is frozen"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "stack-slot coloring transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "stack-slot coloring diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineStackSlotColoringPass::runImplementation(MachineFunction &Function) const { + std::unordered_map<FrameIndex, SlotUseInfo> Uses = collectSlotUses(Function); + std::vector<StackColor> Colors; + std::vector<std::pair<FrameIndex, FrameIndex>> Rewrites; + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + const auto Use = Uses.find(Object.index()); + if (Description == nullptr || Use == Uses.end() || !Use->second.Interval.IsValid || + !isEligibleSlot(*Description)) { + continue; + } + auto Color = std::ranges::find_if(Colors, [&](const StackColor &Candidate) { + return Candidate.Description != nullptr && + compatibleSlots(*Candidate.Description, *Description) && + std::ranges::none_of(Candidate.Intervals, [&](const SlotInterval &Interval) { + return intervalsOverlap(Interval, Use->second.Interval); + }); + }); + if (Color == Colors.end()) { + Colors.push_back({Object.index(), Description, {Use->second.Interval}}); + continue; + } + if (Rewrites.size() == Options.MaximumColoredSlots) + break; + Rewrites.emplace_back(Object.index(), Color->Canonical); + Color->Intervals.push_back(Use->second.Interval); + } + + for (const auto &[OldSlot, NewSlot] : Rewrites) { + const StackObject *OldObject = Function.frameInfo().object(OldSlot); + const StackObject *NewObject = Function.frameInfo().object(NewSlot); + if (OldObject == nullptr || NewObject == nullptr || OldObject->description() == nullptr || + NewObject->description() == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "stack-slot coloring lost a planned stack object"); + } + auto Rewritten = rewriteSlot(Function, OldSlot, NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + StackObjectDesc Combined = *NewObject->description(); + Combined.Lifetime.insert(Combined.Lifetime.end(), OldObject->description()->Lifetime.begin(), + OldObject->description()->Lifetime.end()); + auto Updated = Function.frameInfo().updateObject(NewSlot, std::move(Combined)); + if (!Updated) + return std::unexpected(Updated.error()); + auto Dead = Function.frameInfo().markObjectDead(OldSlot); + if (!Dead) + return std::unexpected(Dead.error()); + } + + MachinePassResult Result; + Result.Changed = !Rewrites.empty(); + if (Result.Changed) { + Result.Changes = {MachinePassChange::Operands, MachinePassChange::Frame}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailDuplication.cpp b/backend/lib/CodeGen/MachineTailDuplication.cpp new file mode 100644 index 0000000..a130680 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailDuplication.cpp @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailDuplication.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailDuplicationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-duplication", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] Expected<bool> isDuplicableInstruction(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::NotDuplicable) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Instruction.flags().has(MachineInstrFlag::NoMerge) || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc) || + hasProtectedAttachment(Instruction)) { + return false; + } + return std::ranges::none_of(Instruction.operands(), [](const MachineOperand &Operand) { + return Operand.kind() == MachineOperand::Kind::BasicBlock || + Operand.kind() == MachineOperand::Kind::DebugInstrRef; + }); +} + +[[nodiscard]] Expected<MachineInstr *> +cloneInstructionBefore(MachineInstr &Source, MachineBasicBlock &Destination, MachineInstr *Before) { + MachineFunction &Function = Destination.function(); + auto Created = Function.createInstruction(Source.opcode()); + if (!Created) + return std::unexpected(Created.error()); + MachineInstr *Clone = *Created; + Clone->reserveOperands(Source.operandCount()); + for (const MachineOperand &Operand : Source.operands()) { + auto Added = Clone->addOperand(Operand.cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + const std::optional<std::size_t> Tied = + Operand == nullptr ? std::nullopt : Operand->tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = Operand->isDef() ? Index : *Tied; + const std::size_t UseIndex = Operand->isDef() ? *Tied : Index; + auto TiedResult = Clone->tieOperands(DefIndex, UseIndex); + if (!TiedResult) + return std::unexpected(TiedResult.error()); + } + for (const MachineMemOperand *Memory : Source.memOperands()) { + if (Memory == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered a null memory operand"); + auto Added = Clone->addMemOperand(*Memory); + if (!Added) + return std::unexpected(Added.error()); + } + Clone->setFlags(Source.flags()); + if (Source.sourceLocation()) + Clone->setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Clone->setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Clone->setOrigin(*Source.origin()); + Expected<void> Inserted = Before == nullptr + ? Destination.appendInstruction(*Clone) + : Destination.insertInstructionBefore(*Before, *Clone); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Clone; +} + +struct TailCandidate final { + MachineBasicBlock *Successor = nullptr; + MachineInstr *Branch = nullptr; + std::vector<MachineInstr *> Body; +}; + +[[nodiscard]] Expected<std::optional<TailCandidate>> +analyzeTail(MachineBasicBlock &Tail, MachineFunction &Function, const TargetDescription &Target, + const TargetBranchInfo &Info, std::size_t MaximumInstructions) { + if (&Tail == Function.firstBlock() || Tail.predecessors().size() < 2 || + Tail.successors().size() != 1 || Tail.successors().front().Block == nullptr || + hasProtectedIdentity(Tail, Function)) { + return std::nullopt; + } + auto Branch = Info.analyzeBranch(Tail, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + MachineBasicBlock *Successor = Tail.successors().front().Block; + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr || + Branch->Unconditional == nullptr || Branch->TrueTarget != Successor || + Branch->Unconditional != Tail.lastInstruction() || + hasProtectedAttachment(*Branch->Unconditional)) { + return std::nullopt; + } + + TailCandidate Candidate{.Successor = Successor, .Branch = Branch->Unconditional, .Body = {}}; + for (MachineInstr *Instruction = Tail.firstInstruction(); Instruction != Candidate.Branch; + Instruction = Instruction->next()) { + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail branch is not reachable from its block body"); + auto Duplicable = isDuplicableInstruction(*Instruction, Target); + if (!Duplicable) + return std::unexpected(Duplicable.error()); + if (!*Duplicable) + return std::nullopt; + Candidate.Body.push_back(Instruction); + if (Candidate.Body.size() > MaximumInstructions) + return std::nullopt; + } + return Candidate; +} + +[[nodiscard]] Expected<bool> +duplicateTailIntoPredecessor(const TailCandidate &Candidate, MachineBasicBlock &Tail, + MachineBasicBlock &Predecessor, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + if (Predecessor.successors().size() != 1 || Predecessor.successors().front().Block != &Tail) { + return false; + } + auto Branch = Info.analyzeBranch(Predecessor, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr) + return false; + MachineInstr *OldBranch = Branch->Unconditional; + if (OldBranch != nullptr) { + if (Branch->TrueTarget != &Tail || OldBranch != Predecessor.lastInstruction() || + hasProtectedAttachment(*OldBranch)) { + return false; + } + } else if (Predecessor.next() != &Tail) { + return false; + } + + for (MachineInstr *Instruction : Candidate.Body) { + auto Cloned = cloneInstructionBefore(*Instruction, Predecessor, OldBranch); + if (!Cloned) + return std::unexpected(Cloned.error()); + } + if (OldBranch != nullptr) { + auto Removed = Info.removeBranch(*OldBranch, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + } + auto NewBranch = + Info.insertUnconditionalBranch(Predecessor, *Candidate.Successor, Function, Target); + if (!NewBranch) + return std::unexpected(NewBranch.error()); + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Tail, *Candidate.Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailDuplicationPass::contract() const noexcept { + return tailDuplicationContract(); +} + +Expected<MachinePassResult> MachineTailDuplicationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "tail duplication transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail duplication diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineTailDuplicationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t DuplicatedInstructions = 0; + for (MachineBasicBlock *Tail = Function.firstBlock(); Tail != nullptr;) { + MachineBasicBlock *Next = Tail->next(); + auto Candidate = analyzeTail(*Tail, Function, Target, *Info, Options.MaximumTailInstructions); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!Candidate->has_value()) { + Tail = Next; + continue; + } + const std::size_t Cost = (*Candidate)->Body.size() + 1U; + const std::vector<MachineBasicBlock *> Predecessors(Tail->predecessors().begin(), + Tail->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || + DuplicatedInstructions > Options.MaximumDuplicatedInstructions || + Cost > Options.MaximumDuplicatedInstructions - DuplicatedInstructions) { + break; + } + auto Duplicated = + duplicateTailIntoPredecessor(**Candidate, *Tail, *Predecessor, Function, Target, *Info); + if (!Duplicated) + return std::unexpected(Duplicated.error()); + if (!*Duplicated) + continue; + DuplicatedInstructions += Cost; + Result.Changed = true; + } + if (Tail->predecessors().empty()) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Tail); + if (!Deleted) + return std::unexpected(Deleted.error()); + Result.Changed = true; + } + Tail = Next; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailMerge.cpp b/backend/lib/CodeGen/MachineTailMerge.cpp new file mode 100644 index 0000000..b754978 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailMerge.cpp @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailMerge.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetBranchInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailMergeContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-merge", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +template <typename ValueT> +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf<ValueT>(); + const ValueT *RightValue = Right.getIf<ValueT>(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +template <typename MaskOperandT> +[[nodiscard]] bool maskEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const MaskOperandT *LeftValue = Left.getIf<MaskOperandT>(); + const MaskOperandT *RightValue = Right.getIf<MaskOperandT>(); + if (LeftValue == nullptr || RightValue == nullptr) + return false; + if (!LeftValue->Mask || !RightValue->Mask) + return LeftValue->Mask == RightValue->Mask; + return *LeftValue->Mask == *RightValue->Mask; +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() || + Left.tiedOperandIndex() != Right.tiedOperandIndex()) { + return false; + } + if (Left.kind() == MachineOperand::Kind::Register) { + const Register *LeftReg = Left.getIf<Register>(); + const Register *RightReg = Right.getIf<Register>(); + if (LeftReg == nullptr || RightReg == nullptr || *LeftReg != *RightReg || + Left.subReg() != Right.subReg()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags; + } + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return false; + case MachineOperand::Kind::Immediate: + return payloadEquals<std::int64_t>(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals<IntegerImmediateOperand>(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals<FloatingImmediateOperand>(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals<BasicBlockOperand>(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals<FrameIndexOperand>(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals<ConstantPoolOperand>(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals<JumpTableOperand>(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals<GlobalSymbolOperand>(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals<ExternalSymbolOperand>(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals<BlockAddressOperand>(Left, Right); + case MachineOperand::Kind::RegisterMask: + return maskEquals<RegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::LiveOutRegisterMask: + return maskEquals<LiveOutRegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::Metadata: + return payloadEquals<MetadataOperand>(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals<TargetIndexOperand>(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals<MCSymbolOperand>(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals<CFIIndexOperand>(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals<IntrinsicOperand>(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals<PredicateOperand>(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals<ShuffleMaskOperand>(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals<DebugInstrRefOperand>(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals<LaneMaskOperand>(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] Expected<bool> instructionEquals(const MachineInstr &Left, const MachineInstr &Right, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Left.opcode()); + if (Descriptor == nullptr || Target.instructionDescriptor(Right.opcode()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge encountered an unknown instruction"); + } + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size() || + Left.sourceLocation() != Right.sourceLocation() || + Left.debugLocation() != Right.debugLocation() || Left.origin() != Right.origin() || + hasProtectedAttachment(Left) || hasProtectedAttachment(Right) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Convergent) || + Descriptor->Flags.has(InstructionFlag::NotDuplicable) || + Left.flags().has(MachineInstrFlag::NoMerge) || + Left.flags().has(MachineInstrFlag::BundledWithPred) || + Left.flags().has(MachineInstrFlag::BundledWithSucc)) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + !operandEquals(*LeftOperand, *RightOperand)) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || + LeftMemory->description() != RightMemory->description()) { + return false; + } + } + return true; +} + +[[nodiscard]] Expected<bool> blockEquals(const MachineBasicBlock &Left, + const MachineBasicBlock &Right, + const MachineFunction &Function, + const TargetDescription &Target) { + if (&Left == &Right || &Left == Function.firstBlock() || &Right == Function.firstBlock() || + hasProtectedIdentity(Left, Function) || hasProtectedIdentity(Right, Function) || + Left.attributes().alignment() != Right.attributes().alignment() || + Left.attributes().sectionID() != Right.attributes().sectionID() || + Left.attributes().callFrameSize() != Right.attributes().callFrameSize() || + !std::ranges::equal(Left.liveIns(), Right.liveIns()) || + !std::ranges::equal(Left.successors(), Right.successors()) || + Left.instructionCount() != Right.instructionCount()) { + return false; + } + const MachineInstr *LeftInstruction = Left.firstInstruction(); + const MachineInstr *RightInstruction = Right.firstInstruction(); + while (LeftInstruction != nullptr && RightInstruction != nullptr) { + auto Equal = instructionEquals(*LeftInstruction, *RightInstruction, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + return false; + LeftInstruction = LeftInstruction->next(); + RightInstruction = RightInstruction->next(); + } + return LeftInstruction == nullptr && RightInstruction == nullptr; +} + +[[nodiscard]] Expected<bool> canRedirectPredecessor(MachineBasicBlock &Predecessor, + MachineBasicBlock &Duplicate, + MachineFunction &Function, + const TargetDescription &Target, + const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + return false; + if ((Analysis->Conditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Conditional)) || + (Analysis->Unconditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Unconditional))) { + return false; + } + if (Analysis->TrueTarget == &Duplicate || Analysis->FalseTarget == &Duplicate) + return true; + return Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate; +} + +[[nodiscard]] Expected<bool> +redirectPredecessor(MachineBasicBlock &Predecessor, MachineBasicBlock &Duplicate, + MachineBasicBlock &Canonical, MachineFunction &Function, + const TargetDescription &Target, const TargetBranchInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + auto Rewritten = Info.rewriteBranchTarget(Predecessor, Duplicate, Canonical, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + const bool IsImplicitEdge = + (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate) || + (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == &Duplicate); + if (!IsImplicitEdge) + return false; + auto Inserted = Info.insertUnconditionalBranch(Predecessor, Canonical, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Duplicate, Canonical); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailMergePass::contract() const noexcept { + return tailMergeContract(); +} + +Expected<MachinePassResult> MachineTailMergePass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "tail merge requires finalized allocated tracked-liveness MIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected(prefixedError(Trial.error(), "tail merge transaction setup failed: ")); + } + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "tail merge diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected<MachinePassResult> +MachineTailMergePass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t MergedBlocks = 0; + bool LocalChange = true; + while (LocalChange && MergedBlocks < Options.MaximumMergedBlocks) { + LocalChange = false; + std::vector<MachineBasicBlock *> Blocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Blocks.push_back(Block); + } + for (std::size_t DuplicateIndex = 1; DuplicateIndex < Blocks.size(); ++DuplicateIndex) { + MachineBasicBlock *Duplicate = Blocks[DuplicateIndex]; + if (Duplicate == nullptr || !Duplicate->isLinked()) + continue; + for (std::size_t CanonicalIndex = 1; CanonicalIndex < DuplicateIndex; ++CanonicalIndex) { + MachineBasicBlock *Canonical = Blocks[CanonicalIndex]; + if (Canonical == nullptr || !Canonical->isLinked()) + continue; + auto Equal = blockEquals(*Canonical, *Duplicate, Function, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + continue; + + const std::vector<MachineBasicBlock *> Predecessors(Duplicate->predecessors().begin(), + Duplicate->predecessors().end()); + bool RedirectedAll = true; + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || Predecessor == Canonical || Predecessor == Duplicate) { + RedirectedAll = false; + break; + } + auto CanRedirect = + canRedirectPredecessor(*Predecessor, *Duplicate, Function, Target, *Info); + if (!CanRedirect) + return std::unexpected(CanRedirect.error()); + if (!*CanRedirect) { + RedirectedAll = false; + break; + } + } + if (!RedirectedAll) + continue; + for (MachineBasicBlock *Predecessor : Predecessors) { + auto Redirected = + redirectPredecessor(*Predecessor, *Duplicate, *Canonical, Function, Target, *Info); + if (!Redirected) + return std::unexpected(Redirected.error()); + if (!*Redirected) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail merge predecessor changed after redirect preflight"); + } + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Duplicate); + if (!Deleted) + return std::unexpected(Deleted.error()); + ++MergedBlocks; + Result.Changed = true; + LocalChange = true; + break; + } + if (LocalChange) + break; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PostRAScheduler.cpp b/backend/lib/CodeGen/PostRAScheduler.cpp new file mode 100644 index 0000000..9aff5ac --- /dev/null +++ b/backend/lib/CodeGen/PostRAScheduler.cpp @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PostRAScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineLivenessRepair.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &postRASchedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-scheduler", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.RequiredAnalyses.push_back(MachineAnalysisManager::id<MachineBlockFrequencyAnalysis>()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Error prefixedError(const Error &Failure, std::string Prefix) { + Prefix.append(Failure.message()); + return Error(Failure.code(), std::move(Prefix)); +} + +struct PhysicalUnitAccess final { + RegisterUnitID Unit; + LaneMask Lanes; +}; + +struct OperandFlagUpdate final { + MachineOperand *Operand = nullptr; + RegisterOperandFlags Flags; +}; + +[[nodiscard]] LaneMask normalizeUnitLanes(LaneMask UnitViewLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +[[nodiscard]] Expected<std::vector<PhysicalUnitAccess>> +physicalOperandUnits(const MachineOperand &Operand, const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a virtual register"); + } + if (Operand.subReg().value() != 0) { + Physical = Target.registers().subRegister(*Physical, Operand.subReg()); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered an invalid subregister"); + } + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr || Description->Units.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a unitless register"); + } + std::vector<PhysicalUnitAccess> Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) { + if (Unit.Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered empty register-unit lanes"); + } + Result.push_back({Unit.Unit, normalizeUnitLanes(Unit.Lanes)}); + } + return Result; +} + +void addPhysicalRegister(RegisterUnitLaneSet &Live, const PhysicalRegisterDesc &Register) { + for (const RegisterUnitLane &Unit : Register.Units) + Live.add(Unit.Unit, normalizeUnitLanes(Unit.Lanes)); +} + +[[nodiscard]] bool intersects(const RegisterUnitLaneSet &Live, + const std::vector<PhysicalUnitAccess> &Units) noexcept { + return std::ranges::any_of(Units, [&Live](const PhysicalUnitAccess &Unit) { + return Live.intersects(Unit.Unit, Unit.Lanes); + }); +} + +void addUnits(RegisterUnitLaneSet &Live, const std::vector<PhysicalUnitAccess> &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.add(Unit.Unit, Unit.Lanes); +} + +void removeUnits(RegisterUnitLaneSet &Live, const std::vector<PhysicalUnitAccess> &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.remove(Unit.Unit, Unit.Lanes); +} + +[[nodiscard]] OperandFlagUpdate &getFlagUpdate(std::vector<OperandFlagUpdate> &Updates, + MachineOperand &Operand) { + const auto Position = std::ranges::find_if( + Updates, [&Operand](const OperandFlagUpdate &Update) { return Update.Operand == &Operand; }); + if (Position != Updates.end()) + return *Position; + Updates.push_back({&Operand, Operand.registerFlags()}); + return Updates.back(); +} + +[[nodiscard]] Expected<std::vector<OperandFlagUpdate>> +buildBlockLivenessRepairPlan(MachineBasicBlock &Block, const TargetDescription &Target, + const MachineBlockLiveness &Boundary) { + RegisterUnitLaneSet Live = Boundary.PhysicalLiveOut; + std::vector<OperandFlagUpdate> Updates; + for (MachineInstr *Instruction = Block.lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null live-out mask"); + } + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->preserves(Register.Reg)) + addPhysicalRegister(Live, Register); + } + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Dead); + if (!Operand->isDef()) + continue; + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Dead); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + removeUnits(Live, *Units); + continue; + } + const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null register mask"); + } + RegisterUnitLaneSet Clobbers; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->clobbers(Register.Reg)) + addPhysicalRegister(Clobbers, Register); + } + Live.subtract(Clobbers); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Kill); + if (!Operand->isUse() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Kill); + addUnits(Live, *Units); + } + } + if (Live != Boundary.PhysicalLiveIn) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA local liveness repair changed the block boundary"); + } + return Updates; +} + +} // namespace + +Expected<bool> repairPostRAMachineBlockLiveness(std::span<MachineBasicBlock *const> Blocks, + const TargetDescription &Target, + const LiveVariablesAnalysis::Result &Liveness) { + std::vector<OperandFlagUpdate> Updates; + for (MachineBasicBlock *Block : Blocks) { + const MachineBlockLiveness *Boundary = Liveness.block(*Block); + if (Boundary == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA scheduler lost a block liveness boundary"); + } + auto BlockUpdates = buildBlockLivenessRepairPlan(*Block, Target, *Boundary); + if (!BlockUpdates) + return std::unexpected(BlockUpdates.error()); + Updates.insert(Updates.end(), BlockUpdates->begin(), BlockUpdates->end()); + } + bool Changed = false; + for (const OperandFlagUpdate &Update : Updates) { + Changed |= Update.Operand->registerFlags() != Update.Flags; + auto Applied = Update.Operand->setRegisterFlags(Update.Flags); + if (!Applied) + return std::unexpected(Applied.error()); + } + return Changed; +} + +const MachinePassContract &PostRASchedulerPass::contract() const noexcept { + return postRASchedulerContract(); +} + +Expected<MachinePassResult> PostRASchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (!Function.properties().containsAll(contract().Requires)) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduler requires allocated tracked-liveness MachineIR"); + } + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "post-RA scheduler transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialLiveness = + TrialAnalyses.getResult<LiveVariablesAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + auto TrialFrequencies = + TrialAnalyses.getResult<MachineBlockFrequencyAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialFrequencies) + return std::unexpected(TrialFrequencies.error()); + auto TrialRun = runMachineScheduler(*Trial->ClonedFunction, Target, *Model, + ScheduleDAGMode::PostRegisterAllocation, **TrialLiveness, + **TrialFrequencies, Options); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialRepair = + repairPostRAMachineBlockLiveness(TrialRun->ChangedBlocks, Target, **TrialLiveness); + if (!TrialRepair) + return std::unexpected(TrialRepair.error()); + + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PostRegisterAllocation, + **Liveness, **Frequencies, Options); + if (!Run) { + return std::unexpected(prefixedError( + Run.error(), "post-RA scheduler diverged after a successful transaction preflight: ")); + } + auto Repaired = repairPostRAMachineBlockLiveness(Run->ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = Run->Changed || *Repaired; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve<LiveVariablesAnalysis>(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterPressure.cpp b/backend/lib/CodeGen/RegisterPressure.cpp new file mode 100644 index 0000000..523a443 --- /dev/null +++ b/backend/lib/CodeGen/RegisterPressure.cpp @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterPressure.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using EntityID = std::size_t; + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +template <typename ValueT> void sortAndUnique(std::vector<ValueT> &Values) { + std::ranges::sort(Values); + const auto [FirstDuplicate, End] = std::ranges::unique(Values); + Values.erase(FirstDuplicate, End); +} + +} // namespace + +namespace detail { + +struct RegisterPressureBuilder final { + RegisterPressureTracker Result; + const ScheduleDAG *DAG = nullptr; + const MachineFunction *Function = nullptr; + const TargetDescription *Target = nullptr; + const TargetSchedulingModel *Model = nullptr; + ScheduleDAGMode Mode = ScheduleDAGMode::PreRegisterAllocation; + std::unordered_map<Register, EntityID> CurrentVirtualEntities; + std::unordered_map<RegisterUnitID, EntityID> CurrentPhysicalEntities; + std::unordered_map<RegisterUnitID, std::vector<RegisterPressureSetID>> PhysicalSets; + + [[nodiscard]] Expected<void> initializePressureSets() { + Result.Pressure.reserve(Model->pressureSets().size()); + for (const RegisterPressureSetDesc &Set : Model->pressureSets()) { + Result.PressureIndexes.emplace(Set.ID, Result.Pressure.size()); + Result.Pressure.push_back({Set.ID, 0, 0, Set.Limit}); + for (const RegisterClassID ClassID : Set.RegisterClasses) { + const RegisterClassDesc *Class = Target->registers().registerClass(ClassID); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure set references an unknown target register class"); + } + for (const PhysicalRegister Physical : Class->Registers) { + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "register class references an unknown physical register"); + } + for (const RegisterUnitLane &Unit : Description->Units) + PhysicalSets[Unit.Unit].push_back(Set.ID); + } + } + } + for (auto &[Unit, Sets] : PhysicalSets) { + (void)Unit; + sortAndUnique(Sets); + } + return {}; + } + + [[nodiscard]] EntityID createEntity(std::vector<RegisterPressureSetID> Sets) { + sortAndUnique(Sets); + const EntityID ID = Result.Entities.size(); + Result.Entities.push_back({std::move(Sets), 0, false, false, false, false}); + return ID; + } + + [[nodiscard]] Expected<std::vector<RegisterPressureSetID>> + virtualPressureSets(Register Reg) const { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pressure tracking encountered a virtual register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().RegClass) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA pressure tracking requires constrained virtual registers"); + } + const std::span<const RegisterPressureSetID> Sets = + Model->pressureSetsForClass(*Info->get().RegClass); + return std::vector<RegisterPressureSetID>{Sets.begin(), Sets.end()}; + } + + [[nodiscard]] Expected<EntityID> boundaryVirtualEntity(Register Reg) { + if (const auto Position = CurrentVirtualEntities.find(Reg); + Position != CurrentVirtualEntities.end()) { + return Position->second; + } + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.emplace(Reg, ID); + return ID; + } + + [[nodiscard]] Expected<EntityID> defineVirtualEntity(Register Reg) { + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.insert_or_assign(Reg, ID); + return ID; + } + + [[nodiscard]] EntityID boundaryPhysicalEntity(RegisterUnitID Unit) { + if (const auto Position = CurrentPhysicalEntities.find(Unit); + Position != CurrentPhysicalEntities.end()) { + return Position->second; + } + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector<RegisterPressureSetID>{} : Sets->second); + CurrentPhysicalEntities.emplace(Unit, ID); + return ID; + } + + [[nodiscard]] EntityID definePhysicalEntity(RegisterUnitID Unit) { + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector<RegisterPressureSetID>{} : Sets->second); + CurrentPhysicalEntities.insert_or_assign(Unit, ID); + return ID; + } + + [[nodiscard]] Expected<std::vector<RegisterUnitID>> physicalUnits(Register Reg, + SubRegIndex SubReg) const { + std::optional<PhysicalRegister> Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical register"); + } + if (SubReg.value() != 0) { + Physical = Target->registers().subRegister(*Physical, SubReg); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical subregister"); + } + } + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(*Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an unknown physical register"); + } + std::vector<RegisterUnitID> Units; + Units.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Units.push_back(Unit.Unit); + sortAndUnique(Units); + return Units; + } + + [[nodiscard]] Expected<void> appendRegisterUse(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = boundaryVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Uses.push_back(*Entity); + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit)); + return {}; + } + + [[nodiscard]] Expected<void> appendRegisterDefinition(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = defineVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Definitions.push_back(*Entity); + Result.Entities[*Entity].HasDefinition = true; + Result.Entities[*Entity].DefinitionPending = true; + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + const EntityID Entity = definePhysicalEntity(Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + return {}; + } + + [[nodiscard]] Expected<void> appendMaskDefinition(RegisterPressureTracker::Event &Event, + const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) { + const EntityID Entity = definePhysicalEntity(Unit.Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + } + return {}; + } + + void appendLiveOutMaskUse(RegisterPressureTracker::Event &Event, const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit.Unit)); + } + } + + [[nodiscard]] Expected<RegisterPressureTracker::Event> + buildEvent(const MachineInstr &Instruction) { + RegisterPressureTracker::Event Event; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null live-out register mask"); + } + appendLiveOutMaskUse(Event, *Mask->Mask); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + if (Operand.isUse()) { + auto Added = appendRegisterUse(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null register mask"); + } + auto Added = appendMaskDefinition(Event, *Mask->Mask); + if (!Added) + return std::unexpected(Added.error()); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse()) + continue; + if (Operand.isDef()) { + auto Added = appendRegisterDefinition(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + sortAndUnique(Event.Uses); + sortAndUnique(Event.Definitions); + return Event; + } + + [[nodiscard]] Expected<void> buildEffects() { + Result.Effects.resize(DAG->units().size()); + Result.Scheduled.assign(DAG->units().size(), false); + for (const ScheduleUnit &Unit : DAG->units()) { + RegisterPressureTracker::UnitEffects &Effects = Result.Effects[Unit.ID]; + Effects.Events.reserve(Unit.Instructions.size()); + for (const MachineInstr *Instruction : Unit.Instructions) { + auto Event = buildEvent(*Instruction); + if (!Event) + return std::unexpected(Event.error()); + for (const EntityID Entity : Event->Uses) + ++Result.Entities[Entity].RemainingUses; + Effects.Events.push_back(std::move(*Event)); + } + } + return {}; + } + + void markEntityLiveOut(EntityID Entity) { + RegisterPressureTracker::Entity &Value = Result.Entities[Entity]; + if (Value.LiveOut) + return; + Value.LiveOut = true; + ++Value.RemainingUses; + } + + [[nodiscard]] Expected<void> markBlockLiveOut(const MachineBlockLiveness &Liveness) { + for (const auto &[Reg, Lanes] : Liveness.VirtualLiveOut.entries()) { + (void)Lanes; + auto Entity = boundaryVirtualEntity(Reg); + if (!Entity) + return std::unexpected(Entity.error()); + markEntityLiveOut(*Entity); + } + for (const auto &[Unit, Lanes] : Liveness.PhysicalLiveOut.entries()) { + (void)Lanes; + markEntityLiveOut(boundaryPhysicalEntity(Unit)); + } + return {}; + } + + [[nodiscard]] Expected<void> markSuffixUses() { + if (DAG->units().empty()) + return {}; + const MachineInstr *Instruction = DAG->units().back().Instructions.back()->next(); + std::unordered_set<Register> KilledVirtual; + std::unordered_set<RegisterUnitID> KilledPhysical; + for (; Instruction != nullptr; Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure suffix scan encountered a null register mask"); + } + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask->Mask->clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + KilledPhysical.insert(Unit.Unit); + } + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isVirtual()) { + if (Operand.isUse() && !KilledVirtual.contains(*Reg)) { + const auto Position = CurrentVirtualEntities.find(*Reg); + if (Position != CurrentVirtualEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledVirtual.insert(*Reg); + continue; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + if (Operand.isUse() && !KilledPhysical.contains(Unit)) { + const auto Position = CurrentPhysicalEntities.find(Unit); + if (Position != CurrentPhysicalEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledPhysical.insert(Unit); + } + } + } + return {}; + } + + [[nodiscard]] Expected<void> initializeLivePressure() { + for (EntityID ID = 0; ID < Result.Entities.size(); ++ID) { + RegisterPressureTracker::Entity &Entity = Result.Entities[ID]; + const bool InitiallyLive = Result.Direction == RegisterPressureDirection::TopDown + ? !Entity.HasDefinition && Entity.RemainingUses != 0 + : Entity.LiveOut; + if (!InitiallyLive) + continue; + auto Added = Result.makeLive(ID); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; + } +}; + +} // namespace detail + +Expected<RegisterPressureTracker> +RegisterPressureTracker::create(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + RegisterPressureDirection Direction) { + detail::RegisterPressureBuilder Builder; + Builder.DAG = &DAG; + Builder.Function = &Function; + Builder.Target = &Target; + Builder.Model = &Model; + Builder.Mode = Mode; + Builder.Result.Direction = Direction; + auto Sets = Builder.initializePressureSets(); + if (!Sets) + return std::unexpected(Sets.error()); + auto Effects = Builder.buildEffects(); + if (!Effects) + return std::unexpected(Effects.error()); + auto Suffix = Builder.markSuffixUses(); + if (!Suffix) + return std::unexpected(Suffix.error()); + auto LiveOut = Builder.markBlockLiveOut(Liveness); + if (!LiveOut) + return std::unexpected(LiveOut.error()); + auto Initial = Builder.initializeLivePressure(); + if (!Initial) + return std::unexpected(Initial.error()); + return std::move(Builder.Result); +} + +Expected<RegisterPressureCost> RegisterPressureTracker::evaluate(std::size_t UnitID) const { + RegisterPressureTracker Trial = *this; + const RegisterPressureCost Before = cost(); + auto ScheduledUnit = Trial.schedule(UnitID); + if (!ScheduledUnit) + return std::unexpected(ScheduledUnit.error()); + RegisterPressureCost After = Trial.cost(); + After.PeakIncrease = + After.PeakExcess > Before.PeakExcess ? After.PeakExcess - Before.PeakExcess : 0; + return After; +} + +Expected<void> RegisterPressureTracker::schedule(std::size_t UnitID) { + if (UnitID >= Effects.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "pressure tracker unit ID is out of range"); + if (Scheduled[UnitID]) + return makeUnexpected(ErrorCode::InvalidArgument, "schedule unit was already issued"); + RegisterPressureTracker Trial = *this; + if (Trial.Direction == RegisterPressureDirection::TopDown) { + for (const Event &Value : Trial.Effects[UnitID].Events) { + auto Applied = Trial.applyEvent(Value); + if (!Applied) + return std::unexpected(Applied.error()); + } + } else { + for (auto Position = Trial.Effects[UnitID].Events.rbegin(); + Position != Trial.Effects[UnitID].Events.rend(); ++Position) { + auto Applied = Trial.applyEvent(*Position); + if (!Applied) + return std::unexpected(Applied.error()); + } + } + Trial.Scheduled[UnitID] = true; + *this = std::move(Trial); + return {}; +} + +RegisterPressureCost RegisterPressureTracker::cost() const noexcept { + RegisterPressureCost Result; + for (const RegisterPressureValue &Value : Pressure) { + Result.CurrentPressure = saturatingAdd(Result.CurrentPressure, Value.Current); + if (Value.Current > Value.Limit) { + Result.CurrentExcess = saturatingAdd(Result.CurrentExcess, Value.Current - Value.Limit); + } + if (Value.Maximum > Value.Limit) + Result.PeakExcess = saturatingAdd(Result.PeakExcess, Value.Maximum - Value.Limit); + } + return Result; +} + +Expected<void> RegisterPressureTracker::applyEvent(const Event &Value) { + if (Direction == RegisterPressureDirection::BottomUp) { + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size() || !Entities[Entity].DefinitionPending) { + return makeUnexpected(ErrorCode::VerificationError, + "bottom-up scheduler crossed an unavailable definition"); + } + Entities[Entity].DefinitionPending = false; + if (Entities[Entity].Live) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown use entity"); + } + if (!Entities[Entity].Live) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size() || !Entities[Entity].Live || + Entities[Entity].RemainingUses == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler selected a use before its pressure value became live"); + } + --Entities[Entity].RemainingUses; + if (Entities[Entity].RemainingUses == 0) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown definition entity"); + } + if (Entities[Entity].Live) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler redefined a pressure value while it was live"); + } + if (Entities[Entity].RemainingUses != 0) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; +} + +Expected<void> RegisterPressureTracker::makeLive(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became live"); + Entity &Value = Entities[EntityIDValue]; + if (Value.Live) + return {}; + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure entity references an unknown pressure set"); + } + RegisterPressureValue &SetPressure = Pressure[Position->second]; + if (SetPressure.Current == std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, "register pressure counter overflowed"); + } + ++SetPressure.Current; + SetPressure.Maximum = std::max(SetPressure.Maximum, SetPressure.Current); + } + Value.Live = true; + return {}; +} + +Expected<void> RegisterPressureTracker::makeDead(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became dead"); + Entity &Value = Entities[EntityIDValue]; + if (!Value.Live) + return makeUnexpected(ErrorCode::InternalError, "dead pressure entity was not live"); + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end() || Pressure[Position->second].Current == 0) { + return makeUnexpected(ErrorCode::InternalError, "register pressure counter underflowed"); + } + --Pressure[Position->second].Current; + } + Value.Live = false; + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleDAG.cpp b/backend/lib/CodeGen/ScheduleDAG.cpp new file mode 100644 index 0000000..8696f2e --- /dev/null +++ b/backend/lib/CodeGen/ScheduleDAG.cpp @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleDAG.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct RegisterDependencyKey final { + bool IsUnit = false; + std::uint32_t Value = 0; + + friend constexpr bool operator==(RegisterDependencyKey, RegisterDependencyKey) noexcept = default; +}; + +struct RegisterDependencyKeyHash final { + [[nodiscard]] std::size_t operator()(RegisterDependencyKey Key) const noexcept { + const std::uint64_t Value = (static_cast<std::uint64_t>(Key.IsUnit) << 32U) | Key.Value; + return std::hash<std::uint64_t>{}(Value); + } +}; + +struct RegisterDependencyState final { + struct Definition final { + std::size_t Unit = 0; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; + }; + + std::optional<Definition> LastDefinition; + std::vector<std::size_t> LastUses; +}; + +struct RegisterAccess final { + RegisterDependencyKey Key; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; +}; + +[[nodiscard]] bool keyLess(RegisterDependencyKey Left, RegisterDependencyKey Right) noexcept { + return Left.IsUnit == Right.IsUnit ? Left.Value < Right.Value : Left.IsUnit < Right.IsUnit; +} + +[[nodiscard]] std::uint32_t saturatingAdd(std::uint32_t Left, std::uint32_t Right) noexcept { + return std::numeric_limits<std::uint32_t>::max() - Left < Right + ? std::numeric_limits<std::uint32_t>::max() + : Left + Right; +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +[[nodiscard]] Expected<std::vector<RegisterDependencyKey>> +operandKeys(const MachineOperand &Operand, const MachineFunction &Function, + const TargetDescription &Target, ScheduleDAGMode Mode) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return std::vector<RegisterDependencyKey>{}; + if (Reg->isVirtual()) { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduling region contains a virtual register"); + } + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown virtual register"); + } + return std::vector<RegisterDependencyKey>{{false, Reg->opaqueValue()}}; + } + + std::optional<PhysicalRegister> Physical = Reg->physicalRegister(); + if (!Physical) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical register"); + if (Operand.subReg().value() != 0) { + const std::optional<PhysicalRegister> SubReg = + Target.registers().subRegister(*Physical, Operand.subReg()); + if (!SubReg) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical subregister"); + } + Physical = SubReg; + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown physical register"); + std::vector<RegisterDependencyKey> Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Result.push_back({true, Unit.Unit.value()}); + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] Expected<std::vector<RegisterDependencyKey>> +maskKeys(const RegisterMask &Mask, const TargetDescription &Target) { + std::vector<RegisterDependencyKey> Result; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Result.push_back({true, Unit.Unit.value()}); + } + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] bool mayAlias(const MachineInstr &Earlier, const MachineInstr &Later, + const MachineAliasAnalysis::Result &Aliases) noexcept { + if (Earlier.memOperands().empty() || Later.memOperands().empty()) + return true; + for (const MachineMemOperand *Left : Earlier.memOperands()) { + if (Left == nullptr) + return true; + for (const MachineMemOperand *Right : Later.memOperands()) { + if (Right == nullptr || Aliases.alias(*Left, *Right) != MachineAliasResult::NoAlias) + return true; + } + } + return false; +} + +[[nodiscard]] bool hasOrderedMemory(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isAtomic() || Memory->isVolatile(); + }); +} + +[[nodiscard]] bool hasLoad(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isLoad(); + }); +} + +[[nodiscard]] bool hasStore(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isStore(); + }); +} + +} // namespace + +Expected<ScheduleDAG> ScheduleDAG::build(MachineBasicBlock &Block, + std::span<MachineInstr *const> Instructions, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode) { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule DAG block belongs to another function"); + ScheduleDAG Result(Block); + auto Units = Result.buildUnits(Instructions, Target, Model); + if (!Units) + return std::unexpected(Units.error()); + auto Registers = Result.buildRegisterDependencies(Function, Target, Model, Mode); + if (!Registers) + return std::unexpected(Registers.error()); + auto Memory = Result.buildMemoryDependencies(Target); + if (!Memory) + return std::unexpected(Memory.error()); + auto Order = Result.buildOrderDependencies(Target); + if (!Order) + return std::unexpected(Order.error()); + auto Mutated = Model.hooks().mutateScheduleDAG(Result, Function, Target, Mode); + if (!Mutated) + return std::unexpected(Mutated.error()); + Result.computeCriticalPaths(); + return Result; +} + +Expected<void> ScheduleDAG::buildUnits(std::span<MachineInstr *const> Instructions, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + MachineInstr *ExpectedInstruction = Instructions.empty() ? nullptr : Instructions.front(); + for (std::size_t Index = 0; Index < Instructions.size();) { + MachineInstr *Instruction = Instructions[Index]; + if (Instruction == nullptr || Instruction->parent() != Block) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region contains a detached or foreign instruction"); + if (Instruction != ExpectedInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region is not a contiguous instruction sequence"); + } + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region begins in the middle of an instruction bundle"); + } + + ScheduleUnit Unit; + Unit.ID = Units.size(); + Unit.StableID = Instruction->stableID(); + std::unordered_map<ProcessorResourceID, std::uint32_t> ResourceCycles; + for (;;) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const SchedulingClassModelDesc *Class = Model.schedulingClass(Descriptor->SchedulingClass); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction references an unknown scheduling class"); + } + Unit.MicroOps = saturatingAdd(Unit.MicroOps, Class->MicroOps); + Unit.Latency = std::max(Unit.Latency, Class->Latency); + for (const SchedulingResourceUse &Use : Class->Resources) { + ResourceCycles[Use.Resource] = saturatingAdd(ResourceCycles[Use.Resource], Use.Cycles); + } + Unit.Instructions.push_back(Instruction); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + ++Index; + if (!HasSuccessor) + break; + if (Index >= Instructions.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region ends in the middle of an instruction bundle"); + } + MachineInstr *Next = Instructions[Index]; + if (Next == nullptr || Next->parent() != Block || + !Next->flags().has(MachineInstrFlag::BundledWithPred) || Instruction->next() != Next) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction bundle links are not reciprocal and contiguous"); + } + Instruction = Next; + } + Unit.Resources.reserve(ResourceCycles.size()); + for (const auto &[Resource, Cycles] : ResourceCycles) + Unit.Resources.push_back({Resource, Cycles}); + std::ranges::sort(Unit.Resources, {}, &SchedulingResourceUse::Resource); + ExpectedInstruction = Unit.Instructions.back()->next(); + Units.push_back(std::move(Unit)); + } + return {}; +} + +Expected<void> ScheduleDAG::buildRegisterDependencies(const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, + ScheduleDAGMode Mode) { + std::unordered_map<RegisterDependencyKey, RegisterDependencyState, RegisterDependencyKeyHash> + States; + for (ScheduleUnit &Unit : Units) { + std::vector<RegisterAccess> Uses; + std::vector<RegisterAccess> Definitions; + for (MachineInstr *Instruction : Unit.Instructions) { + for (std::size_t OperandIndex = 0; OperandIndex < Instruction->operandCount(); + ++OperandIndex) { + const MachineOperand *Operand = Instruction->operandAt(OperandIndex); + if (Operand == nullptr) + continue; + if (const auto *Mask = Operand->getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null register mask"); + auto Keys = maskKeys(*Mask->Mask, Target); + if (!Keys) + return std::unexpected(Keys.error()); + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + continue; + } + if (const auto *Mask = Operand->getIf<LiveOutRegisterMaskOperand>()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null live-out register mask"); + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask->Mask->preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &RegisterUnit : Register.Units) + Uses.push_back({{true, RegisterUnit.Unit.value()}, Instruction, OperandIndex}); + } + continue; + } + if (!Operand->isRegister() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + auto Keys = operandKeys(*Operand, Function, Target, Mode); + if (!Keys) + return std::unexpected(Keys.error()); + if (Operand->isUse()) { + for (const RegisterDependencyKey Key : *Keys) + Uses.push_back({Key, Instruction, OperandIndex}); + } + if (Operand->isDef()) { + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + } + } + } + std::ranges::stable_sort(Uses, [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + std::ranges::stable_sort(Definitions, + [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + + for (std::size_t Begin = 0; Begin < Uses.size();) { + std::size_t End = Begin + 1; + while (End < Uses.size() && Uses[End].Key == Uses[Begin].Key) + ++End; + RegisterDependencyState &State = States[Uses[Begin].Key]; + if (State.LastDefinition) { + std::uint32_t Latency = 0; + for (std::size_t Index = Begin; Index < End; ++Index) { + const InstructionDescriptor *Producer = + Target.instructionDescriptor(State.LastDefinition->Instruction->opcode()); + const InstructionDescriptor *Consumer = + Target.instructionDescriptor(Uses[Index].Instruction->opcode()); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler latency lookup encountered an unknown instruction"); + } + auto OperandLatency = + Model.operandLatency(Producer->SchedulingClass, State.LastDefinition->OperandIndex, + Consumer->SchedulingClass, Uses[Index].OperandIndex); + if (!OperandLatency) + return std::unexpected(OperandLatency.error()); + Latency = std::max(Latency, *OperandLatency); + } + addDependency(State.LastDefinition->Unit, Unit.ID, + Uses[Begin].Key.IsUnit ? ScheduleDependencyKind::RegisterUnit + : ScheduleDependencyKind::Data, + Latency); + } + if (State.LastUses.empty() || State.LastUses.back() != Unit.ID) + State.LastUses.push_back(Unit.ID); + Begin = End; + } + + for (std::size_t Begin = 0; Begin < Definitions.size();) { + std::size_t End = Begin + 1; + while (End < Definitions.size() && Definitions[End].Key == Definitions[Begin].Key) { + ++End; + } + RegisterDependencyState &State = States[Definitions[Begin].Key]; + if (State.LastDefinition) { + addDependency(State.LastDefinition->Unit, Unit.ID, ScheduleDependencyKind::Output, 0); + } + for (const std::size_t Use : State.LastUses) + addDependency(Use, Unit.ID, ScheduleDependencyKind::Anti, 0); + State.LastUses.clear(); + const RegisterAccess &Last = Definitions[End - 1]; + State.LastDefinition = + RegisterDependencyState::Definition{Unit.ID, Last.Instruction, Last.OperandIndex}; + Begin = End; + } + } + return {}; +} + +Expected<void> ScheduleDAG::addArtificialDependency(std::size_t Source, std::size_t Destination, + std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source >= Destination) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "target schedule mutation requires a forward dependency between known units"); + } + addDependency(Source, Destination, ScheduleDependencyKind::Artificial, Latency); + return {}; +} + +Expected<void> ScheduleDAG::buildMemoryDependencies(const TargetDescription &Target) { + MachineAliasAnalysis::Result Aliases; + for (std::size_t LaterIndex = 0; LaterIndex < Units.size(); ++LaterIndex) { + for (std::size_t EarlierIndex = 0; EarlierIndex < LaterIndex; ++EarlierIndex) { + bool NeedsEdge = false; + for (MachineInstr *Earlier : Units[EarlierIndex].Instructions) { + const InstructionDescriptor *EarlierDescriptor = + Target.instructionDescriptor(Earlier->opcode()); + if (EarlierDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool EarlierMemory = EarlierDescriptor->Flags.has(InstructionFlag::MayLoad) || + EarlierDescriptor->Flags.has(InstructionFlag::MayStore) || + !Earlier->memOperands().empty(); + if (!EarlierMemory) + continue; + for (MachineInstr *Later : Units[LaterIndex].Instructions) { + const InstructionDescriptor *LaterDescriptor = + Target.instructionDescriptor(Later->opcode()); + if (LaterDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool LaterMemory = LaterDescriptor->Flags.has(InstructionFlag::MayLoad) || + LaterDescriptor->Flags.has(InstructionFlag::MayStore) || + !Later->memOperands().empty(); + if (!LaterMemory) + continue; + if (hasOrderedMemory(*Earlier) || hasOrderedMemory(*Later) || + ((hasStore(*Earlier) || hasStore(*Later)) && mayAlias(*Earlier, *Later, Aliases)) || + (Earlier->memOperands().empty() || Later->memOperands().empty()) || + (!hasLoad(*Earlier) && !hasStore(*Earlier)) || + (!hasLoad(*Later) && !hasStore(*Later))) { + NeedsEdge = true; + break; + } + } + if (NeedsEdge) + break; + } + if (NeedsEdge) + addDependency(EarlierIndex, LaterIndex, ScheduleDependencyKind::Memory, 0); + } + } + return {}; +} + +Expected<void> ScheduleDAG::buildOrderDependencies(const TargetDescription &Target) { + for (std::size_t Barrier = 0; Barrier < Units.size(); ++Barrier) { + bool IsBarrier = false; + for (const MachineInstr *Instruction : Units[Barrier].Instructions) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + IsBarrier |= Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || + Flags.has(InstructionFlag::Convergent) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::Meta); + } + if (!IsBarrier) + continue; + for (std::size_t Earlier = 0; Earlier < Barrier; ++Earlier) + addDependency(Earlier, Barrier, ScheduleDependencyKind::Barrier, 0); + for (std::size_t Later = Barrier + 1; Later < Units.size(); ++Later) + addDependency(Barrier, Later, ScheduleDependencyKind::Barrier, 0); + } + return {}; +} + +void ScheduleDAG::addDependency(std::size_t Source, std::size_t Destination, + ScheduleDependencyKind Kind, std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source == Destination) + return; + const auto Existing = std::ranges::find_if( + Units[Source].Successors, [Destination, Kind](const ScheduleDependency &Dependency) { + return Dependency.Destination == Destination && Dependency.Kind == Kind; + }); + if (Existing != Units[Source].Successors.end()) { + Existing->Latency = std::max(Existing->Latency, Latency); + const auto Predecessor = std::ranges::find_if( + Units[Destination].Predecessors, [Source, Kind](const ScheduleDependency &Dependency) { + return Dependency.Source == Source && Dependency.Kind == Kind; + }); + if (Predecessor != Units[Destination].Predecessors.end()) + Predecessor->Latency = Existing->Latency; + return; + } + const ScheduleDependency Dependency{Source, Destination, Kind, Latency}; + Units[Source].Successors.push_back(Dependency); + Units[Destination].Predecessors.push_back(Dependency); +} + +void ScheduleDAG::computeCriticalPaths() { + for (ScheduleUnit &Unit : Units) { + Unit.Depth = 0; + for (const ScheduleDependency &Dependency : Unit.Predecessors) { + Unit.Depth = + std::max(Unit.Depth, saturatingAdd(Units[Dependency.Source].Depth, + static_cast<std::uint64_t>(Dependency.Latency))); + } + } + for (auto Position = Units.rbegin(); Position != Units.rend(); ++Position) { + Position->Height = Position->Latency; + for (const ScheduleDependency &Dependency : Position->Successors) { + Position->Height = + std::max(Position->Height, saturatingAdd(Units[Dependency.Destination].Height, + static_cast<std::uint64_t>(Dependency.Latency))); + } + } + for (ScheduleUnit &Unit : Units) { + std::ranges::sort(Unit.Predecessors, {}, &ScheduleDependency::Source); + std::ranges::sort(Unit.Successors, {}, &ScheduleDependency::Destination); + } +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp new file mode 100644 index 0000000..28c032d --- /dev/null +++ b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" + +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +} // namespace + +Expected<ScheduleHazardRecognizer> +ScheduleHazardRecognizer::create(const TargetSchedulingModel &Model, ScheduleDirection Direction) { + ScheduleHazardRecognizer Result(Model, Direction); + Result.TargetHazards = Model.hooks().createHazardRecognizer(Direction); + auto Reset = Result.reset(); + if (!Reset) + return std::unexpected(Reset.error()); + return Result; +} + +ScheduleHazardRecognizer::ScheduleHazardRecognizer(const ScheduleHazardRecognizer &Other) + : Model(Other.Model), Direction(Other.Direction), + TargetHazards(Other.TargetHazards ? Other.TargetHazards->clone() : nullptr), + Resources(Other.Resources), ResourceIndexes(Other.ResourceIndexes), + CurrentCycle(Other.CurrentCycle), DispatchAvailableAt(Other.DispatchAvailableAt), + IssuedMicroOps(Other.IssuedMicroOps), DispatchedUnits(Other.DispatchedUnits) {} + +ScheduleHazardRecognizer & +ScheduleHazardRecognizer::operator=(const ScheduleHazardRecognizer &Other) { + if (this == &Other) + return *this; + Model = Other.Model; + Direction = Other.Direction; + TargetHazards = Other.TargetHazards ? Other.TargetHazards->clone() : nullptr; + Resources = Other.Resources; + ResourceIndexes = Other.ResourceIndexes; + CurrentCycle = Other.CurrentCycle; + DispatchAvailableAt = Other.DispatchAvailableAt; + IssuedMicroOps = Other.IssuedMicroOps; + DispatchedUnits = Other.DispatchedUnits; + return *this; +} + +Expected<void> ScheduleHazardRecognizer::reset() { + Resources.clear(); + ResourceIndexes.clear(); + Resources.reserve(Model->processorResources().size()); + for (const ProcessorResourceDesc &Resource : Model->processorResources()) { + ResourceIndexes.emplace(Resource.ID, Resources.size()); + Resources.push_back({Resource.ID, std::vector<std::uint64_t>(Resource.Units, 0)}); + } + CurrentCycle = 0; + DispatchAvailableAt = 0; + IssuedMicroOps = 0; + DispatchedUnits = 0; + if (TargetHazards != nullptr) { + auto Reset = TargetHazards->reset(); + if (!Reset) + return std::unexpected(Reset.error()); + } + return {}; +} + +Expected<bool> ScheduleHazardRecognizer::canIssue(const ScheduleUnit &Unit, + std::uint64_t Cycle) const { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) { + if (Issued.error().code() == ErrorCode::SchedulingFailed) + return false; + return std::unexpected(Issued.error()); + } + return true; +} + +Expected<std::uint64_t> ScheduleHazardRecognizer::earliestIssueCycle(const ScheduleUnit &Unit, + std::uint64_t Earliest) const { + std::uint64_t Candidate = std::max(Earliest, CurrentCycle); + for (;;) { + auto Available = canIssue(Unit, Candidate); + if (!Available) + return std::unexpected(Available.error()); + if (*Available) + return Candidate; + if (Candidate == std::numeric_limits<std::uint64_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler resource timeline is exhausted"); + } + ++Candidate; + } +} + +Expected<void> ScheduleHazardRecognizer::issue(const ScheduleUnit &Unit, std::uint64_t Cycle) { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + *this = std::move(Trial); + return {}; +} + +Expected<void> ScheduleHazardRecognizer::issueImplementation(const ScheduleUnit &Unit, + std::uint64_t Cycle) { + if (Cycle < CurrentCycle) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduler cannot issue into an earlier cycle"); + } + auto Moved = moveToCycle(Cycle); + if (!Moved) + return std::unexpected(Moved.error()); + if (Cycle < DispatchAvailableAt) { + return makeUnexpected(ErrorCode::SchedulingFailed, "scheduler dispatch pipeline is occupied"); + } + if (TargetHazards != nullptr) { + auto Hazard = TargetHazards->getHazardType(Unit); + if (!Hazard) + return std::unexpected(Hazard.error()); + if (*Hazard != ScheduleHazardType::NoHazard) { + return makeUnexpected(ErrorCode::SchedulingFailed, + *Hazard == ScheduleHazardType::NoopHazard + ? "target hazard requires an unavailable scheduling noop" + : "target dynamic scheduling hazard prevents issue"); + } + } + + const std::uint32_t Width = Model->issueWidth(); + if (Unit.MicroOps <= Width) { + if (Unit.MicroOps > Width - IssuedMicroOps || DispatchedUnits == Model->dispatchLimit()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler issue or dispatch width is exhausted"); + } + IssuedMicroOps += Unit.MicroOps; + ++DispatchedUnits; + } else { + if (IssuedMicroOps != 0 || DispatchedUnits != 0) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "multi-cycle dispatch requires an empty issue cycle"); + } + const std::uint64_t DispatchCycles = + (static_cast<std::uint64_t>(Unit.MicroOps) + Width - 1U) / Width; + DispatchAvailableAt = saturatingAdd(Cycle, DispatchCycles); + IssuedMicroOps = Width; + DispatchedUnits = 1; + } + + for (const SchedulingResourceUse &Use : Unit.Resources) { + auto Reserved = reserveResource(Use.Resource, Use.Cycles, Cycle); + if (!Reserved) + return std::unexpected(Reserved.error()); + } + if (TargetHazards != nullptr) { + auto Emitted = TargetHazards->emitInstruction(Unit); + if (!Emitted) + return std::unexpected(Emitted.error()); + } + return {}; +} + +Expected<void> ScheduleHazardRecognizer::moveToCycle(std::uint64_t Cycle) { + while (CurrentCycle < Cycle) { + if (TargetHazards != nullptr) { + Expected<void> Advanced = Direction == ScheduleDirection::TopDown + ? TargetHazards->advanceCycle() + : TargetHazards->recedeCycle(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + ++CurrentCycle; + IssuedMicroOps = 0; + DispatchedUnits = 0; + } + return {}; +} + +Expected<void> ScheduleHazardRecognizer::reserveResource(ProcessorResourceID Resource, + std::uint32_t Cycles, + std::uint64_t Cycle) { + const auto Position = ResourceIndexes.find(Resource); + const ProcessorResourceDesc *Description = Model->processorResource(Resource); + if (Position == ResourceIndexes.end() || Description == nullptr || Cycles == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule unit references an invalid processor resource"); + } + ResourceState &State = Resources[Position->second]; + std::optional<std::size_t> Selected; + for (std::size_t Index = 0; Index < State.AvailableAt.size(); ++Index) { + if (State.AvailableAt[Index] <= Cycle && + (!Selected || State.AvailableAt[Index] < State.AvailableAt[*Selected])) { + Selected = Index; + } + } + if (!Selected) { + return makeUnexpected(ErrorCode::SchedulingFailed, "processor resource has no available unit"); + } + State.AvailableAt[*Selected] = saturatingAdd(Cycle, Cycles); + if (Description->Parent) { + auto Parent = reserveResource(*Description->Parent, Cycles, Cycle); + if (!Parent) + return std::unexpected(Parent.error()); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetSchedulingModel.cpp b/backend/lib/CodeGen/TargetSchedulingModel.cpp new file mode 100644 index 0000000..97abecd --- /dev/null +++ b/backend/lib/CodeGen/TargetSchedulingModel.cpp @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <memory> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> + +namespace rcc { +namespace { + +template <typename IDT> [[nodiscard]] bool isValidID(IDT ID) noexcept { + return ID.value() != 0; +} + +} // namespace + +Expected<ScheduleHazardType> +TargetScheduleHazardRecognizer::getHazardType(const ScheduleUnit &) const { + return ScheduleHazardType::NoHazard; +} + +Expected<void> TargetScheduleHazardRecognizer::emitInstruction(const ScheduleUnit &) { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::advanceCycle() { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::recedeCycle() { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::reset() { + return {}; +} + +bool TargetSchedulingHooks::isSchedulingBoundary(const MachineInstr &, + const TargetDescription &) const noexcept { + return false; +} + +Expected<void> TargetSchedulingHooks::mutateScheduleDAG(ScheduleDAG &, const MachineFunction &, + const TargetDescription &, + ScheduleDAGMode) const { + return {}; +} + +std::int64_t TargetSchedulingHooks::candidatePreference(const ScheduleUnit &, ScheduleDirection, + const MachineBasicBlock &, + std::uint64_t) const noexcept { + return 0; +} + +std::unique_ptr<TargetScheduleHazardRecognizer> +TargetSchedulingHooks::createHazardRecognizer(ScheduleDirection) const { + return nullptr; +} + +Expected<TargetSchedulingModel> +TargetSchedulingModel::create(TargetSchedulingModelDesc Description) { + if (Description.IssueWidth == 0 || Description.DispatchLimit == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model requires nonzero issue and dispatch widths"); + } + + TargetSchedulingModel Result(std::move(Description)); + std::unordered_set<std::string> Names; + for (std::size_t Index = 0; Index < Result.Description.ProcessorResources.size(); ++Index) { + const ProcessorResourceDesc &Resource = Result.Description.ProcessorResources[Index]; + if (!isValidID(Resource.ID) || Resource.Name.empty() || Resource.Units == 0 || + !Result.ResourceToIndex.emplace(Resource.ID, Index).second || + !Names.insert(Resource.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid processor resource"); + } + } + for (const ProcessorResourceDesc &Resource : Result.Description.ProcessorResources) { + if (Resource.Parent && + (Resource.Parent == Resource.ID || !Result.ResourceToIndex.contains(*Resource.Parent))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource has an invalid parent resource"); + } + std::unordered_set<ProcessorResourceID> Path; + const ProcessorResourceDesc *Current = &Resource; + while (Current->Parent) { + if (!Path.insert(Current->ID).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource hierarchy contains a cycle"); + } + Current = &Result.Description.ProcessorResources[Result.ResourceToIndex.at(*Current->Parent)]; + } + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.SchedulingClasses.size(); ++Index) { + SchedulingClassModelDesc &Class = Result.Description.SchedulingClasses[Index]; + if (!isValidID(Class.ID) || Class.Name.empty() || Class.MicroOps == 0 || Class.Latency == 0 || + !Result.ClassToIndex.emplace(Class.ID, Index).second || !Names.insert(Class.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid scheduling class"); + } + } + for (SchedulingClassModelDesc &Class : Result.Description.SchedulingClasses) { + std::unordered_set<ProcessorResourceID> UsedResources; + for (const SchedulingResourceUse &Use : Class.Resources) { + if (!isValidID(Use.Resource) || Use.Cycles == 0 || + !Result.ResourceToIndex.contains(Use.Resource) || + !UsedResources.insert(Use.Resource).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has an invalid resource use"); + } + } + std::ranges::sort(Class.Resources, {}, + [](const SchedulingResourceUse &Use) { return Use.Resource.value(); }); + + std::unordered_set<std::uint32_t> WriteOperands; + for (const SchedulingWriteLatency &Latency : Class.WriteLatencies) { + if (!WriteOperands.insert(Latency.OperandIndex).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate write latencies"); + } + } + std::ranges::sort(Class.WriteLatencies, {}, &SchedulingWriteLatency::OperandIndex); + + std::unordered_set<std::uint64_t> ReadAdvanceKeys; + for (const SchedulingReadAdvance &Advance : Class.ReadAdvances) { + if (Advance.ProducerClass && !Result.ClassToIndex.contains(*Advance.ProducerClass)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "read advance references an unknown producer class"); + } + const std::uint64_t Producer = Advance.ProducerClass ? Advance.ProducerClass->value() : 0; + const std::uint64_t Key = (Producer << 32U) | Advance.OperandIndex; + if (!ReadAdvanceKeys.insert(Key).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate read advances"); + } + } + std::ranges::sort(Class.ReadAdvances, + [](const SchedulingReadAdvance &Left, const SchedulingReadAdvance &Right) { + if (Left.OperandIndex != Right.OperandIndex) + return Left.OperandIndex < Right.OperandIndex; + return Left.ProducerClass.value_or(SchedulingClassID{}).value() < + Right.ProducerClass.value_or(SchedulingClassID{}).value(); + }); + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.PressureSets.size(); ++Index) { + RegisterPressureSetDesc &Set = Result.Description.PressureSets[Index]; + if (!isValidID(Set.ID) || Set.Name.empty() || Set.Limit == 0 || Set.RegisterClasses.empty() || + !Result.PressureSetToIndex.emplace(Set.ID, Index).second || + !Names.insert(Set.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid register pressure set"); + } + std::ranges::sort(Set.RegisterClasses, [](RegisterClassID Left, RegisterClassID Right) { + return Left.value() < Right.value(); + }); + if (std::ranges::adjacent_find(Set.RegisterClasses) != Set.RegisterClasses.end() || + std::ranges::any_of(Set.RegisterClasses, + [](RegisterClassID Class) { return !isValidID(Class); })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register pressure set has duplicate or invalid register classes"); + } + for (const RegisterClassID Class : Set.RegisterClasses) + Result.ClassToPressureSets[Class].push_back(Set.ID); + } + for (auto &[Class, Sets] : Result.ClassToPressureSets) { + (void)Class; + std::ranges::sort(Sets, [](RegisterPressureSetID Left, RegisterPressureSetID Right) { + return Left.value() < Right.value(); + }); + } + return Result; +} + +const ProcessorResourceDesc * +TargetSchedulingModel::processorResource(ProcessorResourceID ID) const noexcept { + const auto Position = ResourceToIndex.find(ID); + return Position == ResourceToIndex.end() ? nullptr + : &Description.ProcessorResources[Position->second]; +} + +const SchedulingClassModelDesc * +TargetSchedulingModel::schedulingClass(SchedulingClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + return Position == ClassToIndex.end() ? nullptr + : &Description.SchedulingClasses[Position->second]; +} + +const RegisterPressureSetDesc * +TargetSchedulingModel::pressureSet(RegisterPressureSetID ID) const noexcept { + const auto Position = PressureSetToIndex.find(ID); + return Position == PressureSetToIndex.end() ? nullptr + : &Description.PressureSets[Position->second]; +} + +std::span<const RegisterPressureSetID> +TargetSchedulingModel::pressureSetsForClass(RegisterClassID Class) const noexcept { + const auto Position = ClassToPressureSets.find(Class); + return Position == ClassToPressureSets.end() + ? std::span<const RegisterPressureSetID>{} + : std::span<const RegisterPressureSetID>(Position->second); +} + +Expected<std::uint32_t> TargetSchedulingModel::operandLatency(SchedulingClassID ProducerClass, + std::size_t DefinitionOperand, + SchedulingClassID ConsumerClass, + std::size_t UseOperand) const { + if (DefinitionOperand > std::numeric_limits<std::uint32_t>::max() || + UseOperand > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduling operand index exceeds the target model range"); + } + const SchedulingClassModelDesc *Producer = schedulingClass(ProducerClass); + const SchedulingClassModelDesc *Consumer = schedulingClass(ConsumerClass); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand latency references an unknown scheduling class"); + } + + const std::uint32_t DefinitionIndex = static_cast<std::uint32_t>(DefinitionOperand); + const std::uint32_t UseIndex = static_cast<std::uint32_t>(UseOperand); + std::uint32_t Latency = Producer->Latency; + if (const auto Position = std::ranges::find(Producer->WriteLatencies, DefinitionIndex, + &SchedulingWriteLatency::OperandIndex); + Position != Producer->WriteLatencies.end()) { + Latency = Position->Cycles; + } + + const SchedulingReadAdvance *Wildcard = nullptr; + const SchedulingReadAdvance *Exact = nullptr; + for (const SchedulingReadAdvance &Advance : Consumer->ReadAdvances) { + if (Advance.OperandIndex != UseIndex) + continue; + if (!Advance.ProducerClass) + Wildcard = &Advance; + else if (*Advance.ProducerClass == ProducerClass) + Exact = &Advance; + } + const SchedulingReadAdvance *Advance = Exact != nullptr ? Exact : Wildcard; + return Advance == nullptr || Advance->Cycles >= Latency ? (Advance == nullptr ? Latency : 0U) + : Latency - Advance->Cycles; +} + +const TargetSchedulingHooks &TargetSchedulingModel::hooks() const noexcept { + static const TargetSchedulingHooks DefaultHooks; + return Description.Hooks == nullptr ? DefaultHooks : *Description.Hooks; +} + +} // namespace rcc |