From 236082a0fe4dcd0362c5c74923a93e3db6a01291 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Thu, 13 Aug 2026 05:45:37 +0800 Subject: [CodeGen] Align pass manager ownership --- backend/lib/CodeGen/CMakeLists.txt | 11 +- .../lib/CodeGen/GlobalISel/InstructionSelect.cpp | 2 +- backend/lib/CodeGen/GlobalISel/Legalizer.cpp | 2 +- backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp | 2 +- backend/lib/CodeGen/InlineSpiller.cpp | 2 +- backend/lib/CodeGen/LiveIntervals.cpp | 4 +- backend/lib/CodeGen/LiveVariables.cpp | 27 +- backend/lib/CodeGen/MachineAnalyses.cpp | 48 +- backend/lib/CodeGen/MachineBlockPlacement.cpp | 20 +- backend/lib/CodeGen/MachineBranchOptimization.cpp | 2 +- backend/lib/CodeGen/MachineBranchRelaxation.cpp | 6 +- backend/lib/CodeGen/MachineCSE.cpp | 12 +- backend/lib/CodeGen/MachineCombiner.cpp | 2 +- backend/lib/CodeGen/MachineCopyPropagation.cpp | 6 +- backend/lib/CodeGen/MachineDCE.cpp | 2 +- backend/lib/CodeGen/MachineFallthrough.cpp | 6 +- backend/lib/CodeGen/MachineFinalization.cpp | 4 +- backend/lib/CodeGen/MachineLICM.cpp | 22 +- backend/lib/CodeGen/MachineMemoryFolding.cpp | 2 +- .../lib/CodeGen/MachineOptimizationPipeline.cpp | 10 +- backend/lib/CodeGen/MachinePass.cpp | 1497 -------------------- backend/lib/CodeGen/MachinePassManager.cpp | 1099 ++++++++++++++ backend/lib/CodeGen/MachinePseudoExpansion.cpp | 26 +- backend/lib/CodeGen/MachineScheduler.cpp | 9 +- backend/lib/CodeGen/MachineShrinkWrapping.cpp | 10 +- backend/lib/CodeGen/MachineSink.cpp | 18 +- backend/lib/CodeGen/MachineStackSlotColoring.cpp | 2 +- backend/lib/CodeGen/MachineTailDuplication.cpp | 2 +- backend/lib/CodeGen/MachineTailMerge.cpp | 2 +- backend/lib/CodeGen/PHIElimination.cpp | 2 +- backend/lib/CodeGen/PostRAScheduler.cpp | 9 +- backend/lib/CodeGen/PrologueEpilogueInsertion.cpp | 2 +- backend/lib/CodeGen/RegAllocBase.h | 7 +- backend/lib/CodeGen/RegAllocBasic.cpp | 8 +- backend/lib/CodeGen/RegAllocBasic.h | 2 +- backend/lib/CodeGen/RegAllocFast.cpp | 6 +- backend/lib/CodeGen/RegAllocPBQP.cpp | 8 +- backend/lib/CodeGen/SlotIndexes.cpp | 4 +- backend/lib/CodeGen/TwoAddressInstructionPass.cpp | 6 +- backend/lib/CodeGen/VirtRegMap.cpp | 4 +- backend/lib/Target/RISCV/RISCVPipeline.cpp | 22 +- backend/lib/Tooling/PipelineDriver.cpp | 8 +- 42 files changed, 1278 insertions(+), 1667 deletions(-) delete mode 100644 backend/lib/CodeGen/MachinePass.cpp create mode 100644 backend/lib/CodeGen/MachinePassManager.cpp (limited to 'backend/lib') diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index 470571d..4488903 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -48,7 +48,7 @@ set( MachineMetadata.cpp MachineOperand.cpp MachineOptimizationPipeline.cpp - MachinePass.cpp + MachinePassManager.cpp MachinePseudoExpansion.cpp MachineRegisterInfo.cpp MachineScheduler.cpp @@ -136,7 +136,10 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h" - "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/IR/PassManager.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/IR/PassInstrumentation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAnalysisManager.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePassManager.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePseudoExpansion.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PrologueEpilogueInsertion.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterPressure.h" @@ -262,7 +265,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassManager.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" @@ -350,7 +353,7 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationPipeline.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassManager.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineScheduler.cpp" diff --git a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp index ff32eda..344e63e 100644 --- a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -194,7 +194,7 @@ const MachinePassContract &InstructionSelect::contract() const noexcept { Expected InstructionSelect::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (Info == nullptr) return std::unexpected(passFailure("instruction select has no target behavior interface", ErrorCode::InvalidArgument)); diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp index 60f388d..7781818 100644 --- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp +++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -175,7 +175,7 @@ const MachinePassContract &Legalizer::contract() const noexcept { Expected Legalizer::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (Info == nullptr) return std::unexpected( passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument)); diff --git a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp index c36c500..041ab74 100644 --- a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp +++ b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -122,7 +122,7 @@ const MachinePassContract &RegBankSelect::contract() const noexcept { Expected RegBankSelect::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (Info == nullptr) return std::unexpected(passFailure("register-bank select has no target behavior interface", ErrorCode::InvalidArgument)); diff --git a/backend/lib/CodeGen/InlineSpiller.cpp b/backend/lib/CodeGen/InlineSpiller.cpp index 14de321..1d5ba10 100644 --- a/backend/lib/CodeGen/InlineSpiller.cpp +++ b/backend/lib/CodeGen/InlineSpiller.cpp @@ -7,7 +7,7 @@ #include "rcc/CodeGen/MachineIRBuilder.h" #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/MachineOperand.h" -#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/MachinePassManager.h" #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/Spiller.h" #include "rcc/CodeGen/TargetDescription.h" diff --git a/backend/lib/CodeGen/LiveIntervals.cpp b/backend/lib/CodeGen/LiveIntervals.cpp index d8cb539..5a01693 100644 --- a/backend/lib/CodeGen/LiveIntervals.cpp +++ b/backend/lib/CodeGen/LiveIntervals.cpp @@ -307,12 +307,12 @@ Expected LiveIntervalsAnalysis::Result::shrinkToUses(Register Reg, bool LiveIntervalsAnalysis::Result::invalidate(const MachineFunction &, const PreservedAnalyses &Preserved) const noexcept { - return !Preserved.preserves(MachineAnalysisManager::id()); + return !Preserved.preserves(MachineFunctionAnalysisManager::id()); } Expected LiveIntervalsAnalysis::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { auto Indexes = Analyses.getResult(Function, Target); if (!Indexes) return std::unexpected(Indexes.error()); diff --git a/backend/lib/CodeGen/LiveVariables.cpp b/backend/lib/CodeGen/LiveVariables.cpp index 7323e0d..201e09e 100644 --- a/backend/lib/CodeGen/LiveVariables.cpp +++ b/backend/lib/CodeGen/LiveVariables.cpp @@ -884,12 +884,12 @@ bool LiveVariablesAnalysis::Result::isUnitLiveOut(const MachineBasicBlock &Block bool LiveVariablesAnalysis::Result::invalidate(const MachineFunction &, const PreservedAnalyses &Preserved) const noexcept { - return !Preserved.preserves(MachineAnalysisManager::id()); + return !Preserved.preserves(MachineFunctionAnalysisManager::id()); } -Expected LiveVariablesAnalysis::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &) { +Expected +LiveVariablesAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &) { return computeLiveVariables(Function, Target); } const MachinePassContract &RebuildMachineLivenessPass::contract() const noexcept { @@ -911,7 +911,7 @@ const MachinePassContract &RebuildMachineLivenessPass::contract() const noexcept Expected RebuildMachineLivenessPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { const MachineFunctionProperties Missing = contract().Requires.missingFrom(Function.properties()); if (!Missing.empty()) { return makeUnexpected(ErrorCode::VerificationError, @@ -976,11 +976,11 @@ Expected verifyTrackedMachineLiveness(const MachineFunction &Function, return {}; } -Expected registerStage11MachineAnalyses(MachineAnalysisManager &Analyses) { +Expected registerStage11MachineAnalyses(MachineFunctionAnalysisManager &Analyses) { const std::array IDs{ - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), }; const std::size_t Registered = static_cast( std::ranges::count_if(IDs, [&Analyses](AnalysisID ID) { return Analyses.isRegistered(ID); })); @@ -994,7 +994,7 @@ Expected registerStage11MachineAnalyses(MachineAnalysisManager &Analyses) auto Candidate = Analyses.forkEmpty(); if (!Candidate) return std::unexpected(Candidate.error()); - MachineAnalysisManager &Staging = **Candidate; + MachineFunctionAnalysisManager &Staging = **Candidate; auto Slots = Staging.registerAnalysis({ .Name = "SlotIndexes", .Dependencies = {}, @@ -1015,8 +1015,8 @@ Expected registerStage11MachineAnalyses(MachineAnalysisManager &Analyses) return std::unexpected(Variables.error()); auto Intervals = Staging.registerAnalysis({ .Name = "LiveIntervals", - .Dependencies = {MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .Dependencies = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, MachinePassChange::Registers, MachinePassChange::CFG, MachinePassChange::BlockLayout, MachinePassChange::TargetState}, @@ -1024,8 +1024,7 @@ Expected registerStage11MachineAnalyses(MachineAnalysisManager &Analyses) }); if (!Intervals) return std::unexpected(Intervals.error()); - Analyses.Registry = std::move(Staging.Registry); - return {}; + return Analyses.replaceRegistrations(std::move(Staging)); } } // namespace rcc diff --git a/backend/lib/CodeGen/MachineAnalyses.cpp b/backend/lib/CodeGen/MachineAnalyses.cpp index 33dc14f..3f09636 100644 --- a/backend/lib/CodeGen/MachineAnalyses.cpp +++ b/backend/lib/CodeGen/MachineAnalyses.cpp @@ -263,7 +263,7 @@ const MachineBasicBlock *MachineDominatorTreeAnalysis::Result::nearestCommonDomi Expected MachineDominatorTreeAnalysis::run(MachineFunction &Function, const TargetDescription &, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { Result ResultValue; auto Built = ResultValue.build(Function); if (!Built) @@ -287,9 +287,9 @@ bool MachineLoopAnalysis::Result::isIrreducible(const MachineBasicBlock &Block) return IrreducibleBlocks.contains(&Block); } -Expected MachineLoopAnalysis::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineLoopAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { auto Dominators = Analyses.getResult(Function, Target); if (!Dominators) return std::unexpected(Dominators.error()); @@ -496,7 +496,7 @@ std::span MachineBranchProbabilityAnalysis::Result::succ Expected MachineBranchProbabilityAnalysis::run(MachineFunction &Function, const TargetDescription &, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { Result ResultValue; const std::vector Blocks = linkedBlocks(Function); const auto Order = blockOrder(Blocks); @@ -522,7 +522,7 @@ MachineBlockFrequencyAnalysis::Result::frequency(const MachineBasicBlock &Block) Expected MachineBlockFrequencyAnalysis::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { auto Dominators = Analyses.getResult(Function, Target); auto Loops = Analyses.getResult(Function, Target); auto Probabilities = Analyses.getResult(Function, Target); @@ -628,8 +628,9 @@ MachineAliasAnalysis::Result::alias(const MachineMemOperand &Left, return Distance >= LowerSize ? MachineAliasResult::NoAlias : MachineAliasResult::PartialAlias; } -Expected -MachineAliasAnalysis::run(MachineFunction &, const TargetDescription &, MachineAnalysisManager &) { +Expected MachineAliasAnalysis::run(MachineFunction &, + const TargetDescription &, + MachineFunctionAnalysisManager &) { return Result{}; } @@ -659,7 +660,7 @@ std::span MachineMemoryDependenceAnalysis::Result Expected MachineMemoryDependenceAnalysis::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { auto Alias = Analyses.getResult(Function, Target); if (!Alias) return std::unexpected(Alias.error()); @@ -766,14 +767,14 @@ MachineMemoryDependenceAnalysis::run(MachineFunction &Function, const TargetDesc return ResultValue; } -Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { +Expected registerStage8MachineAnalyses(MachineFunctionAnalysisManager &Analyses) { const std::array IDs{ - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), }; const std::size_t Registered = static_cast( std::ranges::count_if(IDs, [&Analyses](AnalysisID ID) { return Analyses.isRegistered(ID); })); @@ -787,7 +788,7 @@ Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { auto Candidate = Analyses.forkEmpty(); if (!Candidate) return std::unexpected(Candidate.error()); - MachineAnalysisManager &Staging = **Candidate; + MachineFunctionAnalysisManager &Staging = **Candidate; auto Dominators = Staging.registerAnalysis({ .Name = "MachineDominatorTree", @@ -799,7 +800,7 @@ Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { return std::unexpected(Dominators.error()); auto Loops = Staging.registerAnalysis({ .Name = "MachineLoopInfo", - .Dependencies = {MachineAnalysisManager::id()}, + .Dependencies = {MachineFunctionAnalysisManager::id()}, .InvalidatedBy = {MachinePassChange::CFG, MachinePassChange::BlockLayout}, .ThreadSafety = PassThreadSafety::ThreadSafe, }); @@ -815,9 +816,9 @@ Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { return std::unexpected(Probabilities.error()); auto Frequencies = Staging.registerAnalysis({ .Name = "MachineBlockFrequency", - .Dependencies = {MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .Dependencies = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .InvalidatedBy = {MachinePassChange::CFG, MachinePassChange::BlockLayout}, .ThreadSafety = PassThreadSafety::ThreadSafe, }); @@ -835,7 +836,7 @@ Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { return std::unexpected(Alias.error()); auto MemoryDependence = Staging.registerAnalysis({ .Name = "MachineMemoryDependence", - .Dependencies = {MachineAnalysisManager::id()}, + .Dependencies = {MachineFunctionAnalysisManager::id()}, .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, MachinePassChange::CFG, MachinePassChange::BlockLayout, MachinePassChange::Frame, MachinePassChange::ConstantPool, @@ -844,8 +845,7 @@ Expected registerStage8MachineAnalyses(MachineAnalysisManager &Analyses) { }); if (!MemoryDependence) return std::unexpected(MemoryDependence.error()); - Analyses.Registry = std::move(Staging.Registry); - return {}; + return Analyses.replaceRegistrations(std::move(Staging)); } } // namespace rcc diff --git a/backend/lib/CodeGen/MachineBlockPlacement.cpp b/backend/lib/CodeGen/MachineBlockPlacement.cpp index daccba0..1e563e1 100644 --- a/backend/lib/CodeGen/MachineBlockPlacement.cpp +++ b/backend/lib/CodeGen/MachineBlockPlacement.cpp @@ -37,10 +37,10 @@ enum class PlacementClass : std::uint8_t { .Clears = {}, .RequiredAnalyses = { - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), }, .VerificationStage = PipelineStage::Selected, }; @@ -101,9 +101,9 @@ const MachinePassContract &MachineBlockPlacementPass::contract() const noexcept return placementContract(); } -Expected MachineBlockPlacementPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineBlockPlacementPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "machine block placement requires selected PHI-free MachineIR"); @@ -113,7 +113,7 @@ Expected MachineBlockPlacementPass::run(MachineFunction &Func MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); if (!Trial) return std::unexpected(prefixedError(Trial.error(), "block placement setup failed: ")); - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; MachineActionReplay Actions(Analyses, Trial->InstructionStableIDs); auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses, Actions); if (!TrialResult) @@ -131,8 +131,8 @@ Expected MachineBlockPlacementPass::run(MachineFunction &Func } Expected MachineBlockPlacementPass::runImplementation( - MachineFunction &Function, const TargetDescription &Target, MachineAnalysisManager &Analyses, - MachineActionReplay &Actions) const { + MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses, MachineActionReplay &Actions) const { auto Probabilities = Analyses.getResult(Function, Target); if (!Probabilities) return std::unexpected(Probabilities.error()); diff --git a/backend/lib/CodeGen/MachineBranchOptimization.cpp b/backend/lib/CodeGen/MachineBranchOptimization.cpp index 5880dbc..c102931 100644 --- a/backend/lib/CodeGen/MachineBranchOptimization.cpp +++ b/backend/lib/CodeGen/MachineBranchOptimization.cpp @@ -301,7 +301,7 @@ const MachinePassContract &MachineBranchOptimizationPass::contract() const noexc Expected MachineBranchOptimizationPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "branch optimization requires selected PHI-free MachineIR"); diff --git a/backend/lib/CodeGen/MachineBranchRelaxation.cpp b/backend/lib/CodeGen/MachineBranchRelaxation.cpp index 78640f1..c11ed8e 100644 --- a/backend/lib/CodeGen/MachineBranchRelaxation.cpp +++ b/backend/lib/CodeGen/MachineBranchRelaxation.cpp @@ -107,9 +107,9 @@ const MachinePassContract &MachineBranchRelaxationPass::contract() const noexcep return relaxationContract(); } -Expected MachineBranchRelaxationPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineBranchRelaxationPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "branch relaxation requires finalized frame and tracked liveness"); diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp index 9d80b4e..92fe1bd 100644 --- a/backend/lib/CodeGen/MachineCSE.cpp +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -33,7 +33,7 @@ namespace { .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::Selected, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -314,7 +314,7 @@ const MachinePassContract &MachineCSEPass::contract() const noexcept { Expected MachineCSEPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "machine CSE requires selected SSA MachineIR"); @@ -325,7 +325,7 @@ Expected MachineCSEPass::run(MachineFunction &Function, if (!Trial) { return std::unexpected(prefixedError(Trial.error(), "machine CSE transaction setup failed: ")); } - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); if (!TrialResult) return std::unexpected(TrialResult.error()); @@ -338,9 +338,9 @@ Expected MachineCSEPass::run(MachineFunction &Function, return Result; } -Expected MachineCSEPass::runImplementation(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineCSEPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { auto Dominators = Analyses.getResult(Function, Target); if (!Dominators) return std::unexpected(Dominators.error()); diff --git a/backend/lib/CodeGen/MachineCombiner.cpp b/backend/lib/CodeGen/MachineCombiner.cpp index df9c002..c3e9795 100644 --- a/backend/lib/CodeGen/MachineCombiner.cpp +++ b/backend/lib/CodeGen/MachineCombiner.cpp @@ -295,7 +295,7 @@ const MachinePassContract &MachineCombinerPass::contract() const noexcept { Expected MachineCombinerPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "machine combiner input does not satisfy its stage contract"); diff --git a/backend/lib/CodeGen/MachineCopyPropagation.cpp b/backend/lib/CodeGen/MachineCopyPropagation.cpp index 581cb4f..9725399 100644 --- a/backend/lib/CodeGen/MachineCopyPropagation.cpp +++ b/backend/lib/CodeGen/MachineCopyPropagation.cpp @@ -290,9 +290,9 @@ const MachinePassContract &MachineCopyPropagationPass::contract() const noexcept return copyPropagationContract(Stage); } -Expected MachineCopyPropagationPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineCopyPropagationPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "machine COPY propagation has unmet stage properties"); diff --git a/backend/lib/CodeGen/MachineDCE.cpp b/backend/lib/CodeGen/MachineDCE.cpp index 4dd0e59..1c53c31 100644 --- a/backend/lib/CodeGen/MachineDCE.cpp +++ b/backend/lib/CodeGen/MachineDCE.cpp @@ -115,7 +115,7 @@ const MachinePassContract &MachineDCEPass::contract() const noexcept { Expected MachineDCEPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "machine DCE requires a selected machine function"); diff --git a/backend/lib/CodeGen/MachineFallthrough.cpp b/backend/lib/CodeGen/MachineFallthrough.cpp index 5c58e67..608b703 100644 --- a/backend/lib/CodeGen/MachineFallthrough.cpp +++ b/backend/lib/CodeGen/MachineFallthrough.cpp @@ -26,7 +26,7 @@ namespace { MachineFunctionProperty::NoVRegs, MachineFunctionProperty::TracksLiveness}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -120,7 +120,7 @@ const MachinePassContract &MachineFallthroughPass::contract() const noexcept { Expected MachineFallthroughPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "fallthrough normalization requires allocated tracked-liveness MIR"); @@ -130,7 +130,7 @@ Expected MachineFallthroughPass::run(MachineFunction &Functio MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); if (!Trial) return std::unexpected(prefixedError(Trial.error(), "fallthrough setup failed: ")); - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialLiveness = TrialAnalyses.getResult(*Trial->ClonedFunction, Target); if (!TrialLiveness) diff --git a/backend/lib/CodeGen/MachineFinalization.cpp b/backend/lib/CodeGen/MachineFinalization.cpp index 8947b86..c33d857 100644 --- a/backend/lib/CodeGen/MachineFinalization.cpp +++ b/backend/lib/CodeGen/MachineFinalization.cpp @@ -91,7 +91,7 @@ const MachinePassContract &FinalizeEmissionPseudosPass::contract() const noexcep Expected FinalizeEmissionPseudosPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::VerificationError, "emission-pseudo finalization requires finalized allocated MIR"); @@ -124,7 +124,7 @@ const MachinePassContract &FinalEncodabilityVerifierPass::contract() const noexc Expected FinalEncodabilityVerifierPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::FinalEncodabilityFailed, "final encodability requires NoExecutablePseudos and finalized MIR"); diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp index 758e13c..31d792b 100644 --- a/backend/lib/CodeGen/MachineLICM.cpp +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -33,9 +33,9 @@ namespace { .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::Selected, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -50,9 +50,9 @@ namespace { MachineFunctionProperty::TracksLiveness}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id(), - MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -332,7 +332,7 @@ const MachinePassContract &MachineLICMPass::contract() const noexcept { Expected MachineLICMPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected( ErrorCode::VerificationError, @@ -346,7 +346,7 @@ Expected MachineLICMPass::run(MachineFunction &Function, if (!Trial) { return std::unexpected(prefixedError(Trial.error(), "machine LICM transaction setup failed: ")); } - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); if (!TrialResult) return std::unexpected(TrialResult.error()); @@ -359,9 +359,9 @@ Expected MachineLICMPass::run(MachineFunction &Function, return Result; } -Expected MachineLICMPass::runImplementation(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineLICMPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { auto Loops = Analyses.getResult(Function, Target); if (!Loops) return std::unexpected(Loops.error()); diff --git a/backend/lib/CodeGen/MachineMemoryFolding.cpp b/backend/lib/CodeGen/MachineMemoryFolding.cpp index 0e15146..e3d796a 100644 --- a/backend/lib/CodeGen/MachineMemoryFolding.cpp +++ b/backend/lib/CodeGen/MachineMemoryFolding.cpp @@ -185,7 +185,7 @@ const MachinePassContract &MachineMemoryFoldingPass::contract() const noexcept { Expected MachineMemoryFoldingPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "memory folding requires selected SSA MachineIR"); diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp index 4f116f0..4a9d7f5 100644 --- a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp +++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp @@ -32,13 +32,13 @@ namespace rcc { namespace { template -[[nodiscard]] Expected addPass(MachinePassManager &Pipeline, Arguments &&...Values) { +[[nodiscard]] Expected addPass(MachineFunctionPassManager &Pipeline, Arguments &&...Values) { return Pipeline.addPass(std::make_unique(std::forward(Values)...)); } } // namespace -Expected registerMachineOptimizationAnalyses(MachineAnalysisManager &Analyses) { +Expected registerMachineOptimizationAnalyses(MachineFunctionAnalysisManager &Analyses) { auto Registered = registerStage8MachineAnalyses(Analyses); if (!Registered) return std::unexpected(Registered.error()); @@ -46,7 +46,7 @@ Expected registerMachineOptimizationAnalyses(MachineAnalysisManager &Analy } Expected -addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, +addSelectedSSAOptimizationPipeline(MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { auto Added = addPass(Pipeline, TargetInfo.Combiner, MachineCombinerStage::SelectedSSA); @@ -68,7 +68,7 @@ addSelectedSSAOptimizationPipeline(MachinePassManager &Pipeline, } Expected addPreRegisterAllocationOptimizationPipeline( - MachinePassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { + MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { auto Added = addPass(Pipeline, TargetInfo.InstrInfo); if (!Added) return std::unexpected(Added.error()); @@ -87,7 +87,7 @@ Expected addPreRegisterAllocationOptimizationPipeline( } Expected -addPostRegisterAllocationPipeline(MachinePassManager &Pipeline, +addPostRegisterAllocationPipeline(MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { auto Added = addPass(Pipeline); if (!Added) diff --git a/backend/lib/CodeGen/MachinePass.cpp b/backend/lib/CodeGen/MachinePass.cpp deleted file mode 100644 index 0ce6f70..0000000 --- a/backend/lib/CodeGen/MachinePass.cpp +++ /dev/null @@ -1,1497 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "rcc/CodeGen/MachinePass.h" - -#include "rcc/CodeGen/CompilationUnit.h" -#include "rcc/CodeGen/MachineBasicBlock.h" -#include "rcc/CodeGen/MachineFunction.h" -#include "rcc/CodeGen/MachineFunctionCloner.h" -#include "rcc/CodeGen/MachineInstr.h" -#include "rcc/CodeGen/MachineOperand.h" -#include "rcc/CodeGen/MachineVerifier.h" -#include "rcc/CodeGen/TargetDescription.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rcc { - -thread_local MachinePassInstrumentation::BufferedEvents *MachinePassInstrumentation::ActiveBuffer = - nullptr; - -namespace { - -using Clock = std::chrono::steady_clock; - -[[nodiscard]] std::uint64_t elapsedNanoseconds(Clock::time_point Start) noexcept { - const auto Elapsed = std::chrono::duration_cast(Clock::now() - Start); - if (Elapsed.count() <= 0) - return 0; - return static_cast(Elapsed.count()); -} - -[[nodiscard]] Error prefixedError(const Error &Source, std::string Prefix) { - Prefix.append(Source.message()); - return Error(Source.code(), std::move(Prefix)); -} - -[[nodiscard]] bool hasDuplicateAnalysisIDs(std::span IDs) { - std::unordered_set Unique; - for (const AnalysisID ID : IDs) { - if (ID == nullptr || !Unique.insert(ID).second) - return true; - } - return false; -} - -[[nodiscard]] Expected validateParameters(std::span Parameters) { - std::unordered_set Names; - for (const PassParameter &Parameter : Parameters) { - if (Parameter.Name.empty()) - return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter name cannot be empty"); - if (!Names.insert(Parameter.Name).second) - return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter names must be unique"); - if (Parameter.Required && Parameter.DefaultValue) { - return makeUnexpected(ErrorCode::InvalidArgument, - "a required pass parameter cannot have a default value"); - } - } - return {}; -} - -void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { - State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); -} - -[[nodiscard]] std::uint64_t machineFunctionFingerprint(const MachineFunction &Function) noexcept { - std::uint64_t Result = 0xcbf29ce484222325ULL; - combineFingerprint(Result, Function.stableID().value()); - combineFingerprint(Result, Function.properties().rawValue()); - combineFingerprint(Result, Function.blockCount()); - combineFingerprint(Result, Function.allInstructions().size()); - combineFingerprint(Result, Function.allMemOperands().size()); - - for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; - Block = Block->next()) { - combineFingerprint(Result, Block->stableID().value()); - combineFingerprint(Result, Block->instructionCount()); - combineFingerprint(Result, Block->successors().size()); - for (const MachineSuccessor &Successor : Block->successors()) { - combineFingerprint(Result, - Successor.Block == nullptr ? 0 : Successor.Block->stableID().value()); - combineFingerprint(Result, Successor.Probability.rawValue()); - } - for (const MachineBasicBlockLiveIn &LiveIn : Block->liveIns()) { - combineFingerprint(Result, LiveIn.PhysReg.number()); - combineFingerprint(Result, LiveIn.Lanes.rawValue()); - } - - for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; - Instruction = Instruction->next()) { - combineFingerprint(Result, Instruction->stableID().value()); - combineFingerprint(Result, Instruction->opcode().value()); - combineFingerprint(Result, Instruction->flags().rawValue()); - combineFingerprint(Result, Instruction->operandCount()); - combineFingerprint(Result, Instruction->memOperands().size()); - for (const MachineOperand &Operand : Instruction->operands()) { - combineFingerprint(Result, static_cast(Operand.kind())); - combineFingerprint(Result, Operand.targetFlags()); - combineFingerprint(Result, Operand.registerFlags().rawValue()); - combineFingerprint(Result, Operand.subReg().value()); - if (const Register *Reg = Operand.getIf()) - combineFingerprint(Result, Reg->opaqueValue()); - else if (const std::int64_t *Immediate = Operand.getIf()) - combineFingerprint(Result, static_cast(*Immediate)); - } - } - } - return Result; -} - -[[nodiscard]] Expected verifyCompilationUnit(const CompilationUnit &Unit, - const TargetDescription &Target) { - auto UnitVerification = Unit.verify(); - if (!UnitVerification) - return std::unexpected(UnitVerification.error()); - for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { - const MachineFunction *Function = Unit.functionAt(Index); - if (Function == nullptr) - return makeUnexpected(ErrorCode::InternalError, - "compilation unit function order contains a null entry"); - auto FunctionVerification = MachineVerifier::verify(*Function, Target); - if (!FunctionVerification) - return std::unexpected(FunctionVerification.error()); - } - return {}; -} - -} // namespace - -std::string MachineActionID::toString() const { - std::string Result; - Result.reserve(Pass.size() + Rule.size() + 48U); - Result.append(Pass); - Result.push_back(':'); - Result.append(Rule); - Result.append(":f"); - Result.append(std::to_string(Function.value())); - Result.append(":i"); - Result.append(std::to_string(Instruction.value())); - return Result; -} - -Expected MachineActionReplay::request(std::string_view Rule, const MachineFunction &Function, - MachineInstrStableID Instruction, std::uint64_t Cost) { - if (!IsReplaying) { - const auto Position = RecordingInstructionIDs.find(Instruction); - const MachineInstrStableID CanonicalInstruction = - Position == RecordingInstructionIDs.end() ? Instruction : Position->second; - auto Accepted = Analyses->requestAction(Rule, Function, CanonicalInstruction, Cost); - if (!Accepted) - return std::unexpected(Accepted.error()); - Decisions.push_back({ - .Rule = std::string(Rule), - .Function = Function.stableID(), - .Instruction = CanonicalInstruction, - .Cost = Cost, - .Accepted = *Accepted, - }); - return *Accepted; - } - - if (NextDecision >= Decisions.size()) { - return makeUnexpected(ErrorCode::InternalError, - "machine action replay requested an unrecorded decision"); - } - const Decision &Recorded = Decisions[NextDecision++]; - if (Recorded.Rule != Rule || Recorded.Function != Function.stableID() || - Recorded.Instruction != Instruction || Recorded.Cost != Cost) { - std::string Message = "machine action replay diverged from transaction preflight: recorded "; - Message.append(Recorded.Rule); - Message.append(":f"); - Message.append(std::to_string(Recorded.Function.value())); - Message.append(":i"); - Message.append(std::to_string(Recorded.Instruction.value())); - Message.append(" cost="); - Message.append(std::to_string(Recorded.Cost)); - Message.append(", requested "); - Message.append(Rule); - Message.append(":f"); - Message.append(std::to_string(Function.stableID().value())); - Message.append(":i"); - Message.append(std::to_string(Instruction.value())); - Message.append(" cost="); - Message.append(std::to_string(Cost)); - return makeUnexpected(ErrorCode::InternalError, std::move(Message)); - } - return Recorded.Accepted; -} - -void MachineActionReplay::beginReplay() noexcept { - IsReplaying = true; - NextDecision = 0; -} - -Expected MachineActionReplay::finishReplay() const { - if (!IsReplaying || NextDecision != Decisions.size()) { - return makeUnexpected(ErrorCode::InternalError, - "machine action replay did not consume every recorded decision"); - } - return {}; -} - -Expected -MachineAnalysisManager::registerAnalysisImpl(MachineAnalysisRegistration Registration, - ErasedRunner Runner, ErasedInvalidator Invalidator) { - if (Registration.ID == nullptr || Registration.Name.empty() || !Runner || !Invalidator) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine analysis registration is incomplete"); - } - if (hasDuplicateAnalysisIDs(Registration.Dependencies)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine analysis dependencies must be non-null and unique"); - } - if (std::ranges::find(Registration.Dependencies, Registration.ID) != - Registration.Dependencies.end()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine analysis cannot directly depend on itself"); - } - if (Registry.contains(Registration.ID)) - return makeUnexpected(ErrorCode::InvalidArgument, "machine analysis ID is already registered"); - if (std::ranges::any_of(Registry, [&Registration](const auto &Entry) { - return Entry.second.Info.Name == Registration.Name; - })) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine analysis name is already registered"); - } - const AnalysisID RegistrationID = Registration.ID; - Registry.emplace(RegistrationID, RegisteredAnalysis{std::move(Registration), std::move(Runner), - std::move(Invalidator)}); - return {}; -} - -bool MachineAnalysisManager::isRegistered(AnalysisID ID) const noexcept { - return ID != nullptr && Registry.contains(ID); -} - -bool MachineAnalysisManager::hasCached(const MachineFunction &Function, - AnalysisID ID) const noexcept { - const auto FunctionPosition = Cache.find(const_cast(&Function)); - if (FunctionPosition == Cache.end()) - return false; - const auto AnalysisPosition = FunctionPosition->second.find(ID); - return AnalysisPosition != FunctionPosition->second.end() && AnalysisPosition->second.Value; -} - -std::size_t -MachineAnalysisManager::cachedResultCount(const MachineFunction &Function) const noexcept { - const auto Position = Cache.find(const_cast(&Function)); - return Position == Cache.end() ? 0 : Position->second.size(); -} - -Expected -MachineAnalysisManager::materializeDependencies(const RegisteredAnalysis &Registration, - MachineFunction &Function, - const TargetDescription &Target) { - for (const AnalysisID Dependency : Registration.Info.Dependencies) { - if (!Registry.contains(Dependency)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine analysis depends on an unregistered analysis"); - } - auto Result = getResultImpl(Dependency, Function, Target); - if (!Result) - return std::unexpected(Result.error()); - } - return {}; -} - -Expected MachineAnalysisManager::getResultImpl(AnalysisID ID, MachineFunction &Function, - const TargetDescription &Target) { - if (ID == nullptr) - return makeUnexpected(ErrorCode::InvalidArgument, "machine analysis ID cannot be null"); - - auto &FunctionCache = Cache[&Function]; - if (const auto Cached = FunctionCache.find(ID); - Cached != FunctionCache.end() && Cached->second.Value) { - const auto Registration = Registry.find(ID); - const std::string_view Name = Registration == Registry.end() - ? std::string_view{"unknown-machine-analysis"} - : std::string_view{Registration->second.Info.Name}; - if (Instrumentation != nullptr) - Instrumentation->afterAnalysis(Name, Function, true, 0); - return Cached->second.Value.get(); - } - - const auto RegistrationPosition = Registry.find(ID); - if (RegistrationPosition == Registry.end()) - return makeUnexpected(ErrorCode::InvalidArgument, "machine analysis ID is not registered"); - const RegisteredAnalysis &Registration = RegistrationPosition->second; - - auto &Stack = ActiveComputations[&Function]; - if (std::ranges::find(Stack, ID) != Stack.end()) { - Error Failure(ErrorCode::InvalidArgument, - "machine analysis dependency cycle reaches '" + Registration.Info.Name + "'"); - if (Instrumentation != nullptr) - Instrumentation->analysisFailure(Registration.Info.Name, Function, Failure, 0); - return std::unexpected(std::move(Failure)); - } - - if (Instrumentation != nullptr) - Instrumentation->beforeAnalysis(Registration.Info.Name, Function); - const Clock::time_point Start = Clock::now(); - Stack.push_back(ID); - auto Dependencies = materializeDependencies(Registration, Function, Target); - if (!Dependencies) { - Stack.pop_back(); - if (Stack.empty()) - ActiveComputations.erase(&Function); - Error Failure = prefixedError(Dependencies.error(), - "machine analysis '" + Registration.Info.Name + "' failed: "); - if (Instrumentation != nullptr) { - Instrumentation->analysisFailure(Registration.Info.Name, Function, Failure, - elapsedNanoseconds(Start)); - } - return std::unexpected(std::move(Failure)); - } - - auto Computed = Registration.Runner(Function, Target, *this); - Stack.pop_back(); - if (Stack.empty()) - ActiveComputations.erase(&Function); - const std::uint64_t Duration = elapsedNanoseconds(Start); - if (!Computed) { - Error Failure = prefixedError(Computed.error(), - "machine analysis '" + Registration.Info.Name + "' failed: "); - if (Instrumentation != nullptr) - Instrumentation->analysisFailure(Registration.Info.Name, Function, Failure, Duration); - return std::unexpected(std::move(Failure)); - } - if (!*Computed) { - Error Failure(ErrorCode::InternalError, - "machine analysis '" + Registration.Info.Name + "' returned a null result"); - if (Instrumentation != nullptr) - Instrumentation->analysisFailure(Registration.Info.Name, Function, Failure, Duration); - return std::unexpected(std::move(Failure)); - } - - void *Result = Computed->get(); - FunctionCache.emplace(ID, CacheEntry{std::move(*Computed), ID}); - if (Instrumentation != nullptr) - Instrumentation->afterAnalysis(Registration.Info.Name, Function, false, Duration); - return Result; -} - -void MachineAnalysisManager::invalidate(MachineFunction &Function, - const PreservedAnalyses &Preserved, - MachinePassChanges Changes) noexcept { - const auto FunctionPosition = Cache.find(&Function); - if (FunctionPosition == Cache.end() || Preserved.preservesAll() || Changes.empty()) - return; - - std::set Invalid; - for (auto &[ID, Entry] : FunctionPosition->second) { - if (Preserved.preserves(ID)) - continue; - const auto Registration = Registry.find(ID); - if (Registration == Registry.end() || - Registration->second.Info.InvalidatedBy.intersects(Changes)) { - const bool ShouldInvalidate = - Registration == Registry.end() || !Entry.Value || !Registration->second.Invalidator || - Registration->second.Invalidator(Entry.Value.get(), Function, Preserved); - if (ShouldInvalidate) - Invalid.insert(ID); - } - } - - bool Added = true; - while (Added) { - Added = false; - for (const auto &[ID, Entry] : FunctionPosition->second) { - (void)Entry; - if (Invalid.contains(ID) || Preserved.preserves(ID)) - continue; - const auto Registration = Registry.find(ID); - if (Registration != Registry.end() && - std::ranges::any_of( - Registration->second.Info.Dependencies, - [&Invalid](AnalysisID Dependency) { return Invalid.contains(Dependency); })) { - Added = Invalid.insert(ID).second || Added; - } - } - } - - for (const AnalysisID ID : Invalid) - FunctionPosition->second.erase(ID); - if (FunctionPosition->second.empty()) - Cache.erase(FunctionPosition); -} - -void MachineAnalysisManager::clear(MachineFunction &Function) noexcept { - Cache.erase(&Function); - ActiveComputations.erase(&Function); -} - -void MachineAnalysisManager::clearAll() noexcept { - Cache.clear(); - ActiveComputations.clear(); -} - -Expected> MachineAnalysisManager::forkEmpty() const { - if (!ActiveComputations.empty()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "cannot fork a machine analysis manager during analysis computation"); - } - auto Result = std::make_unique(); - Result->Registry = Registry; - return Result; -} - -Expected MachineAnalysisManager::validateParallelRequirements( - std::span RequiredAnalyses) const { - std::vector Worklist(RequiredAnalyses.begin(), RequiredAnalyses.end()); - std::unordered_set Visited; - for (std::size_t Index = 0; Index < Worklist.size(); ++Index) { - const AnalysisID ID = Worklist[Index]; - if (ID == nullptr || !Visited.insert(ID).second) - continue; - const auto Position = Registry.find(ID); - if (Position == Registry.end()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "parallel machine pass requires an unregistered analysis"); - } - if (Position->second.Info.ThreadSafety != PassThreadSafety::ThreadSafe) { - return makeUnexpected(ErrorCode::InvalidArgument, - "parallel machine pipeline requires analysis '" + - Position->second.Info.Name + "' to be marked thread-safe"); - } - Worklist.insert(Worklist.end(), Position->second.Info.Dependencies.begin(), - Position->second.Info.Dependencies.end()); - } - return {}; -} - -Expected MachineAnalysisManager::requestAction(std::string_view Action, - const MachineFunction &Function, - std::uint64_t Cost) { - return requestAction(Action, Function, MachineInstrStableID{}, Cost); -} - -Expected MachineAnalysisManager::requestAction(std::string_view Rule, - const MachineFunction &Function, - MachineInstrStableID Instruction, - std::uint64_t Cost) { - if (Rule.empty() || Cost == 0) - return makeUnexpected(ErrorCode::InvalidArgument, - "machine action requires a name and nonzero cost"); - if (Instrumentation == nullptr) - return true; - MachineActionID ID{ - .Pass = ActivePass.empty() ? "unscoped" : ActivePass, - .Rule = std::string(Rule), - .Function = Function.stableID(), - .Instruction = Instruction, - }; - return Instrumentation->requestAction(ID, Function, Cost); -} - -Expected CompilationUnitAnalysisManager::registerAnalysisImpl( - CompilationUnitAnalysisRegistration Registration, ErasedRunner Runner, - ErasedInvalidator Invalidator) { - if (Registration.ID == nullptr || Registration.Name.empty() || !Runner || !Invalidator) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis registration is incomplete"); - } - if (hasDuplicateAnalysisIDs(Registration.Dependencies)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis dependencies must be non-null and unique"); - } - if (std::ranges::find(Registration.Dependencies, Registration.ID) != - Registration.Dependencies.end()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis cannot directly depend on itself"); - } - if (Registry.contains(Registration.ID)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis ID is already registered"); - } - if (std::ranges::any_of(Registry, [&Registration](const auto &Entry) { - return Entry.second.Info.Name == Registration.Name; - })) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis name is already registered"); - } - const AnalysisID RegistrationID = Registration.ID; - Registry.emplace(RegistrationID, RegisteredAnalysis{std::move(Registration), std::move(Runner), - std::move(Invalidator)}); - return {}; -} - -bool CompilationUnitAnalysisManager::isRegistered(AnalysisID ID) const noexcept { - return ID != nullptr && Registry.contains(ID); -} - -bool CompilationUnitAnalysisManager::hasCached(const CompilationUnit &Unit, - AnalysisID ID) const noexcept { - const auto UnitPosition = Cache.find(const_cast(&Unit)); - if (UnitPosition == Cache.end()) - return false; - const auto AnalysisPosition = UnitPosition->second.find(ID); - return AnalysisPosition != UnitPosition->second.end() && AnalysisPosition->second.Value; -} - -Expected -CompilationUnitAnalysisManager::materializeDependencies(const RegisteredAnalysis &Registration, - CompilationUnit &Unit, - const TargetDescription &Target) { - for (const AnalysisID Dependency : Registration.Info.Dependencies) { - if (!Registry.contains(Dependency)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis depends on an unregistered analysis"); - } - auto Result = getResultImpl(Dependency, Unit, Target); - if (!Result) - return std::unexpected(Result.error()); - } - return {}; -} - -Expected CompilationUnitAnalysisManager::getResultImpl(AnalysisID ID, CompilationUnit &Unit, - const TargetDescription &Target) { - if (ID == nullptr) - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis ID cannot be null"); - auto &UnitCache = Cache[&Unit]; - if (const auto Cached = UnitCache.find(ID); Cached != UnitCache.end() && Cached->second.Value) { - const auto Registration = Registry.find(ID); - const std::string_view Name = Registration == Registry.end() - ? std::string_view{"unknown-compilation-unit-analysis"} - : std::string_view{Registration->second.Info.Name}; - if (Instrumentation != nullptr) - Instrumentation->afterUnitAnalysis(Name, Unit, true, 0); - return Cached->second.Value.get(); - } - - const auto RegistrationPosition = Registry.find(ID); - if (RegistrationPosition == Registry.end()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit analysis ID is not registered"); - } - const RegisteredAnalysis &Registration = RegistrationPosition->second; - auto &Stack = ActiveComputations[&Unit]; - if (std::ranges::find(Stack, ID) != Stack.end()) { - Error Failure(ErrorCode::InvalidArgument, - "compilation-unit analysis dependency cycle reaches '" + Registration.Info.Name + - "'"); - if (Instrumentation != nullptr) - Instrumentation->unitAnalysisFailure(Registration.Info.Name, Unit, Failure, 0); - return std::unexpected(std::move(Failure)); - } - - if (Instrumentation != nullptr) - Instrumentation->beforeUnitAnalysis(Registration.Info.Name, Unit); - const Clock::time_point Start = Clock::now(); - Stack.push_back(ID); - auto Dependencies = materializeDependencies(Registration, Unit, Target); - if (!Dependencies) { - Stack.pop_back(); - if (Stack.empty()) - ActiveComputations.erase(&Unit); - Error Failure = prefixedError(Dependencies.error(), "compilation-unit analysis '" + - Registration.Info.Name + "' failed: "); - if (Instrumentation != nullptr) { - Instrumentation->unitAnalysisFailure(Registration.Info.Name, Unit, Failure, - elapsedNanoseconds(Start)); - } - return std::unexpected(std::move(Failure)); - } - - auto Computed = Registration.Runner(Unit, Target, *this); - Stack.pop_back(); - if (Stack.empty()) - ActiveComputations.erase(&Unit); - const std::uint64_t Duration = elapsedNanoseconds(Start); - if (!Computed) { - Error Failure = prefixedError(Computed.error(), "compilation-unit analysis '" + - Registration.Info.Name + "' failed: "); - if (Instrumentation != nullptr) - Instrumentation->unitAnalysisFailure(Registration.Info.Name, Unit, Failure, Duration); - return std::unexpected(std::move(Failure)); - } - if (!*Computed) { - Error Failure(ErrorCode::InternalError, "compilation-unit analysis '" + Registration.Info.Name + - "' returned a null result"); - if (Instrumentation != nullptr) - Instrumentation->unitAnalysisFailure(Registration.Info.Name, Unit, Failure, Duration); - return std::unexpected(std::move(Failure)); - } - - void *Result = Computed->get(); - UnitCache.emplace(ID, CacheEntry{std::move(*Computed), ID}); - if (Instrumentation != nullptr) - Instrumentation->afterUnitAnalysis(Registration.Info.Name, Unit, false, Duration); - return Result; -} - -void CompilationUnitAnalysisManager::invalidate(CompilationUnit &Unit, - const PreservedAnalyses &Preserved, - MachinePassChanges Changes) noexcept { - const auto UnitPosition = Cache.find(&Unit); - if (UnitPosition == Cache.end() || Preserved.preservesAll() || Changes.empty()) - return; - - std::set Invalid; - for (auto &[ID, Entry] : UnitPosition->second) { - if (Preserved.preserves(ID)) - continue; - const auto Registration = Registry.find(ID); - if (Registration == Registry.end() || - Registration->second.Info.InvalidatedBy.intersects(Changes)) { - const bool ShouldInvalidate = - Registration == Registry.end() || !Entry.Value || !Registration->second.Invalidator || - Registration->second.Invalidator(Entry.Value.get(), Unit, Preserved); - if (ShouldInvalidate) - Invalid.insert(ID); - } - } - - bool Added = true; - while (Added) { - Added = false; - for (const auto &[ID, Entry] : UnitPosition->second) { - (void)Entry; - if (Invalid.contains(ID) || Preserved.preserves(ID)) - continue; - const auto Registration = Registry.find(ID); - if (Registration != Registry.end() && - std::ranges::any_of( - Registration->second.Info.Dependencies, - [&Invalid](AnalysisID Dependency) { return Invalid.contains(Dependency); })) { - Added = Invalid.insert(ID).second || Added; - } - } - } - for (const AnalysisID InvalidID : Invalid) - UnitPosition->second.erase(InvalidID); - if (UnitPosition->second.empty()) - Cache.erase(UnitPosition); -} - -void CompilationUnitAnalysisManager::clear(CompilationUnit &Unit) noexcept { - Cache.erase(&Unit); - ActiveComputations.erase(&Unit); -} - -void CompilationUnitAnalysisManager::clearAll() noexcept { - Cache.clear(); - ActiveComputations.clear(); -} - -void MachinePassInstrumentation::addBeforeCallback(BeforeCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - Before.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addAfterCallback(AfterCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - After.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addFailureCallback(FailureCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - Failure.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addBeforeAnalysisCallback(BeforeAnalysisCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - BeforeAnalysis.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addAfterAnalysisCallback(AfterAnalysisCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - AfterAnalysis.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addAnalysisFailureCallback(AnalysisFailureCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - AnalysisFailure.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addIRPrintCallback(IRPrintCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - IRPrint.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addChangeCallback(ChangeCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - Change.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::setActionFilter(ActionFilter NewFilter) { - std::scoped_lock Lock(Mutex); - Filter = std::move(NewFilter); -} - -void MachinePassInstrumentation::setPerFunctionFuel(std::optional Fuel) { - std::scoped_lock Lock(Mutex); - PerFunctionFuel = Fuel; - ConsumedFuel.clear(); - ActionEvents.clear(); -} - -void MachinePassInstrumentation::addBeforeUnitCallback(BeforeUnitCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - BeforeUnit.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addAfterUnitCallback(AfterUnitCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - AfterUnit.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addUnitFailureCallback(UnitFailureCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - UnitFailure.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addBeforeUnitAnalysisCallback( - BeforeUnitAnalysisCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - BeforeUnitAnalysis.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addAfterUnitAnalysisCallback(AfterUnitAnalysisCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - AfterUnitAnalysis.push_back(std::move(Callback)); -} - -void MachinePassInstrumentation::addUnitAnalysisFailureCallback( - UnitAnalysisFailureCallback Callback) { - if (!Callback) - return; - std::scoped_lock Lock(Mutex); - UnitAnalysisFailure.push_back(std::move(Callback)); -} - -InstrumentationStatistic &MachinePassInstrumentation::statistic(std::string_view Name, - bool IsAnalysis) const { - const std::string Key = std::string(IsAnalysis ? "analysis:" : "pass:") + std::string(Name); - auto [Position, Inserted] = Statistics.try_emplace(Key); - if (Inserted) { - Position->second.Name = std::string(Name); - Position->second.IsAnalysis = IsAnalysis; - } - return Position->second; -} - -bool MachinePassInstrumentation::bufferFunctionEvent( - const MachineFunction &Function, std::function Replay) const { - if (ActiveBuffer == nullptr) - return false; - - std::shared_ptr Snapshot; - auto Cloned = cloneMachineFunction( - Function, *ActiveBuffer->Target, - MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); - if (Cloned) { - Snapshot = std::shared_ptr(std::move(Cloned->ClonedFunction)); - } else { - Snapshot = - std::shared_ptr(&Function, [](const MachineFunction *) noexcept {}); - } - ActiveBuffer->Events.emplace_back( - [Snapshot = std::move(Snapshot), Replay = std::move(Replay)]() { Replay(*Snapshot); }); - return true; -} - -void MachinePassInstrumentation::beginBuffering(BufferedEvents &Buffer, - const TargetDescription &Target) const noexcept { - Buffer.Target = &Target; - Buffer.Events.clear(); - ActiveBuffer = &Buffer; -} - -void MachinePassInstrumentation::endBuffering() const noexcept { - ActiveBuffer = nullptr; -} - -void MachinePassInstrumentation::replayBufferedEvents(BufferedEvents &Buffer) const { - for (const std::function &Event : Buffer.Events) - Event(); - Buffer.Events.clear(); - Buffer.Target = nullptr; -} - -void MachinePassInstrumentation::before(std::string_view Name, - const MachineFunction &Function) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name)](const MachineFunction &Input) { - before(Name, Input); - })) { - return; - } - std::vector Callbacks; - std::vector Printers; - { - std::scoped_lock Lock(Mutex); - Callbacks = Before; - Printers = IRPrint; - } - for (const BeforeCallback &Callback : Callbacks) - Callback(Name, Function); - for (const IRPrintCallback &Printer : Printers) - Printer(InstrumentationPhase::Before, Name, Function); -} - -void MachinePassInstrumentation::after(std::string_view Name, const MachineFunction &Function, - const MachinePassResult &Result, - std::uint64_t Nanoseconds) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name), Result, - Nanoseconds](const MachineFunction &Input) { - after(Name, Input, Result, Nanoseconds); - })) { - return; - } - std::vector Callbacks; - std::vector Printers; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic(Name, false); - ++Record.Invocations; - Record.Changes += Result.Changed ? 1U : 0U; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = After; - Printers = IRPrint; - } - for (const AfterCallback &Callback : Callbacks) - Callback(Name, Function, Result); - for (const IRPrintCallback &Printer : Printers) - Printer(InstrumentationPhase::After, Name, Function); -} - -void MachinePassInstrumentation::failure(std::string_view Name, const MachineFunction &Function, - const Error &ErrorValue, std::uint64_t Nanoseconds) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name), ErrorValue, - Nanoseconds](const MachineFunction &Input) { - failure(Name, Input, ErrorValue, Nanoseconds); - })) { - return; - } - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic(Name, false); - ++Record.Invocations; - ++Record.Failures; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = Failure; - } - for (const FailureCallback &Callback : Callbacks) - Callback(Name, Function, ErrorValue); -} - -void MachinePassInstrumentation::beforeAnalysis(std::string_view Name, - const MachineFunction &Function) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name)](const MachineFunction &Input) { - beforeAnalysis(Name, Input); - })) { - return; - } - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - Callbacks = BeforeAnalysis; - } - for (const BeforeAnalysisCallback &Callback : Callbacks) - Callback(Name, Function); -} - -void MachinePassInstrumentation::afterAnalysis(std::string_view Name, - const MachineFunction &Function, bool CacheHit, - std::uint64_t Nanoseconds) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name), CacheHit, - Nanoseconds](const MachineFunction &Input) { - afterAnalysis(Name, Input, CacheHit, Nanoseconds); - })) { - return; - } - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic(Name, true); - ++Record.Invocations; - Record.CacheHits += CacheHit ? 1U : 0U; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = AfterAnalysis; - } - for (const AfterAnalysisCallback &Callback : Callbacks) - Callback(Name, Function, CacheHit, Nanoseconds); -} - -void MachinePassInstrumentation::analysisFailure(std::string_view Name, - const MachineFunction &Function, - const Error &ErrorValue, - std::uint64_t Nanoseconds) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name), ErrorValue, - Nanoseconds](const MachineFunction &Input) { - analysisFailure(Name, Input, ErrorValue, Nanoseconds); - })) { - return; - } - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic(Name, true); - ++Record.Invocations; - ++Record.Failures; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = AnalysisFailure; - } - for (const AnalysisFailureCallback &Callback : Callbacks) - Callback(Name, Function, ErrorValue); -} - -void MachinePassInstrumentation::change(std::string_view Name, const MachineFunction &Function, - MachinePassChanges Changes, bool FingerprintChanged) const { - if (bufferFunctionEvent(Function, [this, Name = std::string(Name), Changes, - FingerprintChanged](const MachineFunction &Input) { - change(Name, Input, Changes, FingerprintChanged); - })) { - return; - } - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - Callbacks = Change; - } - for (const ChangeCallback &Callback : Callbacks) - Callback(Name, Function, Changes, FingerprintChanged); -} - -Expected MachinePassInstrumentation::requestAction(const MachineActionID &Action, - const MachineFunction &Function, - std::uint64_t Cost) { - if (Action.Pass.empty() || Action.Rule.empty() || Action.Function != Function.stableID() || - Cost == 0) { - return makeUnexpected(ErrorCode::InvalidArgument, "machine action identity is invalid"); - } - std::scoped_lock ActionLock(ActionMutex); - ActionFilter CurrentFilter; - { - std::scoped_lock Lock(Mutex); - CurrentFilter = Filter; - } - if (CurrentFilter && !CurrentFilter(Action, Cost)) { - std::scoped_lock Lock(Mutex); - ActionEvents.push_back({Action, Cost, false}); - return false; - } - - std::scoped_lock Lock(Mutex); - if (!PerFunctionFuel) { - ActionEvents.push_back({Action, Cost, true}); - return true; - } - std::uint64_t &Consumed = ConsumedFuel[Function.stableID().value()]; - if (Consumed > *PerFunctionFuel || Cost > *PerFunctionFuel - Consumed) { - ActionEvents.push_back({Action, Cost, false}); - return false; - } - Consumed += Cost; - ActionEvents.push_back({Action, Cost, true}); - return true; -} - -void MachinePassInstrumentation::beforeUnit(std::string_view Name, - const CompilationUnit &Unit) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - Callbacks = BeforeUnit; - } - for (const BeforeUnitCallback &Callback : Callbacks) - Callback(Name, Unit); -} - -void MachinePassInstrumentation::afterUnit(std::string_view Name, const CompilationUnit &Unit, - const CompilationUnitPassResult &Result, - std::uint64_t Nanoseconds) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); - ++Record.Invocations; - Record.Changes += Result.Changed ? 1U : 0U; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = AfterUnit; - } - for (const AfterUnitCallback &Callback : Callbacks) - Callback(Name, Unit, Result); -} - -void MachinePassInstrumentation::unitFailure(std::string_view Name, const CompilationUnit &Unit, - const Error &ErrorValue, - std::uint64_t Nanoseconds) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); - ++Record.Invocations; - ++Record.Failures; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = UnitFailure; - } - for (const UnitFailureCallback &Callback : Callbacks) - Callback(Name, Unit, ErrorValue); -} - -void MachinePassInstrumentation::beforeUnitAnalysis(std::string_view Name, - const CompilationUnit &Unit) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - Callbacks = BeforeUnitAnalysis; - } - for (const BeforeUnitAnalysisCallback &Callback : Callbacks) - Callback(Name, Unit); -} - -void MachinePassInstrumentation::afterUnitAnalysis(std::string_view Name, - const CompilationUnit &Unit, bool CacheHit, - std::uint64_t Nanoseconds) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); - ++Record.Invocations; - Record.CacheHits += CacheHit ? 1U : 0U; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = AfterUnitAnalysis; - } - for (const AfterUnitAnalysisCallback &Callback : Callbacks) - Callback(Name, Unit, CacheHit, Nanoseconds); -} - -void MachinePassInstrumentation::unitAnalysisFailure(std::string_view Name, - const CompilationUnit &Unit, - const Error &ErrorValue, - std::uint64_t Nanoseconds) const { - std::vector Callbacks; - { - std::scoped_lock Lock(Mutex); - InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); - ++Record.Invocations; - ++Record.Failures; - Record.TotalNanoseconds += Nanoseconds; - Callbacks = UnitAnalysisFailure; - } - for (const UnitAnalysisFailureCallback &Callback : Callbacks) - Callback(Name, Unit, ErrorValue); -} - -std::vector MachinePassInstrumentation::statistics() const { - std::vector Result; - { - std::scoped_lock Lock(Mutex); - Result.reserve(Statistics.size()); - for (const auto &[Key, Record] : Statistics) { - (void)Key; - Result.push_back(Record); - } - } - std::ranges::sort( - Result, [](const InstrumentationStatistic &Left, const InstrumentationStatistic &Right) { - if (Left.IsAnalysis != Right.IsAnalysis) - return Left.IsAnalysis < Right.IsAnalysis; - return Left.Name < Right.Name; - }); - return Result; -} - -std::vector MachinePassInstrumentation::actionEvents() const { - std::vector Result; - { - std::scoped_lock Lock(Mutex); - Result = ActionEvents; - } - std::ranges::sort(Result, [](const MachineActionEvent &Left, const MachineActionEvent &Right) { - if (Left.ID.Function.value() != Right.ID.Function.value()) - return Left.ID.Function.value() < Right.ID.Function.value(); - if (Left.ID.Instruction.value() != Right.ID.Instruction.value()) - return Left.ID.Instruction.value() < Right.ID.Instruction.value(); - if (Left.ID.Pass != Right.ID.Pass) - return Left.ID.Pass < Right.ID.Pass; - if (Left.ID.Rule != Right.ID.Rule) - return Left.ID.Rule < Right.ID.Rule; - if (Left.Cost != Right.Cost) - return Left.Cost < Right.Cost; - return Left.Accepted < Right.Accepted; - }); - return Result; -} - -void MachinePassInstrumentation::clearStatistics() { - std::scoped_lock Lock(Mutex); - Statistics.clear(); - ConsumedFuel.clear(); - ActionEvents.clear(); -} - -Expected MachinePassManager::validateContract(const MachinePassContract &Contract) const { - if (Contract.Name.empty()) - return makeUnexpected(ErrorCode::InvalidArgument, "machine pass must have a non-empty name"); - if (Contract.Sets.intersects(Contract.Clears)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine pass cannot set and clear the same property"); - } - if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "machine pass required analyses must be non-null and unique"); - } - return validateParameters(Contract.Parameters); -} - -Expected MachinePassManager::addPass(std::unique_ptr Pass) { - if (!Pass) - return makeUnexpected(ErrorCode::InvalidArgument, - "machine pass manager cannot add a null pass"); - auto Validation = validateContract(Pass->contract()); - if (!Validation) - return std::unexpected(Validation.error()); - Passes.push_back(std::move(Pass)); - return {}; -} - -Expected MachinePassManager::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses, - MachinePassInstrumentation *Instrumentation) const { - Analyses.setInstrumentation(Instrumentation); - for (const std::unique_ptr &Pass : Passes) { - const MachinePassContract &Contract = Pass->contract(); - const std::string_view Name = Contract.Name; - - if (const MachineFunctionProperties Missing = - Contract.Requires.missingFrom(Function.properties()); - !Missing.empty()) { - Error Failure(ErrorCode::VerificationError, - "pass '" + std::string(Name) + "' requires properties: " + Missing.toString()); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - - if (Options.VerifyBefore) { - auto Verification = MachineVerifier::verify(Function, Target); - if (!Verification) { - Error Failure = prefixedError(Verification.error(), "pass '" + std::string(Name) + - "' received invalid MachineIR: "); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - - for (const AnalysisID Required : Contract.RequiredAnalyses) { - auto Result = Analyses.getResultImpl(Required, Function, Target); - if (!Result) { - Error Failure = - prefixedError(Result.error(), "pass '" + std::string(Name) + - "' could not materialize a required analysis: "); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - - const std::uint64_t FingerprintBefore = - Options.DetectUnreportedChanges ? machineFunctionFingerprint(Function) : 0; - if (Instrumentation != nullptr) - Instrumentation->before(Name, Function); - const Clock::time_point Start = Clock::now(); - Analyses.ActivePass = std::string(Name); - auto PassResult = Pass->run(Function, Target, Analyses); - Analyses.ActivePass.clear(); - const std::uint64_t Duration = elapsedNanoseconds(Start); - if (!PassResult) { - Error Failure = - prefixedError(PassResult.error(), "pass '" + std::string(Name) + "' failed: "); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - - if (PassResult->Changed != !PassResult->Changes.empty()) { - Error Failure(ErrorCode::VerificationError, - "pass '" + std::string(Name) + - "' reported inconsistent changed state and modification categories"); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - if (!Contract.DeclaredChanges.contains(PassResult->Changes)) { - Error Failure(ErrorCode::VerificationError, - "pass '" + std::string(Name) + - "' reported a modification category absent from its registration"); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - - bool FingerprintChanged = PassResult->Changed; - if (Options.DetectUnreportedChanges) { - FingerprintChanged = FingerprintBefore != machineFunctionFingerprint(Function); - if (FingerprintChanged && !PassResult->Changed) { - Error Failure(ErrorCode::VerificationError, - "pass '" + std::string(Name) + - "' changed MachineIR without reporting a modification"); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - if (Instrumentation != nullptr) - Instrumentation->change(Name, Function, PassResult->Changes, FingerprintChanged); - - auto Transition = - Function.properties().applyTransition(Contract.Requires, Contract.Sets, Contract.Clears); - if (!Transition) { - Error Failure = - prefixedError(Transition.error(), "pass '" + std::string(Name) + - "' produced an invalid property transition: "); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - Analyses.invalidate(Function, PassResult->Preserved, PassResult->Changes); - - if (Options.VerifyEach) { - MachineVerifierOptions VerifyOptions; - VerifyOptions.Stage = Contract.VerificationStage; - auto Verification = MachineVerifier::verify(Function, Target, VerifyOptions); - if (!Verification) { - Error Failure = prefixedError(Verification.error(), - "pass '" + std::string(Name) + "' left invalid MachineIR: "); - if (Instrumentation != nullptr) - Instrumentation->failure(Name, Function, Failure, Duration); - Analyses.clear(Function); - Analyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - if (Instrumentation != nullptr) - Instrumentation->after(Name, Function, *PassResult, Duration); - } - Analyses.setInstrumentation(nullptr); - return {}; -} - -Expected MachinePassManager::runFunctions(CompilationUnit &Unit, - const TargetDescription &Target, - const MachineAnalysisManager &AnalysisPrototype, - std::size_t ThreadCount, - MachinePassInstrumentation *Instrumentation) const { - if (ThreadCount == 0) - return makeUnexpected(ErrorCode::InvalidArgument, "machine pass thread count must be nonzero"); - auto UnitVerification = verifyCompilationUnit(Unit, Target); - if (!UnitVerification) - return std::unexpected(UnitVerification.error()); - const std::size_t FunctionCount = Unit.functionCount(); - if (FunctionCount == 0) - return {}; - const std::size_t WorkerCount = std::min(ThreadCount, FunctionCount); - if (WorkerCount > 1) { - for (const std::unique_ptr &Pass : Passes) { - if (Pass->contract().ThreadSafety != PassThreadSafety::ThreadSafe) { - return makeUnexpected(ErrorCode::InvalidArgument, - "parallel machine pipeline contains a pass not marked thread-safe"); - } - auto AnalysisSafety = - AnalysisPrototype.validateParallelRequirements(Pass->contract().RequiredAnalyses); - if (!AnalysisSafety) - return std::unexpected(AnalysisSafety.error()); - } - } - std::vector> Managers; - Managers.reserve(FunctionCount); - for (std::size_t Index = 0; Index < FunctionCount; ++Index) { - auto Manager = AnalysisPrototype.forkEmpty(); - if (!Manager) - return std::unexpected(Manager.error()); - Managers.push_back(std::move(*Manager)); - } - - std::vector> Failures(FunctionCount); - std::vector EventBuffers(FunctionCount); - std::atomic NextFunction = 0; - const auto Worker = [&]() { - while (true) { - const std::size_t Index = NextFunction.fetch_add(1, std::memory_order_relaxed); - if (Index >= FunctionCount) - return; - MachineFunction *Function = Unit.functionAt(Index); - if (Function == nullptr) { - Failures[Index] = Error(ErrorCode::InternalError, - "compilation unit contains a null function during parallel run"); - continue; - } - if (Instrumentation != nullptr) - Instrumentation->beginBuffering(EventBuffers[Index], Target); - auto Result = run(*Function, Target, *Managers[Index], Instrumentation); - if (Instrumentation != nullptr) - Instrumentation->endBuffering(); - if (!Result) - Failures[Index] = Result.error(); - } - }; - - std::vector Workers; - Workers.reserve(WorkerCount); - for (std::size_t Index = 0; Index < WorkerCount; ++Index) - Workers.emplace_back(Worker); - for (std::thread &WorkerThread : Workers) - WorkerThread.join(); - - for (std::size_t Index = 0; Index < FunctionCount; ++Index) { - if (Instrumentation != nullptr) - Instrumentation->replayBufferedEvents(EventBuffers[Index]); - const auto &Failure = Failures[Index]; - if (Failure) { - return std::unexpected(prefixedError( - *Failure, "function '" + std::string(Unit.functionAt(Index)->name()) + "' failed: ")); - } - } - return {}; -} - -Expected CompilationUnitPassManager::addPass(std::unique_ptr Pass) { - if (!Pass) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit pass manager cannot add a null pass"); - } - const CompilationUnitPassContract &Contract = Pass->contract(); - if (Contract.Name.empty()) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit pass must have a non-empty name"); - } - if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { - return makeUnexpected(ErrorCode::InvalidArgument, - "compilation-unit pass required analyses must be non-null and unique"); - } - auto Parameters = validateParameters(Contract.Parameters); - if (!Parameters) - return std::unexpected(Parameters.error()); - Passes.push_back(std::move(Pass)); - return {}; -} - -Expected CompilationUnitPassManager::run(CompilationUnit &Unit, - const TargetDescription &Target, - CompilationUnitAnalysisManager &UnitAnalyses, - MachineAnalysisManager &FunctionAnalyses, - MachinePassInstrumentation *Instrumentation) const { - UnitAnalyses.setInstrumentation(Instrumentation); - FunctionAnalyses.setInstrumentation(Instrumentation); - for (const std::unique_ptr &Pass : Passes) { - const CompilationUnitPassContract &Contract = Pass->contract(); - const std::string_view Name = Contract.Name; - - if (Options.VerifyBefore) { - auto Verification = verifyCompilationUnit(Unit, Target); - if (!Verification) { - Error Failure = - prefixedError(Verification.error(), "compilation-unit pass '" + std::string(Name) + - "' received invalid MachineIR: "); - if (Instrumentation != nullptr) - Instrumentation->unitFailure(Name, Unit, Failure); - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - - for (const AnalysisID Required : Contract.RequiredAnalyses) { - auto Result = UnitAnalyses.getResultImpl(Required, Unit, Target); - if (!Result) { - Error Failure = - prefixedError(Result.error(), "compilation-unit pass '" + std::string(Name) + - "' could not materialize a required analysis: "); - if (Instrumentation != nullptr) - Instrumentation->unitFailure(Name, Unit, Failure); - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - - if (Instrumentation != nullptr) - Instrumentation->beforeUnit(Name, Unit); - const Clock::time_point Start = Clock::now(); - auto Result = Pass->run(Unit, Target, UnitAnalyses, FunctionAnalyses); - const std::uint64_t Duration = elapsedNanoseconds(Start); - if (!Result) { - Error Failure = prefixedError(Result.error(), - "compilation-unit pass '" + std::string(Name) + "' failed: "); - if (Instrumentation != nullptr) - Instrumentation->unitFailure(Name, Unit, Failure, Duration); - UnitAnalyses.clearAll(); - FunctionAnalyses.clearAll(); - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - if (Result->Changed != !Result->Changes.empty() || - !Contract.DeclaredChanges.contains(Result->Changes)) { - Error Failure(ErrorCode::VerificationError, - "compilation-unit pass '" + std::string(Name) + - "' reported inconsistent or undeclared modifications"); - if (Instrumentation != nullptr) - Instrumentation->unitFailure(Name, Unit, Failure, Duration); - UnitAnalyses.clearAll(); - FunctionAnalyses.clearAll(); - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - - UnitAnalyses.invalidate(Unit, Result->PreservedUnitAnalyses, Result->Changes); - for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { - MachineFunction *Function = Unit.functionAt(Index); - if (Function != nullptr) { - FunctionAnalyses.invalidate(*Function, Result->PreservedFunctionAnalyses, Result->Changes); - } - } - if (Options.VerifyEach) { - auto Verification = verifyCompilationUnit(Unit, Target); - if (!Verification) { - Error Failure = - prefixedError(Verification.error(), "compilation-unit pass '" + std::string(Name) + - "' left invalid MachineIR: "); - if (Instrumentation != nullptr) - Instrumentation->unitFailure(Name, Unit, Failure, Duration); - UnitAnalyses.clearAll(); - FunctionAnalyses.clearAll(); - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return std::unexpected(std::move(Failure)); - } - } - if (Instrumentation != nullptr) - Instrumentation->afterUnit(Name, Unit, *Result, Duration); - } - UnitAnalyses.setInstrumentation(nullptr); - FunctionAnalyses.setInstrumentation(nullptr); - return {}; -} - -} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePassManager.cpp b/backend/lib/CodeGen/MachinePassManager.cpp new file mode 100644 index 0000000..0810965 --- /dev/null +++ b/backend/lib/CodeGen/MachinePassManager.cpp @@ -0,0 +1,1099 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePassManager.h" + +#include "rcc/CodeGen/CompilationUnit.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { + +thread_local MachinePassInstrumentation::BufferedEvents *MachinePassInstrumentation::ActiveBuffer = + nullptr; + +namespace { + +using Clock = std::chrono::steady_clock; + +[[nodiscard]] std::uint64_t elapsedNanoseconds(Clock::time_point Start) noexcept { + const auto Elapsed = std::chrono::duration_cast(Clock::now() - Start); + if (Elapsed.count() <= 0) + return 0; + return static_cast(Elapsed.count()); +} + +[[nodiscard]] Error prefixedError(const Error &Source, std::string Prefix) { + Prefix.append(Source.message()); + return Error(Source.code(), std::move(Prefix)); +} + +[[nodiscard]] bool hasDuplicateAnalysisIDs(std::span IDs) { + std::unordered_set Unique; + for (const AnalysisID ID : IDs) { + if (ID == nullptr || !Unique.insert(ID).second) + return true; + } + return false; +} + +[[nodiscard]] Expected validateParameters(std::span Parameters) { + std::unordered_set Names; + for (const PassParameter &Parameter : Parameters) { + if (Parameter.Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter name cannot be empty"); + if (!Names.insert(Parameter.Name).second) + return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter names must be unique"); + if (Parameter.Required && Parameter.DefaultValue) { + return makeUnexpected(ErrorCode::InvalidArgument, + "a required pass parameter cannot have a default value"); + } + } + return {}; +} + +void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { + State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); +} + +[[nodiscard]] std::uint64_t machineFunctionFingerprint(const MachineFunction &Function) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + combineFingerprint(Result, Function.stableID().value()); + combineFingerprint(Result, Function.properties().rawValue()); + combineFingerprint(Result, Function.blockCount()); + combineFingerprint(Result, Function.allInstructions().size()); + combineFingerprint(Result, Function.allMemOperands().size()); + + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + combineFingerprint(Result, Block->stableID().value()); + combineFingerprint(Result, Block->instructionCount()); + combineFingerprint(Result, Block->successors().size()); + for (const MachineSuccessor &Successor : Block->successors()) { + combineFingerprint(Result, + Successor.Block == nullptr ? 0 : Successor.Block->stableID().value()); + combineFingerprint(Result, Successor.Probability.rawValue()); + } + for (const MachineBasicBlockLiveIn &LiveIn : Block->liveIns()) { + combineFingerprint(Result, LiveIn.PhysReg.number()); + combineFingerprint(Result, LiveIn.Lanes.rawValue()); + } + + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + combineFingerprint(Result, Instruction->stableID().value()); + combineFingerprint(Result, Instruction->opcode().value()); + combineFingerprint(Result, Instruction->flags().rawValue()); + combineFingerprint(Result, Instruction->operandCount()); + combineFingerprint(Result, Instruction->memOperands().size()); + for (const MachineOperand &Operand : Instruction->operands()) { + combineFingerprint(Result, static_cast(Operand.kind())); + combineFingerprint(Result, Operand.targetFlags()); + combineFingerprint(Result, Operand.registerFlags().rawValue()); + combineFingerprint(Result, Operand.subReg().value()); + if (const Register *Reg = Operand.getIf()) + combineFingerprint(Result, Reg->opaqueValue()); + else if (const std::int64_t *Immediate = Operand.getIf()) + combineFingerprint(Result, static_cast(*Immediate)); + } + } + } + return Result; +} + +[[nodiscard]] Expected verifyCompilationUnit(const CompilationUnit &Unit, + const TargetDescription &Target) { + auto UnitVerification = Unit.verify(); + if (!UnitVerification) + return std::unexpected(UnitVerification.error()); + for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { + const MachineFunction *Function = Unit.functionAt(Index); + if (Function == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "compilation unit function order contains a null entry"); + auto FunctionVerification = MachineVerifier::verify(*Function, Target); + if (!FunctionVerification) + return std::unexpected(FunctionVerification.error()); + } + return {}; +} + +} // namespace + +void AnalysisManagerInstrumentation::beforeAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function) { + Instrumentation->beforeAnalysis(Name, Function); +} + +void AnalysisManagerInstrumentation::afterAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function, bool CacheHit, std::uint64_t Nanoseconds) { + Instrumentation->afterAnalysis(Name, Function, CacheHit, Nanoseconds); +} + +void AnalysisManagerInstrumentation::analysisFailure( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function, const Error &Failure, std::uint64_t Nanoseconds) { + Instrumentation->analysisFailure(Name, Function, Failure, Nanoseconds); +} + +Expected +AnalysisManagerInstrumentation::requestAction( + MachinePassInstrumentation *Instrumentation, std::string_view Pass, std::string_view Rule, + const MachineFunction &Function, MachineInstrStableID Instruction, std::uint64_t Cost) { + return Instrumentation->requestAction(Pass, Rule, Function, Instruction, Cost); +} + +Expected +AnalysisManagerInstrumentation::requestAction( + MachinePassInstrumentation *Instrumentation, std::string_view Pass, std::string_view Rule, + const MachineFunction &Function, std::nullptr_t, std::uint64_t Cost) { + return Instrumentation->requestAction(Pass, Rule, Function, nullptr, Cost); +} + +void AnalysisManagerInstrumentation::beforeAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const CompilationUnit &Unit) { + Instrumentation->beforeAnalysis(Name, Unit); +} + +void AnalysisManagerInstrumentation::afterAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, const CompilationUnit &Unit, + bool CacheHit, std::uint64_t Nanoseconds) { + Instrumentation->afterAnalysis(Name, Unit, CacheHit, Nanoseconds); +} + +void AnalysisManagerInstrumentation::analysisFailure( + MachinePassInstrumentation *Instrumentation, std::string_view Name, const CompilationUnit &Unit, + const Error &Failure, std::uint64_t Nanoseconds) { + Instrumentation->analysisFailure(Name, Unit, Failure, Nanoseconds); +} + +std::string MachineActionID::toString() const { + std::string Result; + Result.reserve(Pass.size() + Rule.size() + 48U); + Result.append(Pass); + Result.push_back(':'); + Result.append(Rule); + Result.append(":f"); + Result.append(std::to_string(Function.value())); + Result.append(":i"); + Result.append(std::to_string(Instruction.value())); + return Result; +} + +Expected MachineActionReplay::request(std::string_view Rule, const MachineFunction &Function, + MachineInstrStableID Instruction, std::uint64_t Cost) { + if (!IsReplaying) { + const auto Position = RecordingInstructionIDs.find(Instruction); + const MachineInstrStableID CanonicalInstruction = + Position == RecordingInstructionIDs.end() ? Instruction : Position->second; + auto Accepted = Request(Rule, Function, CanonicalInstruction, Cost); + if (!Accepted) + return std::unexpected(Accepted.error()); + Decisions.push_back({ + .Rule = std::string(Rule), + .Function = Function.stableID(), + .Instruction = CanonicalInstruction, + .Cost = Cost, + .Accepted = *Accepted, + }); + return *Accepted; + } + + if (NextDecision >= Decisions.size()) { + return makeUnexpected(ErrorCode::InternalError, + "machine action replay requested an unrecorded decision"); + } + const Decision &Recorded = Decisions[NextDecision++]; + if (Recorded.Rule != Rule || Recorded.Function != Function.stableID() || + Recorded.Instruction != Instruction || Recorded.Cost != Cost) { + std::string Message = "machine action replay diverged from transaction preflight: recorded "; + Message.append(Recorded.Rule); + Message.append(":f"); + Message.append(std::to_string(Recorded.Function.value())); + Message.append(":i"); + Message.append(std::to_string(Recorded.Instruction.value())); + Message.append(" cost="); + Message.append(std::to_string(Recorded.Cost)); + Message.append(", requested "); + Message.append(Rule); + Message.append(":f"); + Message.append(std::to_string(Function.stableID().value())); + Message.append(":i"); + Message.append(std::to_string(Instruction.value())); + Message.append(" cost="); + Message.append(std::to_string(Cost)); + return makeUnexpected(ErrorCode::InternalError, std::move(Message)); + } + return Recorded.Accepted; +} + +void MachineActionReplay::beginReplay() noexcept { + IsReplaying = true; + NextDecision = 0; +} + +Expected MachineActionReplay::finishReplay() const { + if (!IsReplaying || NextDecision != Decisions.size()) { + return makeUnexpected(ErrorCode::InternalError, + "machine action replay did not consume every recorded decision"); + } + return {}; +} + +void MachinePassInstrumentation::addBeforeCallback(BeforeCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + Before.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterCallback(AfterCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + After.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addFailureCallback(FailureCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + Failure.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addBeforeAnalysisCallback(BeforeAnalysisCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + BeforeAnalysis.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterAnalysisCallback(AfterAnalysisCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + AfterAnalysis.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addAnalysisFailureCallback(AnalysisFailureCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + AnalysisFailure.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addIRPrintCallback(IRPrintCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + IRPrint.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addChangeCallback(ChangeCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + Change.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::setActionFilter(ActionFilter NewFilter) { + std::scoped_lock Lock(Mutex); + Filter = std::move(NewFilter); +} + +void MachinePassInstrumentation::setPerFunctionFuel(std::optional Fuel) { + std::scoped_lock Lock(Mutex); + PerFunctionFuel = Fuel; + ConsumedFuel.clear(); + ActionEvents.clear(); +} + +void MachinePassInstrumentation::addBeforeUnitCallback(BeforeUnitCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + BeforeUnit.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterUnitCallback(AfterUnitCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + AfterUnit.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addUnitFailureCallback(UnitFailureCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + UnitFailure.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addBeforeUnitAnalysisCallback( + BeforeUnitAnalysisCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + BeforeUnitAnalysis.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterUnitAnalysisCallback(AfterUnitAnalysisCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + AfterUnitAnalysis.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addUnitAnalysisFailureCallback( + UnitAnalysisFailureCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + UnitAnalysisFailure.push_back(std::move(Callback)); +} + +InstrumentationStatistic &MachinePassInstrumentation::statistic(std::string_view Name, + bool IsAnalysis) const { + const std::string Key = std::string(IsAnalysis ? "analysis:" : "pass:") + std::string(Name); + auto [Position, Inserted] = Statistics.try_emplace(Key); + if (Inserted) { + Position->second.Name = std::string(Name); + Position->second.IsAnalysis = IsAnalysis; + } + return Position->second; +} + +bool MachinePassInstrumentation::bufferFunctionEvent( + const MachineFunction &Function, std::function Replay) const { + if (ActiveBuffer == nullptr) + return false; + + std::shared_ptr Snapshot; + auto Cloned = cloneMachineFunction( + Function, *ActiveBuffer->Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (Cloned) { + Snapshot = std::shared_ptr(std::move(Cloned->ClonedFunction)); + } else { + Snapshot = + std::shared_ptr(&Function, [](const MachineFunction *) noexcept {}); + } + ActiveBuffer->Events.emplace_back( + [Snapshot = std::move(Snapshot), Replay = std::move(Replay)]() { Replay(*Snapshot); }); + return true; +} + +void MachinePassInstrumentation::beginBuffering(BufferedEvents &Buffer, + const TargetDescription &Target) const noexcept { + Buffer.Target = &Target; + Buffer.Events.clear(); + ActiveBuffer = &Buffer; +} + +void MachinePassInstrumentation::endBuffering() const noexcept { + ActiveBuffer = nullptr; +} + +void MachinePassInstrumentation::replayBufferedEvents(BufferedEvents &Buffer) const { + for (const std::function &Event : Buffer.Events) + Event(); + Buffer.Events.clear(); + Buffer.Target = nullptr; +} + +void MachinePassInstrumentation::before(std::string_view Name, + const MachineFunction &Function) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name)](const MachineFunction &Input) { + before(Name, Input); + })) { + return; + } + std::vector Callbacks; + std::vector Printers; + { + std::scoped_lock Lock(Mutex); + Callbacks = Before; + Printers = IRPrint; + } + for (const BeforeCallback &Callback : Callbacks) + Callback(Name, Function); + for (const IRPrintCallback &Printer : Printers) + Printer(InstrumentationPhase::Before, Name, Function); +} + +void MachinePassInstrumentation::after(std::string_view Name, const MachineFunction &Function, + const MachinePassResult &Result, + std::uint64_t Nanoseconds) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name), Result, + Nanoseconds](const MachineFunction &Input) { + after(Name, Input, Result, Nanoseconds); + })) { + return; + } + std::vector Callbacks; + std::vector Printers; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, false); + ++Record.Invocations; + Record.Changes += Result.Changed ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = After; + Printers = IRPrint; + } + for (const AfterCallback &Callback : Callbacks) + Callback(Name, Function, Result); + for (const IRPrintCallback &Printer : Printers) + Printer(InstrumentationPhase::After, Name, Function); +} + +void MachinePassInstrumentation::failure(std::string_view Name, const MachineFunction &Function, + const Error &ErrorValue, std::uint64_t Nanoseconds) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name), ErrorValue, + Nanoseconds](const MachineFunction &Input) { + failure(Name, Input, ErrorValue, Nanoseconds); + })) { + return; + } + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, false); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = Failure; + } + for (const FailureCallback &Callback : Callbacks) + Callback(Name, Function, ErrorValue); +} + +void MachinePassInstrumentation::beforeAnalysis(std::string_view Name, + const MachineFunction &Function) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name)](const MachineFunction &Input) { + beforeAnalysis(Name, Input); + })) { + return; + } + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + Callbacks = BeforeAnalysis; + } + for (const BeforeAnalysisCallback &Callback : Callbacks) + Callback(Name, Function); +} + +void MachinePassInstrumentation::afterAnalysis(std::string_view Name, + const MachineFunction &Function, bool CacheHit, + std::uint64_t Nanoseconds) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name), CacheHit, + Nanoseconds](const MachineFunction &Input) { + afterAnalysis(Name, Input, CacheHit, Nanoseconds); + })) { + return; + } + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, true); + ++Record.Invocations; + Record.CacheHits += CacheHit ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = AfterAnalysis; + } + for (const AfterAnalysisCallback &Callback : Callbacks) + Callback(Name, Function, CacheHit, Nanoseconds); +} + +void MachinePassInstrumentation::analysisFailure(std::string_view Name, + const MachineFunction &Function, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name), ErrorValue, + Nanoseconds](const MachineFunction &Input) { + analysisFailure(Name, Input, ErrorValue, Nanoseconds); + })) { + return; + } + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, true); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = AnalysisFailure; + } + for (const AnalysisFailureCallback &Callback : Callbacks) + Callback(Name, Function, ErrorValue); +} + +void MachinePassInstrumentation::change(std::string_view Name, const MachineFunction &Function, + MachinePassChanges Changes, bool FingerprintChanged) const { + if (bufferFunctionEvent(Function, [this, Name = std::string(Name), Changes, + FingerprintChanged](const MachineFunction &Input) { + change(Name, Input, Changes, FingerprintChanged); + })) { + return; + } + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + Callbacks = Change; + } + for (const ChangeCallback &Callback : Callbacks) + Callback(Name, Function, Changes, FingerprintChanged); +} + +Expected MachinePassInstrumentation::requestAction(std::string_view Pass, + std::string_view Rule, + const MachineFunction &Function, + MachineInstrStableID Instruction, + std::uint64_t Cost) { + MachineActionID Action{ + .Pass = std::string(Pass), + .Rule = std::string(Rule), + .Function = Function.stableID(), + .Instruction = Instruction, + }; + if (Action.Pass.empty() || Action.Rule.empty() || Action.Function != Function.stableID() || + Cost == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine action identity is invalid"); + } + std::scoped_lock ActionLock(ActionMutex); + ActionFilter CurrentFilter; + { + std::scoped_lock Lock(Mutex); + CurrentFilter = Filter; + } + if (CurrentFilter && !CurrentFilter(Action, Cost)) { + std::scoped_lock Lock(Mutex); + ActionEvents.push_back({Action, Cost, false}); + return false; + } + + std::scoped_lock Lock(Mutex); + if (!PerFunctionFuel) { + ActionEvents.push_back({Action, Cost, true}); + return true; + } + std::uint64_t &Consumed = ConsumedFuel[Function.stableID().value()]; + if (Consumed > *PerFunctionFuel || Cost > *PerFunctionFuel - Consumed) { + ActionEvents.push_back({Action, Cost, false}); + return false; + } + Consumed += Cost; + ActionEvents.push_back({Action, Cost, true}); + return true; +} + +Expected MachinePassInstrumentation::requestAction(std::string_view Pass, + std::string_view Rule, + const MachineFunction &Function, + std::nullptr_t, std::uint64_t Cost) { + return requestAction(Pass, Rule, Function, MachineInstrStableID{}, Cost); +} + +void MachinePassInstrumentation::beforeUnit(std::string_view Name, + const CompilationUnit &Unit) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + Callbacks = BeforeUnit; + } + for (const BeforeUnitCallback &Callback : Callbacks) + Callback(Name, Unit); +} + +void MachinePassInstrumentation::afterUnit(std::string_view Name, const CompilationUnit &Unit, + const CompilationUnitPassResult &Result, + std::uint64_t Nanoseconds) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); + ++Record.Invocations; + Record.Changes += Result.Changed ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = AfterUnit; + } + for (const AfterUnitCallback &Callback : Callbacks) + Callback(Name, Unit, Result); +} + +void MachinePassInstrumentation::unitFailure(std::string_view Name, const CompilationUnit &Unit, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = UnitFailure; + } + for (const UnitFailureCallback &Callback : Callbacks) + Callback(Name, Unit, ErrorValue); +} + +void MachinePassInstrumentation::beforeAnalysis(std::string_view Name, + const CompilationUnit &Unit) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + Callbacks = BeforeUnitAnalysis; + } + for (const BeforeUnitAnalysisCallback &Callback : Callbacks) + Callback(Name, Unit); +} + +void MachinePassInstrumentation::afterAnalysis(std::string_view Name, const CompilationUnit &Unit, + bool CacheHit, std::uint64_t Nanoseconds) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); + ++Record.Invocations; + Record.CacheHits += CacheHit ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = AfterUnitAnalysis; + } + for (const AfterUnitAnalysisCallback &Callback : Callbacks) + Callback(Name, Unit, CacheHit, Nanoseconds); +} + +void MachinePassInstrumentation::analysisFailure(std::string_view Name, const CompilationUnit &Unit, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + std::vector Callbacks; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + Callbacks = UnitAnalysisFailure; + } + for (const UnitAnalysisFailureCallback &Callback : Callbacks) + Callback(Name, Unit, ErrorValue); +} + +std::vector MachinePassInstrumentation::statistics() const { + std::vector Result; + { + std::scoped_lock Lock(Mutex); + Result.reserve(Statistics.size()); + for (const auto &[Key, Record] : Statistics) { + (void)Key; + Result.push_back(Record); + } + } + std::ranges::sort( + Result, [](const InstrumentationStatistic &Left, const InstrumentationStatistic &Right) { + if (Left.IsAnalysis != Right.IsAnalysis) + return Left.IsAnalysis < Right.IsAnalysis; + return Left.Name < Right.Name; + }); + return Result; +} + +std::vector MachinePassInstrumentation::actionEvents() const { + std::vector Result; + { + std::scoped_lock Lock(Mutex); + Result = ActionEvents; + } + std::ranges::sort(Result, [](const MachineActionEvent &Left, const MachineActionEvent &Right) { + if (Left.ID.Function.value() != Right.ID.Function.value()) + return Left.ID.Function.value() < Right.ID.Function.value(); + if (Left.ID.Instruction.value() != Right.ID.Instruction.value()) + return Left.ID.Instruction.value() < Right.ID.Instruction.value(); + if (Left.ID.Pass != Right.ID.Pass) + return Left.ID.Pass < Right.ID.Pass; + if (Left.ID.Rule != Right.ID.Rule) + return Left.ID.Rule < Right.ID.Rule; + if (Left.Cost != Right.Cost) + return Left.Cost < Right.Cost; + return Left.Accepted < Right.Accepted; + }); + return Result; +} + +Expected +PassManagerCallbacks::validate( + const MachineFunctionPass &Pass) { + const MachinePassContract &Contract = Pass.contract(); + if (Contract.Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "machine pass must have a non-empty name"); + if (Contract.Sets.intersects(Contract.Clears)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine pass cannot set and clear the same property"); + } + if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine pass required analyses must be non-null and unique"); + } + return validateParameters(Contract.Parameters); +} + +Expected +PassManagerCallbacks::run( + const PassList &Passes, const MachinePassManagerOptions &Options, MachineFunction &Function, + const TargetDescription &Target, MachineFunctionAnalysisManager &Analyses, + MachinePassInstrumentation *Instrumentation) { + Analyses.setInstrumentation(Instrumentation); + for (const std::unique_ptr &Pass : Passes) { + const MachinePassContract &Contract = Pass->contract(); + const std::string_view Name = Contract.Name; + + if (const MachineFunctionProperties Missing = + Contract.Requires.missingFrom(Function.properties()); + !Missing.empty()) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + "' requires properties: " + Missing.toString()); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + if (Options.VerifyBefore) { + auto Verification = MachineVerifier::verify(Function, Target); + if (!Verification) { + Error Failure = prefixedError(Verification.error(), "pass '" + std::string(Name) + + "' received invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + for (const AnalysisID Required : Contract.RequiredAnalyses) { + auto Result = Analyses.getResultByID(Required, Function, Target); + if (!Result) { + Error Failure = + prefixedError(Result.error(), "pass '" + std::string(Name) + + "' could not materialize a required analysis: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + const std::uint64_t FingerprintBefore = + Options.DetectUnreportedChanges ? machineFunctionFingerprint(Function) : 0; + if (Instrumentation != nullptr) + Instrumentation->before(Name, Function); + const Clock::time_point Start = Clock::now(); + Analyses.setActivePass(std::string(Name)); + auto PassResult = Pass->run(Function, Target, Analyses); + Analyses.setActivePass({}); + const std::uint64_t Duration = elapsedNanoseconds(Start); + if (!PassResult) { + Error Failure = + prefixedError(PassResult.error(), "pass '" + std::string(Name) + "' failed: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + if (PassResult->Changed != !PassResult->Changes.empty()) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' reported inconsistent changed state and modification categories"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + if (!Contract.DeclaredChanges.contains(PassResult->Changes)) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' reported a modification category absent from its registration"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + bool FingerprintChanged = PassResult->Changed; + if (Options.DetectUnreportedChanges) { + FingerprintChanged = FingerprintBefore != machineFunctionFingerprint(Function); + if (FingerprintChanged && !PassResult->Changed) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' changed MachineIR without reporting a modification"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->change(Name, Function, PassResult->Changes, FingerprintChanged); + + auto Transition = + Function.properties().applyTransition(Contract.Requires, Contract.Sets, Contract.Clears); + if (!Transition) { + Error Failure = + prefixedError(Transition.error(), "pass '" + std::string(Name) + + "' produced an invalid property transition: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + Analyses.invalidate(Function, PassResult->Preserved, PassResult->Changes); + + if (Options.VerifyEach) { + MachineVerifierOptions VerifyOptions; + VerifyOptions.Stage = Contract.VerificationStage; + auto Verification = MachineVerifier::verify(Function, Target, VerifyOptions); + if (!Verification) { + Error Failure = prefixedError(Verification.error(), + "pass '" + std::string(Name) + "' left invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->after(Name, Function, *PassResult, Duration); + } + Analyses.setInstrumentation(nullptr); + return {}; +} + +Expected +PassManagerCallbacks::runFunctions( + const PassList &Passes, const MachinePassManagerOptions &Options, CompilationUnit &Unit, + const TargetDescription &Target, const MachineFunctionAnalysisManager &AnalysisPrototype, + std::size_t ThreadCount, MachinePassInstrumentation *Instrumentation) { + if (ThreadCount == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "machine pass thread count must be nonzero"); + auto UnitVerification = verifyCompilationUnit(Unit, Target); + if (!UnitVerification) + return std::unexpected(UnitVerification.error()); + const std::size_t FunctionCount = Unit.functionCount(); + if (FunctionCount == 0) + return {}; + const std::size_t WorkerCount = std::min(ThreadCount, FunctionCount); + if (WorkerCount > 1) { + for (const std::unique_ptr &Pass : Passes) { + if (Pass->contract().ThreadSafety != PassThreadSafety::ThreadSafe) { + return makeUnexpected(ErrorCode::InvalidArgument, + "parallel machine pipeline contains a pass not marked thread-safe"); + } + auto AnalysisSafety = + AnalysisPrototype.validateParallelRequirements(Pass->contract().RequiredAnalyses); + if (!AnalysisSafety) + return std::unexpected(AnalysisSafety.error()); + } + } + std::vector> Managers; + Managers.reserve(FunctionCount); + for (std::size_t Index = 0; Index < FunctionCount; ++Index) { + auto Manager = AnalysisPrototype.forkEmpty(); + if (!Manager) + return std::unexpected(Manager.error()); + Managers.push_back(std::move(*Manager)); + } + + std::vector> Failures(FunctionCount); + std::vector EventBuffers(FunctionCount); + std::atomic NextFunction = 0; + const auto Worker = [&]() { + while (true) { + const std::size_t Index = NextFunction.fetch_add(1, std::memory_order_relaxed); + if (Index >= FunctionCount) + return; + MachineFunction *Function = Unit.functionAt(Index); + if (Function == nullptr) { + Failures[Index] = Error(ErrorCode::InternalError, + "compilation unit contains a null function during parallel run"); + continue; + } + if (Instrumentation != nullptr) + Instrumentation->beginBuffering(EventBuffers[Index], Target); + auto Result = PassManagerCallbacks::run(Passes, Options, *Function, Target, *Managers[Index], + Instrumentation); + if (Instrumentation != nullptr) + Instrumentation->endBuffering(); + if (!Result) + Failures[Index] = Result.error(); + } + }; + + std::vector Workers; + Workers.reserve(WorkerCount); + for (std::size_t Index = 0; Index < WorkerCount; ++Index) + Workers.emplace_back(Worker); + for (std::thread &WorkerThread : Workers) + WorkerThread.join(); + + for (std::size_t Index = 0; Index < FunctionCount; ++Index) { + if (Instrumentation != nullptr) + Instrumentation->replayBufferedEvents(EventBuffers[Index]); + const auto &Failure = Failures[Index]; + if (Failure) { + return std::unexpected(prefixedError( + *Failure, "function '" + std::string(Unit.functionAt(Index)->name()) + "' failed: ")); + } + } + return {}; +} + +Expected +PassManagerCallbacks::validate(const CompilationUnitPass &Pass) { + const CompilationUnitPassContract &Contract = Pass.contract(); + if (Contract.Name.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compilation-unit pass must have a non-empty name"); + } + if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compilation-unit pass required analyses must be non-null and unique"); + } + auto Parameters = validateParameters(Contract.Parameters); + if (!Parameters) + return std::unexpected(Parameters.error()); + return {}; +} + +Expected +PassManagerCallbacks::run( + const PassList &Passes, const CompilationUnitPassManagerOptions &Options, CompilationUnit &Unit, + const TargetDescription &Target, CompilationUnitAnalysisManager &UnitAnalyses, + MachineFunctionAnalysisManager &FunctionAnalyses, MachinePassInstrumentation *Instrumentation) { + UnitAnalyses.setInstrumentation(Instrumentation); + FunctionAnalyses.setInstrumentation(Instrumentation); + for (const std::unique_ptr &Pass : Passes) { + const CompilationUnitPassContract &Contract = Pass->contract(); + const std::string_view Name = Contract.Name; + + if (Options.VerifyBefore) { + auto Verification = verifyCompilationUnit(Unit, Target); + if (!Verification) { + Error Failure = + prefixedError(Verification.error(), "compilation-unit pass '" + std::string(Name) + + "' received invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + for (const AnalysisID Required : Contract.RequiredAnalyses) { + auto Result = UnitAnalyses.getResultByID(Required, Unit, Target); + if (!Result) { + Error Failure = + prefixedError(Result.error(), "compilation-unit pass '" + std::string(Name) + + "' could not materialize a required analysis: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + if (Instrumentation != nullptr) + Instrumentation->beforeUnit(Name, Unit); + const Clock::time_point Start = Clock::now(); + auto Result = Pass->run(Unit, Target, UnitAnalyses, FunctionAnalyses); + const std::uint64_t Duration = elapsedNanoseconds(Start); + if (!Result) { + Error Failure = prefixedError(Result.error(), + "compilation-unit pass '" + std::string(Name) + "' failed: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + if (Result->Changed != !Result->Changes.empty() || + !Contract.DeclaredChanges.contains(Result->Changes)) { + Error Failure(ErrorCode::VerificationError, + "compilation-unit pass '" + std::string(Name) + + "' reported inconsistent or undeclared modifications"); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + UnitAnalyses.invalidate(Unit, Result->PreservedUnitAnalyses, Result->Changes); + for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { + MachineFunction *Function = Unit.functionAt(Index); + if (Function != nullptr) { + FunctionAnalyses.invalidate(*Function, Result->PreservedFunctionAnalyses, Result->Changes); + } + } + if (Options.VerifyEach) { + auto Verification = verifyCompilationUnit(Unit, Target); + if (!Verification) { + Error Failure = + prefixedError(Verification.error(), "compilation-unit pass '" + std::string(Name) + + "' left invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->afterUnit(Name, Unit, *Result, Duration); + } + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePseudoExpansion.cpp b/backend/lib/CodeGen/MachinePseudoExpansion.cpp index b5afe8e..663b148 100644 --- a/backend/lib/CodeGen/MachinePseudoExpansion.cpp +++ b/backend/lib/CodeGen/MachinePseudoExpansion.cpp @@ -355,7 +355,8 @@ expandImplementation(MachineFunction &Function, const TargetDescription &Target, [[nodiscard]] Expected runPreparationTransaction(MachineFunction &Function, const TargetDescription &Target, - const TargetPseudoExpansionInfo &Info, MachineAnalysisManager &Analyses) { + const TargetPseudoExpansionInfo &Info, + MachineFunctionAnalysisManager &Analyses) { auto Trial = cloneMachineFunction( Function, Target, MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); @@ -380,7 +381,8 @@ runPreparationTransaction(MachineFunction &Function, const TargetDescription &Ta [[nodiscard]] Expected runExpansionTransaction(MachineFunction &Function, const TargetDescription &Target, const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, - const TargetInstrInfo *BranchInfo, MachineAnalysisManager &Analyses) { + const TargetInstrInfo *BranchInfo, + MachineFunctionAnalysisManager &Analyses) { auto Trial = cloneMachineFunction( Function, Target, MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); @@ -409,9 +411,9 @@ const MachinePassContract &PreparePostRAPseudosPass::contract() const noexcept { return preparationContract(); } -Expected PreparePostRAPseudosPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +PreparePostRAPseudosPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::VerificationError, "post-RA pseudo preparation requires selected MachineIR"); @@ -422,9 +424,9 @@ const MachinePassContract &PrePEITargetPseudoExpansionPass::contract() const noe return phaseContract(MachinePseudoExpansionPhase::PrePEI); } -Expected PrePEITargetPseudoExpansionPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +PrePEITargetPseudoExpansionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::VerificationError, "pre-PEI pseudo expansion requires allocated tracked-liveness MIR"); @@ -438,7 +440,7 @@ const MachinePassContract &ExpandPostRAPseudosPass::contract() const noexcept { Expected ExpandPostRAPseudosPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::VerificationError, "post-RA pseudo expansion requires finalized frame and liveness"); @@ -450,9 +452,9 @@ const MachinePassContract &TargetLatePseudoExpansionPass::contract() const noexc return phaseContract(MachinePseudoExpansionPhase::TargetLate); } -Expected TargetLatePseudoExpansionPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +TargetLatePseudoExpansionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) return makeUnexpected(ErrorCode::VerificationError, "target-late pseudo expansion requires finalized allocated MIR"); diff --git a/backend/lib/CodeGen/MachineScheduler.cpp b/backend/lib/CodeGen/MachineScheduler.cpp index e589675..82f0c91 100644 --- a/backend/lib/CodeGen/MachineScheduler.cpp +++ b/backend/lib/CodeGen/MachineScheduler.cpp @@ -37,10 +37,11 @@ namespace { MachineFunctionProperty::TiedOpsRewritten}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::TiedOpsRewritten, }; - Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.RequiredAnalyses.push_back( + MachineFunctionAnalysisManager::id()); Value.ThreadSafety = PassThreadSafety::ThreadSafe; Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; return Value; @@ -395,7 +396,7 @@ const MachinePassContract &MachineSchedulerPass::contract() const noexcept { Expected MachineSchedulerPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "pre-RA machine scheduler requires selected, SSA-exited MachineIR"); @@ -407,7 +408,7 @@ Expected MachineSchedulerPass::run(MachineFunction &Function, return std::unexpected( prefixedError(Trial.error(), "machine scheduler transaction setup failed: ")); } - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialLiveness = TrialAnalyses.getResult(*Trial->ClonedFunction, Target); if (!TrialLiveness) diff --git a/backend/lib/CodeGen/MachineShrinkWrapping.cpp b/backend/lib/CodeGen/MachineShrinkWrapping.cpp index aee3323..7c7bc3a 100644 --- a/backend/lib/CodeGen/MachineShrinkWrapping.cpp +++ b/backend/lib/CodeGen/MachineShrinkWrapping.cpp @@ -32,8 +32,8 @@ namespace { MachineFunctionProperty::TracksLiveness}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -66,9 +66,9 @@ const MachinePassContract &MachineShrinkWrappingPass::contract() const noexcept return shrinkWrappingContract(); } -Expected MachineShrinkWrappingPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineShrinkWrappingPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "shrink wrapping requires allocated tracked-liveness MachineIR"); diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp index 21b17f8..176b75a 100644 --- a/backend/lib/CodeGen/MachineSink.cpp +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -34,8 +34,8 @@ namespace { .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::Selected, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -50,8 +50,8 @@ namespace { MachineFunctionProperty::TracksLiveness}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id(), - MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id(), + MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -354,7 +354,7 @@ const MachinePassContract &MachineSinkPass::contract() const noexcept { Expected MachineSinkPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected( ErrorCode::VerificationError, @@ -368,7 +368,7 @@ Expected MachineSinkPass::run(MachineFunction &Function, if (!Trial) { return std::unexpected(prefixedError(Trial.error(), "machine sink transaction setup failed: ")); } - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); if (!TrialResult) return std::unexpected(TrialResult.error()); @@ -381,9 +381,9 @@ Expected MachineSinkPass::run(MachineFunction &Function, return Result; } -Expected MachineSinkPass::runImplementation(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +MachineSinkPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { auto Dominators = Analyses.getResult(Function, Target); if (!Dominators) return std::unexpected(Dominators.error()); diff --git a/backend/lib/CodeGen/MachineStackSlotColoring.cpp b/backend/lib/CodeGen/MachineStackSlotColoring.cpp index e60a7a2..005872b 100644 --- a/backend/lib/CodeGen/MachineStackSlotColoring.cpp +++ b/backend/lib/CodeGen/MachineStackSlotColoring.cpp @@ -228,7 +228,7 @@ const MachinePassContract &MachineStackSlotColoringPass::contract() const noexce Expected MachineStackSlotColoringPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "stack-slot coloring requires allocated PHI-free MachineIR"); diff --git a/backend/lib/CodeGen/MachineTailDuplication.cpp b/backend/lib/CodeGen/MachineTailDuplication.cpp index 5abd473..8be1eb5 100644 --- a/backend/lib/CodeGen/MachineTailDuplication.cpp +++ b/backend/lib/CodeGen/MachineTailDuplication.cpp @@ -248,7 +248,7 @@ const MachinePassContract &MachineTailDuplicationPass::contract() const noexcept Expected MachineTailDuplicationPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "tail duplication requires finalized allocated tracked-liveness MIR"); diff --git a/backend/lib/CodeGen/MachineTailMerge.cpp b/backend/lib/CodeGen/MachineTailMerge.cpp index c4d6de9..dc07950 100644 --- a/backend/lib/CodeGen/MachineTailMerge.cpp +++ b/backend/lib/CodeGen/MachineTailMerge.cpp @@ -298,7 +298,7 @@ const MachinePassContract &MachineTailMergePass::contract() const noexcept { Expected MachineTailMergePass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "tail merge requires finalized allocated tracked-liveness MIR"); diff --git a/backend/lib/CodeGen/PHIElimination.cpp b/backend/lib/CodeGen/PHIElimination.cpp index 75472f2..6de449f 100644 --- a/backend/lib/CodeGen/PHIElimination.cpp +++ b/backend/lib/CodeGen/PHIElimination.cpp @@ -490,7 +490,7 @@ const MachinePassContract &PHIEliminationPass::contract() const noexcept { Expected PHIEliminationPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { auto Required = requireProperties(Function, contract()); if (!Required) return std::unexpected(Required.error()); diff --git a/backend/lib/CodeGen/PostRAScheduler.cpp b/backend/lib/CodeGen/PostRAScheduler.cpp index f7d6181..b5f7ae9 100644 --- a/backend/lib/CodeGen/PostRAScheduler.cpp +++ b/backend/lib/CodeGen/PostRAScheduler.cpp @@ -37,10 +37,11 @@ namespace { MachineFunctionProperty::TracksLiveness}, .Sets = {}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; - Value.RequiredAnalyses.push_back(MachineAnalysisManager::id()); + Value.RequiredAnalyses.push_back( + MachineFunctionAnalysisManager::id()); Value.ThreadSafety = PassThreadSafety::ThreadSafe; Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; return Value; @@ -61,7 +62,7 @@ const MachinePassContract &PostRASchedulerPass::contract() const noexcept { Expected PostRASchedulerPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (!Function.properties().containsAll(contract().Requires)) { return makeUnexpected(ErrorCode::VerificationError, "post-RA scheduler requires allocated tracked-liveness MachineIR"); @@ -73,7 +74,7 @@ Expected PostRASchedulerPass::run(MachineFunction &Function, return std::unexpected( prefixedError(Trial.error(), "post-RA scheduler transaction setup failed: ")); } - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto TrialLiveness = TrialAnalyses.getResult(*Trial->ClonedFunction, Target); if (!TrialLiveness) diff --git a/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp b/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp index 47b74f6..c935d6a 100644 --- a/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp +++ b/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp @@ -49,7 +49,7 @@ const MachinePassContract &PrologueEpilogueInsertionPass::contract() const noexc Expected PrologueEpilogueInsertionPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (Info == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "PrologueEpilogueInsertion requires target frame lowering"); diff --git a/backend/lib/CodeGen/RegAllocBase.h b/backend/lib/CodeGen/RegAllocBase.h index 318a82d..4ef0bb7 100644 --- a/backend/lib/CodeGen/RegAllocBase.h +++ b/backend/lib/CodeGen/RegAllocBase.h @@ -27,8 +27,9 @@ public: protected: RegAllocBase(MachineFunction &Function, const TargetDescription &Target, - const TargetRegisterAllocationInfo &TargetInfo, MachineAnalysisManager &Analyses, - RegisterAllocationOptions Options, MachineActionReplay *Actions = nullptr) noexcept + const TargetRegisterAllocationInfo &TargetInfo, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options, + MachineActionReplay *Actions = nullptr) noexcept : Function(Function), Target(Target), TargetInfo(TargetInfo), Analyses(Analyses), Options(Options), Actions(Actions) {} @@ -38,7 +39,7 @@ protected: MachineFunction &Function; const TargetDescription &Target; const TargetRegisterAllocationInfo &TargetInfo; - MachineAnalysisManager &Analyses; + MachineFunctionAnalysisManager &Analyses; RegisterAllocationOptions Options; MachineActionReplay *Actions = nullptr; }; diff --git a/backend/lib/CodeGen/RegAllocBasic.cpp b/backend/lib/CodeGen/RegAllocBasic.cpp index 9d14d15..7f730ca 100644 --- a/backend/lib/CodeGen/RegAllocBasic.cpp +++ b/backend/lib/CodeGen/RegAllocBasic.cpp @@ -18,7 +18,7 @@ class BasicAllocatorEngine final : public RegAllocBase { public: BasicAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, const TargetRegisterAllocationInfo &TargetInfo, - MachineAnalysisManager &Analyses, RegisterAllocationOptions Options) + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options) : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} private: @@ -80,7 +80,7 @@ private: MachineFunctionProperty::TracksLiveness}, .Sets = {MachineFunctionProperty::NoVRegs}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -97,7 +97,7 @@ const MachinePassContract &BasicAllocatorPass::contract() const noexcept { Expected BasicAllocatorPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (Info == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "BasicAllocator requires target behavior"); auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); @@ -112,7 +112,7 @@ Expected BasicAllocatorPass::run(MachineFunction &Function, regalloc_detail::reserveFunctionRegisters(*Trial->ClonedFunction, FunctionReserved); if (!TrialReserved) return std::unexpected(TrialReserved.error()); - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto Registered = registerStage11MachineAnalyses(TrialAnalyses); if (!Registered) return std::unexpected(Registered.error()); diff --git a/backend/lib/CodeGen/RegAllocBasic.h b/backend/lib/CodeGen/RegAllocBasic.h index 8a68f5f..f88c977 100644 --- a/backend/lib/CodeGen/RegAllocBasic.h +++ b/backend/lib/CodeGen/RegAllocBasic.h @@ -19,7 +19,7 @@ public: [[nodiscard]] const MachinePassContract &contract() const noexcept override; [[nodiscard]] Expected run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) override; + MachineFunctionAnalysisManager &Analyses) override; private: const TargetRegisterAllocationInfo *Info; diff --git a/backend/lib/CodeGen/RegAllocFast.cpp b/backend/lib/CodeGen/RegAllocFast.cpp index 1399f10..73c3e1a 100644 --- a/backend/lib/CodeGen/RegAllocFast.cpp +++ b/backend/lib/CodeGen/RegAllocFast.cpp @@ -557,7 +557,7 @@ const MachinePassContract &FastAllocatorPass::contract() const noexcept { MachineFunctionProperty::TracksLiveness}, .Sets = {MachineFunctionProperty::NoVRegs}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -568,7 +568,7 @@ const MachinePassContract &FastAllocatorPass::contract() const noexcept { Expected FastAllocatorPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (Info == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "FastAllocator requires target behavior"); auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); @@ -583,7 +583,7 @@ Expected FastAllocatorPass::run(MachineFunction &Function, regalloc_detail::reserveFunctionRegisters(*Trial->ClonedFunction, FunctionReserved); if (!TrialReserved) return std::unexpected(TrialReserved.error()); - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto Registered = registerStage11MachineAnalyses(TrialAnalyses); if (!Registered) return std::unexpected(Registered.error()); diff --git a/backend/lib/CodeGen/RegAllocPBQP.cpp b/backend/lib/CodeGen/RegAllocPBQP.cpp index 3b0e684..6706cd5 100644 --- a/backend/lib/CodeGen/RegAllocPBQP.cpp +++ b/backend/lib/CodeGen/RegAllocPBQP.cpp @@ -86,7 +86,7 @@ class PBQPAllocatorEngine final : public RegAllocBase { public: PBQPAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, const TargetRegisterAllocationInfo &TargetInfo, - MachineAnalysisManager &Analyses, RegisterAllocationOptions Options, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options, MachineActionReplay *Actions) : RegAllocBase(Function, Target, TargetInfo, Analyses, Options, Actions) {} @@ -277,7 +277,7 @@ private: MachineFunctionProperty::TracksLiveness}, .Sets = {MachineFunctionProperty::NoVRegs}, .Clears = {}, - .RequiredAnalyses = {MachineAnalysisManager::id()}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id()}, .VerificationStage = PipelineStage::RegisterAllocated, }; Value.ThreadSafety = PassThreadSafety::ThreadSafe; @@ -294,7 +294,7 @@ const MachinePassContract &PBQPAllocatorPass::contract() const noexcept { Expected PBQPAllocatorPass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &Analyses) { + MachineFunctionAnalysisManager &Analyses) { if (Info == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "PBQPAllocator requires target behavior"); auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); @@ -309,7 +309,7 @@ Expected PBQPAllocatorPass::run(MachineFunction &Function, regalloc_detail::reserveFunctionRegisters(*Trial->ClonedFunction, FunctionReserved); if (!TrialReserved) return std::unexpected(TrialReserved.error()); - MachineAnalysisManager TrialAnalyses; + MachineFunctionAnalysisManager TrialAnalyses; auto Registered = registerStage11MachineAnalyses(TrialAnalyses); if (!Registered) return std::unexpected(Registered.error()); diff --git a/backend/lib/CodeGen/SlotIndexes.cpp b/backend/lib/CodeGen/SlotIndexes.cpp index 57e35b5..4fe753a 100644 --- a/backend/lib/CodeGen/SlotIndexes.cpp +++ b/backend/lib/CodeGen/SlotIndexes.cpp @@ -143,12 +143,12 @@ Expected SlotIndexesAnalysis::Result::renumber(MachineFunction &Function) bool SlotIndexesAnalysis::Result::invalidate(const MachineFunction &, const PreservedAnalyses &Preserved) const noexcept { - return !Preserved.preserves(MachineAnalysisManager::id()); + return !Preserved.preserves(MachineFunctionAnalysisManager::id()); } Expected SlotIndexesAnalysis::run(MachineFunction &Function, const TargetDescription &, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { Result ResultValue; auto Numbered = ResultValue.renumber(Function); if (!Numbered) diff --git a/backend/lib/CodeGen/TwoAddressInstructionPass.cpp b/backend/lib/CodeGen/TwoAddressInstructionPass.cpp index be4db82..b6f2a25 100644 --- a/backend/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/backend/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -459,9 +459,9 @@ const MachinePassContract &TwoAddressInstructionPass::contract() const noexcept return Contract; } -Expected TwoAddressInstructionPass::run(MachineFunction &Function, - const TargetDescription &Target, - MachineAnalysisManager &Analyses) { +Expected +TwoAddressInstructionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { auto Required = requireProperties(Function, contract()); if (!Required) return std::unexpected(Required.error()); diff --git a/backend/lib/CodeGen/VirtRegMap.cpp b/backend/lib/CodeGen/VirtRegMap.cpp index a55a486..4f3ae45 100644 --- a/backend/lib/CodeGen/VirtRegMap.cpp +++ b/backend/lib/CodeGen/VirtRegMap.cpp @@ -169,7 +169,7 @@ Expected rewriteVirtualRegisters(MachineFunction &Function, const TargetDe if (!MappingVerification) return std::unexpected(MappingVerification.error()); - MachineAnalysisManager ValidationAnalyses; + MachineFunctionAnalysisManager ValidationAnalyses; auto Registered = registerStage11MachineAnalyses(ValidationAnalyses); if (!Registered) return std::unexpected(Registered.error()); @@ -282,7 +282,7 @@ const MachinePassContract &VirtualRegisterRewritePass::contract() const noexcept Expected VirtualRegisterRewritePass::run(MachineFunction &Function, const TargetDescription &Target, - MachineAnalysisManager &) { + MachineFunctionAnalysisManager &) { if (Mapping == nullptr) return makeUnexpected(ErrorCode::InvalidArgument, "VirtualRegisterRewrite requires a VirtRegMap"); diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp index c5601ef..60f798c 100644 --- a/backend/lib/Target/RISCV/RISCVPipeline.cpp +++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp @@ -27,11 +27,12 @@ namespace rcc::riscv { namespace { template -[[nodiscard]] Expected addPass(MachinePassManager &Pipeline, ArgumentT &&...Arguments) { +[[nodiscard]] Expected addPass(MachineFunctionPassManager &Pipeline, + ArgumentT &&...Arguments) { return Pipeline.addPass(std::make_unique(std::forward(Arguments)...)); } -[[nodiscard]] Expected addInitialPipeline(MachinePassManager &Pipeline, +[[nodiscard]] Expected addInitialPipeline(MachineFunctionPassManager &Pipeline, const RISCVSubtarget &Subtarget) { auto Added = addPass(Pipeline, machineCombinerInfo(), MachineCombinerStage::PreLegalizer); @@ -46,7 +47,7 @@ template return addPass(Pipeline, Subtarget.getInstructionSelector()); } -[[nodiscard]] Expected addSSAExitPipeline(MachinePassManager &Pipeline) { +[[nodiscard]] Expected addSSAExitPipeline(MachineFunctionPassManager &Pipeline) { auto Added = addSelectedSSAOptimizationPipeline(Pipeline, optimizationPipelineInfo()); if (!Added) return std::unexpected(Added.error()); @@ -56,14 +57,14 @@ template return addPass(Pipeline, instrInfo()); } -[[nodiscard]] Expected addRegisterAllocationPipeline(MachinePassManager &Pipeline) { +[[nodiscard]] Expected addRegisterAllocationPipeline(MachineFunctionPassManager &Pipeline) { auto Added = addPreRegisterAllocationOptimizationPipeline(Pipeline, optimizationPipelineInfo()); if (!Added) return std::unexpected(Added.error()); return addPass(Pipeline, registerAllocationInfo()); } -[[nodiscard]] Expected addCompleteRegistry(MachinePassManager &Pipeline, +[[nodiscard]] Expected addCompleteRegistry(MachineFunctionPassManager &Pipeline, const RISCVSubtarget &Subtarget) { auto Added = addInitialPipeline(Pipeline, Subtarget); if (!Added) @@ -79,7 +80,8 @@ template } // namespace -Expected addCanonicalPipeline(MachinePassManager &Pipeline, const MachineFunction &Function, +Expected addCanonicalPipeline(MachineFunctionPassManager &Pipeline, + const MachineFunction &Function, const RISCVSubtarget &Subtarget) { if (!Function.properties().has(MachineFunctionProperty::Legalized)) { auto Added = addPass(Pipeline, machineCombinerInfo(), @@ -132,11 +134,11 @@ Expected> createCanonicalPipelinePass(std::string_view Name, const RISCVSubtarget &Subtarget) { if (Name.empty()) return makeUnexpected(ErrorCode::InvalidArgument, "pipeline pass name cannot be empty"); - MachinePassManager Registry; + MachineFunctionPassManager Registry; auto Added = addCompleteRegistry(Registry, Subtarget); if (!Added) return std::unexpected(Added.error()); - MachinePassManager::PassList Passes = std::move(Registry).takePasses(); + MachineFunctionPassManager::PassList Passes = std::move(Registry).takePasses(); auto Position = std::ranges::find_if(Passes, [Name](const auto &Pass) { return Pass != nullptr && Pass->contract().Name == Name; }); @@ -149,11 +151,11 @@ createCanonicalPipelinePass(std::string_view Name, const RISCVSubtarget &Subtarg Expected> canonicalPipelinePassNames(const MachineFunction &Function, const RISCVSubtarget &Subtarget) { - MachinePassManager Pipeline; + MachineFunctionPassManager Pipeline; auto Added = addCanonicalPipeline(Pipeline, Function, Subtarget); if (!Added) return std::unexpected(Added.error()); - MachinePassManager::PassList Passes = std::move(Pipeline).takePasses(); + MachineFunctionPassManager::PassList Passes = std::move(Pipeline).takePasses(); std::vector Result; Result.reserve(Passes.size()); for (const std::unique_ptr &Pass : Passes) { diff --git a/backend/lib/Tooling/PipelineDriver.cpp b/backend/lib/Tooling/PipelineDriver.cpp index 495842a..d7a7d16 100644 --- a/backend/lib/Tooling/PipelineDriver.cpp +++ b/backend/lib/Tooling/PipelineDriver.cpp @@ -79,10 +79,10 @@ struct FunctionRunResult final { return std::nullopt; } -[[nodiscard]] Expected +[[nodiscard]] Expected createPasses(const MachineFunction &Function, const BackendInvocation &Invocation, const riscv::RISCVSubtarget &Subtarget, MachinePassManagerOptions Options) { - MachinePassManager Pipeline(Options); + MachineFunctionPassManager Pipeline(Options); if (Invocation.PassPipeline.size() == 1 && Invocation.PassPipeline.front() == "default") { auto Added = riscv::addCanonicalPipeline(Pipeline, Function, Subtarget); if (!Added) @@ -212,7 +212,7 @@ runFunction(MachineFunction &Function, mir::MIRDocument &Document, const TargetD auto Passes = createPasses(Function, Invocation, Subtarget, ManagerOptions); if (!Passes) return std::unexpected(Passes.error()); - MachineAnalysisManager Analyses; + MachineFunctionAnalysisManager Analyses; auto Registered = registerMachineOptimizationAnalyses(Analyses); if (!Registered) return std::unexpected(Registered.error()); @@ -257,7 +257,7 @@ runFunction(MachineFunction &Function, mir::MIRDocument &Document, const TargetD return std::unexpected(Captured.error()); } - MachinePassManager SinglePass(ManagerOptions); + MachineFunctionPassManager SinglePass(ManagerOptions); auto Added = SinglePass.addPass(std::move(Pass)); if (!Added) return std::unexpected(Added.error()); -- cgit v1.2.3