aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--backend/include/rcc/CodeGen/MachineOutliner.h97
-rw-r--r--backend/include/rcc/MIR/EmbeddedIR.h20
-rw-r--r--backend/include/rcc/Target/RISCV/RISCVPipeline.h3
-rw-r--r--backend/lib/CodeGen/CMakeLists.txt2
-rw-r--r--backend/lib/CodeGen/MIRYamlMapping.cpp8
-rw-r--r--backend/lib/CodeGen/MachineOutliner.cpp614
-rw-r--r--backend/lib/MIR/EmbeddedIR.cpp17
-rw-r--r--backend/lib/Target/RISCV/CMakeLists.txt1
-rw-r--r--backend/lib/Target/RISCV/RISCVOutliner.cpp344
-rw-r--r--backend/lib/Tooling/PipelineDriver.cpp67
10 files changed, 1167 insertions, 6 deletions
diff --git a/backend/include/rcc/CodeGen/MachineOutliner.h b/backend/include/rcc/CodeGen/MachineOutliner.h
new file mode 100644
index 0000000..a12ddaf
--- /dev/null
+++ b/backend/include/rcc/CodeGen/MachineOutliner.h
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_CODEGEN_MACHINEOUTLINER_H
+#define RCC_CODEGEN_MACHINEOUTLINER_H
+
+#include "rcc/CodeGen/MachinePassManager.h"
+#include "rcc/CodeGen/Register.h"
+#include "rcc/Support/Error.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace rcc {
+
+class MachineBasicBlock;
+class MachineFunction;
+class MachineInstr;
+class TargetDescription;
+
+enum class MachineOutlinerInstrType : std::uint8_t {
+ Legal,
+ LegalTerminator,
+ Illegal,
+ Invisible,
+};
+
+struct MachineOutlinerFunctionProperties final {
+ bool LinkOnceODR = false;
+ bool HasSection = false;
+ bool HasComdat = false;
+ bool HasSectionPrefix = false;
+};
+
+using MachineOutlinerFunctionPropertyMap =
+ std::unordered_map<std::string, MachineOutlinerFunctionProperties>;
+
+struct MachineOutlinerOptions final {
+ MachineOutlinerFunctionPropertyMap FunctionProperties;
+};
+
+struct MachineOutlinerCandidate final {
+ MachineFunction *Function = nullptr;
+ MachineBasicBlock *Block = nullptr;
+ std::vector<MachineInstr *> Instructions;
+ std::vector<PhysicalRegister> LiveIns;
+ std::vector<PhysicalRegister> DefinedRegisters;
+ std::uint32_t CallConstructionID = 0;
+ std::uint32_t CallOverhead = 0;
+ PhysicalRegister SaveRegister;
+
+ [[nodiscard]] MachineInstr &front() const noexcept;
+ [[nodiscard]] MachineInstr &back() const noexcept;
+ [[nodiscard]] bool containsRegister(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept;
+ [[nodiscard]] bool isRegisterLiveAfter(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept;
+ [[nodiscard]] bool isRegisterAvailable(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept;
+};
+
+class TargetMachineOutlinerInfo {
+public:
+ virtual ~TargetMachineOutlinerInfo();
+
+ [[nodiscard]] virtual bool isFunctionSafeToOutlineFrom(
+ const MachineFunction &Function,
+ const MachineOutlinerFunctionProperties &Properties) const noexcept = 0;
+ [[nodiscard]] virtual MachineOutlinerInstrType
+ getOutliningType(const MachineInstr &Instruction,
+ const MachineOutlinerFunctionProperties &Properties,
+ const TargetDescription &Target) const noexcept = 0;
+ [[nodiscard]] virtual Expected<bool> analyzeCandidate(MachineOutlinerCandidate &Candidate,
+ const TargetDescription &Target) const = 0;
+ [[nodiscard]] virtual Expected<std::uint32_t>
+ instructionSize(const MachineInstr &Instruction, const MachineFunction &Function,
+ const TargetDescription &Target) const = 0;
+ [[nodiscard]] virtual std::uint32_t
+ frameOverhead(const MachineOutlinerCandidate &Candidate) const noexcept = 0;
+ [[nodiscard]] virtual Expected<void>
+ insertOutlinedCall(MachineOutlinerCandidate &Candidate, std::string OutlinedFunctionName,
+ const TargetDescription &Target) const = 0;
+ [[nodiscard]] virtual Expected<void>
+ buildOutlinedFrame(MachineFunction &Function, MachineBasicBlock &Block,
+ const MachineOutlinerCandidate &Candidate,
+ const TargetDescription &Target) const = 0;
+};
+
+[[nodiscard]] std::unique_ptr<CompilationUnitPass>
+createMachineOutlinerPass(std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo,
+ MachineOutlinerOptions Options = {});
+
+} // namespace rcc
+
+#endif // RCC_CODEGEN_MACHINEOUTLINER_H
diff --git a/backend/include/rcc/MIR/EmbeddedIR.h b/backend/include/rcc/MIR/EmbeddedIR.h
index 022f121..80e512a 100644
--- a/backend/include/rcc/MIR/EmbeddedIR.h
+++ b/backend/include/rcc/MIR/EmbeddedIR.h
@@ -58,6 +58,22 @@ public:
return IsUnnamed;
}
+ [[nodiscard]] bool hasLinkOnceODRLinkage() const noexcept {
+ return HasLinkOnceODRLinkage;
+ }
+
+ [[nodiscard]] bool hasSection() const noexcept {
+ return HasSection;
+ }
+
+ [[nodiscard]] bool hasComdat() const noexcept {
+ return HasComdat;
+ }
+
+ [[nodiscard]] bool hasSectionPrefix() const noexcept {
+ return HasSectionPrefix;
+ }
+
[[nodiscard]] std::span<const EmbeddedIRReference> blocks() const noexcept {
return Blocks;
}
@@ -87,6 +103,10 @@ private:
TextRange NameRange;
bool IsDefinition = false;
bool IsUnnamed = false;
+ bool HasLinkOnceODRLinkage = false;
+ bool HasSection = false;
+ bool HasComdat = false;
+ bool HasSectionPrefix = false;
std::uint64_t NextUnnamedSlot = 0;
std::vector<EmbeddedIRReference> Blocks;
std::vector<EmbeddedIRReference> Allocas;
diff --git a/backend/include/rcc/Target/RISCV/RISCVPipeline.h b/backend/include/rcc/Target/RISCV/RISCVPipeline.h
index 50a6a68..3e8ea3d 100644
--- a/backend/include/rcc/Target/RISCV/RISCVPipeline.h
+++ b/backend/include/rcc/Target/RISCV/RISCVPipeline.h
@@ -17,6 +17,7 @@ namespace rcc {
class MachineFunction;
class TargetDescription;
+class TargetMachineOutlinerInfo;
namespace riscv {
@@ -37,6 +38,8 @@ createCanonicalPipelinePass(std::string_view Name, const RISCVSubtarget &Subtarg
RISCVPipelineOptions Options = {});
[[nodiscard]] Expected<std::vector<std::string>>
canonicalPipelinePassNames(const MachineFunction &Function, const RISCVSubtarget &Subtarget);
+[[nodiscard]] Expected<std::unique_ptr<TargetMachineOutlinerInfo>>
+createMachineOutlinerInfo(bool FunctionSections, bool EnableRegisterSave);
} // namespace riscv
} // namespace rcc
diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt
index 4c9cd76..03fec17 100644
--- a/backend/lib/CodeGen/CMakeLists.txt
+++ b/backend/lib/CodeGen/CMakeLists.txt
@@ -49,6 +49,7 @@ set(
MachineMetadata.cpp
MachineOperand.cpp
MachineOptimizationPipeline.cpp
+ MachineOutliner.cpp
MachinePassManager.cpp
MachinePseudoExpansion.cpp
MachineRegisterInfo.cpp
@@ -140,6 +141,7 @@ set(
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOutliner.h"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/IR/PassManager.h"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/IR/PassInstrumentation.h"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAnalysisManager.h"
diff --git a/backend/lib/CodeGen/MIRYamlMapping.cpp b/backend/lib/CodeGen/MIRYamlMapping.cpp
index cd94fc6..c36f006 100644
--- a/backend/lib/CodeGen/MIRYamlMapping.cpp
+++ b/backend/lib/CodeGen/MIRYamlMapping.cpp
@@ -1009,14 +1009,16 @@ Expected<void> verifyMIRDocumentReferences(const MIRDocument &Document,
} else {
IRFunction = Embedded->function(Function.name());
}
- if (IRFunction == nullptr) {
+ if (IRFunction == nullptr && !Function.attributes().has(MachineFunctionAttribute::IsOutlined)) {
return makeUnexpected(ErrorCode::VerificationError,
"machine function has no matching embedded LLVM IR function");
}
}
- const auto ValidateGlobal = [Embedded](std::string_view Name,
- std::string_view Description) -> Expected<void> {
+ const auto ValidateGlobal =
+ [&Document, Embedded](std::string_view Name, std::string_view Description) -> Expected<void> {
+ if (Document.unit().function(Name) != nullptr)
+ return {};
if (Embedded == nullptr) {
return makeUnexpected(ErrorCode::VerificationError,
std::string(Description) +
diff --git a/backend/lib/CodeGen/MachineOutliner.cpp b/backend/lib/CodeGen/MachineOutliner.cpp
new file mode 100644
index 0000000..bdc4d2b
--- /dev/null
+++ b/backend/lib/CodeGen/MachineOutliner.cpp
@@ -0,0 +1,614 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/MachineOutliner.h"
+
+#include "rcc/CodeGen/CompilationUnit.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineCFIInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineFunctionAttributes.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <memory>
+#include <ranges>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+struct InstructionSegment final {
+ MachineFunction *Function = nullptr;
+ MachineBasicBlock *Block = nullptr;
+ std::vector<MachineInstr *> Instructions;
+};
+
+struct SequenceGroup final {
+ std::vector<MachineInstr *> Representative;
+ std::vector<MachineOutlinerCandidate> Candidates;
+ std::size_t DiscoveryOrder = 0;
+ std::uint32_t SequenceSize = 0;
+ std::int64_t Benefit = 0;
+};
+
+[[nodiscard]] RegisterOperandFlags normalizedRegisterFlags(RegisterOperandFlags Flags) noexcept {
+ Flags.clear(RegisterOperandFlag::Kill);
+ Flags.clear(RegisterOperandFlag::Dead);
+ Flags.clear(RegisterOperandFlag::Renamable);
+ return Flags;
+}
+
+template <typename PayloadT>
+[[nodiscard]] bool payloadsEqual(const MachineOperand &Left, const MachineOperand &Right) noexcept {
+ const PayloadT *LeftValue = Left.getIf<PayloadT>();
+ const PayloadT *RightValue = Right.getIf<PayloadT>();
+ return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue;
+}
+
+[[nodiscard]] bool operandsEqual(const MachineOperand &Left, const MachineOperand &Right,
+ const MachineFunction &LeftFunction,
+ const MachineFunction &RightFunction) noexcept {
+ if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() ||
+ Left.subReg() != Right.subReg() || Left.tiedOperandIndex() != Right.tiedOperandIndex()) {
+ return false;
+ }
+ if (Left.isRegister() && normalizedRegisterFlags(Left.registerFlags()) !=
+ normalizedRegisterFlags(Right.registerFlags())) {
+ return false;
+ }
+
+ switch (Left.kind()) {
+ case MachineOperand::Kind::Register:
+ return payloadsEqual<Register>(Left, Right);
+ case MachineOperand::Kind::Immediate:
+ return payloadsEqual<std::int64_t>(Left, Right);
+ case MachineOperand::Kind::IntegerImmediate:
+ return payloadsEqual<IntegerImmediateOperand>(Left, Right);
+ case MachineOperand::Kind::FloatingImmediate:
+ return payloadsEqual<FloatingImmediateOperand>(Left, Right);
+ case MachineOperand::Kind::BasicBlock:
+ return payloadsEqual<BasicBlockOperand>(Left, Right);
+ case MachineOperand::Kind::FrameIndex:
+ return payloadsEqual<FrameIndexOperand>(Left, Right);
+ case MachineOperand::Kind::ConstantPoolIndex:
+ return payloadsEqual<ConstantPoolOperand>(Left, Right);
+ case MachineOperand::Kind::JumpTableIndex:
+ return payloadsEqual<JumpTableOperand>(Left, Right);
+ case MachineOperand::Kind::GlobalSymbol:
+ return payloadsEqual<GlobalSymbolOperand>(Left, Right);
+ case MachineOperand::Kind::ExternalSymbol:
+ return payloadsEqual<ExternalSymbolOperand>(Left, Right);
+ case MachineOperand::Kind::BlockAddress:
+ return payloadsEqual<BlockAddressOperand>(Left, Right);
+ case MachineOperand::Kind::RegisterMask:
+ return payloadsEqual<RegisterMaskOperand>(Left, Right);
+ case MachineOperand::Kind::LiveOutRegisterMask:
+ return payloadsEqual<LiveOutRegisterMaskOperand>(Left, Right);
+ case MachineOperand::Kind::Metadata:
+ return payloadsEqual<MetadataOperand>(Left, Right);
+ case MachineOperand::Kind::TargetIndex:
+ return payloadsEqual<TargetIndexOperand>(Left, Right);
+ case MachineOperand::Kind::MCSymbol:
+ return payloadsEqual<MCSymbolOperand>(Left, Right);
+ case MachineOperand::Kind::CFIIndex: {
+ const CFIIndexOperand *LeftIndex = Left.getIf<CFIIndexOperand>();
+ const CFIIndexOperand *RightIndex = Right.getIf<CFIIndexOperand>();
+ if (LeftIndex == nullptr || RightIndex == nullptr)
+ return false;
+ const MachineCFIInstruction *LeftCFI = LeftFunction.cfiInfo().instruction(LeftIndex->Index);
+ const MachineCFIInstruction *RightCFI = RightFunction.cfiInfo().instruction(RightIndex->Index);
+ return LeftCFI != nullptr && RightCFI != nullptr && *LeftCFI == *RightCFI;
+ }
+ case MachineOperand::Kind::Intrinsic:
+ return payloadsEqual<IntrinsicOperand>(Left, Right);
+ case MachineOperand::Kind::Predicate:
+ return payloadsEqual<PredicateOperand>(Left, Right);
+ case MachineOperand::Kind::ShuffleMask:
+ return payloadsEqual<ShuffleMaskOperand>(Left, Right);
+ case MachineOperand::Kind::DebugInstrRef:
+ return payloadsEqual<DebugInstrRefOperand>(Left, Right);
+ case MachineOperand::Kind::LaneMask:
+ return payloadsEqual<LaneMaskOperand>(Left, Right);
+ case MachineOperand::Kind::Count:
+ return false;
+ }
+ return false;
+}
+
+[[nodiscard]] bool instructionsEqual(const MachineInstr &Left, const MachineInstr &Right) noexcept {
+ if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() ||
+ Left.operandCount() != Right.operandCount() ||
+ Left.memOperands().size() != Right.memOperands().size()) {
+ return false;
+ }
+ for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) {
+ if (!operandsEqual(Left.operands()[Index], Right.operands()[Index], Left.function(),
+ Right.function())) {
+ return false;
+ }
+ }
+ for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) {
+ const MachineMemOperand *LeftMemory = Left.memOperands()[Index];
+ const MachineMemOperand *RightMemory = Right.memOperands()[Index];
+ if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory)
+ return false;
+ }
+ return true;
+}
+
+[[nodiscard]] bool sequencesEqual(const std::vector<MachineInstr *> &Left,
+ const std::vector<MachineInstr *> &Right) noexcept {
+ return Left.size() == Right.size() &&
+ std::ranges::equal(
+ Left, Right,
+ [](const MachineInstr *LeftInstruction, const MachineInstr *RightInstruction) {
+ return LeftInstruction != nullptr && RightInstruction != nullptr &&
+ instructionsEqual(*LeftInstruction, *RightInstruction);
+ });
+}
+
+[[nodiscard]] bool overlaps(PhysicalRegister Left, PhysicalRegister Right,
+ const TargetDescription &Target) noexcept {
+ return Target.registers().regsOverlap(Left, Right);
+}
+
+[[nodiscard]] bool containsRegister(std::span<const PhysicalRegister> Registers,
+ PhysicalRegister Reg,
+ const TargetDescription &Target) noexcept {
+ return std::ranges::any_of(Registers, [Reg, &Target](PhysicalRegister Existing) {
+ return overlaps(Existing, Reg, Target);
+ });
+}
+
+void appendUnique(std::vector<PhysicalRegister> &Registers, PhysicalRegister Reg,
+ const TargetDescription &Target) {
+ if (!containsRegister(Registers, Reg, Target))
+ Registers.push_back(Reg);
+}
+
+void computeCandidateRegisters(MachineOutlinerCandidate &Candidate,
+ const TargetDescription &Target) {
+ std::vector<PhysicalRegister> Defined;
+ for (const MachineInstr *Instruction : Candidate.Instructions) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto RegisterValue = Operand.registerValue();
+ const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt;
+ if (!Physical || Operand.isDebugUse())
+ continue;
+ if (Operand.isUse() && !containsRegister(Defined, *Physical, Target))
+ appendUnique(Candidate.LiveIns, *Physical, Target);
+ }
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto RegisterValue = Operand.registerValue();
+ const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt;
+ if (!Physical || Operand.isDebugUse())
+ continue;
+ if (Operand.isDef()) {
+ appendUnique(Defined, *Physical, Target);
+ appendUnique(Candidate.DefinedRegisters, *Physical, Target);
+ }
+ }
+ }
+ std::ranges::sort(Candidate.DefinedRegisters, {}, &PhysicalRegister::number);
+}
+
+[[nodiscard]] const MachineOutlinerFunctionProperties &
+propertiesFor(const MachineOutlinerOptions &Options, const MachineFunction &Function) noexcept {
+ static const MachineOutlinerFunctionProperties Empty;
+ const auto Position = Options.FunctionProperties.find(std::string(Function.name()));
+ return Position == Options.FunctionProperties.end() ? Empty : Position->second;
+}
+
+[[nodiscard]] std::vector<InstructionSegment>
+collectSegments(CompilationUnit &Unit, const TargetMachineOutlinerInfo &TargetInfo,
+ const MachineOutlinerOptions &Options, const TargetDescription &Target) {
+ std::vector<InstructionSegment> Segments;
+ for (std::size_t FunctionIndex = 0; FunctionIndex < Unit.functionCount(); ++FunctionIndex) {
+ MachineFunction *Function = Unit.functionAt(FunctionIndex);
+ if (Function == nullptr || Function->attributes().has(MachineFunctionAttribute::IsOutlined) ||
+ !TargetInfo.isFunctionSafeToOutlineFrom(*Function, propertiesFor(Options, *Function))) {
+ continue;
+ }
+ for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ InstructionSegment Current{
+ .Function = Function,
+ .Block = Block,
+ .Instructions = {},
+ };
+ const auto Flush = [&]() {
+ if (Current.Instructions.size() >= 2)
+ Segments.push_back(Current);
+ Current.Instructions.clear();
+ };
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const MachineOutlinerInstrType Type =
+ TargetInfo.getOutliningType(*Instruction, propertiesFor(Options, *Function), Target);
+ if (Type == MachineOutlinerInstrType::Illegal ||
+ Type == MachineOutlinerInstrType::Invisible) {
+ Flush();
+ continue;
+ }
+ Current.Instructions.push_back(Instruction);
+ if (Type == MachineOutlinerInstrType::LegalTerminator)
+ Flush();
+ }
+ Flush();
+ }
+ }
+ return Segments;
+}
+
+[[nodiscard]] Expected<std::vector<SequenceGroup>>
+buildSequenceGroups(std::vector<InstructionSegment> &Segments,
+ const TargetMachineOutlinerInfo &TargetInfo, const TargetDescription &Target) {
+ std::vector<SequenceGroup> Groups;
+ std::size_t DiscoveryOrder = 0;
+ for (InstructionSegment &Segment : Segments) {
+ for (std::size_t Begin = 0; Begin + 1 < Segment.Instructions.size(); ++Begin) {
+ for (std::size_t Length = 2; Begin + Length <= Segment.Instructions.size(); ++Length) {
+ std::vector<MachineInstr *> Sequence(
+ Segment.Instructions.begin() + static_cast<std::ptrdiff_t>(Begin),
+ Segment.Instructions.begin() + static_cast<std::ptrdiff_t>(Begin + Length));
+ auto Position = std::ranges::find_if(Groups, [&Sequence](const SequenceGroup &Group) {
+ return sequencesEqual(Group.Representative, Sequence);
+ });
+ if (Position == Groups.end()) {
+ Groups.push_back({
+ .Representative = Sequence,
+ .Candidates = {},
+ .DiscoveryOrder = DiscoveryOrder++,
+ .SequenceSize = 0,
+ .Benefit = 0,
+ });
+ Position = std::prev(Groups.end());
+ }
+ MachineOutlinerCandidate Candidate{
+ .Function = Segment.Function,
+ .Block = Segment.Block,
+ .Instructions = std::move(Sequence),
+ .LiveIns = {},
+ .DefinedRegisters = {},
+ .CallConstructionID = 0,
+ .CallOverhead = 0,
+ .SaveRegister = {},
+ };
+ computeCandidateRegisters(Candidate, Target);
+ Position->Candidates.push_back(std::move(Candidate));
+ }
+ }
+ }
+
+ for (SequenceGroup &Group : Groups) {
+ std::vector<MachineOutlinerCandidate> Viable;
+ for (MachineOutlinerCandidate &Candidate : Group.Candidates) {
+ auto Accepted = TargetInfo.analyzeCandidate(Candidate, Target);
+ if (!Accepted)
+ return std::unexpected(Accepted.error());
+ if (*Accepted)
+ Viable.push_back(std::move(Candidate));
+ }
+ Group.Candidates = std::move(Viable);
+ }
+ std::erase_if(Groups, [](const SequenceGroup &Group) { return Group.Candidates.size() < 2; });
+
+ std::unordered_map<const MachineInstr *, std::size_t> LongestAtStart;
+ for (const SequenceGroup &Group : Groups) {
+ for (const MachineOutlinerCandidate &Candidate : Group.Candidates) {
+ const MachineInstr *Start = Candidate.Instructions.front();
+ LongestAtStart[Start] = std::max(LongestAtStart[Start], Candidate.Instructions.size());
+ }
+ }
+ for (SequenceGroup &Group : Groups) {
+ std::erase_if(Group.Candidates, [&LongestAtStart](const MachineOutlinerCandidate &Candidate) {
+ return Candidate.Instructions.size() < LongestAtStart[Candidate.Instructions.front()];
+ });
+ std::unordered_set<const MachineInstr *> Claimed;
+ std::vector<MachineOutlinerCandidate> NonOverlapping;
+ for (MachineOutlinerCandidate &Candidate : Group.Candidates) {
+ const bool HasOverlap = std::ranges::any_of(
+ Candidate.Instructions, [&Claimed](const auto *I) { return Claimed.contains(I); });
+ if (HasOverlap)
+ continue;
+ Claimed.insert(Candidate.Instructions.begin(), Candidate.Instructions.end());
+ NonOverlapping.push_back(std::move(Candidate));
+ }
+ Group.Candidates = std::move(NonOverlapping);
+ }
+ std::erase_if(Groups, [](const SequenceGroup &Group) { return Group.Candidates.size() < 2; });
+ return Groups;
+}
+
+[[nodiscard]] Expected<void> calculateBenefit(SequenceGroup &Group,
+ const TargetMachineOutlinerInfo &TargetInfo,
+ const TargetDescription &Target) {
+ std::uint64_t SequenceSize = 0;
+ for (const MachineInstr *Instruction : Group.Representative) {
+ auto Size = TargetInfo.instructionSize(*Instruction, Instruction->function(), Target);
+ if (!Size)
+ return std::unexpected(Size.error());
+ SequenceSize += *Size;
+ }
+ if (SequenceSize > std::numeric_limits<std::uint32_t>::max()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "machine outliner sequence size exceeds the supported range");
+ }
+ Group.SequenceSize = static_cast<std::uint32_t>(SequenceSize);
+ std::uint64_t Cost = SequenceSize + TargetInfo.frameOverhead(Group.Candidates.front());
+ for (const MachineOutlinerCandidate &Candidate : Group.Candidates)
+ Cost += Candidate.CallOverhead;
+ const std::uint64_t Original = SequenceSize * Group.Candidates.size();
+ Group.Benefit = static_cast<std::int64_t>(Original) - static_cast<std::int64_t>(Cost);
+ return {};
+}
+
+[[nodiscard]] Expected<MachineOperand> cloneOperand(const MachineOperand &Operand,
+ const MachineFunction &Source,
+ MachineFunction &Destination) {
+ if (Operand.kind() == MachineOperand::Kind::CFIIndex) {
+ const CFIIndexOperand *Index = Operand.getIf<CFIIndexOperand>();
+ const MachineCFIInstruction *Instruction =
+ Index == nullptr ? nullptr : Source.cfiInfo().instruction(Index->Index);
+ if (Instruction == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "machine outliner encountered an invalid CFI reference");
+ }
+ auto NewIndex = Destination.cfiInfo().addInstruction(*Instruction);
+ if (!NewIndex)
+ return std::unexpected(NewIndex.error());
+ return MachineOperand::cfiIndex(*NewIndex, Operand.targetFlags());
+ }
+ return Operand.cloneDetached();
+}
+
+[[nodiscard]] Expected<MachineInstr *> cloneInstruction(const MachineInstr &Source,
+ MachineFunction &Destination,
+ MachineBasicBlock &DestinationBlock) {
+ auto Created = Destination.createInstruction(Source.opcode());
+ if (!Created)
+ return std::unexpected(Created.error());
+ MachineInstr *Clone = *Created;
+ Clone->reserveOperands(Source.operandCount());
+ for (const MachineOperand &Operand : Source.operands()) {
+ auto ClonedOperand = cloneOperand(Operand, Source.function(), Destination);
+ if (!ClonedOperand)
+ return std::unexpected(ClonedOperand.error());
+ auto Added = Clone->addOperand(std::move(*ClonedOperand));
+ if (!Added)
+ return std::unexpected(Added.error());
+ }
+ for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) {
+ const std::optional<std::size_t> Tied = Source.operands()[Index].tiedOperandIndex();
+ if (!Tied || Index > *Tied)
+ continue;
+ const std::size_t DefIndex = Source.operands()[Index].isDef() ? Index : *Tied;
+ const std::size_t UseIndex = Source.operands()[Index].isDef() ? *Tied : Index;
+ auto TiedResult = Clone->tieOperands(DefIndex, UseIndex);
+ if (!TiedResult)
+ return std::unexpected(TiedResult.error());
+ }
+ for (const MachineMemOperand *Memory : Source.memOperands()) {
+ if (Memory == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "machine outliner encountered a null memory operand");
+ }
+ auto ClonedMemory = Destination.createMemOperand(Memory->description());
+ if (!ClonedMemory)
+ return std::unexpected(ClonedMemory.error());
+ auto Added = Clone->addMemOperand(**ClonedMemory);
+ if (!Added)
+ return std::unexpected(Added.error());
+ }
+ Clone->setFlags(Source.flags());
+ auto Appended = DestinationBlock.appendInstruction(*Clone);
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ return Clone;
+}
+
+[[nodiscard]] Expected<std::string> nextOutlinedFunctionName(const CompilationUnit &Unit) {
+ for (std::uint64_t Index = 0; Index < std::numeric_limits<std::uint32_t>::max(); ++Index) {
+ std::string Name = "OUTLINED_FUNCTION_" + std::to_string(Index);
+ if (Unit.function(Name) == nullptr && Unit.globalObject(Name) == nullptr)
+ return Name;
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "machine outliner exhausted outlined function names");
+}
+
+[[nodiscard]] Expected<void> outlineGroup(SequenceGroup &Group, CompilationUnit &Unit,
+ const TargetMachineOutlinerInfo &TargetInfo,
+ const TargetDescription &Target) {
+ auto Name = nextOutlinedFunctionName(Unit);
+ if (!Name)
+ return std::unexpected(Name.error());
+ auto Created = Unit.createMachineFunction(*Name);
+ if (!Created)
+ return std::unexpected(Created.error());
+ MachineFunction &Outlined = **Created;
+ Outlined.properties() = Group.Candidates.front().Function->properties();
+ Outlined.attributes().set(MachineFunctionAttribute::IsOutlined);
+
+ auto CreatedBlock = Outlined.createBasicBlock();
+ if (!CreatedBlock)
+ return std::unexpected(CreatedBlock.error());
+ MachineBasicBlock &OutlinedBlock = **CreatedBlock;
+ auto Appended = Outlined.appendBlock(OutlinedBlock);
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ std::vector<PhysicalRegister> OutlinedLiveIns = Group.Candidates.front().LiveIns;
+ std::ranges::sort(OutlinedLiveIns, std::greater{}, &PhysicalRegister::number);
+ for (const PhysicalRegister LiveIn : OutlinedLiveIns) {
+ auto Added = OutlinedBlock.addLiveIn(LiveIn);
+ if (!Added)
+ return std::unexpected(Added.error());
+ }
+ for (const MachineInstr *Instruction : Group.Representative) {
+ auto Cloned = cloneInstruction(*Instruction, Outlined, OutlinedBlock);
+ if (!Cloned)
+ return std::unexpected(Cloned.error());
+ }
+ auto Frame =
+ TargetInfo.buildOutlinedFrame(Outlined, OutlinedBlock, Group.Candidates.front(), Target);
+ if (!Frame)
+ return std::unexpected(Frame.error());
+
+ for (MachineOutlinerCandidate &Candidate : Group.Candidates) {
+ auto Inserted = TargetInfo.insertOutlinedCall(Candidate, *Name, Target);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ for (MachineInstr *Instruction : Candidate.Instructions) {
+ auto Removed = Candidate.Block->removeInstruction(*Instruction);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ }
+ }
+ return {};
+}
+
+class MachineOutlinerPass final : public CompilationUnitPass {
+public:
+ MachineOutlinerPass(std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo,
+ MachineOutlinerOptions Options)
+ : TargetInfo(std::move(TargetInfo)), Options(std::move(Options)) {}
+
+ [[nodiscard]] const CompilationUnitPassContract &contract() const noexcept override {
+ static const CompilationUnitPassContract Contract{
+ .Name = "machine-outliner",
+ .RequiredAnalyses = {},
+ .Parameters = {},
+ .DeclaredChanges = MachinePassChanges::all(),
+ };
+ return Contract;
+ }
+
+ [[nodiscard]] Expected<CompilationUnitPassResult> run(CompilationUnit &Unit,
+ const TargetDescription &Target,
+ CompilationUnitAnalysisManager &,
+ MachineFunctionAnalysisManager &) override {
+ if (TargetInfo == nullptr) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "machine outliner requires target-specific hooks");
+ }
+ bool Changed = false;
+ while (true) {
+ std::vector<InstructionSegment> Segments =
+ collectSegments(Unit, *TargetInfo, Options, Target);
+ auto Groups = buildSequenceGroups(Segments, *TargetInfo, Target);
+ if (!Groups)
+ return std::unexpected(Groups.error());
+ for (SequenceGroup &Group : *Groups) {
+ auto Calculated = calculateBenefit(Group, *TargetInfo, Target);
+ if (!Calculated)
+ return std::unexpected(Calculated.error());
+ }
+ auto Best = std::ranges::max_element(
+ *Groups, [](const SequenceGroup &Left, const SequenceGroup &Right) {
+ if (Left.Benefit != Right.Benefit)
+ return Left.Benefit < Right.Benefit;
+ if (Left.Representative.size() != Right.Representative.size())
+ return Left.Representative.size() < Right.Representative.size();
+ return Left.DiscoveryOrder > Right.DiscoveryOrder;
+ });
+ if (Best == Groups->end() || Best->Benefit <= 0)
+ break;
+ auto Outlined = outlineGroup(*Best, Unit, *TargetInfo, Target);
+ if (!Outlined)
+ return std::unexpected(Outlined.error());
+ Changed = true;
+ }
+ return CompilationUnitPassResult{
+ .Changed = Changed,
+ .Changes = Changed ? MachinePassChanges::all() : MachinePassChanges{},
+ .PreservedUnitAnalyses = PreservedAnalyses::none(),
+ .PreservedFunctionAnalyses = PreservedAnalyses::none(),
+ };
+ }
+
+private:
+ std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo;
+ MachineOutlinerOptions Options;
+};
+
+} // namespace
+
+MachineInstr &MachineOutlinerCandidate::front() const noexcept {
+ return *Instructions.front();
+}
+
+MachineInstr &MachineOutlinerCandidate::back() const noexcept {
+ return *Instructions.back();
+}
+
+bool MachineOutlinerCandidate::containsRegister(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept {
+ for (const MachineInstr *Instruction : Instructions) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto RegisterValue = Operand.registerValue();
+ const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt;
+ if (Physical && !Operand.isDebugUse() && overlaps(*Physical, Reg, Target))
+ return true;
+ }
+ }
+ return false;
+}
+
+bool MachineOutlinerCandidate::isRegisterLiveAfter(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept {
+ bool Live = false;
+ for (const MachineSuccessor &Successor : Block->successors()) {
+ if (Successor.Block == nullptr)
+ continue;
+ for (const MachineBasicBlockLiveIn &LiveIn : Successor.Block->liveIns()) {
+ if (overlaps(LiveIn.PhysReg, Reg, Target))
+ Live = true;
+ }
+ }
+ for (const MachineInstr *Instruction = Block->lastInstruction();
+ Instruction != nullptr && Instruction != &back(); Instruction = Instruction->previous()) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto RegisterValue = Operand.registerValue();
+ const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt;
+ if (Physical && Operand.isDef() && overlaps(*Physical, Reg, Target))
+ Live = false;
+ }
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto RegisterValue = Operand.registerValue();
+ const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt;
+ if (Physical && Operand.isUse() && !Operand.isDebugUse() && overlaps(*Physical, Reg, Target))
+ Live = true;
+ }
+ }
+ return Live;
+}
+
+bool MachineOutlinerCandidate::isRegisterAvailable(PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept {
+ return !containsRegister(Reg, Target) && !isRegisterLiveAfter(Reg, Target) &&
+ !Function->registerInfo().isReserved(Reg) && !Target.registers().isReservedRegister(Reg);
+}
+
+TargetMachineOutlinerInfo::~TargetMachineOutlinerInfo() = default;
+
+std::unique_ptr<CompilationUnitPass>
+createMachineOutlinerPass(std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo,
+ MachineOutlinerOptions Options) {
+ return std::make_unique<MachineOutlinerPass>(std::move(TargetInfo), std::move(Options));
+}
+
+} // namespace rcc
diff --git a/backend/lib/MIR/EmbeddedIR.cpp b/backend/lib/MIR/EmbeddedIR.cpp
index c727539..d936d66 100644
--- a/backend/lib/MIR/EmbeddedIR.cpp
+++ b/backend/lib/MIR/EmbeddedIR.cpp
@@ -67,6 +67,18 @@ struct ParsedIdentifier final {
return WordEnd == End || !isWordBody(Line[WordEnd]);
}
+[[nodiscard]] bool containsWord(std::string_view Line, std::size_t End,
+ std::string_view Word) noexcept {
+ std::size_t Position = 0;
+ while ((Position = Line.find(Word, Position)) != std::string_view::npos && Position < End) {
+ const bool BeginsWord = Position == 0 || !isWordBody(Line[Position - 1]);
+ if (BeginsWord && startsWord(Line, Position, End, Word))
+ return true;
+ Position += Word.size();
+ }
+ return false;
+}
+
} // namespace
const EmbeddedIRReference *EmbeddedIRFunction::block(std::string_view QueryName) const noexcept {
@@ -459,6 +471,11 @@ private:
auto FunctionIndex = addFunction(*Name, IsDefinition);
if (!FunctionIndex || Failed || !IsDefinition)
return;
+ EmbeddedIRFunction &Function = Index.Functions[*FunctionIndex];
+ Function.HasLinkOnceODRLinkage = containsWord(Line, End, "linkonce_odr");
+ Function.HasSection = containsWord(Line, End, "section");
+ Function.HasComdat = containsWord(Line, End, "comdat");
+ Function.HasSectionPrefix = containsWord(Line, End, "section_prefix");
ActiveFunction = FunctionIndex;
HeaderParenthesisDepth = 0;
parseFunctionHeaderRemainder(Line, LineOffset, Name->End, End);
diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt
index 97e6764..92a77a5 100644
--- a/backend/lib/Target/RISCV/CMakeLists.txt
+++ b/backend/lib/Target/RISCV/CMakeLists.txt
@@ -125,6 +125,7 @@ add_library(
RISCVMergeBaseOffset.cpp
RISCVMoveMerger.cpp
RISCVOptimizationPipeline.cpp
+ RISCVOutliner.cpp
RISCVPipeline.cpp
RISCVPseudoExpansion.cpp
RISCVRegisterAllocation.cpp
diff --git a/backend/lib/Target/RISCV/RISCVOutliner.cpp b/backend/lib/Target/RISCV/RISCVOutliner.cpp
new file mode 100644
index 0000000..a3b7641
--- /dev/null
+++ b/backend/lib/Target/RISCV/RISCVOutliner.cpp
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineOutliner.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+#include "rcc/Target/RISCV/RISCVPipeline.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::uint32_t CallTargetFlag = 1;
+constexpr std::uint32_t PCRelativeLowTargetFlag = 4;
+
+enum class OutlinerConstruction : std::uint32_t {
+ TailCall,
+ Default,
+ RegisterSave,
+};
+
+[[nodiscard]] bool isUnsafeOperand(const MachineOperand &Operand) noexcept {
+ switch (Operand.kind()) {
+ case MachineOperand::Kind::BasicBlock:
+ case MachineOperand::Kind::FrameIndex:
+ case MachineOperand::Kind::ConstantPoolIndex:
+ case MachineOperand::Kind::JumpTableIndex:
+ case MachineOperand::Kind::BlockAddress:
+ case MachineOperand::Kind::TargetIndex:
+ return true;
+ case MachineOperand::Kind::Register: {
+ const auto Reg = Operand.registerValue();
+ return Reg && Reg->isVirtual();
+ }
+ case MachineOperand::Kind::Immediate:
+ case MachineOperand::Kind::IntegerImmediate:
+ case MachineOperand::Kind::FloatingImmediate:
+ case MachineOperand::Kind::GlobalSymbol:
+ case MachineOperand::Kind::ExternalSymbol:
+ case MachineOperand::Kind::RegisterMask:
+ case MachineOperand::Kind::LiveOutRegisterMask:
+ case MachineOperand::Kind::Metadata:
+ case MachineOperand::Kind::MCSymbol:
+ case MachineOperand::Kind::CFIIndex:
+ case MachineOperand::Kind::Intrinsic:
+ case MachineOperand::Kind::Predicate:
+ case MachineOperand::Kind::ShuffleMask:
+ case MachineOperand::Kind::DebugInstrRef:
+ case MachineOperand::Kind::LaneMask:
+ return false;
+ case MachineOperand::Kind::Count:
+ return true;
+ }
+ return true;
+}
+
+[[nodiscard]] std::vector<PhysicalRegister>
+sortedRegisters(std::span<const PhysicalRegister> Registers) {
+ std::vector<PhysicalRegister> Result(Registers.begin(), Registers.end());
+ std::ranges::sort(Result, {}, &PhysicalRegister::number);
+ Result.erase(std::unique(Result.begin(), Result.end()), Result.end());
+ return Result;
+}
+
+class RISCVOutlinerInfo final : public TargetMachineOutlinerInfo {
+public:
+ RISCVOutlinerInfo(bool FunctionSections, bool EnableRegisterSave,
+ std::array<PhysicalRegister, 9> Registers)
+ : FunctionSections(FunctionSections), EnableRegisterSave(EnableRegisterSave),
+ X0(Registers[0]), X2(Registers[1]), X5(Registers[2]), X6(Registers[3]),
+ SaveRegisters{Registers[3], Registers[4], Registers[5],
+ Registers[6], Registers[7], Registers[8]} {}
+
+ [[nodiscard]] bool isFunctionSafeToOutlineFrom(
+ const MachineFunction &,
+ const MachineOutlinerFunctionProperties &Properties) const noexcept override {
+ return !Properties.LinkOnceODR && !Properties.HasSection;
+ }
+
+ [[nodiscard]] MachineOutlinerInstrType
+ getOutliningType(const MachineInstr &Instruction,
+ const MachineOutlinerFunctionProperties &Properties,
+ const TargetDescription &Target) const noexcept override {
+ if (Instruction.isDebugInstr() || Instruction.opcode() == opcode(GenericOpcode::EH_LABEL) ||
+ Instruction.opcode() == opcode(GenericOpcode::GC_LABEL) ||
+ Instruction.opcode() == opcode(GenericOpcode::ANNOTATION_LABEL)) {
+ return MachineOutlinerInstrType::Invisible;
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::CFI_INSTRUCTION))
+ return MachineOutlinerInstrType::Legal;
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+ if (Descriptor == nullptr)
+ return MachineOutlinerInstrType::Illegal;
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ if (isUnsafeOperand(Operand))
+ return MachineOutlinerInstrType::Illegal;
+ if ((Operand.targetFlags() & PCRelativeLowTargetFlag) != 0 &&
+ (FunctionSections || Properties.HasComdat || Properties.HasSection ||
+ Properties.HasSectionPrefix)) {
+ return MachineOutlinerInstrType::Illegal;
+ }
+ }
+ if (Descriptor->Flags.has(InstructionFlag::Return))
+ return MachineOutlinerInstrType::LegalTerminator;
+ if (Descriptor->Flags.has(InstructionFlag::Terminator) ||
+ Descriptor->Flags.has(InstructionFlag::Branch) ||
+ Descriptor->Flags.has(InstructionFlag::Call) ||
+ Descriptor->Flags.has(InstructionFlag::NotDuplicable) ||
+ Descriptor->Flags.has(InstructionFlag::Convergent) ||
+ Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects)) {
+ return MachineOutlinerInstrType::Illegal;
+ }
+ return MachineOutlinerInstrType::Legal;
+ }
+
+ [[nodiscard]] Expected<bool> analyzeCandidate(MachineOutlinerCandidate &Candidate,
+ const TargetDescription &Target) const override {
+ std::size_t CandidateCFICount = 0;
+ for (const MachineInstr *Instruction : Candidate.Instructions) {
+ if (Instruction->opcode() == opcode(GenericOpcode::CFI_INSTRUCTION))
+ ++CandidateCFICount;
+ }
+
+ const InstructionDescriptor *LastDescriptor =
+ Target.instructionDescriptor(Candidate.back().opcode());
+ const bool IsTail =
+ LastDescriptor != nullptr && LastDescriptor->Flags.has(InstructionFlag::Return);
+ if (CandidateCFICount != 0 &&
+ (!IsTail || CandidateCFICount != Candidate.Function->cfiInfo().instructions().size())) {
+ return false;
+ }
+ if (IsTail) {
+ if (!Candidate.isRegisterAvailable(X6, Target))
+ return false;
+ Candidate.CallConstructionID = static_cast<std::uint32_t>(OutlinerConstruction::TailCall);
+ Candidate.CallOverhead = 8;
+ return true;
+ }
+
+ if (!Candidate.isRegisterAvailable(X5, Target)) {
+ if (Candidate.containsRegister(X5, Target) || !EnableRegisterSave)
+ return false;
+ for (const PhysicalRegister Reg : SaveRegisters) {
+ if (Candidate.isRegisterAvailable(Reg, Target)) {
+ Candidate.SaveRegister = Reg;
+ Candidate.CallConstructionID =
+ static_cast<std::uint32_t>(OutlinerConstruction::RegisterSave);
+ Candidate.CallOverhead = 16;
+ return true;
+ }
+ }
+ return false;
+ }
+ Candidate.CallConstructionID = static_cast<std::uint32_t>(OutlinerConstruction::Default);
+ Candidate.CallOverhead = 8;
+ return true;
+ }
+
+ [[nodiscard]] Expected<std::uint32_t>
+ instructionSize(const MachineInstr &Instruction, const MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ return instrInfo().instructionSize(Instruction, Function, Target);
+ }
+
+ [[nodiscard]] std::uint32_t
+ frameOverhead(const MachineOutlinerCandidate &Candidate) const noexcept override {
+ return Candidate.CallConstructionID ==
+ static_cast<std::uint32_t>(OutlinerConstruction::TailCall)
+ ? 0
+ : 4;
+ }
+
+ [[nodiscard]] Expected<void> insertOutlinedCall(MachineOutlinerCandidate &Candidate,
+ std::string OutlinedFunctionName,
+ const TargetDescription &Target) const override {
+ MachineIRBuilder Builder(*Candidate.Function, Target);
+ auto InsertPoint = Builder.setInsertPointBefore(Candidate.front());
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+ const auto Construction = static_cast<OutlinerConstruction>(Candidate.CallConstructionID);
+ const Register StackPointer = Register::fromPhysical(X2);
+ const Register Link = Register::fromPhysical(X5);
+
+ if (Construction == OutlinerConstruction::TailCall) {
+ auto Pending = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoTAIL));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ if (auto Added = Pending->addGlobalSymbol(std::move(OutlinedFunctionName), 0, CallTargetFlag);
+ !Added) {
+ return std::unexpected(Added.error());
+ }
+ if (auto Added = Pending->addImplicitUse(StackPointer); !Added)
+ return std::unexpected(Added.error());
+ for (const PhysicalRegister Reg : Candidate.DefinedRegisters) {
+ if (auto Added = Pending->addImplicitDef(Register::fromPhysical(Reg)); !Added)
+ return std::unexpected(Added.error());
+ }
+ if (auto Added = Pending->addImplicitUse(StackPointer); !Added)
+ return std::unexpected(Added.error());
+ for (const PhysicalRegister Reg : sortedRegisters(Candidate.LiveIns)) {
+ if (auto Added = Pending->addImplicitUse(Register::fromPhysical(Reg)); !Added)
+ return std::unexpected(Added.error());
+ }
+ auto Built = Pending->build();
+ return Built ? Expected<void>{} : std::unexpected(Built.error());
+ }
+
+ if (Construction == OutlinerConstruction::RegisterSave) {
+ auto Save = Builder.buildInstr(instructionOpcode(InstructionOpcode::ADDI));
+ if (!Save)
+ return std::unexpected(Save.error());
+ if (auto Added = Save->addDef(Register::fromPhysical(Candidate.SaveRegister)); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Save->addUse(Link); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Save->addImmediate(0); !Added)
+ return std::unexpected(Added.error());
+ auto Built = Save->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ }
+
+ auto Call = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoCALLReg));
+ if (!Call)
+ return std::unexpected(Call.error());
+ if (auto Added = Call->addDef(Link); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Call->addGlobalSymbol(std::move(OutlinedFunctionName), 0, CallTargetFlag);
+ !Added) {
+ return std::unexpected(Added.error());
+ }
+ if (auto Added = Call->addImplicitDef(Link); !Added)
+ return std::unexpected(Added.error());
+ for (const PhysicalRegister Reg : Candidate.DefinedRegisters) {
+ if (Reg == X5)
+ continue;
+ if (auto Added = Call->addImplicitDef(Register::fromPhysical(Reg)); !Added)
+ return std::unexpected(Added.error());
+ }
+ std::vector<PhysicalRegister> Uses = Candidate.LiveIns;
+ if (Construction == OutlinerConstruction::RegisterSave)
+ Uses.push_back(Candidate.SaveRegister);
+ for (const PhysicalRegister Reg : sortedRegisters(Uses)) {
+ if (auto Added = Call->addImplicitUse(Register::fromPhysical(Reg)); !Added)
+ return std::unexpected(Added.error());
+ }
+ auto BuiltCall = Call->build();
+ if (!BuiltCall)
+ return std::unexpected(BuiltCall.error());
+
+ if (Construction == OutlinerConstruction::RegisterSave) {
+ auto Restore = Builder.buildInstr(instructionOpcode(InstructionOpcode::ADDI));
+ if (!Restore)
+ return std::unexpected(Restore.error());
+ if (auto Added = Restore->addDef(Link); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Restore->addUse(Register::fromPhysical(Candidate.SaveRegister)); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Restore->addImmediate(0); !Added)
+ return std::unexpected(Added.error());
+ auto Built = Restore->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void> buildOutlinedFrame(MachineFunction &Function,
+ MachineBasicBlock &Block,
+ const MachineOutlinerCandidate &Candidate,
+ const TargetDescription &Target) const override {
+ if (Candidate.CallConstructionID ==
+ static_cast<std::uint32_t>(OutlinerConstruction::TailCall)) {
+ return {};
+ }
+ auto AddedLiveIn = Block.addLiveIn(X5);
+ if (!AddedLiveIn)
+ return std::unexpected(AddedLiveIn.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto InsertPoint = Builder.setInsertPoint(Block);
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+ auto Return = Builder.buildInstr(instructionOpcode(InstructionOpcode::JALR));
+ if (!Return)
+ return std::unexpected(Return.error());
+ if (auto Added = Return->addDef(Register::fromPhysical(X0)); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Return->addUse(Register::fromPhysical(X5)); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Return->addImmediate(0); !Added)
+ return std::unexpected(Added.error());
+ auto Built = Return->build();
+ return Built ? Expected<void>{} : std::unexpected(Built.error());
+ }
+
+private:
+ bool FunctionSections = false;
+ bool EnableRegisterSave = false;
+ PhysicalRegister X0;
+ PhysicalRegister X2;
+ PhysicalRegister X5;
+ PhysicalRegister X6;
+ std::array<PhysicalRegister, 6> SaveRegisters;
+};
+
+} // namespace
+
+Expected<std::unique_ptr<TargetMachineOutlinerInfo>>
+createMachineOutlinerInfo(bool FunctionSections, bool EnableRegisterSave) {
+ constexpr std::array IDs{
+ PhysicalRegisterID::X0, PhysicalRegisterID::X2, PhysicalRegisterID::X5,
+ PhysicalRegisterID::X6, PhysicalRegisterID::X7, PhysicalRegisterID::X28,
+ PhysicalRegisterID::X29, PhysicalRegisterID::X30, PhysicalRegisterID::X31,
+ };
+ std::array<PhysicalRegister, IDs.size()> Registers;
+ for (std::size_t Index = 0; Index < IDs.size(); ++Index) {
+ auto Reg = physicalRegister(IDs[Index]);
+ if (!Reg)
+ return std::unexpected(Reg.error());
+ Registers[Index] = *Reg;
+ }
+ return std::make_unique<RISCVOutlinerInfo>(FunctionSections, EnableRegisterSave, Registers);
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Tooling/PipelineDriver.cpp b/backend/lib/Tooling/PipelineDriver.cpp
index ebba1b6..b247326 100644
--- a/backend/lib/Tooling/PipelineDriver.cpp
+++ b/backend/lib/Tooling/PipelineDriver.cpp
@@ -6,6 +6,7 @@
#include "rcc/CodeGen/MIRPrinter.h"
#include "rcc/CodeGen/MachineFunction.h"
#include "rcc/CodeGen/MachineOptimizationPipeline.h"
+#include "rcc/CodeGen/MachineOutliner.h"
#include "rcc/CodeGen/MachineStripDebug.h"
#include "rcc/CodeGen/MachineVerifier.h"
#include "rcc/Target/RISCV/RISCVPipeline.h"
@@ -98,6 +99,12 @@ struct FunctionRunResult final {
return false;
}
+[[nodiscard]] bool booleanTargetOption(const BackendInvocation &Invocation,
+ std::string_view Name) noexcept {
+ const auto Option = std::ranges::find(Invocation.TargetOptions, Name, &BackendOption::Name);
+ return Option != Invocation.TargetOptions.end() && Option->Value == "true";
+}
+
[[nodiscard]] std::optional<BackendStage> stageAfterPass(std::string_view Pass) noexcept {
if (Pass == "legalizer")
return BackendStage::LegalizedMIR;
@@ -175,9 +182,11 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio
Comma == std::string_view::npos ? Selector : Selector.substr(0, Comma);
if (Name.empty())
return makeUnexpected(ErrorCode::InvalidInvocation, "empty pipeline pass selector");
- auto Pass = riscv::createCanonicalPipelinePass(Name, Subtarget);
- if (!Pass)
- return std::unexpected(Pass.error());
+ if (Name != "machine-outliner") {
+ auto Pass = riscv::createCanonicalPipelinePass(Name, Subtarget);
+ if (!Pass)
+ return std::unexpected(Pass.error());
+ }
if (Comma != std::string_view::npos) {
const std::string_view InstanceText = Selector.substr(Comma + 1U);
std::uint32_t Instance = 0;
@@ -383,6 +392,47 @@ runFunction(MachineFunction &Function, mir::MIRDocument &Document, const TargetD
!Controls.PrintAfter.empty();
}
+[[nodiscard]] Expected<PipelineRunResult>
+runMachineOutliner(mir::MIRDocument &Document, const TargetDescription &Target,
+ const BackendInvocation &Invocation, const PipelineControls &Controls,
+ MachinePassInstrumentation *Instrumentation) {
+ auto TargetInfo =
+ riscv::createMachineOutlinerInfo(booleanTargetOption(Invocation, "function-sections"),
+ booleanTargetOption(Invocation, "riscv-outliner-regsave"));
+ if (!TargetInfo)
+ return std::unexpected(TargetInfo.error());
+
+ MachineOutlinerOptions Options;
+ if (const mir::EmbeddedIRReferenceIndex *Embedded = Document.embeddedIR()) {
+ for (const mir::EmbeddedIRFunction &IRFunction : Embedded->functions()) {
+ Options.FunctionProperties.emplace(std::string(IRFunction.name()),
+ MachineOutlinerFunctionProperties{
+ .LinkOnceODR = IRFunction.hasLinkOnceODRLinkage(),
+ .HasSection = IRFunction.hasSection(),
+ .HasComdat = IRFunction.hasComdat(),
+ .HasSectionPrefix = IRFunction.hasSectionPrefix(),
+ });
+ }
+ }
+
+ CompilationUnitPassManager Pipeline({.VerifyBefore = true, .VerifyEach = Controls.VerifyEach});
+ auto Added =
+ Pipeline.addPass(createMachineOutlinerPass(std::move(*TargetInfo), std::move(Options)));
+ if (!Added)
+ return std::unexpected(Added.error());
+ CompilationUnitAnalysisManager UnitAnalyses;
+ MachineFunctionAnalysisManager FunctionAnalyses;
+ auto Registered = registerMachineOptimizationAnalyses(FunctionAnalyses);
+ if (!Registered)
+ return std::unexpected(Registered.error());
+ auto Run = Pipeline.run(Document.unit(), Target, UnitAnalyses, FunctionAnalyses, Instrumentation);
+ if (!Run)
+ return std::unexpected(Run.error());
+ PipelineRunResult Result;
+ Result.ExecutedPasses.push_back({.Function = {}, .Pass = "machine-outliner", .Occurrence = 1});
+ return Result;
+}
+
} // namespace
bool isEarlierStage(BackendStage Left, BackendStage Right) noexcept {
@@ -516,6 +566,17 @@ Expected<PipelineRunResult> runRV64GCPipeline(mir::MIRDocument &Document,
});
}
+ const bool ContainsMachineOutliner =
+ std::ranges::find(Invocation.PassPipeline, "machine-outliner") !=
+ Invocation.PassPipeline.end();
+ if (ContainsMachineOutliner) {
+ if (Invocation.PassPipeline.size() != 1) {
+ return makeUnexpected(ErrorCode::InvalidInvocation,
+ "machine-outliner cannot be mixed with per-function passes");
+ }
+ return runMachineOutliner(Document, Target, Invocation, Controls, ActiveInstrumentation);
+ }
+
const std::size_t FunctionCount = Document.unit().functionCount();
std::vector<FunctionRunResult> FunctionResults(FunctionCount);
const bool Sequential = requiresSequentialExecution(Controls) ||