diff options
70 files changed, 7008 insertions, 194 deletions
diff --git a/backend/include/rcc/CodeGen/GenericConstant.h b/backend/include/rcc/CodeGen/GenericConstant.h index 11c5068..92a8f13 100644 --- a/backend/include/rcc/CodeGen/GenericConstant.h +++ b/backend/include/rcc/CodeGen/GenericConstant.h @@ -50,8 +50,8 @@ struct GenericConstantType final { [[nodiscard]] static GenericConstantType byte() noexcept; [[nodiscard]] static Expected<GenericConstantType> floating(FloatSemantics Semantics); [[nodiscard]] static GenericConstantType pointer(std::uint32_t AddressSpace = 0) noexcept; - [[nodiscard]] static Expected<GenericConstantType> - array(std::uint64_t ElementCount, GenericConstantType Element); + [[nodiscard]] static Expected<GenericConstantType> array(std::uint64_t ElementCount, + GenericConstantType Element); [[nodiscard]] static Expected<GenericConstantType> vector(std::uint64_t ElementCount, GenericConstantType Element, bool Scalable = false); [[nodiscard]] static Expected<GenericConstantType> @@ -152,8 +152,7 @@ struct GenericConstantData final { std::string Block; bool NumericSymbol = false; bool NumericBlock = false; - GenericConstantExpressionOpcode ExpressionOpcode = - GenericConstantExpressionOpcode::Invalid; + GenericConstantExpressionOpcode ExpressionOpcode = GenericConstantExpressionOpcode::Invalid; GenericConstantExpressionFlags ExpressionFlags; std::optional<GenericConstantType> SourceElementType; std::optional<IntegerImmediate> InRangeStart; diff --git a/backend/include/rcc/CodeGen/LiveRangeEdit.h b/backend/include/rcc/CodeGen/LiveRangeEdit.h new file mode 100644 index 0000000..66696df --- /dev/null +++ b/backend/include/rcc/CodeGen/LiveRangeEdit.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_LIVERANGEEDIT_H +#define RCC_CODEGEN_LIVERANGEEDIT_H + +#include "rcc/CodeGen/VirtRegMap.h" + +#include <optional> + +namespace rcc { + +class TargetDescription; + +class LiveRangeEdit final { +public: + LiveRangeEdit(MachineFunction &Function, const TargetDescription &Target, VirtRegMap &Mapping) + : Function(&Function), Target(&Target), Mapping(&Mapping) {} + + [[nodiscard]] Expected<Register> createFrom(Register Parent, bool Spillable, + std::optional<Register> Origin = std::nullopt); + [[nodiscard]] Expected<FrameIndex> getOrCreateSpillSlot(Register VirtualReg); + [[nodiscard]] Expected<void> eraseVirtualRegister(Register VirtualReg); + +private: + MachineFunction *Function; + const TargetDescription *Target; + VirtRegMap *Mapping; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_LIVERANGEEDIT_H diff --git a/backend/include/rcc/CodeGen/LiveRegMatrix.h b/backend/include/rcc/CodeGen/LiveRegMatrix.h new file mode 100644 index 0000000..953828c --- /dev/null +++ b/backend/include/rcc/CodeGen/LiveRegMatrix.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_LIVEREGMATRIX_H +#define RCC_CODEGEN_LIVEREGMATRIX_H + +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <cstdint> +#include <optional> +#include <span> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace rcc { + +class MachineFunction; +class TargetDescription; + +enum class RegisterInterferenceKind : std::uint8_t { + Free, + VirtualRegister, + FixedRegister, + RegisterMask, + RegisterUnit, +}; + +struct RegisterInterference final { + RegisterInterferenceKind Kind = RegisterInterferenceKind::Free; + std::optional<Register> VirtualReg; + std::optional<PhysicalRegister> FixedReg; + std::optional<RegisterUnitID> Unit; + std::optional<MachineInstrStableID> Instruction; +}; + +class LiveRegMatrix final { +public: + LiveRegMatrix(const MachineFunction &Function, const TargetDescription &Target, + const LiveIntervalsAnalysis::Result &Intervals) noexcept + : Function(&Function), Target(&Target), Intervals(&Intervals) {} + + [[nodiscard]] Expected<void> assign(Register VirtualReg, PhysicalRegister PhysicalReg); + [[nodiscard]] Expected<void> unassign(Register VirtualReg); + [[nodiscard]] std::optional<PhysicalRegister> assignment(Register VirtualReg) const noexcept; + [[nodiscard]] RegisterInterference checkInterference(Register VirtualReg, + PhysicalRegister PhysicalReg) const noexcept; + [[nodiscard]] std::vector<RegisterInterference> + collectInterferences(Register VirtualReg, PhysicalRegister PhysicalReg) const; + + [[nodiscard]] std::span<const std::pair<Register, PhysicalRegister>> + assignments() const noexcept { + return OrderedAssignments; + } + +private: + [[nodiscard]] bool intervalsOverlapOnCandidates(const LiveInterval &Left, + PhysicalRegister LeftReg, + const LiveInterval &Right, + PhysicalRegister RightReg) const noexcept; + [[nodiscard]] bool intervalOverlapsUnit(const LiveInterval &Interval, RegisterUnitLane Unit, + const LiveRange &UnitRange) const noexcept; + + const MachineFunction *Function; + const TargetDescription *Target; + const LiveIntervalsAnalysis::Result *Intervals; + std::unordered_map<Register, PhysicalRegister> Assignments; + std::vector<std::pair<Register, PhysicalRegister>> OrderedAssignments; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_LIVEREGMATRIX_H diff --git a/backend/include/rcc/CodeGen/MachineFrameInfo.h b/backend/include/rcc/CodeGen/MachineFrameInfo.h index 48e3d7a..c884afb 100644 --- a/backend/include/rcc/CodeGen/MachineFrameInfo.h +++ b/backend/include/rcc/CodeGen/MachineFrameInfo.h @@ -197,8 +197,7 @@ struct MachineFrameState final { std::vector<MachineFrameSaveRestorePoint> SavePoints; std::vector<MachineFrameSaveRestorePoint> RestorePoints; - friend bool operator==(const MachineFrameState &Left, - const MachineFrameState &Right) = default; + friend bool operator==(const MachineFrameState &Left, const MachineFrameState &Right) = default; }; class MachineFrameInfo final { diff --git a/backend/include/rcc/CodeGen/MachineRegisterInfo.h b/backend/include/rcc/CodeGen/MachineRegisterInfo.h index b6485e3..14e50a3 100644 --- a/backend/include/rcc/CodeGen/MachineRegisterInfo.h +++ b/backend/include/rcc/CodeGen/MachineRegisterInfo.h @@ -134,6 +134,7 @@ public: [[nodiscard]] std::span<const PhysicalRegisterLiveIn> liveIns() const noexcept { return LiveIns; } + void clearLiveInVirtualRegisters() noexcept; [[nodiscard]] Expected<void> reservePhysicalRegister(PhysicalRegister PhysReg); [[nodiscard]] bool isReserved(PhysicalRegister PhysReg) const noexcept; diff --git a/backend/include/rcc/CodeGen/PBQP/Graph.h b/backend/include/rcc/CodeGen/PBQP/Graph.h new file mode 100644 index 0000000..9173bfd --- /dev/null +++ b/backend/include/rcc/CodeGen/PBQP/Graph.h @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_PBQP_GRAPH_H +#define RCC_CODEGEN_PBQP_GRAPH_H + +#include "rcc/CodeGen/PBQP/Math.h" +#include "rcc/Support/Error.h" + +#include <compare> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <optional> +#include <span> +#include <utility> +#include <vector> + +namespace rcc::pbqp { + +class NodeID final { +public: + constexpr NodeID() noexcept = default; + [[nodiscard]] static constexpr NodeID fromValue(std::uint32_t Value) noexcept { + return NodeID(Value); + } + [[nodiscard]] constexpr bool isValid() const noexcept { + return Value != InvalidValue; + } + [[nodiscard]] constexpr std::uint32_t value() const noexcept { + return Value; + } + friend constexpr bool operator==(NodeID, NodeID) noexcept = default; + friend constexpr auto operator<=>(NodeID, NodeID) noexcept = default; + +private: + static constexpr std::uint32_t InvalidValue = std::numeric_limits<std::uint32_t>::max(); + explicit constexpr NodeID(std::uint32_t Value) noexcept : Value(Value) {} + std::uint32_t Value = InvalidValue; +}; + +class EdgeID final { +public: + constexpr EdgeID() noexcept = default; + [[nodiscard]] static constexpr EdgeID fromValue(std::uint32_t Value) noexcept { + return EdgeID(Value); + } + [[nodiscard]] constexpr bool isValid() const noexcept { + return Value != InvalidValue; + } + [[nodiscard]] constexpr std::uint32_t value() const noexcept { + return Value; + } + friend constexpr bool operator==(EdgeID, EdgeID) noexcept = default; + friend constexpr auto operator<=>(EdgeID, EdgeID) noexcept = default; + +private: + static constexpr std::uint32_t InvalidValue = std::numeric_limits<std::uint32_t>::max(); + explicit constexpr EdgeID(std::uint32_t Value) noexcept : Value(Value) {} + std::uint32_t Value = InvalidValue; +}; + +struct Node final { + NodeID ID; + std::uint64_t StableKey = 0; + CostVector Costs; + std::vector<EdgeID> AdjacentEdges; +}; + +struct Edge final { + EdgeID ID; + NodeID First; + NodeID Second; + std::uint64_t StableKey = 0; + CostMatrix Costs; +}; + +class Graph final { +public: + [[nodiscard]] Expected<NodeID> addNode(CostVector Costs, std::uint64_t StableKey = 0) { + if (Costs.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "PBQP node requires at least one option"); + if (Nodes.size() >= std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::Unsupported, "PBQP node ID space is exhausted"); + const NodeID ID = NodeID::fromValue(static_cast<std::uint32_t>(Nodes.size())); + Nodes.push_back(Node{ID, StableKey, std::move(Costs), {}}); + return ID; + } + + [[nodiscard]] Expected<EdgeID> addEdge(NodeID First, NodeID Second, CostMatrix Costs, + std::uint64_t StableKey = 0) { + const Node *FirstNode = node(First); + const Node *SecondNode = node(Second); + if (FirstNode == nullptr || SecondNode == nullptr || First == Second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "PBQP edge requires two distinct known nodes"); + } + if (edgeBetween(First, Second)) + return makeUnexpected(ErrorCode::InvalidArgument, "PBQP graph already contains this edge"); + + if (Second < First) { + std::swap(First, Second); + Costs = Costs.transpose(); + std::swap(FirstNode, SecondNode); + } + if (Costs.rows() != FirstNode->Costs.size() || Costs.columns() != SecondNode->Costs.size()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "PBQP edge matrix dimensions do not match its nodes"); + } + if (Edges.size() >= std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::Unsupported, "PBQP edge ID space is exhausted"); + const EdgeID ID = EdgeID::fromValue(static_cast<std::uint32_t>(Edges.size())); + Edges.push_back(Edge{ID, First, Second, StableKey, std::move(Costs)}); + Nodes[First.value()].AdjacentEdges.push_back(ID); + Nodes[Second.value()].AdjacentEdges.push_back(ID); + return ID; + } + + [[nodiscard]] const Node *node(NodeID ID) const noexcept { + return ID.isValid() && ID.value() < Nodes.size() ? &Nodes[ID.value()] : nullptr; + } + [[nodiscard]] const Edge *edge(EdgeID ID) const noexcept { + return ID.isValid() && ID.value() < Edges.size() ? &Edges[ID.value()] : nullptr; + } + + [[nodiscard]] const Edge *edgeBetween(NodeID Left, NodeID Right) const noexcept { + const Node *LeftNode = node(Left); + if (LeftNode == nullptr || node(Right) == nullptr) + return nullptr; + for (const EdgeID ID : LeftNode->AdjacentEdges) { + const Edge &Candidate = Edges[ID.value()]; + if ((Candidate.First == Left && Candidate.Second == Right) || + (Candidate.First == Right && Candidate.Second == Left)) + return &Candidate; + } + return nullptr; + } + + [[nodiscard]] std::span<const Node> nodes() const noexcept { + return Nodes; + } + [[nodiscard]] std::span<const Edge> edges() const noexcept { + return Edges; + } + [[nodiscard]] std::size_t nodeCount() const noexcept { + return Nodes.size(); + } + + [[nodiscard]] Expected<void> verify() const { + for (std::size_t Index = 0; Index < Nodes.size(); ++Index) { + const Node &Value = Nodes[Index]; + if (Value.ID.value() != Index || Value.Costs.empty()) + return makeUnexpected(ErrorCode::VerificationError, "PBQP graph contains an invalid node"); + for (const EdgeID ID : Value.AdjacentEdges) { + const Edge *Adjacent = edge(ID); + if (Adjacent == nullptr || (Adjacent->First != Value.ID && Adjacent->Second != Value.ID)) { + return makeUnexpected(ErrorCode::VerificationError, + "PBQP node has an invalid adjacency entry"); + } + } + } + for (std::size_t Index = 0; Index < Edges.size(); ++Index) { + const Edge &Value = Edges[Index]; + const Node *First = node(Value.First); + const Node *Second = node(Value.Second); + if (Value.ID.value() != Index || First == nullptr || Second == nullptr || + !(Value.First < Value.Second) || Value.Costs.rows() != First->Costs.size() || + Value.Costs.columns() != Second->Costs.size()) { + return makeUnexpected(ErrorCode::VerificationError, "PBQP graph contains an invalid edge"); + } + } + return {}; + } + +private: + std::vector<Node> Nodes; + std::vector<Edge> Edges; +}; + +} // namespace rcc::pbqp + +template <> struct std::hash<rcc::pbqp::NodeID> { + [[nodiscard]] std::size_t operator()(rcc::pbqp::NodeID ID) const noexcept { + return std::hash<std::uint32_t>{}(ID.value()); + } +}; + +template <> struct std::hash<rcc::pbqp::EdgeID> { + [[nodiscard]] std::size_t operator()(rcc::pbqp::EdgeID ID) const noexcept { + return std::hash<std::uint32_t>{}(ID.value()); + } +}; + +#endif // RCC_CODEGEN_PBQP_GRAPH_H diff --git a/backend/include/rcc/CodeGen/PBQP/Math.h b/backend/include/rcc/CodeGen/PBQP/Math.h new file mode 100644 index 0000000..90fc751 --- /dev/null +++ b/backend/include/rcc/CodeGen/PBQP/Math.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_PBQP_MATH_H +#define RCC_CODEGEN_PBQP_MATH_H + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <span> +#include <utility> +#include <vector> + +namespace rcc::pbqp { + +using Cost = std::int64_t; + +inline constexpr Cost InfiniteCost = std::numeric_limits<Cost>::max() / 4; +inline constexpr Cost NegativeInfiniteCost = -InfiniteCost; + +[[nodiscard]] constexpr bool isInfinite(Cost Value) noexcept { + return Value >= InfiniteCost; +} + +[[nodiscard]] constexpr Cost addCosts(Cost Left, Cost Right) noexcept { + if (isInfinite(Left) || isInfinite(Right)) + return InfiniteCost; + if (Left <= NegativeInfiniteCost || Right <= NegativeInfiniteCost) + return NegativeInfiniteCost; + if (Right > 0 && Left > InfiniteCost - Right) + return InfiniteCost; + if (Right < 0 && Left < NegativeInfiniteCost - Right) + return NegativeInfiniteCost; + return Left + Right; +} + +class CostVector final { +public: + explicit CostVector(std::size_t Size, Cost Initial = 0) : Values(Size, Initial) {} + explicit CostVector(std::vector<Cost> Values) : Values(std::move(Values)) {} + + [[nodiscard]] std::size_t size() const noexcept { + return Values.size(); + } + [[nodiscard]] bool empty() const noexcept { + return Values.empty(); + } + [[nodiscard]] Cost &operator[](std::size_t Index) noexcept { + return Values[Index]; + } + [[nodiscard]] Cost operator[](std::size_t Index) const noexcept { + return Values[Index]; + } + [[nodiscard]] std::span<const Cost> values() const noexcept { + return Values; + } + + [[nodiscard]] std::size_t minimumIndex() const noexcept { + return static_cast<std::size_t>(std::ranges::min_element(Values) - Values.begin()); + } + + void add(const CostVector &Other) noexcept { + for (std::size_t Index = 0; Index < Values.size(); ++Index) + Values[Index] = addCosts(Values[Index], Other.Values[Index]); + } + +private: + std::vector<Cost> Values; +}; + +class CostMatrix final { +public: + CostMatrix(std::size_t Rows, std::size_t Columns, Cost Initial = 0) + : RowCount(Rows), ColumnCount(Columns), Values(Rows * Columns, Initial) {} + + [[nodiscard]] std::size_t rows() const noexcept { + return RowCount; + } + [[nodiscard]] std::size_t columns() const noexcept { + return ColumnCount; + } + [[nodiscard]] bool empty() const noexcept { + return RowCount == 0 || ColumnCount == 0; + } + + [[nodiscard]] Cost &at(std::size_t Row, std::size_t Column) noexcept { + return Values[Row * ColumnCount + Column]; + } + [[nodiscard]] Cost at(std::size_t Row, std::size_t Column) const noexcept { + return Values[Row * ColumnCount + Column]; + } + + [[nodiscard]] CostMatrix transpose() const { + CostMatrix Result(ColumnCount, RowCount); + for (std::size_t Row = 0; Row < RowCount; ++Row) + for (std::size_t Column = 0; Column < ColumnCount; ++Column) + Result.at(Column, Row) = at(Row, Column); + return Result; + } + + void add(const CostMatrix &Other) noexcept { + for (std::size_t Index = 0; Index < Values.size(); ++Index) + Values[Index] = addCosts(Values[Index], Other.Values[Index]); + } + +private: + std::size_t RowCount; + std::size_t ColumnCount; + std::vector<Cost> Values; +}; + +} // namespace rcc::pbqp + +#endif // RCC_CODEGEN_PBQP_MATH_H diff --git a/backend/include/rcc/CodeGen/PBQP/ReductionRules.h b/backend/include/rcc/CodeGen/PBQP/ReductionRules.h new file mode 100644 index 0000000..1b28e33 --- /dev/null +++ b/backend/include/rcc/CodeGen/PBQP/ReductionRules.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_PBQP_REDUCTIONRULES_H +#define RCC_CODEGEN_PBQP_REDUCTIONRULES_H + +#include "rcc/CodeGen/PBQP/Graph.h" + +#include <cstdint> + +namespace rcc::pbqp { + +enum class ReductionKind : std::uint8_t { + DegreeZero, + DegreeOne, + DegreeTwo, + Heuristic, +}; + +struct ReductionRecord final { + ReductionKind Kind = ReductionKind::DegreeZero; + NodeID Node; + NodeID FirstNeighbor; + NodeID SecondNeighbor; + std::size_t FixedSelection = 0; + CostVector NodeCosts{1}; + CostMatrix FirstCosts{1, 1}; + CostMatrix SecondCosts{1, 1}; +}; + +} // namespace rcc::pbqp + +#endif // RCC_CODEGEN_PBQP_REDUCTIONRULES_H diff --git a/backend/include/rcc/CodeGen/PBQP/Solution.h b/backend/include/rcc/CodeGen/PBQP/Solution.h new file mode 100644 index 0000000..512eb57 --- /dev/null +++ b/backend/include/rcc/CodeGen/PBQP/Solution.h @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_PBQP_SOLUTION_H +#define RCC_CODEGEN_PBQP_SOLUTION_H + +#include "rcc/CodeGen/PBQP/Graph.h" +#include "rcc/Support/Error.h" + +#include <cstddef> +#include <limits> +#include <optional> +#include <span> +#include <vector> + +namespace rcc::pbqp { + +class Solution final { +public: + explicit Solution(std::size_t NodeCount) + : Selections(NodeCount, std::numeric_limits<std::size_t>::max()) {} + + [[nodiscard]] Expected<void> setSelection(NodeID ID, std::size_t Selection) { + if (!ID.isValid() || ID.value() >= Selections.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "PBQP solution received an unknown node"); + Selections[ID.value()] = Selection; + return {}; + } + + [[nodiscard]] std::optional<std::size_t> selection(NodeID ID) const noexcept { + if (!ID.isValid() || ID.value() >= Selections.size() || + Selections[ID.value()] == std::numeric_limits<std::size_t>::max()) + return std::nullopt; + return Selections[ID.value()]; + } + + [[nodiscard]] std::span<const std::size_t> selections() const noexcept { + return Selections; + } + [[nodiscard]] Cost totalCost() const noexcept { + return TotalCost; + } + void setTotalCost(Cost Value) noexcept { + TotalCost = Value; + } + + [[nodiscard]] Expected<void> verify(const Graph &Problem) const { + if (Selections.size() != Problem.nodeCount()) + return makeUnexpected(ErrorCode::VerificationError, "PBQP solution has the wrong node count"); + Cost Total = 0; + for (const Node &NodeValue : Problem.nodes()) { + const auto Selection = selection(NodeValue.ID); + if (!Selection) + return makeUnexpected(ErrorCode::VerificationError, + "PBQP solution omits a valid node selection"); + const std::size_t SelectedOption = Selection.value(); + if (SelectedOption >= NodeValue.Costs.size()) + return makeUnexpected(ErrorCode::VerificationError, + "PBQP solution selects an invalid node option"); + Total = addCosts(Total, NodeValue.Costs[SelectedOption]); + } + for (const Edge &EdgeValue : Problem.edges()) { + const auto FirstSelection = selection(EdgeValue.First); + const auto SecondSelection = selection(EdgeValue.Second); + if (!FirstSelection || !SecondSelection) { + return makeUnexpected(ErrorCode::VerificationError, + "PBQP solution omits an edge endpoint selection"); + } + Total = addCosts(Total, EdgeValue.Costs.at(FirstSelection.value(), SecondSelection.value())); + } + if (isInfinite(Total)) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "PBQP problem has no finite solution"); + if (Total != TotalCost) + return makeUnexpected(ErrorCode::VerificationError, "PBQP solution cost is inconsistent"); + return {}; + } + +private: + std::vector<std::size_t> Selections; + Cost TotalCost = 0; +}; + +} // namespace rcc::pbqp + +#endif // RCC_CODEGEN_PBQP_SOLUTION_H diff --git a/backend/include/rcc/CodeGen/PBQP/Solver.h b/backend/include/rcc/CodeGen/PBQP/Solver.h new file mode 100644 index 0000000..ff14857 --- /dev/null +++ b/backend/include/rcc/CodeGen/PBQP/Solver.h @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_PBQP_SOLVER_H +#define RCC_CODEGEN_PBQP_SOLVER_H + +#include "rcc/CodeGen/PBQP/ReductionRules.h" +#include "rcc/CodeGen/PBQP/Solution.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <map> +#include <optional> +#include <set> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc::pbqp { + +class DeterministicSolver final { +public: + [[nodiscard]] Expected<Solution> solve(const Graph &Problem) const { + auto Verified = Problem.verify(); + if (!Verified) + return std::unexpected(Verified.error()); + + WorkingProblem Work(Problem); + std::vector<ReductionRecord> Reductions; + Reductions.reserve(Problem.nodeCount()); + while (Work.activeNodeCount() != 0) { + const NodeID Selected = Work.selectReductionNode(); + const std::size_t Degree = Work.degree(Selected); + Expected<ReductionRecord> Reduced = Degree == 0 ? Work.reduceDegreeZero(Selected) + : Degree == 1 ? Work.reduceDegreeOne(Selected) + : Degree == 2 ? Work.reduceDegreeTwo(Selected) + : Work.reduceHeuristically(Selected); + if (!Reduced) + return std::unexpected(Reduced.error()); + Reductions.push_back(std::move(*Reduced)); + } + + Solution Result(Problem.nodeCount()); + for (auto Position = Reductions.rbegin(); Position != Reductions.rend(); ++Position) { + const ReductionRecord &Record = *Position; + std::size_t Choice = Record.FixedSelection; + if (Record.Kind != ReductionKind::Heuristic) { + Cost Best = InfiniteCost; + for (std::size_t Option = 0; Option < Record.NodeCosts.size(); ++Option) { + Cost Candidate = Record.NodeCosts[Option]; + if (Record.Kind == ReductionKind::DegreeOne || Record.Kind == ReductionKind::DegreeTwo) { + const auto Neighbor = Result.selection(Record.FirstNeighbor); + if (!Neighbor) + return makeUnexpected(ErrorCode::InternalError, + "PBQP back-propagation encountered an unsolved neighbor"); + Candidate = addCosts(Candidate, Record.FirstCosts.at(Option, *Neighbor)); + } + if (Record.Kind == ReductionKind::DegreeTwo) { + const auto Neighbor = Result.selection(Record.SecondNeighbor); + if (!Neighbor) + return makeUnexpected(ErrorCode::InternalError, + "PBQP back-propagation encountered an unsolved neighbor"); + Candidate = addCosts(Candidate, Record.SecondCosts.at(Option, *Neighbor)); + } + if (Candidate < Best) { + Best = Candidate; + Choice = Option; + } + } + if (isInfinite(Best)) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "PBQP reductions produced no finite option"); + } + auto Set = Result.setSelection(Record.Node, Choice); + if (!Set) + return std::unexpected(Set.error()); + } + + Cost Total = 0; + for (const Node &NodeValue : Problem.nodes()) { + const auto Selection = Result.selection(NodeValue.ID); + if (!Selection) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver did not select a reduced node option"); + } + Total = addCosts(Total, NodeValue.Costs[Selection.value()]); + } + for (const Edge &EdgeValue : Problem.edges()) { + const auto FirstSelection = Result.selection(EdgeValue.First); + const auto SecondSelection = Result.selection(EdgeValue.Second); + if (!FirstSelection || !SecondSelection) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver did not select an edge endpoint option"); + } + Total = addCosts(Total, EdgeValue.Costs.at(FirstSelection.value(), SecondSelection.value())); + } + if (isInfinite(Total)) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "PBQP problem has no finite solution"); + Result.setTotalCost(Total); + auto ResultVerification = Result.verify(Problem); + if (!ResultVerification) + return std::unexpected(ResultVerification.error()); + return Result; + } + +private: + struct WorkingEdge final { + NodeID First; + NodeID Second; + CostMatrix Costs; + }; + + class WorkingProblem final { + public: + explicit WorkingProblem(const Graph &Problem) { + Costs.reserve(Problem.nodeCount()); + StableKeys.reserve(Problem.nodeCount()); + Active.assign(Problem.nodeCount(), true); + for (const Node &NodeValue : Problem.nodes()) { + Costs.push_back(NodeValue.Costs); + StableKeys.push_back(NodeValue.StableKey); + } + for (const Edge &EdgeValue : Problem.edges()) + Edges.emplace(key(EdgeValue.First, EdgeValue.Second), + WorkingEdge{EdgeValue.First, EdgeValue.Second, EdgeValue.Costs}); + ActiveCount = Problem.nodeCount(); + } + + [[nodiscard]] std::size_t activeNodeCount() const noexcept { + return ActiveCount; + } + + [[nodiscard]] std::size_t degree(NodeID ID) const noexcept { + std::size_t Result = 0; + for (const auto &[EdgeKey, EdgeValue] : Edges) { + (void)EdgeKey; + if (Active[EdgeValue.First.value()] && Active[EdgeValue.Second.value()] && + (EdgeValue.First == ID || EdgeValue.Second == ID)) + ++Result; + } + return Result; + } + + [[nodiscard]] NodeID selectReductionNode() const noexcept { + std::optional<std::tuple<bool, std::size_t, std::uint64_t, std::uint32_t>> Best; + NodeID Result; + for (std::size_t Index = 0; Index < Active.size(); ++Index) { + if (!Active[Index]) + continue; + const NodeID ID = NodeID::fromValue(static_cast<std::uint32_t>(Index)); + const std::size_t Degree = degree(ID); + const auto Rank = std::tuple{Degree > 2, Degree, StableKeys[Index], ID.value()}; + if (!Best || Rank < *Best) { + Best = Rank; + Result = ID; + } + } + return Result; + } + + [[nodiscard]] Expected<ReductionRecord> reduceDegreeZero(NodeID ID) { + ReductionRecord Record; + Record.Kind = ReductionKind::DegreeZero; + Record.Node = ID; + Record.NodeCosts = Costs[ID.value()]; + deactivate(ID); + return Record; + } + + [[nodiscard]] Expected<ReductionRecord> reduceDegreeOne(NodeID ID) { + const std::vector<WorkingEdge> Incident = incidentEdges(ID); + if (Incident.size() != 1) + return makeUnexpected(ErrorCode::InternalError, + "PBQP degree-one reduction is inconsistent"); + const WorkingEdge &EdgeValue = Incident.front(); + const NodeID Neighbor = other(EdgeValue, ID); + const CostMatrix Oriented = orientFrom(EdgeValue, ID); + + ReductionRecord Record; + Record.Kind = ReductionKind::DegreeOne; + Record.Node = ID; + Record.FirstNeighbor = Neighbor; + Record.NodeCosts = Costs[ID.value()]; + Record.FirstCosts = Oriented; + + for (std::size_t NeighborOption = 0; NeighborOption < Costs[Neighbor.value()].size(); + ++NeighborOption) { + Cost Minimum = InfiniteCost; + for (std::size_t Option = 0; Option < Costs[ID.value()].size(); ++Option) { + Minimum = std::min( + Minimum, addCosts(Costs[ID.value()][Option], Oriented.at(Option, NeighborOption))); + } + Costs[Neighbor.value()][NeighborOption] = + addCosts(Costs[Neighbor.value()][NeighborOption], Minimum); + } + removeIncidentEdges(ID); + deactivate(ID); + return Record; + } + + [[nodiscard]] Expected<ReductionRecord> reduceDegreeTwo(NodeID ID) { + const std::vector<WorkingEdge> Incident = incidentEdges(ID); + if (Incident.size() != 2) + return makeUnexpected(ErrorCode::InternalError, + "PBQP degree-two reduction is inconsistent"); + NodeID First = other(Incident[0], ID); + NodeID Second = other(Incident[1], ID); + CostMatrix FirstCosts = orientFrom(Incident[0], ID); + CostMatrix SecondCosts = orientFrom(Incident[1], ID); + if (Second < First) { + std::swap(First, Second); + std::swap(FirstCosts, SecondCosts); + } + + ReductionRecord Record; + Record.Kind = ReductionKind::DegreeTwo; + Record.Node = ID; + Record.FirstNeighbor = First; + Record.SecondNeighbor = Second; + Record.NodeCosts = Costs[ID.value()]; + Record.FirstCosts = FirstCosts; + Record.SecondCosts = SecondCosts; + + CostMatrix Delta(Costs[First.value()].size(), Costs[Second.value()].size(), InfiniteCost); + for (std::size_t FirstOption = 0; FirstOption < Delta.rows(); ++FirstOption) { + for (std::size_t SecondOption = 0; SecondOption < Delta.columns(); ++SecondOption) { + for (std::size_t Option = 0; Option < Costs[ID.value()].size(); ++Option) { + const Cost Candidate = + addCosts(Costs[ID.value()][Option], addCosts(FirstCosts.at(Option, FirstOption), + SecondCosts.at(Option, SecondOption))); + Delta.at(FirstOption, SecondOption) = + std::min(Delta.at(FirstOption, SecondOption), Candidate); + } + } + } + removeIncidentEdges(ID); + const auto Position = Edges.find(key(First, Second)); + if (Position == Edges.end()) + Edges.emplace(key(First, Second), WorkingEdge{First, Second, std::move(Delta)}); + else + Position->second.Costs.add(Delta); + deactivate(ID); + return Record; + } + + [[nodiscard]] Expected<ReductionRecord> reduceHeuristically(NodeID ID) { + const std::vector<WorkingEdge> Incident = incidentEdges(ID); + Cost Best = InfiniteCost; + std::size_t Choice = 0; + for (std::size_t Option = 0; Option < Costs[ID.value()].size(); ++Option) { + Cost Candidate = Costs[ID.value()][Option]; + for (const WorkingEdge &EdgeValue : Incident) { + const NodeID Neighbor = other(EdgeValue, ID); + const CostMatrix Oriented = orientFrom(EdgeValue, ID); + Cost NeighborMinimum = InfiniteCost; + for (std::size_t NeighborOption = 0; NeighborOption < Costs[Neighbor.value()].size(); + ++NeighborOption) { + NeighborMinimum = std::min(NeighborMinimum, Oriented.at(Option, NeighborOption)); + } + Candidate = addCosts(Candidate, NeighborMinimum); + } + if (Candidate < Best) { + Best = Candidate; + Choice = Option; + } + } + if (isInfinite(Best)) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "PBQP heuristic reduction has no finite option"); + + ReductionRecord Record; + Record.Kind = ReductionKind::Heuristic; + Record.Node = ID; + Record.FixedSelection = Choice; + Record.NodeCosts = Costs[ID.value()]; + for (const WorkingEdge &EdgeValue : Incident) { + const NodeID Neighbor = other(EdgeValue, ID); + const CostMatrix Oriented = orientFrom(EdgeValue, ID); + for (std::size_t Option = 0; Option < Costs[Neighbor.value()].size(); ++Option) { + Costs[Neighbor.value()][Option] = + addCosts(Costs[Neighbor.value()][Option], Oriented.at(Choice, Option)); + } + } + removeIncidentEdges(ID); + deactivate(ID); + return Record; + } + + private: + using EdgeKey = std::pair<std::uint32_t, std::uint32_t>; + + [[nodiscard]] static EdgeKey key(NodeID First, NodeID Second) noexcept { + if (Second < First) + std::swap(First, Second); + return {First.value(), Second.value()}; + } + + [[nodiscard]] static NodeID other(const WorkingEdge &EdgeValue, NodeID ID) noexcept { + return EdgeValue.First == ID ? EdgeValue.Second : EdgeValue.First; + } + + [[nodiscard]] static CostMatrix orientFrom(const WorkingEdge &EdgeValue, NodeID ID) { + return EdgeValue.First == ID ? EdgeValue.Costs : EdgeValue.Costs.transpose(); + } + + [[nodiscard]] std::vector<WorkingEdge> incidentEdges(NodeID ID) const { + std::vector<WorkingEdge> Result; + for (const auto &[EdgeKeyValue, EdgeValue] : Edges) { + (void)EdgeKeyValue; + if (Active[EdgeValue.First.value()] && Active[EdgeValue.Second.value()] && + (EdgeValue.First == ID || EdgeValue.Second == ID)) + Result.push_back(EdgeValue); + } + return Result; + } + + void removeIncidentEdges(NodeID ID) { + for (auto Position = Edges.begin(); Position != Edges.end();) { + if (Position->second.First == ID || Position->second.Second == ID) + Position = Edges.erase(Position); + else + ++Position; + } + } + + void deactivate(NodeID ID) noexcept { + Active[ID.value()] = false; + --ActiveCount; + } + + std::vector<CostVector> Costs; + std::vector<std::uint64_t> StableKeys; + std::vector<bool> Active; + std::size_t ActiveCount = 0; + std::map<EdgeKey, WorkingEdge> Edges; + }; +}; + +} // namespace rcc::pbqp + +#endif // RCC_CODEGEN_PBQP_SOLVER_H diff --git a/backend/include/rcc/CodeGen/RegAllocCommon.h b/backend/include/rcc/CodeGen/RegAllocCommon.h new file mode 100644 index 0000000..0df9799 --- /dev/null +++ b/backend/include/rcc/CodeGen/RegAllocCommon.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_REGALLOCCOMMON_H +#define RCC_CODEGEN_REGALLOCCOMMON_H + +#include "rcc/CodeGen/MachinePass.h" + +#include <memory> + +namespace rcc { + +class TargetRegisterAllocationInfo; + +struct RegisterAllocationOptions final { + bool EnableRematerialization = true; + bool EnableCopyCoalescing = true; +}; + +[[nodiscard]] std::unique_ptr<MachineFunctionPass> +createBasicAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options = {}); + +} // namespace rcc + +#endif // RCC_CODEGEN_REGALLOCCOMMON_H diff --git a/backend/include/rcc/CodeGen/RegAllocFast.h b/backend/include/rcc/CodeGen/RegAllocFast.h new file mode 100644 index 0000000..84cf30c --- /dev/null +++ b/backend/include/rcc/CodeGen/RegAllocFast.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_REGALLOCFAST_H +#define RCC_CODEGEN_REGALLOCFAST_H + +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +namespace rcc { + +class FastAllocatorPass final : public MachineFunctionPass { +public: + explicit FastAllocatorPass(const TargetRegisterAllocationInfo &Info) noexcept : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetRegisterAllocationInfo *Info; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_REGALLOCFAST_H diff --git a/backend/include/rcc/CodeGen/RegAllocPBQP.h b/backend/include/rcc/CodeGen/RegAllocPBQP.h new file mode 100644 index 0000000..368cbee --- /dev/null +++ b/backend/include/rcc/CodeGen/RegAllocPBQP.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_REGALLOCPBQP_H +#define RCC_CODEGEN_REGALLOCPBQP_H + +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +namespace rcc { + +class PBQPAllocatorPass final : public MachineFunctionPass { +public: + explicit PBQPAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetRegisterAllocationInfo *Info; + RegisterAllocationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_REGALLOCPBQP_H diff --git a/backend/include/rcc/CodeGen/Spiller.h b/backend/include/rcc/CodeGen/Spiller.h new file mode 100644 index 0000000..3d1fc66 --- /dev/null +++ b/backend/include/rcc/CodeGen/Spiller.h @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_SPILLER_H +#define RCC_CODEGEN_SPILLER_H + +#include "rcc/CodeGen/LiveRangeEdit.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +#include <optional> +#include <vector> + +namespace rcc { + +class Spiller final { +public: + Spiller(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, VirtRegMap &Mapping, + bool EnableRematerialization = true) + : Function(&Function), Target(&Target), TargetInfo(&TargetInfo), Mapping(&Mapping), + Edit(Function, Target, Mapping), EnableRematerialization(EnableRematerialization) {} + + [[nodiscard]] Expected<std::vector<Register>> spill(Register VirtualReg); + +private: + [[nodiscard]] Expected<std::vector<Register>> spillAtInstructions(Register VirtualReg); + [[nodiscard]] Expected<void> spillLiveIn(Register VirtualReg, RegisterClassID Class, + FrameIndex StackSlot); + [[nodiscard]] Expected<std::optional<std::vector<Register>>> + tryRematerialize(Register VirtualReg); + + MachineFunction *Function; + const TargetDescription *Target; + const TargetRegisterAllocationInfo *TargetInfo; + VirtRegMap *Mapping; + LiveRangeEdit Edit; + bool EnableRematerialization; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_SPILLER_H diff --git a/backend/include/rcc/CodeGen/TargetMIRTextInfo.h b/backend/include/rcc/CodeGen/TargetMIRTextInfo.h index b8eb4ee..3ed6022 100644 --- a/backend/include/rcc/CodeGen/TargetMIRTextInfo.h +++ b/backend/include/rcc/CodeGen/TargetMIRTextInfo.h @@ -68,10 +68,8 @@ struct MIRTargetMappingEntry final { const MIRTargetMappingEntry &Right) = default; }; -using MIRTargetFunctionStateParser = - Expected<TargetFunctionState> (*)(const MIRTargetNode &Node); -using MIRTargetFunctionStatePrinter = - Expected<MIRTargetNode> (*)(const TargetFunctionState &State); +using MIRTargetFunctionStateParser = Expected<TargetFunctionState> (*)(const MIRTargetNode &Node); +using MIRTargetFunctionStatePrinter = Expected<MIRTargetNode> (*)(const TargetFunctionState &State); using MIRTargetConstantParser = Expected<TargetConstant> (*)(std::string_view Text); using MIRTargetConstantPrinter = Expected<std::string> (*)(const TargetConstant &Constant); @@ -235,13 +233,10 @@ public: return Description.FunctionStateType; } - [[nodiscard]] Expected<TargetFunctionState> - parseFunctionState(const MIRTargetNode &Node) const; - [[nodiscard]] Expected<MIRTargetNode> - printFunctionState(const TargetFunctionState &State) const; + [[nodiscard]] Expected<TargetFunctionState> parseFunctionState(const MIRTargetNode &Node) const; + [[nodiscard]] Expected<MIRTargetNode> printFunctionState(const TargetFunctionState &State) const; [[nodiscard]] Expected<TargetConstant> parseTargetConstant(std::string_view Text) const; - [[nodiscard]] Expected<std::string> - printTargetConstant(const TargetConstant &Constant) const; + [[nodiscard]] Expected<std::string> printTargetConstant(const TargetConstant &Constant) const; private: explicit TargetMIRTextInfo(TargetMIRTextInfoDesc Description) diff --git a/backend/include/rcc/CodeGen/TargetRegisterAllocationInfo.h b/backend/include/rcc/CodeGen/TargetRegisterAllocationInfo.h new file mode 100644 index 0000000..6e25465 --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetRegisterAllocationInfo.h @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETREGISTERALLOCATIONINFO_H +#define RCC_CODEGEN_TARGETREGISTERALLOCATIONINFO_H + +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <cstdint> + +namespace rcc { + +class MachineIRBuilder; +class MachineInstr; +class TargetDescription; + +class TargetRegisterAllocationInfo { +public: + virtual ~TargetRegisterAllocationInfo() = default; + + [[nodiscard]] virtual Expected<void> + validateSpillReload(RegisterClassID Class, const TargetDescription &Target) const = 0; + [[nodiscard]] virtual Expected<MachineInstr *> + emitReload(MachineIRBuilder &Builder, Register Destination, RegisterClassID Class, + FrameIndex StackSlot, LaneMask Lanes) const = 0; + [[nodiscard]] virtual Expected<MachineInstr *> emitSpill(MachineIRBuilder &Builder, + Register Source, RegisterClassID Class, + FrameIndex StackSlot, + LaneMask Lanes) const = 0; + + [[nodiscard]] virtual Expected<bool> isRematerializable(const MachineInstr &, Register, + const MachineFunction &, + const TargetDescription &) const { + return false; + } + [[nodiscard]] virtual Expected<MachineInstr *> rematerialize(MachineIRBuilder &, + const MachineInstr &, Register, + Register, + const TargetDescription &) const { + return makeUnexpected(ErrorCode::InternalError, + "target accepted rematerialization without implementing it"); + } + + [[nodiscard]] virtual std::uint64_t physicalRegisterCost(const MachineFunction &, Register, + PhysicalRegister, RegisterClassID, + bool) const noexcept { + return 0; + } + [[nodiscard]] virtual std::uint64_t spillCost(const MachineFunction &, const LiveInterval &, + RegisterClassID) const noexcept { + return 1; + } +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETREGISTERALLOCATIONINFO_H diff --git a/backend/include/rcc/CodeGen/VirtRegMap.h b/backend/include/rcc/CodeGen/VirtRegMap.h new file mode 100644 index 0000000..f37af84 --- /dev/null +++ b/backend/include/rcc/CodeGen/VirtRegMap.h @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_VIRTREGMAP_H +#define RCC_CODEGEN_VIRTREGMAP_H + +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <cstdint> +#include <optional> +#include <unordered_map> +#include <vector> + +namespace rcc { + +class MachineFunction; +class TargetDescription; + +enum class VirtRegAllocationState : std::uint8_t { + Unassigned, + Assigned, + Spilled, + Split, +}; + +struct VirtRegSpillMetadata final { + std::optional<FrameIndex> StackSlot; + LaneMask Lanes = LaneMask::all(); + std::uint64_t SpillCount = 0; + std::uint64_t ReloadCount = 0; + std::uint64_t RematerializationCount = 0; +}; + +struct VirtRegMapEntry final { + Register VirtualReg; + VirtRegAllocationState State = VirtRegAllocationState::Unassigned; + std::optional<PhysicalRegister> PhysicalReg; + std::optional<Register> SplitParent; + std::optional<Register> Origin; + bool Spillable = true; + VirtRegSpillMetadata Spill; +}; + +class VirtRegMap final { +public: + explicit VirtRegMap(MachineFunction &Function) noexcept : Function(&Function) {} + + [[nodiscard]] MachineFunction &function() const noexcept { + return *Function; + } + [[nodiscard]] Expected<void> add(Register VirtualReg, bool Spillable = true, + std::optional<Register> SplitParent = std::nullopt, + std::optional<Register> Origin = std::nullopt); + [[nodiscard]] Expected<void> assign(Register VirtualReg, PhysicalRegister PhysicalReg); + [[nodiscard]] Expected<void> assignStackSlot(Register VirtualReg, FrameIndex StackSlot, + LaneMask Lanes = LaneMask::all()); + [[nodiscard]] Expected<void> markSplit(Register VirtualReg, Register Parent, Register Origin); + [[nodiscard]] Expected<void> setSpillable(Register VirtualReg, bool Spillable); + [[nodiscard]] VirtRegMapEntry *entry(Register VirtualReg) noexcept; + [[nodiscard]] const VirtRegMapEntry *entry(Register VirtualReg) const noexcept; + [[nodiscard]] std::optional<PhysicalRegister> + physicalRegister(Register VirtualReg) const noexcept; + [[nodiscard]] std::optional<FrameIndex> stackSlot(Register VirtualReg) const noexcept; + [[nodiscard]] bool isSpillable(Register VirtualReg) const noexcept; + [[nodiscard]] std::vector<const VirtRegMapEntry *> orderedEntries() const; + [[nodiscard]] Expected<void> verify() const; + +private: + MachineFunction *Function; + std::unordered_map<Register, VirtRegMapEntry> Entries; +}; + +[[nodiscard]] Expected<bool> rewriteVirtualRegisters(MachineFunction &Function, + const TargetDescription &Target, + const VirtRegMap &Mapping); + +class VirtualRegisterRewritePass final : public MachineFunctionPass { +public: + explicit VirtualRegisterRewritePass(const VirtRegMap &Mapping) noexcept : Mapping(&Mapping) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const VirtRegMap *Mapping; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_VIRTREGMAP_H diff --git a/backend/include/rcc/MIR/EmbeddedIR.h b/backend/include/rcc/MIR/EmbeddedIR.h index 50567de..522a217 100644 --- a/backend/include/rcc/MIR/EmbeddedIR.h +++ b/backend/include/rcc/MIR/EmbeddedIR.h @@ -122,8 +122,7 @@ public: return GlobalObjects; } - [[nodiscard]] std::span<const EmbeddedIRMetadataDefinition> - metadataDefinitions() const noexcept { + [[nodiscard]] std::span<const EmbeddedIRMetadataDefinition> metadataDefinitions() const noexcept { return MetadataDefinitions; } @@ -134,7 +133,8 @@ public: [[nodiscard]] const EmbeddedIRFunction *function(std::string_view Name) const noexcept; [[nodiscard]] const EmbeddedIRReference *globalObject(std::string_view Name) const noexcept; [[nodiscard]] const EmbeddedIRMetadataDefinition *metadata(std::string_view Name) const noexcept; - [[nodiscard]] const EmbeddedIRTypeDefinition *typeDefinition(std::string_view Name) const noexcept; + [[nodiscard]] const EmbeddedIRTypeDefinition * + typeDefinition(std::string_view Name) const noexcept; [[nodiscard]] bool hasGlobalValue(std::string_view Name) const noexcept { return function(Name) != nullptr || globalObject(Name) != nullptr; } diff --git a/backend/include/rcc/Support/Error.h b/backend/include/rcc/Support/Error.h index 3d50b46..81568de 100644 --- a/backend/include/rcc/Support/Error.h +++ b/backend/include/rcc/Support/Error.h @@ -16,6 +16,7 @@ enum class ErrorCode : std::uint8_t { ParseError, VerificationError, Unsupported, + RegisterAllocationFailed, IoError, InternalError, }; diff --git a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h index 4db60f7..403bb2d 100644 --- a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h +++ b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h @@ -4,6 +4,7 @@ #define RCC_TARGET_RISCV_RISCVCODEGEN_H #include "rcc/CodeGen/TargetCodeGen.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" namespace rcc::riscv { @@ -11,6 +12,7 @@ namespace rcc::riscv { [[nodiscard]] const TargetRegisterBankInfo ®isterBankInfo() noexcept; [[nodiscard]] const TargetInstructionSelector &instructionSelector() noexcept; [[nodiscard]] const TargetSSAInfo &ssaInfo() noexcept; +[[nodiscard]] const TargetRegisterAllocationInfo ®isterAllocationInfo() noexcept; } // namespace rcc::riscv diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index 4c2887c..d6233c2 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -30,6 +30,14 @@ set( MachinePass.cpp MachinePipelinePasses.cpp MachineRegisterInfo.cpp + InlineSpiller.cpp + LiveRangeEdit.cpp + LiveRegMatrix.cpp + RegAllocBase.cpp + RegAllocBasic.cpp + RegAllocFast.cpp + RegAllocPBQP.cpp + VirtRegMap.cpp MachineSSAExit.cpp MachineVerifier.cpp MachineFunctionProperties.cpp @@ -74,6 +82,19 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePipelinePasses.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Graph.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Math.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/ReductionRules.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Solution.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Solver.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveRangeEdit.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveRegMatrix.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocCommon.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocFast.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocPBQP.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/Spiller.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterAllocationInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/VirtRegMap.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineSSAExit.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineVerifier.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionProperties.h" @@ -146,6 +167,17 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBase.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasic.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocFast.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocPBQP.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/VirtRegMap.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBase.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasic.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocInternal.h" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifier.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionProperties.cpp" @@ -189,6 +221,14 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/InlineSpiller.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRangeEdit.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrix.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBase.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasic.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocFast.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocPBQP.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/VirtRegMap.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExit.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifier.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionProperties.cpp" diff --git a/backend/lib/CodeGen/GenericConstant.cpp b/backend/lib/CodeGen/GenericConstant.cpp index 2ef24bf..d9d9233 100644 --- a/backend/lib/CodeGen/GenericConstant.cpp +++ b/backend/lib/CodeGen/GenericConstant.cpp @@ -66,8 +66,7 @@ struct VectorShape final { friend constexpr bool operator==(VectorShape Left, VectorShape Right) noexcept = default; }; -[[nodiscard]] std::optional<VectorShape> -vectorShape(const GenericConstantType &Type) noexcept { +[[nodiscard]] std::optional<VectorShape> vectorShape(const GenericConstantType &Type) noexcept { const GenericConstantType &Resolved = resolvedType(Type); if (Resolved.Kind != GenericConstantTypeKind::Vector) return std::nullopt; @@ -121,8 +120,7 @@ pointerAddressSpace(const GenericConstantType &Type) noexcept { if (isIntegerScalarType(Type) || isFloatingScalarType(Type)) return true; const GenericConstantType *Element = vectorElementType(Type); - return Element != nullptr && - (isIntegerScalarType(*Element) || isFloatingScalarType(*Element)); + return Element != nullptr && (isIntegerScalarType(*Element) || isFloatingScalarType(*Element)); } [[nodiscard]] std::optional<std::uint64_t> @@ -175,8 +173,7 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { isPointerScalarType(*PointerElement) && isIntegerScalarType(*IntegerElement); } -[[nodiscard]] bool isSizedType(const GenericConstantType &Type, - std::size_t Depth = 0) noexcept { +[[nodiscard]] bool isSizedType(const GenericConstantType &Type, std::size_t Depth = 0) noexcept { if (Depth > MaximumVerificationDepth) return false; const GenericConstantType &Resolved = resolvedType(Type); @@ -224,8 +221,8 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { [[nodiscard]] bool hasOnlyValuePayload(const GenericConstantData &Value) noexcept { return Value.Symbol.empty() && Value.Block.empty() && Value.ExpressionOpcode == GenericConstantExpressionOpcode::Invalid && - Value.ExpressionFlags.rawValue() == 0 && !Value.SourceElementType && - !Value.InRangeStart && !Value.InRangeEnd; + Value.ExpressionFlags.rawValue() == 0 && !Value.SourceElementType && !Value.InRangeStart && + !Value.InRangeEnd; } [[nodiscard]] Expected<void> verifyType(const GenericConstantType &Type, std::size_t Depth, @@ -335,8 +332,7 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { (EffectiveType.Kind != GenericConstantTypeKind::Integer && EffectiveType.Kind != GenericConstantTypeKind::Byte) || Value.IntegerValue->bitWidth() != - (EffectiveType.Kind == GenericConstantTypeKind::Byte ? 8U - : EffectiveType.BitWidth)) { + (EffectiveType.Kind == GenericConstantTypeKind::Byte ? 8U : EffectiveType.BitWidth)) { return makeUnexpected(ErrorCode::VerificationError, "generic integer constant has an inconsistent payload"); } @@ -384,9 +380,8 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { } for (std::size_t Index = 0; Index < Value.Elements.size(); ++Index) { const GenericConstantType &ExpectedType = - EffectiveType.Kind == GenericConstantTypeKind::Struct - ? EffectiveType.Elements[Index] - : EffectiveType.Elements.front(); + EffectiveType.Kind == GenericConstantTypeKind::Struct ? EffectiveType.Elements[Index] + : EffectiveType.Elements.front(); if (Value.Elements[Index].Type != ExpectedType) { return makeUnexpected(ErrorCode::VerificationError, "generic aggregate element type does not match its container"); @@ -417,8 +412,8 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { Value.IntegerValue || Value.FloatingValue || !Value.ByteValue.empty() || !Value.Elements.empty() || !Value.Block.empty() || Value.ExpressionOpcode != GenericConstantExpressionOpcode::Invalid || - Value.ExpressionFlags.rawValue() != 0 || Value.SourceElementType || - Value.InRangeStart || Value.InRangeEnd) { + Value.ExpressionFlags.rawValue() != 0 || Value.SourceElementType || Value.InRangeStart || + Value.InRangeEnd) { return makeUnexpected(ErrorCode::VerificationError, "generic global-address constant has an inconsistent payload"); } @@ -428,13 +423,13 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { Value.Block.empty() || Value.IntegerValue || Value.FloatingValue || !Value.ByteValue.empty() || !Value.Elements.empty() || Value.ExpressionOpcode != GenericConstantExpressionOpcode::Invalid || - Value.ExpressionFlags.rawValue() != 0 || Value.SourceElementType || - Value.InRangeStart || Value.InRangeEnd) { + Value.ExpressionFlags.rawValue() != 0 || Value.SourceElementType || Value.InRangeStart || + Value.InRangeEnd) { return makeUnexpected(ErrorCode::VerificationError, "generic block-address constant has an inconsistent payload"); } return {}; - case GenericConstantKind::PointerAuth: + case GenericConstantKind::PointerAuth: { if (EffectiveType.Kind != GenericConstantTypeKind::Pointer || Value.Elements.size() < 2 || Value.Elements.size() > 5 || Value.IntegerValue || Value.FloatingValue || !Value.ByteValue.empty() || !hasOnlyValuePayload(Value)) { @@ -446,19 +441,20 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { if (!Result) return Result; } + const auto KeyBitWidth = integerScalarBitWidth(Value.Elements[1].Type); if (!isPointerScalarType(Value.Elements[0].Type) || - !sameResolvedType(Value.Type, Value.Elements[0].Type) || - !integerScalarBitWidth(Value.Elements[1].Type) || - *integerScalarBitWidth(Value.Elements[1].Type) != 32) { + !sameResolvedType(Value.Type, Value.Elements[0].Type) || !KeyBitWidth || + KeyBitWidth.value() != 32) { return makeUnexpected(ErrorCode::VerificationError, "generic pointer-auth constant has invalid pointer or key operands"); } - if (Value.Elements.size() >= 3 && - (!integerScalarBitWidth(Value.Elements[2].Type) || - *integerScalarBitWidth(Value.Elements[2].Type) != 64)) { - return makeUnexpected( - ErrorCode::VerificationError, - "generic pointer-auth constant integer discriminator must be a scalar i64"); + if (Value.Elements.size() >= 3) { + const auto DiscriminatorBitWidth = integerScalarBitWidth(Value.Elements[2].Type); + if (!DiscriminatorBitWidth || DiscriminatorBitWidth.value() != 64) { + return makeUnexpected( + ErrorCode::VerificationError, + "generic pointer-auth constant integer discriminator must be a scalar i64"); + } } for (std::size_t Index = 3; Index < Value.Elements.size(); ++Index) { if (!isPointerScalarType(Value.Elements[Index].Type)) { @@ -468,6 +464,7 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { } } return {}; + } case GenericConstantKind::Expression: if (Value.ExpressionOpcode == GenericConstantExpressionOpcode::Invalid || Value.Elements.empty() || Value.IntegerValue || Value.FloatingValue || @@ -495,18 +492,18 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { const bool IsBinary = Value.ExpressionOpcode == GenericConstantExpressionOpcode::Add || Value.ExpressionOpcode == GenericConstantExpressionOpcode::Sub || Value.ExpressionOpcode == GenericConstantExpressionOpcode::Xor; - const bool IsGEP = Value.ExpressionOpcode == - GenericConstantExpressionOpcode::GetElementPtr; + const bool IsGEP = Value.ExpressionOpcode == GenericConstantExpressionOpcode::GetElementPtr; const std::uint8_t Flags = Value.ExpressionFlags.rawValue(); const std::uint8_t BinaryFlags = - (std::uint8_t{1} << static_cast<std::uint8_t>(GenericConstantExpressionFlag::NoUnsignedWrap)) | + (std::uint8_t{1} << static_cast<std::uint8_t>( + GenericConstantExpressionFlag::NoUnsignedWrap)) | (std::uint8_t{1} << static_cast<std::uint8_t>(GenericConstantExpressionFlag::NoSignedWrap)); const std::uint8_t GEPFlags = (std::uint8_t{1} << static_cast<std::uint8_t>(GenericConstantExpressionFlag::InBounds)) | - (std::uint8_t{1} << - static_cast<std::uint8_t>(GenericConstantExpressionFlag::NoUnsignedSignedWrap)) | - (std::uint8_t{1} << - static_cast<std::uint8_t>(GenericConstantExpressionFlag::GEPNoUnsignedWrap)); + (std::uint8_t{1} << static_cast<std::uint8_t>( + GenericConstantExpressionFlag::NoUnsignedSignedWrap)) | + (std::uint8_t{1} << static_cast<std::uint8_t>( + GenericConstantExpressionFlag::GEPNoUnsignedWrap)); if ((!IsBinary && !IsGEP && Flags != 0) || (IsBinary && (Flags & ~BinaryFlags) != 0) || (IsGEP && (Flags & ~GEPFlags) != 0)) { return makeUnexpected(ErrorCode::VerificationError, @@ -594,14 +591,14 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { if (!isIntegerOrIntegerVectorType(Value.Type) || !sameResolvedType(Value.Elements[0].Type, Value.Type) || !sameResolvedType(Value.Elements[1].Type, Value.Type)) { - return makeUnexpected(ErrorCode::VerificationError, - "generic binary constant requires equal integer operand and result types"); + return makeUnexpected( + ErrorCode::VerificationError, + "generic binary constant requires equal integer operand and result types"); } return {}; case GenericConstantExpressionOpcode::GetElementPtr: { if (!Value.SourceElementType || !isSizedType(*Value.SourceElementType) || - !isPointerOrPointerVectorType(FirstType) || - !isPointerOrPointerVectorType(Value.Type)) { + !isPointerOrPointerVectorType(FirstType) || !isPointerOrPointerVectorType(Value.Type)) { return makeUnexpected(ErrorCode::VerificationError, "generic getelementptr constant has invalid source or pointer types"); } @@ -631,9 +628,8 @@ nonPointerFixedBitWidth(const GenericConstantType &Type) noexcept { return makeUnexpected(ErrorCode::VerificationError, "generic getelementptr result has an incompatible vector shape"); } - if (Value.InRangeStart && - (Value.InRangeStart->bitWidth() != Value.InRangeEnd->bitWidth() || - !signedIntegerLess(*Value.InRangeStart, *Value.InRangeEnd))) { + if (Value.InRangeStart && (Value.InRangeStart->bitWidth() != Value.InRangeEnd->bitWidth() || + !signedIntegerLess(*Value.InRangeStart, *Value.InRangeEnd))) { return makeUnexpected(ErrorCode::VerificationError, "generic getelementptr inrange interval is invalid"); } @@ -756,9 +752,8 @@ Expected<GenericConstantType> GenericConstantType::array(std::uint64_t Count, return Type; } -Expected<GenericConstantType> GenericConstantType::vector(std::uint64_t Count, - GenericConstantType Element, - bool IsScalable) { +Expected<GenericConstantType> +GenericConstantType::vector(std::uint64_t Count, GenericConstantType Element, bool IsScalable) { if (Count == 0) return makeUnexpected(ErrorCode::InvalidArgument, "generic vector type cannot be empty"); auto Verification = Element.verify(); @@ -800,8 +795,7 @@ GenericConstantType::function(GenericConstantType Return, return Type; } -Expected<GenericConstantType> GenericConstantType::named(std::string TypeName, - bool IsNumericName) { +Expected<GenericConstantType> GenericConstantType::named(std::string TypeName, bool IsNumericName) { if (TypeName.empty()) return makeUnexpected(ErrorCode::InvalidArgument, "generic named type cannot be empty"); GenericConstantType Type; diff --git a/backend/lib/CodeGen/InlineSpiller.cpp b/backend/lib/CodeGen/InlineSpiller.cpp new file mode 100644 index 0000000..589b7b8 --- /dev/null +++ b/backend/lib/CodeGen/InlineSpiller.cpp @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/Spiller.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<void> clearDebugRegisterOperand(MachineOperand &Operand) { + RegisterOperandFlags Flags = Operand.registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto SetReg = Operand.setRegister(Register{}); + if (!SetReg) + return std::unexpected(SetReg.error()); + auto SetSubReg = Operand.setSubReg(SubRegIndex{}); + if (!SetSubReg) + return std::unexpected(SetSubReg.error()); + return Operand.setRegisterFlags(Flags); +} + +} // namespace + +Expected<std::optional<std::vector<Register>>> Spiller::tryRematerialize(Register VirtualReg) { + if (!EnableRematerialization) + return std::optional<std::vector<Register>>{}; + const MachineOperand *Definition = Function->registerInfo().getUniqueVRegDef(VirtualReg); + const MachineInstr *DefiningInstruction = Definition == nullptr ? nullptr : Definition->parent(); + if (DefiningInstruction == nullptr || DefiningInstruction->parent() == nullptr) + return std::optional<std::vector<Register>>{}; + auto Accepted = + TargetInfo->isRematerializable(*DefiningInstruction, VirtualReg, *Function, *Target); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return std::optional<std::vector<Register>>{}; + + struct UseSite final { + MachineInstr *Instruction = nullptr; + std::vector<std::size_t> OperandIndices; + }; + std::vector<UseSite> Sites; + for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + UseSite Site{Instruction, {}}; + bool HasDefinition = false; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || *Reg != VirtualReg) + continue; + HasDefinition |= Operand->isDef(); + if (Operand->isUse() && !Operand->isDebugUse()) + Site.OperandIndices.push_back(Index); + } + if (HasDefinition && !Site.OperandIndices.empty()) + return std::optional<std::vector<Register>>{}; + if (!Site.OperandIndices.empty()) + Sites.push_back(std::move(Site)); + } + } + if (Sites.empty()) + return std::optional<std::vector<Register>>{}; + + std::vector<Register> NewRegisters; + NewRegisters.reserve(Sites.size()); + for (UseSite &Site : Sites) { + auto Child = Edit.createFrom(VirtualReg, false); + if (!Child) + return std::unexpected(Child.error()); + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointBefore(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Materialized = + TargetInfo->rematerialize(Builder, *DefiningInstruction, *Child, VirtualReg, *Target); + if (!Materialized) + return std::unexpected(Materialized.error()); + for (const std::size_t Index : Site.OperandIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Replaced = Operand->setRegister(*Child); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.RematerializationCount; + NewRegisters.push_back(*Child); + } + + bool HasNonDebugUse = false; + for (const MachineOperand &Operand : Function->registerInfo().useOperands(VirtualReg, false)) { + if (Operand.parent() != nullptr) { + HasNonDebugUse = true; + break; + } + } + if (!HasNonDebugUse) { + MachineBasicBlock *Block = const_cast<MachineBasicBlock *>(DefiningInstruction->parent()); + auto Removed = Block->removeInstruction(*const_cast<MachineInstr *>(DefiningInstruction)); + if (!Removed) + return std::unexpected(Removed.error()); + } + return std::optional<std::vector<Register>>(std::move(NewRegisters)); +} + +Expected<std::vector<Register>> Spiller::spillAtInstructions(Register VirtualReg) { + auto ClassID = regalloc_detail::registerClassFor(*Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Validated = TargetInfo->validateSpillReload(*ClassID, *Target); + if (!Validated) { + std::string Message = + "target declares an allocatable spillable class without spill/reload support: "; + Message.append(Validated.error().message()); + return makeUnexpected(ErrorCode::InternalError, std::move(Message)); + } + auto StackSlot = Edit.getOrCreateSpillSlot(VirtualReg); + if (!StackSlot) + return std::unexpected(StackSlot.error()); + auto LiveInSpill = spillLiveIn(VirtualReg, *ClassID, *StackSlot); + if (!LiveInSpill) + return std::unexpected(LiveInSpill.error()); + + struct SpillSite final { + MachineInstr *Instruction = nullptr; + std::vector<std::size_t> OperandIndices; + std::vector<std::size_t> DebugIndices; + bool HasUse = false; + bool HasDef = false; + }; + std::vector<SpillSite> Sites; + for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + SpillSite Site; + Site.Instruction = Instruction; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || *Reg != VirtualReg) + continue; + if (Operand->isDebugUse()) { + Site.DebugIndices.push_back(Index); + continue; + } + Site.OperandIndices.push_back(Index); + Site.HasUse |= Operand->isUse(); + Site.HasDef |= Operand->isDef(); + } + if (!Site.OperandIndices.empty() || !Site.DebugIndices.empty()) + Sites.push_back(std::move(Site)); + } + } + + std::vector<Register> NewRegisters; + for (SpillSite &Site : Sites) { + for (const std::size_t Index : Site.DebugIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Cleared = clearDebugRegisterOperand(*Operand); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Site.OperandIndices.empty()) + continue; + + auto Child = Edit.createFrom(VirtualReg, false); + if (!Child) + return std::unexpected(Child.error()); + if (Site.HasUse) { + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointBefore(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Reload = TargetInfo->emitReload(Builder, *Child, *ClassID, *StackSlot, LaneMask::all()); + if (!Reload) + return std::unexpected(Reload.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.ReloadCount; + } + for (const std::size_t Index : Site.OperandIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Replaced = Operand->setRegister(*Child); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (Site.HasDef) { + const InstructionDescriptor *Descriptor = + Target->instructionDescriptor(Site.Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "a terminator-defined value cannot be spilled on its outgoing edge"); + } + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointAfter(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo->emitSpill(Builder, *Child, *ClassID, *StackSlot, LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + } + NewRegisters.push_back(*Child); + } + return NewRegisters; +} + +Expected<void> Spiller::spillLiveIn(Register VirtualReg, RegisterClassID Class, + FrameIndex StackSlot) { + const auto LiveIn = std::ranges::find_if(Function->registerInfo().liveIns(), + [VirtualReg](const PhysicalRegisterLiveIn &Value) { + return Value.InitialVirtualRegister == VirtualReg; + }); + if (LiveIn == Function->registerInfo().liveIns().end()) + return {}; + MachineBasicBlock *EntryBlock = Function->firstBlock(); + if (EntryBlock == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "function live-in spill requires an entry block"); + } + MachineIRBuilder Builder(*Function, *Target); + Expected<void> Positioned = EntryBlock->firstInstruction() != nullptr + ? Builder.setInsertPointBefore(*EntryBlock->firstInstruction()) + : Builder.setInsertPoint(*EntryBlock); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo->emitSpill(Builder, Register::fromPhysical(LiveIn->PhysReg), Class, + StackSlot, LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + return {}; +} + +Expected<std::vector<Register>> Spiller::spill(Register VirtualReg) { + if (!VirtualReg.isVirtual() || !Function->registerInfo().isKnownVirtualRegister(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "Spiller requires a known virtual register"); + if (!Mapping->isSpillable(VirtualReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "register pressure requires spilling a non-spillable value"); + } + auto Rematerialized = tryRematerialize(VirtualReg); + if (!Rematerialized) + return std::unexpected(Rematerialized.error()); + if (*Rematerialized) + return std::move(**Rematerialized); + return spillAtInstructions(VirtualReg); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveRangeEdit.cpp b/backend/lib/CodeGen/LiveRangeEdit.cpp new file mode 100644 index 0000000..23a7b3d --- /dev/null +++ b/backend/lib/CodeGen/LiveRangeEdit.cpp @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRangeEdit.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +namespace rcc { + +Expected<Register> LiveRangeEdit::createFrom(Register Parent, bool Spillable, + std::optional<Register> Origin) { + auto ParentInfo = Function->registerInfo().virtualRegisterInfo(Parent); + if (!ParentInfo) + return std::unexpected(ParentInfo.error()); + const VirtualRegisterInfo &Source = ParentInfo->get(); + Expected<Register> Created = Source.Type + ? Function->registerInfo().createVirtualRegister(*Source.Type) + : Function->registerInfo().createVirtualRegister(); + if (!Created) + return std::unexpected(Created.error()); + const Register Child = *Created; + auto Fail = [&](const Error &Failure) -> Expected<Register> { + auto Erased = Function->registerInfo().eraseVirtualRegister(Child); + if (!Erased) + return makeUnexpected(ErrorCode::InternalError, + "split virtual-register setup and rollback both failed"); + return std::unexpected(Failure); + }; + if (Source.Bank) { + auto Set = Function->registerInfo().setRegisterBank(Child, *Source.Bank); + if (!Set) + return Fail(Set.error()); + } + if (Source.RegClass) { + auto Set = Function->registerInfo().setRegisterClass(Child, *Source.RegClass); + if (!Set) + return Fail(Set.error()); + } + for (const PhysicalRegister Hint : Source.AllocationHints) { + auto Added = Function->registerInfo().addAllocationHint(Child, Hint); + if (!Added) + return Fail(Added.error()); + } + if (Source.TargetFlags != 0) { + auto Set = Function->registerInfo().setTargetRegisterFlags(Child, Source.TargetFlags); + if (!Set) + return Fail(Set.error()); + } + const VirtRegMapEntry *ParentEntry = Mapping->entry(Parent); + const Register Root = Origin.value_or( + ParentEntry != nullptr && ParentEntry->Origin ? *ParentEntry->Origin : Parent); + const std::optional<FrameIndex> ParentStackSlot = + ParentEntry == nullptr ? std::nullopt : ParentEntry->Spill.StackSlot; + const LaneMask ParentSpillLanes = + ParentEntry == nullptr ? LaneMask::all() : ParentEntry->Spill.Lanes; + auto Added = Mapping->add(Child, Spillable, Parent, Root); + if (!Added) + return Fail(Added.error()); + auto Split = Mapping->markSplit(Child, Parent, Root); + if (!Split) + return Fail(Split.error()); + if (ParentStackSlot) { + auto SharedSlot = Mapping->assignStackSlot(Child, *ParentStackSlot, ParentSpillLanes); + if (!SharedSlot) + return Fail(SharedSlot.error()); + } + return Child; +} + +Expected<FrameIndex> LiveRangeEdit::getOrCreateSpillSlot(Register VirtualReg) { + if (const auto Existing = Mapping->stackSlot(VirtualReg)) + return *Existing; + const VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + const Register Original = Entry != nullptr && Entry->Origin ? *Entry->Origin : VirtualReg; + if (Original != VirtualReg) { + if (const auto Existing = Mapping->stackSlot(Original)) { + auto Shared = Mapping->assignStackSlot(VirtualReg, *Existing); + if (!Shared) + return std::unexpected(Shared.error()); + return *Existing; + } + } + auto ClassID = regalloc_detail::registerClassFor(*Function, Original); + if (!ClassID) + return std::unexpected(ClassID.error()); + const RegisterClassDesc *Class = Target->registers().registerClass(*ClassID); + if (Class == nullptr || Class->SpillSizeInBits == 0 || Class->SpillAlignmentInBits == 0 || + Class->SpillSizeInBits % 8U != 0 || Class->SpillAlignmentInBits % 8U != 0) { + return makeUnexpected(ErrorCode::InternalError, + "allocatable register class has no valid spill size and alignment"); + } + auto Align = Alignment::create(Class->SpillAlignmentInBits / 8U); + if (!Align) + return std::unexpected(Align.error()); + StackObjectDesc Description; + Description.Kind = StackObjectKind::SpillSlot; + Description.Size = Class->SpillSizeInBits / 8U; + Description.Align = *Align; + auto Slot = Function->frameInfo().createObject(std::move(Description)); + if (!Slot) + return std::unexpected(Slot.error()); + auto AssignedOriginal = Mapping->assignStackSlot(Original, *Slot); + if (!AssignedOriginal) + return std::unexpected(AssignedOriginal.error()); + if (Original != VirtualReg) { + auto AssignedChild = Mapping->assignStackSlot(VirtualReg, *Slot); + if (!AssignedChild) + return std::unexpected(AssignedChild.error()); + } + return *Slot; +} + +Expected<void> LiveRangeEdit::eraseVirtualRegister(Register VirtualReg) { + if (!Function->registerInfo().useEmpty(VirtualReg)) + return makeUnexpected(ErrorCode::InternalError, + "cannot erase a split virtual register with remaining uses"); + for (const MachineOperand &Operand : Function->registerInfo().defOperands(VirtualReg)) { + if (Operand.parent() != nullptr) + return makeUnexpected(ErrorCode::InternalError, + "cannot erase a split virtual register with remaining definitions"); + } + return Function->registerInfo().eraseVirtualRegister(VirtualReg); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveRegMatrix.cpp b/backend/lib/CodeGen/LiveRegMatrix.cpp new file mode 100644 index 0000000..64c7046 --- /dev/null +++ b/backend/lib/CodeGen/LiveRegMatrix.cpp @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRegMatrix.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <bit> +#include <cstdint> +#include <optional> +#include <ranges> +#include <span> +#include <tuple> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::optional<LaneMask> relevantLanes(const LiveInterval &Interval, + RegisterUnitLane Unit) noexcept { + if (Interval.SubRanges.empty()) + return Unit.Lanes; + LaneMask Result; + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (SubRange.Lanes.intersects(Unit.Lanes)) + Result = Result | (SubRange.Lanes & Unit.Lanes); + } + if (Result.empty()) + return std::nullopt; + return Result; +} + +[[nodiscard]] LaneMask normalizeUnitLanes(LaneMask UnitViewLanes, LaneMask RegisterLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + const std::uint64_t Selected = View & RegisterLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + const std::uint64_t RegisterLane = std::uint64_t{1} << std::countr_zero(View); + if ((Selected & RegisterLane) != 0) + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +[[nodiscard]] LaneMask intervalUnitLanes(const LiveInterval &Interval, + RegisterUnitLane Unit) noexcept { + if (Interval.SubRanges.empty()) + return normalizeUnitLanes(Unit.Lanes, Unit.Lanes); + LaneMask Result; + for (const LiveSubRange &SubRange : Interval.SubRanges) + Result = Result | normalizeUnitLanes(Unit.Lanes, SubRange.Lanes); + return Result; +} + +[[nodiscard]] bool intervalLiveAtFixedRegister(const LiveInterval &Interval, + const PhysicalRegisterDesc &Candidate, + const PhysicalRegisterDesc &Fixed, + SlotIndex Position) noexcept { + for (const RegisterUnitLane CandidateUnit : Candidate.Units) { + const auto FixedUnit = + std::ranges::find(Fixed.Units, CandidateUnit.Unit, &RegisterUnitLane::Unit); + if (FixedUnit == Fixed.Units.end()) + continue; + const LaneMask FixedLanes = normalizeUnitLanes(FixedUnit->Lanes, FixedUnit->Lanes); + if (Interval.SubRanges.empty()) { + if (normalizeUnitLanes(CandidateUnit.Lanes, CandidateUnit.Lanes).intersects(FixedLanes) && + Interval.MainRange.liveAt(Position)) { + return true; + } + continue; + } + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (normalizeUnitLanes(CandidateUnit.Lanes, SubRange.Lanes).intersects(FixedLanes) && + SubRange.Range.liveAt(Position)) { + return true; + } + } + } + return false; +} + +[[nodiscard]] bool rangeForUnitOverlaps(const LiveInterval &Interval, RegisterUnitLane Unit, + const LiveRange &Other) noexcept { + if (!relevantLanes(Interval, Unit)) + return false; + if (Interval.SubRanges.empty()) + return Interval.MainRange.overlaps(Other); + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (SubRange.Lanes.intersects(Unit.Lanes) && SubRange.Range.overlaps(Other)) + return true; + } + return false; +} + +} // namespace + +bool regalloc_detail::intervalsOverlapOnPhysicalRegisters( + const LiveInterval &Left, PhysicalRegister LeftReg, const LiveInterval &Right, + PhysicalRegister RightReg, const TargetRegisterInfo &Registers) noexcept { + const PhysicalRegisterDesc *LeftDescription = Registers.physicalRegister(LeftReg); + const PhysicalRegisterDesc *RightDescription = Registers.physicalRegister(RightReg); + if (LeftDescription == nullptr || RightDescription == nullptr) + return true; + for (const RegisterUnitLane LeftUnit : LeftDescription->Units) { + for (const RegisterUnitLane RightUnit : RightDescription->Units) { + if (LeftUnit.Unit != RightUnit.Unit) + continue; + const LaneMask LeftUnitLanes = intervalUnitLanes(Left, LeftUnit); + const LaneMask RightUnitLanes = intervalUnitLanes(Right, RightUnit); + if (!LeftUnitLanes.intersects(RightUnitLanes)) + continue; + if (Left.SubRanges.empty() && Right.SubRanges.empty()) + return Left.MainRange.overlaps(Right.MainRange); + const auto LeftRanges = Left.SubRanges.empty() + ? std::span<const LiveSubRange>{} + : std::span<const LiveSubRange>(Left.SubRanges); + const auto RightRanges = Right.SubRanges.empty() + ? std::span<const LiveSubRange>{} + : std::span<const LiveSubRange>(Right.SubRanges); + if (LeftRanges.empty()) { + for (const LiveSubRange &RightRange : RightRanges) { + const LaneMask RightRangeLanes = normalizeUnitLanes(RightUnit.Lanes, RightRange.Lanes); + if (LeftUnitLanes.intersects(RightRangeLanes) && + Left.MainRange.overlaps(RightRange.Range)) + return true; + } + continue; + } + if (RightRanges.empty()) { + for (const LiveSubRange &LeftRange : LeftRanges) { + const LaneMask LeftRangeLanes = normalizeUnitLanes(LeftUnit.Lanes, LeftRange.Lanes); + if (LeftRangeLanes.intersects(RightUnitLanes) && + LeftRange.Range.overlaps(Right.MainRange)) + return true; + } + continue; + } + for (const LiveSubRange &LeftRange : LeftRanges) { + const LaneMask LeftRangeLanes = normalizeUnitLanes(LeftUnit.Lanes, LeftRange.Lanes); + if (LeftRangeLanes.empty()) + continue; + for (const LiveSubRange &RightRange : RightRanges) { + const LaneMask RightRangeLanes = normalizeUnitLanes(RightUnit.Lanes, RightRange.Lanes); + if (LeftRangeLanes.intersects(RightRangeLanes) && + LeftRange.Range.overlaps(RightRange.Range)) + return true; + } + } + } + } + return false; +} + +bool LiveRegMatrix::intervalOverlapsUnit(const LiveInterval &Interval, RegisterUnitLane Unit, + const LiveRange &UnitRange) const noexcept { + return rangeForUnitOverlaps(Interval, Unit, UnitRange); +} + +bool LiveRegMatrix::intervalsOverlapOnCandidates(const LiveInterval &Left, PhysicalRegister LeftReg, + const LiveInterval &Right, + PhysicalRegister RightReg) const noexcept { + return regalloc_detail::intervalsOverlapOnPhysicalRegisters(Left, LeftReg, Right, RightReg, + Target->registers()); +} + +std::vector<RegisterInterference> +LiveRegMatrix::collectInterferences(Register VirtualReg, PhysicalRegister PhysicalReg) const { + std::vector<RegisterInterference> Result; + const LiveInterval *Interval = Intervals->interval(VirtualReg); + const PhysicalRegisterDesc *Candidate = Target->registers().physicalRegister(PhysicalReg); + if (Interval == nullptr || Candidate == nullptr) + return Result; + + for (const MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const SlotIndex Position = Intervals->slot(*Instruction); + if (!Position.isValid()) + continue; + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical()) + continue; + const auto Fixed = Reg->physicalRegister(); + if (!Fixed) + continue; + const PhysicalRegister FixedReg = Fixed.value(); + if (!Target->registers().regsOverlap(FixedReg, PhysicalReg)) + continue; + const PhysicalRegisterDesc *FixedDescription = + Target->registers().physicalRegister(FixedReg); + if (FixedDescription == nullptr) + continue; + const SlotIndex OperandPosition = + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) + ? Position.earlyClobberSlot() + : Position.registerSlot(); + if (intervalLiveAtFixedRegister(*Interval, *Candidate, *FixedDescription, + OperandPosition)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::FixedRegister, + std::nullopt, FixedReg, std::nullopt, + Instruction->stableID()}); + } + } + } + } + + for (const RegMaskPosition &Mask : Intervals->regMaskPositions()) { + if (Mask.Mask && Mask.Mask->clobbers(PhysicalReg) && + Interval->MainRange.liveAt(Mask.Position)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::RegisterMask, std::nullopt, + std::nullopt, std::nullopt, Mask.Instruction}); + } + } + + for (const RegisterUnitLane Unit : Candidate->Units) { + const LiveRange *FixedRange = Intervals->regUnitRange(Unit.Unit); + if (FixedRange != nullptr && intervalOverlapsUnit(*Interval, Unit, *FixedRange)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::RegisterUnit, std::nullopt, + std::nullopt, Unit.Unit, std::nullopt}); + } + } + + for (const auto &[OtherReg, OtherPhysical] : OrderedAssignments) { + if (OtherReg == VirtualReg) + continue; + const LiveInterval *OtherInterval = Intervals->interval(OtherReg); + if (OtherInterval != nullptr && + intervalsOverlapOnCandidates(*Interval, PhysicalReg, *OtherInterval, OtherPhysical)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::VirtualRegister, OtherReg, + std::nullopt, std::nullopt, std::nullopt}); + } + } + + std::ranges::sort( + Result, [](const RegisterInterference &Left, const RegisterInterference &Right) { + return std::tuple{Left.Kind, Left.VirtualReg.value_or(Register{}).opaqueValue(), + Left.FixedReg.value_or(PhysicalRegister{}).number(), + Left.Unit ? Left.Unit->value() : 0U, + Left.Instruction ? Left.Instruction->value() : 0ULL} < + std::tuple{Right.Kind, Right.VirtualReg.value_or(Register{}).opaqueValue(), + Right.FixedReg.value_or(PhysicalRegister{}).number(), + Right.Unit ? Right.Unit->value() : 0U, + Right.Instruction ? Right.Instruction->value() : 0ULL}; + }); + Result.erase(std::unique(Result.begin(), Result.end(), + [](const RegisterInterference &Left, const RegisterInterference &Right) { + return Left.Kind == Right.Kind && + Left.VirtualReg == Right.VirtualReg && + Left.FixedReg == Right.FixedReg && Left.Unit == Right.Unit && + Left.Instruction == Right.Instruction; + }), + Result.end()); + return Result; +} + +RegisterInterference LiveRegMatrix::checkInterference(Register VirtualReg, + PhysicalRegister PhysicalReg) const noexcept { + const std::vector<RegisterInterference> Values = collectInterferences(VirtualReg, PhysicalReg); + if (Values.empty()) + return {}; + return Values.front(); +} + +Expected<void> LiveRegMatrix::assign(Register VirtualReg, PhysicalRegister PhysicalReg) { + if (!VirtualReg.isVirtual() || Intervals->interval(VirtualReg) == nullptr || + Target->registers().physicalRegister(PhysicalReg) == nullptr) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "LiveRegMatrix assignment requires a known interval and physical register"); + } + if (regalloc_detail::isUnavailablePhysicalRegister(*Function, Target->registers(), PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix cannot assign an unavailable physical register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + const auto ClassID = Info->get().RegClass; + if (ClassID) { + if (!Target->registers().registerClassContains(ClassID.value(), PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix assignment is outside the virtual register class"); + } + } + if (!regalloc_detail::candidateSupportsSubRegisters(*Function, Target->registers(), VirtualReg, + PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix assignment lacks a required subregister view"); + } + if (Assignments.contains(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register is already assigned"); + const RegisterInterference Interference = checkInterference(VirtualReg, PhysicalReg); + if (Interference.Kind != RegisterInterferenceKind::Free) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical register assignment has live interference"); + } + Assignments.emplace(VirtualReg, PhysicalReg); + OrderedAssignments.emplace_back(VirtualReg, PhysicalReg); + std::ranges::sort(OrderedAssignments, [](const auto &Left, const auto &Right) { + return Left.first.opaqueValue() < Right.first.opaqueValue(); + }); + return {}; +} + +Expected<void> LiveRegMatrix::unassign(Register VirtualReg) { + if (!Assignments.erase(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register is not assigned"); + std::erase_if(OrderedAssignments, + [VirtualReg](const auto &Entry) { return Entry.first == VirtualReg; }); + return {}; +} + +std::optional<PhysicalRegister> LiveRegMatrix::assignment(Register VirtualReg) const noexcept { + const auto Position = Assignments.find(VirtualReg); + if (Position == Assignments.end()) + return std::nullopt; + return Position->second; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineAnalyses.cpp b/backend/lib/CodeGen/MachineAnalyses.cpp index cbc46fd..33dc14f 100644 --- a/backend/lib/CodeGen/MachineAnalyses.cpp +++ b/backend/lib/CodeGen/MachineAnalyses.cpp @@ -12,6 +12,7 @@ #include <cstddef> #include <cstdint> #include <limits> +#include <map> #include <optional> #include <ranges> #include <span> @@ -296,7 +297,7 @@ Expected<MachineLoopAnalysis::Result> MachineLoopAnalysis::run(MachineFunction & Result ResultValue; std::vector<MachineBasicBlock *> Blocks = linkedBlocks(Function); const auto Order = blockOrder(Blocks); - std::unordered_map<MachineBasicBlock *, std::unordered_set<MachineBasicBlock *>> LoopBlocks; + std::map<MachineBasicBlock *, std::unordered_set<MachineBasicBlock *>> LoopBlocks; std::unordered_map<MachineBasicBlock *, std::vector<MachineBasicBlock *>> LoopLatches; for (MachineBasicBlock *Latch : Blocks) { diff --git a/backend/lib/CodeGen/MachineCompilationStatus.cpp b/backend/lib/CodeGen/MachineCompilationStatus.cpp index 600deac..1091b52 100644 --- a/backend/lib/CodeGen/MachineCompilationStatus.cpp +++ b/backend/lib/CodeGen/MachineCompilationStatus.cpp @@ -42,6 +42,7 @@ isKnownDiagnosticSeverity(MachineCompilationDiagnosticSeverity Severity) noexcep case ErrorCode::ParseError: case ErrorCode::VerificationError: case ErrorCode::Unsupported: + case ErrorCode::RegisterAllocationFailed: case ErrorCode::IoError: case ErrorCode::InternalError: return true; diff --git a/backend/lib/CodeGen/MachineFrameInfo.cpp b/backend/lib/CodeGen/MachineFrameInfo.cpp index 21a4b78..a6f7a77 100644 --- a/backend/lib/CodeGen/MachineFrameInfo.cpp +++ b/backend/lib/CodeGen/MachineFrameInfo.cpp @@ -66,8 +66,7 @@ parseMachineFramePointerPolicy(std::string_view Name) noexcept { return std::nullopt; } -std::string_view -getMachineFramePointerPolicyName(MachineFramePointerPolicy Policy) noexcept { +std::string_view getMachineFramePointerPolicyName(MachineFramePointerPolicy Policy) noexcept { switch (Policy) { case MachineFramePointerPolicy::None: return "none"; @@ -96,8 +95,8 @@ bool StackObject::isSpill() const noexcept { bool StackObject::isFixed() const noexcept { const StackObjectDesc *Desc = description(); - return Desc != nullptr && (Desc->Kind == StackObjectKind::FixedABI || - Desc->Kind == StackObjectKind::FixedSpillSlot); + return Desc != nullptr && + (Desc->Kind == StackObjectKind::FixedABI || Desc->Kind == StackObjectKind::FixedSpillSlot); } bool StackObject::isVariableSized() const noexcept { @@ -142,9 +141,8 @@ Expected<void> MachineFrameInfo::verifyDescription(const StackObjectDesc &Descri return makeUnexpected(ErrorCode::InvalidArgument, "callee-saved stack objects require a nonzero physical register"); } - if (Description.LocalOffset && - (Description.Kind == StackObjectKind::FixedABI || - Description.Kind == StackObjectKind::FixedSpillSlot)) { + if (Description.LocalOffset && (Description.Kind == StackObjectKind::FixedABI || + Description.Kind == StackObjectKind::FixedSpillSlot)) { return makeUnexpected(ErrorCode::InvalidArgument, "fixed ABI stack objects cannot have a local frame offset"); } @@ -196,8 +194,7 @@ Expected<void> MachineFrameInfo::verifyStateStructure(const MachineFrameState &S for (const PhysicalRegister Reg : Point.Registers) { if (Reg.number() == 0 || !Registers.insert(Reg).second) { return makeUnexpected(ErrorCode::InvalidArgument, - std::string(Kind) + - " point registers must be nonzero and unique"); + std::string(Kind) + " point registers must be nonzero and unique"); } } } diff --git a/backend/lib/CodeGen/MachineFunctionCloner.cpp b/backend/lib/CodeGen/MachineFunctionCloner.cpp index ced3b1a..0048194 100644 --- a/backend/lib/CodeGen/MachineFunctionCloner.cpp +++ b/backend/lib/CodeGen/MachineFunctionCloner.cpp @@ -526,8 +526,7 @@ cloneMemOperand(const MachineMemOperand &SourceOperand, CloneState &State) { } return {}; }; - auto SaveResult = - ClonePoints(State.Source.frameInfo().state().SavePoints, FrameState.SavePoints); + auto SaveResult = ClonePoints(State.Source.frameInfo().state().SavePoints, FrameState.SavePoints); if (!SaveResult) return SaveResult; auto RestoreResult = diff --git a/backend/lib/CodeGen/MachineJumpTableInfo.cpp b/backend/lib/CodeGen/MachineJumpTableInfo.cpp index 5ba0853..c832e7c 100644 --- a/backend/lib/CodeGen/MachineJumpTableInfo.cpp +++ b/backend/lib/CodeGen/MachineJumpTableInfo.cpp @@ -84,9 +84,8 @@ Expected<JumpTableIndex> MachineJumpTableInfo::reserveTable() { return Index; } -Expected<void> -MachineJumpTableInfo::defineReservedTable(JumpTableIndex Index, - MachineJumpTableDesc Description) { +Expected<void> MachineJumpTableInfo::defineReservedTable(JumpTableIndex Index, + MachineJumpTableDesc Description) { if (Frozen) { return makeUnexpected(ErrorCode::InvalidArgument, "frozen jump table info cannot define reserved tables"); diff --git a/backend/lib/CodeGen/MachineLiveness.cpp b/backend/lib/CodeGen/MachineLiveness.cpp index b3d88cd..327ae7d 100644 --- a/backend/lib/CodeGen/MachineLiveness.cpp +++ b/backend/lib/CodeGen/MachineLiveness.cpp @@ -1662,10 +1662,9 @@ LiveIntervalsAnalysis::run(MachineFunction &Function, const TargetDescription &T auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info) return std::unexpected(Info.error()); + // Preserve the target-provided hint order. MachineRegisterInfo already de-duplicates + // hints, and their order is the deterministic preference order consumed by allocators. Interval.AllocationHints = Info->get().AllocationHints; - std::ranges::sort(Interval.AllocationHints, [](PhysicalRegister Left, PhysicalRegister Right) { - return Left.number() < Right.number(); - }); std::uint64_t Weight = 1; for (const MachineOperand &Operand : Function.registerInfo().useOperands(Reg, false)) { diff --git a/backend/lib/CodeGen/MachinePass.cpp b/backend/lib/CodeGen/MachinePass.cpp index e6dccd0..ae69b1c 100644 --- a/backend/lib/CodeGen/MachinePass.cpp +++ b/backend/lib/CodeGen/MachinePass.cpp @@ -20,6 +20,7 @@ #include <memory> #include <optional> #include <ranges> +#include <set> #include <string> #include <string_view> #include <thread> @@ -164,8 +165,9 @@ MachineAnalysisManager::registerAnalysisImpl(MachineAnalysisRegistration Registr return makeUnexpected(ErrorCode::InvalidArgument, "machine analysis name is already registered"); } - Registry.emplace(Registration.ID, RegisteredAnalysis{std::move(Registration), std::move(Runner), - std::move(Invalidator)}); + const AnalysisID RegistrationID = Registration.ID; + Registry.emplace(RegistrationID, RegisteredAnalysis{std::move(Registration), std::move(Runner), + std::move(Invalidator)}); return {}; } @@ -287,7 +289,7 @@ void MachineAnalysisManager::invalidate(MachineFunction &Function, if (FunctionPosition == Cache.end() || Preserved.preservesAll() || Changes.empty()) return; - std::unordered_set<AnalysisID> Invalid; + std::set<AnalysisID> Invalid; for (auto &[ID, Entry] : FunctionPosition->second) { if (Preserved.preserves(ID)) continue; @@ -406,8 +408,9 @@ Expected<void> CompilationUnitAnalysisManager::registerAnalysisImpl( return makeUnexpected(ErrorCode::InvalidArgument, "compilation-unit analysis name is already registered"); } - Registry.emplace(Registration.ID, RegisteredAnalysis{std::move(Registration), std::move(Runner), - std::move(Invalidator)}); + const AnalysisID RegistrationID = Registration.ID; + Registry.emplace(RegistrationID, RegisteredAnalysis{std::move(Registration), std::move(Runner), + std::move(Invalidator)}); return {}; } @@ -524,7 +527,7 @@ void CompilationUnitAnalysisManager::invalidate(CompilationUnit &Unit, if (UnitPosition == Cache.end() || Preserved.preservesAll() || Changes.empty()) return; - std::unordered_set<AnalysisID> Invalid; + std::set<AnalysisID> Invalid; for (auto &[ID, Entry] : UnitPosition->second) { if (Preserved.preserves(ID)) continue; @@ -1238,10 +1241,10 @@ Expected<void> MachinePassManager::runFunctions(CompilationUnit &Unit, for (std::size_t Index = 0; Index < FunctionCount; ++Index) { if (Instrumentation != nullptr) Instrumentation->replayBufferedEvents(EventBuffers[Index]); - if (Failures[Index]) { - return std::unexpected( - prefixedError(*Failures[Index], - "function '" + std::string(Unit.functionAt(Index)->name()) + "' failed: ")); + const auto &Failure = Failures[Index]; + if (Failure) { + return std::unexpected(prefixedError( + *Failure, "function '" + std::string(Unit.functionAt(Index)->name()) + "' failed: ")); } } return {}; diff --git a/backend/lib/CodeGen/MachineRegisterInfo.cpp b/backend/lib/CodeGen/MachineRegisterInfo.cpp index 3784b22..7bd0ef3 100644 --- a/backend/lib/CodeGen/MachineRegisterInfo.cpp +++ b/backend/lib/CodeGen/MachineRegisterInfo.cpp @@ -229,6 +229,11 @@ Expected<void> MachineRegisterInfo::addLiveIn(PhysicalRegister PhysReg, return {}; } +void MachineRegisterInfo::clearLiveInVirtualRegisters() noexcept { + for (PhysicalRegisterLiveIn &LiveIn : LiveIns) + LiveIn.InitialVirtualRegister.reset(); +} + Expected<void> MachineRegisterInfo::reservePhysicalRegister(PhysicalRegister PhysReg) { if (!PhysReg.isValid()) { return makeUnexpected(ErrorCode::InvalidArgument, diff --git a/backend/lib/CodeGen/RegAllocBase.cpp b/backend/lib/CodeGen/RegAllocBase.cpp new file mode 100644 index 0000000..1aae22d --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBase.cpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocBase.h" + +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/Spiller.h" + +#include <algorithm> +#include <cstdint> +#include <unordered_set> +#include <utility> + +namespace rcc { + +std::vector<const LiveInterval *> +orderedAllocationIntervals(const LiveIntervalsAnalysis::Result &Intervals, + const MachineFunction &Function) { + std::vector<const LiveInterval *> Result; + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + continue; + const LiveInterval *Interval = Intervals.interval(*Reg); + if (Interval != nullptr && !Interval->MainRange.empty()) + Result.push_back(Interval); + } + std::ranges::sort(Result, [](const LiveInterval *Left, const LiveInterval *Right) { + const LiveSegment &LeftSegment = Left->MainRange.segments().front(); + const LiveSegment &RightSegment = Right->MainRange.segments().front(); + if (LeftSegment.Start != RightSegment.Start) + return LeftSegment.Start < RightSegment.Start; + if (LeftSegment.End != RightSegment.End) + return LeftSegment.End < RightSegment.End; + return Left->Reg.opaqueValue() < Right->Reg.opaqueValue(); + }); + return Result; +} + +Expected<regalloc_detail::AllocationRunResult> RegAllocBase::run() { + const std::uint64_t InitialFingerprint = regalloc_detail::allocationFingerprint(Function); + const std::size_t InitialFrameObjects = Function.frameInfo().liveObjectCount(); + VirtRegMap Mapping(Function); + auto Initialized = regalloc_detail::initializeMapping(Function, Mapping); + if (!Initialized) + return std::unexpected(Initialized.error()); + + std::unordered_set<std::uint64_t> SeenFingerprints; + while (true) { + const std::uint64_t Fingerprint = regalloc_detail::allocationFingerprint(Function); + if (!SeenFingerprints.insert(Fingerprint).second) { + return makeUnexpected(ErrorCode::InternalError, + "register allocation spill/rebuild cycle made no progress"); + } + auto Intervals = Analyses.getResult<LiveIntervalsAnalysis>(Function, Target); + if (!Intervals) + return std::unexpected(Intervals.error()); + auto Round = allocateRound(**Intervals, Mapping); + if (!Round) + return std::unexpected(Round.error()); + if (Round->Spilled.empty()) + break; + + std::ranges::sort(Round->Spilled, [](Register Left, Register Right) { + return Left.opaqueValue() < Right.opaqueValue(); + }); + Spiller ValueSpiller(Function, Target, TargetInfo, Mapping, Options.EnableRematerialization); + for (const Register VirtualReg : Round->Spilled) { + VirtRegMapEntry *Entry = Mapping.entry(VirtualReg); + if (Entry != nullptr) { + Entry->PhysicalReg.reset(); + Entry->State = VirtRegAllocationState::Unassigned; + } + auto NewRegisters = ValueSpiller.spill(VirtualReg); + if (!NewRegisters) + return std::unexpected(NewRegisters.error()); + } + Analyses.clear(Function); + } + + auto Rewritten = rewriteVirtualRegisters(Function, Target, Mapping); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + const std::uint64_t FinalFingerprint = regalloc_detail::allocationFingerprint(Function); + regalloc_detail::AllocationRunResult Result; + Result.Changed = InitialFingerprint != FinalFingerprint || *Rewritten; + Result.ChangedInstructions = InitialFingerprint != FinalFingerprint; + Result.ChangedFrame = InitialFrameObjects != Function.frameInfo().liveObjectCount(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocBase.h b/backend/lib/CodeGen/RegAllocBase.h new file mode 100644 index 0000000..5af5ed0 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBase.h @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCBASE_H +#define RCC_LIB_CODEGEN_REGALLOCBASE_H + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <vector> + +namespace rcc { + +struct AllocationRoundResult final { + std::vector<Register> Spilled; +}; + +class RegAllocBase { +public: + virtual ~RegAllocBase() = default; + + [[nodiscard]] Expected<regalloc_detail::AllocationRunResult> run(); + +protected: + RegAllocBase(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, MachineAnalysisManager &Analyses, + RegisterAllocationOptions Options) noexcept + : Function(Function), Target(Target), TargetInfo(TargetInfo), Analyses(Analyses), + Options(Options) {} + + [[nodiscard]] virtual Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) = 0; + + MachineFunction &Function; + const TargetDescription &Target; + const TargetRegisterAllocationInfo &TargetInfo; + MachineAnalysisManager &Analyses; + RegisterAllocationOptions Options; +}; + +[[nodiscard]] std::vector<const LiveInterval *> +orderedAllocationIntervals(const LiveIntervalsAnalysis::Result &Intervals, + const MachineFunction &Function); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_REGALLOCBASE_H diff --git a/backend/lib/CodeGen/RegAllocBasic.cpp b/backend/lib/CodeGen/RegAllocBasic.cpp new file mode 100644 index 0000000..4bb7b23 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBasic.cpp @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocBasic.h" + +#include "RegAllocBase.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/MachineLiveness.h" + +#include <algorithm> +#include <memory> +#include <ranges> + +namespace rcc { +namespace { + +class BasicAllocatorEngine final : public RegAllocBase { +public: + BasicAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineAnalysisManager &Analyses, RegisterAllocationOptions Options) + : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} + +private: + [[nodiscard]] Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) override { + LiveRegMatrix Matrix(Function, Target, Intervals); + AllocationRoundResult Result; + std::vector<const LiveInterval *> Worklist = orderedAllocationIntervals(Intervals, Function); + std::ranges::stable_sort(Worklist, [&](const LiveInterval *Left, const LiveInterval *Right) { + const bool LeftSpillable = Mapping.isSpillable(Left->Reg); + const bool RightSpillable = Mapping.isSpillable(Right->Reg); + if (LeftSpillable != RightSpillable) + return !LeftSpillable; + if (Left->SpillWeight != Right->SpillWeight) + return Left->SpillWeight > Right->SpillWeight; + return false; + }); + for (const LiveInterval *Interval : Worklist) { + const Register VirtualReg = Interval->Reg; + if (regalloc_detail::hasTerminatorDefinition(Function, Target, VirtualReg)) { + auto Marked = Mapping.setSpillable(VirtualReg, false); + if (!Marked) + return std::unexpected(Marked.error()); + } + auto Candidates = regalloc_detail::allocationOrder(Function, Target, VirtualReg); + if (!Candidates) + return std::unexpected(Candidates.error()); + bool Assigned = false; + for (const PhysicalRegister Candidate : *Candidates) { + if (Matrix.checkInterference(VirtualReg, Candidate).Kind != RegisterInterferenceKind::Free) + continue; + auto MatrixAssignment = Matrix.assign(VirtualReg, Candidate); + if (!MatrixAssignment) + continue; + auto MapAssignment = Mapping.assign(VirtualReg, Candidate); + if (!MapAssignment) + return std::unexpected(MapAssignment.error()); + Assigned = true; + break; + } + if (!Assigned) { + if (!Mapping.isSpillable(VirtualReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "BasicAllocator cannot assign a non-spillable virtual register"); + } + Result.Spilled.push_back(VirtualReg); + } + } + return Result; + } +}; + +[[nodiscard]] const MachinePassContract &basicContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "BasicAllocator", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + MachineFunctionProperty::TracksLiveness}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveIntervalsAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &BasicAllocatorPass::contract() const noexcept { + return basicContract(); +} + +Expected<MachinePassResult> BasicAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "BasicAllocator requires target behavior"); + auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + auto Trial = regalloc_detail::cloneForAllocation(Function, Target); + if (!Trial) + return std::unexpected(Trial.error()); + MachineAnalysisManager TrialAnalyses; + auto Registered = registerStage11MachineAnalyses(TrialAnalyses); + if (!Registered) + return std::unexpected(Registered.error()); + BasicAllocatorEngine TrialAllocator(*Trial->ClonedFunction, Target, *Info, TrialAnalyses, + Options); + auto TrialRun = TrialAllocator.run(); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialVerification = + regalloc_detail::verifyAllocationTrial(*Trial->ClonedFunction, Target, contract()); + if (!TrialVerification) + return std::unexpected(TrialVerification.error()); + BasicAllocatorEngine Allocator(Function, Target, *Info, Analyses, Options); + auto Allocated = Allocator.run(); + if (!Allocated) { + return std::unexpected(regalloc_detail::withPrefix( + Allocated.error(), "BasicAllocator diverged after a successful transaction preflight: ")); + } + return regalloc_detail::passResult(*Allocated); +} + +std::unique_ptr<MachineFunctionPass> +createBasicAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options) { + return std::make_unique<BasicAllocatorPass>(Info, Options); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocBasic.h b/backend/lib/CodeGen/RegAllocBasic.h new file mode 100644 index 0000000..8a68f5f --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBasic.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCBASIC_H +#define RCC_LIB_CODEGEN_REGALLOCBASIC_H + +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +namespace rcc { + +class BasicAllocatorPass final : public MachineFunctionPass { +public: + explicit BasicAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetRegisterAllocationInfo *Info; + RegisterAllocationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_REGALLOCBASIC_H diff --git a/backend/lib/CodeGen/RegAllocFast.cpp b/backend/lib/CodeGen/RegAllocFast.cpp new file mode 100644 index 0000000..9b087cc --- /dev/null +++ b/backend/lib/CodeGen/RegAllocFast.cpp @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegAllocFast.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/LiveRangeEdit.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/Spiller.h" + +#include <algorithm> +#include <cstddef> +#include <map> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +class FastAllocatorEngine final { +public: + FastAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + const LiveVariablesAnalysis::Result &Liveness) + : Function(Function), Target(Target), TargetInfo(TargetInfo), Liveness(Liveness), + Mapping(Function), Edit(Function, Target, Mapping) {} + + [[nodiscard]] Expected<regalloc_detail::AllocationRunResult> run() { + const std::uint64_t InitialFingerprint = regalloc_detail::allocationFingerprint(Function); + const std::size_t InitialFrameObjects = Function.frameInfo().liveObjectCount(); + auto Initialized = regalloc_detail::initializeMapping(Function, Mapping); + if (!Initialized) + return std::unexpected(Initialized.error()); + + bool IsEntry = true; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Allocated = allocateBlock(*Block, IsEntry); + if (!Allocated) + return std::unexpected(Allocated.error()); + IsEntry = false; + } + + Function.registerInfo().clearLiveInVirtualRegisters(); + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); + ++Index) { + auto Reg = regalloc_detail::virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Erased = Function.registerInfo().eraseVirtualRegister(*Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + + regalloc_detail::AllocationRunResult Result; + const std::uint64_t FinalFingerprint = regalloc_detail::allocationFingerprint(Function); + Result.Changed = InitialFingerprint != FinalFingerprint; + Result.ChangedInstructions = Result.Changed; + Result.ChangedFrame = InitialFrameObjects != Function.frameInfo().liveObjectCount(); + return Result; + } + +private: + struct Location final { + PhysicalRegister PhysicalReg; + bool Dirty = false; + }; + + struct InstructionRegister final { + Register VirtualReg; + std::vector<std::size_t> Operands; + bool HasUse = false; + bool HasDef = false; + std::size_t FirstOperand = 0; + }; + + struct BlockLiveInState final { + PhysicalRegister Reg; + std::size_t RemainingUses = 0; + }; + + [[nodiscard]] bool liveOut(const MachineBasicBlock &Block, Register Reg) const noexcept { + return Liveness.isLiveOut(Block, Reg); + } + + [[nodiscard]] bool neededLater(const MachineBasicBlock &Block, Register Reg) const noexcept { + const auto Position = RemainingUses.find(Reg); + return (Position != RemainingUses.end() && Position->second != 0) || liveOut(Block, Reg); + } + + [[nodiscard]] std::optional<Register> ownerOf(PhysicalRegister Reg) const noexcept { + for (const auto &[VirtualReg, Value] : Locations) { + if (Target.registers().regsOverlap(Value.PhysicalReg, Reg)) + return VirtualReg; + } + return std::nullopt; + } + + [[nodiscard]] bool physicalLiveOut(const MachineBasicBlock &Block, + PhysicalRegister Reg) const noexcept { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg); + if (Description == nullptr) + return true; + return std::ranges::any_of(Description->Units, [&](const RegisterUnitLane &Unit) { + return Liveness.isUnitLiveOut(Block, Unit.Unit, Unit.Lanes); + }); + } + + [[nodiscard]] bool blockedByBlockLiveIn(PhysicalRegister Reg) const noexcept { + return std::ranges::any_of(BlockLiveIns, [&](const BlockLiveInState &LiveIn) { + return Target.registers().regsOverlap(LiveIn.Reg, Reg); + }); + } + + void initializeBlockLiveIns(const MachineBasicBlock &Block) { + BlockLiveIns.clear(); + for (const MachineBasicBlockLiveIn &LiveIn : Block.liveIns()) { + if (LiveIn.Lanes.empty() || + std::ranges::any_of(BlockLiveIns, [&](const BlockLiveInState &Existing) { + return Existing.Reg == LiveIn.PhysReg; + })) + continue; + BlockLiveIns.push_back({LiveIn.PhysReg, 0}); + } + + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical() || !Operand.isUse()) + continue; + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue) + continue; + const PhysicalRegister Physical = PhysicalValue.value(); + for (BlockLiveInState &LiveIn : BlockLiveIns) { + if (Target.registers().regsOverlap(LiveIn.Reg, Physical)) + ++LiveIn.RemainingUses; + } + } + } + } + + void updateBlockLiveIns(const MachineBasicBlock &Block, const MachineInstr &Instruction) { + std::vector<PhysicalRegister> Defined; + std::vector<const RegisterMask *> Masks; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const Register *Reg = Operand.getIf<Register>(); Reg != nullptr && Reg->isPhysical()) { + const auto PhysicalValue = Reg->physicalRegister(); + if (Operand.isDef() && PhysicalValue) + Defined.push_back(PhysicalValue.value()); + } + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr && Mask->Mask) + Masks.push_back(Mask->Mask.get()); + } + + for (auto Position = BlockLiveIns.begin(); Position != BlockLiveIns.end();) { + BlockLiveInState &LiveIn = *Position; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical() || !Operand.isUse()) + continue; + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue || !Target.registers().regsOverlap(LiveIn.Reg, PhysicalValue.value())) + continue; + if (LiveIn.RemainingUses != 0) + --LiveIn.RemainingUses; + } + + const bool DefinedHere = std::ranges::any_of(Defined, [&](PhysicalRegister Reg) { + return Target.registers().regsOverlap(LiveIn.Reg, Reg); + }); + const bool MaskClobbered = std::ranges::any_of( + Masks, [&](const RegisterMask *Mask) { return Mask->clobbers(LiveIn.Reg); }); + const bool KeepForOutput = physicalLiveOut(Block, LiveIn.Reg); + if (!KeepForOutput && (DefinedHere || MaskClobbered || LiveIn.RemainingUses == 0)) + Position = BlockLiveIns.erase(Position); + else + ++Position; + } + } + + [[nodiscard]] bool overlapsAny(PhysicalRegister Candidate, + std::span<const PhysicalRegister> Values) const noexcept { + return std::ranges::any_of(Values, [&](PhysicalRegister Value) { + return Target.registers().regsOverlap(Candidate, Value); + }); + } + + [[nodiscard]] Expected<FrameIndex> spillSlot(Register Reg) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Validated = TargetInfo.validateSpillReload(*ClassID, Target); + if (!Validated) { + return makeUnexpected( + ErrorCode::InternalError, + "target declares an allocatable class without FastAllocator spill hooks"); + } + return Edit.getOrCreateSpillSlot(Reg); + } + + [[nodiscard]] Expected<void> emitSpillBefore(Register Reg, PhysicalRegister PhysicalReg, + MachineInstr *Before, MachineBasicBlock &Block) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Slot = spillSlot(Reg); + if (!Slot) + return std::unexpected(Slot.error()); + MachineIRBuilder Builder(Function, Target); + Expected<void> Positioned = + Before != nullptr ? Builder.setInsertPointBefore(*Before) : Builder.setInsertPoint(Block); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo.emitSpill(Builder, Register::fromPhysical(PhysicalReg), *ClassID, *Slot, + LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping.entry(Reg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + return {}; + } + + [[nodiscard]] Expected<void> evict(Register Reg, MachineInstr &Before, MachineBasicBlock &Block, + bool Preserve) { + const auto Position = Locations.find(Reg); + if (Position == Locations.end()) + return {}; + if (Position->second.Dirty && Preserve) { + auto Stored = emitSpillBefore(Reg, Position->second.PhysicalReg, &Before, Block); + if (!Stored) + return std::unexpected(Stored.error()); + } + Locations.erase(Position); + return {}; + } + + [[nodiscard]] Expected<PhysicalRegister> + ensureLocation(Register Reg, bool NeedsValue, MachineInstr &Instruction, MachineBasicBlock &Block, + std::span<const PhysicalRegister> Fixed, std::span<const PhysicalRegister> Locked, + const std::unordered_set<Register> &CurrentRegisters) { + const auto Existing = Locations.find(Reg); + if (Existing != Locations.end() && !overlapsAny(Existing->second.PhysicalReg, Fixed) && + !overlapsAny(Existing->second.PhysicalReg, Locked)) + return Existing->second.PhysicalReg; + if (Existing != Locations.end()) { + auto Evicted = evict(Reg, Instruction, Block, NeedsValue || neededLater(Block, Reg)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + + auto Candidates = regalloc_detail::allocationOrder(Function, Target, Reg); + if (!Candidates) + return std::unexpected(Candidates.error()); + for (const PhysicalRegister Candidate : *Candidates) { + if (overlapsAny(Candidate, Fixed) || overlapsAny(Candidate, Locked)) + continue; + if (blockedByBlockLiveIn(Candidate)) + continue; + if (const auto Owner = ownerOf(Candidate)) { + if (CurrentRegisters.contains(*Owner)) + continue; + auto Evicted = evict(*Owner, Instruction, Block, neededLater(Block, *Owner)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + if (ownerOf(Candidate)) + continue; + + if (NeedsValue) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Slot = spillSlot(Reg); + if (!Slot) + return std::unexpected(Slot.error()); + MachineIRBuilder Builder(Function, Target); + auto Positioned = Builder.setInsertPointBefore(Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Reload = TargetInfo.emitReload(Builder, Register::fromPhysical(Candidate), *ClassID, + *Slot, LaneMask::all()); + if (!Reload) + return std::unexpected(Reload.error()); + VirtRegMapEntry *Entry = Mapping.entry(Reg); + if (Entry != nullptr) + ++Entry->Spill.ReloadCount; + } + Locations.emplace(Reg, Location{Candidate, false}); + return Candidate; + } + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator cannot satisfy simultaneous register constraints"); + } + + [[nodiscard]] Expected<void> installEntryLiveIns(MachineBasicBlock &Block) { + if (&Block != Function.firstBlock()) + return {}; + for (const PhysicalRegisterLiveIn &LiveIn : Function.registerInfo().liveIns()) { + if (!LiveIn.InitialVirtualRegister) + continue; + const Register Reg = *LiveIn.InitialVirtualRegister; + if (!Function.registerInfo().isKnownVirtualRegister(Reg)) + continue; + auto Candidates = regalloc_detail::allocationOrder(Function, Target, Reg); + if (!Candidates) + return std::unexpected(Candidates.error()); + if (std::ranges::find(*Candidates, LiveIn.PhysReg) == Candidates->end()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "function live-in is outside its allocation order"); + } + if (ownerOf(LiveIn.PhysReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "overlapping function live-ins require the same physical register"); + } + Locations.emplace(Reg, Location{LiveIn.PhysReg, true}); + } + return {}; + } + + [[nodiscard]] Expected<void> allocateInstruction(MachineInstr &Instruction, + MachineBasicBlock &Block) { + std::vector<PhysicalRegister> Fixed; + std::vector<const RegisterMask *> Masks; + std::map<std::uint32_t, InstructionRegister> Registers; + std::vector<std::size_t> DebugOperands; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + if (const auto *Mask = Operand->getIf<RegisterMaskOperand>(); Mask != nullptr && Mask->Mask) + Masks.push_back(Mask->Mask.get()); + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isPhysical()) { + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue) { + return makeUnexpected(ErrorCode::InternalError, + "physical register operand lacks a physical register value"); + } + Fixed.push_back(PhysicalValue.value()); + continue; + } + if (Operand->isDebugUse()) { + DebugOperands.push_back(Index); + continue; + } + auto [Position, Inserted] = Registers.try_emplace( + Reg->opaqueValue(), InstructionRegister{*Reg, {}, false, false, Index}); + if (!Inserted) + Position->second.FirstOperand = std::min(Position->second.FirstOperand, Index); + Position->second.Operands.push_back(Index); + Position->second.HasUse |= Operand->isUse(); + Position->second.HasDef |= Operand->isDef(); + } + std::ranges::sort(Fixed, {}, &PhysicalRegister::number); + Fixed.erase(std::unique(Fixed.begin(), Fixed.end()), Fixed.end()); + + std::unordered_set<Register> CurrentRegisters; + std::unordered_set<Register> DefinedRegisters; + for (const auto &[Key, Value] : Registers) { + (void)Key; + CurrentRegisters.insert(Value.VirtualReg); + if (Value.HasDef) + DefinedRegisters.insert(Value.VirtualReg); + } + for (const auto &[Reg, Value] : + std::vector<std::pair<Register, Location>>(Locations.begin(), Locations.end())) { + bool Clobbered = false; + for (const RegisterMask *Mask : Masks) + Clobbered |= Mask->clobbers(Value.PhysicalReg); + if (Clobbered) { + auto Evicted = evict(Reg, Instruction, Block, neededLater(Block, Reg)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + } + + std::vector<InstructionRegister *> Ordered; + Ordered.reserve(Registers.size()); + for (auto &[Key, Value] : Registers) { + (void)Key; + Ordered.push_back(&Value); + } + std::ranges::sort(Ordered, + [](const InstructionRegister *Left, const InstructionRegister *Right) { + if (Left->HasUse != Right->HasUse) + return Left->HasUse > Right->HasUse; + return Left->FirstOperand < Right->FirstOperand; + }); + + std::vector<PhysicalRegister> Locked; + for (InstructionRegister *Value : Ordered) { + auto Physical = ensureLocation(Value->VirtualReg, Value->HasUse, Instruction, Block, Fixed, + Locked, CurrentRegisters); + if (!Physical) + return std::unexpected(Physical.error()); + Locked.push_back(*Physical); + for (const std::size_t Index : Value->Operands) { + MachineOperand *Operand = Instruction.operandAt(Index); + PhysicalRegister Effective = *Physical; + if (Operand->subReg().value() != 0) { + const auto Sub = Target.registers().subRegister(*Physical, Operand->subReg()); + if (!Sub) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator assignment lacks a required subregister"); + Effective = *Sub; + } + auto Set = Operand->setRegister(Register::fromPhysical(Effective)); + if (!Set) + return std::unexpected(Set.error()); + if (Operand->subReg().value() != 0) { + auto Cleared = Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + } + if (Value->HasDef) + Locations[Value->VirtualReg].Dirty = true; + } + + for (const std::size_t Index : DebugOperands) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand->getIf<Register>(); + const auto Position = Reg == nullptr ? Locations.end() : Locations.find(*Reg); + Register Replacement; + if (Position != Locations.end()) + Replacement = Register::fromPhysical(Position->second.PhysicalReg); + auto Set = Operand->setRegister(Replacement); + if (!Set) + return std::unexpected(Set.error()); + if (Operand->subReg().value() != 0) { + auto Cleared = Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Replacement.isNoRegister()) { + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto Updated = Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + + for (InstructionRegister *Value : Ordered) { + if (Value->HasUse) { + auto Position = RemainingUses.find(Value->VirtualReg); + if (Position != RemainingUses.end() && Position->second != 0) + --Position->second; + } + if (!neededLater(Block, Value->VirtualReg)) + Locations.erase(Value->VirtualReg); + } + for (const RegisterMask *Mask : Masks) { + for (auto Position = Locations.begin(); Position != Locations.end();) { + if (Mask->clobbers(Position->second.PhysicalReg) && + !DefinedRegisters.contains(Position->first)) + Position = Locations.erase(Position); + else + ++Position; + } + } + return {}; + } + + [[nodiscard]] Expected<void> flushLiveOuts(MachineBasicBlock &Block) { + MachineInstr *Terminator = regalloc_detail::firstTerminator(Block, Target); + for (const auto &[Reg, Value] : + std::vector<std::pair<Register, Location>>(Locations.begin(), Locations.end())) { + if (!Value.Dirty || !liveOut(Block, Reg)) + continue; + if (Terminator != nullptr) { + for (const MachineOperand &Operand : Terminator->operands()) { + const Register *OperandReg = Operand.getIf<Register>(); + if (OperandReg != nullptr && *OperandReg == Reg && Operand.isDef()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator cannot spill a terminator-defined live-out"); + } + } + } + auto Stored = emitSpillBefore(Reg, Value.PhysicalReg, Terminator, Block); + if (!Stored) + return std::unexpected(Stored.error()); + } + Locations.clear(); + return {}; + } + + [[nodiscard]] Expected<void> allocateBlock(MachineBasicBlock &Block, bool IsEntry) { + (void)IsEntry; + Locations.clear(); + RemainingUses.clear(); + initializeBlockLiveIns(Block); + std::vector<MachineInstr *> Instructions; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Instructions.push_back(Instruction); + std::unordered_set<Register> UsesInInstruction; + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Reg->isVirtual() && Operand.isUse() && !Operand.isDebugUse()) + UsesInInstruction.insert(*Reg); + } + for (const Register Reg : UsesInInstruction) + ++RemainingUses[Reg]; + } + auto EntryValues = installEntryLiveIns(Block); + if (!EntryValues) + return std::unexpected(EntryValues.error()); + for (MachineInstr *Instruction : Instructions) { + auto Allocated = allocateInstruction(*Instruction, Block); + if (!Allocated) + return std::unexpected(Allocated.error()); + updateBlockLiveIns(Block, *Instruction); + } + auto Flushed = flushLiveOuts(Block); + BlockLiveIns.clear(); + return Flushed; + } + + MachineFunction &Function; + const TargetDescription &Target; + const TargetRegisterAllocationInfo &TargetInfo; + const LiveVariablesAnalysis::Result &Liveness; + VirtRegMap Mapping; + LiveRangeEdit Edit; + std::unordered_map<Register, Location> Locations; + std::unordered_map<Register, std::size_t> RemainingUses; + std::vector<BlockLiveInState> BlockLiveIns; +}; + +} // namespace + +const MachinePassContract &FastAllocatorPass::contract() const noexcept { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "FastAllocator", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + MachineFunctionProperty::TracksLiveness}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +Expected<MachinePassResult> FastAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "FastAllocator requires target behavior"); + auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + auto Trial = regalloc_detail::cloneForAllocation(Function, Target); + if (!Trial) + return std::unexpected(Trial.error()); + MachineAnalysisManager TrialAnalyses; + auto Registered = registerStage11MachineAnalyses(TrialAnalyses); + if (!Registered) + return std::unexpected(Registered.error()); + auto TrialLiveness = + TrialAnalyses.getResult<LiveVariablesAnalysis>(*Trial->ClonedFunction, Target); + if (!TrialLiveness) + return std::unexpected(TrialLiveness.error()); + FastAllocatorEngine TrialAllocator(*Trial->ClonedFunction, Target, *Info, **TrialLiveness); + auto TrialRun = TrialAllocator.run(); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialVerification = + regalloc_detail::verifyAllocationTrial(*Trial->ClonedFunction, Target, contract()); + if (!TrialVerification) + return std::unexpected(TrialVerification.error()); + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + FastAllocatorEngine Allocator(Function, Target, *Info, **Liveness); + auto Allocated = Allocator.run(); + if (!Allocated) { + return std::unexpected(regalloc_detail::withPrefix( + Allocated.error(), "FastAllocator diverged after a successful transaction preflight: ")); + } + return regalloc_detail::passResult(*Allocated); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocInternal.h b/backend/lib/CodeGen/RegAllocInternal.h new file mode 100644 index 0000000..5d0f434 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocInternal.h @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCINTERNAL_H +#define RCC_LIB_CODEGEN_REGALLOCINTERNAL_H + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace rcc::regalloc_detail { + +struct AllocationRunResult final { + bool Changed = false; + bool ChangedInstructions = false; + bool ChangedFrame = false; +}; + +[[nodiscard]] inline Error withPrefix(const Error &Failure, std::string_view Prefix) { + std::string Message(Prefix); + Message.append(Failure.message()); + return Error(Failure.code(), std::move(Message)); +} + +[[nodiscard]] inline Expected<Register> virtualRegisterAt(std::size_t Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] inline Expected<RegisterClassID> registerClassFor(const MachineFunction &Function, + Register VirtualReg) { + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + const auto ClassID = Info->get().RegClass; + if (!ClassID) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "virtual register has no register class after instruction selection"); + } + return ClassID.value(); +} + +[[nodiscard]] inline bool isUnavailablePhysicalRegister(const MachineFunction &Function, + const TargetRegisterInfo &Registers, + PhysicalRegister Reg) noexcept { + const PhysicalRegisterDesc *Description = Registers.physicalRegister(Reg); + return Description == nullptr || Description->IsConstant || Description->IsArtificial || + Description->IsReserved || Registers.isReservedRegister(Reg) || + Function.registerInfo().isReserved(Reg); +} + +[[nodiscard]] inline bool candidateSupportsSubRegisters(const MachineFunction &Function, + const TargetRegisterInfo &Registers, + Register VirtualReg, + PhysicalRegister Candidate) noexcept { + for (const MachineOperand &Operand : + const_cast<MachineRegisterInfo &>(Function.registerInfo()).regOperands(VirtualReg)) { + if (Operand.subReg().value() != 0 && !Registers.subRegister(Candidate, Operand.subReg())) + return false; + } + return true; +} + +[[nodiscard]] inline Expected<std::vector<PhysicalRegister>> +allocationOrder(const MachineFunction &Function, const TargetDescription &Target, + Register VirtualReg) { + auto ClassID = registerClassFor(Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + const RegisterClassDesc *Class = Target.registers().registerClass(*ClassID); + if (Class == nullptr || !Class->Allocatable || Class->AllocationOrder.empty()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "virtual register class has no allocatable physical registers"); + } + + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + std::vector<PhysicalRegister> Result; + Result.reserve(Class->AllocationOrder.size()); + for (const PhysicalRegister Hint : Info->get().AllocationHints) { + if (std::ranges::find(Class->AllocationOrder, Hint) != Class->AllocationOrder.end() && + std::ranges::find(Result, Hint) == Result.end()) + Result.push_back(Hint); + } + for (const PhysicalRegister Candidate : Class->AllocationOrder) { + if (std::ranges::find(Result, Candidate) == Result.end()) + Result.push_back(Candidate); + } + std::erase_if(Result, [&](PhysicalRegister Candidate) { + return isUnavailablePhysicalRegister(Function, Target.registers(), Candidate) || + !candidateSupportsSubRegisters(Function, Target.registers(), VirtualReg, Candidate); + }); + return Result; +} + +[[nodiscard]] inline MachineInstr *firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) noexcept { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] inline bool hasTerminatorDefinition(const MachineFunction &Function, + const TargetDescription &Target, + Register VirtualReg) noexcept { + for (const MachineOperand &Operand : + const_cast<MachineRegisterInfo &>(Function.registerInfo()).defOperands(VirtualReg)) { + const MachineInstr *Instruction = Operand.parent(); + const InstructionDescriptor *Descriptor = + Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) + return true; + } + return false; +} + +[[nodiscard]] inline std::uint64_t allocationFingerprint(const MachineFunction &Function) noexcept { + std::uint64_t Hash = 1469598103934665603ULL; + auto Mix = [&Hash](std::uint64_t Value) { + Hash ^= Value; + Hash *= 1099511628211ULL; + }; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Mix(Block->stableID().value()); + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Mix(Instruction->opcode().value()); + Mix(Instruction->stableID().value()); + for (const MachineOperand &Operand : Instruction->operands()) { + Mix(static_cast<std::uint64_t>(Operand.kind())); + if (const Register *Reg = Operand.getIf<Register>()) + Mix(Reg->opaqueValue()); + } + } + } + return Hash; +} + +[[nodiscard]] inline Expected<void> initializeMapping(MachineFunction &Function, + VirtRegMap &Mapping) { + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Added = Mapping.add(*Reg); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] inline MachinePassResult passResult(const AllocationRunResult &Run) { + MachinePassResult Result; + Result.Changed = Run.Changed; + if (Run.Changed) { + Result.Changes.add(MachinePassChange::Registers); + Result.Changes.add(MachinePassChange::Operands); + } + if (Run.ChangedInstructions) + Result.Changes.add(MachinePassChange::Instructions); + if (Run.ChangedFrame) + Result.Changes.add(MachinePassChange::Frame); + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +[[nodiscard]] inline Expected<void> +requireAllocationProperties(const MachineFunction &Function, const MachinePassContract &Contract) { + const MachineFunctionProperties Missing = Contract.Requires.missingFrom(Function.properties()); + if (!Missing.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + Contract.Name + " requires properties: " + Missing.toString()); + } + return {}; +} + +[[nodiscard]] inline Expected<void> verifyAllocationTrial(MachineFunction &Function, + const TargetDescription &Target, + const MachinePassContract &Contract) { + auto Transition = + Function.properties().applyTransition(Contract.Requires, Contract.Sets, Contract.Clears); + if (!Transition) + return std::unexpected(Transition.error()); + return MachineVerifier::verify(Function, Target, + MachineVerifierOptions{.Mode = MachineVerifierMode::Generic, + .Stage = Contract.VerificationStage, + .VerifyProperties = true}); +} + +[[nodiscard]] inline Expected<MachineFunctionCloneResult> +cloneForAllocation(const MachineFunction &Function, const TargetDescription &Target) { + return cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); +} + +[[nodiscard]] bool +intervalsOverlapOnPhysicalRegisters(const LiveInterval &Left, PhysicalRegister LeftReg, + const LiveInterval &Right, PhysicalRegister RightReg, + const TargetRegisterInfo &Registers) noexcept; + +} // namespace rcc::regalloc_detail + +#endif // RCC_LIB_CODEGEN_REGALLOCINTERNAL_H diff --git a/backend/lib/CodeGen/RegAllocPBQP.cpp b/backend/lib/CodeGen/RegAllocPBQP.cpp new file mode 100644 index 0000000..5e56728 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocPBQP.cpp @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegAllocPBQP.h" + +#include "RegAllocBase.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/PBQP/Solver.h" + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct PBQPAllocationNode final { + Register VirtualReg; + pbqp::NodeID Node; + std::vector<std::optional<PhysicalRegister>> Options; +}; + +[[nodiscard]] std::set<std::pair<std::uint32_t, std::uint32_t>> +copyRelatedRegisters(const MachineFunction &Function) { + std::set<std::pair<std::uint32_t, std::uint32_t>> Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->opcode() != opcode(GenericOpcode::COPY) || Instruction->operandCount() < 2) + continue; + const Register *Destination = Instruction->operands()[0].getIf<Register>(); + const Register *Source = Instruction->operands()[1].getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || *Destination == *Source) + continue; + std::uint32_t Left = Destination->opaqueValue(); + std::uint32_t Right = Source->opaqueValue(); + if (Right < Left) + std::swap(Left, Right); + Result.emplace(Left, Right); + } + } + return Result; +} + +[[nodiscard]] pbqp::Cost boundedCost(std::uint64_t Value) noexcept { + const std::uint64_t Limit = static_cast<std::uint64_t>(pbqp::InfiniteCost - 1); + return static_cast<pbqp::Cost>(std::min(Value, Limit)); +} + +class PBQPAllocatorEngine final : public RegAllocBase { +public: + PBQPAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineAnalysisManager &Analyses, RegisterAllocationOptions Options) + : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} + +private: + [[nodiscard]] Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) override { + LiveRegMatrix FixedMatrix(Function, Target, Intervals); + pbqp::Graph Problem; + std::vector<PBQPAllocationNode> Nodes; + for (const LiveInterval *Interval : orderedAllocationIntervals(Intervals, Function)) { + const Register VirtualReg = Interval->Reg; + const bool TerminatorDefined = + regalloc_detail::hasTerminatorDefinition(Function, Target, VirtualReg); + if (TerminatorDefined) { + auto Marked = Mapping.setSpillable(VirtualReg, false); + if (!Marked) + return std::unexpected(Marked.error()); + } + auto ClassID = regalloc_detail::registerClassFor(Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Candidates = regalloc_detail::allocationOrder(Function, Target, VirtualReg); + if (!Candidates) + return std::unexpected(Candidates.error()); + std::erase_if(*Candidates, [&](PhysicalRegister Candidate) { + return FixedMatrix.checkInterference(VirtualReg, Candidate).Kind != + RegisterInterferenceKind::Free; + }); + + std::vector<std::optional<PhysicalRegister>> NodeOptions; + NodeOptions.reserve(Candidates->size() + 1U); + NodeOptions.push_back(std::nullopt); + for (const PhysicalRegister Candidate : *Candidates) + NodeOptions.push_back(Candidate); + pbqp::CostVector Costs(NodeOptions.size(), 0); + if (!Mapping.isSpillable(VirtualReg)) { + Costs[0] = pbqp::InfiniteCost; + } else { + const std::uint64_t TargetSpillCost = TargetInfo.spillCost(Function, *Interval, *ClassID); + const std::uint64_t Weight = std::max<std::uint64_t>(1, Interval->SpillWeight); + const std::uint64_t Scaled = + TargetSpillCost > std::numeric_limits<std::uint64_t>::max() / Weight + ? std::numeric_limits<std::uint64_t>::max() + : TargetSpillCost * Weight; + const std::uint64_t Adjusted = + Scaled == std::numeric_limits<std::uint64_t>::max() ? Scaled : Scaled + 1U; + Costs[0] = boundedCost(Adjusted); + } + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + for (std::size_t Index = 1; Index < NodeOptions.size(); ++Index) { + const auto CandidateOption = NodeOptions[Index]; + if (!CandidateOption) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP allocation node has an unexpected spill option"); + } + const PhysicalRegister Candidate = CandidateOption.value(); + std::uint64_t Cost = + TargetInfo.physicalRegisterCost(Function, VirtualReg, Candidate, *ClassID, true); + const auto Hint = std::ranges::find(Info->get().AllocationHints, Candidate); + const std::uint64_t HintCost = + Hint == Info->get().AllocationHints.end() + ? 2U + : static_cast<std::uint64_t>(Hint - Info->get().AllocationHints.begin()); + Cost = Cost > std::numeric_limits<std::uint64_t>::max() - HintCost + ? std::numeric_limits<std::uint64_t>::max() + : Cost + HintCost; + Costs[Index] = boundedCost(Cost); + } + auto NodeID = Problem.addNode(std::move(Costs), VirtualReg.opaqueValue()); + if (!NodeID) + return std::unexpected(NodeID.error()); + Nodes.push_back(PBQPAllocationNode{VirtualReg, *NodeID, std::move(NodeOptions)}); + } + + const auto CopyPairs = copyRelatedRegisters(Function); + for (std::size_t LeftIndex = 0; LeftIndex < Nodes.size(); ++LeftIndex) { + const PBQPAllocationNode &Left = Nodes[LeftIndex]; + const LiveInterval *LeftInterval = Intervals.interval(Left.VirtualReg); + for (std::size_t RightIndex = LeftIndex + 1U; RightIndex < Nodes.size(); ++RightIndex) { + const PBQPAllocationNode &Right = Nodes[RightIndex]; + const LiveInterval *RightInterval = Intervals.interval(Right.VirtualReg); + if (LeftInterval == nullptr || RightInterval == nullptr) + continue; + std::uint32_t LeftKey = Left.VirtualReg.opaqueValue(); + std::uint32_t RightKey = Right.VirtualReg.opaqueValue(); + if (RightKey < LeftKey) + std::swap(LeftKey, RightKey); + const bool CopyRelated = + Options.EnableCopyCoalescing && CopyPairs.contains({LeftKey, RightKey}); + pbqp::CostMatrix Costs(Left.Options.size(), Right.Options.size(), 0); + bool HasCost = false; + for (std::size_t LeftOption = 1; LeftOption < Left.Options.size(); ++LeftOption) { + for (std::size_t RightOption = 1; RightOption < Right.Options.size(); ++RightOption) { + const auto LeftRegister = Left.Options[LeftOption]; + const auto RightRegister = Right.Options[RightOption]; + if (!LeftRegister || !RightRegister) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP edge has an unexpected spill option"); + } + const PhysicalRegister LeftReg = LeftRegister.value(); + const PhysicalRegister RightReg = RightRegister.value(); + if (regalloc_detail::intervalsOverlapOnPhysicalRegisters( + *LeftInterval, LeftReg, *RightInterval, RightReg, Target.registers())) { + Costs.at(LeftOption, RightOption) = pbqp::InfiniteCost; + HasCost = true; + } else if (CopyRelated && LeftReg == RightReg) { + Costs.at(LeftOption, RightOption) = -1; + HasCost = true; + } + } + } + if (!HasCost) + continue; + const std::uint64_t StableKey = + (static_cast<std::uint64_t>(Left.VirtualReg.opaqueValue()) << 32U) | + Right.VirtualReg.opaqueValue(); + auto Edge = Problem.addEdge(Left.Node, Right.Node, std::move(Costs), StableKey); + if (!Edge) + return std::unexpected(Edge.error()); + } + } + + pbqp::DeterministicSolver Solver; + auto Solution = Solver.solve(Problem); + if (!Solution) + return std::unexpected(Solution.error()); + + LiveRegMatrix AssignedMatrix(Function, Target, Intervals); + AllocationRoundResult Result; + for (const PBQPAllocationNode &Node : Nodes) { + const auto Selection = Solution->selection(Node.Node); + if (!Selection) + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver returned an invalid selection"); + const std::size_t SelectedOption = Selection.value(); + if (SelectedOption >= Node.Options.size()) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver selected an out-of-range option"); + } + const auto AssignedOption = Node.Options[SelectedOption]; + if (!AssignedOption) { + Result.Spilled.push_back(Node.VirtualReg); + continue; + } + const PhysicalRegister Assigned = AssignedOption.value(); + auto MatrixAssignment = AssignedMatrix.assign(Node.VirtualReg, Assigned); + if (!MatrixAssignment) + return makeUnexpected(ErrorCode::InternalError, + "PBQP solution violates its encoded interference constraints"); + auto MapAssignment = Mapping.assign(Node.VirtualReg, Assigned); + if (!MapAssignment) + return std::unexpected(MapAssignment.error()); + } + return Result; + } +}; + +[[nodiscard]] const MachinePassContract &pbqpContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "PBQPAllocator", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + MachineFunctionProperty::TracksLiveness}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {}, + .RequiredAnalyses = {MachineAnalysisManager::id<LiveIntervalsAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &PBQPAllocatorPass::contract() const noexcept { + return pbqpContract(); +} + +Expected<MachinePassResult> PBQPAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "PBQPAllocator requires target behavior"); + auto Required = regalloc_detail::requireAllocationProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + auto Trial = regalloc_detail::cloneForAllocation(Function, Target); + if (!Trial) + return std::unexpected(Trial.error()); + MachineAnalysisManager TrialAnalyses; + auto Registered = registerStage11MachineAnalyses(TrialAnalyses); + if (!Registered) + return std::unexpected(Registered.error()); + PBQPAllocatorEngine TrialAllocator(*Trial->ClonedFunction, Target, *Info, TrialAnalyses, Options); + auto TrialRun = TrialAllocator.run(); + if (!TrialRun) + return std::unexpected(TrialRun.error()); + auto TrialVerification = + regalloc_detail::verifyAllocationTrial(*Trial->ClonedFunction, Target, contract()); + if (!TrialVerification) + return std::unexpected(TrialVerification.error()); + PBQPAllocatorEngine Allocator(Function, Target, *Info, Analyses, Options); + auto Allocated = Allocator.run(); + if (!Allocated) { + return std::unexpected(regalloc_detail::withPrefix( + Allocated.error(), "PBQPAllocator diverged after a successful transaction preflight: ")); + } + return regalloc_detail::passResult(*Allocated); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetMIRTextInfo.cpp b/backend/lib/CodeGen/TargetMIRTextInfo.cpp index 65e3c65..5d54ea6 100644 --- a/backend/lib/CodeGen/TargetMIRTextInfo.cpp +++ b/backend/lib/CodeGen/TargetMIRTextInfo.cpp @@ -509,8 +509,7 @@ TargetMIRTextInfo::printFunctionState(const TargetFunctionState &State) const { return Node; } -Expected<TargetConstant> -TargetMIRTextInfo::parseTargetConstant(std::string_view Text) const { +Expected<TargetConstant> TargetMIRTextInfo::parseTargetConstant(std::string_view Text) const { if (Description.ParseTargetConstant == nullptr) { return makeUnexpected(ErrorCode::Unsupported, "target does not define a target constant MIR schema"); @@ -524,8 +523,7 @@ TargetMIRTextInfo::parseTargetConstant(std::string_view Text) const { return Constant; } -Expected<std::string> -TargetMIRTextInfo::printTargetConstant(const TargetConstant &Constant) const { +Expected<std::string> TargetMIRTextInfo::printTargetConstant(const TargetConstant &Constant) const { if (Description.PrintTargetConstant == nullptr) { return makeUnexpected(ErrorCode::Unsupported, "target does not define a target constant MIR schema"); diff --git a/backend/lib/CodeGen/VirtRegMap.cpp b/backend/lib/CodeGen/VirtRegMap.cpp new file mode 100644 index 0000000..624a931 --- /dev/null +++ b/backend/lib/CodeGen/VirtRegMap.cpp @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/VirtRegMap.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <utility> +#include <vector> + +namespace rcc { + +Expected<void> VirtRegMap::add(Register VirtualReg, bool Spillable, + std::optional<Register> SplitParent, + std::optional<Register> Origin) { + if (!VirtualReg.isVirtual() || !Function->registerInfo().isKnownVirtualRegister(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, + "VirtRegMap requires a known virtual register"); + if (SplitParent && !SplitParent->isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, "split parent must be a virtual register"); + if (Origin && !Origin->isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, "split origin must be a virtual register"); + auto [Position, Inserted] = + Entries.try_emplace(VirtualReg, VirtRegMapEntry{VirtualReg, + VirtRegAllocationState::Unassigned, + std::nullopt, + SplitParent, + Origin, + Spillable, + {}}); + if (!Inserted) { + Position->second.Spillable = Spillable; + if (SplitParent) + Position->second.SplitParent = SplitParent; + if (Origin) + Position->second.Origin = Origin; + } + return {}; +} + +Expected<void> VirtRegMap::assign(Register VirtualReg, PhysicalRegister PhysicalReg) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || !PhysicalReg.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap assignment is invalid"); + Value->PhysicalReg = PhysicalReg; + Value->State = VirtRegAllocationState::Assigned; + return {}; +} + +Expected<void> VirtRegMap::assignStackSlot(Register VirtualReg, FrameIndex StackSlot, + LaneMask Lanes) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || Function->frameInfo().object(StackSlot) == nullptr || Lanes.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap spill assignment is invalid"); + Value->PhysicalReg.reset(); + Value->State = VirtRegAllocationState::Spilled; + Value->Spill.StackSlot = StackSlot; + Value->Spill.Lanes = Lanes; + return {}; +} + +Expected<void> VirtRegMap::markSplit(Register VirtualReg, Register Parent, Register Origin) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || !Parent.isVirtual() || !Origin.isVirtual() || Parent == VirtualReg || + Origin == VirtualReg || !Function->registerInfo().isKnownVirtualRegister(Parent) || + !Function->registerInfo().isKnownVirtualRegister(Origin)) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap split relationship is invalid"); + Value->State = VirtRegAllocationState::Split; + Value->SplitParent = Parent; + Value->Origin = Origin; + return {}; +} + +Expected<void> VirtRegMap::setSpillable(Register VirtualReg, bool Spillable) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "unknown virtual register in VirtRegMap"); + Value->Spillable = Spillable; + return {}; +} + +VirtRegMapEntry *VirtRegMap::entry(Register VirtualReg) noexcept { + const auto Position = Entries.find(VirtualReg); + return Position == Entries.end() ? nullptr : &Position->second; +} + +const VirtRegMapEntry *VirtRegMap::entry(Register VirtualReg) const noexcept { + const auto Position = Entries.find(VirtualReg); + return Position == Entries.end() ? nullptr : &Position->second; +} + +std::optional<PhysicalRegister> VirtRegMap::physicalRegister(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value == nullptr ? std::nullopt : Value->PhysicalReg; +} + +std::optional<FrameIndex> VirtRegMap::stackSlot(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value == nullptr ? std::nullopt : Value->Spill.StackSlot; +} + +bool VirtRegMap::isSpillable(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value != nullptr && Value->Spillable; +} + +std::vector<const VirtRegMapEntry *> VirtRegMap::orderedEntries() const { + std::vector<const VirtRegMapEntry *> Result; + Result.reserve(Entries.size()); + for (const auto &[Reg, Value] : Entries) { + (void)Reg; + Result.push_back(&Value); + } + std::ranges::sort(Result, [](const VirtRegMapEntry *Left, const VirtRegMapEntry *Right) { + return Left->VirtualReg.opaqueValue() < Right->VirtualReg.opaqueValue(); + }); + return Result; +} + +Expected<void> VirtRegMap::verify() const { + if (Function == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has no MachineFunction"); + for (const VirtRegMapEntry *Value : orderedEntries()) { + if (Value == nullptr || !Value->VirtualReg.isVirtual() || + !Function->registerInfo().isKnownVirtualRegister(Value->VirtualReg)) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap contains an invalid entry"); + if (Value->State == VirtRegAllocationState::Assigned && + (!Value->PhysicalReg || !Value->PhysicalReg->isValid())) + return makeUnexpected(ErrorCode::VerificationError, + "assigned VirtRegMap entry has no register"); + if (Value->State == VirtRegAllocationState::Spilled && !Value->Spill.StackSlot) + return makeUnexpected(ErrorCode::VerificationError, + "spilled VirtRegMap entry has no stack slot"); + if (Value->Spill.StackSlot && Value->Spill.Lanes.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "VirtRegMap spill metadata has an empty lane mask"); + if (Value->SplitParent && + (!Value->SplitParent->isVirtual() || *Value->SplitParent == Value->VirtualReg || + !Function->registerInfo().isKnownVirtualRegister(*Value->SplitParent))) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has an invalid split parent"); + if (Value->Origin && (!Value->Origin->isVirtual() || *Value->Origin == Value->VirtualReg || + !Function->registerInfo().isKnownVirtualRegister(*Value->Origin))) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has an invalid split origin"); + if (Value->State == VirtRegAllocationState::Split && (!Value->SplitParent || !Value->Origin)) + return makeUnexpected(ErrorCode::VerificationError, + "split VirtRegMap entry lacks parent or origin metadata"); + } + return {}; +} + +Expected<bool> rewriteVirtualRegisters(MachineFunction &Function, const TargetDescription &Target, + const VirtRegMap &Mapping) { + if (&Mapping.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "VirtRegMap belongs to a different MachineFunction"); + auto MappingVerification = Mapping.verify(); + if (!MappingVerification) + return std::unexpected(MappingVerification.error()); + + MachineAnalysisManager ValidationAnalyses; + auto Registered = registerStage11MachineAnalyses(ValidationAnalyses); + if (!Registered) + return std::unexpected(Registered.error()); + auto Intervals = ValidationAnalyses.getResult<LiveIntervalsAnalysis>(Function, Target); + if (!Intervals) + return std::unexpected(Intervals.error()); + LiveRegMatrix ValidationMatrix(Function, Target, **Intervals); + for (const VirtRegMapEntry *Entry : Mapping.orderedEntries()) { + if (Entry == nullptr || !Entry->PhysicalReg) + continue; + auto ClassID = regalloc_detail::registerClassFor(Function, Entry->VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + if (!Target.registers().registerClassContains(*ClassID, *Entry->PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "VirtRegMap physical register is outside the virtual register class"); + } + const LiveInterval *Interval = (*Intervals)->interval(Entry->VirtualReg); + if (Interval == nullptr || Interval->MainRange.empty()) + continue; + auto Assigned = ValidationMatrix.assign(Entry->VirtualReg, *Entry->PhysicalReg); + if (!Assigned) + return std::unexpected(Assigned.error()); + } + + struct Rewrite final { + MachineOperand *Operand = nullptr; + Register NewRegister; + }; + std::vector<Rewrite> Rewrites; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + const auto Assigned = Mapping.physicalRegister(*Reg); + if (!Assigned) { + if (Operand->isDebugUse()) { + Rewrites.push_back({Operand, Register{}}); + continue; + } + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical rewrite found an unassigned virtual register " + + Reg->toString()); + } + PhysicalRegister Effective = *Assigned; + if (Operand->subReg().value() != 0) { + const auto Sub = Target.registers().subRegister(*Assigned, Operand->subReg()); + if (!Sub) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical assignment does not support a required subregister"); + } + Effective = *Sub; + } + Rewrites.push_back({Operand, Register::fromPhysical(Effective)}); + } + } + } + + for (const Rewrite &Plan : Rewrites) { + auto SetRegister = Plan.Operand->setRegister(Plan.NewRegister); + if (!SetRegister) + return std::unexpected(SetRegister.error()); + if (Plan.Operand->subReg().value() != 0) { + auto Cleared = Plan.Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Plan.NewRegister.isNoRegister()) { + RegisterOperandFlags Flags = Plan.Operand->registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto Updated = Plan.Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + + Function.registerInfo().clearLiveInVirtualRegisters(); + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = regalloc_detail::virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Erased = Function.registerInfo().eraseVirtualRegister(*Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Liveness = rebuildMachineLiveness(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + return !Rewrites.empty(); +} + +const MachinePassContract &VirtualRegisterRewritePass::contract() const noexcept { + static const MachinePassContract Contract{ + .Name = "VirtualRegisterRewrite", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + return Contract; +} + +Expected<MachinePassResult> VirtualRegisterRewritePass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (Mapping == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, + "VirtualRegisterRewrite requires a VirtRegMap"); + auto Rewritten = rewriteVirtualRegisters(Function, Target, *Mapping); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + MachinePassResult Result; + Result.Changed = *Rewritten; + if (Result.Changed) { + Result.Changes.add(MachinePassChange::Registers); + Result.Changes.add(MachinePassChange::Operands); + } + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/MIR/EmbeddedIR.cpp b/backend/lib/MIR/EmbeddedIR.cpp index b19748e..662faac 100644 --- a/backend/lib/MIR/EmbeddedIR.cpp +++ b/backend/lib/MIR/EmbeddedIR.cpp @@ -322,8 +322,8 @@ private: parseMetadataDefinition(Line, LineOffset, Position, End); } - void parseTypeDefinition(std::string_view Line, std::size_t LineOffset, - std::size_t Position, std::size_t End) { + void parseTypeDefinition(std::string_view Line, std::size_t LineOffset, std::size_t Position, + std::size_t End) { auto Name = parseIdentifier(Line, LineOffset, Position, End, '%'); if (!Name || Failed) return; @@ -350,15 +350,14 @@ private: return; } const std::size_t TypeIndex = Index.TypeDefinitions.size(); - const bool NumericName = !Name->Name.empty() && Name->Name.front() >= '0' && - Name->Name.front() <= '9'; + const bool NumericName = + !Name->Name.empty() && Name->Name.front() >= '0' && Name->Name.front() <= '9'; Index.TypeNameToIndex.emplace(Name->Name, TypeIndex); Index.TypeDefinitions.push_back(EmbeddedIRTypeDefinition{ .Name = std::move(Name->Name), .Range = Name->Range, .Definition = std::string(Line.substr(Position, DefinitionEnd - Position)), - .DefinitionRange = - Index.Source.range(LineOffset + Position, LineOffset + DefinitionEnd), + .DefinitionRange = Index.Source.range(LineOffset + Position, LineOffset + DefinitionEnd), .NumericName = NumericName, }); } @@ -495,10 +494,10 @@ private: std::size_t Previous = IdentifierBegin; while (Previous > 0 && isHorizontalSpace(Line[Previous - 1])) --Previous; - const bool HasTypeBefore = Previous != 0 && Line[Previous - 1] != '(' && - Line[Previous - 1] != ','; - const bool EndsParameter = Following == End || Line[Following] == ',' || - Line[Following] == ')'; + const bool HasTypeBefore = + Previous != 0 && Line[Previous - 1] != '(' && Line[Previous - 1] != ','; + const bool EndsParameter = + Following == End || Line[Following] == ',' || Line[Following] == ')'; if (HasTypeBefore && EndsParameter) addLocalValue(*Name, Index.Functions[ActiveFunction.value()]); Position = Name->End - 1; @@ -630,9 +629,8 @@ private: }); } - void parseLocalValueDefinition(std::string_view Line, std::size_t LineOffset, - std::size_t Begin, std::size_t End, - EmbeddedIRFunction &Function) { + void parseLocalValueDefinition(std::string_view Line, std::size_t LineOffset, std::size_t Begin, + std::size_t End, EmbeddedIRFunction &Function) { std::size_t Position = skipHorizontalSpace(Line, Begin, End); if (Position >= End || Line[Position] != '%') return; diff --git a/backend/lib/MIR/LLVMConstantSyntax.cpp b/backend/lib/MIR/LLVMConstantSyntax.cpp index 543cc1a..8bc2c18 100644 --- a/backend/lib/MIR/LLVMConstantSyntax.cpp +++ b/backend/lib/MIR/LLVMConstantSyntax.cpp @@ -248,9 +248,7 @@ private: Words[Nibble / 16U] |= static_cast<std::uint64_t>(hexDigitValue(*Iterator)) << ((Nibble % 16U) * 4U); } - if ((BitWidth % 64U) != 0 && (Words.back() >> (BitWidth % 64U)) != 0) - return false; - return true; + return (BitWidth % 64U) == 0 || (Words.back() >> (BitWidth % 64U)) == 0; } [[nodiscard]] bool parseLegacyWideFloatHexWords(std::string_view Digits, @@ -1172,7 +1170,7 @@ convertRawDoubleNaN(FloatSemantics Semantics, bool Negative, std::uint64_t Fract "integer hexadecimal spelling contains a non-hexadecimal digit"); } - const auto FirstNonZero = + const auto *const FirstNonZero = std::ranges::find_if(Digits, [](char Character) { return hexDigitValue(Character) != 0; }); std::size_t ActiveBits = 0; if (FirstNonZero != Digits.end()) { @@ -1998,9 +1996,8 @@ Expected<GenericConstantData> ConstantParser::parseExpression(GenericConstantTyp case GenericConstantExpressionOpcode::Xor: case GenericConstantExpressionOpcode::ExtractElement: return 2; - case GenericConstantExpressionOpcode::InsertElement: - return 3; case GenericConstantExpressionOpcode::ShuffleVector: + case GenericConstantExpressionOpcode::InsertElement: return 3; case GenericConstantExpressionOpcode::GetElementPtr: return 1; @@ -2454,15 +2451,22 @@ Expected<std::string> printValue(const GenericConstantData &Value, bool IncludeT case GenericConstantKind::Invalid: break; case GenericConstantKind::Integer: + if (!Value.IntegerValue) + return makeUnexpected(ErrorCode::VerificationError, + "integer constant is missing its integer payload"); if (resolvedType(Value.Type).Kind == GenericConstantTypeKind::Integer && resolvedType(Value.Type).BitWidth == 1) - Result.append(Value.IntegerValue->isZero() ? "false" : "true"); + Result.append(Value.IntegerValue.value().isZero() ? "false" : "true"); else - Result.append(Value.IntegerValue->toUnsignedDecimal()); + Result.append(Value.IntegerValue.value().toUnsignedDecimal()); return Result; case GenericConstantKind::Floating: + if (!Value.FloatingValue) + return makeUnexpected(ErrorCode::VerificationError, + "floating constant is missing its floating payload"); Result.append("f0x"); - Result.append(hexadecimalWords(Value.FloatingValue->words(), Value.FloatingValue->bitWidth())); + Result.append(hexadecimalWords(Value.FloatingValue.value().words(), + Value.FloatingValue.value().bitWidth())); return Result; case GenericConstantKind::NullPointer: Result.append("null"); @@ -2549,15 +2553,25 @@ Expected<std::string> printValue(const GenericConstantData &Value, bool IncludeT if (Value.ExpressionFlags.has(GenericConstantExpressionFlag::NoSignedWrap)) Result.append(" nsw"); if (Value.InRangeStart) { + if (!Value.InRangeEnd) { + return makeUnexpected(ErrorCode::VerificationError, + "constant expression is missing its inrange end"); + } + const auto &InRangeStart = Value.InRangeStart.value(); + const auto &InRangeEnd = Value.InRangeEnd.value(); Result.append(" inrange("); - Result.append(std::to_string(std::bit_cast<std::int64_t>(Value.InRangeStart->words()[0]))); + Result.append(std::to_string(std::bit_cast<std::int64_t>(InRangeStart.words()[0]))); Result.append(", "); - Result.append(std::to_string(std::bit_cast<std::int64_t>(Value.InRangeEnd->words()[0]))); + Result.append(std::to_string(std::bit_cast<std::int64_t>(InRangeEnd.words()[0]))); Result.push_back(')'); } Result.append(" ("); if (Value.ExpressionOpcode == GenericConstantExpressionOpcode::GetElementPtr) { - auto SourceType = printType(*Value.SourceElementType); + if (!Value.SourceElementType) { + return makeUnexpected(ErrorCode::VerificationError, + "getelementptr constant is missing its source element type"); + } + auto SourceType = printType(Value.SourceElementType.value()); if (!SourceType) return std::unexpected(SourceType.error()); Result.append(*SourceType); @@ -3004,14 +3018,12 @@ LayoutAlignment DataLayoutInfo::scalarAlignment(char Kind, std::uint32_t BitWidt if (Entry.Kind != Kind) continue; if (Kind == 'i') { - if (Entry.BitWidth >= BitWidth && - (Selected == nullptr || Entry.BitWidth < Selected->BitWidth)) { + const bool BetterMatch = + Selected == nullptr || + (Entry.BitWidth >= BitWidth && Entry.BitWidth < Selected->BitWidth) || + (Selected->BitWidth < BitWidth && Entry.BitWidth > Selected->BitWidth); + if (BetterMatch) Selected = &Entry; - } else if (Selected == nullptr) { - Selected = &Entry; - } else if (Selected->BitWidth < BitWidth && Entry.BitWidth > Selected->BitWidth) { - Selected = &Entry; - } } else if (Entry.BitWidth == BitWidth) { Selected = &Entry; break; diff --git a/backend/lib/MIR/MIDebugSyntax.cpp b/backend/lib/MIR/MIDebugSyntax.cpp index bee82c4..fc08df6 100644 --- a/backend/lib/MIR/MIDebugSyntax.cpp +++ b/backend/lib/MIR/MIDebugSyntax.cpp @@ -16,15 +16,13 @@ namespace { [[nodiscard]] bool parseIndex(std::string_view Text, std::uint64_t &Value) noexcept { if (Text.empty()) return false; - const auto [End, Error] = - std::from_chars(Text.data(), Text.data() + Text.size(), Value, 10); + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Value, 10); return Error == std::errc{} && End == Text.data() + Text.size(); } } // namespace -std::optional<std::uint64_t> -parseDIExpressionOpcodeName(std::string_view Name) noexcept { +std::optional<std::uint64_t> parseDIExpressionOpcodeName(std::string_view Name) noexcept { const auto NumberedOpcode = [Name](std::string_view Prefix, std::uint64_t Base) -> std::optional<std::uint64_t> { if (!Name.starts_with(Prefix)) @@ -117,7 +115,7 @@ parseDIExpressionOpcodeName(std::string_view Name) noexcept { {"DW_OP_LLVM_implicit_pointer", 0x1004}, {"DW_OP_LLVM_arg", 0x1005}, }); - const auto Found = + const auto *const Found = std::ranges::find_if(NamedOpcodes, [Name](const auto &Entry) { return Entry.first == Name; }); if (Found == NamedOpcodes.end()) return std::nullopt; diff --git a/backend/lib/MIR/MIPrinter.cpp b/backend/lib/MIR/MIPrinter.cpp index 83fce40..8ad6d6c 100644 --- a/backend/lib/MIR/MIPrinter.cpp +++ b/backend/lib/MIR/MIPrinter.cpp @@ -1285,20 +1285,24 @@ Expected<void> BodyPrinter::printInstruction(const MachineInstr &Instruction) { return makeUnexpected(ErrorCode::VerificationError, "machine instruction has no target descriptor"); } - if (Descriptor->NumExplicitDefs > Instruction.operandCount()) { - return makeUnexpected(ErrorCode::VerificationError, - "machine instruction has fewer operands than explicit definitions"); + std::size_t ExplicitDefs = 0; + while (ExplicitDefs < Instruction.operandCount()) { + const MachineOperand *Operand = Instruction.operandAt(ExplicitDefs); + if (Operand == nullptr || !Operand->isRegister() || Operand->isImplicit() || + !Operand->isDef() || Operand->isUse()) + break; + ++ExplicitDefs; } Output.append(" "); - for (std::size_t Index = 0; Index < Descriptor->NumExplicitDefs; ++Index) { + for (std::size_t Index = 0; Index < ExplicitDefs; ++Index) { if (Index != 0) Output.append(", "); if (auto Printed = printOperand(*Instruction.operandAt(Index), true); !Printed) { return Printed; } } - if (Descriptor->NumExplicitDefs != 0) + if (ExplicitDefs != 0) Output.append(" = "); const MachineInstrFlags Flags = Instruction.flags(); @@ -1333,8 +1337,7 @@ Expected<void> BodyPrinter::printInstruction(const MachineInstr &Instruction) { Output.append(Descriptor->Name); bool HasPrintedTail = false; - for (std::size_t Index = Descriptor->NumExplicitDefs; Index < Instruction.operandCount(); - ++Index) { + for (std::size_t Index = ExplicitDefs; Index < Instruction.operandCount(); ++Index) { Output.append(HasPrintedTail ? ", " : " "); HasPrintedTail = true; if (auto Printed = printOperand(*Instruction.operandAt(Index), false); !Printed) { diff --git a/backend/lib/MIR/MIRDocument.cpp b/backend/lib/MIR/MIRDocument.cpp index 4808df7..67e6ffb 100644 --- a/backend/lib/MIR/MIRDocument.cpp +++ b/backend/lib/MIR/MIRDocument.cpp @@ -406,7 +406,8 @@ class MIRDocumentParser final { public: MIRDocumentParser(const TargetDescription &Target, std::string SourceName, std::string SourceText, MIRDocumentParserOptions Options) - : Target(Target), Options(Options), Result(std::move(SourceName), std::move(SourceText)) {} + : Target(Target), Options(std::move(Options)), + Result(std::move(SourceName), std::move(SourceText)) {} [[nodiscard]] MIRDocumentParseResult run(); @@ -2144,8 +2145,7 @@ bool MIRDocumentParser::parseFunctionRecords(const yaml::Node &Root, MachineFunc WideFlags > std::numeric_limits<std::uint32_t>::max()) { return true; } - if (ActiveDocument->EmbeddedIR && - ActiveDocument->EmbeddedIR->hasGlobalValue(Callee) == false) { + if (ActiveDocument->EmbeddedIR && !ActiveDocument->EmbeddedIR->hasGlobalValue(Callee)) { return fail(CalleeNode->range(), "called global references an undefined embedded global"); } auto AddResult = @@ -2765,7 +2765,8 @@ MIRDocumentParseResult::~MIRDocumentParseResult() = default; MIRDocumentParseResult parseMIRDocument(const TargetDescription &Target, std::string SourceName, std::string SourceText, MIRDocumentParserOptions Options) { - return MIRDocumentParser(Target, std::move(SourceName), std::move(SourceText), Options).run(); + return MIRDocumentParser(Target, std::move(SourceName), std::move(SourceText), std::move(Options)) + .run(); } } // namespace rcc::mir diff --git a/backend/lib/MIR/MIRDocumentPrinter.cpp b/backend/lib/MIR/MIRDocumentPrinter.cpp index e9f9e1c..dba1ec7 100644 --- a/backend/lib/MIR/MIRDocumentPrinter.cpp +++ b/backend/lib/MIR/MIRDocumentPrinter.cpp @@ -85,7 +85,7 @@ class MIRDocumentPrinter final { public: MIRDocumentPrinter(const MIRDocument &Document, const TargetDescription &Target, MIRDocumentPrinterOptions Options) - : Document(Document), Target(Target), Options(Options) {} + : Document(Document), Target(Target), Options(std::move(Options)) {} [[nodiscard]] Expected<std::string> run(); @@ -1187,9 +1187,10 @@ Expected<void> MIRDocumentPrinter::printFunction(const MachineFunction &Function Output.append("---\nname: "); Output.append(yamlScalar(Function.name())); Output.push_back('\n'); - if (Function.attributes().alignment()) { + const auto Alignment = Function.attributes().alignment(); + if (Alignment) { Output.append("alignment: "); - Output.append(std::to_string(Function.attributes().alignment()->bytes())); + Output.append(std::to_string(Alignment->bytes())); Output.push_back('\n'); } @@ -1296,7 +1297,7 @@ Expected<std::string> MIRDocumentPrinter::run() { Expected<std::string> printMIRDocument(const MIRDocument &Document, const TargetDescription &Target, MIRDocumentPrinterOptions Options) { - return MIRDocumentPrinter(Document, Target, Options).run(); + return MIRDocumentPrinter(Document, Target, std::move(Options)).run(); } } // namespace rcc::mir diff --git a/backend/lib/MIR/MIRDocumentReferenceValidation.h b/backend/lib/MIR/MIRDocumentReferenceValidation.h index 0992d84..2ab6291 100644 --- a/backend/lib/MIR/MIRDocumentReferenceValidation.h +++ b/backend/lib/MIR/MIRDocumentReferenceValidation.h @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) #ifndef RCC_LIB_MIR_MIRDOCUMENTREFERENCEVALIDATION_H #define RCC_LIB_MIR_MIRDOCUMENTREFERENCEVALIDATION_H diff --git a/backend/lib/Support/Error.cpp b/backend/lib/Support/Error.cpp index dc7d590..817a2bb 100644 --- a/backend/lib/Support/Error.cpp +++ b/backend/lib/Support/Error.cpp @@ -14,6 +14,8 @@ std::string_view getErrorCodeName(ErrorCode Code) noexcept { return "verification error"; case ErrorCode::Unsupported: return "unsupported"; + case ErrorCode::RegisterAllocationFailed: + return "register allocation failed"; case ErrorCode::IoError: return "I/O error"; case ErrorCode::InternalError: diff --git a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp index 3c5e817..9d6b8e1 100644 --- a/backend/lib/Target/Description/TargetDescriptionGenerator.cpp +++ b/backend/lib/Target/Description/TargetDescriptionGenerator.cpp @@ -1364,7 +1364,7 @@ private: const auto Value = unsignedInteger32(*Tie, "instruction operand default_tied_operand"); if (!Value) return std::nullopt; - Result.DefaultTiedOperand = *Value; + Result.DefaultTiedOperand = Value; } if (const yaml::Node *EarlyClobber = optionalNode(Node, "early_clobber")) { const auto Value = boolean(*EarlyClobber, "instruction operand early_clobber"); @@ -1614,7 +1614,7 @@ operandKindEnumerator(std::string_view Value) noexcept { {"debug_instr_ref", "DebugInstrRef"}, {"lane_mask", "LaneMask"}, }}; - const auto Position = std::ranges::find(Kinds, Value, &decltype(Kinds)::value_type::first); + const auto *const Position = std::ranges::find(Kinds, Value, &decltype(Kinds)::value_type::first); if (Position == Kinds.end()) return std::nullopt; return Position->second; @@ -1675,7 +1675,7 @@ instructionFlagEnumerator(std::string_view Value) noexcept { {"generic", "Generic"}, {"terminal_emission", "TerminalEmission"}, }}; - const auto Position = std::ranges::find(Flags, Value, &decltype(Flags)::value_type::first); + const auto *const Position = std::ranges::find(Flags, Value, &decltype(Flags)::value_type::first); if (Position == Flags.end()) return std::nullopt; return Position->second; @@ -1698,7 +1698,8 @@ pseudoOwnerEnumerator(std::string_view Value) noexcept { }}; if (Value == "asm_printer") return "AsmPrinter"; - const auto Position = std::ranges::find(Owners, Value, &decltype(Owners)::value_type::first); + const auto *const Position = + std::ranges::find(Owners, Value, &decltype(Owners)::value_type::first); if (Position == Owners.end()) return std::nullopt; return Position->second; @@ -1718,7 +1719,8 @@ pipelineStageEnumerator(std::string_view Value) noexcept { {"post_ra_finalized", "PostRAFinalized"}, {"emission", "Emission"}, }}; - const auto Position = std::ranges::find(Stages, Value, &decltype(Stages)::value_type::first); + const auto *const Position = + std::ranges::find(Stages, Value, &decltype(Stages)::value_type::first); if (Position == Stages.end()) return std::nullopt; return Position->second; @@ -2066,7 +2068,7 @@ private: fail(Convention.Range, "calling convention register classes must resolve"); return false; } - const auto normalizeRegisterList = [this, + const auto NormalizeRegisterList = [this, &Convention](const std::vector<std::string> &Symbols, const NormalizedRegisterClass &RegisterClass, std::vector<std::uint32_t> &Output) { @@ -2101,13 +2103,13 @@ private: .SupportsVarArgs = Convention.SupportsVarArgs, .SupportsTailCalls = Convention.SupportsTailCalls, }; - if (!normalizeRegisterList(Convention.IntegerArgumentRegisterSymbols, + if (!NormalizeRegisterList(Convention.IntegerArgumentRegisterSymbols, *IntegerClassDescription, Normalized.IntegerArgumentRegisters) || - !normalizeRegisterList(Convention.FloatingArgumentRegisterSymbols, + !NormalizeRegisterList(Convention.FloatingArgumentRegisterSymbols, *FloatingClassDescription, Normalized.FloatingArgumentRegisters) || - !normalizeRegisterList(Convention.IntegerReturnRegisterSymbols, *IntegerClassDescription, + !NormalizeRegisterList(Convention.IntegerReturnRegisterSymbols, *IntegerClassDescription, Normalized.IntegerReturnRegisters) || - !normalizeRegisterList(Convention.FloatingReturnRegisterSymbols, + !NormalizeRegisterList(Convention.FloatingReturnRegisterSymbols, *FloatingClassDescription, Normalized.FloatingReturnRegisters)) { return false; } @@ -3004,8 +3006,9 @@ void emitImmediatePredicate(std::ostream &Output, const NormalizedImmediatePredi } // namespace -DescriptionGenerationResult generateDescription(std::string SourceName, std::string SourceText) { - yaml::ParseResult YAMLResult = yaml::parse(SourceName, SourceText); +DescriptionGenerationResult generateDescription(const std::string &SourceName, + std::string SourceText) { + yaml::ParseResult YAMLResult = yaml::parse(SourceName, std::move(SourceText)); if (!YAMLResult.succeeded()) { DescriptionGenerationResult Result; for (const TextDiagnostic &Diagnostic : YAMLResult.diagnostics()) diff --git a/backend/lib/Target/Description/TargetDescriptionGenerator.h b/backend/lib/Target/Description/TargetDescriptionGenerator.h index c0d8ed5..9a03aa0 100644 --- a/backend/lib/Target/Description/TargetDescriptionGenerator.h +++ b/backend/lib/Target/Description/TargetDescriptionGenerator.h @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) #ifndef RCC_TARGET_DESCRIPTION_TARGETDESCRIPTIONGENERATOR_H #define RCC_TARGET_DESCRIPTION_TARGETDESCRIPTIONGENERATOR_H @@ -27,7 +29,7 @@ struct DescriptionGenerationResult final { } }; -[[nodiscard]] DescriptionGenerationResult generateDescription(std::string SourceName, +[[nodiscard]] DescriptionGenerationResult generateDescription(const std::string &SourceName, std::string SourceText); } // namespace rcc::targetgen diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index 4728117..1d9a27a 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -49,6 +49,7 @@ add_library( RCCBackendRISCV RISCVCodeGen.cpp RISCVCallLowering.cpp + RISCVRegisterAllocation.cpp RISCVTarget.cpp "${RCCBackendRISCVGeneratedRegisterHeader}" "${RCCBackendRISCVGeneratedInstructionHeader}" @@ -93,6 +94,7 @@ install( rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" @@ -101,5 +103,6 @@ rcc_backend_register_format_files( rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" ) diff --git a/backend/lib/Target/RISCV/RISCV.rccdesc b/backend/lib/Target/RISCV/RISCV.rccdesc index a8dfdde..e248bc7 100644 --- a/backend/lib/Target/RISCV/RISCV.rccdesc +++ b/backend/lib/Target/RISCV/RISCV.rccdesc @@ -375,7 +375,7 @@ instructions: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - { kinds: [immediate], immediate_predicate: SIMM20, name: IMM } num_explicit_defs: 1 - flags: [produces_code] + flags: [produces_code, rematerializable] scheduling_class: ALU encoding_category: U assembly_format: RD_IMM @@ -402,7 +402,7 @@ instructions: - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 - flags: [produces_code] + flags: [produces_code, rematerializable] scheduling_class: ALU encoding_category: I assembly_format: RD_RS1_IMM @@ -444,7 +444,7 @@ instructions: - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 - flags: [produces_code] + flags: [produces_code, rematerializable] scheduling_class: ALU encoding_category: I assembly_format: RD_RS1_IMM @@ -709,7 +709,7 @@ instructions: assembly_mnemonic: lw operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, memory_role: value, name: RD } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 flags: [may_load, produces_code] @@ -799,7 +799,7 @@ instructions: assembly_mnemonic: sw operands: - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: value, name: RS2 } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 0 flags: [may_store, produces_code] @@ -1202,7 +1202,7 @@ instructions: assembly_mnemonic: ld operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, memory_role: value, name: RD } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 flags: [may_load, produces_code] @@ -1217,7 +1217,7 @@ instructions: assembly_mnemonic: sd operands: - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: value, name: RS2 } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 0 flags: [may_store, produces_code] @@ -1423,7 +1423,7 @@ instructions: assembly_mnemonic: flw operands: - { kinds: [register], role: def, register_class: FPR32, register_bank: FPR, memory_role: value, name: FRD } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 flags: [may_load, produces_code] @@ -1438,7 +1438,7 @@ instructions: assembly_mnemonic: fsw operands: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, memory_role: value, name: FRS2 } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 0 flags: [may_store, produces_code] @@ -1854,7 +1854,7 @@ instructions: assembly_mnemonic: fld operands: - { kinds: [register], role: def, register_class: FPR64, register_bank: FPR, memory_role: value, name: FRD } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 1 flags: [may_load, produces_code] @@ -1869,7 +1869,7 @@ instructions: assembly_mnemonic: fsd operands: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, memory_role: value, name: FRS2 } - - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } + - { kinds: [register, frame_index], role: use, register_class: GPR, register_bank: GPR, memory_role: address, name: RS1 } - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } num_explicit_defs: 0 flags: [may_store, produces_code] diff --git a/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp new file mode 100644 index 0000000..e437e18 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <utility> + +namespace rcc::riscv { +namespace { + +struct SpillForm final { + InstructionOpcode Load; + InstructionOpcode Store; + std::uint64_t SizeInBytes; +}; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] constexpr RegisterClassID registerClassID(RegisterClass Value) noexcept { + return RegisterClassID::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] constexpr std::optional<SpillForm> spillForm(RegisterClassID Class) noexcept { + if (Class == registerClassID(RegisterClass::GPR) || + Class == registerClassID(RegisterClass::GPRC) || + Class == registerClassID(RegisterClass::GPR_NX0) || + Class == registerClassID(RegisterClass::GPR_NX0_NX2)) { + return SpillForm{InstructionOpcode::LD, InstructionOpcode::SD, 8}; + } + if (Class == registerClassID(RegisterClass::FPR64) || + Class == registerClassID(RegisterClass::FPR64C)) { + return SpillForm{InstructionOpcode::FLD, InstructionOpcode::FSD, 8}; + } + if (Class == registerClassID(RegisterClass::FPR32) || + Class == registerClassID(RegisterClass::FPR32C)) { + return SpillForm{InstructionOpcode::FLW, InstructionOpcode::FSW, 4}; + } + return std::nullopt; +} + +[[nodiscard]] Expected<const MachineMemOperand *> +createStackMemoryOperand(MachineIRBuilder &Builder, FrameIndex StackSlot, MemoryAccessFlag Access, + std::uint64_t SizeInBytes, std::uint64_t AlignmentInBytes) { + auto Size = MemorySize::fixed(SizeInBytes); + if (!Size) + return std::unexpected(Size.error()); + auto Align = Alignment::create(AlignmentInBytes); + if (!Align) + return std::unexpected(Align.error()); + + MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = MemoryBase::frameIndex(StackSlot); + return Builder.function().createMemOperand(std::move(Description)); +} + +[[nodiscard]] bool isCalleeSaved(PhysicalRegister Reg) noexcept { + const std::uint32_t Number = Reg.number(); + const std::array GPRs{ + PhysicalRegisterID::X8, PhysicalRegisterID::X9, PhysicalRegisterID::X18, + PhysicalRegisterID::X19, PhysicalRegisterID::X20, PhysicalRegisterID::X21, + PhysicalRegisterID::X22, PhysicalRegisterID::X23, PhysicalRegisterID::X24, + PhysicalRegisterID::X25, PhysicalRegisterID::X26, PhysicalRegisterID::X27, + }; + const std::array FPRs{ + PhysicalRegisterID::F8, PhysicalRegisterID::F9, PhysicalRegisterID::F18, + PhysicalRegisterID::F19, PhysicalRegisterID::F20, PhysicalRegisterID::F21, + PhysicalRegisterID::F22, PhysicalRegisterID::F23, PhysicalRegisterID::F24, + PhysicalRegisterID::F25, PhysicalRegisterID::F26, PhysicalRegisterID::F27, + }; + const auto Matches = [Number](PhysicalRegisterID ID) { + return Number == static_cast<std::uint32_t>(ID); + }; + return std::ranges::any_of(GPRs, Matches) || std::ranges::any_of(FPRs, Matches); +} + +[[nodiscard]] bool isCompressedRegister(PhysicalRegister Reg) noexcept { + const std::uint32_t Number = Reg.number(); + const std::uint32_t X8 = static_cast<std::uint32_t>(PhysicalRegisterID::X8); + const std::uint32_t X15 = static_cast<std::uint32_t>(PhysicalRegisterID::X15); + const std::uint32_t F8 = static_cast<std::uint32_t>(PhysicalRegisterID::F8); + const std::uint32_t F15 = static_cast<std::uint32_t>(PhysicalRegisterID::F15); + return (Number >= X8 && Number <= X15) || (Number >= F8 && Number <= F15); +} + +class RISCVRegisterAllocationInfo final : public TargetRegisterAllocationInfo { +public: + [[nodiscard]] Expected<void> validateSpillReload(RegisterClassID Class, + const TargetDescription &Target) const override { + const RegisterClassDesc *Description = Target.registers().registerClass(Class); + const auto Form = spillForm(Class); + if (!Form) { + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC register class has no scalar spill/reload form"); + } + if (Description == nullptr || !Description->Allocatable || + Description->SpillSizeInBits != Form->SizeInBytes * 8U || + Description->SpillAlignmentInBits == 0 || Description->SpillAlignmentInBits % 8U != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RV64GC register class has inconsistent spill properties"); + } + for (const InstructionOpcode OpcodeValue : {Form->Load, Form->Store}) { + const InstructionDescriptor *Instruction = + Target.instructionDescriptor(targetOpcode(OpcodeValue)); + if (Instruction == nullptr || Instruction->Operands.size() != 3 || + !Instruction->Operands[1].Kinds.contains(MachineOperand::Kind::FrameIndex) || + !Instruction->RequiresMemoryOperands) { + return makeUnexpected(ErrorCode::InternalError, + "RV64GC spill instruction descriptor is incomplete"); + } + } + return {}; + } + + [[nodiscard]] Expected<MachineInstr *> emitReload(MachineIRBuilder &Builder, Register Destination, + RegisterClassID Class, FrameIndex StackSlot, + LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + const RegisterClassDesc *Description = Builder.target().registers().registerClass(Class); + if (Lanes.empty() || !Lanes.intersects(Description->Lanes)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RV64GC reload requires a nonempty class lane mask"); + } + const auto FormValue = spillForm(Class); + if (!FormValue) { + return makeUnexpected(ErrorCode::InternalError, + "RV64GC reload lost its validated spill form"); + } + const SpillForm Form = FormValue.value(); + auto Memory = + createStackMemoryOperand(Builder, StackSlot, MemoryAccessFlag::Load, Form.SizeInBytes, + Description->SpillAlignmentInBits / 8U); + if (!Memory) + return std::unexpected(Memory.error()); + auto Pending = Builder.buildInstr(targetOpcode(Form.Load)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addImmediate(0); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**Memory); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] Expected<MachineInstr *> emitSpill(MachineIRBuilder &Builder, Register Source, + RegisterClassID Class, FrameIndex StackSlot, + LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + const RegisterClassDesc *Description = Builder.target().registers().registerClass(Class); + if (Lanes.empty() || !Lanes.intersects(Description->Lanes)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RV64GC spill requires a nonempty class lane mask"); + } + const auto FormValue = spillForm(Class); + if (!FormValue) { + return makeUnexpected(ErrorCode::InternalError, "RV64GC spill lost its validated spill form"); + } + const SpillForm Form = FormValue.value(); + auto Memory = + createStackMemoryOperand(Builder, StackSlot, MemoryAccessFlag::Store, Form.SizeInBytes, + Description->SpillAlignmentInBits / 8U); + if (!Memory) + return std::unexpected(Memory.error()); + auto Pending = Builder.buildInstr(targetOpcode(Form.Store)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addImmediate(0); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**Memory); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] Expected<bool> isRematerializable(const MachineInstr &Instruction, + Register VirtualReg, const MachineFunction &, + const TargetDescription &) const override { + const MachineOperand *Definition = Instruction.operandAt(0); + const Register *DefinedReg = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + if (DefinedReg == nullptr || *DefinedReg != VirtualReg || !Definition->isDef()) + return false; + + if (Instruction.opcode() == targetOpcode(InstructionOpcode::LUI)) + return Instruction.operandCount() == 2 && Instruction.operandAt(1) != nullptr; + if (Instruction.opcode() != targetOpcode(InstructionOpcode::ADDI) && + Instruction.opcode() != targetOpcode(InstructionOpcode::ADDIW)) { + return false; + } + if (Instruction.operandCount() != 3) + return false; + const MachineOperand *Source = Instruction.operandAt(1); + const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf<Register>(); + const auto Zero = physicalRegister(PhysicalRegisterID::X0); + return Zero && SourceReg != nullptr && *SourceReg == Register::fromPhysical(*Zero) && + Source->isUse(); + } + + [[nodiscard]] Expected<MachineInstr *> + rematerialize(MachineIRBuilder &Builder, const MachineInstr &Original, Register Destination, + Register OriginalRegister, const TargetDescription &Target) const override { + auto Accepted = isRematerializable(Original, OriginalRegister, Builder.function(), Target); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RV64GC instruction is not rematerializable"); + } + + auto Pending = Builder.buildInstr(Original.opcode()); + if (!Pending) + return std::unexpected(Pending.error()); + const MachineOperand *OriginalDef = Original.operandAt(0); + if (auto Result = Pending->addDef(Destination, {}, {}, OriginalDef->targetFlags()); !Result) + return std::unexpected(Result.error()); + for (std::size_t Index = 1; Index < Original.operandCount(); ++Index) { + const MachineOperand *Operand = Original.operandAt(Index); + if (Operand == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "RV64GC rematerialization found a missing operand"); + } + if (auto Result = Pending->addOperand(Operand->cloneDetached()); !Result) + return std::unexpected(Result.error()); + } + return Pending->build(); + } + + [[nodiscard]] std::uint64_t physicalRegisterCost(const MachineFunction &, Register, + PhysicalRegister Reg, RegisterClassID, + bool) const noexcept override { + if (Reg.number() == static_cast<std::uint32_t>(PhysicalRegisterID::X1)) + return 16; + if (isCalleeSaved(Reg)) + return 4; + return isCompressedRegister(Reg) ? 0 : 1; + } + + [[nodiscard]] std::uint64_t spillCost(const MachineFunction &, const LiveInterval &, + RegisterClassID Class) const noexcept override { + const auto Form = spillForm(Class); + return Form ? std::max<std::uint64_t>(1, Form->SizeInBytes / 4U) : 1; + } +}; + +} // namespace + +const TargetRegisterAllocationInfo ®isterAllocationInfo() noexcept { + static const RISCVRegisterAllocationInfo Info; + return Info; +} + +} // namespace rcc::riscv diff --git a/backend/tools/DescriptionGenerator/DescriptionGeneratorMain.cpp b/backend/tools/DescriptionGenerator/DescriptionGeneratorMain.cpp index 80b4767..13927e1 100644 --- a/backend/tools/DescriptionGenerator/DescriptionGeneratorMain.cpp +++ b/backend/tools/DescriptionGenerator/DescriptionGeneratorMain.cpp @@ -97,9 +97,14 @@ int main(int ArgumentCount, char *Arguments[]) { return 1; } - return writeFile(Command.RegisterHeaderPath, Result.Output->RegisterHeader) && - writeFile(Command.InstructionHeaderPath, Result.Output->InstructionHeader) && - writeFile(Command.TablesPath, Result.Output->TargetTables) + if (!Result.Output) { + std::cerr << "rcc-backend-descgen: successful generation produced no output\n"; + return 1; + } + const rcc::targetgen::GeneratedDescription &Output = Result.Output.value(); + return writeFile(Command.RegisterHeaderPath, Output.RegisterHeader) && + writeFile(Command.InstructionHeaderPath, Output.InstructionHeader) && + writeFile(Command.TablesPath, Output.TargetTables) ? 0 : 1; } diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt index b3f071b..65d9168 100644 --- a/backend/unittests/CodeGen/CMakeLists.txt +++ b/backend/unittests/CodeGen/CMakeLists.txt @@ -22,6 +22,8 @@ set( MachineOperandTest.cpp MachinePassTest.cpp MachinePipelinePassesTest.cpp + PBQPTest.cpp + RegisterAllocationTest.cpp MachineSSAExitTest.cpp MachineVerifierTest.cpp MachineVerifierTargetTest.cpp @@ -48,6 +50,72 @@ target_link_libraries(RCCBackendCodeGenCoreTypesTests PRIVATE RCCBackend::CodeGe add_test(NAME RCCBackend.CodeGen.CoreTypes COMMAND RCCBackendCodeGenCoreTypesTests) +add_executable(RCCBackendCodeGenPBQPTests PBQPTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenPBQPTests) +target_link_libraries(RCCBackendCodeGenPBQPTests PRIVATE RCCBackend::CodeGen) + +add_test(NAME RCCBackend.CodeGen.PBQP COMMAND RCCBackendCodeGenPBQPTests) + +add_executable(RCCBackendCodeGenRegisterAllocationTests RegisterAllocationTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenRegisterAllocationTests) +target_link_libraries( + RCCBackendCodeGenRegisterAllocationTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) + +add_test( + NAME RCCBackend.CodeGen.RegisterAllocation + COMMAND RCCBackendCodeGenRegisterAllocationTests +) + +add_executable(RCCBackendCodeGenLiveRegMatrixTests LiveRegMatrixTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenLiveRegMatrixTests) +target_link_libraries( + RCCBackendCodeGenLiveRegMatrixTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.LiveRegMatrix COMMAND RCCBackendCodeGenLiveRegMatrixTests) + +add_executable(RCCBackendCodeGenVirtRegMapTests VirtRegMapTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenVirtRegMapTests) +target_link_libraries( + RCCBackendCodeGenVirtRegMapTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.VirtRegMap COMMAND RCCBackendCodeGenVirtRegMapTests) + +add_executable(RCCBackendCodeGenSpillerTests SpillerTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenSpillerTests) +target_link_libraries( + RCCBackendCodeGenSpillerTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.Spiller COMMAND RCCBackendCodeGenSpillerTests) + +add_executable(RCCBackendCodeGenBasicAllocatorTests RegAllocBasicTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenBasicAllocatorTests) +target_link_libraries( + RCCBackendCodeGenBasicAllocatorTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.BasicAllocator COMMAND RCCBackendCodeGenBasicAllocatorTests) + +add_executable(RCCBackendCodeGenFastAllocatorTests RegAllocFastTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenFastAllocatorTests) +target_link_libraries( + RCCBackendCodeGenFastAllocatorTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.FastAllocator COMMAND RCCBackendCodeGenFastAllocatorTests) + +add_executable(RCCBackendCodeGenPBQPAllocatorTests RegAllocPBQPTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenPBQPAllocatorTests) +target_link_libraries( + RCCBackendCodeGenPBQPAllocatorTests + PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget +) +add_test(NAME RCCBackend.CodeGen.PBQPAllocator COMMAND RCCBackendCodeGenPBQPAllocatorTests) + add_executable(RCCBackendCodeGenGenericOpcodesTests GenericOpcodesTest.cpp) rcc_backend_configure_target(RCCBackendCodeGenGenericOpcodesTests) target_link_libraries(RCCBackendCodeGenGenericOpcodesTests PRIVATE RCCBackend::CodeGen) @@ -373,6 +441,8 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp" @@ -382,6 +452,13 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionStateTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfoTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfoTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTestSupport.h" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrixTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/VirtRegMapTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/SpillerTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasicTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocFastTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocPBQPTest.cpp" ) rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/CompilationUnitTest.cpp" @@ -404,6 +481,8 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp" @@ -413,4 +492,10 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/TargetFunctionStateTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetMIRTextInfoTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TargetRegisterInfoTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LiveRegMatrixTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/VirtRegMapTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/SpillerTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasicTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocFastTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocPBQPTest.cpp" ) diff --git a/backend/unittests/CodeGen/LiveRegMatrixTest.cpp b/backend/unittests/CodeGen/LiveRegMatrixTest.cpp new file mode 100644 index 0000000..21c0778 --- /dev/null +++ b/backend/unittests/CodeGen/LiveRegMatrixTest.cpp @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRegMatrix.h" + +#include "CodeGen/RegisterAllocationTestSupport.h" + +#include <algorithm> +#include <iostream> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +void testLaneAndAssignment(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("matrix-lanes", 1); + if (!Fixture) + return; + auto Low = createVReg(*Fixture->Function); + auto High = createVReg(*Fixture->Function); + if (!Low || !High || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *Low, 1, TestSubReg::Low32) || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *High, 2, TestSubReg::High32) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *High, TestSubReg::High32) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Low, TestSubReg::Low32)) + return; + + MachineAnalysisManager Analyses; + if (!registerStage11MachineAnalyses(Analyses)) + return; + auto Intervals = Analyses.getResult<LiveIntervalsAnalysis>(*Fixture->Function, Target); + if (!Intervals) + return; + const PhysicalRegister G0 = physical(TestRegister::G0); + LiveRegMatrix Matrix(*Fixture->Function, Target, **Intervals); + Context.expect(Matrix.assign(*Low, G0).has_value() && + Matrix.checkInterference(*High, G0).Kind == RegisterInterferenceKind::Free && + Matrix.assign(*High, G0).has_value() && Matrix.assignment(*Low) == G0, + "LiveRegMatrix must allow disjoint lanes and publish assignments"); + Context.expect(Matrix.unassign(*Low).has_value() && !Matrix.assignment(*Low), + "LiveRegMatrix must remove an assignment without affecting other lanes"); +} + +void testFixedAndMaskInterference(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("matrix-fixed", 2); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + const PhysicalRegister G0 = physical(TestRegister::G0); + const PhysicalRegister G1 = physical(TestRegister::G1); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 7) || + !buildMask(*Fixture->Function, Target, *Fixture->Block, createTestCallPreservedMask()) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, Register::fromPhysical(G0)) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value)) + return; + + MachineAnalysisManager Analyses; + if (!registerStage11MachineAnalyses(Analyses)) + return; + auto Intervals = Analyses.getResult<LiveIntervalsAnalysis>(*Fixture->Function, Target); + if (!Intervals) + return; + LiveRegMatrix Matrix(*Fixture->Function, Target, **Intervals); + const auto G0Interferences = Matrix.collectInterferences(*Value, G0); + const auto G1Interferences = Matrix.collectInterferences(*Value, G1); + const auto HasKind = [](std::span<const RegisterInterference> Values, + RegisterInterferenceKind Kind) { + return std::ranges::any_of( + Values, [Kind](const RegisterInterference &Value) { return Value.Kind == Kind; }); + }; + Context.expect(HasKind(G0Interferences, RegisterInterferenceKind::FixedRegister) && + HasKind(G0Interferences, RegisterInterferenceKind::RegisterMask) && + !HasKind(G1Interferences, RegisterInterferenceKind::FixedRegister) && + Matrix.assign(*Value, G1).has_value(), + "LiveRegMatrix must distinguish fixed-register and RegMask interference"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "LiveRegMatrix test target must be constructible"); + if (!Target) { + std::cerr << Target.error().message() << '\n'; + return Context.result(); + } + testLaneAndAssignment(Context, *Target); + testFixedAndMaskInterference(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/MachineIRBuilderTest.cpp b/backend/unittests/CodeGen/MachineIRBuilderTest.cpp index 00ffffc..c4d5767 100644 --- a/backend/unittests/CodeGen/MachineIRBuilderTest.cpp +++ b/backend/unittests/CodeGen/MachineIRBuilderTest.cpp @@ -427,8 +427,7 @@ void testImplicitAndMemoryValidation(TestContext &Context) { rcc::MachineConstantPoolEntryDesc ConstantDescription; auto Integer = rcc::IntegerImmediate::fromUnsigned(64, 0x2a2a2a2a2a2a2a2aULL); - ConstantDescription.Value = - std::move(*rcc::GenericConstantData::integer(std::move(*Integer))); + ConstantDescription.Value = std::move(*rcc::GenericConstantData::integer(std::move(*Integer))); ConstantDescription.Size = 8; ConstantDescription.Align = *rcc::Alignment::create(8); auto ConstantIndex = Function->constantPool().addEntry(std::move(ConstantDescription)); diff --git a/backend/unittests/CodeGen/PBQPTest.cpp b/backend/unittests/CodeGen/PBQPTest.cpp new file mode 100644 index 0000000..073de16 --- /dev/null +++ b/backend/unittests/CodeGen/PBQPTest.cpp @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PBQP/Graph.h" +#include "rcc/CodeGen/PBQP/Math.h" +#include "rcc/CodeGen/PBQP/Solver.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iostream> +#include <string_view> +#include <vector> + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +void testMath(TestContext &Context) { + using namespace rcc::pbqp; + Context.expect( + addCosts(InfiniteCost, -4) == InfiniteCost && addCosts(InfiniteCost - 1, 2) == InfiniteCost && + addCosts(NegativeInfiniteCost + 1, -2) == NegativeInfiniteCost && addCosts(7, -3) == 4, + "PBQP cost arithmetic must saturate deterministically"); + + CostVector Vector(std::vector<Cost>{8, -2, 4}); + Context.expect(Vector.minimumIndex() == 1, + "PBQP cost vectors must choose the first stable minimum"); + CostMatrix Matrix(2, 3); + Matrix.at(0, 0) = 1; + Matrix.at(0, 1) = 2; + Matrix.at(0, 2) = 3; + Matrix.at(1, 0) = 4; + Matrix.at(1, 1) = 5; + Matrix.at(1, 2) = 6; + const CostMatrix Transposed = Matrix.transpose(); + Context.expect(Transposed.rows() == 3 && Transposed.columns() == 2 && Transposed.at(2, 1) == 6 && + Transposed.at(1, 0) == 2, + "PBQP matrix transposition must preserve orientation"); +} + +void testGraphValidation(TestContext &Context) { + using namespace rcc::pbqp; + Graph Problem; + auto Empty = Problem.addNode(CostVector(0)); + auto First = Problem.addNode(CostVector(std::vector<Cost>{0, 1}), 20); + auto Second = Problem.addNode(CostVector(std::vector<Cost>{2, 0, 3}), 10); + Context.expect(!Empty && First && Second, + "PBQP graphs must reject empty nodes and accept stable nonempty nodes"); + if (!First || !Second) + return; + auto Self = Problem.addEdge(*First, *First, CostMatrix(2, 2)); + auto WrongShape = Problem.addEdge(*First, *Second, CostMatrix(2, 2)); + CostMatrix EdgeCosts(2, 3); + EdgeCosts.at(1, 2) = 7; + auto Edge = Problem.addEdge(*Second, *First, EdgeCosts.transpose(), 99); + auto Duplicate = Problem.addEdge(*First, *Second, CostMatrix(2, 3)); + Context.expect(!Self && !WrongShape && Edge && !Duplicate && Problem.verify().has_value(), + "PBQP graph edges must enforce identity, dimensions, orientation, and uniqueness"); + if (!Edge) + return; + const rcc::pbqp::Edge *Stored = Problem.edge(*Edge); + Context.expect(Stored != nullptr && Stored->First == *First && Stored->Second == *Second && + Stored->Costs.rows() == 2 && Stored->Costs.columns() == 3 && + Stored->Costs.at(1, 2) == 7, + "PBQP graph insertion must normalize reversed edge orientation"); +} + +void testReductions(TestContext &Context) { + using namespace rcc::pbqp; + DeterministicSolver Solver; + + Graph DegreeZero; + auto Only = DegreeZero.addNode(CostVector(std::vector<Cost>{4, 1, 1}), 3); + auto ZeroSolution = Solver.solve(DegreeZero); + Context.expect(Only && ZeroSolution && ZeroSolution->selection(*Only) == 1 && + ZeroSolution->totalCost() == 1, + "PBQP R0 reduction must select the first minimum-cost option"); + + Graph DegreeOne; + auto Left = DegreeOne.addNode(CostVector(std::vector<Cost>{3, 0}), 1); + auto Right = DegreeOne.addNode(CostVector(std::vector<Cost>{0, 2}), 2); + CostMatrix Pair(2, 2); + Pair.at(1, 0) = InfiniteCost; + auto PairEdge = DegreeOne.addEdge(*Left, *Right, std::move(Pair)); + auto OneSolution = Solver.solve(DegreeOne); + Context.expect(Left && Right && PairEdge && OneSolution && OneSolution->selection(*Left) == 1 && + OneSolution->selection(*Right) == 1 && OneSolution->totalCost() == 2, + "PBQP R1 reduction must propagate unary costs to its neighbor"); + + Graph DegreeTwo; + auto A = DegreeTwo.addNode(CostVector(std::vector<Cost>{0, 2}), 1); + auto B = DegreeTwo.addNode(CostVector(std::vector<Cost>{0, 0}), 2); + auto C = DegreeTwo.addNode(CostVector(std::vector<Cost>{2, 0}), 3); + CostMatrix AB(2, 2); + CostMatrix BC(2, 2); + AB.at(0, 1) = 5; + AB.at(1, 0) = 5; + BC.at(0, 1) = 5; + BC.at(1, 0) = 5; + auto ABEdge = DegreeTwo.addEdge(*A, *B, std::move(AB)); + auto BCEdge = DegreeTwo.addEdge(*B, *C, std::move(BC)); + auto TwoSolution = Solver.solve(DegreeTwo); + Context.expect(A && B && C && ABEdge && BCEdge && TwoSolution && + TwoSolution->selection(*A) == 0 && TwoSolution->selection(*B) == 0 && + TwoSolution->selection(*C) == 0 && TwoSolution->totalCost() == 2, + "PBQP R2 reduction must add the induced neighbor edge cost"); +} + +[[nodiscard]] rcc::Expected<rcc::pbqp::Solution> solveClique() { + using namespace rcc::pbqp; + Graph Problem; + std::vector<NodeID> Nodes; + for (std::uint64_t Index = 0; Index < 4; ++Index) { + auto Node = Problem.addNode(CostVector(std::vector<Cost>{9, 0, 0}), 100 - Index); + if (!Node) + return std::unexpected(Node.error()); + Nodes.push_back(*Node); + } + for (std::size_t Left = 0; Left < Nodes.size(); ++Left) { + for (std::size_t Right = Left + 1; Right < Nodes.size(); ++Right) { + CostMatrix Costs(3, 3); + Costs.at(1, 1) = InfiniteCost; + Costs.at(2, 2) = InfiniteCost; + auto Edge = Problem.addEdge(Nodes[Left], Nodes[Right], std::move(Costs), + static_cast<std::uint64_t>(Left * Nodes.size() + Right)); + if (!Edge) + return std::unexpected(Edge.error()); + } + } + return DeterministicSolver{}.solve(Problem); +} + +void testHeuristicAndDeterminism(TestContext &Context) { + auto First = solveClique(); + auto Second = solveClique(); + Context.expect(First && Second && std::ranges::equal(First->selections(), Second->selections()) && + First->totalCost() == Second->totalCost(), + "PBQP higher-degree reduction must be deterministic"); + if (First) { + Context.expect(First->totalCost() == 18, + "a four-node two-register clique must spill exactly two nodes"); + } + + rcc::pbqp::Graph Impossible; + auto Node = Impossible.addNode( + rcc::pbqp::CostVector(std::vector<rcc::pbqp::Cost>{rcc::pbqp::InfiniteCost})); + auto Failure = rcc::pbqp::DeterministicSolver{}.solve(Impossible); + Context.expect(Node && !Failure && + Failure.error().code() == rcc::ErrorCode::RegisterAllocationFailed, + "PBQP solver must diagnose a problem with no finite assignment"); +} + +} // namespace + +int main() { + TestContext Context; + testMath(Context); + testGraphValidation(Context); + testReductions(Context); + testHeuristicAndDeterminism(Context); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/RegAllocBasicTest.cpp b/backend/unittests/CodeGen/RegAllocBasicTest.cpp new file mode 100644 index 0000000..015fe37 --- /dev/null +++ b/backend/unittests/CodeGen/RegAllocBasicTest.cpp @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "CodeGen/RegisterAllocationTestSupport.h" + +#include <iostream> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +[[nodiscard]] Expected<MachineInstr *> buildEarly(MachineFunction &Function, + const TargetDescription &Target, + MachineBasicBlock &Block, Register Destination, + Register Source) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(EarlyOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + RegisterOperandFlags Flags; + Flags.set(RegisterOperandFlag::EarlyClobber); + if (auto Result = Pending->addDef(Destination, Flags); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +void testHintsAndPressure(TestContext &Context, const TargetDescription &Target) { + auto HintFixture = createFunction("basic-hint", 30); + if (!HintFixture) + return; + auto HintValue = createVReg(*HintFixture->Function); + if (!HintValue || !buildDef(*HintFixture->Function, Target, *HintFixture->Block, *HintValue, 1) || + !buildUse(*HintFixture->Function, Target, *HintFixture->Block, *HintValue) || + !HintFixture->Function->registerInfo().addAllocationHint(*HintValue, + physical(TestRegister::G1)) || + !prepareForAllocation(*HintFixture->Function, Target)) + return; + TestRegisterAllocationInfo Info; + auto Allocated = runAllocator(*HintFixture->Function, Target, Info, AllocatorKind::Basic); + const MachineInstr *Use = HintFixture->Block->lastInstruction(); + Context.expect(Allocated && Use != nullptr && + Use->operandAt(0)->registerValue() == + Register::fromPhysical(physical(TestRegister::G1)), + "BasicAllocator must honor the first allocation hint"); + + auto Pressure = createFunction("basic-pressure", 31); + if (!Pressure) + return; + std::vector<Register> Values; + for (std::int64_t Value = 0; Value < 3; ++Value) { + auto Reg = createVReg(*Pressure->Function); + if (!Reg || !buildDef(*Pressure->Function, Target, *Pressure->Block, *Reg, Value)) + return; + Values.push_back(*Reg); + } + for (const Register Reg : Values) + if (!buildUse(*Pressure->Function, Target, *Pressure->Block, Reg)) + return; + if (!prepareForAllocation(*Pressure->Function, Target)) + return; + Allocated = runAllocator(*Pressure->Function, Target, Info, AllocatorKind::Basic, + {.EnableRematerialization = false}); + Context.expect(Allocated && + Pressure->Function->properties().has(MachineFunctionProperty::NoVRegs) && + Pressure->Function->frameInfo().liveObjectCount() != 0, + "BasicAllocator must close spill/reload pressure"); +} + +void testEarlyClobberAndRewrite(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("basic-constraints", 32); + if (!Fixture) + return; + auto Source = createVReg(*Fixture->Function); + auto Destination = createVReg(*Fixture->Function); + if (!Source || !Destination || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *Source, 1) || + !buildEarly(*Fixture->Function, Target, *Fixture->Block, *Destination, *Source) || + !buildTied(*Fixture->Function, Target, *Fixture->Block, *Destination) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Destination) || + !prepareForAllocation(*Fixture->Function, Target)) + return; + TestRegisterAllocationInfo Info; + auto Allocated = runAllocator(*Fixture->Function, Target, Info, AllocatorKind::Basic); + const MachineInstr *Early = Fixture->Block->firstInstruction()->next(); + const MachineInstr *Tied = Early == nullptr ? nullptr : Early->next(); + Context.expect(Allocated && Early != nullptr && Tied != nullptr && + Early->operandAt(0)->registerValue() != Early->operandAt(1)->registerValue() && + Tied->operandAt(0)->registerValue() == Tied->operandAt(1)->registerValue(), + "BasicAllocator must preserve early-clobber and tied constraints"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "BasicAllocator test target must be constructible"); + if (!Target) + return Context.result(); + testHintsAndPressure(Context, *Target); + testEarlyClobberAndRewrite(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/RegAllocFastTest.cpp b/backend/unittests/CodeGen/RegAllocFastTest.cpp new file mode 100644 index 0000000..cf5498f --- /dev/null +++ b/backend/unittests/CodeGen/RegAllocFastTest.cpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "CodeGen/RegisterAllocationTestSupport.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" + +#include <iostream> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +void testBlockBoundary(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("fast-boundary", 40); + if (!Fixture) + return; + auto Exit = Fixture->Function->createBasicBlock(); + if (!Exit || !Fixture->Function->appendBlock(**Exit) || + !Fixture->Function->cfgUpdater().addSuccessor(*Fixture->Block, **Exit, + BranchProbability::one())) + return; + auto Value = createVReg(*Fixture->Function); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 9) || + !buildUse(*Fixture->Function, Target, **Exit, *Value) || + !buildUse(*Fixture->Function, Target, **Exit, + Register::fromPhysical(physical(TestRegister::G0))) || + !prepareForAllocation(*Fixture->Function, Target)) + return; + TestRegisterAllocationInfo Info; + auto Allocated = runAllocator(*Fixture->Function, Target, Info, AllocatorKind::Fast); + Context.expect(Allocated && countOpcode(*Fixture->Function, testOpcode(SpillStoreOpcode)) == 1 && + countOpcode(*Fixture->Function, testOpcode(SpillLoadOpcode)) == 1 && + Fixture->Function->properties().has(MachineFunctionProperty::NoVRegs), + "FastAllocator must preserve values across a block boundary"); +} + +void testFunctionLiveInAndDebug(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("fast-live-in", 41); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + const PhysicalRegister G0 = physical(TestRegister::G0); + if (!Value || !Fixture->Function->registerInfo().addLiveIn(G0, *Value) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value, {}, true) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value) || + !prepareForAllocation(*Fixture->Function, Target)) + return; + TestRegisterAllocationInfo Info; + auto Allocated = runAllocator(*Fixture->Function, Target, Info, AllocatorKind::Fast); + const MachineOperand *DebugOperand = nullptr; + for (const MachineInstr *Instruction = Fixture->Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isDebugUse()) { + DebugOperand = &Operand; + break; + } + } + if (DebugOperand != nullptr) + break; + } + Context.expect( + Allocated && DebugOperand != nullptr && DebugOperand->registerValue() && + (DebugOperand->registerValue()->isNoRegister() || + DebugOperand->registerValue() == Register::fromPhysical(G0)) && + std::ranges::none_of(Fixture->Function->registerInfo().virtualRegisters(), + [](const VirtualRegisterInfo &Info) { return !Info.IsTombstone; }), + "FastAllocator must preserve function live-ins and debug locations"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "FastAllocator test target must be constructible"); + if (!Target) + return Context.result(); + testBlockBoundary(Context, *Target); + testFunctionLiveInAndDebug(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/RegAllocPBQPTest.cpp b/backend/unittests/CodeGen/RegAllocPBQPTest.cpp new file mode 100644 index 0000000..9b70a9b --- /dev/null +++ b/backend/unittests/CodeGen/RegAllocPBQPTest.cpp @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "CodeGen/RegisterAllocationTestSupport.h" + +#include <iostream> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +[[nodiscard]] Expected<FunctionFixture> createPressureFunction(const TargetDescription &Target, + std::uint64_t ID, bool WithCopy) { + auto Fixture = createFunction("pbqp-pressure", ID); + if (!Fixture) + return std::unexpected(Fixture.error()); + std::vector<Register> Values; + for (std::int64_t Value = 0; Value < 3; ++Value) { + auto Reg = createVReg(*Fixture->Function); + if (!Reg || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Reg, Value)) + return makeUnexpected(ErrorCode::InternalError, "failed to construct PBQP pressure value"); + Values.push_back(*Reg); + } + if (WithCopy && !buildCopy(*Fixture->Function, Target, *Fixture->Block, Values[1], Values[0])) + return makeUnexpected(ErrorCode::InternalError, "failed to construct PBQP copy"); + for (const Register Reg : Values) + if (!buildUse(*Fixture->Function, Target, *Fixture->Block, Reg)) + return makeUnexpected(ErrorCode::InternalError, "failed to construct PBQP use"); + auto Prepared = prepareForAllocation(*Fixture->Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + return Fixture; +} + +[[nodiscard]] std::vector<std::uint64_t> signature(const MachineFunction &Function) { + std::vector<std::uint64_t> Result; + Result.push_back(Function.frameInfo().liveObjectCount()); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.push_back(Instruction->opcode().value()); + for (const MachineOperand &Operand : Instruction->operands()) { + Result.push_back(static_cast<std::uint64_t>(Operand.kind())); + if (const Register *Reg = Operand.getIf<Register>()) + Result.push_back(Reg->opaqueValue()); + } + } + } + return Result; +} + +void testHintsAndDeterminism(TestContext &Context, const TargetDescription &Target) { + auto HintFixture = createFunction("pbqp-hint", 50); + if (!HintFixture) + return; + auto Value = createVReg(*HintFixture->Function); + if (!Value || !buildDef(*HintFixture->Function, Target, *HintFixture->Block, *Value, 1) || + !buildUse(*HintFixture->Function, Target, *HintFixture->Block, *Value) || + !HintFixture->Function->registerInfo().addAllocationHint(*Value, + physical(TestRegister::G1)) || + !HintFixture->Function->registerInfo().addAllocationHint(*Value, + physical(TestRegister::G0)) || + !prepareForAllocation(*HintFixture->Function, Target)) + return; + TestRegisterAllocationInfo Info; + auto Allocated = runAllocator(*HintFixture->Function, Target, Info, AllocatorKind::PBQP); + const MachineInstr *Use = HintFixture->Block->lastInstruction(); + Context.expect(Allocated && Use != nullptr && + Use->operandAt(0)->registerValue() == + Register::fromPhysical(physical(TestRegister::G1)), + "PBQP must preserve allocation-hint preference order"); + + auto First = createPressureFunction(Target, 51, false); + auto Second = createPressureFunction(Target, 52, false); + if (!First || !Second) + return; + auto FirstResult = runAllocator(*First->Function, Target, Info, AllocatorKind::PBQP, + {.EnableRematerialization = false}); + auto SecondResult = runAllocator(*Second->Function, Target, Info, AllocatorKind::PBQP, + {.EnableRematerialization = false}); + Context.expect(FirstResult && SecondResult && + signature(*First->Function) == signature(*Second->Function), + "PBQP allocation and spill/rebuild order must be deterministic"); +} + +void testCopyCoalescingSwitch(TestContext &Context, const TargetDescription &Target) { + TestRegisterAllocationInfo Info; + auto Coalesced = createPressureFunction(Target, 53, true); + auto Uncoalesced = createPressureFunction(Target, 54, true); + if (!Coalesced || !Uncoalesced) + return; + auto WithCoalescing = + runAllocator(*Coalesced->Function, Target, Info, AllocatorKind::PBQP, + {.EnableRematerialization = false, .EnableCopyCoalescing = true}); + auto WithoutCoalescing = + runAllocator(*Uncoalesced->Function, Target, Info, AllocatorKind::PBQP, + {.EnableRematerialization = false, .EnableCopyCoalescing = false}); + Context.expect(WithCoalescing && WithoutCoalescing && + Coalesced->Function->properties().has(MachineFunctionProperty::NoVRegs) && + Uncoalesced->Function->properties().has(MachineFunctionProperty::NoVRegs), + "PBQP copy-coalescing modes must both produce valid physical mappings"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "PBQP allocator test target must be constructible"); + if (!Target) + return Context.result(); + testHintsAndDeterminism(Context, *Target); + testCopyCoalescingSwitch(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/RegisterAllocationTest.cpp b/backend/unittests/CodeGen/RegisterAllocationTest.cpp new file mode 100644 index 0000000..a44cd2e --- /dev/null +++ b/backend/unittests/CodeGen/RegisterAllocationTest.cpp @@ -0,0 +1,1043 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "Target/TestTarget.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveRangeEdit.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/RegAllocFast.h" +#include "rcc/CodeGen/RegAllocPBQP.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/Spiller.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iostream> +#include <memory> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace { + +constexpr std::uint32_t DefOpcode = 3000; +constexpr std::uint32_t RematerializableDefOpcode = 3001; +constexpr std::uint32_t UseOpcode = 3002; +constexpr std::uint32_t BinaryOpcode = 3003; +constexpr std::uint32_t SpillLoadOpcode = 3004; +constexpr std::uint32_t SpillStoreOpcode = 3005; +constexpr std::uint32_t MaskOpcode = 3006; +constexpr std::uint32_t EarlyOpcode = 3007; +constexpr std::uint32_t TiedOpcode = 3008; +constexpr std::uint32_t CallDefOpcode = 3009; + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +[[nodiscard]] constexpr rcc::Opcode testOpcode(std::uint32_t Value) noexcept { + return rcc::Opcode::fromValue(Value); +} + +[[nodiscard]] rcc::PhysicalRegister physical(std::uint32_t Number) { + return *rcc::PhysicalRegister::create(Number); +} + +[[nodiscard]] rcc::OperandDescriptor registerOperand(rcc::RegisterOperandRole Role, + bool EarlyClobber = false) { + rcc::OperandDescriptor Result; + Result.Kinds = {rcc::MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.AllowSubReg = true; + Result.EarlyClobber = EarlyClobber; + return Result; +} + +[[nodiscard]] rcc::OperandDescriptor plainOperand(rcc::MachineOperand::Kind Kind) { + rcc::OperandDescriptor Result; + Result.Kinds = {Kind}; + return Result; +} + +[[nodiscard]] rcc::InstructionDescriptor descriptor(std::uint32_t Value, std::string Name) { + rcc::InstructionDescriptor Result; + Result.InstructionOpcode = testOpcode(Value); + Result.Name = std::move(Name); + Result.Flags = {rcc::InstructionFlag::ProducesCode}; + return Result; +} + +[[nodiscard]] rcc::Expected<rcc::TargetDescription> +createTarget(bool AddSharedMultiLaneUnit = false) { + std::vector<rcc::InstructionDescriptor> Descriptors; + + auto Def = descriptor(DefOpcode, "RA_DEF"); + Def.Operands = {registerOperand(rcc::RegisterOperandRole::Def), + plainOperand(rcc::MachineOperand::Kind::Immediate)}; + Def.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Def)); + + auto Remat = descriptor(RematerializableDefOpcode, "RA_REMAT_DEF"); + Remat.Operands = {registerOperand(rcc::RegisterOperandRole::Def), + plainOperand(rcc::MachineOperand::Kind::Immediate)}; + Remat.NumExplicitDefs = 1; + Remat.Flags.set(rcc::InstructionFlag::Rematerializable); + Descriptors.push_back(std::move(Remat)); + + auto Use = descriptor(UseOpcode, "RA_USE"); + Use.Operands = {registerOperand(rcc::RegisterOperandRole::Use)}; + Use.Flags.set(rcc::InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(Use)); + + auto Binary = descriptor(BinaryOpcode, "RA_BINARY"); + Binary.Operands = {registerOperand(rcc::RegisterOperandRole::Def), + registerOperand(rcc::RegisterOperandRole::Use), + registerOperand(rcc::RegisterOperandRole::Use)}; + Binary.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Binary)); + + auto Reload = descriptor(SpillLoadOpcode, "RA_RELOAD"); + Reload.Operands = {registerOperand(rcc::RegisterOperandRole::Def), + plainOperand(rcc::MachineOperand::Kind::FrameIndex)}; + Reload.NumExplicitDefs = 1; + Reload.Flags.set(rcc::InstructionFlag::MayLoad); + Reload.RequiresMemoryOperands = true; + Descriptors.push_back(std::move(Reload)); + + auto Spill = descriptor(SpillStoreOpcode, "RA_SPILL"); + Spill.Operands = {registerOperand(rcc::RegisterOperandRole::Use), + plainOperand(rcc::MachineOperand::Kind::FrameIndex)}; + Spill.Flags.set(rcc::InstructionFlag::MayStore); + Spill.RequiresMemoryOperands = true; + Descriptors.push_back(std::move(Spill)); + + auto Mask = descriptor(MaskOpcode, "RA_MASK"); + Mask.Operands = {plainOperand(rcc::MachineOperand::Kind::RegisterMask)}; + Mask.Flags.set(rcc::InstructionFlag::Call); + Mask.Flags.set(rcc::InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(Mask)); + + auto Early = descriptor(EarlyOpcode, "RA_EARLY"); + Early.Operands = {registerOperand(rcc::RegisterOperandRole::Def, true), + registerOperand(rcc::RegisterOperandRole::Use)}; + Early.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Early)); + + auto Tied = descriptor(TiedOpcode, "RA_TIED"); + auto TiedDef = registerOperand(rcc::RegisterOperandRole::Def); + auto TiedUse = registerOperand(rcc::RegisterOperandRole::Use); + TiedDef.DefaultTiedOperand = 1; + TiedUse.DefaultTiedOperand = 0; + Tied.Operands = {TiedDef, TiedUse}; + Tied.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Tied)); + + auto CallDef = descriptor(CallDefOpcode, "RA_CALL_DEF"); + CallDef.Operands = {registerOperand(rcc::RegisterOperandRole::Def), + plainOperand(rcc::MachineOperand::Kind::RegisterMask)}; + CallDef.NumExplicitDefs = 1; + CallDef.Flags.set(rcc::InstructionFlag::Call); + CallDef.Flags.set(rcc::InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(CallDef)); + + auto Instructions = rcc::InstructionSet::create(std::move(Descriptors), {}); + if (!Instructions) + return std::unexpected(Instructions.error()); + if (AddSharedMultiLaneUnit) { + auto Description = rcc::test::createTestTargetRegisterInfoDescription(); + const rcc::RegisterUnitID SharedUnit = rcc::RegisterUnitID::fromValue(12); + Description.RegisterUnits.push_back({SharedUnit, "shared.multi-lane"}); + for (rcc::PhysicalRegisterDesc &Register : Description.PhysicalRegisters) { + if (Register.Reg == physical(rcc::test::TestRegister::G0) || + Register.Reg == physical(rcc::test::TestRegister::G1)) { + Register.Units.push_back({SharedUnit, rcc::LaneMask::fromRaw(0xF)}); + } + } + auto Registers = rcc::TargetRegisterInfo::create(std::move(Description)); + if (!Registers) + return std::unexpected(Registers.error()); + return rcc::TargetDescription::create(std::move(*Instructions), std::move(*Registers), + rcc::test::createTestTargetMIRTextInfoDescription()); + } + return rcc::test::createTestTargetDescription(std::move(*Instructions)); +} + +[[nodiscard]] rcc::Expected<const rcc::MachineMemOperand *> +createStackMemory(rcc::MachineIRBuilder &Builder, rcc::FrameIndex StackSlot, + rcc::MemoryAccessFlag Access) { + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + if (!Size) + return std::unexpected(Size.error()); + if (!Align) + return std::unexpected(Align.error()); + rcc::MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::frameIndex(StackSlot); + return Builder.function().createMemOperand(std::move(Description)); +} + +class TestRegisterAllocationInfo final : public rcc::TargetRegisterAllocationInfo { +public: + [[nodiscard]] rcc::Expected<void> + validateSpillReload(rcc::RegisterClassID Class, + const rcc::TargetDescription &Target) const override { + const rcc::RegisterClassDesc *Description = Target.registers().registerClass(Class); + if (Description == nullptr || !Description->Allocatable || Description->SpillSizeInBits != 64 || + Description->SpillAlignmentInBits != 64) { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, + "test target register class is not spillable"); + } + return {}; + } + + [[nodiscard]] rcc::Expected<rcc::MachineInstr *> + emitReload(rcc::MachineIRBuilder &Builder, rcc::Register Destination, rcc::RegisterClassID Class, + rcc::FrameIndex StackSlot, rcc::LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + if (Lanes.empty()) + return rcc::makeUnexpected(rcc::ErrorCode::InvalidArgument, "test reload requires lanes"); + auto Memory = createStackMemory(Builder, StackSlot, rcc::MemoryAccessFlag::Load); + if (!Memory) + return std::unexpected(Memory.error()); + auto Pending = Builder.buildInstr(testOpcode(SpillLoadOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**Memory); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] rcc::Expected<rcc::MachineInstr *> + emitSpill(rcc::MachineIRBuilder &Builder, rcc::Register Source, rcc::RegisterClassID Class, + rcc::FrameIndex StackSlot, rcc::LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + if (Lanes.empty()) + return rcc::makeUnexpected(rcc::ErrorCode::InvalidArgument, "test spill requires lanes"); + auto Memory = createStackMemory(Builder, StackSlot, rcc::MemoryAccessFlag::Store); + if (!Memory) + return std::unexpected(Memory.error()); + auto Pending = Builder.buildInstr(testOpcode(SpillStoreOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**Memory); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] rcc::Expected<bool> + isRematerializable(const rcc::MachineInstr &Instruction, rcc::Register VirtualReg, + const rcc::MachineFunction &, const rcc::TargetDescription &) const override { + const rcc::MachineOperand *Operand = Instruction.operandAt(0); + const rcc::Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<rcc::Register>(); + return Instruction.opcode() == testOpcode(RematerializableDefOpcode) && Reg != nullptr && + *Reg == VirtualReg; + } + + [[nodiscard]] rcc::Expected<rcc::MachineInstr *> + rematerialize(rcc::MachineIRBuilder &Builder, const rcc::MachineInstr &Original, + rcc::Register Destination, rcc::Register OriginalRegister, + const rcc::TargetDescription &Target) const override { + auto Accepted = isRematerializable(Original, OriginalRegister, Builder.function(), Target); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return rcc::makeUnexpected(rcc::ErrorCode::InvalidArgument, + "test instruction is not rematerializable"); + auto Pending = Builder.buildInstr(testOpcode(RematerializableDefOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + const rcc::MachineOperand *Immediate = Original.operandAt(1); + if (Immediate == nullptr) + return rcc::makeUnexpected(rcc::ErrorCode::InternalError, + "test rematerialization lost its immediate"); + if (auto Result = Pending->addOperand(Immediate->cloneDetached()); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } +}; + +class RejectingRegisterAllocationInfo final : public rcc::TargetRegisterAllocationInfo { +public: + [[nodiscard]] rcc::Expected<void> + validateSpillReload(rcc::RegisterClassID, const rcc::TargetDescription &) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, + "test target intentionally rejects spills"); + } + + [[nodiscard]] rcc::Expected<rcc::MachineInstr *> emitReload(rcc::MachineIRBuilder &, + rcc::Register, rcc::RegisterClassID, + rcc::FrameIndex, + rcc::LaneMask) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, + "test target intentionally rejects reloads"); + } + + [[nodiscard]] rcc::Expected<rcc::MachineInstr *> emitSpill(rcc::MachineIRBuilder &, rcc::Register, + rcc::RegisterClassID, rcc::FrameIndex, + rcc::LaneMask) const override { + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, + "test target intentionally rejects spills"); + } +}; + +struct FunctionFixture final { + std::unique_ptr<rcc::MachineFunction> Function; + rcc::MachineBasicBlock *Block = nullptr; +}; + +[[nodiscard]] rcc::Expected<FunctionFixture> createFunction(std::string Name, std::uint64_t ID) { + auto Function = + rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID)); + if (!Function) + return std::unexpected(Function.error()); + auto Block = (*Function)->createBasicBlock(); + if (!Block) + return std::unexpected(Block.error()); + auto Appended = (*Function)->appendBlock(**Block); + if (!Appended) + return std::unexpected(Appended.error()); + return FunctionFixture{std::move(*Function), *Block}; +} + +[[nodiscard]] rcc::Expected<rcc::Register> +createVReg(rcc::MachineFunction &Function, + rcc::RegisterClassID Class = rcc::test::TestRegisterClass::GPR64) { + auto Type = rcc::LLT::makeInteger(64); + if (!Type) + return std::unexpected(Type.error()); + auto Reg = Function.registerInfo().createVirtualRegister(*Type); + if (!Reg) + return std::unexpected(Reg.error()); + if (auto Result = Function.registerInfo().setRegisterBank(*Reg, rcc::test::TestRegisterBank::GPR); + !Result) { + return std::unexpected(Result.error()); + } + if (auto Result = Function.registerInfo().setRegisterClass(*Reg, Class); !Result) + return std::unexpected(Result.error()); + return *Reg; +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildDef(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, rcc::Register Reg, std::int64_t Value, + rcc::SubRegIndex SubReg = {}, bool Rematerializable = false) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = + Builder.buildInstr(testOpcode(Rematerializable ? RematerializableDefOpcode : DefOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Reg, {}, SubReg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addImmediate(Value); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildUse(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, rcc::Register Reg, rcc::SubRegIndex SubReg = {}) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(UseOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addUse(Reg, {}, SubReg); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildMask(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, std::shared_ptr<const rcc::RegisterMask> Mask) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(MaskOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addRegisterMask(std::move(Mask)); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildCallDef(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, rcc::Register Destination, + std::shared_ptr<const rcc::RegisterMask> Mask) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(CallDefOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addRegisterMask(std::move(Mask)); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildEarly(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, rcc::Register Destination, rcc::Register Source) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(EarlyOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + rcc::RegisterOperandFlags Flags; + Flags.set(rcc::RegisterOperandFlag::EarlyClobber); + if (auto Result = Pending->addDef(Destination, Flags); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> buildTied(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, + rcc::Register Reg) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(TiedOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Reg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Reg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->tieOperands(0, 1); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<void> prepareForAllocation(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) { + auto Rebuilt = rcc::rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + for (const rcc::MachineFunctionProperty Property : { + rcc::MachineFunctionProperty::TracksLiveness, + rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected, + rcc::MachineFunctionProperty::Selected, + rcc::MachineFunctionProperty::NoPHIs, + rcc::MachineFunctionProperty::TiedOpsRewritten, + }) { + Function.properties().set(Property); + } + return {}; +} + +enum class AllocatorKind : std::uint8_t { + Basic, + Fast, + PBQP, +}; + +[[nodiscard]] rcc::Expected<void> runAllocator(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target, + const rcc::TargetRegisterAllocationInfo &Info, + AllocatorKind Kind, + rcc::RegisterAllocationOptions Options = {}) { + rcc::MachineAnalysisManager Analyses; + auto Registered = rcc::registerStage11MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + std::unique_ptr<rcc::MachineFunctionPass> Pass; + switch (Kind) { + case AllocatorKind::Basic: + Pass = rcc::createBasicAllocatorPass(Info, Options); + break; + case AllocatorKind::Fast: + Pass = std::make_unique<rcc::FastAllocatorPass>(Info); + break; + case AllocatorKind::PBQP: + Pass = std::make_unique<rcc::PBQPAllocatorPass>(Info, Options); + break; + } + rcc::MachinePassManager Pipeline; + auto Added = Pipeline.addPass(std::move(Pass)); + if (!Added) + return std::unexpected(Added.error()); + return Pipeline.run(Function, Target, Analyses); +} + +[[nodiscard]] std::size_t countOpcode(const rcc::MachineFunction &Function, std::uint32_t Value) { + std::size_t Count = 0; + for (const rcc::MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Count += Instruction->opcode() == testOpcode(Value) ? 1U : 0U; + } + } + return Count; +} + +[[nodiscard]] bool hasInterference(std::span<const rcc::RegisterInterference> Values, + rcc::RegisterInterferenceKind Kind) { + return std::ranges::any_of( + Values, [Kind](const rcc::RegisterInterference &Value) { return Value.Kind == Kind; }); +} + +[[nodiscard]] std::vector<std::uint64_t> functionSignature(const rcc::MachineFunction &Function) { + std::vector<std::uint64_t> Result; + Result.push_back(Function.frameInfo().liveObjectCount()); + for (const rcc::MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Result.push_back(Block->stableID().value()); + for (const rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.push_back(Instruction->opcode().value()); + Result.push_back(Instruction->operandCount()); + for (const rcc::MachineOperand &Operand : Instruction->operands()) { + Result.push_back(static_cast<std::uint64_t>(Operand.kind())); + const rcc::Register *Reg = Operand.getIf<rcc::Register>(); + Result.push_back(Reg == nullptr ? 0 : Reg->opaqueValue()); + } + } + } + return Result; +} + +void testVirtRegMapAndEdit(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = createFunction("virtreg-map", 1); + if (!Fixture) + return; + auto Parent = createVReg(*Fixture->Function); + if (!Parent) + return; + rcc::VirtRegMap Mapping(*Fixture->Function); + Context.expect(Mapping.add(*Parent).has_value(), + "VirtRegMap must accept a known virtual register"); + rcc::LiveRangeEdit Edit(*Fixture->Function, Target, Mapping); + auto Child = Edit.createFrom(*Parent, false); + auto Slot = Edit.getOrCreateSpillSlot(*Parent); + Context.expect(Child && Slot && Mapping.stackSlot(*Parent) == Slot && + Mapping.entry(*Child) != nullptr && + Mapping.entry(*Child)->SplitParent == *Parent && + Mapping.entry(*Child)->Origin == *Parent && Mapping.verify().has_value(), + "LiveRangeEdit must preserve class, origin, split, and spill-slot state"); + if (!Child) + return; + const auto G0 = physical(rcc::test::TestRegister::G0); + Context.expect(Mapping.assign(*Child, G0).has_value() && Mapping.physicalRegister(*Child) == G0 && + !Mapping.isSpillable(*Child), + "VirtRegMap must publish deterministic child assignments"); +} + +void testLiveRegMatrix(TestContext &Context, const rcc::TargetDescription &Target) { + auto LaneFixture = createFunction("lane-matrix", 2); + if (!LaneFixture) + return; + auto Low = createVReg(*LaneFixture->Function); + auto High = createVReg(*LaneFixture->Function); + if (!Low || !High) + return; + if (!buildDef(*LaneFixture->Function, Target, *LaneFixture->Block, *Low, 1, + rcc::test::TestSubReg::Low32) || + !buildDef(*LaneFixture->Function, Target, *LaneFixture->Block, *High, 2, + rcc::test::TestSubReg::High32) || + !buildUse(*LaneFixture->Function, Target, *LaneFixture->Block, *High, + rcc::test::TestSubReg::High32) || + !buildUse(*LaneFixture->Function, Target, *LaneFixture->Block, *Low, + rcc::test::TestSubReg::Low32)) { + return; + } + if (!rcc::rebuildMachineLiveness(*LaneFixture->Function, Target)) + return; + rcc::MachineAnalysisManager LaneAnalyses; + if (!rcc::registerStage11MachineAnalyses(LaneAnalyses)) + return; + auto LaneIntervals = + LaneAnalyses.getResult<rcc::LiveIntervalsAnalysis>(*LaneFixture->Function, Target); + if (!LaneIntervals) + return; + const auto G0 = physical(rcc::test::TestRegister::G0); + rcc::LiveRegMatrix LaneMatrix(*LaneFixture->Function, Target, **LaneIntervals); + Context.expect(LaneMatrix.assign(*Low, G0).has_value() && + LaneMatrix.checkInterference(*High, G0).Kind == + rcc::RegisterInterferenceKind::Free && + LaneMatrix.assign(*High, G0).has_value(), + "LiveRegMatrix must permit overlapping intervals on disjoint subregister lanes"); + + auto FixedFixture = createFunction("fixed-matrix", 3); + if (!FixedFixture) + return; + auto Value = createVReg(*FixedFixture->Function); + if (!Value || !buildDef(*FixedFixture->Function, Target, *FixedFixture->Block, *Value, 7) || + !buildMask(*FixedFixture->Function, Target, *FixedFixture->Block, + rcc::test::createTestCallPreservedMask()) || + !buildUse(*FixedFixture->Function, Target, *FixedFixture->Block, + rcc::Register::fromPhysical(G0)) || + !buildUse(*FixedFixture->Function, Target, *FixedFixture->Block, *Value)) { + return; + } + if (!rcc::rebuildMachineLiveness(*FixedFixture->Function, Target)) + return; + rcc::MachineAnalysisManager FixedAnalyses; + if (!rcc::registerStage11MachineAnalyses(FixedAnalyses)) + return; + auto FixedIntervals = + FixedAnalyses.getResult<rcc::LiveIntervalsAnalysis>(*FixedFixture->Function, Target); + if (!FixedIntervals) + return; + rcc::LiveRegMatrix FixedMatrix(*FixedFixture->Function, Target, **FixedIntervals); + const auto G0Interferences = FixedMatrix.collectInterferences(*Value, G0); + const auto G1 = physical(rcc::test::TestRegister::G1); + Context.expect( + hasInterference(G0Interferences, rcc::RegisterInterferenceKind::RegisterMask) && + (hasInterference(G0Interferences, rcc::RegisterInterferenceKind::FixedRegister) || + hasInterference(G0Interferences, rcc::RegisterInterferenceKind::RegisterUnit)) && + FixedMatrix.checkInterference(*Value, G1).Kind == rcc::RegisterInterferenceKind::Free, + "LiveRegMatrix must classify fixed, unit, and RegMask interference separately"); + + auto FixedLaneFixture = createFunction("fixed-lane-matrix", 4); + if (!FixedLaneFixture) + return; + auto HighValue = createVReg(*FixedLaneFixture->Function); + const auto G0Low32 = physical(rcc::test::TestRegister::G0Low32); + if (!HighValue || + !buildDef(*FixedLaneFixture->Function, Target, *FixedLaneFixture->Block, *HighValue, 9, + rcc::test::TestSubReg::High32) || + !buildUse(*FixedLaneFixture->Function, Target, *FixedLaneFixture->Block, + rcc::Register::fromPhysical(G0Low32)) || + !buildUse(*FixedLaneFixture->Function, Target, *FixedLaneFixture->Block, *HighValue, + rcc::test::TestSubReg::High32) || + !rcc::rebuildMachineLiveness(*FixedLaneFixture->Function, Target)) { + return; + } + rcc::MachineAnalysisManager FixedLaneAnalyses; + if (!rcc::registerStage11MachineAnalyses(FixedLaneAnalyses)) + return; + auto FixedLaneIntervals = + FixedLaneAnalyses.getResult<rcc::LiveIntervalsAnalysis>(*FixedLaneFixture->Function, Target); + if (!FixedLaneIntervals) + return; + rcc::LiveRegMatrix FixedLaneMatrix(*FixedLaneFixture->Function, Target, **FixedLaneIntervals); + Context.expect(FixedLaneMatrix.checkInterference(*HighValue, G0).Kind == + rcc::RegisterInterferenceKind::Free, + "a fixed subregister must not interfere with a disjoint virtual lane"); +} + +void testSharedUnitLaneMatrix(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = createFunction("shared-unit-lane-matrix", 5); + if (!Fixture) + return; + auto Low = createVReg(*Fixture->Function); + auto High = createVReg(*Fixture->Function); + if (!Low || !High || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *Low, 1, + rcc::test::TestSubReg::Low32) || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *High, 2, + rcc::test::TestSubReg::High32) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *High, + rcc::test::TestSubReg::High32) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Low, rcc::test::TestSubReg::Low32) || + !rcc::rebuildMachineLiveness(*Fixture->Function, Target)) { + return; + } + rcc::MachineAnalysisManager Analyses; + if (!rcc::registerStage11MachineAnalyses(Analyses)) + return; + auto Intervals = Analyses.getResult<rcc::LiveIntervalsAnalysis>(*Fixture->Function, Target); + if (!Intervals) + return; + const auto G0 = physical(rcc::test::TestRegister::G0); + const auto G1 = physical(rcc::test::TestRegister::G1); + rcc::LiveRegMatrix Matrix(*Fixture->Function, Target, **Intervals); + Context.expect(Target.registers().regsOverlap(G0, G1) && Matrix.assign(*Low, G0).has_value() && + Matrix.checkInterference(*High, G1).Kind == + rcc::RegisterInterferenceKind::Free && + Matrix.assign(*High, G1).has_value(), + "disjoint lanes must coexist on a shared multi-lane RegisterUnit"); +} + +[[nodiscard]] rcc::Expected<FunctionFixture> +createPressureFunction(const rcc::TargetDescription &Target, std::uint64_t ID) { + auto Fixture = createFunction("pressure-" + std::to_string(ID), ID); + if (!Fixture) + return std::unexpected(Fixture.error()); + std::vector<rcc::Register> Values; + for (std::int64_t Index = 0; Index < 3; ++Index) { + auto Reg = createVReg(*Fixture->Function); + if (!Reg) + return std::unexpected(Reg.error()); + if (auto Built = buildDef(*Fixture->Function, Target, *Fixture->Block, *Reg, Index); !Built) + return std::unexpected(Built.error()); + Values.push_back(*Reg); + } + for (const rcc::Register Reg : Values) { + if (auto Built = buildUse(*Fixture->Function, Target, *Fixture->Block, Reg); !Built) + return std::unexpected(Built.error()); + } + auto Prepared = prepareForAllocation(*Fixture->Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + return Fixture; +} + +void testAllocatorsUnderPressure(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + for (const auto [Kind, ID, Name] : { + std::tuple{AllocatorKind::Basic, std::uint64_t{10}, std::string_view{"Basic"}}, + std::tuple{AllocatorKind::Fast, std::uint64_t{11}, std::string_view{"Fast"}}, + std::tuple{AllocatorKind::PBQP, std::uint64_t{12}, std::string_view{"PBQP"}}, + }) { + auto Fixture = createPressureFunction(Target, ID); + if (!Fixture) + continue; + auto Allocated = + runAllocator(*Fixture->Function, Target, Info, Kind, {.EnableRematerialization = false}); + if (!Allocated) + std::cerr << Name << " allocation error: " << Allocated.error().message() << '\n'; + Context.expect(Allocated.has_value() && + Fixture->Function->properties().has(rcc::MachineFunctionProperty::NoVRegs) && + Fixture->Function->frameInfo().liveObjectCount() != 0 && + countOpcode(*Fixture->Function, SpillLoadOpcode) != 0 && + countOpcode(*Fixture->Function, SpillStoreOpcode) != 0, + "all allocators must close a deterministic spill/reload cycle under pressure"); + } +} + +void testFailureAtomicity(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = createPressureFunction(Target, 13); + if (!Fixture) + return; + const std::vector<std::uint64_t> Before = functionSignature(*Fixture->Function); + const std::size_t KnownBefore = static_cast<std::size_t>(std::ranges::count_if( + Fixture->Function->registerInfo().virtualRegisters(), + [](const rcc::VirtualRegisterInfo &Info) { return !Info.IsTombstone; })); + RejectingRegisterAllocationInfo Rejecting; + auto Failed = runAllocator(*Fixture->Function, Target, Rejecting, AllocatorKind::Basic, + {.EnableRematerialization = false}); + const std::size_t KnownAfter = static_cast<std::size_t>(std::ranges::count_if( + Fixture->Function->registerInfo().virtualRegisters(), + [](const rcc::VirtualRegisterInfo &Info) { return !Info.IsTombstone; })); + Context.expect(!Failed && Failed.error().code() == rcc::ErrorCode::InternalError && + functionSignature(*Fixture->Function) == Before && KnownAfter == KnownBefore && + !Fixture->Function->properties().has(rcc::MachineFunctionProperty::NoVRegs), + "failed register allocation must leave the original MachineFunction unchanged"); +} + +void testFastBlockBoundary(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + auto Fixture = createFunction("fast-block-boundary", 14); + if (!Fixture) + return; + auto Exit = Fixture->Function->createBasicBlock(); + if (!Exit || !Fixture->Function->appendBlock(**Exit) || + !Fixture->Function->cfgUpdater().addSuccessor(*Fixture->Block, **Exit, + rcc::BranchProbability::one())) { + return; + } + auto Value = createVReg(*Fixture->Function); + const rcc::Register G0 = rcc::Register::fromPhysical(physical(rcc::test::TestRegister::G0)); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 9) || + !buildUse(*Fixture->Function, Target, **Exit, *Value) || + !buildUse(*Fixture->Function, Target, **Exit, G0) || + !prepareForAllocation(*Fixture->Function, Target)) { + return; + } + auto Allocated = runAllocator(*Fixture->Function, Target, Info, AllocatorKind::Fast); + Context.expect( + Allocated && countOpcode(*Fixture->Function, SpillStoreOpcode) == 1 && + countOpcode(*Fixture->Function, SpillLoadOpcode) == 1 && + (*Exit)->firstInstruction()->next() != nullptr && + (*Exit)->firstInstruction()->next()->operandAt(0)->registerValue() != G0, + "FastAllocator must preserve block live-ins and virtual values across a CFG boundary"); +} + +void testFastRegMaskCalls(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + auto LiveThrough = createFunction("fast-regmask-live-through", 16); + if (!LiveThrough) + return; + auto Value = createVReg(*LiveThrough->Function); + if (!Value || !buildDef(*LiveThrough->Function, Target, *LiveThrough->Block, *Value, 11) || + !buildMask(*LiveThrough->Function, Target, *LiveThrough->Block, + rcc::test::createTestCallPreservedMask()) || + !buildUse(*LiveThrough->Function, Target, *LiveThrough->Block, *Value) || + !prepareForAllocation(*LiveThrough->Function, Target)) { + return; + } + auto LiveThroughAllocated = + runAllocator(*LiveThrough->Function, Target, Info, AllocatorKind::Fast); + Context.expect(LiveThroughAllocated && + countOpcode(*LiveThrough->Function, SpillStoreOpcode) == 1 && + countOpcode(*LiveThrough->Function, SpillLoadOpcode) == 1, + "FastAllocator must spill and reload a value live through a clobbering RegMask"); + + auto CallResult = createFunction("fast-regmask-call-result", 17); + if (!CallResult) + return; + auto Result = createVReg(*CallResult->Function); + if (!Result || + !buildCallDef(*CallResult->Function, Target, *CallResult->Block, *Result, + rcc::test::createTestCallPreservedMask()) || + !buildUse(*CallResult->Function, Target, *CallResult->Block, *Result) || + !prepareForAllocation(*CallResult->Function, Target)) { + return; + } + auto ResultAllocated = runAllocator(*CallResult->Function, Target, Info, AllocatorKind::Fast); + Context.expect(ResultAllocated && countOpcode(*CallResult->Function, SpillStoreOpcode) == 0 && + countOpcode(*CallResult->Function, SpillLoadOpcode) == 0, + "FastAllocator must retain a call-defined result without reloading an " + "uninitialized spill slot"); +} + +void testTiedAndEarlyClobber(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + auto Fixture = createFunction("tied-early", 15); + if (!Fixture) + return; + auto Source = createVReg(*Fixture->Function); + auto Destination = createVReg(*Fixture->Function); + if (!Source || !Destination || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *Source, 1)) { + return; + } + auto Early = buildEarly(*Fixture->Function, Target, *Fixture->Block, *Destination, *Source); + auto Tied = buildTied(*Fixture->Function, Target, *Fixture->Block, *Destination); + if (!Early || !Tied || !buildUse(*Fixture->Function, Target, *Fixture->Block, *Destination) || + !prepareForAllocation(*Fixture->Function, Target)) { + return; + } + auto Allocated = runAllocator(*Fixture->Function, Target, Info, AllocatorKind::Basic); + const std::optional<rcc::Register> EarlyDef = (*Early)->operandAt(0)->registerValue(); + const std::optional<rcc::Register> EarlyUse = (*Early)->operandAt(1)->registerValue(); + const std::optional<rcc::Register> TiedDef = (*Tied)->operandAt(0)->registerValue(); + const std::optional<rcc::Register> TiedUse = (*Tied)->operandAt(1)->registerValue(); + Context.expect(Allocated && EarlyDef && EarlyUse && *EarlyDef != *EarlyUse && TiedDef && + TiedUse && *TiedDef == *TiedUse, + "allocation must separate early-clobber operands and preserve rewritten ties"); +} + +void testRematerialization(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + auto Fixture = createFunction("rematerialization", 20); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 42, {}, true) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value)) { + return; + } + rcc::VirtRegMap Mapping(*Fixture->Function); + if (!Mapping.add(*Value)) + return; + rcc::Spiller Spiller(*Fixture->Function, Target, Info, Mapping, true); + auto Children = Spiller.spill(*Value); + Context.expect(Children && Children->size() == 2 && + Fixture->Function->frameInfo().liveObjectCount() == 0 && + countOpcode(*Fixture->Function, RematerializableDefOpcode) == 2 && + countOpcode(*Fixture->Function, SpillLoadOpcode) == 0 && + countOpcode(*Fixture->Function, SpillStoreOpcode) == 0, + "Spiller must rematerialize each use without creating a stack object"); + + auto Unspillable = createVReg(*Fixture->Function); + if (!Unspillable || !Mapping.add(*Unspillable, false)) + return; + auto Rejected = Spiller.spill(*Unspillable); + Context.expect(!Rejected && Rejected.error().code() == rcc::ErrorCode::RegisterAllocationFailed, + "Spiller must reject pressure on a non-spillable virtual register"); +} + +void testRewriteAndConstraints(TestContext &Context, const rcc::TargetDescription &Target, + const TestRegisterAllocationInfo &Info) { + auto RewriteFixture = createFunction("subregister-rewrite", 30); + if (!RewriteFixture) + return; + auto Value = createVReg(*RewriteFixture->Function); + if (!Value || + !buildDef(*RewriteFixture->Function, Target, *RewriteFixture->Block, *Value, 1, + rcc::test::TestSubReg::Low32) || + !buildUse(*RewriteFixture->Function, Target, *RewriteFixture->Block, *Value, + rcc::test::TestSubReg::Low32)) { + return; + } + rcc::VirtRegMap Mapping(*RewriteFixture->Function); + const auto G0 = physical(rcc::test::TestRegister::G0); + if (!Mapping.add(*Value) || !Mapping.assign(*Value, G0)) + return; + auto Rewritten = rcc::rewriteVirtualRegisters(*RewriteFixture->Function, Target, Mapping); + const auto G0Low32 = physical(rcc::test::TestRegister::G0Low32); + Context.expect(Rewritten && *Rewritten && + RewriteFixture->Block->firstInstruction()->operandAt(0)->registerValue() == + rcc::Register::fromPhysical(G0Low32) && + RewriteFixture->Block->firstInstruction()->operandAt(0)->subReg().value() == 0, + "physical rewrite must project and clear virtual subregister operands"); + + auto MaskFixture = createFunction("mask-allocation", 31); + if (!MaskFixture) + return; + auto AcrossCall = createVReg(*MaskFixture->Function); + if (!AcrossCall || + !buildDef(*MaskFixture->Function, Target, *MaskFixture->Block, *AcrossCall, 5) || + !buildMask(*MaskFixture->Function, Target, *MaskFixture->Block, + rcc::test::createTestCallPreservedMask()) || + !buildUse(*MaskFixture->Function, Target, *MaskFixture->Block, *AcrossCall) || + !prepareForAllocation(*MaskFixture->Function, Target)) { + return; + } + auto Allocated = runAllocator(*MaskFixture->Function, Target, Info, AllocatorKind::Basic); + const rcc::MachineInstr *FinalUse = MaskFixture->Block->lastInstruction(); + const auto G1 = physical(rcc::test::TestRegister::G1); + Context.expect(Allocated && FinalUse != nullptr && + FinalUse->operandAt(0)->registerValue() == rcc::Register::fromPhysical(G1), + "global allocation must keep call-crossing values in a preserved register"); +} + +void testInvalidRewriteMappings(TestContext &Context, const rcc::TargetDescription &Target) { + auto ExpectAtomicFailure = [&](FunctionFixture &Fixture, const rcc::VirtRegMap &Mapping, + std::string_view Message) { + const std::vector<std::uint64_t> Before = functionSignature(*Fixture.Function); + auto Rewritten = rcc::rewriteVirtualRegisters(*Fixture.Function, Target, Mapping); + Context.expect(!Rewritten && + Rewritten.error().code() == rcc::ErrorCode::RegisterAllocationFailed && + functionSignature(*Fixture.Function) == Before, + Message); + }; + + auto ClassFixture = createFunction("invalid-class-mapping", 40); + if (!ClassFixture) + return; + auto ClassValue = createVReg(*ClassFixture->Function); + if (!ClassValue || + !buildDef(*ClassFixture->Function, Target, *ClassFixture->Block, *ClassValue, 1) || + !buildUse(*ClassFixture->Function, Target, *ClassFixture->Block, *ClassValue) || + !prepareForAllocation(*ClassFixture->Function, Target)) { + return; + } + rcc::VirtRegMap ClassMapping(*ClassFixture->Function); + const auto F0 = physical(rcc::test::TestRegister::F0); + if (!ClassMapping.add(*ClassValue) || !ClassMapping.assign(*ClassValue, F0)) + return; + ExpectAtomicFailure(*ClassFixture, ClassMapping, + "class-incompatible rewrite mappings must fail atomically"); + + auto ReservedFixture = createFunction("reserved-register-mapping", 41); + if (!ReservedFixture) + return; + auto ReservedValue = createVReg(*ReservedFixture->Function); + const auto G0 = physical(rcc::test::TestRegister::G0); + if (!ReservedValue || + !buildDef(*ReservedFixture->Function, Target, *ReservedFixture->Block, *ReservedValue, 2) || + !buildUse(*ReservedFixture->Function, Target, *ReservedFixture->Block, *ReservedValue) || + !ReservedFixture->Function->registerInfo().reservePhysicalRegister(G0) || + !prepareForAllocation(*ReservedFixture->Function, Target)) { + return; + } + rcc::VirtRegMap ReservedMapping(*ReservedFixture->Function); + if (!ReservedMapping.add(*ReservedValue) || !ReservedMapping.assign(*ReservedValue, G0)) + return; + ExpectAtomicFailure(*ReservedFixture, ReservedMapping, + "reserved-register rewrite mappings must fail atomically"); + + auto OverlapFixture = createFunction("overlapping-register-mapping", 42); + if (!OverlapFixture) + return; + auto Left = createVReg(*OverlapFixture->Function); + auto Right = createVReg(*OverlapFixture->Function); + if (!Left || !Right || + !buildDef(*OverlapFixture->Function, Target, *OverlapFixture->Block, *Left, 3) || + !buildDef(*OverlapFixture->Function, Target, *OverlapFixture->Block, *Right, 4) || + !buildUse(*OverlapFixture->Function, Target, *OverlapFixture->Block, *Left) || + !buildUse(*OverlapFixture->Function, Target, *OverlapFixture->Block, *Right) || + !prepareForAllocation(*OverlapFixture->Function, Target)) { + return; + } + rcc::VirtRegMap OverlapMapping(*OverlapFixture->Function); + if (!OverlapMapping.add(*Left) || !OverlapMapping.add(*Right) || + !OverlapMapping.assign(*Left, G0) || !OverlapMapping.assign(*Right, G0)) { + return; + } + ExpectAtomicFailure(*OverlapFixture, OverlapMapping, + "interfering rewrite mappings must fail atomically"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + auto SharedUnitTarget = createTarget(true); + Context.expect(Target.has_value(), "register-allocation TestTarget must be constructible"); + Context.expect(SharedUnitTarget.has_value(), + "shared-unit register-allocation TestTarget must be constructible"); + if (!Target) { + std::cerr << "target construction error: " << Target.error().message() << '\n'; + return Context.result(); + } + TestRegisterAllocationInfo Info; + testVirtRegMapAndEdit(Context, *Target); + testLiveRegMatrix(Context, *Target); + if (SharedUnitTarget) + testSharedUnitLaneMatrix(Context, *SharedUnitTarget); + testAllocatorsUnderPressure(Context, *Target, Info); + testFailureAtomicity(Context, *Target); + testFastBlockBoundary(Context, *Target, Info); + testFastRegMaskCalls(Context, *Target, Info); + testTiedAndEarlyClobber(Context, *Target, Info); + testRematerialization(Context, *Target, Info); + testRewriteAndConstraints(Context, *Target, Info); + testInvalidRewriteMappings(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/RegisterAllocationTestSupport.h b/backend/unittests/CodeGen/RegisterAllocationTestSupport.h new file mode 100644 index 0000000..5a3bae2 --- /dev/null +++ b/backend/unittests/CodeGen/RegisterAllocationTestSupport.h @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private test header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_BACKEND_REGISTER_ALLOCATION_TEST_SUPPORT_H +#define RCC_BACKEND_REGISTER_ALLOCATION_TEST_SUPPORT_H + +#include "Target/TestTarget.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveRangeEdit.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/RegAllocFast.h" +#include "rcc/CodeGen/RegAllocPBQP.h" +#include "rcc/CodeGen/Spiller.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <algorithm> +#include <cstdint> +#include <iostream> +#include <memory> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc::test::ra { + +enum class AllocatorKind : std::uint8_t { Basic, Fast, PBQP }; + +inline constexpr std::uint32_t DefOpcode = 3000; +inline constexpr std::uint32_t RematerializableDefOpcode = 3001; +inline constexpr std::uint32_t UseOpcode = 3002; +inline constexpr std::uint32_t BinaryOpcode = 3003; +inline constexpr std::uint32_t SpillLoadOpcode = 3004; +inline constexpr std::uint32_t SpillStoreOpcode = 3005; +inline constexpr std::uint32_t MaskOpcode = 3006; +inline constexpr std::uint32_t EarlyOpcode = 3007; +inline constexpr std::uint32_t TiedOpcode = 3008; +inline constexpr std::uint32_t CallDefOpcode = 3009; + +[[nodiscard]] constexpr Opcode testOpcode(std::uint32_t Value) noexcept { + return Opcode::fromValue(Value); +} + +[[nodiscard]] inline PhysicalRegister physical(std::uint32_t Number) { + return *PhysicalRegister::create(Number); +} + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +[[nodiscard]] inline OperandDescriptor registerOperand(RegisterOperandRole Role, + bool EarlyClobber = false) { + OperandDescriptor Result; + Result.Kinds = {MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.AllowSubReg = true; + Result.EarlyClobber = EarlyClobber; + return Result; +} + +[[nodiscard]] inline OperandDescriptor plainOperand(MachineOperand::Kind Kind) { + OperandDescriptor Result; + Result.Kinds = {Kind}; + return Result; +} + +[[nodiscard]] inline InstructionDescriptor descriptor(std::uint32_t Value, std::string Name) { + InstructionDescriptor Result; + Result.InstructionOpcode = testOpcode(Value); + Result.Name = std::move(Name); + Result.Flags = {InstructionFlag::ProducesCode}; + return Result; +} + +[[nodiscard]] inline Expected<TargetDescription> createTarget() { + std::vector<InstructionDescriptor> Descriptors; + + auto Def = descriptor(DefOpcode, "RA_DEF"); + Def.Operands = {registerOperand(RegisterOperandRole::Def), + plainOperand(MachineOperand::Kind::Immediate)}; + Def.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Def)); + + auto Remat = descriptor(RematerializableDefOpcode, "RA_REMAT_DEF"); + Remat.Operands = {registerOperand(RegisterOperandRole::Def), + plainOperand(MachineOperand::Kind::Immediate)}; + Remat.NumExplicitDefs = 1; + Remat.Flags.set(InstructionFlag::Rematerializable); + Descriptors.push_back(std::move(Remat)); + + auto Use = descriptor(UseOpcode, "RA_USE"); + Use.Operands = {registerOperand(RegisterOperandRole::Use)}; + Use.Flags.set(InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(Use)); + + auto Binary = descriptor(BinaryOpcode, "RA_BINARY"); + Binary.Operands = {registerOperand(RegisterOperandRole::Def), + registerOperand(RegisterOperandRole::Use), + registerOperand(RegisterOperandRole::Use)}; + Binary.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Binary)); + + auto Reload = descriptor(SpillLoadOpcode, "RA_RELOAD"); + Reload.Operands = {registerOperand(RegisterOperandRole::Def), + plainOperand(MachineOperand::Kind::FrameIndex)}; + Reload.NumExplicitDefs = 1; + Reload.Flags.set(InstructionFlag::MayLoad); + Reload.RequiresMemoryOperands = true; + Descriptors.push_back(std::move(Reload)); + + auto Spill = descriptor(SpillStoreOpcode, "RA_SPILL"); + Spill.Operands = {registerOperand(RegisterOperandRole::Use), + plainOperand(MachineOperand::Kind::FrameIndex)}; + Spill.Flags.set(InstructionFlag::MayStore); + Spill.RequiresMemoryOperands = true; + Descriptors.push_back(std::move(Spill)); + + auto Mask = descriptor(MaskOpcode, "RA_MASK"); + Mask.Operands = {plainOperand(MachineOperand::Kind::RegisterMask)}; + Mask.Flags.set(InstructionFlag::Call); + Mask.Flags.set(InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(Mask)); + + auto Early = descriptor(EarlyOpcode, "RA_EARLY"); + Early.Operands = {registerOperand(RegisterOperandRole::Def, true), + registerOperand(RegisterOperandRole::Use)}; + Early.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Early)); + + auto Tied = descriptor(TiedOpcode, "RA_TIED"); + auto TiedDef = registerOperand(RegisterOperandRole::Def); + auto TiedUse = registerOperand(RegisterOperandRole::Use); + TiedDef.DefaultTiedOperand = 1; + TiedUse.DefaultTiedOperand = 0; + Tied.Operands = {TiedDef, TiedUse}; + Tied.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Tied)); + + auto CallDef = descriptor(CallDefOpcode, "RA_CALL_DEF"); + CallDef.Operands = {registerOperand(RegisterOperandRole::Def), + plainOperand(MachineOperand::Kind::RegisterMask)}; + CallDef.NumExplicitDefs = 1; + CallDef.Flags.set(InstructionFlag::Call); + CallDef.Flags.set(InstructionFlag::HasSideEffects); + Descriptors.push_back(std::move(CallDef)); + + auto Copy = descriptor(static_cast<std::uint32_t>(GenericOpcode::COPY), "COPY"); + Copy.Operands = {registerOperand(RegisterOperandRole::Def), + registerOperand(RegisterOperandRole::Use)}; + Copy.NumExplicitDefs = 1; + Descriptors.push_back(std::move(Copy)); + + auto Instructions = InstructionSet::create(std::move(Descriptors), {}); + if (!Instructions) + return std::unexpected(Instructions.error()); + return test::createTestTargetDescription(std::move(*Instructions)); +} + +struct FunctionFixture final { + std::unique_ptr<MachineFunction> Function; + MachineBasicBlock *Block = nullptr; +}; + +[[nodiscard]] inline Expected<FunctionFixture> createFunction(std::string Name, std::uint64_t ID) { + auto Function = MachineFunction::create(std::move(Name), MachineFunctionStableID::fromValue(ID)); + if (!Function) + return std::unexpected(Function.error()); + auto Block = (*Function)->createBasicBlock(); + if (!Block) + return std::unexpected(Block.error()); + auto Appended = (*Function)->appendBlock(**Block); + if (!Appended) + return std::unexpected(Appended.error()); + return FunctionFixture{std::move(*Function), *Block}; +} + +[[nodiscard]] inline Expected<Register> +createVReg(MachineFunction &Function, RegisterClassID Class = TestRegisterClass::GPR64) { + auto Type = LLT::makeInteger(64); + if (!Type) + return std::unexpected(Type.error()); + auto Reg = Function.registerInfo().createVirtualRegister(*Type); + if (!Reg) + return std::unexpected(Reg.error()); + if (auto Result = Function.registerInfo().setRegisterBank(*Reg, TestRegisterBank::GPR); !Result) + return std::unexpected(Result.error()); + if (auto Result = Function.registerInfo().setRegisterClass(*Reg, Class); !Result) + return std::unexpected(Result.error()); + return *Reg; +} + +[[nodiscard]] inline Expected<MachineInstr *> +buildDef(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, + Register Reg, std::int64_t Value, SubRegIndex SubReg = {}, bool Rematerializable = false) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = + Builder.buildInstr(testOpcode(Rematerializable ? RematerializableDefOpcode : DefOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Reg, {}, SubReg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addImmediate(Value); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] inline Expected<MachineInstr *> +buildUse(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, + Register Reg, SubRegIndex SubReg = {}, bool Debug = false) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(UseOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + RegisterOperandFlags Flags; + if (Debug) + Flags.set(RegisterOperandFlag::Debug); + if (auto Result = Pending->addUse(Reg, Flags, SubReg); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] inline Expected<MachineInstr *> buildCopy(MachineFunction &Function, + const TargetDescription &Target, + MachineBasicBlock &Block, + Register Destination, Register Source) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::COPY)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] inline Expected<MachineInstr *> buildMask(MachineFunction &Function, + const TargetDescription &Target, + MachineBasicBlock &Block, + std::shared_ptr<const RegisterMask> Mask) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(MaskOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addRegisterMask(std::move(Mask)); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] inline Expected<MachineInstr *> buildTied(MachineFunction &Function, + const TargetDescription &Target, + MachineBasicBlock &Block, Register Reg) { + MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(testOpcode(TiedOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Reg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Reg); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->tieOperands(0, 1); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] inline Expected<void> prepareForAllocation(MachineFunction &Function, + const TargetDescription &Target) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + for (const MachineFunctionProperty Property : { + MachineFunctionProperty::TracksLiveness, + MachineFunctionProperty::Legalized, + MachineFunctionProperty::RegBankSelected, + MachineFunctionProperty::Selected, + MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + }) + Function.properties().set(Property); + return {}; +} + +class TestRegisterAllocationInfo : public TargetRegisterAllocationInfo { +public: + std::uint64_t RegisterCost = 0; + std::uint64_t SpillCost = 1; + + [[nodiscard]] Expected<void> validateSpillReload(RegisterClassID Class, + const TargetDescription &Target) const override { + const RegisterClassDesc *Description = Target.registers().registerClass(Class); + if (Description == nullptr || !Description->Allocatable || Description->SpillSizeInBits == 0 || + Description->SpillAlignmentInBits == 0) + return makeUnexpected(ErrorCode::Unsupported, "test register class is not spillable"); + return {}; + } + + [[nodiscard]] Expected<MachineInstr *> emitReload(MachineIRBuilder &Builder, Register Destination, + RegisterClassID Class, FrameIndex StackSlot, + LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + if (Lanes.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "test reload requires lanes"); + auto Size = MemorySize::fixed(8); + auto Align = Alignment::create(8); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "test memory shape is invalid"); + MachineMemOperandDesc Memory; + Memory.Accesses = {MemoryAccessFlag::Load}; + Memory.Size = *Size; + Memory.Align = *Align; + Memory.Base = MemoryBase::frameIndex(StackSlot); + auto MMO = Builder.function().createMemOperand(std::move(Memory)); + if (!MMO) + return std::unexpected(MMO.error()); + auto Pending = Builder.buildInstr(testOpcode(SpillLoadOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**MMO); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] Expected<MachineInstr *> emitSpill(MachineIRBuilder &Builder, Register Source, + RegisterClassID Class, FrameIndex StackSlot, + LaneMask Lanes) const override { + auto Validated = validateSpillReload(Class, Builder.target()); + if (!Validated) + return std::unexpected(Validated.error()); + if (Lanes.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "test spill requires lanes"); + auto Size = MemorySize::fixed(8); + auto Align = Alignment::create(8); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "test memory shape is invalid"); + MachineMemOperandDesc Memory; + Memory.Accesses = {MemoryAccessFlag::Store}; + Memory.Size = *Size; + Memory.Align = *Align; + Memory.Base = MemoryBase::frameIndex(StackSlot); + auto MMO = Builder.function().createMemOperand(std::move(Memory)); + if (!MMO) + return std::unexpected(MMO.error()); + auto Pending = Builder.buildInstr(testOpcode(SpillStoreOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addFrameIndex(StackSlot); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addMemOperand(**MMO); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] Expected<bool> isRematerializable(const MachineInstr &Instruction, + Register VirtualReg, const MachineFunction &, + const TargetDescription &) const override { + const Register *Defined = Instruction.operandAt(0)->getIf<Register>(); + return Instruction.opcode() == testOpcode(RematerializableDefOpcode) && Defined != nullptr && + *Defined == VirtualReg; + } + + [[nodiscard]] Expected<MachineInstr *> + rematerialize(MachineIRBuilder &Builder, const MachineInstr &Original, Register Destination, + Register OriginalRegister, const TargetDescription &Target) const override { + auto Accepted = isRematerializable(Original, OriginalRegister, Builder.function(), Target); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return makeUnexpected(ErrorCode::InvalidArgument, "test instruction is not rematerializable"); + auto Pending = Builder.buildInstr(testOpcode(RematerializableDefOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addOperand(Original.operandAt(1)->cloneDetached()); !Result) + return std::unexpected(Result.error()); + return Pending->build(); + } + + [[nodiscard]] std::uint64_t physicalRegisterCost(const MachineFunction &, Register, + PhysicalRegister, RegisterClassID, + bool) const noexcept override { + return RegisterCost; + } + + [[nodiscard]] std::uint64_t spillCost(const MachineFunction &, const LiveInterval &, + RegisterClassID) const noexcept override { + return SpillCost; + } +}; + +[[nodiscard]] inline Expected<void> runAllocator(MachineFunction &Function, + const TargetDescription &Target, + const TargetRegisterAllocationInfo &Info, + AllocatorKind Kind, + RegisterAllocationOptions Options = {}) { + MachineAnalysisManager Analyses; + auto Registered = registerStage11MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + std::unique_ptr<MachineFunctionPass> Pass; + switch (Kind) { + case AllocatorKind::Basic: + Pass = createBasicAllocatorPass(Info, Options); + break; + case AllocatorKind::Fast: + Pass = std::make_unique<FastAllocatorPass>(Info); + break; + case AllocatorKind::PBQP: + Pass = std::make_unique<PBQPAllocatorPass>(Info, Options); + break; + } + MachinePassManager Pipeline; + auto Added = Pipeline.addPass(std::move(Pass)); + if (!Added) + return std::unexpected(Added.error()); + return Pipeline.run(Function, Target, Analyses); +} + +[[nodiscard]] inline std::size_t countOpcode(const MachineFunction &Function, Opcode OpcodeValue) { + std::size_t Count = 0; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Count += Instruction->opcode() == OpcodeValue ? 1U : 0U; + return Count; +} + +} // namespace rcc::test::ra + +#endif // RCC_BACKEND_REGISTER_ALLOCATION_TEST_SUPPORT_H diff --git a/backend/unittests/CodeGen/SpillerTest.cpp b/backend/unittests/CodeGen/SpillerTest.cpp new file mode 100644 index 0000000..1f22d8f --- /dev/null +++ b/backend/unittests/CodeGen/SpillerTest.cpp @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/Spiller.h" + +#include "CodeGen/RegisterAllocationTestSupport.h" + +#include <iostream> +#include <optional> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +void testRematerialization(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("spiller-remat", 20); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 42, {}, true) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value)) + return; + VirtRegMap Mapping(*Fixture->Function); + TestRegisterAllocationInfo Info; + if (!Mapping.add(*Value)) + return; + Spiller ValueSpiller(*Fixture->Function, Target, Info, Mapping, true); + auto Children = ValueSpiller.spill(*Value); + Context.expect(Children && Children->size() == 2 && + Fixture->Function->frameInfo().liveObjectCount() == 0 && + countOpcode(*Fixture->Function, testOpcode(RematerializableDefOpcode)) == 2 && + countOpcode(*Fixture->Function, testOpcode(SpillLoadOpcode)) == 0, + "Spiller must rematerialize each use without allocating a stack slot"); +} + +void testDirectSpillAndDebugUse(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("spiller-debug", 21); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + if (!Value || !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 7) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value, {}, true) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value)) + return; + VirtRegMap Mapping(*Fixture->Function); + TestRegisterAllocationInfo Info; + if (!Mapping.add(*Value)) + return; + Spiller ValueSpiller(*Fixture->Function, Target, Info, Mapping, false); + auto Children = ValueSpiller.spill(*Value); + const MachineOperand *DebugOperand = nullptr; + for (const MachineInstr *Instruction = Fixture->Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isDebugUse()) { + DebugOperand = &Operand; + break; + } + } + if (DebugOperand != nullptr) + break; + } + const std::optional<Register> DebugRegister = + DebugOperand == nullptr ? std::nullopt : DebugOperand->registerValue(); + const bool DebugRegisterCleared = DebugRegister && DebugRegister.value().isNoRegister(); + Context.expect(Children && !Children->empty() && + countOpcode(*Fixture->Function, testOpcode(SpillLoadOpcode)) != 0 && + countOpcode(*Fixture->Function, testOpcode(SpillStoreOpcode)) != 0 && + DebugOperand != nullptr && DebugRegisterCleared && + DebugOperand->registerFlags().has(RegisterOperandFlag::Undef), + "Spiller must preserve direct spill code and clear debug-only uses"); + + auto Unspillable = createVReg(*Fixture->Function); + if (!Unspillable || !Mapping.add(*Unspillable, false)) + return; + auto Rejected = ValueSpiller.spill(*Unspillable); + Context.expect(!Rejected && Rejected.error().code() == ErrorCode::RegisterAllocationFailed, + "Spiller must reject an explicitly non-spillable value"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "Spiller test target must be constructible"); + if (!Target) + return Context.result(); + testRematerialization(Context, *Target); + testDirectSpillAndDebugUse(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/CodeGen/VirtRegMapTest.cpp b/backend/unittests/CodeGen/VirtRegMapTest.cpp new file mode 100644 index 0000000..a3bdfc1 --- /dev/null +++ b/backend/unittests/CodeGen/VirtRegMapTest.cpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "CodeGen/RegisterAllocationTestSupport.h" +#include "rcc/CodeGen/LiveRangeEdit.h" + +#include <iostream> + +namespace { + +using namespace rcc; +using namespace rcc::test; +using namespace rcc::test::ra; + +void testSplitMetadata(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("virt-map", 10); + if (!Fixture) + return; + auto Parent = createVReg(*Fixture->Function); + if (!Parent) + return; + VirtRegMap Mapping(*Fixture->Function); + LiveRangeEdit Edit(*Fixture->Function, Target, Mapping); + Context.expect(Mapping.add(*Parent).has_value(), "VirtRegMap must accept a known vreg"); + auto Slot = Edit.getOrCreateSpillSlot(*Parent); + auto Child = Edit.createFrom(*Parent, false); + Context.expect( + Slot && Child && Mapping.verify().has_value() && Mapping.stackSlot(*Child) == Slot && + Mapping.entry(*Child)->SplitParent == *Parent && Mapping.entry(*Child)->Origin == *Parent, + "LiveRangeEdit must preserve split origin and spill metadata"); + if (!Child) + return; + const PhysicalRegister G0 = physical(TestRegister::G0); + Context.expect(Mapping.assign(*Child, G0).has_value() && Mapping.physicalRegister(*Child) == G0 && + Mapping.entry(*Child)->State == VirtRegAllocationState::Assigned, + "VirtRegMap must transition a split child to an assigned state"); +} + +void testRewriteAndAtomicValidation(TestContext &Context, const TargetDescription &Target) { + auto Fixture = createFunction("virt-rewrite", 11); + if (!Fixture) + return; + auto Value = createVReg(*Fixture->Function); + if (!Value || + !buildDef(*Fixture->Function, Target, *Fixture->Block, *Value, 1, TestSubReg::Low32) || + !buildUse(*Fixture->Function, Target, *Fixture->Block, *Value, TestSubReg::Low32) || + !prepareForAllocation(*Fixture->Function, Target)) + return; + VirtRegMap Mapping(*Fixture->Function); + const PhysicalRegister G0 = physical(TestRegister::G0); + Context.expect(Mapping.add(*Value).has_value() && Mapping.assign(*Value, G0).has_value(), + "VirtRegMap must accept a physical assignment before rewrite"); + auto Rewritten = rewriteVirtualRegisters(*Fixture->Function, Target, Mapping); + const MachineOperand *Def = Fixture->Block->firstInstruction()->operandAt(0); + Context.expect( + Rewritten && *Rewritten && Def != nullptr && Def->registerValue() && + Def->registerValue() == Register::fromPhysical(physical(TestRegister::G0Low32)) && + Def->subReg().value() == 0 && + std::ranges::none_of(Fixture->Function->registerInfo().virtualRegisters(), + [](const VirtualRegisterInfo &Info) { return !Info.IsTombstone; }), + "VirtualRegisterRewrite must project and clear subregister operands"); + + auto InvalidFixture = createFunction("virt-rewrite-invalid", 12); + if (!InvalidFixture) + return; + auto Invalid = createVReg(*InvalidFixture->Function); + if (!Invalid || + !buildDef(*InvalidFixture->Function, Target, *InvalidFixture->Block, *Invalid, 2) || + !buildUse(*InvalidFixture->Function, Target, *InvalidFixture->Block, *Invalid) || + !prepareForAllocation(*InvalidFixture->Function, Target)) + return; + VirtRegMap InvalidMapping(*InvalidFixture->Function); + const PhysicalRegister F0 = physical(TestRegister::F0); + if (!InvalidMapping.add(*Invalid) || !InvalidMapping.assign(*Invalid, F0)) + return; + auto Failed = rewriteVirtualRegisters(*InvalidFixture->Function, Target, InvalidMapping); + Context.expect(!Failed && Failed.error().code() == ErrorCode::RegisterAllocationFailed, + "VirtualRegisterRewrite must reject class-incompatible mappings atomically"); +} + +} // namespace + +int main() { + TestContext Context; + auto Target = createTarget(); + Context.expect(Target.has_value(), "VirtRegMap test target must be constructible"); + if (!Target) + return Context.result(); + testSplitMetadata(Context, *Target); + testRewriteAndAtomicValidation(Context, *Target); + return Context.result(); +} diff --git a/backend/unittests/Target/CMakeLists.txt b/backend/unittests/Target/CMakeLists.txt index 0bffb47..ca376da 100644 --- a/backend/unittests/Target/CMakeLists.txt +++ b/backend/unittests/Target/CMakeLists.txt @@ -38,12 +38,24 @@ rcc_backend_configure_target(RCCBackendRISCVSSAExitTests) target_link_libraries(RCCBackendRISCVSSAExitTests PRIVATE RCCBackend::RISCV RCCBackend::CodeGen) add_test(NAME RCCBackendRISCVSSAExitTests COMMAND RCCBackendRISCVSSAExitTests) +add_executable(RCCBackendRISCVRegisterAllocationTests RISCVRegisterAllocationTest.cpp) +rcc_backend_configure_target(RCCBackendRISCVRegisterAllocationTests) +target_link_libraries( + RCCBackendRISCVRegisterAllocationTests + PRIVATE RCCBackend::RISCV RCCBackend::CodeGen +) +add_test( + NAME RCCBackendRISCVRegisterAllocationTests + COMMAND RCCBackendRISCVRegisterAllocationTests +) + rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/DescriptionGeneratorTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLoweringTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTargetTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.h" ) @@ -53,5 +65,6 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTargetTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" ) diff --git a/backend/unittests/Target/DescriptionGeneratorTest.cpp b/backend/unittests/Target/DescriptionGeneratorTest.cpp index d41d2d4..0f8d1e9 100644 --- a/backend/unittests/Target/DescriptionGeneratorTest.cpp +++ b/backend/unittests/Target/DescriptionGeneratorTest.cpp @@ -78,27 +78,28 @@ void testGeneration(TestContext &Context) { } Context.expect(First.succeeded() && Second.succeeded(), "a complete restricted target description must generate successfully"); - if (!First.succeeded() || !Second.succeeded()) + if (!First.succeeded() || !Second.succeeded() || !First.Output || !Second.Output) return; + const auto &FirstOutput = First.Output.value(); + const auto &SecondOutput = Second.Output.value(); - Context.expect(First.Output->RegisterHeader == Second.Output->RegisterHeader && - First.Output->InstructionHeader == Second.Output->InstructionHeader && - First.Output->TargetTables == Second.Output->TargetTables, + Context.expect(FirstOutput.RegisterHeader == SecondOutput.RegisterHeader && + FirstOutput.InstructionHeader == SecondOutput.InstructionHeader && + FirstOutput.TargetTables == SecondOutput.TargetTables, "target description generation must be deterministic"); Context.expect( - First.Output->RegisterHeader.find("enum class PhysicalRegisterID") != std::string::npos && - First.Output->RegisterHeader.find("R1 = 1") != std::string::npos && - First.Output->InstructionHeader.find("enum class InstructionOpcode") != + FirstOutput.RegisterHeader.find("enum class PhysicalRegisterID") != std::string::npos && + FirstOutput.RegisterHeader.find("R1 = 1") != std::string::npos && + FirstOutput.InstructionHeader.find("enum class InstructionOpcode") != std::string::npos && + FirstOutput.InstructionHeader.find("LOAD_IMM = 1024") != std::string::npos && + FirstOutput.TargetTables.find("appendGeneratedRegisterInfo") != std::string::npos && + FirstOutput.TargetTables.find("appendGeneratedInstructionInfo") != std::string::npos && + FirstOutput.TargetTables.find("GeneratedAssemblyInstructions") != std::string::npos && + FirstOutput.TargetTables.find("Description.DirectOperandFlagMask = 31") != std::string::npos && - First.Output->InstructionHeader.find("LOAD_IMM = 1024") != std::string::npos && - First.Output->TargetTables.find("appendGeneratedRegisterInfo") != std::string::npos && - First.Output->TargetTables.find("appendGeneratedInstructionInfo") != std::string::npos && - First.Output->TargetTables.find("GeneratedAssemblyInstructions") != std::string::npos && - First.Output->TargetTables.find("Description.DirectOperandFlagMask = 31") != - std::string::npos && - First.Output->TargetTables.find("\"tiny-call\", 1") != std::string::npos && - First.Output->TargetTables.find("\"load.imm\"") != std::string::npos && - First.Output->TargetTables.find("makePreservedMask") != std::string::npos, + FirstOutput.TargetTables.find("\"tiny-call\", 1") != std::string::npos && + FirstOutput.TargetTables.find("\"load.imm\"") != std::string::npos && + FirstOutput.TargetTables.find("makePreservedMask") != std::string::npos, "generated output must contain typed target IDs, assembly mnemonics, and static target " "tables"); } diff --git a/backend/unittests/Target/RISCVRegisterAllocationTest.cpp b/backend/unittests/Target/RISCVRegisterAllocationTest.cpp new file mode 100644 index 0000000..f9031da --- /dev/null +++ b/backend/unittests/Target/RISCVRegisterAllocationTest.cpp @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineLiveness.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/RegAllocFast.h" +#include "rcc/CodeGen/RegAllocPBQP.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iostream> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +[[nodiscard]] constexpr rcc::Opcode targetOpcode(rcc::riscv::InstructionOpcode Value) noexcept { + return rcc::Opcode::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] constexpr rcc::RegisterClassID +registerClass(rcc::riscv::RegisterClass Value) noexcept { + return rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] constexpr rcc::RegisterBankID registerBank(rcc::riscv::RegisterBank Value) noexcept { + return rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] rcc::Register physical(rcc::riscv::PhysicalRegisterID ID) { + return rcc::Register::fromPhysical(*rcc::riscv::physicalRegister(ID)); +} + +struct FunctionFixture final { + std::unique_ptr<rcc::MachineFunction> Function; + rcc::MachineBasicBlock *Block = nullptr; +}; + +[[nodiscard]] rcc::Expected<FunctionFixture> createFunction(std::string Name, std::uint64_t ID) { + auto Function = + rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID)); + if (!Function) + return std::unexpected(Function.error()); + auto Block = (*Function)->createBasicBlock(); + if (!Block) + return std::unexpected(Block.error()); + auto Appended = (*Function)->appendBlock(**Block); + if (!Appended) + return std::unexpected(Appended.error()); + return FunctionFixture{std::move(*Function), *Block}; +} + +[[nodiscard]] rcc::Expected<rcc::Register> createVReg(rcc::MachineFunction &Function, rcc::LLT Type, + rcc::riscv::RegisterBank Bank, + rcc::riscv::RegisterClass Class) { + auto Reg = Function.registerInfo().createVirtualRegister(Type); + if (!Reg) + return std::unexpected(Reg.error()); + if (auto Result = Function.registerInfo().setRegisterBank(*Reg, registerBank(Bank)); !Result) + return std::unexpected(Result.error()); + if (auto Result = Function.registerInfo().setRegisterClass(*Reg, registerClass(Class)); !Result) + return std::unexpected(Result.error()); + return *Reg; +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> +buildAddImmediate(rcc::MachineFunction &Function, const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, rcc::Register Destination, rcc::Register Source, + std::int64_t Immediate) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(targetOpcode(rcc::riscv::InstructionOpcode::ADDI)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Source); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addImmediate(Immediate); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<rcc::MachineInstr *> buildAdd(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target, + rcc::MachineBasicBlock &Block, + rcc::Register Destination, + rcc::Register Left, rcc::Register Right) { + rcc::MachineIRBuilder Builder(Function, Target); + if (auto Result = Builder.setInsertPoint(Block); !Result) + return std::unexpected(Result.error()); + auto Pending = Builder.buildInstr(targetOpcode(rcc::riscv::InstructionOpcode::ADD)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Result = Pending->addDef(Destination); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Left); !Result) + return std::unexpected(Result.error()); + if (auto Result = Pending->addUse(Right); !Result) + return std::unexpected(Result.error()); + return Pending->build(); +} + +[[nodiscard]] rcc::Expected<void> prepareForAllocation(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) { + auto Rebuilt = rcc::rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + for (const rcc::MachineFunctionProperty Property : { + rcc::MachineFunctionProperty::TracksLiveness, + rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected, + rcc::MachineFunctionProperty::Selected, + rcc::MachineFunctionProperty::NoPHIs, + rcc::MachineFunctionProperty::TiedOpsRewritten, + }) { + Function.properties().set(Property); + } + return {}; +} + +enum class AllocatorKind : std::uint8_t { + Basic, + Fast, + PBQP, +}; + +[[nodiscard]] rcc::Expected<void> runAllocator(rcc::MachineFunction &Function, + const rcc::TargetDescription &Target, + AllocatorKind Kind, + rcc::RegisterAllocationOptions Options = {}) { + rcc::MachineAnalysisManager Analyses; + auto Registered = rcc::registerStage11MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + std::unique_ptr<rcc::MachineFunctionPass> Pass; + switch (Kind) { + case AllocatorKind::Basic: + Pass = rcc::createBasicAllocatorPass(rcc::riscv::registerAllocationInfo(), Options); + break; + case AllocatorKind::Fast: + Pass = std::make_unique<rcc::FastAllocatorPass>(rcc::riscv::registerAllocationInfo()); + break; + case AllocatorKind::PBQP: + Pass = std::make_unique<rcc::PBQPAllocatorPass>(rcc::riscv::registerAllocationInfo(), Options); + break; + } + rcc::MachinePassManager Pipeline; + auto Added = Pipeline.addPass(std::move(Pass)); + if (!Added) + return std::unexpected(Added.error()); + return Pipeline.run(Function, Target, Analyses); +} + +[[nodiscard]] std::size_t countOpcode(const rcc::MachineFunction &Function, + rcc::riscv::InstructionOpcode Value) { + std::size_t Count = 0; + for (const rcc::MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Count += Instruction->opcode() == targetOpcode(Value) ? 1U : 0U; + } + } + return Count; +} + +[[nodiscard]] std::vector<std::uint64_t> allocationSignature(const rcc::MachineFunction &Function) { + std::vector<std::uint64_t> Result; + Result.push_back(Function.frameInfo().liveObjectCount()); + for (const rcc::MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.push_back(Instruction->opcode().value()); + for (const rcc::MachineOperand &Operand : Instruction->operands()) { + Result.push_back(static_cast<std::uint64_t>(Operand.kind())); + const rcc::Register *Reg = Operand.getIf<rcc::Register>(); + Result.push_back(Reg == nullptr ? 0 : Reg->opaqueValue()); + const auto *Frame = Operand.getIf<rcc::FrameIndexOperand>(); + Result.push_back(Frame == nullptr ? 0 + : static_cast<std::uint64_t>(Frame->Index.value()) + 1U); + } + } + } + return Result; +} + +void testTargetHooks(TestContext &Context, const rcc::TargetDescription &Target) { + const rcc::TargetRegisterAllocationInfo &Info = rcc::riscv::registerAllocationInfo(); + for (const rcc::riscv::RegisterClass Class : { + rcc::riscv::RegisterClass::GPR, + rcc::riscv::RegisterClass::GPRC, + rcc::riscv::RegisterClass::FPR64, + rcc::riscv::RegisterClass::FPR64C, + rcc::riscv::RegisterClass::FPR32, + rcc::riscv::RegisterClass::FPR32C, + rcc::riscv::RegisterClass::GPR_NX0, + rcc::riscv::RegisterClass::GPR_NX0_NX2, + }) { + Context.expect(Info.validateSpillReload(registerClass(Class), Target).has_value(), + "every allocatable RV64GC scalar class must have spill/reload hooks"); + } + + auto Fixture = createFunction("riscv-spill-hooks", 1); + if (!Fixture) + return; + rcc::StackObjectDesc StackDescription; + StackDescription.Kind = rcc::StackObjectKind::SpillSlot; + StackDescription.Size = 8; + auto Align = rcc::Alignment::create(8); + if (!Align) + return; + StackDescription.Align = *Align; + auto Slot = Fixture->Function->frameInfo().createObject(std::move(StackDescription)); + if (!Slot) + return; + rcc::MachineIRBuilder Builder(*Fixture->Function, Target); + if (!Builder.setInsertPoint(*Fixture->Block)) + return; + + auto GPRReload = + Info.emitReload(Builder, physical(rcc::riscv::PhysicalRegisterID::X5), + registerClass(rcc::riscv::RegisterClass::GPR), *Slot, rcc::LaneMask::all()); + auto GPRSpill = + Info.emitSpill(Builder, physical(rcc::riscv::PhysicalRegisterID::X5), + registerClass(rcc::riscv::RegisterClass::GPR), *Slot, rcc::LaneMask::all()); + auto F32Reload = + Info.emitReload(Builder, physical(rcc::riscv::PhysicalRegisterID::F10), + registerClass(rcc::riscv::RegisterClass::FPR32), *Slot, rcc::LaneMask::all()); + auto F32Spill = + Info.emitSpill(Builder, physical(rcc::riscv::PhysicalRegisterID::F10), + registerClass(rcc::riscv::RegisterClass::FPR32), *Slot, rcc::LaneMask::all()); + auto F64Reload = + Info.emitReload(Builder, physical(rcc::riscv::PhysicalRegisterID::F11), + registerClass(rcc::riscv::RegisterClass::FPR64), *Slot, rcc::LaneMask::all()); + auto F64Spill = + Info.emitSpill(Builder, physical(rcc::riscv::PhysicalRegisterID::F11), + registerClass(rcc::riscv::RegisterClass::FPR64), *Slot, rcc::LaneMask::all()); + Context.expect(GPRReload && GPRSpill && F32Reload && F32Spill && F64Reload && F64Spill && + (*GPRReload)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::LD) && + (*GPRSpill)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::SD) && + (*F32Reload)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::FLW) && + (*F32Spill)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::FSW) && + (*F64Reload)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::FLD) && + (*F64Spill)->opcode() == targetOpcode(rcc::riscv::InstructionOpcode::FSD), + "RV64GC hooks must select width- and bank-correct stack instructions"); + if (GPRReload && F32Reload && F64Reload) { + Context.expect( + (*GPRReload)->operandAt(1)->getIf<rcc::FrameIndexOperand>() != nullptr && + (*GPRReload)->memOperands().front()->description().Size.minimumBytes() == 8 && + (*F32Reload)->memOperands().front()->description().Size.minimumBytes() == 4 && + (*F64Reload)->memOperands().front()->description().Size.minimumBytes() == 8, + "RV64GC stack instructions must retain precise FrameIndex MMO size metadata"); + } + auto EmptyLane = Info.emitReload(Builder, physical(rcc::riscv::PhysicalRegisterID::X5), + registerClass(rcc::riscv::RegisterClass::GPR), *Slot, {}); + Context.expect(!EmptyLane && EmptyLane.error().code() == rcc::ErrorCode::InvalidArgument, + "RV64GC spill hooks must reject an empty lane request"); +} + +void testRematerialization(TestContext &Context, const rcc::TargetDescription &Target) { + auto Fixture = createFunction("riscv-rematerialization", 2); + auto I64 = rcc::LLT::makeInteger(64); + if (!Fixture || !I64) + return; + auto Original = createVReg(*Fixture->Function, *I64, rcc::riscv::RegisterBank::GPR, + rcc::riscv::RegisterClass::GPR); + auto Child = createVReg(*Fixture->Function, *I64, rcc::riscv::RegisterBank::GPR, + rcc::riscv::RegisterClass::GPR); + if (!Original || !Child) + return; + auto Definition = buildAddImmediate(*Fixture->Function, Target, *Fixture->Block, *Original, + physical(rcc::riscv::PhysicalRegisterID::X0), 17); + if (!Definition) + return; + const rcc::TargetRegisterAllocationInfo &Info = rcc::riscv::registerAllocationInfo(); + auto Accepted = Info.isRematerializable(**Definition, *Original, *Fixture->Function, Target); + rcc::MachineIRBuilder Builder(*Fixture->Function, Target); + if (!Builder.setInsertPointAfter(**Definition)) + return; + auto Materialized = Info.rematerialize(Builder, **Definition, *Child, *Original, Target); + Context.expect(Accepted && *Accepted && Materialized && + (*Materialized)->opcode() == + targetOpcode(rcc::riscv::InstructionOpcode::ADDI) && + (*Materialized)->operandAt(0)->registerValue() == *Child && + (*Materialized)->operandAt(1)->registerValue() == + physical(rcc::riscv::PhysicalRegisterID::X0), + "RV64GC must rematerialize constant ADDI forms with a replacement definition"); +} + +[[nodiscard]] rcc::Expected<FunctionFixture> +createSimpleAllocationFunction(const rcc::TargetDescription &Target, std::uint64_t ID) { + auto Fixture = createFunction("riscv-ra-" + std::to_string(ID), ID); + auto I64 = rcc::LLT::makeInteger(64); + if (!Fixture) + return std::unexpected(Fixture.error()); + if (!I64) + return std::unexpected(I64.error()); + std::vector<rcc::Register> Values; + for (std::int64_t Immediate : {1, 2, 3}) { + auto Reg = createVReg(*Fixture->Function, *I64, rcc::riscv::RegisterBank::GPR, + rcc::riscv::RegisterClass::GPR); + if (!Reg) + return std::unexpected(Reg.error()); + if (auto Built = buildAddImmediate(*Fixture->Function, Target, *Fixture->Block, *Reg, + physical(rcc::riscv::PhysicalRegisterID::X0), Immediate); + !Built) { + return std::unexpected(Built.error()); + } + Values.push_back(*Reg); + } + auto Sum = createVReg(*Fixture->Function, *I64, rcc::riscv::RegisterBank::GPR, + rcc::riscv::RegisterClass::GPR); + if (!Sum) + return std::unexpected(Sum.error()); + if (auto Built = + buildAdd(*Fixture->Function, Target, *Fixture->Block, *Sum, Values[0], Values[1]); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = buildAdd(*Fixture->Function, Target, *Fixture->Block, + physical(rcc::riscv::PhysicalRegisterID::X0), *Sum, Values[2]); + !Built) { + return std::unexpected(Built.error()); + } + auto Prepared = prepareForAllocation(*Fixture->Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + return Fixture; +} + +void testRealAllocators(TestContext &Context, const rcc::TargetDescription &Target) { + for (const auto [Kind, ID, Name] : { + std::tuple{AllocatorKind::Basic, std::uint64_t{10}, std::string_view{"Basic"}}, + std::tuple{AllocatorKind::Fast, std::uint64_t{11}, std::string_view{"Fast"}}, + std::tuple{AllocatorKind::PBQP, std::uint64_t{12}, std::string_view{"PBQP"}}, + }) { + auto Fixture = createSimpleAllocationFunction(Target, ID); + if (!Fixture) + continue; + auto Allocated = runAllocator(*Fixture->Function, Target, Kind); + if (!Allocated) + std::cerr << "RV64GC " << Name << " error: " << Allocated.error().message() << '\n'; + Context.expect( + Allocated && Fixture->Function->properties().has(rcc::MachineFunctionProperty::NoVRegs) && + Fixture->Function->registerInfo().virtualRegisters().size() == 4, + "every allocator must rewrite real RV64GC selected instructions to physical registers"); + if (!Allocated) + continue; + bool OnlyPhysical = true; + for (const rcc::MachineInstr *Instruction : Fixture->Function->allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + for (const rcc::MachineOperand &Operand : Instruction->operands()) { + const rcc::Register *Reg = Operand.getIf<rcc::Register>(); + OnlyPhysical &= Reg == nullptr || !Reg->isVirtual(); + } + } + Context.expect(OnlyPhysical, + "RV64GC allocator output must not retain a linked virtual-register operand"); + } +} + +[[nodiscard]] rcc::Expected<FunctionFixture> +createPressureFunction(const rcc::TargetDescription &Target, std::uint64_t ID) { + auto Fixture = createFunction("riscv-pressure-" + std::to_string(ID), ID); + auto I64 = rcc::LLT::makeInteger(64); + if (!Fixture) + return std::unexpected(Fixture.error()); + if (!I64) + return std::unexpected(I64.error()); + + std::vector<rcc::Register> Values; + Values.reserve(30); + for (std::int64_t Immediate = 0; Immediate < 30; ++Immediate) { + auto Reg = createVReg(*Fixture->Function, *I64, rcc::riscv::RegisterBank::GPR, + rcc::riscv::RegisterClass::GPR); + if (!Reg) + return std::unexpected(Reg.error()); + auto Built = buildAddImmediate(*Fixture->Function, Target, *Fixture->Block, *Reg, + physical(rcc::riscv::PhysicalRegisterID::X0), Immediate); + if (!Built) + return std::unexpected(Built.error()); + Values.push_back(*Reg); + } + const rcc::Register Zero = physical(rcc::riscv::PhysicalRegisterID::X0); + for (const rcc::Register Value : Values) { + auto Built = buildAdd(*Fixture->Function, Target, *Fixture->Block, Zero, Value, Zero); + if (!Built) + return std::unexpected(Built.error()); + } + auto Prepared = prepareForAllocation(*Fixture->Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + return Fixture; +} + +void testRealPressureAndDeterminism(TestContext &Context, const rcc::TargetDescription &Target) { + std::vector<std::uint64_t> FirstPBQPSignature; + for (const auto [Kind, ID, Name] : { + std::tuple{AllocatorKind::Basic, std::uint64_t{20}, std::string_view{"Basic"}}, + std::tuple{AllocatorKind::PBQP, std::uint64_t{21}, std::string_view{"PBQP"}}, + std::tuple{AllocatorKind::PBQP, std::uint64_t{22}, std::string_view{"PBQP-repeat"}}, + }) { + auto Fixture = createPressureFunction(Target, ID); + if (!Fixture) + continue; + auto Allocated = + runAllocator(*Fixture->Function, Target, Kind, {.EnableRematerialization = false}); + if (!Allocated) + std::cerr << "RV64GC pressure " << Name << " error: " << Allocated.error().message() << '\n'; + Context.expect(Allocated && Fixture->Function->frameInfo().liveObjectCount() != 0 && + countOpcode(*Fixture->Function, rcc::riscv::InstructionOpcode::LD) != 0 && + countOpcode(*Fixture->Function, rcc::riscv::InstructionOpcode::SD) != 0, + "RV64GC global allocators must spill and reload real GPR pressure"); + if (!Allocated || Kind != AllocatorKind::PBQP) + continue; + const std::vector<std::uint64_t> Signature = allocationSignature(*Fixture->Function); + if (FirstPBQPSignature.empty()) + FirstPBQPSignature = Signature; + else + Context.expect(Signature == FirstPBQPSignature, + "RV64GC PBQP allocation must be stable across identical functions"); + } + + auto RematerializedPressure = createPressureFunction(Target, 23); + if (RematerializedPressure) { + auto Allocated = runAllocator(*RematerializedPressure->Function, Target, AllocatorKind::PBQP); + if (!Allocated) + std::cerr << "RV64GC rematerialization pressure error: " << Allocated.error().message() + << '\n'; + Context.expect(Allocated && RematerializedPressure->Function->properties().has( + rcc::MachineFunctionProperty::NoVRegs), + "RV64GC PBQP rematerialization must finish with a valid register map"); + } +} + +} // namespace + +int main() { + TestContext Context; + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC register-allocation target must be constructible"); + if (!Target) { + std::cerr << "target construction error: " << Target.error().message() << '\n'; + return Context.result(); + } + testTargetHooks(Context, *Target); + testRematerialization(Context, *Target); + testRealAllocators(Context, *Target); + testRealPressureAndDeterminism(Context, *Target); + return Context.result(); +} |