From f99ef4322a7a2c09380fff61f88ab9440f05a92c Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Fri, 7 Aug 2026 22:06:28 +0800 Subject: [CodeGen] Complete machine pass and ABI lowering infrastructure --- backend/lib/CodeGen/GenericOpcodes.cpp | 1498 ++++++++++++++++++++++++++++++++ 1 file changed, 1498 insertions(+) create mode 100644 backend/lib/CodeGen/GenericOpcodes.cpp (limited to 'backend/lib/CodeGen/GenericOpcodes.cpp') diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp new file mode 100644 index 0000000..0b0e0fe --- /dev/null +++ b/backend/lib/CodeGen/GenericOpcodes.cpp @@ -0,0 +1,1498 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" + +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +using GO = GenericOpcode; + +[[nodiscard]] Expected verifyGenericExplicitOperands(Opcode InstructionOpcode, + std::span Operands, + const MachineRegisterInfo &RegInfo); +[[nodiscard]] Expected verifyGenericInstructionSemantics(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo); + +[[nodiscard]] LLTKindSet anyValueKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Float, + LLT::Kind::Pointer, LLT::Kind::VectorAny, LLT::Kind::VectorInteger, + LLT::Kind::VectorFloat, LLT::Kind::VectorPointer}; +} + +[[nodiscard]] LLTKindSet integerKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::VectorAny, LLT::Kind::VectorInteger}; +} + +[[nodiscard]] LLTKindSet scalarIntegerKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Integer}; +} + +[[nodiscard]] LLTKindSet scalarValueKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Float, LLT::Kind::Pointer}; +} + +[[nodiscard]] LLTKindSet floatingKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Float, LLT::Kind::VectorAny, LLT::Kind::VectorFloat}; +} + +[[nodiscard]] LLTKindSet scalarFloatingKinds() { + return {LLT::Kind::AnyScalar, LLT::Kind::Float}; +} + +[[nodiscard]] LLTKindSet pointerKinds() { + return {LLT::Kind::Pointer, LLT::Kind::VectorPointer}; +} + +[[nodiscard]] LLTKindSet scalarPointerKinds() { + return {LLT::Kind::Pointer}; +} + +[[nodiscard]] LLTKindSet vectorKinds() { + return {LLT::Kind::VectorAny, LLT::Kind::VectorInteger, LLT::Kind::VectorFloat, + LLT::Kind::VectorPointer}; +} + +[[nodiscard]] OperandKindSet allOperandKinds() { + return {MachineOperand::Kind::Register, + MachineOperand::Kind::Immediate, + MachineOperand::Kind::IntegerImmediate, + MachineOperand::Kind::FloatingImmediate, + MachineOperand::Kind::BasicBlock, + MachineOperand::Kind::FrameIndex, + MachineOperand::Kind::ConstantPoolIndex, + MachineOperand::Kind::JumpTableIndex, + MachineOperand::Kind::GlobalSymbol, + MachineOperand::Kind::ExternalSymbol, + MachineOperand::Kind::BlockAddress, + MachineOperand::Kind::RegisterMask, + MachineOperand::Kind::LiveOutRegisterMask, + MachineOperand::Kind::Metadata, + MachineOperand::Kind::TargetIndex, + MachineOperand::Kind::MCSymbol, + MachineOperand::Kind::CFIIndex, + MachineOperand::Kind::Intrinsic, + MachineOperand::Kind::Predicate, + MachineOperand::Kind::ShuffleMask, + MachineOperand::Kind::DebugInstrRef, + MachineOperand::Kind::LaneMask}; +} + +[[nodiscard]] OperandDescriptor typedRegister(RegisterOperandRole Role, LLTKindSet Kinds, + std::uint16_t Group) { + OperandDescriptor Result; + Result.Kinds = {MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.TypeConstraint = OperandTypeConstraint{std::move(Kinds), Group}; + return Result; +} + +[[nodiscard]] OperandDescriptor untypedRegister(RegisterOperandRole Role, bool AllowSubReg = true) { + OperandDescriptor Result; + Result.Kinds = {MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.AllowSubReg = AllowSubReg; + return Result; +} + +[[nodiscard]] OperandDescriptor operand(OperandKindSet Kinds) { + OperandDescriptor Result; + Result.Kinds = Kinds; + if (Kinds.contains(MachineOperand::Kind::Register)) { + Result.RegisterRole = RegisterOperandRole::Any; + Result.AllowSubReg = true; + } + return Result; +} + +[[nodiscard]] OperandDescriptor kind(MachineOperand::Kind Kind) { + return operand(OperandKindSet{Kind}); +} + +[[nodiscard]] OperandDescriptor immediate() { + return operand({MachineOperand::Kind::Immediate, MachineOperand::Kind::IntegerImmediate}); +} + +[[nodiscard]] OperandDescriptor symbol() { + return operand({MachineOperand::Kind::GlobalSymbol, MachineOperand::Kind::ExternalSymbol, + MachineOperand::Kind::MCSymbol}); +} + +[[nodiscard]] OperandDescriptor anyOperand() { + return operand(allOperandKinds()); +} + +[[nodiscard]] OperandDescriptor repeated(OperandDescriptor Descriptor) { + Descriptor.VariadicGroup = true; + return Descriptor; +} + +class DescriptorRegistry final { +public: + DescriptorRegistry() { + Descriptors.reserve(CorePseudoOpcodeCount + GenericOpcodeCount); + Configured.reserve(CorePseudoOpcodeCount + GenericOpcodeCount); +#define RCC_CORE_OPCODE(Name) add(GO::Name, #Name, false); +#define RCC_GENERIC_OPCODE(Name) add(GO::Name, #Name, true); +#include "rcc/CodeGen/GenericOpcode.def" +#undef RCC_GENERIC_OPCODE +#undef RCC_CORE_OPCODE + } + + void configure(std::initializer_list Opcodes, std::vector Operands, + std::size_t NumDefs, std::initializer_list Flags = {}, + bool Variadic = false, std::size_t MinimumRepetitions = 0, + bool RequiresMemoryOperands = false, PseudoOwner Owner = PseudoOwner::None, + PipelineStage Deadline = PipelineStage::Emission, + bool AllowExtraImplicitOperands = false) { + for (const GO OpcodeValue : Opcodes) { + const std::size_t Index = index(OpcodeValue); + if (Index >= Descriptors.size() || Configured[Index]) + std::terminate(); + Configured[Index] = true; + InstructionDescriptor &Descriptor = Descriptors[Index]; + Descriptor.Operands = Operands; + Descriptor.NumExplicitDefs = NumDefs; + Descriptor.Variadic = Variadic; + Descriptor.MinimumVariadicRepetitions = MinimumRepetitions; + Descriptor.Flags = {}; + Descriptor.Flags.set(isGenericOpcode(Descriptor.InstructionOpcode) ? InstructionFlag::Generic + : InstructionFlag::Pseudo); + Descriptor.Flags.set(InstructionFlag::AllowedPreLegalizer); + for (const InstructionFlag Flag : Flags) + Descriptor.Flags.set(Flag); + Descriptor.RequiresMemoryOperands = RequiresMemoryOperands; + Descriptor.Owner = Owner; + Descriptor.Deadline = Deadline; + Descriptor.AllowExtraImplicitOperands = AllowExtraImplicitOperands; + if (isGenericOpcode(Descriptor.InstructionOpcode)) { + Descriptor.VerifyExplicitOperands = &verifyGenericExplicitOperands; + Descriptor.VerifyInstructionSemantics = &verifyGenericInstructionSemantics; + } + } + } + + void addFlag(std::initializer_list Opcodes, InstructionFlag Flag) { + for (const GO OpcodeValue : Opcodes) { + const std::size_t Index = index(OpcodeValue); + if (Index >= Descriptors.size() || !Configured[Index]) + std::terminate(); + Descriptors[Index].Flags.set(Flag); + } + } + + void tie(GO OpcodeValue, std::size_t Left, std::size_t Right) { + InstructionDescriptor &Descriptor = Descriptors[index(OpcodeValue)]; + if (!Configured[index(OpcodeValue)] || Left >= Descriptor.Operands.size() || + Right >= Descriptor.Operands.size()) { + std::terminate(); + } + Descriptor.Operands[Left].DefaultTiedOperand = Right; + Descriptor.Operands[Right].DefaultTiedOperand = Left; + } + + [[nodiscard]] std::vector take() { + if (!std::ranges::all_of(Configured, [](bool Value) { return Value; })) + std::terminate(); + return std::move(Descriptors); + } + +private: + static std::size_t index(GO OpcodeValue) { + const std::size_t Value = static_cast(OpcodeValue); + if (Value < static_cast(GO::CORE_PSEUDO_END)) + return Value - static_cast(GO::PHI); + return CorePseudoOpcodeCount + Value - static_cast(GO::G_ASSERT_SEXT); + } + + void add(GO OpcodeValue, std::string_view Name, bool IsGeneric) { + InstructionDescriptor Descriptor; + Descriptor.InstructionOpcode = opcode(OpcodeValue); + Descriptor.Name = Name; + if (IsGeneric != isGenericOpcode(Descriptor.InstructionOpcode)) + std::terminate(); + Descriptors.push_back(std::move(Descriptor)); + Configured.push_back(false); + } + + std::vector Descriptors; + std::vector Configured; +}; + +[[nodiscard]] Expected semanticError(std::string_view Message) { + return makeUnexpected(ErrorCode::VerificationError, std::string(Message)); +} + +[[nodiscard]] std::optional plainImmediate(const MachineOperand &Operand) noexcept { + const auto *Value = Operand.getIf(); + if (Operand.kind() != MachineOperand::Kind::Immediate || Value == nullptr) + return std::nullopt; + return *Value; +} + +[[nodiscard]] std::uint64_t minimumBitWidth(LLT Type) noexcept { + const std::uint64_t Elements = Type.isVector() ? Type.minimumElementCount() : 1; + return Elements * Type.scalarBitWidth(); +} + +[[nodiscard]] std::uint64_t minimumStoreBytes(LLT Type) noexcept { + return (minimumBitWidth(Type) + 7) / 8; +} + +[[nodiscard]] LLT scalarElementType(LLT Type) noexcept { + if (const auto Element = Type.scalarElementType()) + return *Element; + return Type; +} + +[[nodiscard]] bool hasSameShape(LLT Left, LLT Right) noexcept { + if (Left.isVector() != Right.isVector()) + return false; + if (!Left.isVector()) + return true; + return Left.isScalableVector() == Right.isScalableVector() && + Left.minimumElementCount() == Right.minimumElementCount(); +} + +[[nodiscard]] bool isI1(LLT Type) noexcept { + return Type.isIntegerOrIntegerVector() && Type.scalarBitWidth() == 1; +} + +[[nodiscard]] Expected requireSameShape(LLT Left, LLT Right, std::string_view Message) { + if (!hasSameShape(Left, Right)) + return semanticError(Message); + return {}; +} + +[[nodiscard]] Expected requireI1WithShape(LLT Condition, LLT Value, + bool AllowScalarCondition) { + if (!isI1(Condition)) + return semanticError("generic condition or status result must use an i1 LLT"); + if (AllowScalarCondition && !Condition.isVector()) + return {}; + return requireSameShape(Condition, Value, + "generic condition or status vector shape does not match its value"); +} + +[[nodiscard]] Expected verifyGenericExplicitOperands(Opcode InstructionOpcode, + std::span Operands, + const MachineRegisterInfo &RegInfo) { + std::vector> Types(Operands.size()); + bool SawNonDefinition = false; + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const MachineOperand &Operand = Operands[Index]; + if (!Operand.isRegister()) { + SawNonDefinition = true; + continue; + } + + if (Operand.isDef() == Operand.isUse()) + return semanticError("generic register operand must be either a pure definition or use"); + if (Operand.isDef()) { + if (SawNonDefinition) + return semanticError("generic register definitions must form the explicit operand prefix"); + } else { + SawNonDefinition = true; + if (Operand.registerFlags().has(RegisterOperandFlag::Undef)) + return semanticError("generic virtual-register uses cannot carry the undef flag"); + } + + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return semanticError("generic instruction register operands must be virtual registers"); + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return semanticError("generic virtual register is missing its LLT"); + Types[Index] = *Info->get().Type; + } + + const auto TypeAt = [&Types](std::size_t Index) -> LLT { return *Types[Index]; }; + const GO Generic = static_cast(InstructionOpcode.value()); + + const auto VerifyWidthChange = [&](bool IsExtension) -> Expected { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (auto Shape = + requireSameShape(Dst, Src, "generic width conversion must preserve vector shape"); + !Shape) { + return Shape; + } + if ((IsExtension && Dst.scalarBitWidth() <= Src.scalarBitWidth()) || + (!IsExtension && Dst.scalarBitWidth() >= Src.scalarBitWidth())) { + return semanticError(IsExtension + ? "generic extension destination must be wider than its source" + : "generic truncation destination must be narrower than its source"); + } + return {}; + }; + + const auto VerifyImmediateBitWidth = [&](std::size_t ImmediateIndex, std::size_t TypeIndex, + bool StrictlySmaller) -> Expected { + const auto Value = plainImmediate(Operands[ImmediateIndex]); + if (!Value || *Value <= 0) + return semanticError("generic bit-width immediate must be a positive machine immediate"); + const std::uint64_t Width = TypeAt(TypeIndex).scalarBitWidth(); + if ((StrictlySmaller && static_cast(*Value) >= Width) || + (!StrictlySmaller && static_cast(*Value) > Width)) { + return semanticError("generic bit-width immediate exceeds its register element width"); + } + return {}; + }; + + switch (Generic) { + case GO::G_ASSERT_SEXT: + case GO::G_ASSERT_ZEXT: + case GO::G_SEXT_INREG: + return VerifyImmediateBitWidth(2, 1, true); + case GO::G_ASSERT_ALIGN: { + const auto AlignmentValue = plainImmediate(Operands[2]); + if (!AlignmentValue || *AlignmentValue <= 0 || + !std::has_single_bit(static_cast(*AlignmentValue))) { + return semanticError("G_ASSERT_ALIGN requires a positive power-of-two alignment"); + } + return {}; + } + case GO::G_CONSTANT: { + if (const auto *Payload = Operands[1].getIf(); + Payload != nullptr && Payload->Value.bitWidth() != TypeAt(0).scalarBitWidth()) { + return semanticError("G_CONSTANT integer width must match its destination LLT"); + } + return {}; + } + case GO::G_FCONSTANT: { + const auto *Payload = Operands[1].getIf(); + if (Payload == nullptr || Payload->Bits.semantics() != TypeAt(0).floatSemantics()) + return semanticError("G_FCONSTANT semantics must match its destination LLT"); + return {}; + } + case GO::G_BITCAST: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (Dst.isPointerOrPointerVector() != Src.isPointerOrPointerVector()) + return semanticError("G_BITCAST cannot cross the pointer/non-pointer boundary"); + if (minimumBitWidth(Dst) != minimumBitWidth(Src) || + Dst.isScalableVector() != Src.isScalableVector()) { + return semanticError("G_BITCAST source and destination sizes must match"); + } + if (Dst == Src) + return semanticError("G_BITCAST must change the LLT"); + return {}; + } + case GO::G_INTTOPTR: + case GO::G_PTRTOINT: + return requireSameShape(TypeAt(0), TypeAt(1), + "pointer/integer conversion must preserve vector shape"); + case GO::G_ADDRSPACE_CAST: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (auto Shape = requireSameShape(Dst, Src, "G_ADDRSPACE_CAST must preserve vector shape"); + !Shape) { + return Shape; + } + if (Dst.addressSpace() == Src.addressSpace()) + return semanticError("G_ADDRSPACE_CAST must change the address space"); + return {}; + } + case GO::G_ANYEXT: + case GO::G_SEXT: + case GO::G_ZEXT: + case GO::G_FPEXT: + return VerifyWidthChange(true); + case GO::G_TRUNC: + case GO::G_TRUNC_SSAT_S: + case GO::G_TRUNC_SSAT_U: + case GO::G_TRUNC_USAT_U: + case GO::G_FPTRUNC: + return VerifyWidthChange(false); + case GO::G_PTR_ADD: + case GO::G_PTRMASK: + return requireSameShape(TypeAt(1), TypeAt(2), + "pointer arithmetic operands must have matching scalar/vector shape"); + case GO::G_ICMP: + case GO::G_FCMP: + case GO::G_STRICT_FCMP: + case GO::G_STRICT_FCMPS: { + const PredicateOperand *Predicate = Operands[1].getIf(); + if (Predicate == nullptr || ((Generic == GO::G_ICMP) != Predicate->isInteger())) { + return semanticError("generic comparison predicate kind does not match its opcode"); + } + return requireI1WithShape(TypeAt(0), TypeAt(2), false); + } + case GO::G_SCMP: + case GO::G_UCMP: + if (TypeAt(0).scalarBitWidth() < 2) + return semanticError("G_SCMP/G_UCMP result elements must be at least two bits wide"); + return requireSameShape(TypeAt(0), TypeAt(1), + "G_SCMP/G_UCMP must preserve scalar/vector shape"); + case GO::G_SELECT: + return requireI1WithShape(TypeAt(1), TypeAt(0), true); + case GO::G_UADDO: + case GO::G_USUBO: + case GO::G_SADDO: + case GO::G_SSUBO: + case GO::G_UMULO: + case GO::G_SMULO: + case GO::G_UADDE: + case GO::G_USUBE: + case GO::G_SADDE: + case GO::G_SSUBE: + return requireI1WithShape(TypeAt(1), TypeAt(0), false); + case GO::G_CTTZ: + case GO::G_CTTZ_ZERO_POISON: + case GO::G_CTLZ: + case GO::G_CTLZ_ZERO_POISON: + case GO::G_CTLS: + case GO::G_CTPOP: + return requireSameShape(TypeAt(0), TypeAt(1), + "generic bit-count operation must preserve scalar/vector shape"); + case GO::G_BSWAP: + if ((TypeAt(0).scalarBitWidth() % 16) != 0) + return semanticError("G_BSWAP element width must be a multiple of 16 bits"); + return {}; + case GO::G_FPTOSI: + case GO::G_FPTOUI: + case GO::G_FPTOSI_SAT: + case GO::G_FPTOUI_SAT: + case GO::G_SITOFP: + case GO::G_UITOFP: + case GO::G_LROUND: + case GO::G_LLROUND: + return requireSameShape(TypeAt(0), TypeAt(1), + "generic numeric conversion must preserve scalar/vector shape"); + case GO::G_FCOPYSIGN: + return requireSameShape(TypeAt(0), TypeAt(2), + "G_FCOPYSIGN operands must have matching scalar/vector shape"); + case GO::G_IS_FPCLASS: { + if (auto Shape = requireI1WithShape(TypeAt(0), TypeAt(1), false); !Shape) + return Shape; + const auto Classes = plainImmediate(Operands[2]); + if (!Classes || *Classes < 0 || *Classes > 0x3ff) + return semanticError("G_IS_FPCLASS class mask must fit the supported ten-bit mask"); + return {}; + } + case GO::G_FPOWI: + case GO::G_FLDEXP: + case GO::G_STRICT_FLDEXP: + return requireSameShape(TypeAt(0), TypeAt(2), + "floating operation exponent must match the result vector shape"); + case GO::G_FFREXP: + return requireSameShape(TypeAt(0), TypeAt(1), + "G_FFREXP mantissa and exponent results must have matching shape"); + case GO::G_MERGE_VALUES: { + if (TypeAt(0).isVector() || TypeAt(1).isVector()) + return semanticError("G_MERGE_VALUES only merges scalar values"); + if (minimumBitWidth(TypeAt(0)) != minimumBitWidth(TypeAt(1)) * (Operands.size() - 1)) + return semanticError("G_MERGE_VALUES source widths do not fill the destination"); + return {}; + } + case GO::G_UNMERGE_VALUES: { + const std::size_t DestinationCount = Operands.size() - 1; + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(DestinationCount); + if (minimumBitWidth(Src) != minimumBitWidth(Dst) * DestinationCount) + return semanticError("G_UNMERGE_VALUES destination widths do not cover the source"); + if (Dst.isVector() && (!Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src))) { + return semanticError("vector G_UNMERGE_VALUES operands have incompatible element types"); + } + return {}; + } + case GO::G_EXTRACT: { + const auto Offset = plainImmediate(Operands[2]); + const std::uint64_t DstBits = minimumBitWidth(TypeAt(0)); + const std::uint64_t SrcBits = minimumBitWidth(TypeAt(1)); + if (!Offset || *Offset < 0 || DstBits >= SrcBits || + static_cast(*Offset) > SrcBits - DstBits) { + return semanticError("G_EXTRACT offset and result width must stay inside its source"); + } + return {}; + } + case GO::G_INSERT: { + const auto Offset = plainImmediate(Operands[3]); + const std::uint64_t DstBits = minimumBitWidth(TypeAt(0)); + const std::uint64_t InsertBits = minimumBitWidth(TypeAt(2)); + if (!Offset || *Offset < 0 || InsertBits >= DstBits || + static_cast(*Offset) > DstBits - InsertBits) { + return semanticError("G_INSERT offset and inserted width must stay inside its result"); + } + return {}; + } + case GO::G_BUILD_VECTOR: + case GO::G_BUILD_VECTOR_TRUNC: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (!Dst.isVector() || Dst.isScalableVector() || Src.isVector()) + return semanticError("G_BUILD_VECTOR requires a fixed vector result and scalar sources"); + if (Dst.minimumElementCount() != Operands.size() - 1) + return semanticError("G_BUILD_VECTOR source count must equal the result lane count"); + const LLT Element = scalarElementType(Dst); + if (Generic == GO::G_BUILD_VECTOR && Element != Src) + return semanticError("G_BUILD_VECTOR source LLT must equal the result element LLT"); + if (Generic == GO::G_BUILD_VECTOR_TRUNC && Src.scalarBitWidth() <= Element.scalarBitWidth()) + return semanticError("G_BUILD_VECTOR_TRUNC sources must be wider than result elements"); + return {}; + } + case GO::G_CONCAT_VECTORS: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src) || + Dst.minimumElementCount() != Src.minimumElementCount() * (Operands.size() - 1)) { + return semanticError("G_CONCAT_VECTORS operands do not exactly tile the result vector"); + } + return {}; + } + case GO::G_VSCALE: { + const auto *Scale = Operands[1].getIf(); + if (TypeAt(0).isVector() || Scale == nullptr || Scale->Value.isZero() || + Scale->Value.bitWidth() != TypeAt(0).scalarBitWidth()) { + return semanticError("G_VSCALE requires a nonzero typed constant matching a scalar result"); + } + return {}; + } + case GO::G_SHL: + case GO::G_LSHR: + case GO::G_ASHR: + case GO::G_ROTR: + case GO::G_ROTL: + case GO::G_USHLSAT: + case GO::G_SSHLSAT: + case GO::G_FSHL: + case GO::G_FSHR: + return requireSameShape(TypeAt(1), TypeAt(Operands.size() - 1), + "shift amount must match the shifted value's scalar/vector shape"); + case GO::G_INSERT_SUBVECTOR: + case GO::G_EXTRACT_SUBVECTOR: { + const bool IsInsert = Generic == GO::G_INSERT_SUBVECTOR; + const LLT Outer = TypeAt(IsInsert ? 0 : 1); + const LLT Inner = TypeAt(IsInsert ? 2 : 0); + const auto Index = plainImmediate(Operands[IsInsert ? 3 : 2]); + if (!Outer.isVector() || !Inner.isVector() || + scalarElementType(Outer) != scalarElementType(Inner) || !Index || *Index < 0) { + return semanticError("subvector operation has incompatible vector types or index"); + } + if ((!Outer.isScalableVector() && Inner.isScalableVector()) || + Inner.minimumElementCount() > Outer.minimumElementCount()) { + return semanticError("subvector operation cannot place a larger vector inside a smaller one"); + } + const std::uint64_t Start = static_cast(*Index); + const std::uint64_t InnerLength = Inner.minimumElementCount(); + const std::uint64_t OuterLength = Outer.minimumElementCount(); + const bool MixedScalability = Outer.isScalableVector() && !Inner.isScalableVector(); + if ((Start % InnerLength) != 0 || Start >= OuterLength || + (!MixedScalability && Start + InnerLength > OuterLength)) { + return semanticError("subvector index is misaligned or outside the containing vector"); + } + return {}; + } + case GO::G_SHUFFLE_VECTOR: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + const auto *Mask = Operands[3].getIf(); + if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() || Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src) || Mask == nullptr || + Mask->Elements.size() != Dst.minimumElementCount()) { + return semanticError("G_SHUFFLE_VECTOR types and mask length are inconsistent"); + } + const std::uint64_t SourceLanes = Src.minimumElementCount(); + for (const std::int32_t Element : Mask->Elements) { + if (Element < ShuffleMaskOperand::Undef || + (Element >= 0 && static_cast(Element) >= 2 * SourceLanes)) { + return semanticError("G_SHUFFLE_VECTOR mask contains an invalid lane index"); + } + } + return {}; + } + case GO::G_SPLAT_VECTOR: { + const LLT Dst = TypeAt(0); + const LLT Src = TypeAt(1); + if (!Dst.isScalableVector() || Src.isVector() || + scalarElementType(Dst).scalarBitWidth() > Src.scalarBitWidth()) { + return semanticError("G_SPLAT_VECTOR requires a scalable result with no wider elements"); + } + return {}; + } + case GO::G_STEP_VECTOR: { + const LLT Dst = TypeAt(0); + const auto *Step = Operands[1].getIf(); + if (!Dst.isScalableVector() || Step == nullptr || Step->Value.isZero() || + Step->Value.isNegative() || Step->Value.bitWidth() != Dst.scalarBitWidth()) { + return semanticError("G_STEP_VECTOR step must be a positive typed element-width constant"); + } + return {}; + } + case GO::G_INSERT_VECTOR_ELT: { + const LLT Vector = TypeAt(0); + if (scalarElementType(Vector) != TypeAt(2) || TypeAt(3).isVector()) + return semanticError("G_INSERT_VECTOR_ELT element or index LLT is incompatible"); + return {}; + } + case GO::G_EXTRACT_VECTOR_ELT: { + const LLT Vector = TypeAt(1); + if (scalarElementType(Vector) != TypeAt(0) || TypeAt(2).isVector()) + return semanticError("G_EXTRACT_VECTOR_ELT result or index LLT is incompatible"); + return {}; + } + case GO::G_VECTOR_COMPRESS: + return requireI1WithShape(TypeAt(2), TypeAt(0), false); + case GO::G_VECREDUCE_SEQ_FADD: + case GO::G_VECREDUCE_SEQ_FMUL: + if (TypeAt(0).isVector() || TypeAt(1).isVector() || !TypeAt(2).isVector() || + scalarElementType(TypeAt(2)) != TypeAt(0) || TypeAt(0) != TypeAt(1)) { + return semanticError("sequential vector reduction accumulator and element LLTs must match"); + } + return {}; + case GO::G_VECREDUCE_FADD: + case GO::G_VECREDUCE_FMUL: + case GO::G_VECREDUCE_FMAX: + case GO::G_VECREDUCE_FMIN: + case GO::G_VECREDUCE_FMAXIMUM: + case GO::G_VECREDUCE_FMINIMUM: + case GO::G_VECREDUCE_ADD: + case GO::G_VECREDUCE_MUL: + case GO::G_VECREDUCE_AND: + case GO::G_VECREDUCE_OR: + case GO::G_VECREDUCE_XOR: + case GO::G_VECREDUCE_SMAX: + case GO::G_VECREDUCE_SMIN: + case GO::G_VECREDUCE_UMAX: + case GO::G_VECREDUCE_UMIN: + if (TypeAt(0).isVector() || !TypeAt(1).isVector() || + scalarElementType(TypeAt(1)) != TypeAt(0)) { + return semanticError("vector reduction result must equal the source element LLT"); + } + return {}; + case GO::G_SBFX: + case GO::G_UBFX: + if (TypeAt(0).isVector() || TypeAt(2).isVector() || TypeAt(3).isVector()) + return semanticError("generic bit-field extraction only supports scalar operands"); + return {}; + case GO::G_BRCOND: + if (!isI1(TypeAt(0)) || TypeAt(0).isVector()) + return semanticError("G_BRCOND condition must be a scalar i1 virtual register"); + return {}; + case GO::G_BRJT: + if (TypeAt(0).isVector() || TypeAt(2).isVector()) + return semanticError("G_BRJT table pointer and index must be scalar values"); + return {}; + case GO::G_INTRINSIC: + case GO::G_INTRINSIC_W_SIDE_EFFECTS: + case GO::G_INTRINSIC_CONVERGENT: + case GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: { + const auto FirstUse = std::ranges::find_if_not(Operands, [](const MachineOperand &Operand) { + return Operand.isRegister() && Operand.isDef(); + }); + if (FirstUse == Operands.end() || FirstUse->kind() != MachineOperand::Kind::Intrinsic) + return semanticError("generic intrinsic must place its intrinsic ID after all definitions"); + return {}; + } + case GO::G_INDEXED_LOAD: + case GO::G_INDEXED_SEXTLOAD: + case GO::G_INDEXED_ZEXTLOAD: + if (const auto Mode = plainImmediate(Operands[4]); !Mode || (*Mode != 0 && *Mode != 1)) + return semanticError("generic indexed-load address mode must be immediate 0 or 1"); + return {}; + case GO::G_INDEXED_STORE: + if (const auto Mode = plainImmediate(Operands[4]); !Mode || (*Mode != 0 && *Mode != 1)) + return semanticError("G_INDEXED_STORE address mode must be immediate 0 or 1"); + return {}; + case GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + return requireI1WithShape(TypeAt(1), TypeAt(0), false); + case GO::G_FENCE: { + const auto Ordering = plainImmediate(Operands[0]); + const auto Scope = plainImmediate(Operands[1]); + if (!Ordering || *Ordering < 4 || *Ordering > 7 || !Scope || *Scope < 0 || + static_cast(*Scope) > std::numeric_limits::max()) { + return semanticError("G_FENCE requires acquire-or-stronger ordering and a uint32 scope"); + } + return {}; + } + case GO::G_PREFETCH: { + const auto ReadWrite = plainImmediate(Operands[1]); + const auto Locality = plainImmediate(Operands[2]); + const auto CacheType = plainImmediate(Operands[3]); + if (!ReadWrite || *ReadWrite < 0 || *ReadWrite > 1 || !Locality || *Locality < 0 || + *Locality > 3 || !CacheType || *CacheType < 0 || *CacheType > 1) { + return semanticError("G_PREFETCH control operands are outside their defined ranges"); + } + return {}; + } + case GO::G_VAARG: { + const auto Align = plainImmediate(Operands[2]); + if (!Align || *Align <= 0 || !std::has_single_bit(static_cast(*Align))) + return semanticError("G_VAARG alignment must be a positive power of two"); + return {}; + } + case GO::G_DYN_STACKALLOC: { + const auto Align = plainImmediate(Operands[2]); + if (!Align || *Align <= 0 || !std::has_single_bit(static_cast(*Align))) + return semanticError("G_DYN_STACKALLOC alignment must be a positive power of two"); + return {}; + } + case GO::G_MEMCPY: + case GO::G_MEMMOVE: + if (const auto Tail = plainImmediate(Operands[3]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("generic memory-transfer tail flag must be immediate 0 or 1"); + return {}; + case GO::G_MEMSET: + if (TypeAt(1).isVector() || TypeAt(1).scalarBitWidth() != 8) + return semanticError("G_MEMSET fill value must be a scalar i8 value"); + if (const auto Tail = plainImmediate(Operands[3]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("G_MEMSET tail flag must be immediate 0 or 1"); + return {}; + case GO::G_BZERO: + if (const auto Tail = plainImmediate(Operands[2]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("G_BZERO tail flag must be immediate 0 or 1"); + return {}; + case GO::G_UBSANTRAP: { + const auto Kind = plainImmediate(Operands[0]); + if (!Kind || *Kind < -128 || *Kind > 127) + return semanticError("G_UBSANTRAP kind must fit in a signed eight-bit immediate"); + return {}; + } + case GO::G_INTRINSIC_FPTRUNC_ROUND: { + const auto Rounding = plainImmediate(Operands[2]); + if (!Rounding || !((*Rounding >= 0 && *Rounding <= 4) || *Rounding == 7)) + return semanticError("floating truncation rounding mode is invalid"); + return VerifyWidthChange(false); + } + default: + return {}; + } +} + +[[nodiscard]] Expected verifyGenericInstructionSemantics(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo) { + const auto Operands = Instruction.operands(); + const auto MemoryOperands = Instruction.memOperands(); + const GO Generic = static_cast(Instruction.opcode().value()); + + const auto TypeAt = [&](std::size_t Index) -> Expected { + if (Index >= Operands.size()) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier operand index is out of range"); + const Register *Reg = Operands[Index].getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier expected a virtual register"); + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier found a register without an LLT"); + return *Info->get().Type; + }; + + const auto VerifyAddress = [&](std::size_t OperandIndex, + const MachineMemOperand &Memory) -> Expected { + auto Type = TypeAt(OperandIndex); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isPointer() || Type->addressSpace() != Memory.description().AddressSpace) + return semanticError("generic memory address LLT and MMO address space do not match"); + return {}; + }; + + enum class SizeRelation : std::uint8_t { + Equal, + NoLarger, + StrictlySmaller, + }; + const auto VerifySize = [&](LLT ValueType, const MachineMemOperand &Memory, + SizeRelation Relation) -> Expected { + const MemorySize Size = Memory.description().Size; + if (!Size.isKnown()) + return semanticError("typed generic memory access requires a known MMO size"); + if (Size.isScalable() != ValueType.isScalableVector()) + return semanticError("generic value and MMO scalability do not match"); + const std::uint64_t MemoryBytes = Size.minimumBytes(); + const std::uint64_t ValueBytes = minimumStoreBytes(ValueType); + const bool Valid = Relation == SizeRelation::Equal + ? MemoryBytes == ValueBytes + : (Relation == SizeRelation::NoLarger ? MemoryBytes <= ValueBytes + : MemoryBytes < ValueBytes); + if (!Valid) + return semanticError("generic memory access size is incompatible with its value LLT"); + return {}; + }; + + const auto VerifySingleAccess = [&](std::size_t AddressIndex, std::size_t ValueIndex, + bool MustLoad, bool MustStore, + SizeRelation Relation) -> Expected { + if (MemoryOperands.size() != 1) + return semanticError("generic scalar memory instruction requires exactly one MMO"); + const MachineMemOperand &Memory = *MemoryOperands.front(); + if (Memory.isLoad() != MustLoad || Memory.isStore() != MustStore) + return semanticError("generic instruction and MMO access directions do not match"); + if (auto Address = VerifyAddress(AddressIndex, Memory); !Address) + return Address; + auto ValueType = TypeAt(ValueIndex); + if (!ValueType) + return std::unexpected(ValueType.error()); + return VerifySize(*ValueType, Memory, Relation); + }; + + switch (Generic) { + case GO::G_LOAD: + return VerifySingleAccess(1, 0, true, false, SizeRelation::NoLarger); + case GO::G_SEXTLOAD: + case GO::G_ZEXTLOAD: + case GO::G_FPEXTLOAD: + return VerifySingleAccess(1, 0, true, false, SizeRelation::StrictlySmaller); + case GO::G_STORE: + return VerifySingleAccess(1, 0, false, true, SizeRelation::NoLarger); + case GO::G_FPTRUNCSTORE: + return VerifySingleAccess(1, 0, false, true, SizeRelation::StrictlySmaller); + case GO::G_INDEXED_LOAD: + return VerifySingleAccess(2, 0, true, false, SizeRelation::NoLarger); + case GO::G_INDEXED_SEXTLOAD: + case GO::G_INDEXED_ZEXTLOAD: + return VerifySingleAccess(2, 0, true, false, SizeRelation::StrictlySmaller); + case GO::G_INDEXED_STORE: + return VerifySingleAccess(2, 1, false, true, SizeRelation::NoLarger); + case GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) + return semanticError("G_ATOMIC_CMPXCHG_WITH_SUCCESS requires one compare-exchange MMO"); + return VerifySingleAccess(2, 0, true, true, SizeRelation::Equal); + case GO::G_ATOMIC_CMPXCHG: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) + return semanticError("G_ATOMIC_CMPXCHG requires one compare-exchange MMO"); + return VerifySingleAccess(1, 0, true, true, SizeRelation::Equal); + case GO::G_ATOMICRMW_XCHG: + case GO::G_ATOMICRMW_ADD: + case GO::G_ATOMICRMW_SUB: + case GO::G_ATOMICRMW_AND: + case GO::G_ATOMICRMW_NAND: + case GO::G_ATOMICRMW_OR: + case GO::G_ATOMICRMW_XOR: + case GO::G_ATOMICRMW_MAX: + case GO::G_ATOMICRMW_MIN: + case GO::G_ATOMICRMW_UMAX: + case GO::G_ATOMICRMW_UMIN: + case GO::G_ATOMICRMW_FADD: + case GO::G_ATOMICRMW_FSUB: + case GO::G_ATOMICRMW_FMAX: + case GO::G_ATOMICRMW_FMIN: + case GO::G_ATOMICRMW_FMAXIMUM: + case GO::G_ATOMICRMW_FMINIMUM: + case GO::G_ATOMICRMW_FMAXIMUMNUM: + case GO::G_ATOMICRMW_FMINIMUMNUM: + case GO::G_ATOMICRMW_UINC_WRAP: + case GO::G_ATOMICRMW_UDEC_WRAP: + case GO::G_ATOMICRMW_USUB_COND: + case GO::G_ATOMICRMW_USUB_SAT: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::ReadModifyWrite) + return semanticError("generic atomic RMW requires one read-modify-write MMO"); + return VerifySingleAccess(1, 0, true, true, SizeRelation::Equal); + case GO::G_PREFETCH: { + if (MemoryOperands.size() != 1) + return semanticError("G_PREFETCH requires exactly one MMO"); + const auto ReadWrite = plainImmediate(Operands[1]); + if (!ReadWrite) + return semanticError("G_PREFETCH is missing its validated read/write selector"); + const MachineMemOperand &Memory = *MemoryOperands.front(); + if ((*ReadWrite == 0 && (!Memory.isLoad() || Memory.isStore())) || + (*ReadWrite == 1 && (Memory.isLoad() || !Memory.isStore()))) { + return semanticError("G_PREFETCH selector and MMO access direction do not match"); + } + return VerifyAddress(0, Memory); + } + case GO::G_VASTART: + if (MemoryOperands.size() != 1 || MemoryOperands.front()->isLoad() || + !MemoryOperands.front()->isStore()) { + return semanticError("G_VASTART requires exactly one store-only MMO"); + } + return VerifyAddress(0, *MemoryOperands.front()); + case GO::G_VAARG: { + if (MemoryOperands.size() != 2) + return semanticError("G_VAARG requires separate value-load and list-update MMOs"); + const MachineMemOperand *ValueLoad = nullptr; + const MachineMemOperand *ListStore = nullptr; + for (const MachineMemOperand *Memory : MemoryOperands) { + if (Memory->isLoad() == Memory->isStore()) { + return semanticError("G_VAARG MMOs must each describe exactly one access direction"); + } + if (Memory->isLoad()) + ValueLoad = Memory; + else + ListStore = Memory; + if (auto Address = VerifyAddress(1, *Memory); !Address) + return Address; + } + if (ValueLoad == nullptr || ListStore == nullptr) + return semanticError("G_VAARG MMOs must cover both the value load and list update"); + auto ResultType = TypeAt(0); + if (!ResultType) + return std::unexpected(ResultType.error()); + return VerifySize(*ResultType, *ValueLoad, SizeRelation::Equal); + } + case GO::G_MEMCPY: + case GO::G_MEMCPY_INLINE: + case GO::G_MEMMOVE: { + if (MemoryOperands.size() != 2 || MemoryOperands[0]->isLoad() || + !MemoryOperands[0]->isStore() || !MemoryOperands[1]->isLoad() || + MemoryOperands[1]->isStore()) { + return semanticError("generic memory transfer requires store and load MMOs in that order"); + } + if (MemoryOperands[0]->description().Size != MemoryOperands[1]->description().Size) + return semanticError("generic memory-transfer MMOs must have identical sizes"); + if (auto Destination = VerifyAddress(0, *MemoryOperands[0]); !Destination) + return Destination; + return VerifyAddress(1, *MemoryOperands[1]); + } + case GO::G_MEMSET: + case GO::G_BZERO: + if (MemoryOperands.size() != 1 || MemoryOperands.front()->isLoad() || + !MemoryOperands.front()->isStore()) { + return semanticError("generic memory fill requires exactly one store-only MMO"); + } + return VerifyAddress(0, *MemoryOperands.front()); + default: + return {}; + } +} + +void configureCoreOpcodes(DescriptorRegistry &Registry) { + const auto RegDef = untypedRegister(RegisterOperandRole::Def); + const auto RegUse = untypedRegister(RegisterOperandRole::Use); + const auto Any = anyOperand(); + const auto VariadicAny = repeated(Any); + const auto VariadicUse = repeated(RegUse); + + Registry.configure({GO::PHI}, + {RegDef, VariadicUse, repeated(kind(MachineOperand::Kind::BasicBlock))}, 1, + {InstructionFlag::Phi}, true, 1, false, PseudoOwner::PhiElimination, + PipelineStage::SSAEliminated); + Registry.configure({GO::INLINEASM}, {VariadicAny}, 0, + {InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission, + InstructionFlag::UnmodeledSideEffects}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::INLINEASM_BR}, {VariadicAny}, 0, + {InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission, + InstructionFlag::UnmodeledSideEffects, InstructionFlag::HasSideEffects}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::CFI_INSTRUCTION}, {kind(MachineOperand::Kind::CFIIndex)}, 0, + {InstructionFlag::Meta, InstructionFlag::NotDuplicable}); + Registry.configure({GO::EH_LABEL, GO::GC_LABEL, GO::ANNOTATION_LABEL}, + {kind(MachineOperand::Kind::MCSymbol)}, 0, + {InstructionFlag::Meta, InstructionFlag::NotDuplicable}); + Registry.configure({GO::KILL}, {VariadicUse}, 0, {InstructionFlag::Meta}, true); + Registry.configure({GO::EXTRACT_SUBREG}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized); + Registry.configure({GO::INSERT_SUBREG}, {RegDef, RegUse, RegUse, immediate()}, 1, {}, false, 0, + false, PseudoOwner::TwoAddressRewrite, PipelineStage::TiedOpsRewritten); + Registry.tie(GO::INSERT_SUBREG, 0, 1); + Registry.configure({GO::IMPLICIT_DEF}, {RegDef}, 1, + {InstructionFlag::Meta, InstructionFlag::Rematerializable}); + Registry.configure({GO::INIT_UNDEF}, {RegDef}, 1, {}, false, 0, false, + PseudoOwner::RegisterAllocator, PipelineStage::RegisterAllocated); + Registry.configure({GO::SUBREG_TO_REG}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized); + Registry.configure({GO::COPY_TO_REGCLASS}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::RegisterAllocator, PipelineStage::RegisterAllocated); + Registry.configure({GO::DBG_VALUE, GO::DBG_VALUE_LIST, GO::DBG_INSTR_REF, GO::DBG_PHI}, + {VariadicAny}, 0, {InstructionFlag::Meta}, true); + Registry.configure({GO::DBG_LABEL}, {kind(MachineOperand::Kind::Metadata)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::REG_SEQUENCE}, {RegDef, repeated(RegUse), repeated(immediate())}, 1, {}, + true, 1, false, PseudoOwner::TwoAddressRewrite, + PipelineStage::TiedOpsRewritten); + Registry.configure({GO::COPY}, {RegDef, RegUse}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized); + Registry.configure({GO::COPY_LANEMASK}, {RegDef, RegUse, kind(MachineOperand::Kind::LaneMask)}, 1, + {}, false, 0, false, PseudoOwner::PostRAExpansion, + PipelineStage::PostRAFinalized); + Registry.configure({GO::BUNDLE}, {VariadicAny}, 0, {InstructionFlag::Meta}, true, 0, false, + PseudoOwner::None, PipelineStage::Emission, true); + Registry.configure({GO::LIFETIME_START, GO::LIFETIME_END}, {immediate()}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::PSEUDO_PROBE}, {immediate(), immediate(), immediate(), immediate()}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::ARITH_FENCE}, {RegDef, RegUse}, 1, {InstructionFlag::Meta}); + Registry.tie(GO::ARITH_FENCE, 0, 1); + Registry.configure({GO::STACKMAP}, {immediate(), immediate(), VariadicAny}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::FENTRY_CALL}, {}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure( + {GO::PATCHPOINT}, + {RegDef, immediate(), immediate(), Any, immediate(), immediate(), VariadicAny}, 1, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::HasSideEffects, + InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::LOAD_STACK_GUARD}, {RegDef}, 1, + {InstructionFlag::MayLoad, InstructionFlag::Rematerializable}, false, 0, false, + PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::PREALLOCATED_SETUP}, {immediate()}, 0, {InstructionFlag::HasSideEffects}, + false, 0, false, PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::PREALLOCATED_ARG}, {RegDef, immediate(), immediate()}, 1, + {InstructionFlag::HasSideEffects}, false, 0, false, + PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::STATEPOINT}, {VariadicAny}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::LOCAL_ESCAPE}, {symbol(), kind(MachineOperand::Kind::FrameIndex)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::FAULTING_OP}, {RegDef, VariadicAny}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::Terminator, + InstructionFlag::Branch, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_OP}, {VariadicAny}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_FUNCTION_ENTER, GO::PATCHABLE_FUNCTION_EXIT}, {}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission); + Registry.configure({GO::PATCHABLE_RET, GO::PATCHABLE_TAIL_CALL}, {VariadicAny}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission, InstructionFlag::Terminator, + InstructionFlag::Return}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_EVENT_CALL}, {RegUse, Any}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_TYPED_EVENT_CALL}, {Any, RegUse, Any}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::ICALL_BRANCH_FUNNEL}, {VariadicAny}, 0, {InstructionFlag::HasSideEffects}, + true, 0, false, PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized, + true); + Registry.configure({GO::FAKE_USE}, {VariadicUse}, 0, + {InstructionFlag::Meta, InstructionFlag::FakeUse}, true); + Registry.configure({GO::MEMBARRIER}, {}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::JUMP_TABLE_DEBUG_INFO}, {kind(MachineOperand::Kind::JumpTableIndex)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::RELOC_NONE}, {symbol()}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::CONVERGENCECTRL_ENTRY, GO::CONVERGENCECTRL_ANCHOR}, {RegDef}, 1, + {InstructionFlag::Meta, InstructionFlag::Convergent}); + Registry.configure({GO::CONVERGENCECTRL_LOOP}, {RegDef, RegUse}, 1, + {InstructionFlag::Meta, InstructionFlag::Convergent}); + Registry.configure({GO::CONVERGENCECTRL_GLUE}, {RegUse}, 0, + {InstructionFlag::Meta, InstructionFlag::Convergent}); +} + +void configureGenericOpcodes(DescriptorRegistry &Registry) { + const LLTKindSet AnyTypes = anyValueKinds(); + const LLTKindSet IntTypes = integerKinds(); + const LLTKindSet ScalarValues = scalarValueKinds(); + const LLTKindSet ScalarInts = scalarIntegerKinds(); + const LLTKindSet FloatTypes = floatingKinds(); + const LLTKindSet ScalarFloats = scalarFloatingKinds(); + const LLTKindSet PointerTypes = pointerKinds(); + const LLTKindSet ScalarPointers = scalarPointerKinds(); + const LLTKindSet VectorTypes = vectorKinds(); + + const auto AnyDef0 = typedRegister(RegisterOperandRole::Def, AnyTypes, 0); + const auto AnyUse0 = typedRegister(RegisterOperandRole::Use, AnyTypes, 0); + const auto AnyUse1 = typedRegister(RegisterOperandRole::Use, AnyTypes, 1); + const auto ScalarDef0 = typedRegister(RegisterOperandRole::Def, ScalarValues, 0); + const auto ScalarUse0 = typedRegister(RegisterOperandRole::Use, ScalarValues, 0); + const auto IntDef0 = typedRegister(RegisterOperandRole::Def, IntTypes, 0); + const auto IntDef1 = typedRegister(RegisterOperandRole::Def, IntTypes, 1); + const auto IntUse0 = typedRegister(RegisterOperandRole::Use, IntTypes, 0); + const auto IntUse1 = typedRegister(RegisterOperandRole::Use, IntTypes, 1); + const auto ScalarIntDef0 = typedRegister(RegisterOperandRole::Def, ScalarInts, 0); + const auto ScalarIntDef1 = typedRegister(RegisterOperandRole::Def, ScalarInts, 1); + const auto ScalarIntUse0 = typedRegister(RegisterOperandRole::Use, ScalarInts, 0); + const auto ScalarIntUse1 = typedRegister(RegisterOperandRole::Use, ScalarInts, 1); + const auto ScalarIntUse2 = typedRegister(RegisterOperandRole::Use, ScalarInts, 2); + const auto FloatDef0 = typedRegister(RegisterOperandRole::Def, FloatTypes, 0); + const auto FloatUse0 = typedRegister(RegisterOperandRole::Use, FloatTypes, 0); + const auto FloatUse1 = typedRegister(RegisterOperandRole::Use, FloatTypes, 1); + const auto ScalarFloatDef0 = typedRegister(RegisterOperandRole::Def, ScalarFloats, 0); + const auto ScalarFloatUse0 = typedRegister(RegisterOperandRole::Use, ScalarFloats, 0); + const auto PointerDef0 = typedRegister(RegisterOperandRole::Def, PointerTypes, 0); + const auto PointerUse0 = typedRegister(RegisterOperandRole::Use, PointerTypes, 0); + const auto PointerUse1 = typedRegister(RegisterOperandRole::Use, PointerTypes, 1); + const auto ScalarPointerDef0 = typedRegister(RegisterOperandRole::Def, ScalarPointers, 0); + const auto ScalarPointerDef1 = typedRegister(RegisterOperandRole::Def, ScalarPointers, 1); + const auto ScalarPointerUse0 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 0); + const auto ScalarPointerUse1 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 1); + const auto ScalarPointerUse2 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 2); + + Registry.configure({GO::G_ASSERT_SEXT, GO::G_ASSERT_ZEXT}, {IntDef0, IntUse0, immediate()}, 1); + Registry.configure({GO::G_ASSERT_ALIGN}, {AnyDef0, AnyUse0, immediate()}, 1); + + Registry.configure( + {GO::G_ADD, GO::G_SUB, GO::G_MUL, GO::G_SDIV, GO::G_UDIV, GO::G_SREM, + GO::G_UREM, GO::G_AND, GO::G_OR, GO::G_XOR, GO::G_ABDS, GO::G_ABDU, + GO::G_UAVGFLOOR, GO::G_UAVGCEIL, GO::G_SAVGFLOOR, GO::G_SAVGCEIL, GO::G_UMULH, GO::G_SMULH, + GO::G_UADDSAT, GO::G_SADDSAT, GO::G_USUBSAT, GO::G_SSUBSAT, GO::G_SMIN, GO::G_SMAX, + GO::G_UMIN, GO::G_UMAX, GO::G_CLMUL}, + {IntDef0, IntUse0, IntUse0}, 1); + Registry.addFlag({GO::G_ADD, GO::G_MUL, GO::G_AND, GO::G_OR, GO::G_XOR, GO::G_ABDS, GO::G_ABDU, + GO::G_UMULH, GO::G_SMULH, GO::G_UADDSAT, GO::G_SADDSAT, GO::G_SMIN, GO::G_SMAX, + GO::G_UMIN, GO::G_UMAX, GO::G_CLMUL}, + InstructionFlag::Commutable); + + Registry.configure({GO::G_SDIVREM, GO::G_UDIVREM}, {IntDef0, IntDef0, IntUse0, IntUse0}, 2); + Registry.configure({GO::G_IMPLICIT_DEF}, {AnyDef0}, 1); + Registry.configure({GO::G_PHI}, + {AnyDef0, repeated(AnyUse0), repeated(kind(MachineOperand::Kind::BasicBlock))}, + 1, {InstructionFlag::Phi}, true, 1); + Registry.configure({GO::G_FRAME_INDEX}, + {ScalarPointerDef0, kind(MachineOperand::Kind::FrameIndex)}, 1); + Registry.configure({GO::G_GLOBAL_VALUE}, + {ScalarPointerDef0, kind(MachineOperand::Kind::GlobalSymbol)}, 1); + Registry.configure( + {GO::G_PTRAUTH_GLOBAL_VALUE}, + {ScalarPointerDef0, ScalarPointerUse1, immediate(), ScalarIntUse2, immediate()}, 1); + Registry.configure({GO::G_CONSTANT_POOL}, + {ScalarPointerDef0, kind(MachineOperand::Kind::ConstantPoolIndex)}, 1); + Registry.configure({GO::G_EXTRACT}, {AnyDef0, AnyUse1, immediate()}, 1); + Registry.configure({GO::G_UNMERGE_VALUES}, {AnyDef0, repeated(AnyDef0), AnyUse1}, 2, {}, true, 1); + Registry.configure({GO::G_INSERT}, {AnyDef0, AnyUse0, AnyUse1, immediate()}, 1); + Registry.configure({GO::G_MERGE_VALUES}, {AnyDef0, AnyUse1, repeated(AnyUse1)}, 1, {}, true, 1); + + const auto VectorDef0 = typedRegister(RegisterOperandRole::Def, VectorTypes, 0); + Registry.configure({GO::G_BUILD_VECTOR, GO::G_BUILD_VECTOR_TRUNC}, + {VectorDef0, AnyUse1, repeated(AnyUse1)}, 1, {}, true); + Registry.configure({GO::G_CONCAT_VECTORS}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + repeated(typedRegister(RegisterOperandRole::Use, VectorTypes, 1))}, + 1, {}, true, 1); + + Registry.configure({GO::G_PTRTOINT}, {IntDef0, PointerUse1}, 1); + Registry.configure({GO::G_INTTOPTR}, {PointerDef0, IntUse1}, 1); + Registry.configure({GO::G_BITCAST}, {AnyDef0, AnyUse1}, 1); + Registry.configure({GO::G_FREEZE, GO::G_CONSTANT_FOLD_BARRIER}, {AnyDef0, AnyUse0}, 1); + Registry.configure({GO::G_INTRINSIC_FPTRUNC_ROUND}, {FloatDef0, FloatUse1, immediate()}, 1); + Registry.configure({GO::G_INTRINSIC_TRUNC, GO::G_INTRINSIC_ROUND, GO::G_INTRINSIC_ROUNDEVEN}, + {FloatDef0, FloatUse0}, 1); + Registry.configure({GO::G_INTRINSIC_LRINT, GO::G_INTRINSIC_LLRINT}, {IntDef0, FloatUse1}, 1); + Registry.configure({GO::G_READCYCLECOUNTER, GO::G_READSTEADYCOUNTER}, + {typedRegister(RegisterOperandRole::Def, ScalarInts, 0)}, 1, + {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_LOAD}, {AnyDef0, ScalarPointerUse1}, 1, {InstructionFlag::MayLoad}, + false, 0, true); + Registry.configure({GO::G_SEXTLOAD, GO::G_ZEXTLOAD}, {IntDef0, ScalarPointerUse1}, 1, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_FPEXTLOAD}, {FloatDef0, ScalarPointerUse1}, 1, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_INDEXED_LOAD}, + {AnyDef0, ScalarPointerDef1, ScalarPointerUse1, ScalarIntUse2, immediate()}, 2, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_INDEXED_SEXTLOAD, GO::G_INDEXED_ZEXTLOAD}, + {IntDef0, ScalarPointerDef1, ScalarPointerUse1, ScalarIntUse2, immediate()}, 2, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_STORE}, {AnyUse0, ScalarPointerUse1}, 0, {InstructionFlag::MayStore}, + false, 0, true); + Registry.configure({GO::G_FPTRUNCSTORE}, {FloatUse0, ScalarPointerUse1}, 0, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_INDEXED_STORE}, + {ScalarPointerDef0, AnyUse1, ScalarPointerUse0, ScalarIntUse2, immediate()}, 1, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS}, + {ScalarDef0, ScalarIntDef1, ScalarPointerUse2, ScalarUse0, ScalarUse0}, 2, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMIC_CMPXCHG}, + {ScalarDef0, ScalarPointerUse1, ScalarUse0, ScalarUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_XCHG}, {ScalarDef0, ScalarPointerUse1, ScalarUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_ADD, GO::G_ATOMICRMW_SUB, GO::G_ATOMICRMW_AND, + GO::G_ATOMICRMW_NAND, GO::G_ATOMICRMW_OR, GO::G_ATOMICRMW_XOR, + GO::G_ATOMICRMW_MAX, GO::G_ATOMICRMW_MIN, GO::G_ATOMICRMW_UMAX, + GO::G_ATOMICRMW_UMIN, GO::G_ATOMICRMW_UINC_WRAP, GO::G_ATOMICRMW_UDEC_WRAP, + GO::G_ATOMICRMW_USUB_COND, GO::G_ATOMICRMW_USUB_SAT}, + {ScalarIntDef0, ScalarPointerUse1, ScalarIntUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_FADD, GO::G_ATOMICRMW_FSUB, GO::G_ATOMICRMW_FMAX, + GO::G_ATOMICRMW_FMIN, GO::G_ATOMICRMW_FMAXIMUM, GO::G_ATOMICRMW_FMINIMUM, + GO::G_ATOMICRMW_FMAXIMUMNUM, GO::G_ATOMICRMW_FMINIMUMNUM}, + {ScalarFloatDef0, ScalarPointerUse1, ScalarFloatUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_FENCE}, {immediate(), immediate()}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure( + {GO::G_PREFETCH}, {ScalarPointerUse0, immediate(), immediate(), immediate()}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, InstructionFlag::HasSideEffects}, false, + 0, true); + Registry.configure({GO::G_BRCOND}, {ScalarIntUse0, kind(MachineOperand::Kind::BasicBlock)}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch}); + Registry.configure({GO::G_BRINDIRECT}, {ScalarPointerUse0}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, + InstructionFlag::IndirectBranch, InstructionFlag::Barrier}); + Registry.configure({GO::G_INVOKE_REGION_START}, {}, 0, {InstructionFlag::Terminator}); + Registry.configure({GO::G_INTRINSIC, GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT, + GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + {repeated(anyOperand())}, 0, {}, true, 1, false, PseudoOwner::None, + PipelineStage::Emission, true); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::HasSideEffects); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::MayLoad); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::MayStore); + Registry.addFlag({GO::G_INTRINSIC_CONVERGENT, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::Convergent); + + Registry.configure({GO::G_ANYEXT, GO::G_SEXT, GO::G_ZEXT}, {IntDef0, IntUse1}, 1); + Registry.configure({GO::G_TRUNC, GO::G_TRUNC_SSAT_S, GO::G_TRUNC_SSAT_U, GO::G_TRUNC_USAT_U}, + {IntDef0, IntUse1}, 1); + Registry.configure( + {GO::G_CONSTANT}, + {typedRegister(RegisterOperandRole::Def, ScalarInts, 0), + operand({MachineOperand::Kind::Immediate, MachineOperand::Kind::IntegerImmediate})}, + 1); + Registry.configure({GO::G_FCONSTANT}, + {ScalarFloatDef0, kind(MachineOperand::Kind::FloatingImmediate)}, 1); + Registry.configure({GO::G_VASTART}, {ScalarPointerUse0}, 0, {InstructionFlag::MayStore}, false, 0, + true); + Registry.configure({GO::G_VAARG}, {AnyDef0, ScalarPointerUse1, immediate()}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_SEXT_INREG}, {IntDef0, IntUse0, immediate()}, 1); + Registry.configure( + {GO::G_SHL, GO::G_LSHR, GO::G_ASHR, GO::G_ROTR, GO::G_ROTL, GO::G_USHLSAT, GO::G_SSHLSAT}, + {IntDef0, IntUse0, IntUse1}, 1); + Registry.configure({GO::G_FSHL, GO::G_FSHR}, {IntDef0, IntUse0, IntUse0, IntUse1}, 1); + Registry.configure({GO::G_ICMP}, + {IntDef0, kind(MachineOperand::Kind::Predicate), IntUse1, IntUse1}, 1); + Registry.configure({GO::G_FCMP}, + {IntDef0, kind(MachineOperand::Kind::Predicate), FloatUse1, FloatUse1}, 1); + Registry.configure({GO::G_SCMP, GO::G_UCMP}, {IntDef0, IntUse1, IntUse1}, 1); + Registry.configure({GO::G_SELECT}, {AnyDef0, IntUse1, AnyUse0, AnyUse0}, 1); + + Registry.configure({GO::G_UADDO, GO::G_USUBO, GO::G_SADDO, GO::G_SSUBO, GO::G_UMULO, GO::G_SMULO}, + {IntDef0, IntDef1, IntUse0, IntUse0}, 2); + Registry.configure({GO::G_UADDE, GO::G_USUBE, GO::G_SADDE, GO::G_SSUBE}, + {IntDef0, IntDef1, IntUse0, IntUse0, IntUse1}, 2); + Registry.addFlag({GO::G_UADDO, GO::G_SADDO, GO::G_UMULO, GO::G_SMULO}, + InstructionFlag::Commutable); + Registry.configure({GO::G_SMULFIX, GO::G_UMULFIX, GO::G_SMULFIXSAT, GO::G_UMULFIXSAT, + GO::G_SDIVFIX, GO::G_UDIVFIX, GO::G_SDIVFIXSAT, GO::G_UDIVFIXSAT}, + {IntDef0, IntUse0, IntUse0, immediate()}, 1); + Registry.addFlag({GO::G_SMULFIX, GO::G_UMULFIX, GO::G_SMULFIXSAT, GO::G_UMULFIXSAT}, + InstructionFlag::Commutable); + + Registry.configure({GO::G_FNEG, GO::G_FABS, GO::G_FCANONICALIZE, GO::G_FEXP, GO::G_FEXP2, + GO::G_FEXP10, GO::G_FLOG, GO::G_FLOG2, GO::G_FLOG10, GO::G_FCEIL, + GO::G_FCOS, GO::G_FSIN, GO::G_FTAN, GO::G_FACOS, GO::G_FASIN, + GO::G_FATAN, GO::G_FCOSH, GO::G_FSINH, GO::G_FTANH, GO::G_FSQRT, + GO::G_FFLOOR, GO::G_FRINT, GO::G_FNEARBYINT}, + {FloatDef0, FloatUse0}, 1); + Registry.configure({GO::G_FPEXT, GO::G_FPTRUNC}, {FloatDef0, FloatUse1}, 1); + Registry.configure({GO::G_FPTOSI, GO::G_FPTOUI, GO::G_FPTOSI_SAT, GO::G_FPTOUI_SAT}, + {IntDef0, FloatUse1}, 1); + Registry.configure({GO::G_SITOFP, GO::G_UITOFP}, {FloatDef0, IntUse1}, 1); + Registry.configure({GO::G_FCOPYSIGN}, {FloatDef0, FloatUse0, FloatUse1}, 1); + Registry.configure({GO::G_IS_FPCLASS}, {IntDef0, FloatUse1, immediate()}, 1); + Registry.configure({GO::G_FADD, GO::G_FSUB, GO::G_FMUL, GO::G_FDIV, GO::G_FREM, GO::G_FPOW, + GO::G_FATAN2, GO::G_FMINNUM, GO::G_FMAXNUM, GO::G_FMINNUM_IEEE, + GO::G_FMAXNUM_IEEE, GO::G_FMINIMUM, GO::G_FMAXIMUM, GO::G_FMINIMUMNUM, + GO::G_FMAXIMUMNUM}, + {FloatDef0, FloatUse0, FloatUse0}, 1); + Registry.addFlag({GO::G_FADD, GO::G_FMUL, GO::G_FMINNUM, GO::G_FMAXNUM, GO::G_FMINNUM_IEEE, + GO::G_FMAXNUM_IEEE, GO::G_FMINIMUM, GO::G_FMAXIMUM, GO::G_FMINIMUMNUM, + GO::G_FMAXIMUMNUM}, + InstructionFlag::Commutable); + Registry.configure({GO::G_FMA, GO::G_FMAD}, {FloatDef0, FloatUse0, FloatUse0, FloatUse0}, 1); + Registry.configure({GO::G_FMODF, GO::G_FSINCOS}, {FloatDef0, FloatDef0, FloatUse0}, 2); + Registry.configure({GO::G_FPOWI, GO::G_FLDEXP}, {FloatDef0, FloatUse0, IntUse1}, 1); + Registry.configure({GO::G_FFREXP}, {FloatDef0, IntDef1, FloatUse0}, 2); + + Registry.configure({GO::G_GET_FPENV, GO::G_GET_FPMODE}, {AnyDef0}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_SET_FPENV, GO::G_SET_FPMODE}, {AnyUse0}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_RESET_FPENV, GO::G_RESET_FPMODE}, {}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_GET_ROUNDING}, {IntDef0}, 1, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_SET_ROUNDING}, {IntUse0}, 0, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_PTR_ADD}, {PointerDef0, PointerUse0, IntUse1}, 1); + Registry.configure({GO::G_PTRMASK}, {PointerDef0, PointerUse0, IntUse1}, 1); + Registry.configure({GO::G_ABS}, {IntDef0, IntUse0}, 1); + Registry.configure({GO::G_LROUND, GO::G_LLROUND}, {IntDef0, FloatUse1}, 1); + Registry.configure( + {GO::G_BR}, {kind(MachineOperand::Kind::BasicBlock)}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, InstructionFlag::Barrier}); + Registry.configure({GO::G_BRJT}, + {ScalarPointerUse0, kind(MachineOperand::Kind::JumpTableIndex), ScalarIntUse1}, + 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, + InstructionFlag::IndirectBranch, InstructionFlag::Barrier}); + Registry.configure({GO::G_VSCALE}, {ScalarIntDef0, immediate()}, 1); + + Registry.configure({GO::G_INSERT_SUBVECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), immediate()}, + 1); + Registry.configure( + {GO::G_EXTRACT_SUBVECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), immediate()}, 1); + Registry.configure( + {GO::G_INSERT_VECTOR_ELT}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), AnyUse1, ScalarIntUse2}, + 1); + Registry.configure( + {GO::G_EXTRACT_VECTOR_ELT}, + {AnyDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), ScalarIntUse2}, 1); + Registry.configure({GO::G_SHUFFLE_VECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + kind(MachineOperand::Kind::ShuffleMask)}, + 1); + Registry.configure({GO::G_SPLAT_VECTOR}, {VectorDef0, AnyUse1}, 1); + Registry.configure({GO::G_STEP_VECTOR}, {VectorDef0, immediate()}, 1); + Registry.configure({GO::G_VECTOR_COMPRESS}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 0)}, + 1); + + Registry.configure({GO::G_CTTZ, GO::G_CTTZ_ZERO_POISON, GO::G_CTLZ, GO::G_CTLZ_ZERO_POISON, + GO::G_CTLS, GO::G_CTPOP, GO::G_BSWAP, GO::G_BITREVERSE}, + {IntDef0, IntUse0}, 1); + Registry.configure({GO::G_ADDRSPACE_CAST}, {PointerDef0, PointerUse1}, 1); + Registry.configure({GO::G_BLOCK_ADDR}, + {ScalarPointerDef0, kind(MachineOperand::Kind::BlockAddress)}, 1); + Registry.configure({GO::G_JUMP_TABLE}, + {ScalarPointerDef0, kind(MachineOperand::Kind::JumpTableIndex)}, 1); + Registry.configure({GO::G_DYN_STACKALLOC}, {ScalarPointerDef0, ScalarIntUse1, immediate()}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STACKSAVE}, {ScalarPointerDef0}, 1, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STACKRESTORE}, {ScalarPointerUse0}, 0, + {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_STRICT_FADD, GO::G_STRICT_FSUB, GO::G_STRICT_FMUL, GO::G_STRICT_FDIV, + GO::G_STRICT_FREM}, + {FloatDef0, FloatUse0, FloatUse0}, 1, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STRICT_FMA}, {FloatDef0, FloatUse0, FloatUse0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STRICT_FSQRT}, {FloatDef0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STRICT_FLDEXP}, {FloatDef0, FloatUse0, IntUse1}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STRICT_FCMP, GO::G_STRICT_FCMPS}, + {IntDef0, kind(MachineOperand::Kind::Predicate), FloatUse1, FloatUse1}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_READ_REGISTER}, {AnyDef0, kind(MachineOperand::Kind::Metadata)}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::Convergent}); + Registry.configure({GO::G_WRITE_REGISTER}, {kind(MachineOperand::Kind::Metadata), AnyUse0}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::Convergent}); + Registry.configure({GO::G_MEMCPY, GO::G_MEMMOVE}, + {ScalarPointerUse0, ScalarPointerUse1, ScalarIntUse2, immediate()}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_MEMCPY_INLINE}, {ScalarPointerUse0, ScalarPointerUse1, ScalarIntUse2}, + 0, {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_MEMSET}, {ScalarPointerUse0, ScalarIntUse1, ScalarIntUse2, immediate()}, + 0, {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_BZERO}, {ScalarPointerUse0, ScalarIntUse1, immediate()}, 0, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_TRAP}, {}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::MayStore}); + Registry.configure({GO::G_DEBUGTRAP}, {}, 0, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_UBSANTRAP}, {immediate()}, 0, {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_VECREDUCE_SEQ_FADD, GO::G_VECREDUCE_SEQ_FMUL}, + {ScalarFloatDef0, typedRegister(RegisterOperandRole::Use, ScalarFloats, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 2)}, + 1); + Registry.configure({GO::G_VECREDUCE_FADD, GO::G_VECREDUCE_FMUL, GO::G_VECREDUCE_FMAX, + GO::G_VECREDUCE_FMIN, GO::G_VECREDUCE_FMAXIMUM, GO::G_VECREDUCE_FMINIMUM}, + {ScalarFloatDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1)}, 1); + Registry.configure({GO::G_VECREDUCE_ADD, GO::G_VECREDUCE_MUL, GO::G_VECREDUCE_AND, + GO::G_VECREDUCE_OR, GO::G_VECREDUCE_XOR, GO::G_VECREDUCE_SMAX, + GO::G_VECREDUCE_SMIN, GO::G_VECREDUCE_UMAX, GO::G_VECREDUCE_UMIN}, + {ScalarIntDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1)}, 1); + Registry.configure({GO::G_SBFX, GO::G_UBFX}, + {ScalarIntDef0, ScalarIntUse0, ScalarIntUse1, ScalarIntUse1}, 1); +} + +[[nodiscard]] std::vector buildDescriptors() { + DescriptorRegistry Registry; + configureCoreOpcodes(Registry); + configureGenericOpcodes(Registry); + return Registry.take(); +} + +[[nodiscard]] const std::vector &descriptors() { + static const std::vector Value = buildDescriptors(); + return Value; +} + +} // namespace + +std::string_view getGenericOpcodeName(GenericOpcode OpcodeValue) noexcept { + const Opcode Value = opcode(OpcodeValue); + const auto Position = + std::ranges::find_if(descriptors(), [Value](const InstructionDescriptor &Descriptor) { + return Descriptor.InstructionOpcode == Value; + }); + return Position == descriptors().end() ? std::string_view{} : std::string_view(Position->Name); +} + +std::span genericInstructionDescriptors() noexcept { + return descriptors(); +} + +std::span genericImmediatePredicates() noexcept { + static const std::vector Predicates; + return Predicates; +} + +Expected appendGenericInstructionInfo(std::vector &Descriptors, + std::vector &Predicates) { + const auto ExistingOpcode = [&Descriptors](Opcode Value) { + return std::ranges::any_of(Descriptors, [Value](const InstructionDescriptor &Descriptor) { + return Descriptor.InstructionOpcode == Value; + }); + }; + for (const InstructionDescriptor &Descriptor : descriptors()) { + if (ExistingOpcode(Descriptor.InstructionOpcode)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target instruction set already contains a target-independent opcode"); + } + Descriptors.push_back(Descriptor); + } + for (const ImmediatePredicate &Predicate : genericImmediatePredicates()) + Predicates.push_back(Predicate); + return {}; +} + +} // namespace rcc -- cgit v1.2.3