diff options
Diffstat (limited to 'backend/lib')
19 files changed, 651 insertions, 766 deletions
diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp index 0cfd83f..f3ff5f1 100644 --- a/backend/lib/CodeGen/GenericOpcodes.cpp +++ b/backend/lib/CodeGen/GenericOpcodes.cpp @@ -33,52 +33,50 @@ using GO = GenericOpcode; [[nodiscard]] Expected<void> verifyGenericInstructionSemantics(const MachineInstr &Instruction, const MachineRegisterInfo &RegInfo); -[[nodiscard]] LLTKindSet anyValueKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Float, - LLT::Kind::Pointer, LLT::Kind::VectorAny, LLT::Kind::VectorInteger, - LLT::Kind::VectorFloat, LLT::Kind::VectorPointer}; +[[nodiscard]] LowLevelTypeKindSet anyValueKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer, + LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; } -[[nodiscard]] LLTKindSet integerKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::VectorAny, LLT::Kind::VectorInteger}; +[[nodiscard]] LowLevelTypeKindSet integerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Vector}; } -[[nodiscard]] LLTKindSet integerOrPointerKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Pointer, - LLT::Kind::VectorAny, LLT::Kind::VectorInteger, LLT::Kind::VectorPointer}; +[[nodiscard]] LowLevelTypeKindSet integerOrPointerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer, + LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; } -[[nodiscard]] LLTKindSet scalarIntegerKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer}; +[[nodiscard]] LowLevelTypeKindSet scalarIntegerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar}; } -[[nodiscard]] LLTKindSet scalarIntegerOrPointerKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Pointer}; +[[nodiscard]] LowLevelTypeKindSet scalarIntegerOrPointerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer}; } -[[nodiscard]] LLTKindSet scalarValueKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Integer, LLT::Kind::Float, LLT::Kind::Pointer}; +[[nodiscard]] LowLevelTypeKindSet scalarValueKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer}; } -[[nodiscard]] LLTKindSet floatingKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Float, LLT::Kind::VectorAny, LLT::Kind::VectorFloat}; +[[nodiscard]] LowLevelTypeKindSet floatingKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Vector}; } -[[nodiscard]] LLTKindSet scalarFloatingKinds() { - return {LLT::Kind::AnyScalar, LLT::Kind::Float}; +[[nodiscard]] LowLevelTypeKindSet scalarFloatingKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar}; } -[[nodiscard]] LLTKindSet pointerKinds() { - return {LLT::Kind::Pointer, LLT::Kind::VectorPointer}; +[[nodiscard]] LowLevelTypeKindSet pointerKinds() { + return {LowLevelType::Kind::Pointer, LowLevelType::Kind::VectorPointer}; } -[[nodiscard]] LLTKindSet scalarPointerKinds() { - return {LLT::Kind::Pointer}; +[[nodiscard]] LowLevelTypeKindSet scalarPointerKinds() { + return {LowLevelType::Kind::Pointer}; } -[[nodiscard]] LLTKindSet vectorKinds() { - return {LLT::Kind::VectorAny, LLT::Kind::VectorInteger, LLT::Kind::VectorFloat, - LLT::Kind::VectorPointer}; +[[nodiscard]] LowLevelTypeKindSet vectorKinds() { + return {LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; } [[nodiscard]] OperandKindSet allOperandKinds() { @@ -106,7 +104,7 @@ using GO = GenericOpcode; MachineOperand::Kind::LaneMask}; } -[[nodiscard]] OperandDescriptor typedRegister(RegisterOperandRole Role, LLTKindSet Kinds, +[[nodiscard]] OperandDescriptor typedRegister(RegisterOperandRole Role, LowLevelTypeKindSet Kinds, std::uint16_t Group) { OperandDescriptor Result; Result.Kinds = {MachineOperand::Kind::Register}; @@ -263,22 +261,22 @@ private: return *Value; } -[[nodiscard]] std::uint64_t minimumBitWidth(LLT Type) noexcept { +[[nodiscard]] std::uint64_t minimumBitWidth(LowLevelType Type) noexcept { const std::uint64_t Elements = Type.isVector() ? Type.minimumElementCount() : 1; return Elements * Type.scalarBitWidth(); } -[[nodiscard]] std::uint64_t minimumStoreBytes(LLT Type) noexcept { +[[nodiscard]] std::uint64_t minimumStoreBytes(LowLevelType Type) noexcept { return (minimumBitWidth(Type) + 7) / 8; } -[[nodiscard]] LLT scalarElementType(LLT Type) noexcept { +[[nodiscard]] LowLevelType scalarElementType(LowLevelType Type) noexcept { if (const auto Element = Type.scalarElementType()) return *Element; return Type; } -[[nodiscard]] bool hasSameShape(LLT Left, LLT Right) noexcept { +[[nodiscard]] bool hasSameShape(LowLevelType Left, LowLevelType Right) noexcept { if (Left.isVector() != Right.isVector()) return false; if (!Left.isVector()) @@ -287,18 +285,18 @@ private: Left.minimumElementCount() == Right.minimumElementCount(); } -[[nodiscard]] Expected<void> requireSameShape(LLT Left, LLT Right, std::string_view Message) { +[[nodiscard]] Expected<void> requireSameShape(LowLevelType Left, LowLevelType Right, + std::string_view Message) { if (!hasSameShape(Left, Right)) return semanticError(Message); return {}; } -[[nodiscard]] Expected<void> requireI1WithShape(LLT Condition, LLT Value, +[[nodiscard]] Expected<void> requireI1WithShape(LowLevelType Condition, LowLevelType Value, bool AllowScalarCondition) { - const bool IsScalarInteger = Condition.isAnyScalar() || Condition.isInteger(); - const bool IsVectorInteger = Condition.isAnyVector() || Condition.isIntegerVector(); - if ((!IsScalarInteger && !IsVectorInteger) || Condition.scalarBitWidth() == 0) - return semanticError("generic condition or status result must use an integer LLT"); + if ((!Condition.isCategoryNeutralScalar() && !Condition.isVectorValue()) || + Condition.scalarBitWidth() == 0) + return semanticError("generic condition or status result must use an integer LowLevelType"); if (AllowScalarCondition && !Condition.isVector()) return {}; return requireSameShape(Condition, Value, @@ -308,7 +306,7 @@ private: [[nodiscard]] Expected<void> verifyGenericExplicitOperands(Opcode InstructionOpcode, std::span<const MachineOperand> Operands, const MachineRegisterInfo &RegInfo) { - std::vector<std::optional<LLT>> Types(Operands.size()); + std::vector<std::optional<LowLevelType>> Types(Operands.size()); bool SawNonDefinition = false; for (std::size_t Index = 0; Index < Operands.size(); ++Index) { const MachineOperand &Operand = Operands[Index]; @@ -334,18 +332,18 @@ private: auto Info = RegInfo.virtualRegisterInfo(*Reg); if (!Info) return std::unexpected(Info.error()); - const std::optional<LLT> &Type = Info->get().Type; + const std::optional<LowLevelType> &Type = Info->get().Type; if (!Type) - return semanticError("generic virtual register is missing its LLT"); + return semanticError("generic virtual register is missing its LowLevelType"); Types[Index] = Type.value(); } - const auto TypeAt = [&Types](std::size_t Index) -> LLT { return *Types[Index]; }; + const auto TypeAt = [&Types](std::size_t Index) -> LowLevelType { return *Types[Index]; }; const GO Generic = static_cast<GO>(InstructionOpcode.value()); const auto VerifyWidthChange = [&](bool IsExtension) -> Expected<void> { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (auto Shape = requireSameShape(Dst, Src, "generic width conversion must preserve vector shape"); !Shape) { @@ -389,7 +387,7 @@ private: case GO::G_CONSTANT: { if (const auto *Payload = Operands[1].getIf<IntegerImmediateOperand>(); Payload != nullptr && Payload->Value.bitWidth() != TypeAt(0).scalarBitWidth()) { - return semanticError("G_CONSTANT integer width must match its destination LLT"); + return semanticError("G_CONSTANT integer width must match its destination LowLevelType"); } return {}; } @@ -397,17 +395,14 @@ private: const auto *Payload = Operands[1].getIf<FloatingImmediateOperand>(); if (Payload == nullptr) return semanticError("G_FCONSTANT requires a floating-point immediate"); - const LLT DestinationType = TypeAt(0); - const std::optional<FloatSemantics> DestinationSemantics = DestinationType.floatSemantics(); - if ((DestinationSemantics && Payload->Bits.semantics() != *DestinationSemantics) || - (!DestinationSemantics && Payload->Bits.bitWidth() != DestinationType.scalarBitWidth())) { - return semanticError("G_FCONSTANT semantics must match its destination LLT"); - } + const LowLevelType DestinationType = TypeAt(0); + if (Payload->Bits.bitWidth() != DestinationType.scalarBitWidth()) + return semanticError("G_FCONSTANT width must match its destination LowLevelType"); return {}; } case GO::G_BITCAST: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (Dst.isPointerOrPointerVector() != Src.isPointerOrPointerVector()) return semanticError("G_BITCAST cannot cross the pointer/non-pointer boundary"); if (minimumBitWidth(Dst) != minimumBitWidth(Src) || @@ -415,7 +410,7 @@ private: return semanticError("G_BITCAST source and destination sizes must match"); } if (Dst == Src) - return semanticError("G_BITCAST must change the LLT"); + return semanticError("G_BITCAST must change the LowLevelType"); return {}; } case GO::G_INTTOPTR: @@ -423,8 +418,8 @@ private: return requireSameShape(TypeAt(0), TypeAt(1), "pointer/integer conversion must preserve vector shape"); case GO::G_ADDRSPACE_CAST: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (auto Shape = requireSameShape(Dst, Src, "G_ADDRSPACE_CAST must preserve vector shape"); !Shape) { return Shape; @@ -527,8 +522,8 @@ private: } case GO::G_UNMERGE_VALUES: { const std::size_t DestinationCount = Operands.size() - 1; - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(DestinationCount); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(DestinationCount); if (minimumBitWidth(Src) != minimumBitWidth(Dst) * DestinationCount) return semanticError("G_UNMERGE_VALUES destination widths do not cover the source"); if (Dst.isVector() && (!Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || @@ -559,22 +554,23 @@ private: } case GO::G_BUILD_VECTOR: case GO::G_BUILD_VECTOR_TRUNC: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (!Dst.isVector() || Dst.isScalableVector() || Src.isVector()) return semanticError("G_BUILD_VECTOR requires a fixed vector result and scalar sources"); if (Dst.minimumElementCount() != Operands.size() - 1) return semanticError("G_BUILD_VECTOR source count must equal the result lane count"); - const LLT Element = scalarElementType(Dst); + const LowLevelType Element = scalarElementType(Dst); if (Generic == GO::G_BUILD_VECTOR && Element != Src) - return semanticError("G_BUILD_VECTOR source LLT must equal the result element LLT"); + return semanticError( + "G_BUILD_VECTOR source LowLevelType must equal the result element LowLevelType"); if (Generic == GO::G_BUILD_VECTOR_TRUNC && Src.scalarBitWidth() <= Element.scalarBitWidth()) return semanticError("G_BUILD_VECTOR_TRUNC sources must be wider than result elements"); return {}; } case GO::G_CONCAT_VECTORS: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || scalarElementType(Dst) != scalarElementType(Src) || Dst.minimumElementCount() != Src.minimumElementCount() * (Operands.size() - 1)) { @@ -604,8 +600,8 @@ private: case GO::G_INSERT_SUBVECTOR: case GO::G_EXTRACT_SUBVECTOR: { const bool IsInsert = Generic == GO::G_INSERT_SUBVECTOR; - const LLT Outer = TypeAt(IsInsert ? 0 : 1); - const LLT Inner = TypeAt(IsInsert ? 2 : 0); + const LowLevelType Outer = TypeAt(IsInsert ? 0 : 1); + const LowLevelType Inner = TypeAt(IsInsert ? 2 : 0); const auto Index = plainImmediate(Operands[IsInsert ? 3 : 2]); if (!Outer.isVector() || !Inner.isVector() || scalarElementType(Outer) != scalarElementType(Inner) || !Index || *Index < 0) { @@ -626,8 +622,8 @@ private: return {}; } case GO::G_SHUFFLE_VECTOR: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); const auto *Mask = Operands[3].getIf<ShuffleMaskOperand>(); if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() || Src.isScalableVector() || scalarElementType(Dst) != scalarElementType(Src) || Mask == nullptr || @@ -644,8 +640,8 @@ private: return {}; } case GO::G_SPLAT_VECTOR: { - const LLT Dst = TypeAt(0); - const LLT Src = TypeAt(1); + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); if (!Dst.isScalableVector() || Src.isVector() || scalarElementType(Dst).scalarBitWidth() > Src.scalarBitWidth()) { return semanticError("G_SPLAT_VECTOR requires a scalable result with no wider elements"); @@ -653,7 +649,7 @@ private: return {}; } case GO::G_STEP_VECTOR: { - const LLT Dst = TypeAt(0); + const LowLevelType Dst = TypeAt(0); const auto *Step = Operands[1].getIf<IntegerImmediateOperand>(); if (!Dst.isScalableVector() || Step == nullptr || Step->Value.isZero() || Step->Value.isNegative() || Step->Value.bitWidth() != Dst.scalarBitWidth()) { @@ -662,15 +658,15 @@ private: return {}; } case GO::G_INSERT_VECTOR_ELT: { - const LLT Vector = TypeAt(0); + const LowLevelType Vector = TypeAt(0); if (scalarElementType(Vector) != TypeAt(2) || TypeAt(3).isVector()) - return semanticError("G_INSERT_VECTOR_ELT element or index LLT is incompatible"); + return semanticError("G_INSERT_VECTOR_ELT element or index LowLevelType is incompatible"); return {}; } case GO::G_EXTRACT_VECTOR_ELT: { - const LLT Vector = TypeAt(1); + const LowLevelType Vector = TypeAt(1); if (scalarElementType(Vector) != TypeAt(0) || TypeAt(2).isVector()) - return semanticError("G_EXTRACT_VECTOR_ELT result or index LLT is incompatible"); + return semanticError("G_EXTRACT_VECTOR_ELT result or index LowLevelType is incompatible"); return {}; } case GO::G_VECTOR_COMPRESS: @@ -699,7 +695,7 @@ private: case GO::G_VECREDUCE_UMIN: if (TypeAt(0).isVector() || !TypeAt(1).isVector() || scalarElementType(TypeAt(1)) != TypeAt(0)) { - return semanticError("vector reduction result must equal the source element LLT"); + return semanticError("vector reduction result must equal the source element LowLevelType"); } return {}; case GO::G_SBFX: @@ -807,7 +803,7 @@ private: const auto MemoryOperands = Instruction.memOperands(); const GO Generic = static_cast<GO>(Instruction.opcode().value()); - const auto TypeAt = [&](std::size_t Index) -> Expected<LLT> { + const auto TypeAt = [&](std::size_t Index) -> Expected<LowLevelType> { if (Index >= Operands.size()) return makeUnexpected(ErrorCode::InternalError, "generic semantic verifier operand index is out of range"); @@ -820,7 +816,7 @@ private: return std::unexpected(Info.error()); if (!Info->get().Type) return makeUnexpected(ErrorCode::InternalError, - "generic semantic verifier found a register without an LLT"); + "generic semantic verifier found a register without an LowLevelType"); return *Info->get().Type; }; @@ -830,7 +826,8 @@ private: if (!Type) return std::unexpected(Type.error()); if (!Type->isPointer() || Type->addressSpace() != Memory.description().AddressSpace) - return semanticError("generic memory address LLT and MMO address space do not match"); + return semanticError( + "generic memory address LowLevelType and MMO address space do not match"); return {}; }; @@ -839,7 +836,7 @@ private: NoLarger, StrictlySmaller, }; - const auto VerifySize = [&](LLT ValueType, const MachineMemOperand &Memory, + const auto VerifySize = [&](LowLevelType ValueType, const MachineMemOperand &Memory, SizeRelation Relation) -> Expected<void> { const MemorySize Size = Memory.description().Size; if (!Size.isKnown()) @@ -853,7 +850,8 @@ private: : (Relation == SizeRelation::NoLarger ? MemoryBytes <= ValueBytes : MemoryBytes < ValueBytes); if (!Valid) - return semanticError("generic memory access size is incompatible with its value LLT"); + return semanticError( + "generic memory access size is incompatible with its value LowLevelType"); return {}; }; @@ -1121,17 +1119,17 @@ void configureCoreOpcodes(DescriptorRegistry &Registry) { } void configureGenericOpcodes(DescriptorRegistry &Registry) { - const LLTKindSet AnyTypes = anyValueKinds(); - const LLTKindSet IntTypes = integerKinds(); - const LLTKindSet IntegerOrPointers = integerOrPointerKinds(); - const LLTKindSet ScalarValues = scalarValueKinds(); - const LLTKindSet ScalarInts = scalarIntegerKinds(); - const LLTKindSet ScalarIntegerOrPointers = scalarIntegerOrPointerKinds(); - const LLTKindSet FloatTypes = floatingKinds(); - const LLTKindSet ScalarFloats = scalarFloatingKinds(); - const LLTKindSet PointerTypes = pointerKinds(); - const LLTKindSet ScalarPointers = scalarPointerKinds(); - const LLTKindSet VectorTypes = vectorKinds(); + const LowLevelTypeKindSet AnyTypes = anyValueKinds(); + const LowLevelTypeKindSet IntTypes = integerKinds(); + const LowLevelTypeKindSet IntegerOrPointers = integerOrPointerKinds(); + const LowLevelTypeKindSet ScalarValues = scalarValueKinds(); + const LowLevelTypeKindSet ScalarInts = scalarIntegerKinds(); + const LowLevelTypeKindSet ScalarIntegerOrPointers = scalarIntegerOrPointerKinds(); + const LowLevelTypeKindSet FloatTypes = floatingKinds(); + const LowLevelTypeKindSet ScalarFloats = scalarFloatingKinds(); + const LowLevelTypeKindSet PointerTypes = pointerKinds(); + const LowLevelTypeKindSet ScalarPointers = scalarPointerKinds(); + const LowLevelTypeKindSet VectorTypes = vectorKinds(); const auto AnyDef0 = typedRegister(RegisterOperandRole::Def, AnyTypes, 0); const auto AnyUse0 = typedRegister(RegisterOperandRole::Use, AnyTypes, 0); @@ -1414,16 +1412,17 @@ void configureGenericOpcodes(DescriptorRegistry &Registry) { Registry.configure({GO::G_STRICT_FADD, GO::G_STRICT_FSUB, GO::G_STRICT_FMUL, GO::G_STRICT_FDIV, GO::G_STRICT_FREM}, - {FloatDef0, FloatUse0, FloatUse0}, 1, {InstructionFlag::HasSideEffects}); + {FloatDef0, FloatUse0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); Registry.configure({GO::G_STRICT_FMA}, {FloatDef0, FloatUse0, FloatUse0, FloatUse0}, 1, - {InstructionFlag::HasSideEffects}); + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); Registry.configure({GO::G_STRICT_FSQRT}, {FloatDef0, FloatUse0}, 1, - {InstructionFlag::HasSideEffects}); + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); Registry.configure({GO::G_STRICT_FLDEXP}, {FloatDef0, FloatUse0, IntUse1}, 1, - {InstructionFlag::HasSideEffects}); + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); Registry.configure({GO::G_STRICT_FCMP, GO::G_STRICT_FCMPS}, {IntDef0, kind(MachineOperand::Kind::Predicate), FloatUse1, FloatUse1}, 1, - {InstructionFlag::HasSideEffects}); + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); Registry.configure({GO::G_READ_REGISTER}, {AnyDef0, kind(MachineOperand::Kind::Metadata)}, 1, {InstructionFlag::HasSideEffects, InstructionFlag::Convergent}); Registry.configure({GO::G_WRITE_REGISTER}, {kind(MachineOperand::Kind::Metadata), AnyUse0}, 0, diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index 2c99939..d164cde 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -80,7 +80,7 @@ namespace { } if (Operand.TypeConstraint) { if (!isRegisterOnly(Operand.Kinds) || Operand.TypeConstraint->Kinds.empty() || - Operand.TypeConstraint->Kinds.contains(LLT::Kind::Invalid)) { + Operand.TypeConstraint->Kinds.contains(LowLevelType::Kind::Invalid)) { return makeUnexpected(ErrorCode::InvalidArgument, "typed operand descriptor requires a nonempty register type set"); } @@ -307,8 +307,8 @@ OperandKindSet::OperandKindSet(std::initializer_list<MachineOperand::Kind> Kinds add(Kind); } -LLTKindSet::LLTKindSet(std::initializer_list<LLT::Kind> Kinds) noexcept { - for (const LLT::Kind Kind : Kinds) +LowLevelTypeKindSet::LowLevelTypeKindSet(std::initializer_list<LowLevelType::Kind> Kinds) noexcept { + for (const LowLevelType::Kind Kind : Kinds) add(Kind); } @@ -386,7 +386,7 @@ Expected<void> InstructionSet::validateExplicitOperands(const InstructionDescrip return std::unexpected(Shape.error()); bool SawUse = false; - std::unordered_map<std::uint16_t, LLT> EqualityGroups; + std::unordered_map<std::uint16_t, LowLevelType> EqualityGroups; for (std::size_t Index = 0; Index < Operands.size(); ++Index) { const MachineOperand &Operand = Operands[Index]; if (Operand.isImplicit()) { @@ -423,22 +423,24 @@ Expected<void> InstructionSet::validateExplicitOperands(const InstructionDescrip auto Info = RegInfo.virtualRegisterInfo(*Reg); if (!Info) return std::unexpected(Info.error()); - const std::optional<LLT> &RegisterType = Info->get().Type; + const std::optional<LowLevelType> &RegisterType = Info->get().Type; if (!RegisterType) { return makeUnexpected(ErrorCode::VerificationError, - "typed instruction operand virtual register has no LLT"); + "typed instruction operand virtual register has no LowLevelType"); } - const LLT Type = RegisterType.value(); + const LowLevelType Type = RegisterType.value(); if (!OperandDesc.TypeConstraint->Kinds.contains(Type.kind())) { - return makeUnexpected(ErrorCode::VerificationError, - "instruction operand LLT kind is not permitted by its descriptor"); + return makeUnexpected( + ErrorCode::VerificationError, + "instruction operand LowLevelType kind is not permitted by its descriptor"); } if (OperandDesc.TypeConstraint->EqualityGroup) { const auto [Position, Inserted] = EqualityGroups.emplace(*OperandDesc.TypeConstraint->EqualityGroup, Type); if (!Inserted && Position->second != Type) { - return makeUnexpected(ErrorCode::VerificationError, - "instruction operands in one LLT group have different types"); + return makeUnexpected( + ErrorCode::VerificationError, + "instruction operands in one LowLevelType group have different types"); } } } diff --git a/backend/lib/CodeGen/LowLevelType.cpp b/backend/lib/CodeGen/LowLevelType.cpp index b50ea6b..50ee0e6 100644 --- a/backend/lib/CodeGen/LowLevelType.cpp +++ b/backend/lib/CodeGen/LowLevelType.cpp @@ -8,130 +8,99 @@ namespace rcc { namespace { -[[nodiscard]] LLT::Kind getVectorKind(LLT ElementType) noexcept { +[[nodiscard]] LowLevelType::Kind getVectorKind(LowLevelType ElementType) noexcept { switch (ElementType.kind()) { - case LLT::Kind::AnyScalar: - return LLT::Kind::VectorAny; - case LLT::Kind::Integer: - return LLT::Kind::VectorInteger; - case LLT::Kind::Float: - return LLT::Kind::VectorFloat; - case LLT::Kind::Pointer: - return LLT::Kind::VectorPointer; - case LLT::Kind::Invalid: - case LLT::Kind::Token: - case LLT::Kind::VectorAny: - case LLT::Kind::VectorInteger: - case LLT::Kind::VectorFloat: - case LLT::Kind::VectorPointer: - return LLT::Kind::Invalid; + case LowLevelType::Kind::CategoryNeutralScalar: + return LowLevelType::Kind::Vector; + case LowLevelType::Kind::Pointer: + return LowLevelType::Kind::VectorPointer; + case LowLevelType::Kind::Invalid: + case LowLevelType::Kind::Token: + case LowLevelType::Kind::Vector: + case LowLevelType::Kind::VectorPointer: + return LowLevelType::Kind::Invalid; } - return LLT::Kind::Invalid; + return LowLevelType::Kind::Invalid; } } // namespace -Expected<LLT> LLT::makeScalar(Kind TypeKind, std::uint64_t BitWidth) { +Expected<LowLevelType> LowLevelType::makeCategoryNeutralScalar(std::uint64_t BitWidth) { if (BitWidth == 0 || BitWidth > MaxBitWidth) { return makeUnexpected(ErrorCode::InvalidArgument, - "LLT bit width must be in the range [1, 65535]"); + "LowLevelType bit width must be in the range [1, 65535]"); } - return LLT(encode(TypeKind, static_cast<std::uint32_t>(BitWidth), 0, 0, false)); + return LowLevelType( + encode(Kind::CategoryNeutralScalar, static_cast<std::uint32_t>(BitWidth), 0, 0, false)); } -Expected<LLT> LLT::makeAnyScalar(std::uint64_t BitWidth) { - return makeScalar(Kind::AnyScalar, BitWidth); -} - -Expected<LLT> LLT::makeInteger(std::uint64_t BitWidth) { - return makeScalar(Kind::Integer, BitWidth); -} - -Expected<LLT> LLT::makeFloat(FloatSemantics Semantics) { - const auto BitWidth = getFloatSemanticsBitWidth(Semantics); - if (!BitWidth) { - return makeUnexpected(ErrorCode::InvalidArgument, "unknown floating-point semantics"); - } - - return LLT(encode(Kind::Float, *BitWidth, 0, static_cast<std::uint32_t>(Semantics), false)); -} - -Expected<LLT> LLT::makePointer(std::uint64_t AddressSpace, std::uint64_t BitWidth) { +Expected<LowLevelType> LowLevelType::makePointer(std::uint64_t AddressSpace, + std::uint64_t BitWidth) { if (BitWidth == 0 || BitWidth > MaxBitWidth) { return makeUnexpected(ErrorCode::InvalidArgument, - "LLT pointer bit width must be in the range [1, 65535]"); + "LowLevelType pointer bit width must be in the range [1, 65535]"); } if (AddressSpace > MaxAddressSpace) { - return makeUnexpected(ErrorCode::InvalidArgument, "LLT address space must fit in 24 bits"); + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType address space must fit in 24 bits"); } - return LLT(encode(Kind::Pointer, static_cast<std::uint32_t>(BitWidth), 0, - static_cast<std::uint32_t>(AddressSpace), false)); + return LowLevelType(encode(Kind::Pointer, static_cast<std::uint32_t>(BitWidth), 0, + static_cast<std::uint32_t>(AddressSpace), false)); } -Expected<LLT> LLT::makeFixedVector(std::uint64_t ElementCount, LLT ElementType) { +Expected<LowLevelType> LowLevelType::makeFixedVector(std::uint64_t ElementCount, + LowLevelType ElementType) { return makeVector(ElementCount, false, ElementType); } -Expected<LLT> LLT::makeScalableVector(std::uint64_t MinimumElementCount, LLT ElementType) { +Expected<LowLevelType> LowLevelType::makeScalableVector(std::uint64_t MinimumElementCount, + LowLevelType ElementType) { return makeVector(MinimumElementCount, true, ElementType); } -Expected<LLT> LLT::makeVector(std::uint64_t ElementCount, bool IsScalable, LLT ElementType) { +Expected<LowLevelType> LowLevelType::makeVector(std::uint64_t ElementCount, bool IsScalable, + LowLevelType ElementType) { if (ElementCount == 0 || ElementCount > MaxElementCount) { return makeUnexpected(ErrorCode::InvalidArgument, - "LLT vector element count must be in the range [1, 65535]"); + "LowLevelType vector element count must be in the range [1, 65535]"); } if (!IsScalable && ElementCount == 1) { return makeUnexpected(ErrorCode::InvalidArgument, - "a fixed LLT vector must contain at least two elements"); + "a fixed LowLevelType vector must contain at least two elements"); } const Kind VectorKind = getVectorKind(ElementType); if (VectorKind == Kind::Invalid) { return makeUnexpected(ErrorCode::InvalidArgument, - "LLT vector elements must be non-vector value types"); + "LowLevelType vector elements must be non-vector value types"); } std::uint32_t Auxiliary = 0; if (ElementType.isPointer()) Auxiliary = *ElementType.addressSpace(); - else if (ElementType.isFloat()) - Auxiliary = static_cast<std::uint32_t>(*ElementType.floatSemantics()); - return LLT(encode(VectorKind, ElementType.scalarBitWidth(), - static_cast<std::uint32_t>(ElementCount), Auxiliary, IsScalable)); + return LowLevelType(encode(VectorKind, ElementType.scalarBitWidth(), + static_cast<std::uint32_t>(ElementCount), Auxiliary, IsScalable)); } -std::optional<std::uint32_t> LLT::addressSpace() const noexcept { +std::optional<std::uint32_t> LowLevelType::addressSpace() const noexcept { if (!isPointerOrPointerVector()) return std::nullopt; return auxiliaryValue(); } -std::optional<FloatSemantics> LLT::floatSemantics() const noexcept { - if (!isFloatOrFloatVector()) - return std::nullopt; - return static_cast<FloatSemantics>(auxiliaryValue()); -} - -std::optional<LLT> LLT::scalarElementType() const noexcept { +std::optional<LowLevelType> LowLevelType::scalarElementType() const noexcept { switch (kind()) { - case Kind::VectorAny: - return LLT(encode(Kind::AnyScalar, scalarBitWidth(), 0, 0, false)); - case Kind::VectorInteger: - return LLT(encode(Kind::Integer, scalarBitWidth(), 0, 0, false)); - case Kind::VectorFloat: - return LLT(encode(Kind::Float, scalarBitWidth(), 0, auxiliaryValue(), false)); + case Kind::Vector: + return LowLevelType(encode(Kind::CategoryNeutralScalar, scalarBitWidth(), 0, 0, false)); case Kind::VectorPointer: - return LLT(encode(Kind::Pointer, scalarBitWidth(), 0, auxiliaryValue(), false)); + return LowLevelType(encode(Kind::Pointer, scalarBitWidth(), 0, auxiliaryValue(), false)); case Kind::Invalid: case Kind::Token: - case Kind::AnyScalar: - case Kind::Integer: - case Kind::Float: + case Kind::CategoryNeutralScalar: case Kind::Pointer: return std::nullopt; } @@ -139,7 +108,7 @@ std::optional<LLT> LLT::scalarElementType() const noexcept { return std::nullopt; } -std::string LLT::toString() const { +std::string LowLevelType::toString() const { if (!isValid()) return "LLT_invalid"; if (isToken()) @@ -147,19 +116,10 @@ std::string LLT::toString() const { std::string ScalarSpelling; switch (kind()) { - case Kind::AnyScalar: - case Kind::VectorAny: + case Kind::CategoryNeutralScalar: + case Kind::Vector: ScalarSpelling = "s" + std::to_string(scalarBitWidth()); break; - case Kind::Integer: - case Kind::VectorInteger: - ScalarSpelling = "i" + std::to_string(scalarBitWidth()); - break; - case Kind::Float: - case Kind::VectorFloat: - ScalarSpelling = - std::string(getFloatSemanticsSpelling(static_cast<FloatSemantics>(auxiliaryValue()))); - break; case Kind::Pointer: case Kind::VectorPointer: ScalarSpelling = diff --git a/backend/lib/CodeGen/MIRParser/MIParser.cpp b/backend/lib/CodeGen/MIRParser/MIParser.cpp index 5932111..b21e713 100644 --- a/backend/lib/CodeGen/MIRParser/MIParser.cpp +++ b/backend/lib/CodeGen/MIRParser/MIParser.cpp @@ -159,7 +159,7 @@ namespace { return APInt::create(BitWidth, Words); } -[[nodiscard]] std::optional<std::uint64_t> minimumTypeSizeInBits(LLT Type) noexcept { +[[nodiscard]] std::optional<std::uint64_t> minimumTypeSizeInBits(LowLevelType Type) noexcept { if (!Type.isValid() || Type.isToken()) return std::nullopt; const std::uint64_t ScalarSize = Type.scalarBitWidth(); @@ -377,8 +377,8 @@ private: [[nodiscard]] bool parseSigned64(const MIToken &Token, std::int64_t &Value); [[nodiscard]] bool parseOptionalOffset(std::int64_t &Offset); [[nodiscard]] bool parseAlignment(Alignment &Result); - [[nodiscard]] bool parseLowLevelType(LLT &Result); - [[nodiscard]] bool parseScalarLowLevelType(const MIToken &Token, LLT &Result); + [[nodiscard]] bool parseLowLevelType(LowLevelType &Result); + [[nodiscard]] bool parseScalarLowLevelType(const MIToken &Token, LowLevelType &Result); [[nodiscard]] bool parseFirstPass(); [[nodiscard]] bool parseSecondPass(); @@ -553,30 +553,20 @@ bool MIParser::Impl::parseAlignment(Alignment &Result) { return false; } -bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Result) { +bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LowLevelType &Result) { if (TypeToken.kind() != MITokenKind::Identifier) return fail(TypeToken, "expected a scalar low-level type"); std::string_view Spelling = TypeToken.stringValue(); - enum class ScalarKind : std::uint8_t { - Any, - Integer, - Float, - BFloat, - Pointer, - }; - ScalarKind Kind = ScalarKind::Any; + enum class ScalarKind : std::uint8_t { CategoryNeutral, Pointer }; + ScalarKind Kind = ScalarKind::CategoryNeutral; std::string_view Digits; if (Spelling.starts_with("bf")) { - Kind = ScalarKind::BFloat; Digits = Spelling.substr(2); } else if (Spelling.starts_with('s')) { - Kind = ScalarKind::Any; Digits = Spelling.substr(1); } else if (Spelling.starts_with('i')) { - Kind = ScalarKind::Integer; Digits = Spelling.substr(1); } else if (Spelling.starts_with('f')) { - Kind = ScalarKind::Float; Digits = Spelling.substr(1); } else if (Spelling.starts_with('p')) { Kind = ScalarKind::Pointer; @@ -589,43 +579,20 @@ bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Resu if (!parseUnsignedDigits(Digits, Value)) return fail(TypeToken, "expected integers after 's'/'i'/'f'/'bf'/'p' type identifier"); - Expected<LLT> Created = makeUnexpected(ErrorCode::ParseError, "invalid low-level type"); + Expected<LowLevelType> Created = makeUnexpected(ErrorCode::ParseError, "invalid low-level type"); switch (Kind) { - case ScalarKind::Any: + case ScalarKind::CategoryNeutral: if (Value == 0) { - Result = LLT::token(); + Result = LowLevelType::token(); return false; } - Created = LLT::makeAnyScalar(Value); - break; - case ScalarKind::Integer: - Created = LLT::makeInteger(Value); - break; - case ScalarKind::Float: { - std::optional<FloatSemantics> Semantics; - if (Value == 16) - Semantics = FloatSemantics::IEEEHalf; - else if (Value == 32) - Semantics = FloatSemantics::IEEESingle; - else if (Value == 64) - Semantics = FloatSemantics::IEEEDouble; - else if (Value == 128) - Semantics = FloatSemantics::IEEEQuad; - if (!Semantics) - return fail(TypeToken, "unsupported IEEE floating low-level type width"); - Created = LLT::makeFloat(*Semantics); - break; - } - case ScalarKind::BFloat: - if (Value != 16) - return fail(TypeToken, "bfloat low-level type must have width 16"); - Created = LLT::makeFloat(FloatSemantics::BFloat); + Created = LowLevelType::makeCategoryNeutralScalar(Value); break; case ScalarKind::Pointer: if (!Context.PointerSizeInBits) return fail(TypeToken, "pointer low-level type requires an explicit pointer size"); if (auto PointerSize = Context.PointerSizeInBits(static_cast<std::uint32_t>(Value))) - Created = LLT::makePointer(Value, *PointerSize); + Created = LowLevelType::makePointer(Value, *PointerSize); else return importError(TypeToken, PointerSize.error()); break; @@ -636,7 +603,7 @@ bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Resu return false; } -bool MIParser::Impl::parseLowLevelType(LLT &Result) { +bool MIParser::Impl::parseLowLevelType(LowLevelType &Result) { const MIToken Begin = token(); const auto HasScalarPrefix = [](std::string_view Spelling) { return Spelling.starts_with('s') || Spelling.starts_with('i') || Spelling.starts_with('f') || @@ -685,7 +652,7 @@ bool MIParser::Impl::parseLowLevelType(LLT &Result) { if (!at(MITokenKind::Identifier) || !HasScalarPrefix(token().stringValue())) return VectorError(); - LLT ElementType; + LowLevelType ElementType; if (parseScalarLowLevelType(token(), ElementType)) return true; if (ElementType.isToken()) @@ -694,8 +661,9 @@ bool MIParser::Impl::parseLowLevelType(LLT &Result) { if (!consume(MITokenKind::Greater)) return VectorError(); - Expected<LLT> Created = Scalable ? LLT::makeScalableVector(ElementCount, ElementType) - : LLT::makeFixedVector(ElementCount, ElementType); + Expected<LowLevelType> Created = Scalable + ? LowLevelType::makeScalableVector(ElementCount, ElementType) + : LowLevelType::makeFixedVector(ElementCount, ElementType); if (!Created) return importError(Begin, Created.error()); Result = *Created; @@ -1078,7 +1046,7 @@ bool MIParser::Impl::parseRegisterOperand(bool IsDef, ParsedOperand &Result) { } else { if (!Reg.isVirtual()) return fail("low-level type is only valid on a virtual register"); - LLT Type; + LowLevelType Type; if (parseLowLevelType(Type)) return true; if (expect(MITokenKind::RightParen, "')' after the low-level type")) @@ -1883,7 +1851,7 @@ bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result, } else if (atKeyword("unknown-size")) { advance(); } else if (consume(MITokenKind::LeftParen)) { - LLT Type; + LowLevelType Type; if (parseLowLevelType(Type)) return true; if (expect(MITokenKind::RightParen, "')' after the memory low-level type")) @@ -1945,7 +1913,7 @@ bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result, } else if (atKeyword("addrspace")) { advance(); if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Description.AddressSpace) || - Description.AddressSpace > LLT::MaxAddressSpace) { + Description.AddressSpace > LowLevelType::MaxAddressSpace) { return fail("invalid machine memory operand address space"); } advance(); diff --git a/backend/lib/CodeGen/MIRPrinter.cpp b/backend/lib/CodeGen/MIRPrinter.cpp index d2cc8eb..1ba8ac1 100644 --- a/backend/lib/CodeGen/MIRPrinter.cpp +++ b/backend/lib/CodeGen/MIRPrinter.cpp @@ -1647,13 +1647,11 @@ namespace { return {}; } -[[nodiscard]] Expected<std::string> llvmMIRScalarLowLevelType(LLT Type) { +[[nodiscard]] Expected<std::string> llvmMIRScalarLowLevelType(LowLevelType Type) { if (Type.isToken()) return "s0"; - if (Type.isAnyScalar()) + if (Type.isCategoryNeutralScalar()) return "s" + std::to_string(Type.scalarBitWidth()); - if (Type.isInteger()) - return "i" + std::to_string(Type.scalarBitWidth()); if (Type.isPointer()) { const auto AddressSpace = Type.addressSpace(); if (!AddressSpace) { @@ -1662,49 +1660,11 @@ namespace { } return "p" + std::to_string(*AddressSpace); } - if (!Type.isFloat()) { - return makeUnexpected(ErrorCode::VerificationError, - "invalid scalar low-level type cannot be printed as LLVM MIR"); - } - - const auto Semantics = Type.floatSemantics(); - if (!Semantics) { - return makeUnexpected(ErrorCode::VerificationError, - "floating low-level type has no floating-point semantics"); - } - switch (*Semantics) { - case FloatSemantics::IEEEHalf: - return "f16"; - case FloatSemantics::BFloat: - return "bf16"; - case FloatSemantics::IEEESingle: - return "f32"; - case FloatSemantics::IEEEDouble: - return "f64"; - case FloatSemantics::IEEEQuad: - return "f128"; - case FloatSemantics::PPCDoubleDouble: - case FloatSemantics::Float8E5M2: - case FloatSemantics::Float8E5M2FNUZ: - case FloatSemantics::Float8E4M3: - case FloatSemantics::Float8E4M3FN: - case FloatSemantics::Float8E4M3FNUZ: - case FloatSemantics::Float8E4M3B11FNUZ: - case FloatSemantics::Float8E3M4: - case FloatSemantics::FloatTF32: - case FloatSemantics::Float8E8M0FNU: - case FloatSemantics::Float6E3M2FN: - case FloatSemantics::Float6E2M3FN: - case FloatSemantics::Float4E2M1FN: - case FloatSemantics::X87DoubleExtended: - case FloatSemantics::Count: - return makeUnexpected(ErrorCode::Unsupported, - "fixed LLVM MIR cannot spell this floating low-level type"); - } - return makeUnexpected(ErrorCode::InternalError, "floating low-level type is unhandled"); + return makeUnexpected(ErrorCode::VerificationError, + "invalid scalar low-level type cannot be printed as LLVM MIR"); } -[[nodiscard]] Expected<std::string> llvmMIRLowLevelType(LLT Type) { +[[nodiscard]] Expected<std::string> llvmMIRLowLevelType(LowLevelType Type) { if (!Type.isValid()) { return makeUnexpected(ErrorCode::VerificationError, "invalid low-level type cannot be printed as LLVM MIR"); diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp index 52a1484..30129bc 100644 --- a/backend/lib/CodeGen/MachineCSE.cpp +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -149,24 +149,7 @@ template <typename ValueT> return true; } -[[nodiscard]] bool hasFloatingRegister(const MachineInstr &Instruction, - const MachineRegisterInfo &Registers) { - for (const MachineOperand &Operand : Instruction.operands()) { - const Register *Reg = Operand.getIf<Register>(); - if (Reg == nullptr || !Reg->isVirtual()) - continue; - auto Info = Registers.virtualRegisterInfo(*Reg); - if (Info) { - const auto &Type = Info->get().Type; - if (Type && Type->isFloat()) - return true; - } - } - return false; -} - [[nodiscard]] Expected<bool> isCandidate(const MachineInstr &Instruction, - const MachineFunction &Function, const TargetDescription &Target) { const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); if (Descriptor == nullptr) { @@ -187,8 +170,8 @@ template <typename ValueT> return false; if (!Flags.has(InstructionFlag::MayLoad) && !Instruction.memOperands().empty()) return false; - if (!Instruction.flags().has(MachineInstrFlag::NoFPExcept) && - hasFloatingRegister(Instruction, Function.registerInfo())) { + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { return false; } @@ -351,7 +334,7 @@ MachineCSEPass::runImplementation(MachineFunction &Function, const TargetDescrip Block = Block->next()) { for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { MachineInstr *Next = Instruction->next(); - auto Candidate = isCandidate(*Instruction, Function, Target); + auto Candidate = isCandidate(*Instruction, Target); if (!Candidate) return std::unexpected(Candidate.error()); if (!*Candidate) { diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp index 8a34d3b..627e59f 100644 --- a/backend/lib/CodeGen/MachineLICM.cpp +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -113,6 +113,9 @@ namespace { hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { return false; } + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) + return false; bool HasDefinition = false; for (const MachineOperand &Operand : Instruction.operands()) { @@ -141,14 +144,6 @@ namespace { auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info) return std::unexpected(Info.error()); - const auto &Type = Info->get().Type; - if (Type && Type->isFloat() && !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { - return false; - } - } - if (Stage == MachineCodeMotionStage::PostRegisterAllocation && - !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { - return false; } return HasDefinition; } diff --git a/backend/lib/CodeGen/MachineMemOperand.cpp b/backend/lib/CodeGen/MachineMemOperand.cpp index cb90e61..0ae6941 100644 --- a/backend/lib/CodeGen/MachineMemOperand.cpp +++ b/backend/lib/CodeGen/MachineMemOperand.cpp @@ -257,7 +257,7 @@ Expected<MachineMemOperand> MachineMemOperand::create(MachineMemOperandDesc Desc return makeUnexpected(ErrorCode::InvalidArgument, "machine memory operand must describe a load or store"); } - if (Description.AddressSpace > LLT::MaxAddressSpace) { + if (Description.AddressSpace > LowLevelType::MaxAddressSpace) { return makeUnexpected(ErrorCode::InvalidArgument, "machine memory operand address space must fit in 24 bits"); } diff --git a/backend/lib/CodeGen/MachineRegisterInfo.cpp b/backend/lib/CodeGen/MachineRegisterInfo.cpp index ccd7d89..a88bfb0 100644 --- a/backend/lib/CodeGen/MachineRegisterInfo.cpp +++ b/backend/lib/CodeGen/MachineRegisterInfo.cpp @@ -68,9 +68,10 @@ Expected<Register> MachineRegisterInfo::createVirtualRegister() { return *Reg; } -Expected<Register> MachineRegisterInfo::createVirtualRegister(LLT Type) { +Expected<Register> MachineRegisterInfo::createVirtualRegister(LowLevelType Type) { if (!Type.isValid()) { - return makeUnexpected(ErrorCode::InvalidArgument, "virtual register type must be a valid LLT"); + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual register type must be a valid LowLevelType"); } auto Reg = createVirtualRegister(); @@ -124,9 +125,10 @@ MachineRegisterInfo::virtualRegisterInfo(Register Reg) const { return std::cref(VirtualRegisters[*Index]); } -Expected<void> MachineRegisterInfo::setType(Register Reg, LLT Type) { +Expected<void> MachineRegisterInfo::setType(Register Reg, LowLevelType Type) { if (!Type.isValid()) { - return makeUnexpected(ErrorCode::InvalidArgument, "virtual register type must be a valid LLT"); + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual register type must be a valid LowLevelType"); } auto Info = virtualRegisterInfo(Reg); if (!Info) diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp index 7c540e0..95f902d 100644 --- a/backend/lib/CodeGen/MachineSink.cpp +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -90,6 +90,9 @@ namespace { hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { return false; } + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) + return false; bool HasDefinition = false; for (const MachineOperand &Operand : Instruction.operands()) { @@ -118,14 +121,6 @@ namespace { auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info) return std::unexpected(Info.error()); - const auto &Type = Info->get().Type; - if (Type && Type->isFloat() && !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { - return false; - } - } - if (Stage == MachineCodeMotionStage::PostRegisterAllocation && - !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { - return false; } return HasDefinition; } diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp index 725a288..14ce8ef 100644 --- a/backend/lib/CodeGen/MachineVerifier.cpp +++ b/backend/lib/CodeGen/MachineVerifier.cpp @@ -849,7 +849,7 @@ private: MissingVirtualBank |= !Info.Bank; MissingVirtualClass |= !Info.RegClass; if (Info.Type && !Info.Type->isValid()) - return fail("virtual register has an invalid LLT"); + return fail("virtual register has an invalid LowLevelType"); std::unordered_set<PhysicalRegister> Hints; for (const PhysicalRegister Hint : Info.AllocationHints) { @@ -924,7 +924,7 @@ private: Has(MachineFunctionProperty::Legalized) || Has(MachineFunctionProperty::RegBankSelected); if (NeedsTypes && MissingVirtualType) - return fail("typed pipeline stage has a virtual register without LLT"); + return fail("typed pipeline stage has a virtual register without LowLevelType"); if (Options.Mode == MachineVerifierMode::PreLegalizer) { if (!Has(MachineFunctionProperty::IsSSA) || !Has(MachineFunctionProperty::TracksLiveness)) { diff --git a/backend/lib/CodeGen/TargetDescription.cpp b/backend/lib/CodeGen/TargetDescription.cpp index e8c7d79..b9569be 100644 --- a/backend/lib/CodeGen/TargetDescription.cpp +++ b/backend/lib/CodeGen/TargetDescription.cpp @@ -30,7 +30,7 @@ namespace { return Descriptor.Operands.back(); } -[[nodiscard]] std::uint64_t minimumTypeSizeInBits(LLT Type) noexcept { +[[nodiscard]] std::uint64_t minimumTypeSizeInBits(LowLevelType Type) noexcept { if (Type.isToken()) return 0; const std::uint64_t ScalarSize = Type.scalarBitWidth(); diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp index b4d513a..b0c22fc 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp @@ -54,25 +54,22 @@ constexpr std::uint32_t AllowedDirectCallFlags = return Value + Increment; } -[[nodiscard]] std::uint64_t valueSizeBytes(LLT Type) noexcept { +[[nodiscard]] std::uint64_t valueSizeBytes(LowLevelType Type) noexcept { return (static_cast<std::uint64_t>(Type.scalarBitWidth()) + 7U) / 8U; } -[[nodiscard]] bool isSupportedFloat(LLT Type) noexcept { - if (!Type.isFloat()) - return false; - const auto Semantics = Type.floatSemantics(); - return Semantics && - (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +[[nodiscard]] bool isSupportedFloat(const ABIValue &Value) noexcept { + return Value.Category == ABIValueCategory::FloatingPoint && + (Value.Type.scalarBitWidth() == 32 || Value.Type.scalarBitWidth() == 64); } [[nodiscard]] Expected<PhysicalRegister> floatingRegisterView(PhysicalRegister DoubleView, - LLT Type) { - if (!isSupportedFloat(Type)) { + const ABIValue &Value) { + if (!isSupportedFloat(Value)) { return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V floating-register view requires f32 or f64"); } - if (Type.scalarBitWidth() == 64) + if (Value.Type.scalarBitWidth() == 64) return DoubleView; constexpr std::uint32_t FirstDouble = static_cast<std::uint32_t>(PhysicalRegisterID::F0_D); @@ -87,7 +84,7 @@ constexpr std::uint32_t AllowedDirectCallFlags = } [[nodiscard]] bool requiresIntegerTransport(const ABIValue &Value) noexcept { - return !isSupportedFloat(Value.Type) || Value.Flags.has(ABIValueFlag::ByVal) || + return !isSupportedFloat(Value) || Value.Flags.has(ABIValueFlag::ByVal) || Value.Flags.has(ABIValueFlag::StructReturn) || Value.Flags.has(ABIValueFlag::Indirect); } @@ -101,14 +98,21 @@ constexpr std::uint32_t AllowedDirectCallFlags = ErrorCode::Unsupported, "RISC-V ABI accepts explicitly classified integer, pointer, f32, and f64 scalar fragments"); } - if (Value.Type.isFloat() && !isSupportedFloat(Value.Type)) { + if (Value.Category == ABIValueCategory::Invalid || Value.Category >= ABIValueCategory::Count) { + return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V ABI value category is invalid"); + } + if (Value.Category == ABIValueCategory::FloatingPoint && !isSupportedFloat(Value)) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V LP64 ABIs only support f32 and f64 floating fragments"); } + if ((Value.Category == ABIValueCategory::Pointer) != Value.Type.isPointer()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V ABI pointer category must match its pointer LowLevelType"); + } if (Value.Type.isPointer() && Value.Type.scalarBitWidth() != 64) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V LP64 ABIs require 64-bit pointers"); } - if ((Value.Type.isInteger() || Value.Type.isAnyScalar()) && Value.Type.scalarBitWidth() < 64) { + if (Value.Category == ABIValueCategory::Integer && Value.Type.scalarBitWidth() < 64) { if (Value.Extension == ABIExtensionKind::None) { return makeUnexpected(ErrorCode::InvalidArgument, "narrow RISC-V ABI integers require an explicit extension kind"); @@ -226,10 +230,9 @@ struct AssignmentState final { bool VariadicStackOnly = false; }; -[[nodiscard]] Expected<CCAssignment> registerAssignment(const ABIValue &Value, - std::size_t ValueIndex, - PhysicalRegister Location, LLT LocationType, - bool IsVariadic, bool IsBitcast) { +[[nodiscard]] Expected<CCAssignment> +registerAssignment(const ABIValue &Value, std::size_t ValueIndex, PhysicalRegister Location, + LowLevelType LocationType, bool IsVariadic, bool IsBitcast) { auto Align = alignment(std::min<std::uint64_t>(Value.OriginalAlignment.bytes(), ABIStackAlignmentBytes)); if (!Align) @@ -254,8 +257,8 @@ struct AssignmentState final { } [[nodiscard]] Expected<CCAssignment> stackAssignment(const ABIValue &Value, std::size_t ValueIndex, - std::uint64_t StackOffset, LLT LocationType, - bool IsVariadic, + std::uint64_t StackOffset, + LowLevelType LocationType, bool IsVariadic, std::uint64_t StackAlignment) { auto Align = alignment(StackAlignment); if (!Align) @@ -282,7 +285,7 @@ struct AssignmentState final { [[nodiscard]] Expected<ArgumentAssignmentResult> assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount, bool IsVariadic, const CallingConventionInfo &Convention) { - auto I64 = LLT::makeInteger(64); + auto I64 = LowLevelType::makeCategoryNeutralScalar(64); auto StackAlign = alignment(Convention.StackArgumentAlignmentBytes); if (!I64 || !StackAlign) return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid"); @@ -325,7 +328,7 @@ assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount false, false); } else { auto View = - floatingRegisterView(Convention.FloatingArgumentRegisters[State.FPR++], Value.Type); + floatingRegisterView(Convention.FloatingArgumentRegisters[State.FPR++], Value); if (!View) return std::unexpected(View.error()); Assignment = registerAssignment(Value, Index + Fragment, *View, Value.Type, false, false); @@ -356,7 +359,7 @@ assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount const bool HasGPR = !ForceEntireGroupToStack && State.GPR < Convention.IntegerArgumentRegisters.size(); if (HasGPR) { - const bool Bitcast = isSupportedFloat(Value.Type); + const bool Bitcast = isSupportedFloat(Value); auto Assignment = registerAssignment(Value, Index + Fragment, Convention.IntegerArgumentRegisters[State.GPR++], *I64, VariadicArgument, Bitcast); @@ -380,7 +383,7 @@ assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount if (!AlignedOffset) return std::unexpected(AlignedOffset.error()); State.StackOffset = *AlignedOffset; - const LLT LocationType = isSupportedFloat(Value.Type) ? Value.Type : *I64; + const LowLevelType LocationType = isSupportedFloat(Value) ? Value.Type : *I64; auto Assignment = stackAssignment(Value, Index + Fragment, State.StackOffset, LocationType, VariadicArgument, RequiredAlignment); if (!Assignment) @@ -415,7 +418,7 @@ assignReturns(std::span<const ABIValue> Values, bool RequireRegisters, return makeUnexpected(ErrorCode::Unsupported, "RISC-V direct returns are limited to two explicit fragments"); } - auto I64 = LLT::makeInteger(64); + auto I64 = LowLevelType::makeCategoryNeutralScalar(64); if (!I64) return std::unexpected(I64.error()); @@ -430,17 +433,17 @@ assignReturns(std::span<const ABIValue> Values, bool RequireRegisters, "return fragments cannot carry argument-only ABI flags"); } const bool UseFloatingRegister = - isSupportedFloat(Value.Type) && FPR < Convention.FloatingReturnRegisters.size(); + isSupportedFloat(Value) && FPR < Convention.FloatingReturnRegisters.size(); Expected<CCAssignment> Assignment = makeUnexpected( ErrorCode::Unsupported, "RISC-V calling convention has no return register available"); if (UseFloatingRegister) { - auto View = floatingRegisterView(Convention.FloatingReturnRegisters[FPR++], Value.Type); + auto View = floatingRegisterView(Convention.FloatingReturnRegisters[FPR++], Value); if (!View) return std::unexpected(View.error()); Assignment = registerAssignment(Value, Index, *View, Value.Type, false, false); } else if (GPR < Convention.IntegerReturnRegisters.size()) { Assignment = registerAssignment(Value, Index, Convention.IntegerReturnRegisters[GPR++], *I64, - false, isSupportedFloat(Value.Type)); + false, isSupportedFloat(Value)); } if (!Assignment) return std::unexpected(Assignment.error()); @@ -924,8 +927,9 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std:: if (!Info) return std::unexpected(Info.error()); if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) { - return makeUnexpected(ErrorCode::VerificationError, - "call-lowering plan register LLT differs from MachineRegisterInfo"); + return makeUnexpected( + ErrorCode::VerificationError, + "call-lowering plan register LowLevelType differs from MachineRegisterInfo"); } return {}; }; @@ -952,10 +956,12 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std:: auto Info = Registers.virtualRegisterInfo(CalleeReg); if (!Info) return std::unexpected(Info.error()); - const std::optional<LLT> Type = Info->get().Type; - if (!Type || ((!Type->isPointer() && !Type->isInteger()) || Type->scalarBitWidth() != 64)) { - return makeUnexpected(ErrorCode::VerificationError, - "indirect RISC-V callee requires a 64-bit pointer or integer LLT"); + const std::optional<LowLevelType> Type = Info->get().Type; + if (!Type || ((!Type->isPointer() && !Type->isCategoryNeutralScalar()) || + Type->scalarBitWidth() != 64)) { + return makeUnexpected( + ErrorCode::VerificationError, + "indirect RISC-V callee requires a 64-bit pointer or integer LowLevelType"); } } } diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index a7160b4..e4c937f 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -31,8 +31,8 @@ namespace rcc::riscv { namespace { -[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function, - const MachineOperand &Operand) { +[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function, + const MachineOperand &Operand) { const Register *Reg = Operand.getIf<Register>(); if (Reg == nullptr || !Reg->isVirtual()) return std::nullopt; @@ -42,43 +42,59 @@ namespace { return Info->get().Type; } -[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function, - const MachineInstr &Instruction) { +[[nodiscard]] std::optional<LowLevelType> resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { const MachineOperand *Result = Instruction.operandAt(0); return Result == nullptr ? std::nullopt : registerType(Function, *Result); } -[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function, - const MachineInstr &Instruction, std::size_t Index) { +[[nodiscard]] std::optional<LowLevelType> +operandType(const MachineFunction &Function, const MachineInstr &Instruction, std::size_t Index) { const MachineOperand *Operand = Instruction.operandAt(Index); return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); } -[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { - if (!Type.isFloat()) - return false; - const std::optional<FloatSemantics> Semantics = Type.floatSemantics(); - return Semantics && - (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && + (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept { + return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isSupportedIntegerScalar(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64; } -[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept { + return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } +[[nodiscard]] constexpr RegisterBankID fprBank() noexcept { + return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::FPR)); +} + +[[nodiscard]] Expected<RegisterBankID> registerBank(const MachineFunction &Function, Register Reg) { + if (!Reg.isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V generic instruction requires a virtual register"); + } + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Bank) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction selection requires an assigned register bank"); + } + return *Info->get().Bank; +} + [[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept { const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U)); const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1; @@ -146,9 +162,9 @@ public: const MachineFunction &Function, const TargetDescription &Target) const override { if (Instruction.opcode() == instructionOpcode(InstructionOpcode::G_SLLW)) { - const std::optional<LLT> Type = resultType(Function, Instruction); - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); return Type && Left && Right && isScalarGPRType(*Type) && Type->scalarBitWidth() == 64 && *Left == *Type && *Right == *Type ? Expected<void>{} @@ -158,9 +174,9 @@ public: if (!isGenericOpcode(Instruction.opcode())) return {}; const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); - const std::optional<LLT> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; - const auto Float = [Type] { return Type && isRVFloatType(*Type); }; + const auto Float = [Type] { return Type && isSupportedFloatingScalar(*Type); }; auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target); if (!Classification) return std::unexpected(Classification.error()); @@ -226,8 +242,7 @@ public: const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>(); - const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt; - if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + if (!Float() || Floating == nullptr || Floating->Bits.bitWidth() != Type->scalarBitWidth()) return makeUnexpected(ErrorCode::Unsupported, "RISC-V selector requires a matching floating constant"); return {}; @@ -240,7 +255,6 @@ public: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: case GenericOpcode::G_PTRTOINT: case GenericOpcode::G_INTTOPTR: - case GenericOpcode::G_BITCAST: case GenericOpcode::G_TRUNC: case GenericOpcode::G_SEXT: case GenericOpcode::G_SEXT_INREG: @@ -307,7 +321,7 @@ public: if (Instruction.opcode() == instructionOpcode(InstructionOpcode::G_SLLW)) return Instruction.setOpcode(instructionOpcode(InstructionOpcode::SLLW)); const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); - const std::optional<LLT> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); const auto SelectDirect = [&Instruction](InstructionOpcode Selected) { return Instruction.setOpcode(instructionOpcode(Selected)); @@ -364,7 +378,7 @@ public: case GenericOpcode::G_PTRMASK: return SelectDirect(InstructionOpcode::AND); case GenericOpcode::G_LOAD: - return selectLoad(Instruction, Type); + return selectLoad(Instruction, Function, Type); case GenericOpcode::G_SEXTLOAD: return selectExtLoad(Instruction, Type, true); case GenericOpcode::G_ZEXTLOAD: @@ -397,8 +411,6 @@ public: case GenericOpcode::G_PTRTOINT: case GenericOpcode::G_INTTOPTR: return Instruction.setOpcode(opcode(GenericOpcode::COPY)); - case GenericOpcode::G_BITCAST: - return selectBitcast(Instruction, Function); case GenericOpcode::G_FADD: return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S, InstructionOpcode::FADD_D); @@ -526,11 +538,17 @@ public: } private: - [[nodiscard]] Expected<Register> createTemporary(MachineFunction &Function, LLT Type, - const TargetDescription &Target) const { + [[nodiscard]] Expected<Register> + createTemporary(MachineFunction &Function, LowLevelType Type, const TargetDescription &Target, + std::optional<RegisterBankID> Bank = std::nullopt) const { auto Temporary = Function.registerInfo().createVirtualRegister(Type); if (!Temporary) return std::unexpected(Temporary.error()); + if (Bank) { + auto SetBank = Function.registerInfo().setRegisterBank(*Temporary, *Bank); + if (!SetBank) + return std::unexpected(SetBank.error()); + } auto Constraint = RegisterBanks.constrainRegister(Function, *Temporary, Target); if (!Constraint) return std::unexpected(Constraint.error()); @@ -624,7 +642,7 @@ private: [[nodiscard]] Expected<Register> normalizeIntegerBefore(MachineInstr &Anchor, MachineFunction &Function, const TargetDescription &Target, - Register Source, LLT Type, + Register Source, LowLevelType Type, bool Signed) const { if (!isSupportedIntegerScalar(Type)) return makeUnexpected(ErrorCode::Unsupported, @@ -689,9 +707,9 @@ private: [[nodiscard]] Expected<MachineInstr *> buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, - const TargetDescription &Target, LLT ValueType, Register Destination, + const TargetDescription &Target, LowLevelType ValueType, Register Destination, Register Condition, Register TrueValue, Register FalseValue) const { - auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth()); + auto IntegerType = LowLevelType::makeCategoryNeutralScalar(ValueType.scalarBitWidth()); if (!IntegerType) return std::unexpected(IntegerType.error()); auto Mask = createTemporary(Function, *IntegerType, Target); @@ -725,7 +743,7 @@ private: [[nodiscard]] Expected<void> selectDivRem(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, bool Signed) const { + std::optional<LowLevelType> Type, bool Signed) const { const MachineOperand *RemainderOperand = Instruction.operandAt(1); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); @@ -772,7 +790,8 @@ private: [[nodiscard]] Expected<void> selectMultiplyHigh(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, bool Signed) const { + std::optional<LowLevelType> Type, + bool Signed) const { if (!Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V high multiplication has no result type"); @@ -793,7 +812,7 @@ private: "RISC-V high multiplication has invalid register operands"); const Register DestinationRegister = *Destination; const Register RightRegister = *Right; - auto I64 = LLT::makeInteger(64); + auto I64 = LowLevelType::makeCategoryNeutralScalar(64); if (!I64) return std::unexpected(I64.error()); auto ExtendedLeft = createTemporary(Function, *I64, Target); @@ -868,7 +887,7 @@ private: [[nodiscard]] Expected<void> selectIntegerMinMax(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, GenericOpcode Generic) const { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); @@ -880,7 +899,7 @@ private: if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer min/max has invalid operands"); - auto I1 = LLT::makeInteger(1); + auto I1 = LowLevelType::makeCategoryNeutralScalar(1); if (!I1) return std::unexpected(I1.error()); auto Condition = createTemporary(Function, *I1, Target); @@ -922,7 +941,7 @@ private: [[nodiscard]] Expected<void> selectValue(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type) const { + std::optional<LowLevelType> Type) const { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *ConditionOperand = Instruction.operandAt(1); const MachineOperand *TrueOperand = Instruction.operandAt(2); @@ -942,7 +961,7 @@ private: const Register DestinationRegister = *Destination; const Register TrueRegister = *TrueValue; const Register FalseRegister = *FalseValue; - auto I1 = LLT::makeInteger(1); + auto I1 = LowLevelType::makeCategoryNeutralScalar(1); if (!I1) return std::unexpected(I1.error()); auto NormalizedCondition = createTemporary(Function, *I1, Target); @@ -964,8 +983,12 @@ private: if (!SetNormalize) return std::unexpected(SetNormalize.error()); + auto DestinationBank = registerBank(Function, DestinationRegister); + if (!DestinationBank) + return std::unexpected(DestinationBank.error()); + MachineIRBuilder Builder(Function, Target); - if (!Type->isFloat()) { + if (*DestinationBank != fprBank()) { auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, *NormalizedCondition, TrueRegister, FalseRegister); @@ -974,7 +997,7 @@ private: return {}; } - auto IntegerType = LLT::makeInteger(Type->scalarBitWidth()); + auto IntegerType = LowLevelType::makeCategoryNeutralScalar(Type->scalarBitWidth()); if (!IntegerType) return std::unexpected(IntegerType.error()); auto IntegerTrue = createTemporary(Function, *IntegerType, Target); @@ -1010,7 +1033,7 @@ private: [[nodiscard]] Expected<void> selectAbsoluteValue(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type) const { + std::optional<LowLevelType> Type) const { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = @@ -1059,12 +1082,13 @@ private: [[nodiscard]] Expected<std::pair<Register, MachineInstr *>> buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, - MachineFunction &Function, const TargetDescription &Target, LLT Type, - Register Result, Register Left, Register Right, bool IsAdd) const { + MachineFunction &Function, const TargetDescription &Target, + LowLevelType Type, Register Result, Register Left, Register Right, + bool IsAdd) const { auto FirstDifference = createTemporary(Function, Type, Target); auto SecondDifference = createTemporary(Function, Type, Target); auto SignBits = createTemporary(Function, Type, Target); - auto I1 = LLT::makeInteger(1); + auto I1 = LowLevelType::makeCategoryNeutralScalar(1); if (!I1) return std::unexpected(I1.error()); auto Overflow = createTemporary(Function, *I1, Target); @@ -1101,7 +1125,7 @@ private: [[nodiscard]] Expected<void> selectOverflow(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, GenericOpcode Generic) const { const MachineOperand *ResultOperand = Instruction.operandAt(0); const MachineOperand *OverflowOperand = Instruction.operandAt(1); @@ -1183,7 +1207,7 @@ private: return {}; } - auto I64 = LLT::makeInteger(64); + auto I64 = LowLevelType::makeCategoryNeutralScalar(64); if (!I64) return std::unexpected(I64.error()); auto Zero = physicalRegister(PhysicalRegisterID::X0); @@ -1305,7 +1329,7 @@ private: [[nodiscard]] Expected<void> selectExtendedOverflow(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, GenericOpcode Generic) const { const MachineOperand *ResultOperand = Instruction.operandAt(0); const MachineOperand *CarryOutOperand = Instruction.operandAt(1); @@ -1336,7 +1360,7 @@ private: IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB); auto Partial = createTemporary(Function, *Type, Target); - auto I1 = LLT::makeInteger(1); + auto I1 = LowLevelType::makeCategoryNeutralScalar(1); if (!Partial || !I1) return std::unexpected(!Partial ? Partial.error() : I1.error()); auto NormalizedCarry = createTemporary(Function, *I1, Target); @@ -1404,7 +1428,7 @@ private: [[nodiscard]] Expected<void> selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, GenericOpcode Generic) const { const MachineOperand *AddressOperand = Instruction.operandAt(1); const MachineOperand *ValueOperand = Instruction.operandAt(2); @@ -1484,8 +1508,8 @@ private: MachineFunction &Function, const TargetDescription &Target, GenericOpcode Generic, - std::optional<LLT> Type) const { - const std::optional<LLT> SourceType = operandType(Function, Instruction, 1); + std::optional<LowLevelType> Type) const { + const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, 1); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = @@ -1539,7 +1563,7 @@ private: [[nodiscard]] Expected<void> selectSignExtendInRegister(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type) const { + std::optional<LowLevelType> Type) const { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = @@ -1610,7 +1634,7 @@ private: DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); - const std::optional<LLT> LeftType = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> LeftType = operandType(Function, Instruction, 2); if (Predicate == nullptr || Predicate->integerPredicate() == nullptr || Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType) return makeUnexpected(ErrorCode::VerificationError, @@ -1779,8 +1803,8 @@ private: DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); - const std::optional<LLT> FloatType = operandType(Function, Instruction, 2); - const std::optional<LLT> ResultType = resultType(Function, Instruction); + const std::optional<LowLevelType> FloatType = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> ResultType = resultType(Function, Instruction); if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr || Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType) return makeUnexpected(ErrorCode::VerificationError, @@ -1951,8 +1975,9 @@ private: [[nodiscard]] Expected<void> selectFloatToInteger(MachineInstr &Instruction, MachineFunction &Function, - std::optional<LLT> Type, bool Unsigned) const { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + std::optional<LowLevelType> Type, + bool Unsigned) const { + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); if (!Type || !Source) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating-to-integer conversion has invalid types"); @@ -1973,8 +1998,9 @@ private: [[nodiscard]] Expected<void> selectIntegerToFloat(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type, bool Unsigned) const { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + std::optional<LowLevelType> Type, + bool Unsigned) const { + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *SourceRegister = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); @@ -2003,12 +2029,12 @@ private: } [[nodiscard]] Expected<void> selectFloatingMinMax(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode Single, InstructionOpcode Double) const { - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(instructionOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(instructionOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating min/max type"); @@ -2068,40 +2094,20 @@ private: return Instruction.setOpcode(instructionOpcode(InstructionOpcode::EBREAK)); } - [[nodiscard]] Expected<void> selectBitcast(MachineInstr &Instruction, - MachineFunction &Function) const { - const std::optional<LLT> DestinationType = resultType(Function, Instruction); - const std::optional<LLT> SourceType = operandType(Function, Instruction, 1); - if (!DestinationType || !SourceType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V bitcast has invalid register types"); - if (DestinationType->isInteger() && SourceType->isFloat()) { - return Instruction.setOpcode(instructionOpcode(SourceType->scalarBitWidth() == 32 - ? InstructionOpcode::FMV_X_W - : InstructionOpcode::FMV_X_D)); - } - if (DestinationType->isFloat() && SourceType->isInteger()) { - return Instruction.setOpcode(instructionOpcode(DestinationType->scalarBitWidth() == 32 - ? InstructionOpcode::FMV_W_X - : InstructionOpcode::FMV_D_X)); - } - return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported"); - } - [[nodiscard]] Expected<void> selectIntegerBinary(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode DoubleWord, InstructionOpcode Word) const { if (!Type || !isSupportedIntegerType(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer instruction requires a supported scalar integer"); - const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + const bool IsWord = Type->isCategoryNeutralScalar() && Type->scalarBitWidth() <= 32; return Instruction.setOpcode(instructionOpcode(IsWord ? Word : DoubleWord)); } [[nodiscard]] Expected<void> selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, std::optional<LLT> Type, + const TargetDescription &Target, std::optional<LowLevelType> Type, InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed, bool NormalizeRight) const { const MachineOperand *LeftOperand = Instruction.operandAt(1); @@ -2134,8 +2140,8 @@ private: return Instruction.setOpcode(instructionOpcode(IsWord ? Word : DoubleWord)); } - [[nodiscard]] Expected<void> selectLoad(MachineInstr &Instruction, - std::optional<LLT> Type) const { + [[nodiscard]] Expected<void> selectLoad(MachineInstr &Instruction, MachineFunction &Function, + std::optional<LowLevelType> Type) const { if (!Type) return makeUnexpected(ErrorCode::VerificationError, "load result has no virtual-register type"); @@ -2143,8 +2149,16 @@ private: if (!Bytes) return makeUnexpected(ErrorCode::Unsupported, "RISC-V load size does not match its result type"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V load has no destination"); + auto Bank = registerBank(Function, *Destination); + if (!Bank) + return std::unexpected(Bank.error()); if (Instruction.memOperands().front()->isAtomic()) { - if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + if (*Bank == fprBank() || !isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V atomic load requires a 32-bit or 64-bit GPR value"); return Instruction.setOpcode( @@ -2153,7 +2167,7 @@ private: auto AppendOffset = appendImmediate(Instruction, 0); if (!AppendOffset) return std::unexpected(AppendOffset.error()); - if (isSupportedIntegerType(*Type)) { + if (*Bank != fprBank() && isSupportedIntegerType(*Type)) { const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB : *Bytes == 2 ? InstructionOpcode::LH : *Bytes == 4 ? InstructionOpcode::LW @@ -2163,15 +2177,15 @@ private: return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported"); return Instruction.setOpcode(instructionOpcode(Selected)); } - if (Type->isFloat() && Type->scalarBitWidth() == 32) + if (*Bank == fprBank() && Type->scalarBitWidth() == 32 && *Bytes == 4) return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FLW)); - if (Type->isFloat() && Type->scalarBitWidth() == 64) + if (*Bank == fprBank() && Type->scalarBitWidth() == 64 && *Bytes == 8) return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FLD)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type"); } - [[nodiscard]] Expected<void> selectExtLoad(MachineInstr &Instruction, std::optional<LLT> Type, - bool Signed) const { + [[nodiscard]] Expected<void> selectExtLoad(MachineInstr &Instruction, + std::optional<LowLevelType> Type, bool Signed) const { if (!Type || !isSupportedIntegerType(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load requires an integer result"); @@ -2201,10 +2215,10 @@ private: return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating extension load has no destination register"); const Register DestinationRegister = *Destination; - auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + auto F32 = LowLevelType::makeCategoryNeutralScalar(32); if (!F32) return std::unexpected(F32.error()); - auto Temporary = createTemporary(Function, *F32, Target); + auto Temporary = createTemporary(Function, *F32, Target, fprBank()); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); @@ -2233,10 +2247,10 @@ private: return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating truncation store has no value register"); const Register ValueRegister = *Value; - auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + auto F32 = LowLevelType::makeCategoryNeutralScalar(32); if (!F32) return std::unexpected(F32.error()); - auto Temporary = createTemporary(Function, *F32, Target); + auto Temporary = createTemporary(Function, *F32, Target, fprBank()); if (!Temporary) return std::unexpected(Temporary.error()); MachineIRBuilder Builder(Function, Target); @@ -2270,7 +2284,7 @@ private: [[nodiscard]] Expected<void> selectStore(MachineInstr &Instruction, MachineFunction &Function) const { const MachineOperand *Value = Instruction.operandAt(0); - const std::optional<LLT> Type = + const std::optional<LowLevelType> Type = Value == nullptr ? std::nullopt : registerType(Function, *Value); if (!Type) return makeUnexpected(ErrorCode::VerificationError, @@ -2279,8 +2293,14 @@ private: if (!Bytes) return makeUnexpected(ErrorCode::Unsupported, "RISC-V store size does not match its value type"); + const Register *ValueRegister = Value->getIf<Register>(); + if (ValueRegister == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V store has no value register"); + auto Bank = registerBank(Function, *ValueRegister); + if (!Bank) + return std::unexpected(Bank.error()); if (Instruction.memOperands().front()->isAtomic()) { - if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + if (*Bank == fprBank() || !isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V atomic store requires a 32-bit or 64-bit GPR value"); auto Zero = physicalRegister(PhysicalRegisterID::X0); @@ -2295,7 +2315,7 @@ private: auto AppendOffset = appendImmediate(Instruction, 0); if (!AppendOffset) return std::unexpected(AppendOffset.error()); - if (isSupportedIntegerType(*Type)) { + if (*Bank != fprBank() && isSupportedIntegerType(*Type)) { const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB : *Bytes == 2 ? InstructionOpcode::SH : *Bytes == 4 ? InstructionOpcode::SW @@ -2305,9 +2325,9 @@ private: return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported"); return Instruction.setOpcode(instructionOpcode(Selected)); } - if (Type->isFloat() && Type->scalarBitWidth() == 32) + if (*Bank == fprBank() && Type->scalarBitWidth() == 32 && *Bytes == 4) return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FSW)); - if (Type->isFloat() && Type->scalarBitWidth() == 64) + if (*Bank == fprBank() && Type->scalarBitWidth() == 64 && *Bytes == 8) return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FSD)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type"); } @@ -2335,7 +2355,7 @@ private: [[nodiscard]] Expected<void> selectIntegerConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type) const { + std::optional<LowLevelType> Type) const { if (!Type || !isSupportedIntegerType(*Type)) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant selection requires an i1-i64 or pointer result"); @@ -2357,7 +2377,7 @@ private: const Register Destination = *DestinationValue; const std::int64_t Value = *ImmediateValue; const std::uint32_t ImmediateTargetFlags = ImmediateOperand->targetFlags(); - const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + const bool IsWord = Type->isCategoryNeutralScalar() && Type->scalarBitWidth() <= 32; const InstructionOpcode AddImmediate = IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI; const InstructionOpcode ShiftLeftImmediate = @@ -2488,14 +2508,13 @@ private: [[nodiscard]] Expected<void> selectFloatingConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, - std::optional<LLT> Type) const { - if (!Type || !isRVFloatType(*Type)) + std::optional<LowLevelType> Type) const { + if (!Type || !isSupportedFloatingScalar(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64"); const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>(); - const std::optional<FloatSemantics> Semantics = Type->floatSemantics(); - if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + if (Floating == nullptr || Floating->Bits.bitWidth() != Type->scalarBitWidth()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating constant has mismatched bit semantics"); const MachineOperand *DestinationOperand = Instruction.operandAt(0); @@ -2511,7 +2530,7 @@ private: auto Integer = APInt::fromUnsigned(64, Bits); if (!Integer) return std::unexpected(Integer.error()); - auto IntegerType = LLT::makeInteger(64); + auto IntegerType = LowLevelType::makeCategoryNeutralScalar(64); if (!IntegerType) return std::unexpected(IntegerType.error()); auto Temporary = createTemporary(Function, *IntegerType, Target); @@ -2564,35 +2583,35 @@ private: } [[nodiscard]] Expected<void> selectRoundingBinary(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode Single, InstructionOpcode Double) const { auto RoundingMode = appendImmediate(Instruction, 0); if (!RoundingMode) return std::unexpected(RoundingMode.error()); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(instructionOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(instructionOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type"); } [[nodiscard]] Expected<void> selectRoundingTernary(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode Single, InstructionOpcode Double) const { return selectRoundingBinary(Instruction, Type, Single, Double); } [[nodiscard]] Expected<void> selectRoundingUnary(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode Single, InstructionOpcode Double) const { return selectRoundingBinary(Instruction, Type, Single, Double); } [[nodiscard]] Expected<void> selectSignInjection(MachineInstr &Instruction, - std::optional<LLT> Type, + std::optional<LowLevelType> Type, InstructionOpcode Single, InstructionOpcode Double) const { const MachineOperand *Source = Instruction.operandAt(1); @@ -2603,9 +2622,9 @@ private: auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg); if (!Duplicate) return std::unexpected(Duplicate.error()); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(instructionOpcode(Single)); - if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + if (Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(instructionOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type"); } diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index eace5ad..1a29ebf 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -31,7 +31,8 @@ namespace rcc::riscv { namespace { -[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function, Register Reg) { +[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function, + Register Reg) { if (!Reg.isVirtual()) return std::nullopt; auto Info = Function.registerInfo().virtualRegisterInfo(Reg); @@ -40,62 +41,60 @@ namespace { return Info->get().Type; } -[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function, - const MachineOperand &Operand) { +[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function, + const MachineOperand &Operand) { const Register *Reg = Operand.getIf<Register>(); return Reg == nullptr ? std::nullopt : registerType(Function, *Reg); } -[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function, - const MachineInstr &Instruction) { +[[nodiscard]] std::optional<LowLevelType> resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { const MachineOperand *Result = Instruction.operandAt(0); return Result == nullptr ? std::nullopt : registerType(Function, *Result); } -[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function, - const MachineInstr &Instruction, std::size_t Index) { +[[nodiscard]] std::optional<LowLevelType> +operandType(const MachineFunction &Function, const MachineInstr &Instruction, std::size_t Index) { const MachineOperand *Operand = Instruction.operandAt(Index); return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); } -[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { - if (!Type.isFloat()) - return false; - const std::optional<FloatSemantics> Semantics = Type.floatSemantics(); - return Semantics && - (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && + (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept { + return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isSupportedIntegerScalar(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64; } -[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && +[[nodiscard]] bool isNativeIntegerScalar(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { - return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && +[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept { + return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } -[[nodiscard]] bool isI1Type(LLT Type) noexcept { - return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; +[[nodiscard]] bool isI1Type(LowLevelType Type) noexcept { + return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() == 1; } -[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { - return (Destination.isInteger() || Destination.isAnyScalar()) && - (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && - Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && +[[nodiscard]] bool isScalarIntegerExtension(LowLevelType Destination, + LowLevelType Source) noexcept { + return Destination.isCategoryNeutralScalar() && Source.isCategoryNeutralScalar() && + Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 && + Source.scalarBitWidth() < Destination.scalarBitWidth() && Destination.scalarBitWidth() <= 64; } @@ -394,11 +393,12 @@ wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Fun std::uint64_t Value); [[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Left, - Register Right); + GenericOpcode Generic, LowLevelType Type, + Register Left, Register Right); [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Source); + GenericOpcode Generic, LowLevelType Type, + Register Source); [[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction); @@ -462,7 +462,7 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); const MachineOperand *WideOperand = Truncation->operandAt(1); const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); - const std::optional<LLT> WideType = + const std::optional<LowLevelType> WideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || *WideType != *DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) @@ -533,7 +533,7 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow constant legalization has invalid operands"); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto SetType = Function.registerInfo().setType(*Destination, *Type); @@ -554,7 +554,8 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun [[nodiscard]] Expected<Register> widenAtomicCompareExchangeSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, - Register Source, LLT WideType) { + Register Source, + LowLevelType WideType) { const auto SourceType = registerType(Function, Source); if (!SourceType) return makeUnexpected(ErrorCode::VerificationError, @@ -578,7 +579,7 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun [[nodiscard]] Expected<void> lowerAtomicCompareExchange(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { - const std::optional<LLT> OriginalType = resultType(Function, Instruction); + const std::optional<LowLevelType> OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *CompareOperand = Instruction.operandAt(2); const MachineOperand *NewValueOperand = Instruction.operandAt(3); @@ -593,7 +594,7 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun "RISC-V narrow compare-exchange has invalid operands"); const Register OriginalDestination = *Destination; - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -610,7 +611,7 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && operandType(Function, *FoldedExtension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); const Register *FoldedDestination = @@ -668,7 +669,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf<Register>(); const Register *NewValue = NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf<Register>(); - const std::optional<LLT> ValueType = resultType(Function, Instruction); + const std::optional<LowLevelType> ValueType = resultType(Function, Instruction); if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) return makeUnexpected(ErrorCode::VerificationError, @@ -727,7 +728,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction [[nodiscard]] Expected<Register> widenIntegerCompareSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, - Register Source, LLT WideType) { + Register Source, LowLevelType WideType) { const auto SourceType = registerType(Function, Source); if (!SourceType) return makeUnexpected(ErrorCode::VerificationError, @@ -751,7 +752,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction const MachineOperand *ExtendedOperand = User->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - const std::optional<LLT> ExtendedType = + const std::optional<LowLevelType> ExtendedType = ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && @@ -778,7 +779,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction Def->operandCount() == 2) { const MachineOperand *WideOperand = Def->operandAt(1); const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); - const std::optional<LLT> ExistingWideType = + const std::optional<LowLevelType> ExistingWideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { auto Result = Function.registerInfo().createVirtualRegister(WideType); @@ -825,7 +826,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction const Register OriginalLeft = *Left; const Register OriginalRight = *Right; - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -855,7 +856,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && operandType(Function, *FoldedExtension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); const Register *Extended = @@ -923,7 +924,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction const std::uint64_t Mask = Plan->SourceWidth == 64 ? std::numeric_limits<std::uint64_t>::max() : (std::uint64_t{1} << Plan->SourceWidth) - 1U; - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -964,7 +965,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction if (Plan->SourceWidth == 32 && (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); @@ -1043,7 +1044,7 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction if (!PoolIndex) return std::unexpected(PoolIndex.error()); - auto PointerType = LLT::makePointer(0, 64); + auto PointerType = LowLevelType::makePointer(0, 64); if (!PointerType) return std::unexpected(PointerType.error()); auto Address = Function.registerInfo().createVirtualRegister(*PointerType); @@ -1143,11 +1144,11 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept { return Description.Size.minimumBytes(); } -[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { +[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LowLevelType Type) noexcept { const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) + if (!Bytes || (!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type))) return false; - if (isRVFloatType(Type)) + if (isSupportedFloatingScalar(Type)) return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; @@ -1224,7 +1225,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, MachineInstr &Anchor, std::uint32_t Width, std::uint64_t Value) { - auto Type = LLT::makeAnyScalar(Width); + auto Type = LowLevelType::makeCategoryNeutralScalar(Width); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); @@ -1250,7 +1251,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, } [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, + GenericOpcode Generic, LowLevelType Type, Register Source) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) @@ -1272,8 +1273,8 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, } [[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LLT Type, Register Left, - Register Right) { + GenericOpcode Generic, LowLevelType Type, + Register Left, Register Right) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); @@ -1321,7 +1322,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, [[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { - const std::optional<LLT> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); @@ -1342,7 +1343,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, return foldNarrowAnyExtBinary(*Extension, Function, Target); } - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -1365,7 +1366,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, const TargetDescription &Target, CallingConvention Convention, const TargetCallLowering &CallLowering) { - const std::optional<LLT> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); @@ -1387,6 +1388,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, return ABIValue{ .Value = Reg, .Type = Type, + .Category = ABIValueCategory::Integer, .OriginalValue = Number, .FragmentIndex = 0, .FragmentCount = 1, @@ -1488,7 +1490,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, } } } - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); @@ -1562,7 +1564,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, [[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { - const std::optional<LLT> DestinationType = resultType(Function, Instruction); + const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction); if (!DestinationType || DestinationType->scalarBitWidth() == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge destination has no scalar type"); @@ -1582,7 +1584,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, MachineInstr *Use = singleUseInstruction(Function, *Destination); if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && operandType(Function, *Use, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *ExtendedOperand = Use->operandAt(0); const Register *Extended = @@ -1594,7 +1596,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, } } - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -1613,7 +1615,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, Terms.push_back(*WideSource); continue; } - const std::optional<LLT> SourceType = registerType(Function, *Operand); + const std::optional<LowLevelType> SourceType = registerType(Function, *Operand); if (!SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); const std::uint64_t Shift = @@ -1758,8 +1760,8 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge has invalid register operands"); const std::size_t OutputCount = Instruction.operandCount() - 1U; - const std::optional<LLT> OutputType = operandType(Function, Instruction, 0); - const std::optional<LLT> SourceType = operandType(Function, Instruction, OutputCount); + const std::optional<LowLevelType> OutputType = operandType(Function, Instruction, 0); + const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, OutputCount); if (!OutputType || !SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge operands have no scalar types"); @@ -1770,7 +1772,7 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); - auto WideType = LLT::makeAnyScalar(64); + auto WideType = LowLevelType::makeCategoryNeutralScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); @@ -1794,11 +1796,11 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, } MachineInstr *Extension = singleUseInstruction(Function, *Output); - const bool FoldZExt = Extension != nullptr && - Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && - operandType(Function, *Extension, 0) - .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false); + const bool FoldZExt = + Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && + operandType(Function, *Extension, 0) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false); if (FoldZExt) { const MachineOperand *ExtendedOperand = Extension->operandAt(0); const Register *Extended = @@ -1851,7 +1853,7 @@ collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth const auto Type = registerType(Function, Value); if (!Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); - if ((!Type->isInteger() && !Type->isAnyScalar()) || Type->scalarBitWidth() == 0) + if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value decomposition requires an integer scalar"); if (Type->scalarBitWidth() <= 64) @@ -1869,7 +1871,7 @@ collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { const MachineOperand *Operand = Def->operandAt(Index); const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - const std::optional<LLT> PartType = + const std::optional<LowLevelType> PartType = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, @@ -1932,7 +1934,7 @@ struct WideResultPlan final { for (std::size_t Index = 0; Index < RequiredParts; ++Index) { const MachineOperand *Operand = Use->operandAt(Index); const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - const std::optional<LLT> Type = + const std::optional<LowLevelType> Type = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, @@ -1946,7 +1948,7 @@ struct WideResultPlan final { MachineInstr &Anchor, IntegerPredicate Predicate, Register Left, Register Right) { - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); @@ -1973,8 +1975,8 @@ struct WideResultPlan final { } [[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - LLT Type, Register Condition, Register TrueValue, - Register FalseValue) { + LowLevelType Type, Register Condition, + Register TrueValue, Register FalseValue) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); @@ -2007,7 +2009,7 @@ struct TwoLimbShiftResult final { emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register AmountExcess, Register AmountLack, Register Zero, Register IsShort, Register IsZero) { - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); @@ -2042,7 +2044,7 @@ emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Regist [[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount) { - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); @@ -2074,7 +2076,7 @@ emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Regist emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register Boundary, Register Zero) { - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto AmountExcess = @@ -2119,7 +2121,7 @@ emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, if (Parts.size() != 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V three-limb shift requires exactly three limbs"); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); @@ -2291,7 +2293,7 @@ emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide add requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector<Register> Results; @@ -2356,7 +2358,7 @@ emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide sub requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector<Register> Results; @@ -2415,7 +2417,7 @@ emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode G if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide bitwise operation requires matching multi-limb operands"); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector<Register> Results; @@ -2500,7 +2502,8 @@ emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode G Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) return false; return operandType(Function, *Truncation, 1) - .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) + .transform( + [](LowLevelType Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) .value_or(false); } @@ -2515,7 +2518,7 @@ emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode G const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent()); const Register Source = *Truncation->operandAt(1)->getIf<Register>(); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); @@ -2549,7 +2552,7 @@ struct NarrowPhiIncoming final { [[nodiscard]] Expected<Register> buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, - LLT Type, Register Source) { + LowLevelType Type, Register Source) { auto Result = Function.registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); @@ -2573,14 +2576,14 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ [[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { - const std::optional<LLT> OriginalType = resultType(Function, Instruction); + const std::optional<LowLevelType> OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); - auto LegalType = LLT::makeAnyScalar(32); + auto LegalType = LowLevelType::makeCategoryNeutralScalar(32); if (!LegalType) return std::unexpected(LegalType.error()); @@ -2603,7 +2606,7 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && operandType(Function, *Def, 1) - .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() >= 32; }) .value_or(false)) { Reusable = Def; } @@ -2731,7 +2734,7 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); if (!ResultPlan) return std::unexpected(ResultPlan.error()); - auto Type = LLT::makeAnyScalar(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); @@ -2800,7 +2803,7 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != &Use) return false; - const std::optional<LLT> Result = resultType(Function, *Trunc); + const std::optional<LowLevelType> Result = resultType(Function, *Trunc); if (!Result || Result->scalarBitWidth() > LowType->scalarBitWidth()) return false; } @@ -2813,13 +2816,14 @@ struct RV64ScalarLayout final { std::uint64_t ABIAlignment = 0; }; -[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LLT Type) noexcept { +[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LowLevelType Type) noexcept { if (Type.isPointer()) { return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 ? std::optional<RV64ScalarLayout>{{8, 8, 8}} : std::nullopt; } - if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) { + if ((!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)) || + Type.scalarBitWidth() == 0) { return std::nullopt; } const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; @@ -2828,7 +2832,7 @@ struct RV64ScalarLayout final { } [[nodiscard]] Expected<const MachineMemOperand *> -createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, +createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LowLevelType Type, std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { auto Size = MemorySize::fixed(SizeBytes); auto Align = Alignment::create(AlignmentBytes); @@ -2845,8 +2849,8 @@ createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, struct VAArgPlan final { Register Destination; Register ListPointer; - LLT DestinationType; - LLT PointerType; + LowLevelType DestinationType; + LowLevelType PointerType; std::uint64_t Alignment = 0; RV64ScalarLayout ValueLayout; }; @@ -2863,8 +2867,8 @@ struct VAArgPlan final { const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>(); - const std::optional<LLT> DestinationType = resultType(Function, Instruction); - const std::optional<LLT> PointerType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction); + const std::optional<LowLevelType> PointerType = operandType(Function, Instruction, 1); const auto AlignmentValue = AlignmentOperand == nullptr ? Expected<std::int64_t>(std::unexpected( Error(ErrorCode::InvalidArgument, "missing alignment"))) @@ -2989,7 +2993,7 @@ struct VAArgPlan final { struct VACopyPlan final { Register Destination; Register Source; - LLT PointerType; + LowLevelType PointerType; }; [[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction, @@ -3006,8 +3010,8 @@ struct VACopyPlan final { const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - const std::optional<LLT> DestinationType = operandType(Function, Instruction, 1); - const std::optional<LLT> SourceType = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> DestinationType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, 2); if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || Destination == nullptr || Source == nullptr || !Destination->isVirtual() || !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || @@ -3095,7 +3099,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In return LegalizationAction::Legal; const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); - const std::optional<LLT> Type = resultType(Function, Instruction); + const std::optional<LowLevelType> Type = resultType(Function, Instruction); switch (Generic) { case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom @@ -3110,7 +3114,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_BR: return LegalizationAction::Legal; case GenericOpcode::G_BRCOND: { - const std::optional<LLT> Condition = operandType(Function, Instruction, 0); + const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 0); if (!Condition || !isScalarGPRType(*Condition)) return LegalizationAction::Unsupported; return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom @@ -3120,28 +3124,28 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In if (!Type) return LegalizationAction::Unsupported; for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const std::optional<LLT> Incoming = operandType(Function, Instruction, Index); + const std::optional<LowLevelType> Incoming = operandType(Function, Instruction, Index); if (!Incoming || *Incoming != *Type) return LegalizationAction::Unsupported; } - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) + if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() > 64) return LegalizationAction::Custom; - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) + if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() < 32) return LegalizationAction::Custom; - if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) + if (!isSupportedIntegerType(*Type)) return LegalizationAction::Unsupported; return LegalizationAction::Legal; } case GenericOpcode::G_IMPLICIT_DEF: - return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; + return Type && isSupportedIntegerType(*Type) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; case GenericOpcode::G_STORE: if (Instruction.operandCount() == 0) return LegalizationAction::Unsupported; - if (const std::optional<LLT> ValueType = registerType(Function, *Instruction.operandAt(0)); + if (const std::optional<LowLevelType> ValueType = + registerType(Function, *Instruction.operandAt(0)); ValueType) { - const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; @@ -3153,8 +3157,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In ? LegalizationAction::Legal : LegalizationAction::Unsupported; } - if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && - isSupportedNonAtomicMemoryAccess(Instruction) && + if (isSupportedIntegerType(*ValueType) && isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *ValueType)) { return LegalizationAction::Legal; } @@ -3163,7 +3166,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_SEXTLOAD: case GenericOpcode::G_ZEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); - const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && @@ -3175,19 +3178,19 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_FPEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); - const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); - return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && - AddressType->isPointer() && AddressType->addressSpace() == 0 && - AddressType->scalarBitWidth() == 64 && *Bytes == 4 && - isSupportedNonAtomicMemoryAccess(Instruction) + const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && Type->isCategoryNeutralScalar() && + Type->scalarBitWidth() == 64 && AddressType->isPointer() && + AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && + *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPTRUNCSTORE: { - const std::optional<LLT> ValueType = operandType(Function, Instruction, 0); - const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> ValueType = operandType(Function, Instruction, 0); + const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); const auto Bytes = memoryAccessBytes(Instruction); - return ValueType && AddressType && Bytes && ValueType->isFloat() && + return ValueType && AddressType && Bytes && ValueType->isCategoryNeutralScalar() && ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) @@ -3196,7 +3199,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_LOAD: if (Type) { - const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; @@ -3208,8 +3211,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In ? LegalizationAction::Legal : LegalizationAction::Unsupported; } - if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && - isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) { + if (isSupportedIntegerType(*Type) && isSupportedNonAtomicMemoryAccess(Instruction) && + memoryMatchesType(Instruction, *Type)) { return LegalizationAction::Legal; } } @@ -3218,33 +3221,35 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_FSUB: case GenericOpcode::G_FMUL: case GenericOpcode::G_FDIV: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && + *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMA: { - const std::optional<LLT> First = operandType(Function, Instruction, 1); - const std::optional<LLT> Second = operandType(Function, Instruction, 2); - const std::optional<LLT> Third = operandType(Function, Instruction, 3); - return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && - *Second == *Type && *Third == *Type + const std::optional<LowLevelType> First = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Second = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Third = operandType(Function, Instruction, 3); + return Type && First && Second && Third && isSupportedFloatingScalar(*Type) && + *First == *Type && *Second == *Type && *Third == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FNEG: case GenericOpcode::G_FABS: case GenericOpcode::G_FSQRT: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - return Type && Source && isRVFloatType(*Type) && *Source == *Type + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedFloatingScalar(*Type) && *Source == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FCOPYSIGN: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && + *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } @@ -3252,16 +3257,15 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>(); - const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt; - return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && - Floating->Bits.semantics() == *Semantics + return Type && isSupportedFloatingScalar(*Type) && Floating != nullptr && + Floating->Bits.bitWidth() == Type->scalarBitWidth() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ASSERT_SEXT: case GenericOpcode::G_ASSERT_ZEXT: case GenericOpcode::G_ASSERT_ALIGN: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Instruction.operandCount() == 3 ? LegalizationAction::Legal @@ -3269,38 +3273,35 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_FREEZE: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - return Type && Source && *Type == *Source && - (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && isSupportedIntegerType(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRTOINT: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64 && Source->isPointer() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_INTTOPTR: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && - Source->isInteger() && Source->scalarBitWidth() == 64 + Source->isCategoryNeutralScalar() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_BITCAST: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - const bool IntegerFloatPair = - Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && - ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && - (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); - return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && *Type != *Source && Type->scalarBitWidth() == Source->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; } case GenericOpcode::G_ANYEXT: case GenericOpcode::G_SEXT: case GenericOpcode::G_ZEXT: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); if (!Type || !Source) return LegalizationAction::Unsupported; if (!isScalarIntegerExtension(*Type, *Source)) { @@ -3309,8 +3310,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); MachineInstr *OnlyUse = Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); - return (Type->isInteger() || Type->isAnyScalar()) && - (Source->isInteger() || Source->isAnyScalar()) && + return Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() && Type->scalarBitWidth() > 64 && Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) @@ -3334,10 +3334,9 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In return LegalizationAction::Legal; } case GenericOpcode::G_TRUNC: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && - (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && - Source->scalarBitWidth() > Type->scalarBitWidth()) + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + if (Type && Source && Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() && + Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth()) return LegalizationAction::Legal; return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() @@ -3351,19 +3350,20 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; - const std::optional<LLT> SmallType = operandType(Function, Instruction, SmallTypeIndex); - const std::optional<LLT> BigType = operandType(Function, Instruction, BigTypeIndex); + const std::optional<LowLevelType> SmallType = + operandType(Function, Instruction, SmallTypeIndex); + const std::optional<LowLevelType> BigType = operandType(Function, Instruction, BigTypeIndex); if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) return LegalizationAction::Legal; - return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) && - (BigType->isInteger() || BigType->isAnyScalar()) && - SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 + return SmallType && BigType && SmallType->isCategoryNeutralScalar() && + BigType->isCategoryNeutralScalar() && SmallType->scalarBitWidth() != 0 && + BigType->scalarBitWidth() != 0 ? LegalizationAction::Custom : LegalizationAction::Unsupported; } case GenericOpcode::G_SEXT_INREG: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); const MachineOperand *WidthOperand = Instruction.operandAt(2); const auto Width = WidthOperand == nullptr @@ -3379,8 +3379,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In const MachineOperand *Predicate = Instruction.operandAt(1); const auto *IntegerPredicate = Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); - const std::optional<LLT> Left = operandType(Function, Instruction, 2); - const std::optional<LLT> Right = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || !isSupportedIntegerType(*Left)) { @@ -3394,19 +3394,20 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In const MachineOperand *Predicate = Instruction.operandAt(1); const auto *FloatingPredicate = Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); - const std::optional<LLT> Left = operandType(Function, Instruction, 2); - const std::optional<LLT> Right = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); return Type && isI1Type(*Type) && FloatingPredicate != nullptr && FloatingPredicate->floatingPredicate() != nullptr && Left && Right && - *Left == *Right && isRVFloatType(*Left) + *Left == *Right && isSupportedFloatingScalar(*Left) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPEXT: case GenericOpcode::G_FPTRUNC: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); const bool IsExtension = Generic == GenericOpcode::G_FPEXT; - return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && + return Type && Source && isSupportedFloatingScalar(*Type) && + isSupportedFloatingScalar(*Source) && ((IsExtension && Source->scalarBitWidth() == 32 && Type->scalarBitWidth() == 64) || (!IsExtension && Source->scalarBitWidth() == 64 && @@ -3416,39 +3417,39 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_FPTOSI: case GenericOpcode::G_FPTOUI: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedIntegerType(*Type) && isSupportedFloatingScalar(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SITOFP: case GenericOpcode::G_UITOFP: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); - return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedFloatingScalar(*Type) && isSupportedIntegerType(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMINNUM: case GenericOpcode::G_FMAXNUM: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && + *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SELECT: { - const std::optional<LLT> Condition = operandType(Function, Instruction, 1); - const std::optional<LLT> TrueValue = operandType(Function, Instruction, 2); - const std::optional<LLT> FalseValue = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> TrueValue = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> FalseValue = operandType(Function, Instruction, 3); return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && - *TrueValue == *Type && *FalseValue == *Type && - (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + *TrueValue == *Type && *FalseValue == *Type && isSupportedIntegerType(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRMASK: { - const std::optional<LLT> Pointer = operandType(Function, Instruction, 1); - const std::optional<LLT> Mask = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Mask = operandType(Function, Instruction, 2); return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 @@ -3456,16 +3457,16 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In : LegalizationAction::Unsupported; } case GenericOpcode::G_ABS: { - const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SDIVREM: case GenericOpcode::G_UDIVREM: { - const std::optional<LLT> Remainder = operandType(Function, Instruction, 1); - const std::optional<LLT> Left = operandType(Function, Instruction, 2); - const std::optional<LLT> Right = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Remainder = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && *Remainder == *Type && *Left == *Type && *Right == *Type ? LegalizationAction::Legal @@ -3473,8 +3474,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_SMULH: case GenericOpcode::G_UMULH: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal @@ -3484,8 +3485,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: case GenericOpcode::G_UMAX: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal @@ -3497,9 +3498,9 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_SSUBO: case GenericOpcode::G_UMULO: case GenericOpcode::G_SMULO: { - const std::optional<LLT> Overflow = operandType(Function, Instruction, 1); - const std::optional<LLT> Left = operandType(Function, Instruction, 2); - const std::optional<LLT> Right = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Overflow = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && isI1Type(*Overflow) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal @@ -3509,10 +3510,10 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_USUBE: case GenericOpcode::G_SADDE: case GenericOpcode::G_SSUBE: { - const std::optional<LLT> CarryOut = operandType(Function, Instruction, 1); - const std::optional<LLT> Left = operandType(Function, Instruction, 2); - const std::optional<LLT> Right = operandType(Function, Instruction, 3); - const std::optional<LLT> CarryIn = operandType(Function, Instruction, 4); + const std::optional<LowLevelType> CarryOut = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> CarryIn = operandType(Function, Instruction, 4); return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal @@ -3528,8 +3529,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_ATOMICRMW_MIN: case GenericOpcode::G_ATOMICRMW_UMAX: case GenericOpcode::G_ATOMICRMW_UMIN: { - const std::optional<LLT> Address = operandType(Function, Instruction, 1); - const std::optional<LLT> Value = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Value = operandType(Function, Instruction, 2); const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && Address->isPointer() && Address->addressSpace() == 0 && @@ -3540,11 +3541,11 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In : LegalizationAction::Unsupported; } case GenericOpcode::G_ATOMIC_CMPXCHG: { - const std::optional<LLT> Address = operandType(Function, Instruction, 1); - const std::optional<LLT> Compare = operandType(Function, Instruction, 2); - const std::optional<LLT> NewValue = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 3); if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || - (!Type->isInteger() && !Type->isAnyScalar()) || + !Type->isCategoryNeutralScalar() || (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || @@ -3555,13 +3556,12 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; } case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { - const std::optional<LLT> Success = operandType(Function, Instruction, 1); - const std::optional<LLT> Address = operandType(Function, Instruction, 2); - const std::optional<LLT> Compare = operandType(Function, Instruction, 3); - const std::optional<LLT> NewValue = operandType(Function, Instruction, 4); + const std::optional<LowLevelType> Success = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Address = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 3); + const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 4); return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && - *Compare == *Type && *NewValue == *Type && - (Type->isInteger() || Type->isAnyScalar()) && + *Compare == *Type && *NewValue == *Type && Type->isCategoryNeutralScalar() && (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && Address->isPointer() && Address->addressSpace() == 0 && @@ -3605,7 +3605,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In } case GenericOpcode::G_BRINDIRECT: return operandType(Function, Instruction, 0) - .transform([](LLT Value) { + .transform([](LowLevelType Value) { return Value.isPointer() && Value.addressSpace() == 0 && Value.scalarBitWidth() == 64; }) @@ -3629,7 +3629,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In return wideConstantUnmergePlan(Instruction, Function).has_value() ? LegalizationAction::Custom : LegalizationAction::Unsupported; - if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { + if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return LegalizationAction::Custom; @@ -3650,10 +3650,10 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In case GenericOpcode::G_SHL: case GenericOpcode::G_LSHR: case GenericOpcode::G_ASHR: { - const std::optional<LLT> Left = operandType(Function, Instruction, 1); - const std::optional<LLT> Right = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || - (!Type->isInteger() && !Type->isAnyScalar())) + !Type->isCategoryNeutralScalar()) return LegalizationAction::Unsupported; if (Type->scalarBitWidth() > 64) { return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || @@ -3670,7 +3670,7 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); MachineInstr *Extension = Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); - const std::optional<LLT> ExtensionType = + const std::optional<LowLevelType> ExtensionType = Extension == nullptr ? std::nullopt : resultType(Function, *Extension); if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && ExtensionType && ExtensionType->scalarBitWidth() == 64) { @@ -3682,8 +3682,8 @@ Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &In : LegalizationAction::Unsupported; } case GenericOpcode::G_PTR_ADD: { - const std::optional<LLT> Pointer = operandType(Function, Instruction, 1); - const std::optional<LLT> Offset = operandType(Function, Instruction, 2); + const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1); + const std::optional<LowLevelType> Offset = operandType(Function, Instruction, 2); return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 @@ -3737,7 +3737,7 @@ Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunct } if (Action == LegalizationAction::Custom && resultType(Function, Instruction) - .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) + .transform([](LowLevelType Type) { return Type.scalarBitWidth() > 64; }) .value_or(false)) { const Opcode Value = Instruction.opcode(); if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || @@ -3783,8 +3783,8 @@ Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunct if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { const MachineOperand *Immediate = Instruction.operandAt(1); - const std::optional<LLT> Type = resultType(Function, Instruction); - if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && + const std::optional<LowLevelType> Type = resultType(Function, Instruction); + if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp index 7c60219..66a3371 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp @@ -20,13 +20,9 @@ namespace rcc::riscv { namespace { -[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { - return Type.isFloat() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { +[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept { return (Type.isPointer() && Type.addressSpace() == 0 && Type.scalarBitWidth() == 64) || - ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() >= 1 && + (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() >= 1 && Type.scalarBitWidth() <= 64); } @@ -219,9 +215,9 @@ requiredBankForRegister(const MachineFunction &Function, Register Reg, "RISC-V selection requires a type for every virtual register"); } const VirtualRegisterInfo &RegisterInfo = Info->get(); - const LLT Type = RegisterInfo.Type.value_or(LLT{}); + const LowLevelType Type = RegisterInfo.Type.value_or(LowLevelType{}); RegisterClassID RequiredClass; - if ((RegisterInfo.Bank && *RegisterInfo.Bank == fprBank()) || Type.isFloat()) { + if (RegisterInfo.Bank && *RegisterInfo.Bank == fprBank()) { if (Type.scalarBitWidth() != 32 && Type.scalarBitWidth() != 64) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V RV64GC has no floating register class for type '" + @@ -269,8 +265,7 @@ RISCVRegisterBankInfo::selectBank(const MachineFunction &Function, Register Reg, if (!Required) return std::unexpected(Required.error()); const bool IsConstrained = Required->has_value() || Info.Bank.has_value(); - const RegisterBankID Selected = - Required->value_or(isRVFloatType(*Info.Type) ? fprBank() : gprBank()); + const RegisterBankID Selected = Required->value_or(gprBank()); if (Info.Bank && *Info.Bank != Selected) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V virtual register has a conflicting register bank"); diff --git a/backend/lib/Target/RISCV/RISCV.rccdesc b/backend/lib/Target/RISCV/RISCV.rccdesc index 910ea3f..91cad89 100644 --- a/backend/lib/Target/RISCV/RISCV.rccdesc +++ b/backend/lib/Target/RISCV/RISCV.rccdesc @@ -1923,7 +1923,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -1941,7 +1941,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -1959,7 +1959,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -1977,7 +1977,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -1994,7 +1994,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2011,7 +2011,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2028,7 +2028,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2045,7 +2045,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_DIV encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2061,7 +2061,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_SQRT encoding_category: R assembly_format: FRD_FRS1_RM @@ -2121,7 +2121,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2 @@ -2136,7 +2136,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2 @@ -2152,7 +2152,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2168,7 +2168,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2184,7 +2184,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2200,7 +2200,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2229,7 +2229,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2244,7 +2244,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2259,7 +2259,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR32, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2289,7 +2289,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2305,7 +2305,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2321,7 +2321,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2337,7 +2337,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2399,7 +2399,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -2417,7 +2417,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -2435,7 +2435,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -2453,7 +2453,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R4 assembly_format: FRD_FRS1_FRS2_FRS3_RM @@ -2470,7 +2470,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2487,7 +2487,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2504,7 +2504,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_MUL encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2521,7 +2521,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_DIV encoding_category: R assembly_format: FRD_FRS1_FRS2_RM @@ -2537,7 +2537,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_SQRT encoding_category: R assembly_format: FRD_FRS1_RM @@ -2597,7 +2597,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2 @@ -2612,7 +2612,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: FRD_FRS1_FRS2 @@ -2628,7 +2628,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2644,7 +2644,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2660,7 +2660,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2676,7 +2676,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: RD_FRS1_RM @@ -2705,7 +2705,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2720,7 +2720,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2735,7 +2735,7 @@ instructions: - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS1 } - { kinds: [register], role: use, register_class: FPR64, register_bank: FPR, name: FRS2 } num_explicit_defs: 1 - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_ADD encoding_category: R assembly_format: RD_FRS1_FRS2 @@ -2797,7 +2797,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2813,7 +2813,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_RS1_RM @@ -2843,7 +2843,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_FRS1_RM @@ -3373,7 +3373,7 @@ instructions: - { kinds: [immediate], immediate_predicate: RM, name: RM } num_explicit_defs: 1 allow_extra_implicit_operands: true - flags: [produces_code] + flags: [may_raise_fp_exception, produces_code] scheduling_class: FP_CONVERT encoding_category: R assembly_format: FRD_FRS1_RM diff --git a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp index d5d932c..b9cdb48 100644 --- a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp +++ b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp @@ -45,7 +45,8 @@ registerBank(Register Reg, const MachineFunction &Function, const TargetDescript return std::nullopt; } -[[nodiscard]] std::optional<LLT> registerType(Register Reg, const MachineFunction &Function) { +[[nodiscard]] std::optional<LowLevelType> registerType(Register Reg, + const MachineFunction &Function) { if (!Reg.isVirtual()) return std::nullopt; auto Info = Function.registerInfo().virtualRegisterInfo(Reg); @@ -156,7 +157,7 @@ copyRegisters(const MachineInstr &Instruction) { const auto DestinationType = registerType(Registers->first, Function); const auto SourceType = registerType(Registers->second, Function); - const std::optional<LLT> Type = DestinationType ? DestinationType : SourceType; + const std::optional<LowLevelType> Type = DestinationType ? DestinationType : SourceType; if (!Type || Type->isVector() || (Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64)) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V cross-bank COPY requires a scalar 32-bit or 64-bit type"); diff --git a/backend/lib/Tooling/MIRFuzzer.cpp b/backend/lib/Tooling/MIRFuzzer.cpp index 467291c..dc6b8d1 100644 --- a/backend/lib/Tooling/MIRFuzzer.cpp +++ b/backend/lib/Tooling/MIRFuzzer.cpp @@ -186,7 +186,7 @@ private: MachineBasicBlock *Block = Function->firstBlock(); if (Block == nullptr) return false; - auto Type = LLT::makeInteger(64); + auto Type = LowLevelType::makeCategoryNeutralScalar(64); if (!Type) return std::unexpected(Type.error()); auto Register = Function->registerInfo().createVirtualRegister(*Type); |