diff options
Diffstat (limited to 'backend/lib/MIR')
| -rw-r--r-- | backend/lib/MIR/CMakeLists.txt | 8 | ||||
| -rw-r--r-- | backend/lib/MIR/MILexer.cpp | 42 | ||||
| -rw-r--r-- | backend/lib/MIR/MIParser.cpp | 3043 | ||||
| -rw-r--r-- | backend/lib/MIR/MIPrinter.cpp | 1464 |
4 files changed, 4545 insertions, 12 deletions
diff --git a/backend/lib/MIR/CMakeLists.txt b/backend/lib/MIR/CMakeLists.txt index ca213aa..7f66685 100644 --- a/backend/lib/MIR/CMakeLists.txt +++ b/backend/lib/MIR/CMakeLists.txt @@ -4,11 +4,15 @@ set( RCCBackendMIRSources EmbeddedIR.cpp MILexer.cpp + MIParser.cpp + MIPrinter.cpp ) set( RCCBackendMIRHeaders "${RCC_BACKEND_SOURCE_DIR}/include/rcc/MIR/EmbeddedIR.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/MIR/MILexer.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/MIR/MIParser.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/MIR/MIPrinter.h" ) add_library(RCCBackendMIR ${RCCBackendMIRSources}) @@ -42,9 +46,13 @@ install( rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/EmbeddedIR.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MILexer.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MIParser.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MIPrinter.cpp" ${RCCBackendMIRHeaders} ) rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/EmbeddedIR.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MILexer.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MIParser.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MIPrinter.cpp" ) diff --git a/backend/lib/MIR/MILexer.cpp b/backend/lib/MIR/MILexer.cpp index 816264e..f568cef 100644 --- a/backend/lib/MIR/MILexer.cpp +++ b/backend/lib/MIR/MILexer.cpp @@ -569,14 +569,25 @@ private: ++Offset; const std::string_view Number = slice(NumberBegin, Offset); - std::size_t NameBegin = Offset; - if (peek() == '.') { - ++Offset; - NameBegin = Offset; - while (!atEnd() && isIdentifierCharacter(peek())) - ++Offset; + if (peek() != '.') { + return stringToken(IsReference ? MITokenKind::MachineBasicBlock + : MITokenKind::MachineBasicBlockLabel, + Begin, Offset, {}, Number); } + ++Offset; + const std::size_t NameBegin = Offset; + if (peek() == '"') { + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + return ownedStringToken(IsReference ? MITokenKind::MachineBasicBlock + : MITokenKind::MachineBasicBlockLabel, + Begin, Offset, std::move(Quoted->Value), Number); + } + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; return stringToken(IsReference ? MITokenKind::MachineBasicBlock : MITokenKind::MachineBasicBlockLabel, Begin, Offset, slice(NameBegin, Offset), Number); @@ -597,13 +608,20 @@ private: if (!HasOptionalName) return token(Kind, Begin, Offset, Number); - std::size_t NameBegin = Offset; - if (peek() == '.') { - ++Offset; - NameBegin = Offset; - while (!atEnd() && isIdentifierCharacter(peek())) - ++Offset; + if (peek() != '.') + return stringToken(Kind, Begin, Offset, {}, Number); + + ++Offset; + const std::size_t NameBegin = Offset; + if (peek() == '"') { + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + return ownedStringToken(Kind, Begin, Offset, std::move(Quoted->Value), Number); } + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; return stringToken(Kind, Begin, Offset, slice(NameBegin, Offset), Number); } diff --git a/backend/lib/MIR/MIParser.cpp b/backend/lib/MIR/MIParser.cpp new file mode 100644 index 0000000..b632415 --- /dev/null +++ b/backend/lib/MIR/MIParser.cpp @@ -0,0 +1,3043 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/MIR/MIParser.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/MIR/MILexer.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <charconv> +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <limits> +#include <memory> +#include <optional> +#include <span> +#include <string> +#include <string_view> +#include <system_error> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc::mir { +namespace { + +[[nodiscard]] bool parseUnsignedDecimal(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.empty() || Spelling.front() == '-') + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseSignedDecimal(std::string_view Spelling, std::int64_t &Value) { + if (Spelling.empty()) + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseHexadecimal(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.size() < 3 || Spelling[0] != '0' || (Spelling[1] != 'x' && Spelling[1] != 'X')) + return false; + const char *Begin = Spelling.data() + 2; + const char *End = Spelling.data() + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 16); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseUnsignedDigits(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.empty()) + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] std::optional<FloatSemantics> floatSemanticsForKeyword(std::string_view Keyword) { + if (Keyword == "half") + return FloatSemantics::IEEEHalf; + if (Keyword == "bfloat") + return FloatSemantics::BFloat; + if (Keyword == "float") + return FloatSemantics::IEEESingle; + if (Keyword == "double") + return FloatSemantics::IEEEDouble; + if (Keyword == "x86_fp80") + return FloatSemantics::X87DoubleExtended; + if (Keyword == "fp128") + return FloatSemantics::IEEEQuad; + if (Keyword == "ppc_fp128") + return FloatSemantics::PPCDoubleDouble; + return std::nullopt; +} + +[[nodiscard]] bool isFloatingSemanticsKeyword(const MIToken &Token) { + return Token.kind() == MITokenKind::Keyword && + floatSemanticsForKeyword(Token.stringValue()).has_value(); +} + +[[nodiscard]] bool isCFIOperationToken(const MIToken &Token) { + return Token.isKeyword("same_value") || Token.isKeyword("remember_state") || + Token.isKeyword("restore_state") || Token.isKeyword("offset") || + Token.isKeyword("llvm_def_aspace_cfa") || Token.isKeyword("def_cfa_register") || + Token.isKeyword("def_cfa_offset") || Token.isKeyword("def_cfa") || + Token.isKeyword("rel_offset") || Token.isKeyword("adjust_cfa_offset") || + Token.isKeyword("escape") || Token.isKeyword("restore") || Token.isKeyword("undefined") || + Token.isKeyword("register") || Token.isKeyword("window_save") || + Token.isKeyword("negate_ra_sign_state") || + Token.isKeyword("negate_ra_sign_state_with_pc") || Token.isKeyword("llvm_register_pair") || + Token.isKeyword("llvm_vector_registers") || Token.isKeyword("llvm_vector_offset") || + Token.isKeyword("llvm_vector_register_mask"); +} + +[[nodiscard]] std::optional<unsigned> hexDigit(char Character) noexcept { + if (Character >= '0' && Character <= '9') + return static_cast<unsigned>(Character - '0'); + if (Character >= 'a' && Character <= 'f') + return static_cast<unsigned>(Character - 'a') + 10U; + if (Character >= 'A' && Character <= 'F') + return static_cast<unsigned>(Character - 'A') + 10U; + return std::nullopt; +} + +[[nodiscard]] bool parseHexWords(std::string_view Digits, std::uint32_t BitWidth, + std::vector<std::uint64_t> &Words) { + if (Digits.empty() || Digits.size() > (static_cast<std::size_t>(BitWidth) + 3U) / 4U) + return false; + Words.assign((static_cast<std::size_t>(BitWidth) + 63U) / 64U, 0); + std::size_t BitOffset = 0; + for (auto Iterator = Digits.rbegin(); Iterator != Digits.rend(); ++Iterator) { + const auto Digit = hexDigit(*Iterator); + if (!Digit) + return false; + const std::size_t WordIndex = BitOffset / 64U; + const unsigned Shift = static_cast<unsigned>(BitOffset % 64U); + Words[WordIndex] |= static_cast<std::uint64_t>(*Digit) << Shift; + BitOffset += 4U; + } + if (BitWidth % 64U != 0) + Words.back() &= (std::uint64_t{1} << (BitWidth % 64U)) - 1U; + return true; +} + +[[nodiscard]] bool isZeroWords(std::span<const std::uint64_t> Words) noexcept { + return std::ranges::all_of(Words, [](std::uint64_t Word) { return Word == 0; }); +} + +[[nodiscard]] bool magnitudeFitsSignedNegative(std::span<const std::uint64_t> Words, + std::uint32_t BitWidth) noexcept { + const std::size_t SignWord = (BitWidth - 1U) / 64U; + const unsigned SignBit = (BitWidth - 1U) % 64U; + for (std::size_t Index = SignWord + 1U; Index < Words.size(); ++Index) { + if (Words[Index] != 0) + return false; + } + const std::uint64_t SignMask = std::uint64_t{1} << SignBit; + if ((Words[SignWord] & ~SignMask) != 0) + return false; + if ((Words[SignWord] & SignMask) == 0) + return true; + for (std::size_t Index = 0; Index < SignWord; ++Index) { + if (Words[Index] != 0) + return false; + } + return true; +} + +[[nodiscard]] Expected<IntegerImmediate> parseIntegerImmediateValue(std::uint32_t BitWidth, + std::string_view Spelling) { + if (BitWidth == 0 || BitWidth > IntegerImmediate::MaxBitWidth) + return makeUnexpected(ErrorCode::ParseError, "typed integer width is out of range"); + + const bool Negative = Spelling.starts_with('-'); + if (Negative) + Spelling.remove_prefix(1); + if (Spelling.empty()) + return makeUnexpected(ErrorCode::ParseError, "typed integer has no digits"); + + std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U, 0); + for (char Character : Spelling) { + if (Character < '0' || Character > '9') + return makeUnexpected(ErrorCode::ParseError, "typed integer contains a non-decimal digit"); + std::uint64_t Carry = static_cast<unsigned>(Character - '0'); + for (std::uint64_t &Word : Words) { + const unsigned __int128 Product = static_cast<unsigned __int128>(Word) * 10U + Carry; + Word = static_cast<std::uint64_t>(Product); + Carry = static_cast<std::uint64_t>(Product >> 64U); + } + if (Carry != 0) + return makeUnexpected(ErrorCode::ParseError, "typed integer does not fit its bit width"); + if (BitWidth % 64U != 0 && (Words.back() >> (BitWidth % 64U)) != 0) { + return makeUnexpected(ErrorCode::ParseError, "typed integer does not fit its bit width"); + } + } + + if (Negative) { + if (isZeroWords(Words) || !magnitudeFitsSignedNegative(Words, BitWidth)) + return makeUnexpected(ErrorCode::ParseError, + "negative typed integer does not fit its bit width"); + for (std::uint64_t &Word : Words) + Word = ~Word; + std::uint64_t Carry = 1; + for (std::uint64_t &Word : Words) { + const std::uint64_t Previous = Word; + Word += Carry; + Carry = Carry != 0 && Word < Previous ? 1 : 0; + } + if (BitWidth % 64U != 0) + Words.back() &= (std::uint64_t{1} << (BitWidth % 64U)) - 1U; + } + return IntegerImmediate::create(BitWidth, Words); +} + +[[nodiscard]] std::uint16_t floatToHalfBits(float Value) noexcept { + const std::uint32_t Bits = std::bit_cast<std::uint32_t>(Value); + const std::uint32_t Sign = (Bits >> 16U) & 0x8000U; + const std::uint32_t Exponent = (Bits >> 23U) & 0xFFU; + const std::uint32_t Fraction = Bits & 0x7FFFFFU; + if (Exponent == 0xFFU) { + const std::uint32_t Payload = Fraction == 0 ? 0 : std::max(1U, Fraction >> 13U); + return static_cast<std::uint16_t>(Sign | 0x7C00U | Payload); + } + + const int HalfExponent = static_cast<int>(Exponent) - 127 + 15; + if (HalfExponent >= 31) + return static_cast<std::uint16_t>(Sign | 0x7C00U); + if (HalfExponent <= 0) { + if (HalfExponent < -10) + return static_cast<std::uint16_t>(Sign); + const std::uint32_t Mantissa = Fraction | 0x800000U; + const unsigned Shift = static_cast<unsigned>(14 - HalfExponent); + std::uint32_t Rounded = Mantissa >> Shift; + const std::uint32_t Remainder = Mantissa & ((std::uint32_t{1} << Shift) - 1U); + const std::uint32_t Halfway = std::uint32_t{1} << (Shift - 1U); + if (Remainder > Halfway || (Remainder == Halfway && (Rounded & 1U) != 0)) + ++Rounded; + return static_cast<std::uint16_t>(Sign | Rounded); + } + + std::uint32_t Rounded = Fraction >> 13U; + const std::uint32_t Remainder = Fraction & 0x1FFFU; + if (Remainder > 0x1000U || (Remainder == 0x1000U && (Rounded & 1U) != 0)) { + ++Rounded; + if (Rounded == 0x400U) + return static_cast<std::uint16_t>(Sign | + (static_cast<std::uint32_t>(HalfExponent + 1) << 10U)); + } + return static_cast<std::uint16_t>(Sign | (static_cast<std::uint32_t>(HalfExponent) << 10U) | + Rounded); +} + +[[nodiscard]] std::uint16_t floatToBFloatBits(float Value) noexcept { + std::uint32_t Bits = std::bit_cast<std::uint32_t>(Value); + const std::uint32_t Bias = 0x7FFFU + ((Bits >> 16U) & 1U); + Bits += Bias; + return static_cast<std::uint16_t>(Bits >> 16U); +} + +[[nodiscard]] std::optional<std::uint64_t> debugExpressionOpcode(std::string_view Name) noexcept { + if (Name.starts_with("DW_OP_lit")) { + std::uint64_t Index = 0; + if (parseUnsignedDigits(Name.substr(9), Index) && Index <= 31) + return 0x30U + Index; + } + if (Name.starts_with("DW_OP_reg")) { + std::uint64_t Index = 0; + if (parseUnsignedDigits(Name.substr(9), Index) && Index <= 31) + return 0x50U + Index; + } + if (Name.starts_with("DW_OP_breg")) { + std::uint64_t Index = 0; + if (parseUnsignedDigits(Name.substr(10), Index) && Index <= 31) + return 0x70U + Index; + } + + constexpr auto NamedOpcodes = std::to_array<std::pair<std::string_view, std::uint64_t>>({ + {"DW_OP_addr", 0x03}, + {"DW_OP_deref", 0x06}, + {"DW_OP_const1u", 0x08}, + {"DW_OP_const1s", 0x09}, + {"DW_OP_const2u", 0x0A}, + {"DW_OP_const2s", 0x0B}, + {"DW_OP_const4u", 0x0C}, + {"DW_OP_const4s", 0x0D}, + {"DW_OP_const8u", 0x0E}, + {"DW_OP_const8s", 0x0F}, + {"DW_OP_constu", 0x10}, + {"DW_OP_consts", 0x11}, + {"DW_OP_dup", 0x12}, + {"DW_OP_drop", 0x13}, + {"DW_OP_over", 0x14}, + {"DW_OP_pick", 0x15}, + {"DW_OP_swap", 0x16}, + {"DW_OP_rot", 0x17}, + {"DW_OP_xderef", 0x18}, + {"DW_OP_abs", 0x19}, + {"DW_OP_and", 0x1A}, + {"DW_OP_div", 0x1B}, + {"DW_OP_minus", 0x1C}, + {"DW_OP_mod", 0x1D}, + {"DW_OP_mul", 0x1E}, + {"DW_OP_neg", 0x1F}, + {"DW_OP_not", 0x20}, + {"DW_OP_or", 0x21}, + {"DW_OP_plus", 0x22}, + {"DW_OP_plus_uconst", 0x23}, + {"DW_OP_shl", 0x24}, + {"DW_OP_shr", 0x25}, + {"DW_OP_shra", 0x26}, + {"DW_OP_xor", 0x27}, + {"DW_OP_bra", 0x28}, + {"DW_OP_eq", 0x29}, + {"DW_OP_ge", 0x2A}, + {"DW_OP_gt", 0x2B}, + {"DW_OP_le", 0x2C}, + {"DW_OP_lt", 0x2D}, + {"DW_OP_ne", 0x2E}, + {"DW_OP_skip", 0x2F}, + {"DW_OP_regx", 0x90}, + {"DW_OP_fbreg", 0x91}, + {"DW_OP_bregx", 0x92}, + {"DW_OP_piece", 0x93}, + {"DW_OP_deref_size", 0x94}, + {"DW_OP_xderef_size", 0x95}, + {"DW_OP_nop", 0x96}, + {"DW_OP_push_object_address", 0x97}, + {"DW_OP_call2", 0x98}, + {"DW_OP_call4", 0x99}, + {"DW_OP_call_ref", 0x9A}, + {"DW_OP_form_tls_address", 0x9B}, + {"DW_OP_call_frame_cfa", 0x9C}, + {"DW_OP_bit_piece", 0x9D}, + {"DW_OP_implicit_value", 0x9E}, + {"DW_OP_stack_value", 0x9F}, + {"DW_OP_implicit_pointer", 0xA0}, + {"DW_OP_addrx", 0xA1}, + {"DW_OP_constx", 0xA2}, + {"DW_OP_entry_value", 0xA3}, + {"DW_OP_const_type", 0xA4}, + {"DW_OP_regval_type", 0xA5}, + {"DW_OP_deref_type", 0xA6}, + {"DW_OP_xderef_type", 0xA7}, + {"DW_OP_convert", 0xA8}, + {"DW_OP_reinterpret", 0xA9}, + {"DW_OP_LLVM_fragment", 0x1000}, + {"DW_OP_LLVM_convert", 0x1001}, + {"DW_OP_LLVM_tag_offset", 0x1002}, + {"DW_OP_LLVM_entry_value", 0x1003}, + {"DW_OP_LLVM_implicit_pointer", 0x1004}, + {"DW_OP_LLVM_arg", 0x1005}, + }); + const auto *const Found = + std::ranges::find_if(NamedOpcodes, [Name](const auto &Entry) { return Entry.first == Name; }); + if (Found == NamedOpcodes.end()) + return std::nullopt; + return Found->second; +} + +[[nodiscard]] std::optional<std::uint64_t> minimumTypeSizeInBits(LLT Type) noexcept { + if (!Type.isValid() || Type.isToken()) + return std::nullopt; + const std::uint64_t ScalarSize = Type.scalarBitWidth(); + if (!Type.isVector()) + return ScalarSize; + const std::uint64_t ElementCount = Type.minimumElementCount(); + if (ScalarSize != 0 && ElementCount > std::numeric_limits<std::uint64_t>::max() / ScalarSize) + return std::nullopt; + return ScalarSize * ElementCount; +} + +[[nodiscard]] std::optional<AtomicOrdering> atomicOrdering(std::string_view Name) noexcept { + if (Name == "unordered") + return AtomicOrdering::Unordered; + if (Name == "monotonic") + return AtomicOrdering::Monotonic; + if (Name == "acquire") + return AtomicOrdering::Acquire; + if (Name == "release") + return AtomicOrdering::Release; + if (Name == "acq_rel") + return AtomicOrdering::AcquireRelease; + if (Name == "seq_cst") + return AtomicOrdering::SequentiallyConsistent; + return std::nullopt; +} + +[[nodiscard]] bool isInstructionFlag(const MIToken &Token) { + return Token.isKeyword("frame-setup") || Token.isKeyword("frame-destroy") || + Token.isKeyword("nnan") || Token.isKeyword("ninf") || Token.isKeyword("nsz") || + Token.isKeyword("arcp") || Token.isKeyword("contract") || Token.isKeyword("afn") || + Token.isKeyword("reassoc") || Token.isKeyword("nuw") || Token.isKeyword("nsw") || + Token.isKeyword("nusw") || Token.isKeyword("exact") || Token.isKeyword("nofpexcept") || + Token.isKeyword("unpredictable") || Token.isKeyword("noconvergent") || + Token.isKeyword("nneg") || Token.isKeyword("disjoint") || Token.isKeyword("samesign") || + Token.isKeyword("inbounds"); +} + +[[nodiscard]] std::optional<MachineInstrFlag> instructionFlag(std::string_view Name) { + if (Name == "frame-setup") + return MachineInstrFlag::FrameSetup; + if (Name == "frame-destroy") + return MachineInstrFlag::FrameDestroy; + if (Name == "nnan") + return MachineInstrFlag::FastMathNoNaNs; + if (Name == "ninf") + return MachineInstrFlag::FastMathNoInfs; + if (Name == "nsz") + return MachineInstrFlag::FastMathNoSignedZeros; + if (Name == "arcp") + return MachineInstrFlag::FastMathAllowReciprocal; + if (Name == "contract") + return MachineInstrFlag::FastMathAllowContract; + if (Name == "afn") + return MachineInstrFlag::FastMathApproxFunc; + if (Name == "reassoc") + return MachineInstrFlag::FastMathAllowReassoc; + if (Name == "nuw") + return MachineInstrFlag::NoUnsignedWrap; + if (Name == "nsw") + return MachineInstrFlag::NoSignedWrap; + if (Name == "nusw") + return MachineInstrFlag::NoUSWrap; + if (Name == "exact") + return MachineInstrFlag::Exact; + if (Name == "nofpexcept") + return MachineInstrFlag::NoFPExcept; + if (Name == "unpredictable") + return MachineInstrFlag::Unpredictable; + if (Name == "noconvergent") + return MachineInstrFlag::NoConvergent; + if (Name == "nneg") + return MachineInstrFlag::NonNeg; + if (Name == "disjoint") + return MachineInstrFlag::Disjoint; + if (Name == "samesign") + return MachineInstrFlag::SameSign; + if (Name == "inbounds") + return MachineInstrFlag::InBounds; + return std::nullopt; +} + +[[nodiscard]] std::optional<IntegerPredicate> integerPredicate(std::string_view Name) { + if (Name == "eq") + return IntegerPredicate::Equal; + if (Name == "ne") + return IntegerPredicate::NotEqual; + if (Name == "sgt") + return IntegerPredicate::SignedGreaterThan; + if (Name == "sge") + return IntegerPredicate::SignedGreaterEqual; + if (Name == "slt") + return IntegerPredicate::SignedLessThan; + if (Name == "sle") + return IntegerPredicate::SignedLessEqual; + if (Name == "ugt") + return IntegerPredicate::UnsignedGreaterThan; + if (Name == "uge") + return IntegerPredicate::UnsignedGreaterEqual; + if (Name == "ult") + return IntegerPredicate::UnsignedLessThan; + if (Name == "ule") + return IntegerPredicate::UnsignedLessEqual; + return std::nullopt; +} + +[[nodiscard]] std::optional<FloatingPredicate> floatingPredicate(std::string_view Name) { + if (Name == "false") + return FloatingPredicate::AlwaysFalse; + if (Name == "oeq") + return FloatingPredicate::OrderedEqual; + if (Name == "ogt") + return FloatingPredicate::OrderedGreaterThan; + if (Name == "oge") + return FloatingPredicate::OrderedGreaterEqual; + if (Name == "olt") + return FloatingPredicate::OrderedLessThan; + if (Name == "ole") + return FloatingPredicate::OrderedLessEqual; + if (Name == "one") + return FloatingPredicate::OrderedNotEqual; + if (Name == "ord") + return FloatingPredicate::Ordered; + if (Name == "uno") + return FloatingPredicate::Unordered; + if (Name == "ueq") + return FloatingPredicate::UnorderedEqual; + if (Name == "ugt") + return FloatingPredicate::UnorderedGreaterThan; + if (Name == "uge") + return FloatingPredicate::UnorderedGreaterEqual; + if (Name == "ult") + return FloatingPredicate::UnorderedLessThan; + if (Name == "ule") + return FloatingPredicate::UnorderedLessEqual; + if (Name == "une") + return FloatingPredicate::UnorderedNotEqual; + if (Name == "true") + return FloatingPredicate::AlwaysTrue; + return std::nullopt; +} + +} // namespace + +class MIParser::Impl final { +public: + Impl(MachineFunction &Function, const TargetDescription &Target, std::string SourceName, + std::string SourceText, MIParserContext Context, MIParserOptions Options) + : Function(Function), Target(Target), + Lexer(std::move(SourceName), std::move(SourceText), + MILexerOptions{.MaximumTokenLength = Options.MaximumTokenLength, + .MaximumTokens = Options.MaximumTokens}), + Context(std::move(Context)), Options(Options) {} + + [[nodiscard]] Expected<void> parse(); + + [[nodiscard]] const TextSource &source() const noexcept { + return Lexer.source(); + } + + [[nodiscard]] std::span<const TextDiagnostic> diagnostics() const noexcept { + return Diagnostics; + } + + [[nodiscard]] bool failed() const noexcept { + return !Diagnostics.empty(); + } + +private: + struct ParsedOperand final { + MachineOperand Operand; + std::optional<std::size_t> TiedDef; + TextRange Range; + }; + + [[nodiscard]] const MIToken &token() const noexcept { + return Tokens[Position]; + } + + [[nodiscard]] bool at(MITokenKind Kind) const noexcept { + return token().is(Kind); + } + + [[nodiscard]] bool nextIs(MITokenKind Kind) const noexcept { + return Position + 1U < Tokens.size() && Tokens[Position + 1U].is(Kind); + } + + [[nodiscard]] bool atKeyword(std::string_view Keyword) const noexcept { + return token().isKeyword(Keyword); + } + + void advance() noexcept { + if (!at(MITokenKind::EndOfFile)) + ++Position; + } + + [[nodiscard]] bool consume(MITokenKind Kind) noexcept { + if (!at(Kind)) + return false; + advance(); + return true; + } + + [[nodiscard]] bool expect(MITokenKind Kind, std::string_view Description); + [[nodiscard]] bool fail(const MIToken &At, std::string Message); + [[nodiscard]] bool fail(std::string Message) { + return fail(token(), std::move(Message)); + } + [[nodiscard]] Expected<void> failureResult() const; + [[nodiscard]] bool importError(const MIToken &At, const Error &Failure); + [[nodiscard]] bool tokenize(); + + [[nodiscard]] bool parseUnsigned(const MIToken &Token, std::uint64_t &Value); + [[nodiscard]] bool parseUnsigned32(const MIToken &Token, std::uint32_t &Value); + [[nodiscard]] bool parseSigned64(const MIToken &Token, std::int64_t &Value); + [[nodiscard]] bool parseOptionalOffset(std::int64_t &Offset); + [[nodiscard]] bool parseAlignment(Alignment &Result); + [[nodiscard]] bool parseLowLevelType(LLT &Result); + [[nodiscard]] bool parseScalarLowLevelType(const MIToken &Token, LLT &Result); + + [[nodiscard]] bool parseFirstPass(); + [[nodiscard]] bool parseSecondPass(); + [[nodiscard]] bool parseBlockDefinition(bool Create, std::uint32_t &ID); + [[nodiscard]] bool parseBlockAttributes(MachineBasicBlock *Block); + [[nodiscard]] bool parseBlockLiveIns(MachineBasicBlock &Block); + [[nodiscard]] bool parseBlockSuccessors(MachineBasicBlock &Block); + [[nodiscard]] bool parseInstruction(MachineBasicBlock &Block, bool InBundle, + MachineInstr *PreviousInstruction, MachineInstr *&Result); + [[nodiscard]] bool inferSuccessors(); + + [[nodiscard]] bool parseRegister(Register &Result); + [[nodiscard]] bool parseRegisterOperand(bool IsDef, ParsedOperand &Result); + [[nodiscard]] bool parseTargetFlags(std::uint32_t &Flags); + [[nodiscard]] bool parseOperand(bool IsDef, ParsedOperand &Result); + [[nodiscard]] bool parseTypedIntegerOperand(ParsedOperand &Result); + [[nodiscard]] bool parseFloatingImmediateOperand(ParsedOperand &Result); + [[nodiscard]] bool parseMetadataReference(MachineMetadataReference &Result); + [[nodiscard]] bool parseMetadataOperand(ParsedOperand &Result); + [[nodiscard]] bool + parseInlineDILocation(std::vector<MachineDILocationFrame> &InlineChain, + std::optional<MachineMetadataReference> &TerminalInlinedAt); + [[nodiscard]] bool parseCFIRegister(DwarfRegisterNumber &Result); + [[nodiscard]] bool parseCFISigned(std::int64_t &Result); + [[nodiscard]] bool parseCFIUnsigned(std::uint32_t &Result); + [[nodiscard]] bool parseCFIOperand(ParsedOperand &Result); + [[nodiscard]] bool parseMemoryBase(MemoryBase &Result, std::int64_t &Offset); + [[nodiscard]] bool parseMemoryOperand(const MachineMemOperand *&Result); + [[nodiscard]] bool parsePredicateOperand(ParsedOperand &Result, bool IsFloating); + [[nodiscard]] bool parseShuffleMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseDebugInstrRefOperand(ParsedOperand &Result); + [[nodiscard]] bool parseLaneMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseIntrinsicOperand(ParsedOperand &Result); + [[nodiscard]] bool parseTargetIndexOperand(ParsedOperand &Result); + [[nodiscard]] bool parseLiveOutMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseCustomMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseBlockAddressOperand(ParsedOperand &Result); + + [[nodiscard]] Register getOrCreateVirtualRegister(const MIToken &Token); + [[nodiscard]] MachineBasicBlock *blockFor(const MIToken &Token); + [[nodiscard]] std::optional<FrameIndex> frameIndexFor(const MIToken &Token, bool Fixed); + [[nodiscard]] std::optional<ConstantPoolIndex> constantPoolIndexFor(const MIToken &Token); + [[nodiscard]] std::optional<JumpTableIndex> jumpTableIndexFor(const MIToken &Token); + [[nodiscard]] std::string symbolName(const MIToken &Token) const; + + MachineFunction &Function; + const TargetDescription &Target; + MILexer Lexer; + MIParserContext Context; + MIParserOptions Options; + std::vector<MIToken> Tokens; + std::size_t Position = 0; + std::vector<TextDiagnostic> Diagnostics; + std::unordered_map<std::uint32_t, MachineBasicBlock *> Blocks; + std::unordered_map<std::uint32_t, Register> NumericVirtualRegisters; + std::unordered_map<std::string, Register> NamedVirtualRegisters; + std::unordered_set<MachineBasicBlock *> ExplicitSuccessorBlocks; + std::size_t ParsedInstructionCount = 0; + bool ParseAttempted = false; +}; + +bool MIParser::Impl::expect(MITokenKind Kind, std::string_view Description) { + if (consume(Kind)) + return false; + return fail("expected " + std::string(Description) + ", found " + + std::string(getMITokenKindName(token().kind()))); +} + +bool MIParser::Impl::fail(const MIToken &At, std::string Message) { + if (Diagnostics.empty()) { + Diagnostics.push_back(TextDiagnostic{ + .Severity = DiagnosticSeverity::Error, + .Category = "mir-parser", + .Message = std::move(Message), + .SourceName = std::string(Lexer.source().name()), + .Range = At.range(), + }); + } + return true; +} + +Expected<void> MIParser::Impl::failureResult() const { + const std::string Message = + Diagnostics.empty() ? "machine instruction parsing failed" : Diagnostics.front().Message; + return makeUnexpected(ErrorCode::ParseError, Message); +} + +bool MIParser::Impl::importError(const MIToken &At, const Error &Failure) { + return fail(At, std::string(Failure.message())); +} + +bool MIParser::Impl::tokenize() { + while (true) { + MIToken Token = Lexer.next(); + const MITokenKind Kind = Token.kind(); + Tokens.push_back(std::move(Token)); + if (Kind == MITokenKind::Error) { + const auto LexerDiagnostics = Lexer.diagnostics(); + Diagnostics.assign(LexerDiagnostics.begin(), LexerDiagnostics.end()); + return true; + } + if (Kind == MITokenKind::EndOfFile) + return false; + } +} + +bool MIParser::Impl::parseUnsigned(const MIToken &Token, std::uint64_t &Value) { + if (Token.kind() == MITokenKind::HexLiteral) + return !parseHexadecimal(Token.spelling(), Value); + if (Token.hasIntegerValue()) + return !parseUnsignedDecimal(Token.integerValue(), Value); + return true; +} + +bool MIParser::Impl::parseUnsigned32(const MIToken &Token, std::uint32_t &Value) { + std::uint64_t Wide = 0; + if (parseUnsigned(Token, Wide) || Wide > std::numeric_limits<std::uint32_t>::max()) + return true; + Value = static_cast<std::uint32_t>(Wide); + return false; +} + +bool MIParser::Impl::parseSigned64(const MIToken &Token, std::int64_t &Value) { + if (Token.kind() != MITokenKind::IntegerLiteral) + return true; + if (parseSignedDecimal(Token.integerValue(), Value)) + return false; + std::uint64_t Unsigned = 0; + if (!parseUnsignedDecimal(Token.integerValue(), Unsigned)) + return true; + Value = std::bit_cast<std::int64_t>(Unsigned); + return false; +} + +bool MIParser::Impl::parseOptionalOffset(std::int64_t &Offset) { + Offset = 0; + if (!at(MITokenKind::Plus) && !at(MITokenKind::Minus)) + return false; + const bool Negative = at(MITokenKind::Minus); + advance(); + std::uint64_t Magnitude = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Magnitude) || + Magnitude > (std::uint64_t{1} << 63U)) { + return fail("expected a 64-bit integer offset"); + } + if (Negative) { + if (Magnitude == (std::uint64_t{1} << 63U)) + Offset = std::numeric_limits<std::int64_t>::min(); + else + Offset = -static_cast<std::int64_t>(Magnitude); + } else { + if (Magnitude > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return fail("positive offset exceeds signed 64-bit range"); + Offset = static_cast<std::int64_t>(Magnitude); + } + advance(); + return false; +} + +bool MIParser::Impl::parseAlignment(Alignment &Result) { + advance(); + std::uint64_t Bytes = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Bytes)) + return fail("expected an unsigned alignment"); + auto Parsed = Alignment::create(Bytes); + if (!Parsed) + return importError(token(), Parsed.error()); + Result = *Parsed; + advance(); + return false; +} + +bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Result) { + if (TypeToken.kind() != MITokenKind::Identifier) + return fail(TypeToken, "expected a scalar low-level type"); + std::string_view Spelling = TypeToken.stringValue(); + enum class ScalarKind : std::uint8_t { + Any, + Integer, + Float, + BFloat, + Pointer, + }; + ScalarKind Kind = ScalarKind::Any; + std::string_view Digits; + if (Spelling.starts_with("bf")) { + Kind = ScalarKind::BFloat; + Digits = Spelling.substr(2); + } else if (Spelling.starts_with('s')) { + Kind = ScalarKind::Any; + Digits = Spelling.substr(1); + } else if (Spelling.starts_with('i')) { + Kind = ScalarKind::Integer; + Digits = Spelling.substr(1); + } else if (Spelling.starts_with('f')) { + Kind = ScalarKind::Float; + Digits = Spelling.substr(1); + } else if (Spelling.starts_with('p')) { + Kind = ScalarKind::Pointer; + Digits = Spelling.substr(1); + } else { + return fail(TypeToken, "unknown scalar low-level type '" + std::string(Spelling) + "'"); + } + + std::uint64_t Value = 0; + if (!parseUnsignedDigits(Digits, Value)) + return fail(TypeToken, "low-level type suffix must be an unsigned decimal integer"); + + Expected<LLT> Created = makeUnexpected(ErrorCode::ParseError, "invalid low-level type"); + switch (Kind) { + case ScalarKind::Any: + if (Value == 0) { + Result = LLT::token(); + return false; + } + Created = LLT::makeAnyScalar(Value); + break; + case ScalarKind::Integer: + Created = LLT::makeInteger(Value); + break; + case ScalarKind::Float: { + std::optional<FloatSemantics> Semantics; + if (Value == 16) + Semantics = FloatSemantics::IEEEHalf; + else if (Value == 32) + Semantics = FloatSemantics::IEEESingle; + else if (Value == 64) + Semantics = FloatSemantics::IEEEDouble; + else if (Value == 128) + Semantics = FloatSemantics::IEEEQuad; + if (!Semantics) + return fail(TypeToken, "unsupported IEEE floating low-level type width"); + Created = LLT::makeFloat(*Semantics); + break; + } + case ScalarKind::BFloat: + if (Value != 16) + return fail(TypeToken, "bfloat low-level type must have width 16"); + Created = LLT::makeFloat(FloatSemantics::BFloat); + break; + case ScalarKind::Pointer: + if (Context.DefaultPointerSizeInBits == 0) + return fail(TypeToken, "pointer low-level type requires an explicit pointer size"); + Created = LLT::makePointer(Value, Context.DefaultPointerSizeInBits); + break; + } + if (!Created) + return importError(TypeToken, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parseLowLevelType(LLT &Result) { + const MIToken Begin = token(); + if (at(MITokenKind::Identifier)) { + if (parseScalarLowLevelType(token(), Result)) + return true; + advance(); + return false; + } + if (!consume(MITokenKind::Less)) + return fail(Begin, "expected a scalar, pointer, or vector low-level type"); + + bool Scalable = false; + if (at(MITokenKind::Identifier) && token().stringValue() == "vscale") { + Scalable = true; + advance(); + if (!at(MITokenKind::Identifier) || token().stringValue() != "x") + return fail("expected 'x' after 'vscale' in a scalable vector type"); + advance(); + } + + std::uint64_t ElementCount = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), ElementCount)) + return fail("expected an unsigned vector element count"); + advance(); + if (!at(MITokenKind::Identifier) || token().stringValue() != "x") + return fail("expected 'x' after the vector element count"); + advance(); + + LLT ElementType; + if (parseScalarLowLevelType(token(), ElementType)) + return true; + if (ElementType.isToken()) + return fail("token cannot be a vector element type"); + advance(); + if (expect(MITokenKind::Greater, "'>' after the vector low-level type")) + return true; + + Expected<LLT> Created = Scalable ? LLT::makeScalableVector(ElementCount, ElementType) + : LLT::makeFixedVector(ElementCount, ElementType); + if (!Created) + return importError(Begin, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parseBlockAttributes(MachineBasicBlock *Block) { + std::unordered_set<std::string> Seen; + while (!at(MITokenKind::RightParen)) { + const MIToken AttributeToken = token(); + if (atKeyword("machine-block-address-taken") || atKeyword("landing-pad") || + atKeyword("inlineasm-br-indirect-target") || atKeyword("ehfunclet-entry") || + atKeyword("ehscope-entry")) { + const std::string Name(AttributeToken.stringValue()); + if (!Seen.insert(Name).second) + return fail("duplicate basic block attribute '" + Name + "'"); + if (Block != nullptr) { + if (Name == "machine-block-address-taken") + Block->attributes().set(MachineBasicBlockAttribute::MachineBlockAddressTaken); + else if (Name == "landing-pad") + Block->attributes().set(MachineBasicBlockAttribute::LandingPad); + else if (Name == "inlineasm-br-indirect-target") + Block->attributes().set(MachineBasicBlockAttribute::InlineAsmBrIndirectTarget); + else if (Name == "ehfunclet-entry") + Block->attributes().set(MachineBasicBlockAttribute::EHFuncletEntry); + else + Block->attributes().set(MachineBasicBlockAttribute::EHScopeEntry); + } + advance(); + } else if (atKeyword("ir-block-address-taken")) { + if (!Seen.insert("ir-block-address-taken").second) + return fail("duplicate basic block attribute 'ir-block-address-taken'"); + advance(); + if (!at(MITokenKind::IRBlock) && !at(MITokenKind::NamedIRBlock)) + return fail("expected an IR block after 'ir-block-address-taken'"); + if (Block != nullptr) { + Block->attributes().set(MachineBasicBlockAttribute::IRBlockAddressTaken); + auto SetName = Block->attributes().setIRBlockName(symbolName(token())); + if (!SetName) + return importError(token(), SetName.error()); + } + advance(); + } else if (at(MITokenKind::IRBlock) || at(MITokenKind::NamedIRBlock)) { + if (!Seen.insert("ir-block").second) + return fail("duplicate IR block attribute"); + if (Block != nullptr) { + auto SetName = Block->attributes().setIRBlockName(symbolName(token())); + if (!SetName) + return importError(token(), SetName.error()); + } + advance(); + } else if (atKeyword("align")) { + if (!Seen.insert("align").second) + return fail("duplicate basic block alignment"); + Alignment Align; + if (parseAlignment(Align)) + return true; + if (Block != nullptr) + Block->attributes().setAlignment(Align); + } else if (atKeyword("bbsections")) { + if (!Seen.insert("bbsections").second) + return fail("duplicate basic block section"); + advance(); + MachineBasicBlockSectionID Section; + std::uint32_t Number = 0; + if (at(MITokenKind::IntegerLiteral) && !parseUnsigned32(token(), Number)) + Section = MachineBasicBlockSectionID::numbered(Number); + else if (at(MITokenKind::Identifier) && token().stringValue() == "Exception") + Section = MachineBasicBlockSectionID::exception(); + else if (at(MITokenKind::Identifier) && token().stringValue() == "Cold") + Section = MachineBasicBlockSectionID::cold(); + else + return fail("unknown basic block section ID"); + if (Block != nullptr) + Block->attributes().setSectionID(Section); + advance(); + } else if (atKeyword("bb_id")) { + if (!Seen.insert("bb_id").second) + return fail("duplicate basic block profile ID"); + advance(); + std::uint32_t BaseID = 0; + std::uint32_t CloneID = 0; + if (at(MITokenKind::FloatingPointLiteral)) { + const std::string_view Spelling = token().spelling(); + const std::size_t Dot = Spelling.find('.'); + std::uint64_t Base = 0; + std::uint64_t Clone = 0; + if (Dot == std::string_view::npos || !parseUnsignedDecimal(Spelling.substr(0, Dot), Base) || + !parseUnsignedDecimal(Spelling.substr(Dot + 1), Clone) || + Base > std::numeric_limits<std::uint32_t>::max() || + Clone > std::numeric_limits<std::uint32_t>::max()) { + return fail("invalid basic block profile ID"); + } + BaseID = static_cast<std::uint32_t>(Base); + CloneID = static_cast<std::uint32_t>(Clone); + advance(); + } else { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), BaseID)) + return fail("invalid basic block base ID"); + advance(); + if (consume(MITokenKind::Dot) || consume(MITokenKind::Comma)) { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), CloneID)) + return fail("invalid basic block clone ID"); + advance(); + } + } + if (Block != nullptr) + Block->attributes().setProfileID({BaseID, CloneID}); + } else if (atKeyword("call-frame-size")) { + if (!Seen.insert("call-frame-size").second) + return fail("duplicate basic block call frame size"); + advance(); + std::uint32_t Size = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Size)) + return fail("invalid basic block call frame size"); + if (Block != nullptr) + Block->attributes().setCallFrameSize(Size); + advance(); + } else { + return fail("unknown basic block attribute '" + std::string(token().spelling()) + "'"); + } + + if (!consume(MITokenKind::Comma)) + break; + } + return expect(MITokenKind::RightParen, "')'"); +} + +bool MIParser::Impl::parseBlockDefinition(bool Create, std::uint32_t &ID) { + const MIToken Label = token(); + if (!at(MITokenKind::MachineBasicBlockLabel) || parseUnsigned32(Label, ID)) + return fail("invalid machine basic block label"); + advance(); + + MachineBasicBlock *Block = nullptr; + if (Create) { + if (Blocks.size() >= Options.MaximumBlocks) + return fail(Label, "machine basic block count exceeds parser limit"); + if (Blocks.contains(ID)) + return fail(Label, "redefinition of machine basic block #" + std::to_string(ID)); + auto Created = Function.createBasicBlock(); + if (!Created) + return importError(Label, Created.error()); + Block = *Created; + auto Appended = Function.appendBlock(*Block); + if (!Appended) + return importError(Label, Appended.error()); + if (!Label.stringValue().empty()) { + auto SetName = Block->attributes().setName(std::string(Label.stringValue())); + if (!SetName) + return importError(Label, SetName.error()); + } + Blocks.emplace(ID, Block); + } else { + const auto Found = Blocks.find(ID); + if (Found == Blocks.end()) + return fail(Label, "use of undefined machine basic block #" + std::to_string(ID)); + Block = Found->second; + } + + if (consume(MITokenKind::LeftParen) && parseBlockAttributes(Create ? Block : nullptr)) + return true; + return expect(MITokenKind::Colon, "':' after the basic block definition"); +} + +bool MIParser::Impl::parseFirstPass() { + Position = 0; + bool AtLineStart = true; + std::size_t BraceDepth = 0; + while (!at(MITokenKind::EndOfFile)) { + if (consume(MITokenKind::Newline)) { + AtLineStart = true; + continue; + } + if (at(MITokenKind::MachineBasicBlockLabel)) { + if (!AtLineStart || token().range().Begin.Column != 1) + return fail("basic block definition must be at the start of a line"); + std::uint32_t ID = 0; + if (parseBlockDefinition(true, ID)) + return true; + AtLineStart = false; + continue; + } + if (at(MITokenKind::LeftBrace)) + ++BraceDepth; + else if (at(MITokenKind::RightBrace)) { + if (BraceDepth == 0) + return fail("extraneous closing instruction bundle brace"); + --BraceDepth; + } + AtLineStart = false; + advance(); + } + if (BraceDepth != 0) + return fail("unterminated instruction bundle"); + return false; +} + +Register MIParser::Impl::getOrCreateVirtualRegister(const MIToken &Token) { + if (Token.kind() == MITokenKind::VirtualRegister) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) + return {}; + if (const auto Found = NumericVirtualRegisters.find(ID); + Found != NumericVirtualRegisters.end()) { + return Found->second; + } + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) { + static_cast<void>(importError(Token, Created.error())); + return {}; + } + NumericVirtualRegisters.emplace(ID, *Created); + return *Created; + } + + const std::string Name(Token.stringValue()); + if (const auto Found = NamedVirtualRegisters.find(Name); Found != NamedVirtualRegisters.end()) + return Found->second; + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) { + static_cast<void>(importError(Token, Created.error())); + return {}; + } + NamedVirtualRegisters.emplace(Name, *Created); + return *Created; +} + +bool MIParser::Impl::parseRegister(Register &Result) { + const MIToken RegisterToken = token(); + if (consume(MITokenKind::Underscore)) { + Result = {}; + return false; + } + if (at(MITokenKind::NamedRegister)) { + if (RegisterToken.stringValue() == "noreg") { + Result = {}; + advance(); + return false; + } + const PhysicalRegisterDesc *Description = + Target.registers().physicalRegister(RegisterToken.stringValue()); + if (Description == nullptr) + return fail("unknown physical register '$" + std::string(RegisterToken.stringValue()) + "'"); + Result = Register::fromPhysical(Description->Reg); + advance(); + return false; + } + if (at(MITokenKind::VirtualRegister) || at(MITokenKind::NamedVirtualRegister)) { + Result = getOrCreateVirtualRegister(RegisterToken); + if (Result.isNoRegister()) + return true; + advance(); + return false; + } + return fail("expected a machine register"); +} + +bool MIParser::Impl::parseRegisterOperand(bool IsDef, ParsedOperand &Result) { + const MIToken Begin = token(); + RegisterOperandFlags Flags{IsDef ? RegisterOperandFlag::Define : RegisterOperandFlag::Use}; + std::unordered_set<std::string> SeenFlags; + while (token().isRegisterFlag()) { + const std::string Name(token().stringValue()); + if (!SeenFlags.insert(Name).second) + return fail("duplicate '" + Name + "' register flag"); + if (Name == "implicit") + Flags.set(RegisterOperandFlag::Implicit); + else if (Name == "implicit-def") { + Flags.clear(RegisterOperandFlag::Use); + Flags.set(RegisterOperandFlag::Define); + Flags.set(RegisterOperandFlag::Implicit); + IsDef = true; + } else if (Name == "def") { + Flags.clear(RegisterOperandFlag::Use); + Flags.set(RegisterOperandFlag::Define); + IsDef = true; + } else if (Name == "dead") + Flags.set(RegisterOperandFlag::Dead); + else if (Name == "killed") + Flags.set(RegisterOperandFlag::Kill); + else if (Name == "undef") + Flags.set(RegisterOperandFlag::Undef); + else if (Name == "internal") + Flags.set(RegisterOperandFlag::InternalRead); + else if (Name == "early-clobber") + Flags.set(RegisterOperandFlag::EarlyClobber); + else if (Name == "debug-use") + Flags.set(RegisterOperandFlag::Debug); + else if (Name == "renamable") + Flags.set(RegisterOperandFlag::Renamable); + advance(); + } + + Register Reg; + if (parseRegister(Reg)) + return true; + + SubRegIndex SubReg; + if (consume(MITokenKind::Dot)) { + if (!Reg.isVirtual() || !at(MITokenKind::Identifier)) + return fail("subregister index requires a virtual register"); + const SubRegIndexDesc *Description = Target.registers().subRegIndex(token().stringValue()); + if (Description == nullptr) + return fail("unknown subregister index '" + std::string(token().stringValue()) + "'"); + SubReg = Description->ID; + advance(); + } + + if (consume(MITokenKind::Colon)) { + if (!Reg.isVirtual()) + return fail("register class or bank requires a virtual register"); + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + VirtualRegisterInfo &Info = InfoResult->get(); + if (consume(MITokenKind::Underscore)) { + if (Info.RegClass || Info.Bank) + return fail("conflicting generic virtual register constraint"); + } else if (at(MITokenKind::Identifier)) { + if (const RegisterClassDesc *Class = + Target.registers().registerClass(token().stringValue())) { + if ((Info.RegClass && *Info.RegClass != Class->ID) || Info.Bank) + return fail("conflicting virtual register class"); + auto SetClass = Function.registerInfo().setRegisterClass(Reg, Class->ID); + if (!SetClass) + return importError(token(), SetClass.error()); + } else if (const RegisterBankDesc *Bank = + Target.registers().registerBank(token().stringValue())) { + if ((Info.Bank && *Info.Bank != Bank->ID) || Info.RegClass) + return fail("conflicting virtual register bank"); + auto SetBank = Function.registerInfo().setRegisterBank(Reg, Bank->ID); + if (!SetBank) + return importError(token(), SetBank.error()); + } else { + return fail("unknown register class or bank '" + std::string(token().stringValue()) + "'"); + } + advance(); + } else { + return fail("expected '_', register class, or register bank name"); + } + } + + std::optional<std::size_t> TiedDef; + if (consume(MITokenKind::LeftParen)) { + if (atKeyword("tied-def")) { + if (IsDef) + return fail("'tied-def' is only valid on register uses"); + advance(); + std::uint64_t Index = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Index) || + Index > std::numeric_limits<std::size_t>::max()) { + return fail("invalid tied-def operand index"); + } + TiedDef = static_cast<std::size_t>(Index); + advance(); + if (expect(MITokenKind::RightParen, "')' after tied-def")) + return true; + } else { + if (!Reg.isVirtual()) + return fail("low-level type is only valid on a virtual register"); + LLT Type; + if (parseLowLevelType(Type)) + return true; + if (expect(MITokenKind::RightParen, "')' after the low-level type")) + return true; + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + VirtualRegisterInfo &Info = InfoResult->get(); + if (Info.Type && *Info.Type != Type) + return fail(Begin, "conflicting virtual register low-level type"); + auto SetType = Function.registerInfo().setType(Reg, Type); + if (!SetType) + return importError(Begin, SetType.error()); + } + } + + if (IsDef && Reg.isVirtual()) { + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + const VirtualRegisterInfo &Info = InfoResult->get(); + if (!Info.RegClass && !Info.Type) + return fail(Begin, "generic virtual register definition requires a low-level type"); + } + + if (Flags.has(RegisterOperandFlag::Define) && Flags.has(RegisterOperandFlag::Kill)) + return fail(Begin, "defined register operand cannot be killed"); + if (!Flags.has(RegisterOperandFlag::Define) && Flags.has(RegisterOperandFlag::Dead)) + return fail(Begin, "register use cannot be dead"); + + auto Operand = MachineOperand::registerOperand(Reg, Flags, SubReg); + if (!Operand) + return importError(Begin, Operand.error()); + Result = + ParsedOperand{std::move(*Operand), TiedDef, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +MachineBasicBlock *MIParser::Impl::blockFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid machine basic block reference")); + return nullptr; + } + const auto Found = Blocks.find(ID); + if (Found == Blocks.end()) { + static_cast<void>(fail(Token, "use of undefined machine basic block #" + std::to_string(ID))); + return nullptr; + } + if (!Token.stringValue().empty()) { + const std::optional<std::string_view> Name = Found->second->attributes().name(); + if (!Name || *Name != Token.stringValue()) { + static_cast<void>(fail(Token, "machine basic block reference has a mismatched name")); + return nullptr; + } + } + return Found->second; +} + +std::optional<FrameIndex> MIParser::Impl::frameIndexFor(const MIToken &Token, bool Fixed) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid stack object reference")); + return std::nullopt; + } + const auto &Slots = Fixed ? Context.FixedStackObjects : Context.StackObjects; + for (const MIFrameObjectSlot &Slot : Slots) { + if (Slot.TextID != ID) + continue; + if (!Token.stringValue().empty() && (!Slot.Name || *Slot.Name != Token.stringValue())) { + static_cast<void>(fail(Token, "stack object reference has a mismatched name")); + return std::nullopt; + } + if (Function.frameInfo().object(Slot.Index) == nullptr) { + static_cast<void>(fail(Token, "stack object slot maps to a missing frame object")); + return std::nullopt; + } + return Slot.Index; + } + static_cast<void>(fail(Token, std::string("use of undefined ") + + (Fixed ? "fixed stack" : "stack") + " object #" + + std::to_string(ID))); + return std::nullopt; +} + +std::optional<ConstantPoolIndex> MIParser::Impl::constantPoolIndexFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid constant pool reference")); + return std::nullopt; + } + for (const MIConstantPoolSlot &Slot : Context.ConstantPoolItems) { + if (Slot.TextID == ID && Function.constantPool().entry(Slot.Index) != nullptr) + return Slot.Index; + } + static_cast<void>(fail(Token, "use of undefined constant pool item #" + std::to_string(ID))); + return std::nullopt; +} + +std::optional<JumpTableIndex> MIParser::Impl::jumpTableIndexFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid jump table reference")); + return std::nullopt; + } + for (const MIJumpTableSlot &Slot : Context.JumpTables) { + if (Slot.TextID == ID && Function.jumpTableInfo().table(Slot.Index) != nullptr) + return Slot.Index; + } + static_cast<void>(fail(Token, "use of undefined jump table #" + std::to_string(ID))); + return std::nullopt; +} + +std::string MIParser::Impl::symbolName(const MIToken &Token) const { + if (Token.hasStringValue()) + return std::string(Token.stringValue()); + return std::string(Token.integerValue()); +} + +bool MIParser::Impl::parseTargetFlags(std::uint32_t &Flags) { + Flags = 0; + if (!atKeyword("target-flags")) + return false; + advance(); + if (expect(MITokenKind::LeftParen, "'(' after target-flags")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a target flag name"); + + const std::string First(token().stringValue()); + if (const auto Direct = Target.mirText().directOperandFlag(First)) + Flags = *Direct; + else if (const auto Bit = Target.mirText().bitmaskOperandFlag(First)) + Flags = *Bit; + else + return fail("unknown target operand flag '" + First + "'"); + advance(); + + std::unordered_set<std::string> Seen{First}; + while (consume(MITokenKind::Comma)) { + if (!at(MITokenKind::Identifier)) + return fail("expected a target bit flag name"); + const std::string Name(token().stringValue()); + const auto Bit = Target.mirText().bitmaskOperandFlag(Name); + if (!Bit) + return fail("unknown target bit operand flag '" + Name + "'"); + if (!Seen.insert(Name).second || (Flags & *Bit) == *Bit) + return fail("duplicate target operand flag '" + Name + "'"); + Flags |= *Bit; + advance(); + } + if (expect(MITokenKind::RightParen, "')' after target flags")) + return true; + const auto Valid = Target.mirText().validateOperandFlags(Flags); + return !Valid && importError(token(), Valid.error()); +} + +bool MIParser::Impl::parseTypedIntegerOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const std::string_view TypeSpelling = Begin.stringValue(); + if (!TypeSpelling.starts_with('i')) + return fail(Begin, "typed integer operand must use an 'iN' type"); + std::uint64_t WideBitWidth = 0; + if (!parseUnsignedDigits(TypeSpelling.substr(1), WideBitWidth) || + WideBitWidth > IntegerImmediate::MaxBitWidth) { + return fail(Begin, "typed integer operand has an invalid bit width"); + } + const auto BitWidth = static_cast<std::uint32_t>(WideBitWidth); + advance(); + + std::string_view ValueSpelling; + if (at(MITokenKind::IntegerLiteral)) { + ValueSpelling = token().integerValue(); + } else if (at(MITokenKind::Identifier) && token().stringValue() == "true") { + ValueSpelling = "1"; + } else if (at(MITokenKind::Identifier) && token().stringValue() == "false") { + ValueSpelling = "0"; + } else { + return fail("expected a decimal integer, true, or false after the typed integer type"); + } + + auto Value = parseIntegerImmediateValue(BitWidth, ValueSpelling); + if (!Value) + return importError(token(), Value.error()); + advance(); + Result = ParsedOperand{ + MachineOperand::integerImmediate(std::move(*Value)), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseFloatingImmediateOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const auto Semantics = floatSemanticsForKeyword(Begin.stringValue()); + if (!Semantics) + return fail(Begin, "unknown floating-point immediate type"); + const std::uint32_t BitWidth = getFloatSemanticsBitWidth(*Semantics).value_or(0); + advance(); + if (!at(MITokenKind::FloatingPointLiteral) && !at(MITokenKind::HexLiteral)) + return fail("expected a floating-point literal"); + + const MIToken Literal = token(); + const std::string_view Spelling = Literal.spelling(); + std::vector<std::uint64_t> Words; + if (Spelling.starts_with("f0x") || Spelling.starts_with("f0X")) { + if (!parseHexWords(Spelling.substr(3), BitWidth, Words)) + return fail(Literal, "raw floating-point literal does not fit its semantics"); + } else if (Spelling.starts_with("0x") || Spelling.starts_with("0X")) { + std::string_view Digits = Spelling.substr(2); + char Prefix = '\0'; + if (!Digits.empty() && !hexDigit(Digits.front())) { + Prefix = Digits.front(); + Digits.remove_prefix(1); + } + char ExpectedPrefix = '\0'; + switch (*Semantics) { + case FloatSemantics::IEEEHalf: + ExpectedPrefix = 'H'; + break; + case FloatSemantics::BFloat: + ExpectedPrefix = 'R'; + break; + case FloatSemantics::X87DoubleExtended: + ExpectedPrefix = 'K'; + break; + case FloatSemantics::IEEEQuad: + ExpectedPrefix = 'L'; + break; + case FloatSemantics::PPCDoubleDouble: + case FloatSemantics::PPCDoubleDoubleLegacy: + ExpectedPrefix = 'M'; + break; + default: + break; + } + if (ExpectedPrefix != '\0') { + if (Prefix != ExpectedPrefix || !parseHexWords(Digits, BitWidth, Words)) + return fail(Literal, "floating-point hexadecimal prefix does not match its semantics"); + } else if (Prefix != '\0') { + return fail(Literal, "unexpected floating-point hexadecimal prefix"); + } else if (*Semantics == FloatSemantics::IEEESingle) { + std::uint64_t DoubleBits = 0; + if (!parseHexadecimal(Spelling, DoubleBits)) + return fail(Literal, "invalid IEEE single hexadecimal literal"); + const float Value = static_cast<float>(std::bit_cast<double>(DoubleBits)); + Words = {std::bit_cast<std::uint32_t>(Value)}; + } else if (*Semantics == FloatSemantics::IEEEDouble) { + std::uint64_t DoubleBits = 0; + if (!parseHexadecimal(Spelling, DoubleBits)) + return fail(Literal, "invalid IEEE double hexadecimal literal"); + Words = {DoubleBits}; + } else { + return fail(Literal, "floating-point literal requires its semantics prefix"); + } + } else { + const char *First = Spelling.data(); + const char *Last = First + Spelling.size(); + if (*Semantics == FloatSemantics::IEEEDouble) { + double Value = 0; + const auto Parsed = std::from_chars(First, Last, Value, std::chars_format::general); + if (Parsed.ec != std::errc{} || Parsed.ptr != Last) + return fail(Literal, "invalid IEEE double decimal literal"); + Words = {std::bit_cast<std::uint64_t>(Value)}; + } else if (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEHalf || + *Semantics == FloatSemantics::BFloat) { + float Value = 0; + const auto Parsed = std::from_chars(First, Last, Value, std::chars_format::general); + if (Parsed.ec != std::errc{} || Parsed.ptr != Last) + return fail(Literal, "invalid floating-point decimal literal"); + if (*Semantics == FloatSemantics::IEEESingle) + Words = {std::bit_cast<std::uint32_t>(Value)}; + else if (*Semantics == FloatSemantics::IEEEHalf) + Words = {floatToHalfBits(Value)}; + else + Words = {floatToBFloatBits(Value)}; + } else { + return fail(Literal, "this floating-point semantics requires an exact hexadecimal literal"); + } + } + + auto Bits = FloatingPointBits::create(*Semantics, Words); + if (!Bits) + return importError(Literal, Bits.error()); + advance(); + Result = ParsedOperand{ + MachineOperand::floatingImmediate(*Bits), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseMetadataReference(MachineMetadataReference &Result) { + const MIToken Begin = token(); + if (expect(MITokenKind::Exclaim, "'!' before a metadata slot")) + return true; + std::uint32_t TextID = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), TextID)) + return fail("expected a 32-bit metadata slot number"); + + for (const MIMetadataSlot &Slot : Context.Metadata) { + if (Slot.TextID != TextID) + continue; + if (!Slot.Reference.isValid()) + return fail(Begin, "metadata slot maps to an invalid reference"); + Result = Slot.Reference; + advance(); + return false; + } + if (Function.metadata().node(TextID) != nullptr) { + Result = MachineMetadataReference::machineFunction(TextID); + advance(); + return false; + } + return fail(Begin, "use of undefined metadata slot !" + std::to_string(TextID)); +} + +bool MIParser::Impl::parseMetadataOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + MachineMetadataValue Value; + if (at(MITokenKind::Exclaim)) { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + auto Created = MachineMetadataValue::reference(Reference); + if (!Created) + return importError(Begin, Created.error()); + Value = std::move(*Created); + } else if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DIExpression")) { + advance(); + if (expect(MITokenKind::LeftParen, "'(' after !DIExpression")) + return true; + std::vector<std::uint64_t> Elements; + while (!at(MITokenKind::RightParen)) { + if (at(MITokenKind::Identifier)) { + const auto Opcode = debugExpressionOpcode(token().stringValue()); + if (!Opcode) + return fail("unknown debug expression opcode '" + std::string(token().stringValue()) + + "'"); + Elements.push_back(*Opcode); + advance(); + } else if (at(MITokenKind::IntegerLiteral)) { + std::int64_t Signed = 0; + if (parseSigned64(token(), Signed)) + return fail("debug expression element exceeds 64 bits"); + Elements.push_back(std::bit_cast<std::uint64_t>(Signed)); + advance(); + } else { + return fail("expected a debug expression opcode or integer"); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after !DIExpression")) + return true; + Value = MachineMetadataValue::expression(MachineDIExpression(std::move(Elements))); + } else { + return fail("expected a metadata reference or !DIExpression"); + } + + auto Operand = MachineOperand::metadata(std::move(Value)); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseInlineDILocation( + std::vector<MachineDILocationFrame> &InlineChain, + std::optional<MachineMetadataReference> &TerminalInlinedAt) { + if (!at(MITokenKind::MetadataKeyword) || !token().isMetadataKeyword("!DILocation")) { + return fail("expected an inline !DILocation"); + } + advance(); + if (expect(MITokenKind::LeftParen, "'(' after !DILocation")) + return true; + + MachineDILocationFrame Frame; + bool HasLine = false; + bool HasScope = false; + std::vector<MachineDILocationFrame> NestedChain; + std::optional<MachineMetadataReference> NestedTerminal; + std::unordered_set<std::string> SeenFields; + while (!at(MITokenKind::RightParen)) { + if (!at(MITokenKind::Identifier)) + return fail("expected a DILocation field name"); + const MIToken FieldToken = token(); + const std::string Field(token().stringValue()); + if (!SeenFields.insert(Field).second) + return fail("duplicate DILocation field '" + Field + "'"); + advance(); + if (expect(MITokenKind::Colon, "':' after a DILocation field name")) + return true; + + if (Field == "line") { + std::uint32_t Line = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Line)) + return fail("expected an unsigned 32-bit DILocation line"); + Frame.Line = Line; + HasLine = true; + advance(); + } else if (Field == "column") { + std::uint64_t Column = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Column) || + Column > std::numeric_limits<std::uint16_t>::max()) { + return fail("expected an unsigned 16-bit DILocation column"); + } + Frame.Column = static_cast<std::uint16_t>(Column); + advance(); + } else if (Field == "scope") { + MachineMetadataReference Scope; + if (parseMetadataReference(Scope)) + return true; + if (Scope.scope() != MachineMetadataScope::EmbeddedIR) + return fail(FieldToken, "DILocation scope must reference embedded IR metadata"); + Frame.Scope = Scope; + HasScope = true; + } else if (Field == "inlinedAt") { + if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DILocation")) { + if (parseInlineDILocation(NestedChain, NestedTerminal)) + return true; + } else { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (Reference.scope() != MachineMetadataScope::EmbeddedIR) { + return fail(FieldToken, "DILocation inlinedAt must reference embedded IR metadata"); + } + NestedTerminal = Reference; + } + } else if (Field == "isImplicitCode") { + if (!at(MITokenKind::Identifier) || + (token().stringValue() != "true" && token().stringValue() != "false")) { + return fail("expected 'true' or 'false' for DILocation isImplicitCode"); + } + Frame.IsImplicitCode = token().stringValue() == "true"; + advance(); + } else if (Field == "atomGroup") { + std::uint64_t Group = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Group) || + Group > MachineDILocation::MaxAtomGroup) { + return fail("DILocation atomGroup exceeds 61 bits"); + } + Frame.AtomGroup = Group; + advance(); + } else if (Field == "atomRank") { + std::uint64_t Rank = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Rank) || Rank > 7) { + return fail("DILocation atomRank exceeds three bits"); + } + Frame.AtomRank = static_cast<std::uint8_t>(Rank); + advance(); + } else { + return fail(FieldToken, "unknown DILocation field '" + Field + "'"); + } + + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after !DILocation")) + return true; + if (!HasLine) + return fail("DILocation requires a line field"); + if (!HasScope) + return fail("DILocation requires a scope field"); + + InlineChain.push_back(Frame); + InlineChain.insert(InlineChain.end(), NestedChain.begin(), NestedChain.end()); + TerminalInlinedAt = NestedTerminal; + return false; +} + +bool MIParser::Impl::parseCFIRegister(DwarfRegisterNumber &Result) { + if (!at(MITokenKind::NamedRegister)) + return fail("expected a physical register in the CFI instruction"); + const MIToken RegisterToken = token(); + Register Reg; + if (parseRegister(Reg) || !Reg.isPhysical()) + return true; + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail(RegisterToken, "CFI register is not a physical register"); + const auto Dwarf = Target.mirText().dwarfRegister(*Physical); + if (!Dwarf) + return fail(RegisterToken, "physical register has no canonical DWARF register number"); + Result = DwarfRegisterNumber::fromValue(*Dwarf); + return false; +} + +bool MIParser::Impl::parseCFISigned(std::int64_t &Result) { + if (!at(MITokenKind::IntegerLiteral) || parseSigned64(token(), Result)) + return fail("expected a signed 64-bit CFI integer"); + advance(); + return false; +} + +bool MIParser::Impl::parseCFIUnsigned(std::uint32_t &Result) { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Result)) + return fail("expected an unsigned 32-bit CFI integer"); + advance(); + return false; +} + +bool MIParser::Impl::parseCFIOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const std::string Operation(token().stringValue()); + advance(); + + const auto Comma = [this]() { return expect(MITokenKind::Comma, "',' in CFI instruction"); }; + DwarfRegisterNumber Register; + DwarfRegisterNumber OtherRegister; + std::int64_t Offset = 0; + std::optional<MachineCFIInstruction> Instruction; + if (Operation == "same_value") { + if (parseCFIRegister(Register)) + return true; + Instruction = MachineCFIInstruction::sameValue(Register); + } else if (Operation == "remember_state") { + Instruction = MachineCFIInstruction::rememberState(); + } else if (Operation == "restore_state") { + Instruction = MachineCFIInstruction::restoreState(); + } else if (Operation == "offset" || Operation == "rel_offset") { + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset)) + return true; + Instruction = Operation == "offset" ? MachineCFIInstruction::offset(Register, Offset) + : MachineCFIInstruction::relativeOffset(Register, Offset); + } else if (Operation == "def_cfa_register") { + if (parseCFIRegister(Register)) + return true; + Instruction = MachineCFIInstruction::defCFARegister(Register); + } else if (Operation == "def_cfa_offset" || Operation == "adjust_cfa_offset") { + if (parseCFISigned(Offset)) + return true; + Instruction = Operation == "def_cfa_offset" ? MachineCFIInstruction::defCFAOffset(Offset) + : MachineCFIInstruction::adjustCFAOffset(Offset); + } else if (Operation == "def_cfa") { + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset)) + return true; + Instruction = MachineCFIInstruction::defCFA(Register, Offset); + } else if (Operation == "llvm_def_aspace_cfa") { + std::uint32_t AddressSpace = 0; + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset) || Comma() || + parseCFIUnsigned(AddressSpace)) { + return true; + } + Instruction = MachineCFIInstruction::llvmDefAddressSpaceCFA(Register, Offset, AddressSpace); + } else if (Operation == "restore" || Operation == "undefined") { + if (parseCFIRegister(Register)) + return true; + Instruction = Operation == "restore" ? MachineCFIInstruction::restore(Register) + : MachineCFIInstruction::undefined(Register); + } else if (Operation == "register") { + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(OtherRegister)) + return true; + Instruction = MachineCFIInstruction::registerRule(Register, OtherRegister); + } else if (Operation == "window_save") { + Instruction = MachineCFIInstruction::windowSave(); + } else if (Operation == "negate_ra_sign_state") { + Instruction = MachineCFIInstruction::negateReturnAddressState(); + } else if (Operation == "negate_ra_sign_state_with_pc") { + Instruction = MachineCFIInstruction::negateReturnAddressStateWithPC(); + } else if (Operation == "llvm_register_pair") { + DwarfRegisterNumber First; + DwarfRegisterNumber Second; + std::uint32_t FirstSize = 0; + std::uint32_t SecondSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(First) || Comma() || + parseCFIUnsigned(FirstSize) || Comma() || parseCFIRegister(Second) || Comma() || + parseCFIUnsigned(SecondSize)) { + return true; + } + Instruction = MachineCFIInstruction::llvmRegisterPair({.Register = Register, + .FirstRegister = First, + .FirstSizeInBits = FirstSize, + .SecondRegister = Second, + .SecondSizeInBits = SecondSize}); + } else if (Operation == "llvm_vector_registers") { + if (parseCFIRegister(Register) || Comma()) + return true; + std::vector<MachineCFIVectorRegisterLane> Registers; + do { + DwarfRegisterNumber VectorRegister; + std::uint32_t Lane = 0; + std::uint32_t Size = 0; + if (parseCFIRegister(VectorRegister) || Comma() || parseCFIUnsigned(Lane) || Comma() || + parseCFIUnsigned(Size)) { + return true; + } + Registers.push_back({.Register = VectorRegister, .Lane = Lane, .SizeInBits = Size}); + if (!at(MITokenKind::Comma)) + break; + advance(); + if (!at(MITokenKind::NamedRegister)) + return fail("expected another vector CFI register tuple"); + } while (true); + auto Created = MachineCFIInstruction::llvmVectorRegisters( + {.Register = Register, .VectorRegisters = std::move(Registers)}); + if (!Created) + return importError(Begin, Created.error()); + Instruction = std::move(*Created); + } else if (Operation == "llvm_vector_offset") { + DwarfRegisterNumber MaskRegister; + std::uint32_t RegisterSize = 0; + std::uint32_t MaskSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIUnsigned(RegisterSize) || Comma() || + parseCFIRegister(MaskRegister) || Comma() || parseCFIUnsigned(MaskSize) || Comma() || + parseCFISigned(Offset)) { + return true; + } + Instruction = MachineCFIInstruction::llvmVectorOffset({.Register = Register, + .RegisterSizeInBits = RegisterSize, + .MaskRegister = MaskRegister, + .MaskRegisterSizeInBits = MaskSize, + .Offset = Offset}); + } else if (Operation == "llvm_vector_register_mask") { + DwarfRegisterNumber SpillRegister; + DwarfRegisterNumber MaskRegister; + std::uint32_t SpillLaneSize = 0; + std::uint32_t MaskSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(SpillRegister) || Comma() || + parseCFIUnsigned(SpillLaneSize) || Comma() || parseCFIRegister(MaskRegister) || Comma() || + parseCFIUnsigned(MaskSize)) { + return true; + } + Instruction = + MachineCFIInstruction::llvmVectorRegisterMask({.Register = Register, + .SpillRegister = SpillRegister, + .SpillRegisterLaneSizeInBits = SpillLaneSize, + .MaskRegister = MaskRegister, + .MaskRegisterSizeInBits = MaskSize}); + } else if (Operation == "escape") { + std::vector<std::uint8_t> Bytes; + while (true) { + std::uint64_t Byte = 0; + if (!at(MITokenKind::HexLiteral) || parseUnsigned(token(), Byte) || + Byte > std::numeric_limits<std::uint8_t>::max()) { + return fail("CFI escape element must be an 8-bit hexadecimal literal"); + } + Bytes.push_back(static_cast<std::uint8_t>(Byte)); + advance(); + if (!at(MITokenKind::Comma) || !nextIs(MITokenKind::HexLiteral)) + break; + advance(); + } + auto Created = MachineCFIInstruction::escape(std::move(Bytes)); + if (!Created) + return importError(Begin, Created.error()); + Instruction = std::move(*Created); + } else { + return fail(Begin, "unsupported CFI operation '" + Operation + "'"); + } + + auto Index = Function.cfiInfo().addInstruction(std::move(*Instruction)); + if (!Index) + return importError(Begin, Index.error()); + Result = ParsedOperand{ + MachineOperand::cfiIndex(*Index), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseMemoryBase(MemoryBase &Result, std::int64_t &Offset) { + const MIToken BaseToken = token(); + switch (BaseToken.kind()) { + case MITokenKind::StackObject: + case MITokenKind::FixedStackObject: { + const auto Index = frameIndexFor(BaseToken, BaseToken.kind() == MITokenKind::FixedStackObject); + if (!Index) + return true; + Result = MemoryBase::frameIndex(*Index); + advance(); + break; + } + case MITokenKind::ConstantPoolItem: { + const auto Index = constantPoolIndexFor(BaseToken); + if (!Index) + return true; + Result = MemoryBase::constantPool(*Index); + advance(); + break; + } + case MITokenKind::JumpTableIndex: { + const auto Index = jumpTableIndexFor(BaseToken); + if (!Index) + return true; + Result = MemoryBase::jumpTable(*Index); + advance(); + break; + } + case MITokenKind::GlobalValue: + case MITokenKind::NamedGlobalValue: + Result = MemoryBase::globalSymbol(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::ExternalSymbol: + Result = MemoryBase::externalSymbol(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::IRValue: + case MITokenKind::NamedIRValue: + case MITokenKind::QuotedIRValue: + Result = MemoryBase::irValue(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::Keyword: + if (BaseToken.isKeyword("unknown-address")) { + Result = MemoryBase(); + advance(); + } else if (BaseToken.isKeyword("stack")) { + Result = MemoryBase::stack(); + advance(); + } else if (BaseToken.isKeyword("got")) { + Result = MemoryBase::got(); + advance(); + } else if (BaseToken.isKeyword("constant-pool")) { + Result = MemoryBase::constantPoolPseudo(); + advance(); + } else if (BaseToken.isKeyword("jump-table")) { + Result = MemoryBase::jumpTablePseudo(); + advance(); + } else if (BaseToken.isKeyword("call-entry")) { + advance(); + if (at(MITokenKind::GlobalValue) || at(MITokenKind::NamedGlobalValue)) { + Result = MemoryBase::callEntryGlobal(symbolName(token())); + } else if (at(MITokenKind::ExternalSymbol)) { + Result = MemoryBase::callEntryExternal(symbolName(token())); + } else { + return fail("call-entry memory base requires a global or external symbol"); + } + advance(); + } else if (BaseToken.isKeyword("custom")) { + advance(); + if (!at(MITokenKind::Identifier) && !at(MITokenKind::StringConstant)) + return fail("custom memory base requires a target spelling"); + Result = MemoryBase::custom(std::string(token().stringValue())); + advance(); + } else { + return fail(BaseToken, "unsupported machine memory base"); + } + break; + default: + return fail(BaseToken, "expected a machine memory base"); + } + return parseOptionalOffset(Offset); +} + +bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result) { + const MIToken Begin = token(); + if (expect(MITokenKind::LeftParen, "'(' before a machine memory operand")) + return true; + + MachineMemOperandDesc Description; + std::unordered_set<std::string> SeenFlags; + while (token().isMemoryOperandFlag()) { + const std::string Name(token().stringValue()); + if (!SeenFlags.insert(Name).second) + return fail("duplicate machine memory operand flag '" + Name + "'"); + if (atKeyword("volatile")) + Description.Accesses.set(MemoryAccessFlag::Volatile); + else if (atKeyword("non-temporal")) + Description.Accesses.set(MemoryAccessFlag::NonTemporal); + else if (atKeyword("dereferenceable")) + Description.Accesses.set(MemoryAccessFlag::Dereferenceable); + else if (atKeyword("invariant")) + Description.Accesses.set(MemoryAccessFlag::Invariant); + else { + const auto Flag = Target.mirText().machineMemOperandFlag(token().stringValue()); + if (!Flag) + return fail("unknown target machine memory operand flag '" + Name + "'"); + if ((Description.TargetFlags & *Flag) != 0) + return fail("overlapping target machine memory operand flag '" + Name + "'"); + Description.TargetFlags |= *Flag; + } + advance(); + } + + if (!at(MITokenKind::Identifier) || + (token().stringValue() != "load" && token().stringValue() != "store")) { + return fail("expected load or store in a machine memory operand"); + } + if (token().stringValue() == "load") + Description.Accesses.set(MemoryAccessFlag::Load); + else + Description.Accesses.set(MemoryAccessFlag::Store); + advance(); + if (at(MITokenKind::Identifier) && token().stringValue() == "store") { + if (Description.Accesses.has(MemoryAccessFlag::Store)) + return fail("duplicate store memory operation"); + Description.Accesses.set(MemoryAccessFlag::Store); + advance(); + } + + if (at(MITokenKind::Identifier) && token().stringValue() == "syncscope") { + advance(); + if (expect(MITokenKind::LeftParen, "'(' after syncscope")) + return true; + if (!at(MITokenKind::StringConstant)) + return fail("syncscope requires a quoted scope name"); + const std::string_view Name = token().stringValue(); + const auto Scope = std::ranges::find_if( + Context.SyncScopes, [Name](const MISyncScopeSlot &Slot) { return Slot.Name == Name; }); + if (Scope == Context.SyncScopes.end()) + return fail("unknown synchronization scope '" + std::string(Name) + "'"); + Description.SyncScope = Scope->ID; + advance(); + if (expect(MITokenKind::RightParen, "')' after syncscope")) + return true; + } + + std::size_t OrderingCount = 0; + while (at(MITokenKind::Identifier)) { + const auto Ordering = atomicOrdering(token().stringValue()); + if (!Ordering) + break; + if (OrderingCount == 0) + Description.SuccessOrdering = *Ordering; + else if (OrderingCount == 1) + Description.FailureOrdering = *Ordering; + else + return fail("machine memory operand has more than two atomic orderings"); + ++OrderingCount; + advance(); + } + const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); + const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); + if (OrderingCount == 1) + Description.AtomicKind = IsLoad && IsStore + ? AtomicOperation::ReadModifyWrite + : (IsLoad ? AtomicOperation::Load : AtomicOperation::Store); + else if (OrderingCount == 2) + Description.AtomicKind = AtomicOperation::CompareExchange; + + std::uint64_t MinimumBytes = 0; + if (at(MITokenKind::IntegerLiteral)) { + if (parseUnsigned(token(), MinimumBytes) || MinimumBytes == 0) + return fail("machine memory operand size must be a nonzero byte count"); + auto Size = MemorySize::fixed(MinimumBytes); + if (!Size) + return importError(token(), Size.error()); + Description.Size = *Size; + advance(); + } else if (atKeyword("unknown-size")) { + advance(); + } else if (consume(MITokenKind::LeftParen)) { + LLT Type; + if (parseLowLevelType(Type)) + return true; + if (expect(MITokenKind::RightParen, "')' after the memory low-level type")) + return true; + const auto MinimumBits = minimumTypeSizeInBits(Type); + if (!MinimumBits || *MinimumBits == 0) + return fail(Begin, "memory low-level type has no representable size"); + MinimumBytes = (*MinimumBits + 7U) / 8U; + auto Size = Type.isScalableVector() ? MemorySize::scalable(MinimumBytes) + : MemorySize::fixed(MinimumBytes); + if (!Size) + return importError(Begin, Size.error()); + Description.Size = *Size; + } else { + return fail("expected a memory size, unknown-size, or memory low-level type"); + } + + if (MinimumBytes != 0) { + if (MinimumBytes > (std::uint64_t{1} << 63U)) + return fail(Begin, "machine memory operand size has no representable default alignment"); + auto DefaultAlignment = Alignment::create(std::bit_ceil(MinimumBytes)); + if (!DefaultAlignment) + return importError(Begin, DefaultAlignment.error()); + Description.Align = *DefaultAlignment; + } + + if (at(MITokenKind::Identifier)) { + const std::string_view ExpectedPreposition = + IsLoad && IsStore ? "on" : (IsLoad ? "from" : "into"); + if (token().stringValue() != ExpectedPreposition) { + return fail("expected '" + std::string(ExpectedPreposition) + + "' before the machine memory base"); + } + advance(); + if (parseMemoryBase(Description.Base, Description.Offset)) + return true; + } + + std::unordered_set<std::string> SeenOptions; + while (consume(MITokenKind::Comma)) { + const std::string Name(token().spelling()); + if (!SeenOptions.insert(Name).second) + return fail("duplicate machine memory operand option '" + Name + "'"); + if (atKeyword("align")) { + if (parseAlignment(Description.Align)) + return true; + } else if (atKeyword("basealign")) { + Alignment BaseAlignment; + if (parseAlignment(BaseAlignment)) + return true; + Description.BaseAlign = BaseAlignment; + } else if (atKeyword("addrspace")) { + advance(); + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Description.AddressSpace) || + Description.AddressSpace > LLT::MaxAddressSpace) { + return fail("invalid machine memory operand address space"); + } + advance(); + } else if (at(MITokenKind::MetadataKeyword)) { + const std::string MetadataKind(token().stringValue()); + advance(); + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (MetadataKind == "!tbaa") + Description.TBAA = Reference; + else if (MetadataKind == "!alias.scope") + Description.AliasScope = Reference; + else if (MetadataKind == "!noalias") + Description.NoAlias = Reference; + else if (MetadataKind == "!noalias.addrspace") + Description.NoAliasAddressSpace = Reference; + else if (MetadataKind == "!range") + Description.RangeMetadata = Reference; + else + return fail("unsupported machine memory operand metadata attachment"); + } else { + return fail("unknown machine memory operand option '" + Name + "'"); + } + } + if (expect(MITokenKind::RightParen, "')' after the machine memory operand")) + return true; + + auto Created = Function.createMemOperand(std::move(Description)); + if (!Created) + return importError(Begin, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parsePredicateOperand(ParsedOperand &Result, bool IsFloating) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after predicate kind")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a predicate spelling"); + + Expected<MachineOperand> Operand = IsFloating ? [&]() -> Expected<MachineOperand> { + const auto Predicate = floatingPredicate(token().stringValue()); + if (!Predicate) { + return makeUnexpected(ErrorCode::ParseError, "invalid floating-point predicate"); + } + return MachineOperand::floatingPredicate(*Predicate); + }() + : [&]() -> Expected<MachineOperand> { + const auto Predicate = integerPredicate(token().stringValue()); + if (!Predicate) { + return makeUnexpected(ErrorCode::ParseError, "invalid integer predicate"); + } + return MachineOperand::integerPredicate(*Predicate); + }(); + if (!Operand) + return importError(token(), Operand.error()); + advance(); + if (expect(MITokenKind::RightParen, "')' after predicate")) + return true; + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseShuffleMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after shufflemask")) + return true; + std::vector<std::int32_t> Elements; + while (!at(MITokenKind::RightParen)) { + if (atKeyword("undef")) { + Elements.push_back(ShuffleMaskOperand::Undef); + advance(); + } else { + std::int64_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || parseSigned64(token(), Value) || + Value < std::numeric_limits<std::int32_t>::min() || + Value > std::numeric_limits<std::int32_t>::max()) { + return fail("shuffle mask element must be a 32-bit integer or undef"); + } + Elements.push_back(static_cast<std::int32_t>(Value)); + advance(); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after shufflemask")) + return true; + auto Operand = MachineOperand::shuffleMask(std::move(Elements)); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseDebugInstrRefOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after dbg-instr-ref")) + return true; + std::uint64_t Instruction = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Instruction)) + return fail("invalid debug instruction number"); + advance(); + if (expect(MITokenKind::Comma, "',' in dbg-instr-ref")) + return true; + std::uint64_t OperandNumber = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), OperandNumber)) + return fail("invalid debug operand number"); + advance(); + if (expect(MITokenKind::RightParen, "')' after dbg-instr-ref")) + return true; + auto Operand = MachineOperand::debugInstrRef(Instruction, OperandNumber); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseLaneMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after lanemask")) + return true; + std::uint64_t Value = 0; + if (parseUnsigned(token(), Value)) + return fail("invalid 64-bit lane mask"); + advance(); + if (expect(MITokenKind::RightParen, "')' after lanemask")) + return true; + Result = ParsedOperand{MachineOperand::laneMask(LaneMask::fromRaw(Value)), + std::nullopt, + {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseIntrinsicOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after intrinsic")) + return true; + if (!at(MITokenKind::NamedGlobalValue)) + return fail("expected a named intrinsic"); + const auto ID = Target.mirText().intrinsic(token().stringValue()); + if (!ID) + return fail("unknown intrinsic '" + std::string(token().stringValue()) + "'"); + advance(); + if (expect(MITokenKind::RightParen, "')' after intrinsic")) + return true; + auto Operand = MachineOperand::intrinsic(*ID); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseTargetIndexOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after target-index")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a target index name"); + const auto Index = Target.mirText().targetIndex(token().stringValue()); + if (!Index) + return fail("unknown target index '" + std::string(token().stringValue()) + "'"); + advance(); + if (expect(MITokenKind::RightParen, "')' after target index")) + return true; + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Result = ParsedOperand{MachineOperand::targetIndex(*Index, Offset), + std::nullopt, + {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseCustomMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after CustomRegMask")) + return true; + std::uint32_t MaximumRegister = 0; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) + MaximumRegister = std::max(MaximumRegister, Register.Reg.number()); + std::vector<std::uint64_t> Words(MaximumRegister / 64U + 1U, 0); + if (!at(MITokenKind::RightParen)) { + do { + Register Reg; + if (!at(MITokenKind::NamedRegister) || parseRegister(Reg) || !Reg.isPhysical()) + return fail("CustomRegMask requires physical registers"); + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("CustomRegMask requires physical registers"); + const std::uint32_t Number = Physical->number(); + Words[Number / 64U] |= std::uint64_t{1} << (Number % 64U); + } while (consume(MITokenKind::Comma)); + } + if (expect(MITokenKind::RightParen, "')' after CustomRegMask")) + return true; + auto Operand = + MachineOperand::registerMask(std::make_shared<const RegisterMask>(std::move(Words))); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseLiveOutMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after liveout")) + return true; + std::uint32_t MaximumRegister = 0; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) + MaximumRegister = std::max(MaximumRegister, Register.Reg.number()); + std::vector<std::uint64_t> Words(MaximumRegister / 64U + 1U, 0); + do { + Register Reg; + if (!at(MITokenKind::NamedRegister) || parseRegister(Reg) || !Reg.isPhysical()) + return fail("liveout requires physical registers"); + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("liveout requires physical registers"); + const std::uint32_t Number = Physical->number(); + Words[Number / 64U] |= std::uint64_t{1} << (Number % 64U); + } while (consume(MITokenKind::Comma)); + if (expect(MITokenKind::RightParen, "')' after liveout")) + return true; + auto Operand = + MachineOperand::liveOutRegisterMask(std::make_shared<const RegisterMask>(std::move(Words))); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseBlockAddressOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after blockaddress")) + return true; + if (!at(MITokenKind::GlobalValue) && !at(MITokenKind::NamedGlobalValue)) + return fail("blockaddress requires a function reference"); + const std::string FunctionName = symbolName(token()); + advance(); + if (expect(MITokenKind::Comma, "',' in blockaddress")) + return true; + if (!at(MITokenKind::IRBlock) && !at(MITokenKind::NamedIRBlock)) + return fail("blockaddress requires an IR block reference"); + const std::string BlockName = symbolName(token()); + advance(); + if (expect(MITokenKind::RightParen, "')' after blockaddress")) + return true; + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Operand = MachineOperand::blockAddress(FunctionName, BlockName, Offset); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseOperand(bool IsDef, ParsedOperand &Result) { + const MIToken Begin = token(); + if (token().isRegister() || token().isRegisterFlag()) + return parseRegisterOperand(IsDef, Result); + + std::uint32_t TargetFlags = 0; + if (parseTargetFlags(TargetFlags)) + return true; + const MIToken OperandToken = token(); + if (isFloatingSemanticsKeyword(OperandToken)) { + if (parseFloatingImmediateOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + if (OperandToken.kind() == MITokenKind::Exclaim || + (OperandToken.kind() == MITokenKind::MetadataKeyword && + OperandToken.isMetadataKeyword("!DIExpression"))) { + if (parseMetadataOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + if (isCFIOperationToken(OperandToken)) { + if (parseCFIOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + MachineOperand Operand = MachineOperand::immediate(0); + bool HasOperand = true; + + switch (OperandToken.kind()) { + case MITokenKind::IntegerLiteral: { + std::int64_t Value = 0; + if (parseSigned64(OperandToken, Value)) + return fail("integer literal is too large for a machine immediate"); + Operand = MachineOperand::immediate(Value, TargetFlags); + advance(); + break; + } + case MITokenKind::MachineBasicBlock: { + MachineBasicBlock *Block = blockFor(OperandToken); + if (Block == nullptr) + return true; + Operand = MachineOperand::basicBlock(*Block, TargetFlags); + advance(); + break; + } + case MITokenKind::StackObject: + case MITokenKind::FixedStackObject: { + const bool Fixed = OperandToken.kind() == MITokenKind::FixedStackObject; + const auto Index = frameIndexFor(OperandToken, Fixed); + if (!Index) + return true; + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Operand = MachineOperand::frameIndex(*Index, Offset, TargetFlags); + break; + } + case MITokenKind::ConstantPoolItem: { + const auto Index = constantPoolIndexFor(OperandToken); + if (!Index) + return true; + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Operand = MachineOperand::constantPool(*Index, Offset, TargetFlags); + break; + } + case MITokenKind::JumpTableIndex: { + const auto Index = jumpTableIndexFor(OperandToken); + if (!Index) + return true; + Operand = MachineOperand::jumpTable(*Index, TargetFlags); + advance(); + break; + } + case MITokenKind::NamedGlobalValue: + case MITokenKind::GlobalValue: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Created = MachineOperand::globalSymbol(Name, Offset, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::ExternalSymbol: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Created = MachineOperand::externalSymbol(Name, Offset, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::MCSymbol: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Created = MachineOperand::mcSymbol(Name, Offset, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::SubRegisterIndex: { + const SubRegIndexDesc *Description = Target.registers().subRegIndex(OperandToken.stringValue()); + if (Description == nullptr) + return fail("unknown subregister index '" + std::string(OperandToken.stringValue()) + "'"); + Operand = + MachineOperand::immediate(static_cast<std::int64_t>(Description->ID.value()), TargetFlags); + advance(); + break; + } + case MITokenKind::Identifier: { + if (const auto Mask = Target.mirText().registerMask(OperandToken.stringValue())) { + auto Created = MachineOperand::registerMask(Mask, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + advance(); + } else if (OperandToken.stringValue() == "CustomRegMask") { + if (TargetFlags != 0) + return fail("custom register masks cannot have target flags"); + return parseCustomMaskOperand(Result); + } else if (OperandToken.stringValue().starts_with('i')) { + if (parseTypedIntegerOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } else { + return fail("unknown named machine operand '" + std::string(OperandToken.stringValue()) + + "'"); + } + break; + } + case MITokenKind::Keyword: + HasOperand = false; + break; + default: + return fail("expected a machine operand"); + } + + if (!HasOperand) { + bool Failed = false; + if (OperandToken.isKeyword("intpred")) + Failed = parsePredicateOperand(Result, false); + else if (OperandToken.isKeyword("floatpred")) + Failed = parsePredicateOperand(Result, true); + else if (OperandToken.isKeyword("shufflemask")) + Failed = parseShuffleMaskOperand(Result); + else if (OperandToken.isKeyword("dbg-instr-ref")) + Failed = parseDebugInstrRefOperand(Result); + else if (OperandToken.isKeyword("lanemask")) + Failed = parseLaneMaskOperand(Result); + else if (OperandToken.isKeyword("intrinsic")) + Failed = parseIntrinsicOperand(Result); + else if (OperandToken.isKeyword("target-index")) + Failed = parseTargetIndexOperand(Result); + else if (OperandToken.isKeyword("liveout")) + Failed = parseLiveOutMaskOperand(Result); + else if (OperandToken.isKeyword("blockaddress")) + Failed = parseBlockAddressOperand(Result); + else + return fail("unsupported machine operand spelling '" + std::string(OperandToken.spelling()) + + "'"); + if (!Failed) + Result.Operand.setTargetFlags(TargetFlags); + return Failed; + } + + Result = + ParsedOperand{std::move(Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseBlockLiveIns(MachineBasicBlock &Block) { + advance(); + if (expect(MITokenKind::Colon, "':' after liveins")) + return true; + if (token().isNewlineOrEnd()) + return false; + do { + if (!at(MITokenKind::NamedRegister)) + return fail("expected a physical register in liveins"); + Register Reg; + if (parseRegister(Reg) || !Reg.isPhysical()) + return true; + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("expected a physical register in liveins"); + LaneMask Lanes = LaneMask::all(); + if (consume(MITokenKind::Colon)) { + std::uint64_t Raw = 0; + if (parseUnsigned(token(), Raw)) + return fail("invalid live-in lane mask"); + Lanes = LaneMask::fromRaw(Raw); + advance(); + } + auto Added = Block.addLiveIn(*Physical, Lanes); + if (!Added) + return importError(token(), Added.error()); + } while (consume(MITokenKind::Comma)); + return false; +} + +bool MIParser::Impl::parseBlockSuccessors(MachineBasicBlock &Block) { + advance(); + if (expect(MITokenKind::Colon, "':' after successors")) + return true; + ExplicitSuccessorBlocks.insert(&Block); + if (token().isNewlineOrEnd()) + return false; + do { + if (!at(MITokenKind::MachineBasicBlock)) + return fail("expected a machine basic block successor"); + MachineBasicBlock *Successor = blockFor(token()); + if (Successor == nullptr) + return true; + advance(); + BranchProbability Probability = BranchProbability::zero(); + if (consume(MITokenKind::LeftParen)) { + std::uint32_t Weight = 0; + if (parseUnsigned32(token(), Weight)) + return fail("invalid successor probability weight"); + Probability = BranchProbability::fromRaw(Weight); + advance(); + if (expect(MITokenKind::RightParen, "')' after successor weight")) + return true; + } + auto Added = Function.cfgUpdater().addSuccessor(Block, *Successor, Probability); + if (!Added) + return importError(token(), Added.error()); + } while (consume(MITokenKind::Comma)); + return false; +} + +bool MIParser::Impl::parseInstruction(MachineBasicBlock &Block, bool InBundle, + MachineInstr *PreviousInstruction, MachineInstr *&Result) { + if (ParsedInstructionCount >= Options.MaximumInstructions) + return fail("machine instruction count exceeds parser limit"); + + std::vector<ParsedOperand> Operands; + while (token().isRegister() || token().isRegisterFlag()) { + ParsedOperand Operand{MachineOperand::immediate(0), std::nullopt, {}}; + if (parseRegisterOperand(true, Operand)) + return true; + Operands.push_back(std::move(Operand)); + if (!consume(MITokenKind::Comma)) + break; + } + if (!Operands.empty() && expect(MITokenKind::Equal, "'=' after register definitions")) + return true; + + MachineInstrFlags Flags; + while (isInstructionFlag(token())) { + const auto Flag = instructionFlag(token().stringValue()); + if (!Flag) + return fail("unknown instruction flag"); + if (Flags.has(*Flag)) + return fail("duplicate instruction flag '" + std::string(token().stringValue()) + "'"); + Flags.set(*Flag); + advance(); + } + if (!at(MITokenKind::Identifier)) + return fail("expected a machine instruction name"); + const MIToken OpcodeToken = token(); + const InstructionDescriptor *Descriptor = + Target.instructions().descriptor(OpcodeToken.stringValue()); + if (Descriptor == nullptr) + return fail("unknown machine instruction '" + std::string(OpcodeToken.stringValue()) + "'"); + advance(); + + while (!token().isNewlineOrEnd() && !at(MITokenKind::DoubleColon) && + !at(MITokenKind::LeftBrace) && !atKeyword("debug-instr-number") && + !atKeyword("debug-location") && !atKeyword("pre-instr-symbol") && + !atKeyword("post-instr-symbol") && !atKeyword("heap-alloc-marker") && + !atKeyword("pcsections") && !atKeyword("mmra") && !atKeyword("cfi-type") && + !atKeyword("deactivation-symbol")) { + if (Operands.size() >= Options.MaximumOperandsPerInstruction) + return fail("instruction operand count exceeds parser limit"); + ParsedOperand Operand{MachineOperand::immediate(0), std::nullopt, {}}; + if (parseOperand(false, Operand)) + return true; + Operands.push_back(std::move(Operand)); + if (token().isNewlineOrEnd() || at(MITokenKind::DoubleColon) || at(MITokenKind::LeftBrace) || + atKeyword("debug-instr-number") || atKeyword("debug-location")) + break; + if (expect(MITokenKind::Comma, "',' before the next machine operand")) + return true; + } + + std::uint32_t DebugNumber = 0; + std::optional<MachineDebugLocation> DebugLocation; + std::optional<std::string> PreInstrSymbol; + std::optional<std::string> PostInstrSymbol; + std::optional<MachineMetadataReference> HeapAllocMarker; + std::optional<MachineMetadataReference> PCSections; + std::optional<MachineMetadataReference> MMRA; + std::optional<std::uint32_t> CFIType; + std::optional<std::string> DeactivationSymbol; + std::unordered_set<std::string> SeenAttachments; + while (atKeyword("debug-instr-number") || atKeyword("debug-location") || + atKeyword("pre-instr-symbol") || atKeyword("post-instr-symbol") || + atKeyword("heap-alloc-marker") || atKeyword("pcsections") || atKeyword("mmra") || + atKeyword("cfi-type") || atKeyword("deactivation-symbol")) { + const std::string Name(token().stringValue()); + if (!SeenAttachments.insert(Name).second) + return fail("duplicate instruction attachment '" + Name + "'"); + advance(); + if (Name == "debug-instr-number") { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), DebugNumber) || + DebugNumber == 0) { + return fail("invalid debug instruction number"); + } + advance(); + } else if (Name == "debug-location") { + if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DILocation")) { + std::vector<MachineDILocationFrame> InlineChain; + std::optional<MachineMetadataReference> TerminalInlinedAt; + if (parseInlineDILocation(InlineChain, TerminalInlinedAt)) + return true; + auto Location = MachineDILocation::create(std::move(InlineChain), TerminalInlinedAt); + if (!Location) + return importError(OpcodeToken, Location.error()); + DebugLocation = MachineDebugLocation::inlineLocation(std::move(*Location)); + } else { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + auto Created = MachineDebugLocation::reference(Reference); + if (!Created) + return importError(OpcodeToken, Created.error()); + DebugLocation = std::move(*Created); + } + } else if (Name == "pre-instr-symbol" || Name == "post-instr-symbol") { + if (!at(MITokenKind::MCSymbol)) + return fail("expected an MC symbol after '" + Name + "'"); + if (Name == "pre-instr-symbol") + PreInstrSymbol = symbolName(token()); + else + PostInstrSymbol = symbolName(token()); + advance(); + } else if (Name == "heap-alloc-marker" || Name == "pcsections" || Name == "mmra") { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (Name == "heap-alloc-marker") + HeapAllocMarker = Reference; + else if (Name == "pcsections") + PCSections = Reference; + else + MMRA = Reference; + } else if (Name == "cfi-type") { + std::uint32_t Type = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Type) || Type == 0) + return fail("invalid nonzero CFI type"); + CFIType = Type; + advance(); + } else { + if (!at(MITokenKind::GlobalValue) && !at(MITokenKind::NamedGlobalValue)) + return fail("expected a global value after 'deactivation-symbol'"); + DeactivationSymbol = symbolName(token()); + advance(); + } + static_cast<void>(consume(MITokenKind::Comma)); + } + std::vector<const MachineMemOperand *> MemOperands; + if (consume(MITokenKind::DoubleColon)) { + while (!token().isNewlineOrEnd() && !at(MITokenKind::LeftBrace)) { + if (MemOperands.size() >= Options.MaximumMemoryOperandsPerInstruction) + return fail("machine memory operand count exceeds parser limit"); + const MachineMemOperand *MemOperand = nullptr; + if (parseMemoryOperand(MemOperand)) + return true; + MemOperands.push_back(MemOperand); + if (token().isNewlineOrEnd() || at(MITokenKind::LeftBrace)) + break; + if (expect(MITokenKind::Comma, "',' before the next machine memory operand")) + return true; + } + } + + auto Created = Function.createInstruction(Descriptor->InstructionOpcode); + if (!Created) + return importError(OpcodeToken, Created.error()); + MachineInstr *Instruction = *Created; + Instruction->setFlags(Flags); + for (ParsedOperand &Parsed : Operands) { + auto Added = Instruction->addOperand(std::move(Parsed.Operand)); + if (!Added) + return importError(OpcodeToken, Added.error()); + } + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + if (!Operands[Index].TiedDef) + continue; + if (*Operands[Index].TiedDef >= Operands.size()) + return fail(OpcodeToken, "tied-def operand index is out of range"); + auto Tied = Instruction->tieOperands(*Operands[Index].TiedDef, Index); + if (!Tied) + return importError(OpcodeToken, Tied.error()); + } + for (const MachineMemOperand *MemOperand : MemOperands) { + auto Added = Instruction->addMemOperand(*MemOperand); + if (!Added) + return importError(OpcodeToken, Added.error()); + } + if (DebugNumber != 0) { + auto SetNumber = Instruction->setDebugInstrNumber(DebugNumber); + if (!SetNumber) + return importError(OpcodeToken, SetNumber.error()); + } + if (DebugLocation) { + auto SetLocation = Instruction->setDebugLocation(std::move(*DebugLocation)); + if (!SetLocation) + return importError(OpcodeToken, SetLocation.error()); + } + if (PreInstrSymbol) { + auto SetSymbol = Instruction->setPreInstrSymbol(std::move(*PreInstrSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (PostInstrSymbol) { + auto SetSymbol = Instruction->setPostInstrSymbol(std::move(*PostInstrSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (HeapAllocMarker) { + auto SetMarker = Instruction->setHeapAllocMarker(*HeapAllocMarker); + if (!SetMarker) + return importError(OpcodeToken, SetMarker.error()); + } + if (PCSections) { + auto SetSections = Instruction->setPCSections(*PCSections); + if (!SetSections) + return importError(OpcodeToken, SetSections.error()); + } + if (MMRA) { + auto SetMMRA = Instruction->setMMRA(*MMRA); + if (!SetMMRA) + return importError(OpcodeToken, SetMMRA.error()); + } + if (CFIType) { + auto SetType = Instruction->setCFIType(*CFIType); + if (!SetType) + return importError(OpcodeToken, SetType.error()); + } + if (DeactivationSymbol) { + auto SetSymbol = Instruction->setDeactivationSymbol(std::move(*DeactivationSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (InBundle) { + Flags.set(MachineInstrFlag::BundledWithPred); + Instruction->setFlags(Flags); + if (PreviousInstruction != nullptr) { + MachineInstrFlags PreviousFlags = PreviousInstruction->flags(); + PreviousFlags.set(MachineInstrFlag::BundledWithSucc); + PreviousInstruction->setFlags(PreviousFlags); + } + } + auto Appended = Block.appendInstruction(*Instruction); + if (!Appended) + return importError(OpcodeToken, Appended.error()); + + ++ParsedInstructionCount; + Result = Instruction; + return false; +} + +bool MIParser::Impl::parseSecondPass() { + Position = 0; + while (consume(MITokenKind::Newline)) { + } + while (!at(MITokenKind::EndOfFile)) { + if (!at(MITokenKind::MachineBasicBlockLabel)) + return fail("expected a machine basic block definition"); + std::uint32_t ID = 0; + if (parseBlockDefinition(false, ID)) + return true; + MachineBasicBlock *Block = Blocks.at(ID); + + while (consume(MITokenKind::Newline)) { + } + while (atKeyword("liveins") || atKeyword("successors")) { + if ((atKeyword("liveins") && parseBlockLiveIns(*Block)) || + (atKeyword("successors") && parseBlockSuccessors(*Block))) { + return true; + } + if (!token().isNewlineOrEnd()) + return fail("expected a line break after the block list"); + while (consume(MITokenKind::Newline)) { + } + } + + bool InBundle = false; + MachineInstr *PreviousInstruction = nullptr; + while (!at(MITokenKind::MachineBasicBlockLabel) && !at(MITokenKind::EndOfFile)) { + if (consume(MITokenKind::Newline)) + continue; + if (consume(MITokenKind::RightBrace)) { + if (!InBundle) + return fail("closing brace without an active instruction bundle"); + InBundle = false; + continue; + } + + MachineInstr *Instruction = nullptr; + if (parseInstruction(*Block, InBundle, PreviousInstruction, Instruction)) + return true; + PreviousInstruction = Instruction; + if (consume(MITokenKind::LeftBrace)) { + if (InBundle) + return fail("nested instruction bundles are not allowed"); + MachineInstrFlags BundleFlags = Instruction->flags(); + BundleFlags.set(MachineInstrFlag::BundledWithSucc); + Instruction->setFlags(BundleFlags); + InBundle = true; + } + if (!token().isNewlineOrEnd() && !at(MITokenKind::RightBrace)) + return fail("expected a line break after the machine instruction"); + } + if (InBundle) + return fail("unterminated instruction bundle"); + } + return false; +} + +bool MIParser::Impl::inferSuccessors() { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (ExplicitSuccessorBlocks.contains(Block)) + continue; + + bool BlocksFallthrough = true; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = + Target.instructions().descriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return fail("instruction has no target descriptor while inferring successors"); + if (Descriptor->Flags.has(InstructionFlag::Branch)) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const BasicBlockOperand *TargetBlock = Operand.getIf<BasicBlockOperand>()) { + auto Added = Function.cfgUpdater().addSuccessor(*Block, *TargetBlock->Block); + if (!Added) + return importError(token(), Added.error()); + } + } + } + if (Descriptor->Flags.has(InstructionFlag::Barrier)) + BlocksFallthrough = false; + } + if (BlocksFallthrough && Block->next() != nullptr) { + auto Added = Function.cfgUpdater().addSuccessor(*Block, *Block->next()); + if (!Added) + return importError(token(), Added.error()); + } + } + return false; +} + +Expected<void> MIParser::Impl::parse() { + if (ParseAttempted) + return makeUnexpected(ErrorCode::InvalidArgument, "MI parser can only be run once"); + ParseAttempted = true; + if (Function.blockCount() != 0 || !Function.allInstructions().empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "MI parser requires a staging machine function with an empty body"); + } + if (tokenize()) + return failureResult(); + if (parseFirstPass() || parseSecondPass() || inferSuccessors()) + return failureResult(); + + Function.renumberBlocks(); + auto Verified = MachineVerifier::verify( + Function, Target, + MachineVerifierOptions{ + .Mode = MachineVerifierMode::Generic, .Stage = std::nullopt, .VerifyProperties = false}); + if (!Verified) { + static_cast<void>(fail(token(), "parsed machine function failed verification: " + + std::string(Verified.error().message()))); + return failureResult(); + } + return {}; +} + +MIParser::MIParser(MachineFunction &Function, const TargetDescription &Target, + std::string SourceName, std::string SourceText, MIParserContext Context, + MIParserOptions Options) + : Implementation(std::make_unique<Impl>(Function, Target, std::move(SourceName), + std::move(SourceText), std::move(Context), Options)) {} + +MIParser::MIParser(MIParser &&) noexcept = default; + +MIParser &MIParser::operator=(MIParser &&) noexcept = default; + +MIParser::~MIParser() = default; + +Expected<void> MIParser::parse() { + return Implementation->parse(); +} + +const TextSource &MIParser::source() const noexcept { + return Implementation->source(); +} + +std::span<const TextDiagnostic> MIParser::diagnostics() const noexcept { + return Implementation->diagnostics(); +} + +bool MIParser::failed() const noexcept { + return Implementation->failed(); +} + +} // namespace rcc::mir diff --git a/backend/lib/MIR/MIPrinter.cpp b/backend/lib/MIR/MIPrinter.cpp new file mode 100644 index 0000000..e3f504b --- /dev/null +++ b/backend/lib/MIR/MIPrinter.cpp @@ -0,0 +1,1464 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/MIR/MIPrinter.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFIInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace rcc::mir { +namespace { + +[[nodiscard]] bool isBareNameCharacter(char Character) noexcept { + return (Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z') || + (Character >= '0' && Character <= '9') || Character == '_' || Character == '-' || + Character == '.' || Character == '$'; +} + +[[nodiscard]] std::string quoteName(std::string_view Name) { + if (!Name.empty() && std::ranges::all_of(Name, isBareNameCharacter)) + return std::string(Name); + + constexpr std::string_view Hex = "0123456789ABCDEF"; + std::string Result; + Result.reserve(Name.size() + 2U); + Result.push_back('"'); + for (const char RawCharacter : Name) { + const auto Character = static_cast<unsigned char>(RawCharacter); + if (Character >= 0x20U && Character <= 0x7EU && Character != '"' && Character != '\\') { + Result.push_back(static_cast<char>(Character)); + continue; + } + Result.push_back('\\'); + Result.push_back(Hex[Character >> 4U]); + Result.push_back(Hex[Character & 0xFU]); + } + Result.push_back('"'); + return Result; +} + +[[nodiscard]] std::string hexadecimalWords(std::span<const std::uint64_t> Words, + std::uint32_t BitWidth) { + constexpr std::string_view Hex = "0123456789ABCDEF"; + const std::size_t DigitCount = (static_cast<std::size_t>(BitWidth) + 3U) / 4U; + std::string Result(DigitCount, '0'); + for (std::size_t Digit = 0; Digit < DigitCount; ++Digit) { + const std::size_t BitOffset = Digit * 4U; + const std::uint64_t Nibble = (Words[BitOffset / 64U] >> (BitOffset % 64U)) & 0xFU; + Result[DigitCount - Digit - 1U] = Hex[Nibble]; + } + return Result; +} + +[[nodiscard]] std::string unsignedDecimal(std::span<const std::uint64_t> InputWords) { + std::vector<std::uint64_t> Words(InputWords.begin(), InputWords.end()); + while (!Words.empty() && Words.back() == 0) + Words.pop_back(); + if (Words.empty()) + return "0"; + + std::string Reversed; + while (!Words.empty()) { + std::uint64_t Remainder = 0; + for (std::size_t Index = Words.size(); Index != 0; --Index) { + const unsigned __int128 Dividend = + (static_cast<unsigned __int128>(Remainder) << 64U) | Words[Index - 1U]; + Words[Index - 1U] = static_cast<std::uint64_t>(Dividend / 10U); + Remainder = static_cast<std::uint64_t>(Dividend % 10U); + } + Reversed.push_back(static_cast<char>('0' + Remainder)); + while (!Words.empty() && Words.back() == 0) + Words.pop_back(); + } + std::ranges::reverse(Reversed); + return Reversed; +} + +[[nodiscard]] std::string_view floatKeyword(FloatSemantics Semantics) noexcept { + switch (Semantics) { + case FloatSemantics::IEEEHalf: + return "half"; + case FloatSemantics::BFloat: + return "bfloat"; + case FloatSemantics::IEEESingle: + return "float"; + case FloatSemantics::IEEEDouble: + return "double"; + case FloatSemantics::IEEEQuad: + return "fp128"; + case FloatSemantics::PPCDoubleDouble: + case FloatSemantics::PPCDoubleDoubleLegacy: + return "ppc_fp128"; + case FloatSemantics::X87DoubleExtended: + return "x86_fp80"; + default: + return {}; + } +} + +[[nodiscard]] std::string_view integerPredicateName(IntegerPredicate Predicate) noexcept { + switch (Predicate) { + case IntegerPredicate::Equal: + return "eq"; + case IntegerPredicate::NotEqual: + return "ne"; + case IntegerPredicate::SignedGreaterThan: + return "sgt"; + case IntegerPredicate::SignedGreaterEqual: + return "sge"; + case IntegerPredicate::SignedLessThan: + return "slt"; + case IntegerPredicate::SignedLessEqual: + return "sle"; + case IntegerPredicate::UnsignedGreaterThan: + return "ugt"; + case IntegerPredicate::UnsignedGreaterEqual: + return "uge"; + case IntegerPredicate::UnsignedLessThan: + return "ult"; + case IntegerPredicate::UnsignedLessEqual: + return "ule"; + case IntegerPredicate::Count: + return {}; + } + return {}; +} + +[[nodiscard]] std::string_view floatingPredicateName(FloatingPredicate Predicate) noexcept { + switch (Predicate) { + case FloatingPredicate::AlwaysFalse: + return "false"; + case FloatingPredicate::OrderedEqual: + return "oeq"; + case FloatingPredicate::OrderedGreaterThan: + return "ogt"; + case FloatingPredicate::OrderedGreaterEqual: + return "oge"; + case FloatingPredicate::OrderedLessThan: + return "olt"; + case FloatingPredicate::OrderedLessEqual: + return "ole"; + case FloatingPredicate::OrderedNotEqual: + return "one"; + case FloatingPredicate::Ordered: + return "ord"; + case FloatingPredicate::Unordered: + return "uno"; + case FloatingPredicate::UnorderedEqual: + return "ueq"; + case FloatingPredicate::UnorderedGreaterThan: + return "ugt"; + case FloatingPredicate::UnorderedGreaterEqual: + return "uge"; + case FloatingPredicate::UnorderedLessThan: + return "ult"; + case FloatingPredicate::UnorderedLessEqual: + return "ule"; + case FloatingPredicate::UnorderedNotEqual: + return "une"; + case FloatingPredicate::AlwaysTrue: + return "true"; + case FloatingPredicate::Count: + return {}; + } + return {}; +} + +[[nodiscard]] std::string_view atomicOrderingName(AtomicOrdering Ordering) noexcept { + switch (Ordering) { + case AtomicOrdering::Unordered: + return "unordered"; + case AtomicOrdering::Monotonic: + return "monotonic"; + case AtomicOrdering::Acquire: + return "acquire"; + case AtomicOrdering::Release: + return "release"; + case AtomicOrdering::AcquireRelease: + return "acq_rel"; + case AtomicOrdering::SequentiallyConsistent: + return "seq_cst"; + case AtomicOrdering::NotAtomic: + return {}; + } + return {}; +} + +class BodyPrinter final { +public: + BodyPrinter(const MachineFunction &Function, const TargetDescription &Target, + const MIParserContext &Context, MIPrinterOptions Options) + : Function(Function), Target(Target), Context(Context), Options(Options) {} + + [[nodiscard]] Expected<std::string> run(); + +private: + [[nodiscard]] Expected<void> initialize(); + [[nodiscard]] Expected<void> printBlock(const MachineBasicBlock &Block); + [[nodiscard]] Expected<void> printInstruction(const MachineInstr &Instruction); + [[nodiscard]] Expected<void> printOperand(const MachineOperand &Operand, bool IsExplicitDef); + [[nodiscard]] Expected<void> printRegisterOperand(const MachineOperand &Operand, + bool IsExplicitDef); + [[nodiscard]] Expected<void> printTargetFlags(std::uint32_t Flags); + [[nodiscard]] Expected<void> printMetadataReference(MachineMetadataReference Reference); + [[nodiscard]] Expected<void> printDILocation(const MachineDILocation &Location, + std::size_t FrameIndex); + [[nodiscard]] Expected<void> printCFIInstruction(CFIInstructionIndex Index); + [[nodiscard]] Expected<void> printMemoryOperand(const MachineMemOperand &Operand); + [[nodiscard]] Expected<void> printMemoryBase(const MemoryBase &Base, std::int64_t Offset); + + [[nodiscard]] Expected<std::uint32_t> blockID(const MachineBasicBlock &Block) const; + [[nodiscard]] Expected<std::uint32_t> frameID(FrameIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> constantPoolID(ConstantPoolIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> jumpTableID(JumpTableIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> metadataID(MachineMetadataReference Reference) const; + [[nodiscard]] Expected<void> appendPhysicalRegister(PhysicalRegister Register); + void appendBlockReference(const MachineBasicBlock &Block); + void appendOffset(std::int64_t Offset); + + const MachineFunction &Function; + const TargetDescription &Target; + const MIParserContext &Context; + MIPrinterOptions Options; + std::string Output; + std::unordered_map<const MachineBasicBlock *, std::uint32_t> BlockIDs; +}; + +Expected<void> BodyPrinter::initialize() { + std::uint64_t NextID = 0; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + if (NextID > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function has too many blocks to print"); + } + BlockIDs.emplace(Block, static_cast<std::uint32_t>(NextID)); + ++NextID; + } + return {}; +} + +Expected<std::uint32_t> BodyPrinter::blockID(const MachineBasicBlock &Block) const { + const auto Found = BlockIDs.find(&Block); + if (Found == BlockIDs.end()) + return makeUnexpected(ErrorCode::VerificationError, + "machine operand references an unlinked basic block"); + return Found->second; +} + +Expected<std::uint32_t> BodyPrinter::frameID(FrameIndex Index) const { + const auto InStack = std::ranges::find_if( + Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (InStack != Context.StackObjects.end()) + return InStack->TextID; + const auto InFixed = + std::ranges::find_if(Context.FixedStackObjects, + [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (InFixed != Context.FixedStackObjects.end()) + return InFixed->TextID; + return makeUnexpected(ErrorCode::VerificationError, "frame index has no MIR text slot"); +} + +Expected<std::uint32_t> BodyPrinter::constantPoolID(ConstantPoolIndex Index) const { + const auto Found = + std::ranges::find_if(Context.ConstantPoolItems, + [Index](const MIConstantPoolSlot &Slot) { return Slot.Index == Index; }); + if (Found == Context.ConstantPoolItems.end()) { + return makeUnexpected(ErrorCode::VerificationError, "constant-pool index has no MIR text slot"); + } + return Found->TextID; +} + +Expected<std::uint32_t> BodyPrinter::jumpTableID(JumpTableIndex Index) const { + const auto Found = std::ranges::find_if( + Context.JumpTables, [Index](const MIJumpTableSlot &Slot) { return Slot.Index == Index; }); + if (Found == Context.JumpTables.end()) + return makeUnexpected(ErrorCode::VerificationError, "jump-table index has no MIR text slot"); + return Found->TextID; +} + +Expected<std::uint32_t> BodyPrinter::metadataID(MachineMetadataReference Reference) const { + const auto Found = + std::ranges::find_if(Context.Metadata, [Reference](const MIMetadataSlot &Slot) { + return Slot.Reference == Reference; + }); + if (Found != Context.Metadata.end()) + return Found->TextID; + if (Reference.scope() == MachineMetadataScope::MachineFunction) + return Reference.slot(); + return makeUnexpected(ErrorCode::VerificationError, + "embedded metadata reference has no MIR text slot"); +} + +Expected<void> BodyPrinter::appendPhysicalRegister(PhysicalRegister Register) { + const auto Name = Target.registers().physicalRegisterMIRName(Register); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "physical register has no MIR name"); + Output.push_back('$'); + Output.append(*Name); + return {}; +} + +void BodyPrinter::appendBlockReference(const MachineBasicBlock &Block) { + Output.append("%bb."); + Output.append(std::to_string(BlockIDs.at(&Block))); + const auto &Name = Block.attributes().name(); + if (Name) { + Output.push_back('.'); + Output.append(quoteName(*Name)); + } +} + +void BodyPrinter::appendOffset(std::int64_t Offset) { + if (Offset == 0) + return; + if (Offset > 0) { + Output.append(" + "); + Output.append(std::to_string(Offset)); + return; + } + Output.append(" - "); + const std::uint64_t Magnitude = Offset == std::numeric_limits<std::int64_t>::min() + ? std::uint64_t{1} << 63U + : static_cast<std::uint64_t>(-Offset); + Output.append(std::to_string(Magnitude)); +} + +Expected<void> BodyPrinter::printTargetFlags(std::uint32_t Flags) { + if (Flags == 0) + return {}; + auto Names = Target.mirText().operandFlagNames(Flags); + if (!Names) + return std::unexpected(Names.error()); + Output.append("target-flags("); + for (std::size_t Index = 0; Index < Names->size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append((*Names)[Index]); + } + Output.append(") "); + return {}; +} + +Expected<void> BodyPrinter::printMetadataReference(MachineMetadataReference Reference) { + auto ID = metadataID(Reference); + if (!ID) + return std::unexpected(ID.error()); + Output.push_back('!'); + Output.append(std::to_string(*ID)); + return {}; +} + +Expected<void> BodyPrinter::printDILocation(const MachineDILocation &Location, + std::size_t FrameIndex) { + const auto Frames = Location.inlineChain(); + if (FrameIndex >= Frames.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline DILocation has an invalid frame index"); + } + const MachineDILocationFrame &Frame = Frames[FrameIndex]; + Output.append("!DILocation(line: "); + Output.append(std::to_string(Frame.Line)); + if (Options.Mode == MIPrintMode::Complete || Frame.Column != 0) { + Output.append(", column: "); + Output.append(std::to_string(Frame.Column)); + } + Output.append(", scope: "); + if (auto Printed = printMetadataReference(Frame.Scope); !Printed) + return Printed; + if (FrameIndex + 1U < Frames.size()) { + Output.append(", inlinedAt: "); + if (auto Printed = printDILocation(Location, FrameIndex + 1U); !Printed) + return Printed; + } else if (Location.terminalInlinedAt()) { + Output.append(", inlinedAt: "); + if (auto Printed = printMetadataReference(*Location.terminalInlinedAt()); !Printed) { + return Printed; + } + } + if (Options.Mode == MIPrintMode::Complete || Frame.IsImplicitCode) { + Output.append(", isImplicitCode: "); + Output.append(Frame.IsImplicitCode ? "true" : "false"); + } + if (Options.Mode == MIPrintMode::Complete || Frame.AtomGroup != 0) { + Output.append(", atomGroup: "); + Output.append(std::to_string(Frame.AtomGroup)); + } + if (Options.Mode == MIPrintMode::Complete || Frame.AtomRank != 0) { + Output.append(", atomRank: "); + Output.append(std::to_string(Frame.AtomRank)); + } + Output.push_back(')'); + return {}; +} + +Expected<void> BodyPrinter::printRegisterOperand(const MachineOperand &Operand, + bool IsExplicitDef) { + const RegisterOperandFlags Flags = Operand.registerFlags(); + if (Flags.has(RegisterOperandFlag::Implicit) && Flags.has(RegisterOperandFlag::Define)) { + Output.append("implicit-def "); + } else { + if (!IsExplicitDef && Flags.has(RegisterOperandFlag::Define)) + Output.append("def "); + if (Flags.has(RegisterOperandFlag::Implicit)) + Output.append("implicit "); + } + if (Flags.has(RegisterOperandFlag::Dead)) + Output.append("dead "); + if (Flags.has(RegisterOperandFlag::Kill)) + Output.append("killed "); + if (Flags.has(RegisterOperandFlag::Undef)) + Output.append("undef "); + if (Flags.has(RegisterOperandFlag::InternalRead)) + Output.append("internal "); + if (Flags.has(RegisterOperandFlag::EarlyClobber)) + Output.append("early-clobber "); + if (Flags.has(RegisterOperandFlag::Debug)) + Output.append("debug-use "); + if (Flags.has(RegisterOperandFlag::Renamable)) + Output.append("renamable "); + + const auto RegisterValue = Operand.registerValue(); + if (!RegisterValue) { + return makeUnexpected(ErrorCode::VerificationError, "register operand has no register value"); + } + const Register Reg = *RegisterValue; + if (Reg.isNoRegister()) { + Output.append("$noreg"); + } else if (Reg.isPhysical()) { + const auto Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register has no physical-register value"); + } + auto Printed = appendPhysicalRegister(*Physical); + if (!Printed) + return Printed; + } else { + const auto VirtualIndex = Reg.virtualIndex(); + if (!VirtualIndex) + return makeUnexpected(ErrorCode::VerificationError, "virtual register has no numeric index"); + Output.push_back('%'); + Output.append(std::to_string(*VirtualIndex)); + if (Operand.subReg().value() != 0) { + const SubRegIndexDesc *SubReg = Target.registers().subRegIndex(Operand.subReg()); + if (SubReg == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register has an unknown subregister"); + Output.push_back('.'); + Output.append(SubReg->Name); + } + + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const VirtualRegisterInfo &RegisterInfo = Info->get(); + if (RegisterInfo.RegClass) { + const auto Name = Target.registers().registerClassMIRName(*RegisterInfo.RegClass); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register class has no MIR name"); + Output.push_back(':'); + Output.append(*Name); + } else if (RegisterInfo.Bank) { + const auto Name = Target.registers().registerBankMIRName(*RegisterInfo.Bank); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register bank has no MIR name"); + Output.push_back(':'); + Output.append(*Name); + } else if (RegisterInfo.Type) { + Output.append(":_"); + } + if (RegisterInfo.Type) { + Output.push_back('('); + Output.append(RegisterInfo.Type->toString()); + Output.push_back(')'); + } + } + + if (Operand.tiedOperandIndex() && Operand.isUse()) { + Output.append("(tied-def "); + Output.append(std::to_string(*Operand.tiedOperandIndex())); + Output.push_back(')'); + } + return {}; +} + +Expected<void> BodyPrinter::printCFIInstruction(CFIInstructionIndex Index) { + const MachineCFIInstruction *Instruction = Function.cfiInfo().instruction(Index); + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "CFI operand references an unknown instruction"); + const auto PrintRegister = [this](DwarfRegisterNumber Dwarf) -> Expected<void> { + const auto Register = Target.mirText().physicalRegister(Dwarf.value()); + if (!Register) { + return makeUnexpected(ErrorCode::VerificationError, + "CFI instruction has no canonical physical register"); + } + return appendPhysicalRegister(*Register); + }; + const auto PrintRegisterOffset = + [this, &PrintRegister](std::string_view Name, + const MachineCFIRegisterOffsetRule &Rule) -> Expected<void> { + Output.append(Name); + Output.push_back(' '); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + return {}; + }; + + switch (Instruction->operation()) { + case MachineCFIOperation::SameValue: { + Output.append("same_value "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + } + case MachineCFIOperation::RememberState: + Output.append("remember_state"); + return {}; + case MachineCFIOperation::RestoreState: + Output.append("restore_state"); + return {}; + case MachineCFIOperation::Offset: + return PrintRegisterOffset("offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::LLVMDefAddressSpaceCFA: { + const auto &Rule = *Instruction->getIf<MachineCFIAddressSpaceCFARule>(); + Output.append("llvm_def_aspace_cfa "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + Output.append(", "); + Output.append(std::to_string(Rule.AddressSpace)); + return {}; + } + case MachineCFIOperation::DefCFARegister: + Output.append("def_cfa_register "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::DefCFAOffset: + Output.append("def_cfa_offset "); + Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset)); + return {}; + case MachineCFIOperation::DefCFA: + return PrintRegisterOffset("def_cfa", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::RelativeOffset: + return PrintRegisterOffset("rel_offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::AdjustCFAOffset: + Output.append("adjust_cfa_offset "); + Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset)); + return {}; + case MachineCFIOperation::Escape: { + Output.append("escape "); + constexpr std::string_view Hex = "0123456789ABCDEF"; + const auto &Bytes = Instruction->getIf<MachineCFIEscapeRule>()->Bytes; + for (std::size_t ByteIndex = 0; ByteIndex < Bytes.size(); ++ByteIndex) { + if (ByteIndex != 0) + Output.append(", "); + Output.append("0x"); + Output.push_back(Hex[Bytes[ByteIndex] >> 4U]); + Output.push_back(Hex[Bytes[ByteIndex] & 0xFU]); + } + return {}; + } + case MachineCFIOperation::Restore: + Output.append("restore "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::Undefined: + Output.append("undefined "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::Register: { + const auto &Rule = *Instruction->getIf<MachineCFIRegisterRegisterRule>(); + Output.append("register "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + return PrintRegister(Rule.SourceRegister); + } + case MachineCFIOperation::WindowSave: + Output.append("window_save"); + return {}; + case MachineCFIOperation::NegateReturnAddressState: + Output.append("negate_ra_sign_state"); + return {}; + case MachineCFIOperation::NegateReturnAddressStateWithPC: + Output.append("negate_ra_sign_state_with_pc"); + return {}; + case MachineCFIOperation::GNUArgsSize: + case MachineCFIOperation::Label: + case MachineCFIOperation::ValueOffset: + return makeUnexpected( + ErrorCode::Unsupported, + "CFI operation has no textual representation in the fixed LLVM MIR baseline"); + case MachineCFIOperation::LLVMRegisterPair: { + const auto &Rule = *Instruction->getIf<MachineCFIRegisterPairRule>(); + Output.append("llvm_register_pair "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + if (auto Printed = PrintRegister(Rule.FirstRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.FirstSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.SecondRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.SecondSizeInBits)); + return {}; + } + case MachineCFIOperation::LLVMVectorRegisters: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorRegistersRule>(); + Output.append("llvm_vector_registers "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + for (const MachineCFIVectorRegisterLane &Lane : Rule.VectorRegisters) { + Output.append(", "); + if (auto Printed = PrintRegister(Lane.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Lane.Lane)); + Output.append(", "); + Output.append(std::to_string(Lane.SizeInBits)); + } + return {}; + } + case MachineCFIOperation::LLVMVectorOffset: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorOffsetRule>(); + Output.append("llvm_vector_offset "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.RegisterSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.MaskRegisterSizeInBits)); + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + return {}; + } + case MachineCFIOperation::LLVMVectorRegisterMask: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorRegisterMaskRule>(); + Output.append("llvm_vector_register_mask "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + if (auto Printed = PrintRegister(Rule.SpillRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.SpillRegisterLaneSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.MaskRegisterSizeInBits)); + return {}; + } + case MachineCFIOperation::Count: + return makeUnexpected(ErrorCode::VerificationError, "CFI instruction has an invalid operation"); + } + return makeUnexpected(ErrorCode::InternalError, "CFI printer did not handle an operation"); +} + +Expected<void> BodyPrinter::printOperand(const MachineOperand &Operand, bool IsExplicitDef) { + if (Operand.isRegister()) + return printRegisterOperand(Operand, IsExplicitDef); + if (auto Flags = printTargetFlags(Operand.targetFlags()); !Flags) + return Flags; + + switch (Operand.kind()) { + case MachineOperand::Kind::Register: + return makeUnexpected(ErrorCode::InternalError, + "register operand reached the non-register printer"); + case MachineOperand::Kind::Immediate: + Output.append(std::to_string(*Operand.getIf<std::int64_t>())); + return {}; + case MachineOperand::Kind::IntegerImmediate: { + const IntegerImmediate &Value = Operand.getIf<IntegerImmediateOperand>()->Value; + Output.push_back('i'); + Output.append(std::to_string(Value.bitWidth())); + Output.push_back(' '); + Output.append(unsignedDecimal(Value.words())); + return {}; + } + case MachineOperand::Kind::FloatingImmediate: { + const FloatingPointBits &Bits = Operand.getIf<FloatingImmediateOperand>()->Bits; + const std::string_view Keyword = floatKeyword(Bits.semantics()); + if (Keyword.empty()) { + return makeUnexpected(ErrorCode::Unsupported, + "floating-point semantics has no LLVM MIR spelling"); + } + Output.append(Keyword); + Output.append(" f0x"); + Output.append(hexadecimalWords(Bits.words(), Bits.bitWidth())); + return {}; + } + case MachineOperand::Kind::BasicBlock: { + const MachineBasicBlock *Block = Operand.getIf<BasicBlockOperand>()->Block; + if (Block == nullptr || !BlockIDs.contains(Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "basic-block operand references an unlinked block"); + } + appendBlockReference(*Block); + return {}; + } + case MachineOperand::Kind::FrameIndex: { + const auto &Frame = *Operand.getIf<FrameIndexOperand>(); + auto ID = frameID(Frame.Index); + if (!ID) + return std::unexpected(ID.error()); + const auto Stack = std::ranges::find_if( + Context.StackObjects, + [Index = Frame.Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (Stack != Context.StackObjects.end()) { + Output.append("%stack."); + Output.append(std::to_string(*ID)); + if (Stack->Name) { + Output.push_back('.'); + Output.append(quoteName(*Stack->Name)); + } + } else { + Output.append("%fixed-stack."); + Output.append(std::to_string(*ID)); + } + appendOffset(Frame.Offset); + return {}; + } + case MachineOperand::Kind::ConstantPoolIndex: { + const auto &Constant = *Operand.getIf<ConstantPoolOperand>(); + auto ID = constantPoolID(Constant.Index); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%const."); + Output.append(std::to_string(*ID)); + appendOffset(Constant.Offset); + return {}; + } + case MachineOperand::Kind::JumpTableIndex: { + auto ID = jumpTableID(Operand.getIf<JumpTableOperand>()->Index); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%jump-table."); + Output.append(std::to_string(*ID)); + return {}; + } + case MachineOperand::Kind::GlobalSymbol: { + const auto &Global = *Operand.getIf<GlobalSymbolOperand>(); + Output.push_back('@'); + Output.append(quoteName(Global.Name)); + appendOffset(Global.Offset); + return {}; + } + case MachineOperand::Kind::ExternalSymbol: { + const auto &External = *Operand.getIf<ExternalSymbolOperand>(); + Output.push_back('&'); + Output.append(quoteName(External.Name)); + appendOffset(External.Offset); + return {}; + } + case MachineOperand::Kind::BlockAddress: { + const auto &Address = *Operand.getIf<BlockAddressOperand>(); + Output.append("blockaddress(@"); + Output.append(quoteName(Address.FunctionName)); + Output.append(", %ir-block."); + Output.append(quoteName(Address.BlockName)); + Output.push_back(')'); + appendOffset(Address.Offset); + return {}; + } + case MachineOperand::Kind::RegisterMask: { + const RegisterMask &Mask = *Operand.getIf<RegisterMaskOperand>()->Mask; + if (const auto Name = Target.mirText().registerMaskName(Mask)) { + Output.append(*Name); + return {}; + } + Output.append("CustomRegMask("); + bool First = true; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + if (!First) + Output.append(", "); + First = false; + if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed) + return Printed; + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::LiveOutRegisterMask: { + const RegisterMask &Mask = *Operand.getIf<LiveOutRegisterMaskOperand>()->Mask; + Output.append("liveout("); + bool First = true; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + if (!First) + Output.append(", "); + First = false; + if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed) + return Printed; + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::Metadata: { + const MachineMetadataValue &Value = Operand.getIf<MetadataOperand>()->Value; + if (const auto *Reference = Value.reference()) + return printMetadataReference(*Reference); + const MachineDIExpression *Expression = Value.expression(); + if (Expression == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "metadata operand has no printable value"); + } + Output.append("!DIExpression("); + const auto Elements = Expression->elements(); + for (std::size_t Index = 0; Index < Elements.size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append(std::to_string(Elements[Index])); + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::TargetIndex: { + const auto &TargetIndexValue = *Operand.getIf<TargetIndexOperand>(); + const auto Name = Target.mirText().targetIndexName(TargetIndexValue.Index); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "target index has no MIR name"); + Output.append("target-index("); + Output.append(*Name); + Output.push_back(')'); + appendOffset(TargetIndexValue.Offset); + return {}; + } + case MachineOperand::Kind::MCSymbol: { + const auto &Symbol = *Operand.getIf<MCSymbolOperand>(); + Output.append("<mcsymbol "); + Output.append(quoteName(Symbol.Name)); + Output.push_back('>'); + appendOffset(Symbol.Offset); + return {}; + } + case MachineOperand::Kind::CFIIndex: + return printCFIInstruction(Operand.getIf<CFIIndexOperand>()->Index); + case MachineOperand::Kind::Intrinsic: { + const auto Name = Target.mirText().intrinsicName(Operand.getIf<IntrinsicOperand>()->ID); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "intrinsic has no MIR name"); + Output.append("intrinsic(@"); + Output.append(*Name); + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::Predicate: { + const PredicateOperand &Predicate = *Operand.getIf<PredicateOperand>(); + if (const auto *Integer = Predicate.integerPredicate()) { + Output.append("intpred("); + Output.append(integerPredicateName(*Integer)); + } else { + Output.append("floatpred("); + Output.append(floatingPredicateName(*Predicate.floatingPredicate())); + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::ShuffleMask: { + Output.append("shufflemask("); + const auto &Elements = Operand.getIf<ShuffleMaskOperand>()->Elements; + for (std::size_t Index = 0; Index < Elements.size(); ++Index) { + if (Index != 0) + Output.append(", "); + if (Elements[Index] == ShuffleMaskOperand::Undef) + Output.append("undef"); + else + Output.append(std::to_string(Elements[Index])); + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::DebugInstrRef: { + const auto &Reference = *Operand.getIf<DebugInstrRefOperand>(); + Output.append("dbg-instr-ref("); + Output.append(std::to_string(Reference.InstructionNumber)); + Output.append(", "); + Output.append(std::to_string(Reference.OperandNumber)); + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::LaneMask: + Output.append("lanemask(0x"); + Output.append( + hexadecimalWords(std::array{Operand.getIf<LaneMaskOperand>()->Mask.rawValue()}, 64)); + Output.push_back(')'); + return {}; + case MachineOperand::Kind::Count: + return makeUnexpected(ErrorCode::VerificationError, "machine operand has an invalid kind"); + } + return makeUnexpected(ErrorCode::InternalError, "machine operand printer did not handle a kind"); +} + +Expected<void> BodyPrinter::printMemoryBase(const MemoryBase &Base, std::int64_t Offset) { + switch (Base.kind()) { + case MemoryBase::Kind::Unknown: + Output.append("unknown-address"); + break; + case MemoryBase::Kind::FrameIndex: { + const auto Frame = Base.frameIndex(); + if (!Frame) { + return makeUnexpected(ErrorCode::VerificationError, + "frame-index memory base has no frame index"); + } + const FrameIndex Index = *Frame; + auto ID = frameID(Index); + if (!ID) + return std::unexpected(ID.error()); + const auto Stack = + std::ranges::find_if(Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { + return Slot.Index == Index; + }); + if (Stack != Context.StackObjects.end()) { + Output.append("%stack."); + Output.append(std::to_string(*ID)); + if (Stack->Name) { + Output.push_back('.'); + Output.append(quoteName(*Stack->Name)); + } + } else { + Output.append("%fixed-stack."); + Output.append(std::to_string(*ID)); + } + break; + } + case MemoryBase::Kind::ConstantPool: { + const auto Constant = Base.constantPoolIndex(); + if (!Constant) { + return makeUnexpected(ErrorCode::VerificationError, "constant-pool memory base has no index"); + } + auto ID = constantPoolID(*Constant); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%const."); + Output.append(std::to_string(*ID)); + break; + } + case MemoryBase::Kind::JumpTable: { + const auto JumpTable = Base.jumpTableIndex(); + if (!JumpTable) { + return makeUnexpected(ErrorCode::VerificationError, "jump-table memory base has no index"); + } + auto ID = jumpTableID(*JumpTable); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%jump-table."); + Output.append(std::to_string(*ID)); + break; + } + case MemoryBase::Kind::GlobalSymbol: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "global memory base has no symbol name"); + } + Output.push_back('@'); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::ExternalSymbol: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "external memory base has no symbol name"); + } + Output.push_back('&'); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::IRValue: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "IR-value memory base has no value name"); + } + Output.append("%ir."); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::Stack: + Output.append("stack"); + break; + case MemoryBase::Kind::GOT: + Output.append("got"); + break; + case MemoryBase::Kind::ConstantPoolPseudo: + Output.append("constant-pool"); + break; + case MemoryBase::Kind::JumpTablePseudo: + Output.append("jump-table"); + break; + case MemoryBase::Kind::CallEntryGlobal: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "global call-entry memory base has no symbol name"); + } + Output.append("call-entry @"); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::CallEntryExternal: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "external call-entry memory base has no symbol name"); + } + Output.append("call-entry &"); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::Custom: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "custom memory base has no name"); + } + Output.append("custom "); + Output.append(quoteName(*Name)); + break; + } + } + appendOffset(Offset); + return {}; +} + +Expected<void> BodyPrinter::printMemoryOperand(const MachineMemOperand &Operand) { + const MachineMemOperandDesc &Description = Operand.description(); + Output.push_back('('); + if (Description.Accesses.has(MemoryAccessFlag::Volatile)) + Output.append("volatile "); + if (Description.Accesses.has(MemoryAccessFlag::NonTemporal)) + Output.append("non-temporal "); + if (Description.Accesses.has(MemoryAccessFlag::Dereferenceable)) + Output.append("dereferenceable "); + if (Description.Accesses.has(MemoryAccessFlag::Invariant)) + Output.append("invariant "); + if (Description.TargetFlags != 0) { + auto Names = Target.mirText().machineMemOperandFlagNames(Description.TargetFlags); + if (!Names) + return std::unexpected(Names.error()); + for (std::string_view Name : *Names) { + Output.append(quoteName(Name)); + Output.push_back(' '); + } + } + + if (Operand.isLoad()) + Output.append("load"); + else + Output.append("store"); + if (Operand.isLoad() && Operand.isStore()) + Output.append(" store"); + + if (Description.SyncScope.value() != 0) { + const auto Scope = + std::ranges::find_if(Context.SyncScopes, [&Description](const MISyncScopeSlot &Slot) { + return Slot.ID == Description.SyncScope; + }); + if (Scope == Context.SyncScopes.end()) { + return makeUnexpected(ErrorCode::VerificationError, "synchronization scope has no MIR name"); + } + Output.append(" syncscope("); + Output.append(quoteName(Scope->Name)); + Output.push_back(')'); + } + if (Description.SuccessOrdering != AtomicOrdering::NotAtomic) { + Output.push_back(' '); + Output.append(atomicOrderingName(Description.SuccessOrdering)); + } + if (Description.FailureOrdering != AtomicOrdering::NotAtomic) { + Output.push_back(' '); + Output.append(atomicOrderingName(Description.FailureOrdering)); + } + + Output.push_back(' '); + switch (Description.Size.kind()) { + case MemorySize::Kind::Unknown: + Output.append("unknown-size"); + break; + case MemorySize::Kind::Fixed: + Output.append(std::to_string(Description.Size.minimumBytes())); + break; + case MemorySize::Kind::Scalable: + Output.append("(<vscale x "); + Output.append(std::to_string(Description.Size.minimumBytes())); + Output.append(" x s8>)"); + break; + } + + const std::string_view Preposition = + Operand.isLoad() && Operand.isStore() ? " on " : (Operand.isLoad() ? " from " : " into "); + Output.append(Preposition); + if (auto Printed = printMemoryBase(Description.Base, Description.Offset); !Printed) { + return Printed; + } + + Output.append(", align "); + Output.append(std::to_string(Description.Align.bytes())); + if (Description.BaseAlign) { + Output.append(", basealign "); + Output.append(std::to_string(Description.BaseAlign->bytes())); + } + if (Description.AddressSpace != 0) { + Output.append(", addrspace "); + Output.append(std::to_string(Description.AddressSpace)); + } + + const auto PrintMetadata = + [this](std::string_view Name, + const std::optional<MachineMetadataReference> &Reference) -> Expected<void> { + if (!Reference) + return {}; + Output.append(", "); + Output.append(Name); + Output.push_back(' '); + return printMetadataReference(*Reference); + }; + if (auto Printed = PrintMetadata("!tbaa", Description.TBAA); !Printed) + return Printed; + if (auto Printed = PrintMetadata("!alias.scope", Description.AliasScope); !Printed) { + return Printed; + } + if (auto Printed = PrintMetadata("!noalias", Description.NoAlias); !Printed) + return Printed; + if (auto Printed = PrintMetadata("!noalias.addrspace", Description.NoAliasAddressSpace); + !Printed) { + return Printed; + } + if (auto Printed = PrintMetadata("!range", Description.RangeMetadata); !Printed) { + return Printed; + } + Output.push_back(')'); + return {}; +} + +Expected<void> BodyPrinter::printInstruction(const MachineInstr &Instruction) { + const InstructionDescriptor *Descriptor = Target.instructions().descriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine instruction has no target descriptor"); + } + if (Descriptor->NumExplicitDefs > Instruction.operandCount()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine instruction has fewer operands than explicit definitions"); + } + + Output.append(" "); + for (std::size_t Index = 0; Index < Descriptor->NumExplicitDefs; ++Index) { + if (Index != 0) + Output.append(", "); + if (auto Printed = printOperand(*Instruction.operandAt(Index), true); !Printed) { + return Printed; + } + } + if (Descriptor->NumExplicitDefs != 0) + Output.append(" = "); + + const MachineInstrFlags Flags = Instruction.flags(); + constexpr std::array FlagSpellings{ + std::pair{MachineInstrFlag::FrameSetup, std::string_view{"frame-setup"}}, + std::pair{MachineInstrFlag::FrameDestroy, std::string_view{"frame-destroy"}}, + std::pair{MachineInstrFlag::FastMathNoNaNs, std::string_view{"nnan"}}, + std::pair{MachineInstrFlag::FastMathNoInfs, std::string_view{"ninf"}}, + std::pair{MachineInstrFlag::FastMathNoSignedZeros, std::string_view{"nsz"}}, + std::pair{MachineInstrFlag::FastMathAllowReciprocal, std::string_view{"arcp"}}, + std::pair{MachineInstrFlag::FastMathAllowContract, std::string_view{"contract"}}, + std::pair{MachineInstrFlag::FastMathApproxFunc, std::string_view{"afn"}}, + std::pair{MachineInstrFlag::FastMathAllowReassoc, std::string_view{"reassoc"}}, + std::pair{MachineInstrFlag::NoUnsignedWrap, std::string_view{"nuw"}}, + std::pair{MachineInstrFlag::NoSignedWrap, std::string_view{"nsw"}}, + std::pair{MachineInstrFlag::NoUSWrap, std::string_view{"nusw"}}, + std::pair{MachineInstrFlag::Exact, std::string_view{"exact"}}, + std::pair{MachineInstrFlag::NoFPExcept, std::string_view{"nofpexcept"}}, + std::pair{MachineInstrFlag::Unpredictable, std::string_view{"unpredictable"}}, + std::pair{MachineInstrFlag::NoConvergent, std::string_view{"noconvergent"}}, + std::pair{MachineInstrFlag::NonNeg, std::string_view{"nneg"}}, + std::pair{MachineInstrFlag::Disjoint, std::string_view{"disjoint"}}, + std::pair{MachineInstrFlag::SameSign, std::string_view{"samesign"}}, + std::pair{MachineInstrFlag::InBounds, std::string_view{"inbounds"}}, + }; + for (const auto &[Flag, Spelling] : FlagSpellings) { + if (Flags.has(Flag)) { + Output.append(Spelling); + Output.push_back(' '); + } + } + Output.append(Descriptor->Name); + + bool HasPrintedTail = false; + for (std::size_t Index = Descriptor->NumExplicitDefs; Index < Instruction.operandCount(); + ++Index) { + Output.append(HasPrintedTail ? ", " : " "); + HasPrintedTail = true; + if (auto Printed = printOperand(*Instruction.operandAt(Index), false); !Printed) { + return Printed; + } + } + + const auto BeginAttachment = [this, &HasPrintedTail](std::string_view Name) { + Output.append(HasPrintedTail ? ", " : " "); + HasPrintedTail = true; + Output.append(Name); + Output.push_back(' '); + }; + if (Instruction.preInstrSymbol()) { + BeginAttachment("pre-instr-symbol"); + Output.append("<mcsymbol "); + Output.append(quoteName(*Instruction.preInstrSymbol())); + Output.push_back('>'); + } + if (Instruction.postInstrSymbol()) { + BeginAttachment("post-instr-symbol"); + Output.append("<mcsymbol "); + Output.append(quoteName(*Instruction.postInstrSymbol())); + Output.push_back('>'); + } + if (Instruction.heapAllocMarker()) { + BeginAttachment("heap-alloc-marker"); + if (auto Printed = printMetadataReference(*Instruction.heapAllocMarker()); !Printed) { + return Printed; + } + } + if (Instruction.pcSections()) { + BeginAttachment("pcsections"); + if (auto Printed = printMetadataReference(*Instruction.pcSections()); !Printed) { + return Printed; + } + } + if (Instruction.mmra()) { + BeginAttachment("mmra"); + if (auto Printed = printMetadataReference(*Instruction.mmra()); !Printed) + return Printed; + } + if (Instruction.cfiType()) { + BeginAttachment("cfi-type"); + Output.append(std::to_string(*Instruction.cfiType())); + } + if (Instruction.deactivationSymbol()) { + BeginAttachment("deactivation-symbol"); + Output.push_back('@'); + Output.append(quoteName(*Instruction.deactivationSymbol())); + } + if (Instruction.debugInstrNumber() != 0) { + BeginAttachment("debug-instr-number"); + Output.append(std::to_string(Instruction.debugInstrNumber())); + } + if (Instruction.debugLocation()) { + BeginAttachment("debug-location"); + if (const MachineMetadataReference *Reference = Instruction.debugLocation()->reference()) { + if (auto Printed = printMetadataReference(*Reference); !Printed) + return Printed; + } else if (const MachineDILocation *Location = Instruction.debugLocation()->inlineLocation()) { + if (auto Printed = printDILocation(*Location, 0); !Printed) + return Printed; + } else { + return makeUnexpected(ErrorCode::VerificationError, + "machine debug location has no printable value"); + } + } + + if (!Instruction.memOperands().empty()) { + Output.append(" :: "); + for (std::size_t Index = 0; Index < Instruction.memOperands().size(); ++Index) { + if (Index != 0) + Output.append(", "); + if (auto Printed = printMemoryOperand(*Instruction.memOperands()[Index]); !Printed) { + return Printed; + } + } + } + return {}; +} + +Expected<void> BodyPrinter::printBlock(const MachineBasicBlock &Block) { + auto ID = blockID(Block); + if (!ID) + return std::unexpected(ID.error()); + Output.append("bb."); + Output.append(std::to_string(*ID)); + const auto &Name = Block.attributes().name(); + if (Name) { + Output.push_back('.'); + Output.append(quoteName(*Name)); + } + + std::vector<std::string> Attributes; + const auto AddFlag = [&Block, &Attributes](MachineBasicBlockAttribute Attribute, + std::string_view Spelling) { + if (Block.attributes().has(Attribute)) + Attributes.emplace_back(Spelling); + }; + AddFlag(MachineBasicBlockAttribute::MachineBlockAddressTaken, "machine-block-address-taken"); + if (Block.attributes().has(MachineBasicBlockAttribute::IRBlockAddressTaken)) { + std::string Attribute = "ir-block-address-taken"; + const auto &IRBlockName = Block.attributes().irBlockName(); + if (IRBlockName) { + Attribute.append(" %ir-block."); + Attribute.append(quoteName(*IRBlockName)); + } + Attributes.push_back(std::move(Attribute)); + } else { + const auto &IRBlockName = Block.attributes().irBlockName(); + if (IRBlockName) + Attributes.push_back("%ir-block." + quoteName(*IRBlockName)); + } + AddFlag(MachineBasicBlockAttribute::LandingPad, "landing-pad"); + AddFlag(MachineBasicBlockAttribute::InlineAsmBrIndirectTarget, "inlineasm-br-indirect-target"); + AddFlag(MachineBasicBlockAttribute::EHFuncletEntry, "ehfunclet-entry"); + AddFlag(MachineBasicBlockAttribute::EHScopeEntry, "ehscope-entry"); + if (Block.attributes().alignment().bytes() != 1) { + Attributes.push_back("align " + std::to_string(Block.attributes().alignment().bytes())); + } + const MachineBasicBlockSectionID Section = Block.attributes().sectionID(); + if (!Section.isDefault()) { + std::string Value; + if (Section.kind() == MachineBasicBlockSectionKind::Exception) + Value = "Exception"; + else if (Section.kind() == MachineBasicBlockSectionKind::Cold) + Value = "Cold"; + else + Value = std::to_string(Section.number()); + Attributes.push_back("bbsections " + Value); + } + const auto &ProfileID = Block.attributes().profileID(); + if (ProfileID) { + const MachineBasicBlockProfileID Profile = *ProfileID; + Attributes.push_back("bb_id " + std::to_string(Profile.BaseID) + "." + + std::to_string(Profile.CloneID)); + } + if (Block.attributes().callFrameSize() != 0) { + Attributes.push_back("call-frame-size " + std::to_string(Block.attributes().callFrameSize())); + } + if (!Attributes.empty()) { + Output.append(" ("); + for (std::size_t Index = 0; Index < Attributes.size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append(Attributes[Index]); + } + Output.push_back(')'); + } + Output.append(":\n"); + + if (!Block.liveIns().empty()) { + Output.append(" liveins: "); + for (std::size_t Index = 0; Index < Block.liveIns().size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachineBasicBlockLiveIn &LiveIn = Block.liveIns()[Index]; + if (auto Printed = appendPhysicalRegister(LiveIn.PhysReg); !Printed) + return Printed; + if (LiveIn.Lanes != LaneMask::all()) { + Output.append(": 0x"); + Output.append(hexadecimalWords(std::array{LiveIn.Lanes.rawValue()}, 64)); + } + } + Output.push_back('\n'); + } + if (!Block.successors().empty()) { + Output.append(" successors: "); + for (std::size_t Index = 0; Index < Block.successors().size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachineSuccessor &Successor = Block.successors()[Index]; + if (Successor.Block == nullptr || !BlockIDs.contains(Successor.Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "successor references an unlinked block"); + } + appendBlockReference(*Successor.Block); + Output.push_back('('); + Output.append(std::to_string(Successor.Probability.rawValue())); + Output.push_back(')'); + } + Output.push_back('\n'); + } + + bool InBundle = false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (auto Printed = printInstruction(*Instruction); !Printed) + return Printed; + const bool HasPredecessor = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!InBundle && HasSuccessor && !HasPredecessor) { + Output.append(" {"); + InBundle = true; + } + Output.push_back('\n'); + if (InBundle && !HasSuccessor) { + Output.append(" }\n"); + InBundle = false; + } + } + if (InBundle) { + return makeUnexpected(ErrorCode::VerificationError, + "unterminated instruction bundle cannot be printed"); + } + return {}; +} + +Expected<std::string> BodyPrinter::run() { + auto Verification = MachineVerifier::verify( + Function, Target, + MachineVerifierOptions{ + .Mode = MachineVerifierMode::Generic, .Stage = std::nullopt, .VerifyProperties = false}); + if (!Verification) + return std::unexpected(Verification.error()); + if (auto Initialized = initialize(); !Initialized) + return std::unexpected(Initialized.error()); + + bool First = true; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + if (!First) + Output.push_back('\n'); + First = false; + if (auto Printed = printBlock(*Block); !Printed) + return std::unexpected(Printed.error()); + } + return std::move(Output); +} + +} // namespace + +Expected<std::string> printMachineFunctionBody(const MachineFunction &Function, + const TargetDescription &Target, + const MIParserContext &Context, + MIPrinterOptions Options) { + return BodyPrinter(Function, Target, Context, Options).run(); +} + +} // namespace rcc::mir |