[MIR] Parse one-ordering compare-exchange MMOs

4 files changed, 94 insertions, 52 deletions

diff --git a/backend/lib/CodeGen/MachineMemOperand.cpp b/backend/lib/CodeGen/MachineMemOperand.cpp
index 5f126fd..cb90e61 100644
--- a/backend/lib/CodeGen/MachineMemOperand.cpp
+++ b/backend/lib/CodeGen/MachineMemOperand.cpp
@@ -37,13 +37,15 @@ namespace {
 }
 
 [[nodiscard]] bool isValidCompareExchangeFailureOrdering(AtomicOrdering Ordering) noexcept {
-  return Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Acquire ||
-         Ordering == AtomicOrdering::SequentiallyConsistent;
+  return Ordering == AtomicOrdering::NotAtomic || Ordering == AtomicOrdering::Monotonic ||
+         Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::SequentiallyConsistent;
 }
 
 [[nodiscard]] bool isFailureOrderingNoStrongerThanSuccess(AtomicOrdering Failure,
                                                           AtomicOrdering Success) noexcept {
   switch (Failure) {
+  case AtomicOrdering::NotAtomic:
+    return true;
   case AtomicOrdering::Monotonic:
     return isValidAtomicReadModifyWriteOrdering(Success);
   case AtomicOrdering::Acquire:
@@ -51,7 +53,6 @@ namespace {
            Success == AtomicOrdering::SequentiallyConsistent;
   case AtomicOrdering::SequentiallyConsistent:
     return Success == AtomicOrdering::SequentiallyConsistent;
-  case AtomicOrdering::NotAtomic:
   case AtomicOrdering::Unordered:
   case AtomicOrdering::Release:
   case AtomicOrdering::AcquireRelease:
diff --git a/backend/lib/MIR/MIParser.cpp b/backend/lib/MIR/MIParser.cpp
index 01ecbf5..f634119 100644
--- a/backend/lib/MIR/MIParser.cpp
+++ b/backend/lib/MIR/MIParser.cpp
@@ -2,6 +2,7 @@
 
 #include "rcc/MIR/MIParser.h"
 
+#include "rcc/CodeGen/GenericOpcodes.h"
 #include "rcc/CodeGen/InlineAsm.h"
 #include "rcc/CodeGen/InstructionDescriptor.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
@@ -406,7 +407,7 @@ private:
   [[nodiscard]] bool parseCFIUnsigned(std::uint32_t &Result);
   [[nodiscard]] bool parseCFIOperand(ParsedOperand &Result);
   [[nodiscard]] bool parseMemoryBase(MemoryBase &Result, std::int64_t &Offset);
-  [[nodiscard]] bool parseMemoryOperand(const MachineMemOperand *&Result);
+  [[nodiscard]] bool parseMemoryOperand(const MachineMemOperand *&Result, Opcode InstructionOpcode);
   [[nodiscard]] bool parsePredicateOperand(ParsedOperand &Result, bool IsFloating);
   [[nodiscard]] bool parseShuffleMaskOperand(ParsedOperand &Result);
   [[nodiscard]] bool parseDebugInstrRefOperand(ParsedOperand &Result);
@@ -585,8 +586,7 @@ bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Resu
 
   std::uint64_t Value = 0;
   if (!parseUnsignedDigits(Digits, Value))
-    return fail(TypeToken,
-                "expected integers after 's'/'i'/'f'/'bf'/'p' type identifier");
+    return fail(TypeToken, "expected integers after 's'/'i'/'f'/'bf'/'p' type identifier");
 
   Expected<LLT> Created = makeUnexpected(ErrorCode::ParseError, "invalid low-level type");
   switch (Kind) {
@@ -642,15 +642,13 @@ bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LLT &Resu
 bool MIParser::Impl::parseLowLevelType(LLT &Result) {
   const MIToken Begin = token();
   const auto HasScalarPrefix = [](std::string_view Spelling) {
-    return Spelling.starts_with('s') || Spelling.starts_with('i') ||
-           Spelling.starts_with('f') || Spelling.starts_with("bf") ||
-           Spelling.starts_with('p');
+    return Spelling.starts_with('s') || Spelling.starts_with('i') || Spelling.starts_with('f') ||
+           Spelling.starts_with("bf") || Spelling.starts_with('p');
   };
   if (at(MITokenKind::Identifier)) {
     if (!HasScalarPrefix(token().stringValue())) {
-      return fail(Begin,
-                  "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or "
-                  "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}");
+      return fail(Begin, "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or "
+                         "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}");
     }
     if (parseScalarLowLevelType(token(), Result))
       return true;
@@ -658,9 +656,8 @@ bool MIParser::Impl::parseLowLevelType(LLT &Result) {
     return false;
   }
   if (!consume(MITokenKind::Less)) {
-    return fail(Begin,
-                "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or "
-                "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}");
+    return fail(Begin, "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or "
+                       "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}");
   }
 
   bool Scalable = false;
@@ -668,20 +665,17 @@ bool MIParser::Impl::parseLowLevelType(LLT &Result) {
     Scalable = true;
     advance();
     if (!at(MITokenKind::Identifier) || token().stringValue() != "x") {
-      return fail(
-          "expected <vscale x M x tN>, where t = {'s', 'i', 'f', 'bf', 'p'}");
+      return fail("expected <vscale x M x tN>, where t = {'s', 'i', 'f', 'bf', 'p'}");
     }
     advance();
   }
 
   const auto VectorError = [this, &Begin, Scalable] {
     if (Scalable) {
-      return fail(Begin,
-                  "expected <vscale x M x tN> for vector type, where t = "
-                  "{'s', 'i', 'f', 'bf', 'p'}");
+      return fail(Begin, "expected <vscale x M x tN> for vector type, where t = "
+                         "{'s', 'i', 'f', 'bf', 'p'}");
     }
-    return fail(Begin,
-                "expected <M x tN> for vector type, where t = {'s', 'i', 'f', 'bf', 'p'}");
+    return fail(Begin, "expected <M x tN> for vector type, where t = {'s', 'i', 'f', 'bf', 'p'}");
   };
 
   std::uint64_t ElementCount = 0;
@@ -846,13 +840,12 @@ bool MIParser::Impl::parseBlockDefinition(bool Create, std::uint32_t &ID) {
     if (Blocks.size() >= Options.MaximumBlocks)
       return fail(Label, "machine basic block count exceeds parser limit");
     if (Blocks.contains(ID))
-      return fail(Label,
-                  "redefinition of machine basic block with id #" + std::to_string(ID));
+      return fail(Label, "redefinition of machine basic block with id #" + std::to_string(ID));
     if (!Label.stringValue().empty() && Context.HasIRBlock &&
         !Context.HasIRBlock(Label.stringValue())) {
       return fail(Label, "basic block '" + std::string(Label.stringValue()) +
-                             "' is not defined in the function '" +
-                             std::string(Function.name()) + "'");
+                             "' is not defined in the function '" + std::string(Function.name()) +
+                             "'");
     }
     auto Created = Function.createBasicBlock();
     if (!Created)
@@ -1783,7 +1776,8 @@ bool MIParser::Impl::parseMemoryBase(MemoryBase &Result, std::int64_t &Offset) {
   return parseOptionalOffset(Offset);
 }
 
-bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result) {
+bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result,
+                                        Opcode InstructionOpcode) {
   const MIToken Begin = token();
   if (expect(MITokenKind::LeftParen, "'(' before a machine memory operand"))
     return true;
@@ -1870,11 +1864,16 @@ bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result) {
   }
   const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load);
   const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store);
-  if (OrderingCount == 1)
-    Description.AtomicKind = IsLoad && IsStore
-                                 ? AtomicOperation::ReadModifyWrite
-                                 : (IsLoad ? AtomicOperation::Load : AtomicOperation::Store);
-  else if (OrderingCount == 2)
+  if (OrderingCount == 1) {
+    if (InstructionOpcode == opcode(GenericOpcode::G_ATOMIC_CMPXCHG) ||
+        InstructionOpcode == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) {
+      Description.AtomicKind = AtomicOperation::CompareExchange;
+    } else {
+      Description.AtomicKind = IsLoad && IsStore
+                                   ? AtomicOperation::ReadModifyWrite
+                                   : (IsLoad ? AtomicOperation::Load : AtomicOperation::Store);
+    }
+  } else if (OrderingCount == 2)
     Description.AtomicKind = AtomicOperation::CompareExchange;
 
   std::uint64_t MinimumBytes = 0;
@@ -2608,16 +2607,15 @@ bool MIParser::Impl::parseInstruction(MachineBasicBlock &Block, bool InBundle,
     if (Operands.size() >= Options.MaximumOperandsPerInstruction)
       return fail("instruction operand count exceeds parser limit");
     ParsedOperand Operand{MachineOperand::immediate(0), std::nullopt, {}};
-    const bool IsInlineAsmExtraInfo =
-        IsInlineAsm && Operands.size() == InlineAsmExtraInfoOperandIndex &&
-        at(MITokenKind::Identifier);
-    const bool IsInlineAsmOperandFlag =
-        IsInlineAsm && Operands.size() >= InlineAsmFirstConstraintOperandIndex &&
-        at(MITokenKind::Identifier);
-    if (IsInlineAsmExtraInfo
-            ? parseInlineAsmExtraInfoOperand(Operand)
-            : (IsInlineAsmOperandFlag ? parseInlineAsmOperandFlag(Operand)
-                                      : parseOperand(false, Operand)))
+    const bool IsInlineAsmExtraInfo = IsInlineAsm &&
+                                      Operands.size() == InlineAsmExtraInfoOperandIndex &&
+                                      at(MITokenKind::Identifier);
+    const bool IsInlineAsmOperandFlag = IsInlineAsm &&
+                                        Operands.size() >= InlineAsmFirstConstraintOperandIndex &&
+                                        at(MITokenKind::Identifier);
+    if (IsInlineAsmExtraInfo ? parseInlineAsmExtraInfoOperand(Operand)
+                             : (IsInlineAsmOperandFlag ? parseInlineAsmOperandFlag(Operand)
+                                                       : parseOperand(false, Operand)))
       return true;
     Operands.push_back(std::move(Operand));
     if (token().isNewlineOrEnd() || at(MITokenKind::DoubleColon) || at(MITokenKind::LeftBrace) ||
@@ -2708,7 +2706,7 @@ bool MIParser::Impl::parseInstruction(MachineBasicBlock &Block, bool InBundle,
       if (MemOperands.size() >= Options.MaximumMemoryOperandsPerInstruction)
         return fail("machine memory operand count exceeds parser limit");
       const MachineMemOperand *MemOperand = nullptr;
-      if (parseMemoryOperand(MemOperand))
+      if (parseMemoryOperand(MemOperand, Descriptor->InstructionOpcode))
         return true;
       MemOperands.push_back(MemOperand);
       if (token().isNewlineOrEnd() || at(MITokenKind::LeftBrace))
diff --git a/backend/unittests/CodeGen/MachineValueTypesTest.cpp b/backend/unittests/CodeGen/MachineValueTypesTest.cpp
index 50482ba..64d4465 100644
--- a/backend/unittests/CodeGen/MachineValueTypesTest.cpp
+++ b/backend/unittests/CodeGen/MachineValueTypesTest.cpp
@@ -174,6 +174,11 @@ void testMachineMemOperand(TestContext &Context) {
   Context.expect(rcc::MachineMemOperand::create(CompareExchange).has_value(),
                  "compatible compare-exchange orderings must be accepted");
 
+  CompareExchange.FailureOrdering = rcc::AtomicOrdering::NotAtomic;
+  Context.expect(rcc::MachineMemOperand::create(CompareExchange).has_value(),
+                 "LLVM MIR compare-exchange MMO without failure ordering must be accepted");
+
+  CompareExchange.FailureOrdering = rcc::AtomicOrdering::Acquire;
   CompareExchange.SuccessOrdering = rcc::AtomicOrdering::Release;
   Context.expect(!rcc::MachineMemOperand::create(CompareExchange),
                  "failure ordering stronger than success must be rejected");
diff --git a/backend/unittests/MIR/MIParserTest.cpp b/backend/unittests/MIR/MIParserTest.cpp
index 918de62..ab6b69d 100644
--- a/backend/unittests/MIR/MIParserTest.cpp
+++ b/backend/unittests/MIR/MIParserTest.cpp
@@ -3,6 +3,7 @@
 #include "rcc/MIR/MIParser.h"
 
 #include "Target/TestTarget.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
 #include "rcc/CodeGen/InstructionDescriptor.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFunction.h"
@@ -161,6 +162,13 @@ registerOperand(rcc::RegisterOperandRole Role,
   Descriptors.push_back(std::move(Metadata));
   Descriptors.push_back(std::move(Load));
   Descriptors.push_back(std::move(InlineAsm));
+  for (const rcc::InstructionDescriptor &Generic : rcc::genericInstructionDescriptors()) {
+    if (Generic.InstructionOpcode == rcc::opcode(rcc::GenericOpcode::G_ATOMIC_CMPXCHG) ||
+        Generic.InstructionOpcode ==
+            rcc::opcode(rcc::GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) {
+      Descriptors.push_back(Generic);
+    }
+  }
   auto Instructions = rcc::InstructionSet::create(std::move(Descriptors));
   if (!Instructions)
     return std::unexpected(Instructions.error());
@@ -464,17 +472,14 @@ void testTypedIntegerTruncation(TestContext &Context) {
                              ? Instructions[0]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
                              : nullptr;
   const auto *NarrowNegative =
-      Instructions.size() > 1
-          ? Instructions[1]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
-          : nullptr;
+      Instructions.size() > 1 ? Instructions[1]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
+                              : nullptr;
   const auto *NarrowMinimum =
-      Instructions.size() > 2
-          ? Instructions[2]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
-          : nullptr;
+      Instructions.size() > 2 ? Instructions[2]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
+                              : nullptr;
   const auto *WidePositive =
-      Instructions.size() > 3
-          ? Instructions[3]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
-          : nullptr;
+      Instructions.size() > 3 ? Instructions[3]->operandAt(1)->getIf<rcc::IntegerImmediateOperand>()
+                              : nullptr;
   Context.expect(Parsed.has_value() && MinusOne != nullptr &&
                      MinusOne->Value.words()[0] == std::numeric_limits<std::uint64_t>::max() &&
                      NarrowNegative != nullptr && NarrowNegative->Value.bitWidth() == 16 &&
@@ -532,6 +537,38 @@ void testLLVMInlineAsmAndConstantMemorySyntax(TestContext &Context) {
                  "canonical MIR must print symbolic inline-asm flags and LLVM constant bases");
 }
 
+void testLLVMCompareExchangeMemorySyntax(TestContext &Context) {
+  auto Target = createTarget();
+  auto Function = createFunction(Context, 12);
+  Context.expect(Target.has_value(), "compare-exchange target description must be valid");
+  if (!Target || !Function)
+    return;
+
+  const std::string Body = R"(bb.0:
+  %0:gpr64(p0) = TEST_COPY $g0
+  %1:_(s8) = TEST_CONSTANT i8 0
+  %2:_(s8) = TEST_CONSTANT i8 1
+  %3:_(s8) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store monotonic (s8))
+)";
+  rcc::mir::MIParserContext ParserContext;
+  ParserContext.DefaultPointerSizeInBits = 64;
+  rcc::mir::MIParser Parser(*Function, *Target, "llvm-cmpxchg.mir", Body, ParserContext);
+  const auto Parsed = Parser.parse();
+  Context.expect(Parsed.has_value(), "LLVM one-ordering compare-exchange MMO syntax must parse");
+  if (!Parsed) {
+    for (const auto &Diagnostic : Parser.diagnostics())
+      std::cerr << Diagnostic.Category << ": " << Diagnostic.Message << '\n';
+    return;
+  }
+
+  const rcc::MachineInstr *CompareExchange = Function->allInstructions().back();
+  const rcc::MachineMemOperandDesc &Memory = CompareExchange->memOperands().front()->description();
+  Context.expect(Memory.AtomicKind == rcc::AtomicOperation::CompareExchange &&
+                     Memory.SuccessOrdering == rcc::AtomicOrdering::Monotonic &&
+                     Memory.FailureOrdering == rcc::AtomicOrdering::NotAtomic,
+                 "compare-exchange opcode must disambiguate a one-ordering LLVM MMO");
+}
+
 void expectParseError(TestContext &Context, const rcc::TargetDescription &Target, std::string Body,
                       std::string_view Message) {
   auto Function = createFunction(Context, 10);
@@ -583,6 +620,7 @@ int main() {
   testCompatibleRegisterClassAndBankAnnotations(Context);
   testTypedIntegerTruncation(Context);
   testLLVMInlineAsmAndConstantMemorySyntax(Context);
+  testLLVMCompareExchangeMemorySyntax(Context);
   testDiagnostics(Context);
   return Context.result();
 }