// SPDX-License-Identifier: GPL-3.0-or-later #include "rcc/CodeGen/MachineMemOperand.h" #include #include namespace rcc { namespace { [[nodiscard]] bool isValidAtomicLoadOrdering(AtomicOrdering Ordering) noexcept { return Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::SequentiallyConsistent; } [[nodiscard]] bool isValidAtomicStoreOrdering(AtomicOrdering Ordering) noexcept { return Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Release || Ordering == AtomicOrdering::SequentiallyConsistent; } [[nodiscard]] bool isValidAtomicReadModifyWriteOrdering(AtomicOrdering Ordering) noexcept { return Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::Release || Ordering == AtomicOrdering::AcquireRelease || Ordering == AtomicOrdering::SequentiallyConsistent; } [[nodiscard]] bool isValidCompareExchangeFailureOrdering(AtomicOrdering Ordering) noexcept { return Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::SequentiallyConsistent; } [[nodiscard]] bool isFailureOrderingNoStrongerThanSuccess(AtomicOrdering Failure, AtomicOrdering Success) noexcept { switch (Failure) { case AtomicOrdering::Monotonic: return isValidAtomicReadModifyWriteOrdering(Success); case AtomicOrdering::Acquire: return Success == AtomicOrdering::Acquire || Success == AtomicOrdering::AcquireRelease || Success == AtomicOrdering::SequentiallyConsistent; case AtomicOrdering::SequentiallyConsistent: return Success == AtomicOrdering::SequentiallyConsistent; case AtomicOrdering::NotAtomic: case AtomicOrdering::Unordered: case AtomicOrdering::Release: case AtomicOrdering::AcquireRelease: return false; } return false; } [[nodiscard]] Expected verifyAtomicDescription(const MachineMemOperandDesc &Description) { const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); switch (Description.AtomicKind) { case AtomicOperation::None: if (Description.SuccessOrdering != AtomicOrdering::NotAtomic || Description.FailureOrdering != AtomicOrdering::NotAtomic) { return makeUnexpected(ErrorCode::InvalidArgument, "non-atomic memory operands cannot carry atomic orderings"); } return {}; case AtomicOperation::Load: if (!IsLoad || IsStore) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic load memory operands must be load-only"); } if (!isValidAtomicLoadOrdering(Description.SuccessOrdering) || Description.FailureOrdering != AtomicOrdering::NotAtomic) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic load memory operand has an invalid ordering"); } return {}; case AtomicOperation::Store: if (IsLoad || !IsStore) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic store memory operands must be store-only"); } if (!isValidAtomicStoreOrdering(Description.SuccessOrdering) || Description.FailureOrdering != AtomicOrdering::NotAtomic) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic store memory operand has an invalid ordering"); } return {}; case AtomicOperation::ReadModifyWrite: if (!IsLoad || !IsStore) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic read-modify-write memory operands must load and store"); } if (!isValidAtomicReadModifyWriteOrdering(Description.SuccessOrdering) || Description.FailureOrdering != AtomicOrdering::NotAtomic) { return makeUnexpected(ErrorCode::InvalidArgument, "atomic read-modify-write memory operand has an invalid ordering"); } return {}; case AtomicOperation::CompareExchange: if (!IsLoad || !IsStore) { return makeUnexpected(ErrorCode::InvalidArgument, "compare-exchange memory operands must load and store"); } if (!isValidAtomicReadModifyWriteOrdering(Description.SuccessOrdering) || !isValidCompareExchangeFailureOrdering(Description.FailureOrdering) || !isFailureOrderingNoStrongerThanSuccess(Description.FailureOrdering, Description.SuccessOrdering)) { return makeUnexpected(ErrorCode::InvalidArgument, "compare-exchange memory operand has incompatible orderings"); } return {}; } return makeUnexpected(ErrorCode::InternalError, "unknown atomic operation"); } } // namespace MemoryAccessFlags::MemoryAccessFlags(std::initializer_list Flags) noexcept { for (const MemoryAccessFlag Flag : Flags) set(Flag); } Expected Alignment::create(std::uint64_t Bytes) { if (!std::has_single_bit(Bytes)) { return makeUnexpected(ErrorCode::InvalidArgument, "alignment must be a nonzero power of two"); } return Alignment(Bytes); } Expected MemorySize::fixed(std::uint64_t Bytes) { if (Bytes == 0) return makeUnexpected(ErrorCode::InvalidArgument, "fixed memory size must be nonzero"); return MemorySize(Kind::Fixed, Bytes); } Expected MemorySize::scalable(std::uint64_t MinimumBytes) { if (MinimumBytes == 0) { return makeUnexpected(ErrorCode::InvalidArgument, "scalable memory size minimum must be nonzero"); } return MemorySize(Kind::Scalable, MinimumBytes); } Expected MemoryOffsetRange::create(std::int64_t Lower, std::int64_t Upper) { if (Lower > Upper) { return makeUnexpected(ErrorCode::InvalidArgument, "memory offset range lower bound cannot exceed its upper bound"); } return MemoryOffsetRange{Lower, Upper}; } MemoryBase MemoryBase::frameIndex(FrameIndex Index) { return MemoryBase(Kind::FrameIndex, Index); } MemoryBase MemoryBase::constantPool(ConstantPoolIndex Index) { return MemoryBase(Kind::ConstantPool, Index); } MemoryBase MemoryBase::jumpTable(JumpTableIndex Index) { return MemoryBase(Kind::JumpTable, Index); } MemoryBase MemoryBase::globalSymbol(std::string Name) { return MemoryBase(Kind::GlobalSymbol, std::move(Name)); } MemoryBase MemoryBase::externalSymbol(std::string Name) { return MemoryBase(Kind::ExternalSymbol, std::move(Name)); } std::optional MemoryBase::frameIndex() const noexcept { if (BaseKind != Kind::FrameIndex) return std::nullopt; return std::get(Value); } std::optional MemoryBase::constantPoolIndex() const noexcept { if (BaseKind != Kind::ConstantPool) return std::nullopt; return std::get(Value); } std::optional MemoryBase::jumpTableIndex() const noexcept { if (BaseKind != Kind::JumpTable) return std::nullopt; return std::get(Value); } std::optional MemoryBase::symbolName() const noexcept { if (BaseKind != Kind::GlobalSymbol && BaseKind != Kind::ExternalSymbol) return std::nullopt; return std::get(Value); } Expected MachineMemOperand::create(MachineMemOperandDesc Description) { const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); if (!IsLoad && !IsStore) { return makeUnexpected(ErrorCode::InvalidArgument, "machine memory operand must describe a load or store"); } if (Description.AddressSpace > LLT::MaxAddressSpace) { return makeUnexpected(ErrorCode::InvalidArgument, "machine memory operand address space must fit in 24 bits"); } if (Description.Accesses.has(MemoryAccessFlag::Invariant) && (!IsLoad || IsStore)) { return makeUnexpected(ErrorCode::InvalidArgument, "invariant memory operands must be load-only"); } const auto AtomicResult = verifyAtomicDescription(Description); if (!AtomicResult) return std::unexpected(AtomicResult.error()); const auto SymbolName = Description.Base.symbolName(); if (SymbolName && SymbolName->empty()) { return makeUnexpected(ErrorCode::InvalidArgument, "memory base symbol name cannot be empty"); } return MachineMemOperand(std::move(Description)); } std::string_view getAtomicOrderingName(AtomicOrdering Ordering) noexcept { switch (Ordering) { case AtomicOrdering::NotAtomic: return "not-atomic"; case AtomicOrdering::Unordered: return "unordered"; case AtomicOrdering::Monotonic: return "monotonic"; case AtomicOrdering::Acquire: return "acquire"; case AtomicOrdering::Release: return "release"; case AtomicOrdering::AcquireRelease: return "acquire-release"; case AtomicOrdering::SequentiallyConsistent: return "sequentially-consistent"; } return "unknown"; } } // namespace rcc