code:
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <memory>

#include "common/common_funcs.h"
#include "common/page_table.h"
#include "core/core.h"
#include "core/hle/kernel/k_dynamic_resource_manager.h"
#include "core/hle/kernel/k_light_lock.h"
#include "core/hle/kernel/k_memory_block.h"
#include "core/hle/kernel/k_memory_block_manager.h"
#include "core/hle/kernel/k_memory_layout.h"
#include "core/hle/kernel/k_memory_manager.h"
#include "core/hle/kernel/k_typed_address.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/result.h"
#include "core/memory.h"

namespace Kernel {

enum class DisableMergeAttribute : u8 {
    None = (0U << 0),

    DisableHead = (1U << 0),
    DisableHeadAndBody = (1U << 1),
    EnableHeadAndBody = (1U << 2),
    DisableTail = (1U << 3),
    EnableTail = (1U << 4),
    EnableAndMergeHeadBodyTail = (1U << 5),

    EnableHeadBodyTail = EnableHeadAndBody | EnableTail,
    DisableHeadBodyTail = DisableHeadAndBody | DisableTail,
};
DECLARE_ENUM_FLAG_OPERATORS(DisableMergeAttribute);

struct KPageProperties {
    KMemoryPermission perm;
    bool io;
    bool uncached;
    DisableMergeAttribute disable_merge_attributes;
};
static_assert(std::is_trivial_v<KPageProperties>);
static_assert(sizeof(KPageProperties) == sizeof(u32));

class KResourceLimit;
class KSystemResource;

class KPageTableBase {
    YUZU_NON_COPYABLE(KPageTableBase);
    YUZU_NON_MOVEABLE(KPageTableBase);

public:
    using TraversalEntry = Common::PageTable::TraversalEntry;
    using TraversalContext = Common::PageTable::TraversalContext;

    class MemoryRange {
    private:
        KernelCore& m_kernel;
        KPhysicalAddress m_address;
        size_t m_size;
        bool m_heap;

    public:
        explicit MemoryRange(KernelCore& kernel)
            : m_kernel(kernel), m_address(0), m_size(0), m_heap(false) {}

        void Set(KPhysicalAddress address, size_t size, bool heap) {
            m_address = address;
            m_size = size;
            m_heap = heap;
        }

        KPhysicalAddress GetAddress() const {
            return m_address;
        }
        size_t GetSize() const {
            return m_size;
        }
        bool IsHeap() const {
            return m_heap;
        }

        void Open();
        void Close();
    };

protected:
    enum MemoryFillValue : u8 {
        MemoryFillValue_Zero = 0,
        MemoryFillValue_Stack = 'X',
        MemoryFillValue_Ipc = 'Y',
        MemoryFillValue_Heap = 'Z',
    };

    enum class OperationType {
        Map = 0,
        MapGroup = 1,
        MapFirstGroup = 2,
        Unmap = 3,
        ChangePermissions = 4,
        ChangePermissionsAndRefresh = 5,
        ChangePermissionsAndRefreshAndFlush = 6,
        Separate = 7,

        MapFirstGroupPhysical = 65000,
        UnmapPhysical = 65001,
    };

    static constexpr size_t MaxPhysicalMapAlignment = 1_GiB;
    static constexpr size_t RegionAlignment = 2_MiB;
    static_assert(RegionAlignment == KernelAslrAlignment);

    struct PageLinkedList {
    private:
        struct Node {
            Node* m_next;
            std::array<u8, PageSize - sizeof(Node*)> m_buffer;
        };
        static_assert(std::is_trivial_v<Node>);

    private:
        Node* m_root{};

    public:
        constexpr PageLinkedList() : m_root(nullptr) {}

        void Push(Node* n) {
            ASSERT(Common::IsAligned(reinterpret_cast<uintptr_t>(n), PageSize));
            n->m_next = m_root;
            m_root = n;
        }

        Node* Peek() const {
            return m_root;
        }

        Node* Pop() {
            Node* const r = m_root;

            m_root = r->m_next;
            r->m_next = nullptr;

            return r;
        }
    };
    static_assert(std::is_trivially_destructible_v<PageLinkedList>);

    static constexpr auto DefaultMemoryIgnoreAttr =
        KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared;

    static constexpr size_t GetAddressSpaceWidth(Svc::CreateProcessFlag as_type) {
        switch (static_cast<Svc::CreateProcessFlag>(as_type &
                                                    Svc::CreateProcessFlag::AddressSpaceMask)) {
        case Svc::CreateProcessFlag::AddressSpace64Bit:
            return 39;
        case Svc::CreateProcessFlag::AddressSpace64BitDeprecated:
            return 36;
        case Svc::CreateProcessFlag::AddressSpace32Bit:
        case Svc::CreateProcessFlag::AddressSpace32BitWithoutAlias:
            return 32;
        default:
            UNREACHABLE();
        }
    }

private:
    class KScopedPageTableUpdater {
    private:
        KPageTableBase* m_pt;
        PageLinkedList m_ll;

    public:
        explicit KScopedPageTableUpdater(KPageTableBase* pt) : m_pt(pt), m_ll() {}
        explicit KScopedPageTableUpdater(KPageTableBase& pt)
            : KScopedPageTableUpdater(std::addressof(pt)) {}
        ~KScopedPageTableUpdater() {
            m_pt->FinalizeUpdate(this->GetPageList());
        }

        PageLinkedList* GetPageList() {
            return std::addressof(m_ll);
        }
    };

private:
    KernelCore& m_kernel;
    Core::System& m_system;
    KProcessAddress m_address_space_start{};
    KProcessAddress m_address_space_end{};
    KProcessAddress m_heap_region_start{};
    KProcessAddress m_heap_region_end{};
    KProcessAddress m_current_heap_end{};
    KProcessAddress m_alias_region_start{};
    KProcessAddress m_alias_region_end{};
    KProcessAddress m_stack_region_start{};
    KProcessAddress m_stack_region_end{};
    KProcessAddress m_kernel_map_region_start{};
    KProcessAddress m_kernel_map_region_end{};
    KProcessAddress m_alias_code_region_start{};
    KProcessAddress m_alias_code_region_end{};
    KProcessAddress m_code_region_start{};
    KProcessAddress m_code_region_end{};
    size_t m_max_heap_size{};
    size_t m_mapped_physical_memory_size{};
    size_t m_mapped_unsafe_physical_memory{};
    size_t m_mapped_insecure_memory{};
    size_t m_mapped_ipc_server_memory{};
    mutable KLightLock m_general_lock;
    mutable KLightLock m_map_physical_memory_lock;
    KLightLock m_device_map_lock;
    std::unique_ptr<Common::PageTable> m_impl{};
    Core::Memory::Memory* m_memory{};
    KMemoryBlockManager m_memory_block_manager{};
    u32 m_allocate_option{};
    u32 m_address_space_width{};
    bool m_is_kernel{};
    bool m_enable_aslr{};
    bool m_enable_device_address_space_merge{};
    KMemoryBlockSlabManager* m_memory_block_slab_manager{};
    KBlockInfoManager* m_block_info_manager{};
    KResourceLimit* m_resource_limit{};
    const KMemoryRegion* m_cached_physical_linear_region{};
    const KMemoryRegion* m_cached_physical_heap_region{};
    MemoryFillValue m_heap_fill_value{};
    MemoryFillValue m_ipc_fill_value{};
    MemoryFillValue m_stack_fill_value{};

public:
    explicit KPageTableBase(KernelCore& kernel);
    ~KPageTableBase();

    Result InitializeForKernel(bool is_64_bit, KVirtualAddress start, KVirtualAddress end,
                               Core::Memory::Memory& memory);
    Result InitializeForProcess(Svc::CreateProcessFlag as_type, bool enable_aslr,
                                bool enable_device_address_space_merge, bool from_back,
                                KMemoryManager::Pool pool, KProcessAddress code_address,
                                size_t code_size, KSystemResource* system_resource,
                                KResourceLimit* resource_limit, Core::Memory::Memory& memory,
                                KProcessAddress aslr_space_start);

    Result FinalizeProcess();
    void Finalize();

    bool IsKernel() const {
        return m_is_kernel;
    }
    bool IsAslrEnabled() const {
        return m_enable_aslr;
    }

    bool Contains(KProcessAddress addr) const {
        return m_address_space_start <= addr && addr <= m_address_space_end - 1;
    }

    bool Contains(KProcessAddress addr, size_t size) const {
        return m_address_space_start <= addr && addr < addr + size &&
               addr + size - 1 <= m_address_space_end - 1;
    }

    bool IsInAliasRegion(KProcessAddress addr, size_t size) const {
        return this->Contains(addr, size) && m_alias_region_start <= addr &&
               addr + size - 1 <= m_alias_region_end - 1;
    }

    bool IsInHeapRegion(KProcessAddress addr, size_t size) const {
        return this->Contains(addr, size) && m_heap_region_start <= addr &&
               addr + size - 1 <= m_heap_region_end - 1;
    }

    bool IsInUnsafeAliasRegion(KProcessAddress addr, size_t size) const {
        // Even though Unsafe physical memory is KMemoryState_Normal, it must be mapped inside the
        // alias code region.
        return this->CanContain(addr, size, Svc::MemoryState::AliasCode);
    }

    KScopedLightLock AcquireDeviceMapLock() {
        return KScopedLightLock(m_device_map_lock);
    }

    KProcessAddress GetRegionAddress(Svc::MemoryState state) const;
    size_t GetRegionSize(Svc::MemoryState state) const;
    bool CanContain(KProcessAddress addr, size_t size, Svc::MemoryState state) const;

    KProcessAddress GetRegionAddress(KMemoryState state) const {
        return this->GetRegionAddress(static_cast<Svc::MemoryState>(state & KMemoryState::Mask));
    }
    size_t GetRegionSize(KMemoryState state) const {
        return this->GetRegionSize(static_cast<Svc::MemoryState>(state & KMemoryState::Mask));
    }
    bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const {
        return this->CanContain(addr, size,
                                static_cast<Svc::MemoryState>(state & KMemoryState::Mask));
    }

public:
    Core::Memory::Memory& GetMemory() {
        return *m_memory;
    }

    Core::Memory::Memory& GetMemory() const {
        return *m_memory;
    }

    Common::PageTable& GetImpl() {
        return *m_impl;
    }

    Common::PageTable& GetImpl() const {
        return *m_impl;
    }

    size_t GetNumGuardPages() const {
        return this->IsKernel() ? 1 : 4;
    }

protected:
    // NOTE: These three functions (Operate, Operate, FinalizeUpdate) are virtual functions
    // in Nintendo's kernel. We devirtualize them, since KPageTable is the only derived
    // class, and this avoids unnecessary virtual function calls.
    Result Operate(PageLinkedList* page_list, KProcessAddress virt_addr, size_t num_pages,
                   KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties,
                   OperationType operation, bool reuse_ll);
    Result Operate(PageLinkedList* page_list, KProcessAddress virt_addr, size_t num_pages,
                   const KPageGroup& page_group, const KPageProperties properties,
                   OperationType operation, bool reuse_ll);
    void FinalizeUpdate(PageLinkedList* page_list);

    bool IsLockedByCurrentThread() const {
        return m_general_lock.IsLockedByCurrentThread();
    }

    bool IsLinearMappedPhysicalAddress(KPhysicalAddress phys_addr) {
        ASSERT(this->IsLockedByCurrentThread());

        return m_kernel.MemoryLayout().IsLinearMappedPhysicalAddress(
            m_cached_physical_linear_region, phys_addr);
    }

    bool IsLinearMappedPhysicalAddress(KPhysicalAddress phys_addr, size_t size) {
        ASSERT(this->IsLockedByCurrentThread());

        return m_kernel.MemoryLayout().IsLinearMappedPhysicalAddress(
            m_cached_physical_linear_region, phys_addr, size);
    }

    bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr) {
        ASSERT(this->IsLockedByCurrentThread());

        return m_kernel.MemoryLayout().IsHeapPhysicalAddress(m_cached_physical_heap_region,
                                                             phys_addr);
    }

    bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr, size_t size) {
        ASSERT(this->IsLockedByCurrentThread());

        return m_kernel.MemoryLayout().IsHeapPhysicalAddress(m_cached_physical_heap_region,
                                                             phys_addr, size);
    }

    bool IsHeapPhysicalAddressForFinalize(KPhysicalAddress phys_addr) {
        ASSERT(!this->IsLockedByCurrentThread());

        return m_kernel.MemoryLayout().IsHeapPhysicalAddress(m_cached_physical_heap_region,
                                                             phys_addr);
    }

    bool ContainsPages(KProcessAddress addr, size_t num_pages) const {
        return (m_address_space_start <= addr) &&
               (num_pages <= (m_address_space_end - m_address_space_start) / PageSize) &&
               (addr + num_pages * PageSize - 1 <= m_address_space_end - 1);
    }

private:
    KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages,
                                 size_t num_pages, size_t alignment, size_t offset,
                                 size_t guard_pages) const;

    Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
                                      KMemoryState state_mask, KMemoryState state,
                                      KMemoryPermission perm_mask, KMemoryPermission perm,
                                      KMemoryAttribute attr_mask, KMemoryAttribute attr) const;
    Result CheckMemoryStateContiguous(KProcessAddress addr, size_t size, KMemoryState state_mask,
                                      KMemoryState state, KMemoryPermission perm_mask,
                                      KMemoryPermission perm, KMemoryAttribute attr_mask,
                                      KMemoryAttribute attr) const {
        R_RETURN(this->CheckMemoryStateContiguous(nullptr, addr, size, state_mask, state, perm_mask,
                                                  perm, attr_mask, attr));
    }

    Result CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, KMemoryState state,
                            KMemoryPermission perm_mask, KMemoryPermission perm,
                            KMemoryAttribute attr_mask, KMemoryAttribute attr) const;
    Result CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm,
                            KMemoryAttribute* out_attr, size_t* out_blocks_needed,
                            KMemoryBlockManager::const_iterator it, KProcessAddress last_addr,
                            KMemoryState state_mask, KMemoryState state,
                            KMemoryPermission perm_mask, KMemoryPermission perm,
                            KMemoryAttribute attr_mask, KMemoryAttribute attr,
                            KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const;
    Result CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm,
                            KMemoryAttribute* out_attr, size_t* out_blocks_needed,
                            KProcessAddress addr, size_t size, KMemoryState state_mask,
                            KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm,
                            KMemoryAttribute attr_mask, KMemoryAttribute attr,
                            KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const;
    Result CheckMemoryState(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
                            KMemoryState state_mask, KMemoryState state,
                            KMemoryPermission perm_mask, KMemoryPermission perm,
                            KMemoryAttribute attr_mask, KMemoryAttribute attr,
                            KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const {
        R_RETURN(this->CheckMemoryState(nullptr, nullptr, nullptr, out_blocks_needed, addr, size,
                                        state_mask, state, perm_mask, perm, attr_mask, attr,
                                        ignore_attr));
    }
    Result CheckMemoryState(KProcessAddress addr, size_t size, KMemoryState state_mask,
                            KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm,
                            KMemoryAttribute attr_mask, KMemoryAttribute attr,
                            KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const {
        R_RETURN(this->CheckMemoryState(nullptr, addr, size, state_mask, state, perm_mask, perm,
                                        attr_mask, attr, ignore_attr));
    }

    Result LockMemoryAndOpen(KPageGroup* out_pg, KPhysicalAddress* out_paddr, KProcessAddress addr,
                             size_t size, KMemoryState state_mask, KMemoryState state,
                             KMemoryPermission perm_mask, KMemoryPermission perm,
                             KMemoryAttribute attr_mask, KMemoryAttribute attr,
                             KMemoryPermission new_perm, KMemoryAttribute lock_attr);
    Result UnlockMemory(KProcessAddress addr, size_t size, KMemoryState state_mask,
                        KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm,
                        KMemoryAttribute attr_mask, KMemoryAttribute attr,
                        KMemoryPermission new_perm, KMemoryAttribute lock_attr,
                        const KPageGroup* pg);

    Result QueryInfoImpl(KMemoryInfo* out_info, Svc::PageInfo* out_page,
                         KProcessAddress address) const;

    Result QueryMappingImpl(KProcessAddress* out, KPhysicalAddress address, size_t size,
                            Svc::MemoryState state) const;

    Result AllocateAndMapPagesImpl(PageLinkedList* page_list, KProcessAddress address,
                                   size_t num_pages, KMemoryPermission perm);
    Result MapPageGroupImpl(PageLinkedList* page_list, KProcessAddress address,
                            const KPageGroup& pg, const KPageProperties properties, bool reuse_ll);

    void RemapPageGroup(PageLinkedList* page_list, KProcessAddress address, size_t size,
                        const KPageGroup& pg);

    Result MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_t num_pages);
    bool IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr, size_t num_pages);

    Result GetContiguousMemoryRangeWithState(MemoryRange* out, KProcessAddress address, size_t size,
                                             KMemoryState state_mask, KMemoryState state,
                                             KMemoryPermission perm_mask, KMemoryPermission perm,
                                             KMemoryAttribute attr_mask, KMemoryAttribute attr);

    Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment,
                    KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start,
                    size_t region_num_pages, KMemoryState state, KMemoryPermission perm);

    Result MapIoImpl(KProcessAddress* out, PageLinkedList* page_list, KPhysicalAddress phys_addr,
                     size_t size, KMemoryState state, KMemoryPermission perm);
    Result ReadIoMemoryImpl(KProcessAddress dst_addr, KPhysicalAddress phys_addr, size_t size,
                            KMemoryState state);
    Result WriteIoMemoryImpl(KPhysicalAddress phys_addr, KProcessAddress src_addr, size_t size,
                             KMemoryState state);

    Result SetupForIpcClient(PageLinkedList* page_list, size_t* out_blocks_needed,
                             KProcessAddress address, size_t size, KMemoryPermission test_perm,
                             KMemoryState dst_state);
    Result SetupForIpcServer(KProcessAddress* out_addr, size_t size, KProcessAddress src_addr,
                             KMemoryPermission test_perm, KMemoryState dst_state,
                             KPageTableBase& src_page_table, bool send);
    void CleanupForIpcClientOnServerSetupFailure(PageLinkedList* page_list, KProcessAddress address,
                                                 size_t size, KMemoryPermission prot_perm);

    size_t GetSize(KMemoryState state) const;

    bool GetPhysicalAddressLocked(KPhysicalAddress* out, KProcessAddress virt_addr) const {
        // Validate pre-conditions.
        ASSERT(this->IsLockedByCurrentThread());

        return this->GetImpl().GetPhysicalAddress(out, virt_addr);
    }

public:
    bool GetPhysicalAddress(KPhysicalAddress* out, KProcessAddress virt_addr) const {
        // Validate pre-conditions.
        ASSERT(!this->IsLockedByCurrentThread());

        // Acquire exclusive access to the table while doing address translation.
        KScopedLightLock lk(m_general_lock);

        return this->GetPhysicalAddressLocked(out, virt_addr);
    }

    KBlockInfoManager* GetBlockInfoManager() const {
        return m_block_info_manager;
    }

    Result SetMemoryPermission(KProcessAddress addr, size_t size, Svc::MemoryPermission perm);
    Result SetProcessMemoryPermission(KProcessAddress addr, size_t size,
                                      Svc::MemoryPermission perm);
    Result SetMemoryAttribute(KProcessAddress addr, size_t size, KMemoryAttribute mask,
                              KMemoryAttribute attr);
    Result SetHeapSize(KProcessAddress* out, size_t size);
    Result SetMaxHeapSize(size_t size);
    Result QueryInfo(KMemoryInfo* out_info, Svc::PageInfo* out_page_info,
                     KProcessAddress addr) const;
    Result QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, KProcessAddress address) const;
    Result QueryStaticMapping(KProcessAddress* out, KPhysicalAddress address, size_t size) const {
        R_RETURN(this->QueryMappingImpl(out, address, size, Svc::MemoryState::Static));
    }
    Result QueryIoMapping(KProcessAddress* out, KPhysicalAddress address, size_t size) const {
        R_RETURN(this->QueryMappingImpl(out, address, size, Svc::MemoryState::Io));
    }
    Result MapMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result UnmapMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result MapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result UnmapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
    Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size,
                       Svc::MemoryMapping mapping, Svc::MemoryPermission perm);
    Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size,
                         Svc::MemoryMapping mapping);
    Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
    Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm);
    Result MapInsecureMemory(KProcessAddress address, size_t size);
    Result UnmapInsecureMemory(KProcessAddress address, size_t size);

    Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment,
                    KPhysicalAddress phys_addr, KProcessAddress region_start,
                    size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
        R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true, region_start,
                                region_num_pages, state, perm));
    }

    Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment,
                    KPhysicalAddress phys_addr, KMemoryState state, KMemoryPermission perm) {
        R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true,
                                this->GetRegionAddress(state),
                                this->GetRegionSize(state) / PageSize, state, perm));
    }

    Result MapPages(KProcessAddress* out_addr, size_t num_pages, KMemoryState state,
                    KMemoryPermission perm) {
        R_RETURN(this->MapPages(out_addr, num_pages, PageSize, 0, false,
                                this->GetRegionAddress(state),
                                this->GetRegionSize(state) / PageSize, state, perm));
    }

    Result MapPages(KProcessAddress address, size_t num_pages, KMemoryState state,
                    KMemoryPermission perm);
    Result UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state);

    Result MapPageGroup(KProcessAddress* out_addr, const KPageGroup& pg,
                        KProcessAddress region_start, size_t region_num_pages, KMemoryState state,
                        KMemoryPermission perm);
    Result MapPageGroup(KProcessAddress address, const KPageGroup& pg, KMemoryState state,
                        KMemoryPermission perm);
    Result UnmapPageGroup(KProcessAddress address, const KPageGroup& pg, KMemoryState state);

    Result MakeAndOpenPageGroup(KPageGroup* out, KProcessAddress address, size_t num_pages,
                                KMemoryState state_mask, KMemoryState state,
                                KMemoryPermission perm_mask, KMemoryPermission perm,
                                KMemoryAttribute attr_mask, KMemoryAttribute attr);

    Result InvalidateProcessDataCache(KProcessAddress address, size_t size);
    Result InvalidateCurrentProcessDataCache(KProcessAddress address, size_t size);

    Result ReadDebugMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result ReadDebugIoMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size,
                             KMemoryState state);

    Result WriteDebugMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
    Result WriteDebugIoMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size,
                              KMemoryState state);

    Result LockForMapDeviceAddressSpace(bool* out_is_io, KProcessAddress address, size_t size,
                                        KMemoryPermission perm, bool is_aligned, bool check_heap);
    Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size, bool check_heap);

    Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
    Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size);

    Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange* out,
                                                   KProcessAddress address, size_t size,
                                                   KMemoryPermission perm, bool is_aligned);
    Result OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange* out, KProcessAddress address,
                                                     size_t size);

    Result LockForIpcUserBuffer(KPhysicalAddress* out, KProcessAddress address, size_t size);
    Result UnlockForIpcUserBuffer(KProcessAddress address, size_t size);

    Result LockForTransferMemory(KPageGroup* out, KProcessAddress address, size_t size,
                                 KMemoryPermission perm);
    Result UnlockForTransferMemory(KProcessAddress address, size_t size, const KPageGroup& pg);
    Result LockForCodeMemory(KPageGroup* out, KProcessAddress address, size_t size);
    Result UnlockForCodeMemory(KProcessAddress address, size_t size, const KPageGroup& pg);

    Result OpenMemoryRangeForProcessCacheOperation(MemoryRange* out, KProcessAddress address,
                                                   size_t size);

    Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size,
                                      KProcessAddress src_addr, KMemoryState src_state_mask,
                                      KMemoryState src_state, KMemoryPermission src_test_perm,
                                      KMemoryAttribute src_attr_mask, KMemoryAttribute src_attr);
    Result CopyMemoryFromLinearToKernel(void* buffer, size_t size, KProcessAddress src_addr,
                                        KMemoryState src_state_mask, KMemoryState src_state,
                                        KMemoryPermission src_test_perm,
                                        KMemoryAttribute src_attr_mask, KMemoryAttribute src_attr);
    Result CopyMemoryFromUserToLinear(KProcessAddress dst_addr, size_t size,
                                      KMemoryState dst_state_mask, KMemoryState dst_state,
                                      KMemoryPermission dst_test_perm,
                                      KMemoryAttribute dst_attr_mask, KMemoryAttribute dst_attr,
                                      KProcessAddress src_addr);
    Result CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, size_t size,
                                        KMemoryState dst_state_mask, KMemoryState dst_state,
                                        KMemoryPermission dst_test_perm,
                                        KMemoryAttribute dst_attr_mask, KMemoryAttribute dst_attr,
                                        void* buffer);
    Result CopyMemoryFromHeapToHeap(KPageTableBase& dst_page_table, KProcessAddress dst_addr,
                                    size_t size, KMemoryState dst_state_mask,
                                    KMemoryState dst_state, KMemoryPermission dst_test_perm,
                                    KMemoryAttribute dst_attr_mask, KMemoryAttribute dst_attr,
                                    KProcessAddress src_addr, KMemoryState src_state_mask,
                                    KMemoryState src_state, KMemoryPermission src_test_perm,
                                    KMemoryAttribute src_attr_mask, KMemoryAttribute src_attr);
    Result CopyMemoryFromHeapToHeapWithoutCheckDestination(
        KPageTableBase& dst_page_table, KProcessAddress dst_addr, size_t size,
        KMemoryState dst_state_mask, KMemoryState dst_state, KMemoryPermission dst_test_perm,
        KMemoryAttribute dst_attr_mask, KMemoryAttribute dst_attr, KProcessAddress src_addr,
        KMemoryState src_state_mask, KMemoryState src_state, KMemoryPermission src_test_perm,
        KMemoryAttribute src_attr_mask, KMemoryAttribute src_attr);

    Result SetupForIpc(KProcessAddress* out_dst_addr, size_t size, KProcessAddress src_addr,
                       KPageTableBase& src_page_table, KMemoryPermission test_perm,
                       KMemoryState dst_state, bool send);
    Result CleanupForIpcServer(KProcessAddress address, size_t size, KMemoryState dst_state);
    Result CleanupForIpcClient(KProcessAddress address, size_t size, KMemoryState dst_state);

    Result MapPhysicalMemory(KProcessAddress address, size_t size);
    Result UnmapPhysicalMemory(KProcessAddress address, size_t size);

    Result MapPhysicalMemoryUnsafe(KProcessAddress address, size_t size);
    Result UnmapPhysicalMemoryUnsafe(KProcessAddress address, size_t size);

    Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KPageTableBase& src_pt,
                              KProcessAddress src_address);

public:
    KProcessAddress GetAddressSpaceStart() const {
        return m_address_space_start;
    }
    KProcessAddress GetHeapRegionStart() const {
        return m_heap_region_start;
    }
    KProcessAddress GetAliasRegionStart() const {
        return m_alias_region_start;
    }
    KProcessAddress GetStackRegionStart() const {
        return m_stack_region_start;
    }
    KProcessAddress GetKernelMapRegionStart() const {
        return m_kernel_map_region_start;
    }
    KProcessAddress GetCodeRegionStart() const {
        return m_code_region_start;
    }
    KProcessAddress GetAliasCodeRegionStart() const {
        return m_alias_code_region_start;
    }

    size_t GetAddressSpaceSize() const {
        return m_address_space_end - m_address_space_start;
    }
    size_t GetHeapRegionSize() const {
        return m_heap_region_end - m_heap_region_start;
    }
    size_t GetAliasRegionSize() const {
        return m_alias_region_end - m_alias_region_start;
    }
    size_t GetStackRegionSize() const {
        return m_stack_region_end - m_stack_region_start;
    }
    size_t GetKernelMapRegionSize() const {
        return m_kernel_map_region_end - m_kernel_map_region_start;
    }
    size_t GetCodeRegionSize() const {
        return m_code_region_end - m_code_region_start;
    }
    size_t GetAliasCodeRegionSize() const {
        return m_alias_code_region_end - m_alias_code_region_start;
    }

    size_t GetNormalMemorySize() const {
        // Lock the table.
        KScopedLightLock lk(m_general_lock);

        return (m_current_heap_end - m_heap_region_start) + m_mapped_physical_memory_size;
    }

    size_t GetCodeSize() const;
    size_t GetCodeDataSize() const;
    size_t GetAliasCodeSize() const;
    size_t GetAliasCodeDataSize() const;

    u32 GetAllocateOption() const {
        return m_allocate_option;
    }

    u32 GetAddressSpaceWidth() const {
        return m_address_space_width;
    }

public:
    // Linear mapped
    static u8* GetLinearMappedVirtualPointer(KernelCore& kernel, KPhysicalAddress addr) {
        return kernel.System().DeviceMemory().GetPointer<u8>(addr);
    }

    static KPhysicalAddress GetLinearMappedPhysicalAddress(KernelCore& kernel,
                                                           KVirtualAddress addr) {
        return kernel.MemoryLayout().GetLinearPhysicalAddress(addr);
    }

    static KVirtualAddress GetLinearMappedVirtualAddress(KernelCore& kernel,
                                                         KPhysicalAddress addr) {
        return kernel.MemoryLayout().GetLinearVirtualAddress(addr);
    }

    // Heap
    static u8* GetHeapVirtualPointer(KernelCore& kernel, KPhysicalAddress addr) {
        return kernel.System().DeviceMemory().GetPointer<u8>(addr);
    }

    static KPhysicalAddress GetHeapPhysicalAddress(KernelCore& kernel, KVirtualAddress addr) {
        return GetLinearMappedPhysicalAddress(kernel, addr);
    }

    static KVirtualAddress GetHeapVirtualAddress(KernelCore& kernel, KPhysicalAddress addr) {
        return GetLinearMappedVirtualAddress(kernel, addr);
    }

    // Member heap
    u8* GetHeapVirtualPointer(KPhysicalAddress addr) {
        return GetHeapVirtualPointer(m_kernel, addr);
    }

    KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress addr) {
        return GetHeapPhysicalAddress(m_kernel, addr);
    }

    KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress addr) {
        return GetHeapVirtualAddress(m_kernel, addr);
    }

    // TODO: GetPageTableVirtualAddress
    // TODO: GetPageTablePhysicalAddress
};

} // namespace Kernel

repo_name: litucks/torzu

license: mit


code:
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <atomic>

#include "core/hle/kernel/k_dynamic_resource_manager.h"
#include "core/hle/kernel/k_page_table_slab_heap.h"
#include "core/hle/kernel/k_typed_address.h"

namespace Kernel {

class KPageTableManager : public KDynamicResourceManager<impl::PageTablePage, true> {
public:
    using RefCount = KPageTableSlabHeap::RefCount;
    static constexpr size_t PageTableSize = KPageTableSlabHeap::PageTableSize;

public:
    KPageTableManager() = default;

    void Initialize(KDynamicPageManager* page_allocator, KPageTableSlabHeap* pt_heap) {
        m_pt_heap = pt_heap;

        static_assert(std::derived_from<KPageTableSlabHeap, DynamicSlabType>);
        BaseHeap::Initialize(page_allocator, pt_heap);
    }

    KVirtualAddress Allocate() {
        return KVirtualAddress(BaseHeap::Allocate());
    }

    RefCount GetRefCount(KVirtualAddress addr) const {
        return m_pt_heap->GetRefCount(addr);
    }

    void Open(KVirtualAddress addr, int count) {
        return m_pt_heap->Open(addr, count);
    }

    bool Close(KVirtualAddress addr, int count) {
        return m_pt_heap->Close(addr, count);
    }

    bool IsInPageTableHeap(KVirtualAddress addr) const {
        return m_pt_heap->IsInRange(addr);
    }

private:
    using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;

    KPageTableSlabHeap* m_pt_heap{};
};

} // namespace Kernel


code:
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <array>
#include <vector>

#include "core/hle/kernel/k_dynamic_slab_heap.h"
#include "core/hle/kernel/k_typed_address.h"
#include "core/hle/kernel/slab_helpers.h"

namespace Kernel {

namespace impl {

class PageTablePage {
public:
    // Do not initialize anything.
    PageTablePage() = default;

private:
    // Initializer intentionally skipped
    std::array<u8, PageSize> m_buffer;
};
static_assert(sizeof(PageTablePage) == PageSize);

} // namespace impl

class KPageTableSlabHeap : public KDynamicSlabHeap<impl::PageTablePage, true> {
public:
    using RefCount = u16;
    static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
    static_assert(PageTableSize == PageSize);

public:
    KPageTableSlabHeap() = default;

    static constexpr size_t CalculateReferenceCountSize(size_t size) {
        return (size / PageSize) * sizeof(RefCount);
    }

    void Initialize(KDynamicPageManager* page_allocator, size_t object_count, RefCount* rc) {
        BaseHeap::Initialize(page_allocator, object_count);
        this->Initialize(rc);
    }

    RefCount GetRefCount(KVirtualAddress addr) {
        ASSERT(this->IsInRange(addr));
        return *this->GetRefCountPointer(addr);
    }

    void Open(KVirtualAddress addr, int count) {
        ASSERT(this->IsInRange(addr));

        *this->GetRefCountPointer(addr) += static_cast<RefCount>(count);

        ASSERT(this->GetRefCount(addr) > 0);
    }

    bool Close(KVirtualAddress addr, int count) {
        ASSERT(this->IsInRange(addr));
        ASSERT(this->GetRefCount(addr) >= count);

        *this->GetRefCountPointer(addr) -= static_cast<RefCount>(count);
        return this->GetRefCount(addr) == 0;
    }

    bool IsInPageTableHeap(KVirtualAddress addr) const {
        return this->IsInRange(addr);
    }

private:
    void Initialize([[maybe_unused]] RefCount* rc) {
        // TODO(bunnei): Use rc once we support kernel virtual memory allocations.
        const auto count = this->GetSize() / PageSize;
        m_ref_counts.resize(count);

        for (size_t i = 0; i < count; i++) {
            m_ref_counts[i] = 0;
        }
    }

    RefCount* GetRefCountPointer(KVirtualAddress addr) {
        return m_ref_counts.data() + ((addr - this->GetAddress()) / PageSize);
    }

private:
    using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;

    std::vector<RefCount> m_ref_counts;
};

} // namespace Kernel


code:
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#include "core/hle/kernel/k_port.h"
#include "core/hle/kernel/k_scheduler.h"
#include "core/hle/kernel/svc_results.h"

namespace Kernel {

KPort::KPort(KernelCore& kernel)
    : KAutoObjectWithSlabHeapAndContainer{kernel}, m_server{kernel}, m_client{kernel} {}

KPort::~KPort() = default;

void KPort::Initialize(s32 max_sessions, bool is_light, uintptr_t name) {
    // Open a new reference count to the initialized port.
    this->Open();

    // Create and initialize our server/client pair.
    KAutoObject::Create(std::addressof(m_server));
    KAutoObject::Create(std::addressof(m_client));
    m_server.Initialize(this);
    m_client.Initialize(this, max_sessions);

    // Set our member variables.
    m_is_light = is_light;
    m_name = name;
    m_state = State::Normal;
}

void KPort::OnClientClosed() {
    KScopedSchedulerLock sl{m_kernel};

    if (m_state == State::Normal) {
        m_state = State::ClientClosed;
    }
}

void KPort::OnServerClosed() {
    KScopedSchedulerLock sl{m_kernel};

    if (m_state == State::Normal) {
        m_state = State::ServerClosed;
    }
}

bool KPort::IsServerClosed() const {
    KScopedSchedulerLock sl{m_kernel};
    return m_state == State::ServerClosed;
}

Result KPort::EnqueueSession(KServerSession* session) {
    KScopedSchedulerLock sl{m_kernel};

    R_UNLESS(m_state == State::Normal, ResultPortClosed);

    m_server.EnqueueSession(session);
    R_SUCCEED();
}

Result KPort::EnqueueSession(KLightServerSession* session) {
    KScopedSchedulerLock sl{m_kernel};

    R_UNLESS(m_state == State::Normal, ResultPortClosed);

    m_server.EnqueueSession(session);
    R_SUCCEED();
}

} // namespace Kernel


code:
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <string>

#include "common/common_types.h"
#include "core/hle/kernel/k_client_port.h"
#include "core/hle/kernel/k_server_port.h"
#include "core/hle/kernel/slab_helpers.h"
#include "core/hle/result.h"

namespace Kernel {

class KLightServerSession;
class KServerSession;

class KPort final : public KAutoObjectWithSlabHeapAndContainer<KPort, KAutoObjectWithList> {
    KERNEL_AUTOOBJECT_TRAITS(KPort, KAutoObject);

public:
    explicit KPort(KernelCore& kernel);
    ~KPort() override;

    static void PostDestroy(uintptr_t arg) {}

    void Initialize(s32 max_sessions, bool is_light, uintptr_t name);
    void OnClientClosed();
    void OnServerClosed();

    uintptr_t GetName() const {
        return m_name;
    }
    bool IsLight() const {
        return m_is_light;
    }

    bool IsServerClosed() const;

    Result EnqueueSession(KServerSession* session);
    Result EnqueueSession(KLightServerSession* session);

    KClientPort& GetClientPort() {
        return m_client;
    }
    KServerPort& GetServerPort() {
        return m_server;
    }
    const KClientPort& GetClientPort() const {
        return m_client;
    }
    const KServerPort& GetServerPort() const {
        return m_server;
    }

private:
    enum class State : u8 {
        Invalid = 0,
        Normal = 1,
        ClientClosed = 2,
        ServerClosed = 3,
    };

    KServerPort m_server;
    KClientPort m_client;
    uintptr_t m_name;
    State m_state{State::Invalid};
    bool m_is_light{};
};

} // namespace Kernel


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <array>
#include <bit>
#include <concepts>

#include "common/assert.h"
#include "common/bit_set.h"
#include "common/common_types.h"
#include "common/concepts.h"

namespace Kernel {

class KThread;

template <typename T>
concept KPriorityQueueAffinityMask = !
std::is_reference_v<T>&& requires(T& t) {
                             { t.GetAffinityMask() } -> Common::ConvertibleTo<u64>;
                             { t.SetAffinityMask(0) };

                             { t.GetAffinity(0) } -> std::same_as<bool>;
                             { t.SetAffinity(0, false) };
                             { t.SetAll() };
                         };

template <typename T>
concept KPriorityQueueMember = !
std::is_reference_v<T>&& requires(T& t) {
                             { typename T::QueueEntry() };
                             { (typename T::QueueEntry()).Initialize() };
                             { (typename T::QueueEntry()).SetPrev(std::addressof(t)) };
                             { (typename T::QueueEntry()).SetNext(std::addressof(t)) };
                             { (typename T::QueueEntry()).GetNext() } -> std::same_as<T*>;
                             { (typename T::QueueEntry()).GetPrev() } -> std::same_as<T*>;
                             {
                                 t.GetPriorityQueueEntry(0)
                                 } -> std::same_as<typename T::QueueEntry&>;

                             { t.GetAffinityMask() };
                             {
                                 std::remove_cvref_t<decltype(t.GetAffinityMask())>()
                                 } -> KPriorityQueueAffinityMask;

                             { t.GetActiveCore() } -> Common::ConvertibleTo<s32>;
                             { t.GetPriority() } -> Common::ConvertibleTo<s32>;
                             { t.IsDummyThread() } -> Common::ConvertibleTo<bool>;
                         };

template <typename Member, size_t NumCores_, int LowestPriority, int HighestPriority>
    requires KPriorityQueueMember<Member>
class KPriorityQueue {
public:
    using AffinityMaskType = std::remove_cv_t<
        std::remove_reference_t<decltype(std::declval<Member>().GetAffinityMask())>>;

    static_assert(LowestPriority >= 0);
    static_assert(HighestPriority >= 0);
    static_assert(LowestPriority >= HighestPriority);
    static constexpr size_t NumPriority = LowestPriority - HighestPriority + 1;
    static constexpr size_t NumCores = NumCores_;

    static constexpr bool IsValidCore(s32 core) {
        return 0 <= core && core < static_cast<s32>(NumCores);
    }

    static constexpr bool IsValidPriority(s32 priority) {
        return HighestPriority <= priority && priority <= LowestPriority + 1;
    }

private:
    using Entry = typename Member::QueueEntry;

public:
    class KPerCoreQueue {
    private:
        std::array<Entry, NumCores> m_root{};

    public:
        constexpr KPerCoreQueue() {
            for (auto& per_core_root : m_root) {
                per_core_root.Initialize();
            }
        }

        constexpr bool PushBack(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entry associated with the end of the queue.
            Member* tail = m_root[core].GetPrev();
            Entry& tail_entry =
                (tail != nullptr) ? tail->GetPriorityQueueEntry(core) : m_root[core];

            // Link the entries.
            member_entry.SetPrev(tail);
            member_entry.SetNext(nullptr);
            tail_entry.SetNext(member);
            m_root[core].SetPrev(member);

            return tail == nullptr;
        }

        constexpr bool PushFront(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entry associated with the front of the queue.
            Member* head = m_root[core].GetNext();
            Entry& head_entry =
                (head != nullptr) ? head->GetPriorityQueueEntry(core) : m_root[core];

            // Link the entries.
            member_entry.SetPrev(nullptr);
            member_entry.SetNext(head);
            head_entry.SetPrev(member);
            m_root[core].SetNext(member);

            return (head == nullptr);
        }

        constexpr bool Remove(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entries associated with next and prev.
            Member* prev = member_entry.GetPrev();
            Member* next = member_entry.GetNext();
            Entry& prev_entry =
                (prev != nullptr) ? prev->GetPriorityQueueEntry(core) : m_root[core];
            Entry& next_entry =
                (next != nullptr) ? next->GetPriorityQueueEntry(core) : m_root[core];

            // Unlink.
            prev_entry.SetNext(next);
            next_entry.SetPrev(prev);

            return (this->GetFront(core) == nullptr);
        }

        constexpr Member* GetFront(s32 core) const {
            return m_root[core].GetNext();
        }
    };

    class KPriorityQueueImpl {
    public:
        constexpr KPriorityQueueImpl() = default;

        constexpr void PushBack(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].PushBack(core, member)) {
                m_available_priorities[core].SetBit(priority);
            }
        }

        constexpr void PushFront(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].PushFront(core, member)) {
                m_available_priorities[core].SetBit(priority);
            }
        }

        constexpr void Remove(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].Remove(core, member)) {
                m_available_priorities[core].ClearBit(priority);
            }
        }

        constexpr Member* GetFront(s32 core) const {
            ASSERT(IsValidCore(core));

            const s32 priority = static_cast<s32>(m_available_priorities[core].CountLeadingZero());
            if (priority <= LowestPriority) {
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

        constexpr Member* GetFront(s32 priority, s32 core) const {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

        constexpr Member* GetNext(s32 core, const Member* member) const {
            ASSERT(IsValidCore(core));

            Member* next = member->GetPriorityQueueEntry(core).GetNext();
            if (next == nullptr) {
                const s32 priority = static_cast<s32>(
                    m_available_priorities[core].GetNextSet(member->GetPriority()));
                if (priority <= LowestPriority) {
                    next = m_queues[priority].GetFront(core);
                }
            }
            return next;
        }

        constexpr void MoveToFront(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                m_queues[priority].Remove(core, member);
                m_queues[priority].PushFront(core, member);
            }
        }

        constexpr Member* MoveToBack(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                m_queues[priority].Remove(core, member);
                m_queues[priority].PushBack(core, member);
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

    private:
        std::array<KPerCoreQueue, NumPriority> m_queues{};
        std::array<Common::BitSet64<NumPriority>, NumCores> m_available_priorities{};
    };

private:
    KPriorityQueueImpl m_scheduled_queue;
    KPriorityQueueImpl m_suggested_queue;

private:
    static constexpr void ClearAffinityBit(u64& affinity, s32 core) {
        affinity &= ~(UINT64_C(1) << core);
    }

    static constexpr s32 GetNextCore(u64& affinity) {
        const s32 core = std::countr_zero(affinity);
        ClearAffinityBit(affinity, core);
        return core;
    }

    constexpr void PushBack(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Push onto the scheduled queue for its core, if we can.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.PushBack(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // And suggest the thread for all other cores.
        while (affinity) {
            m_suggested_queue.PushBack(priority, GetNextCore(affinity), member);
        }
    }

    constexpr void PushFront(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Push onto the scheduled queue for its core, if we can.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.PushFront(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // And suggest the thread for all other cores.
        // Note: Nintendo pushes onto the back of the suggested queue, not the front.
        while (affinity) {
            m_suggested_queue.PushBack(priority, GetNextCore(affinity), member);
        }
    }

    constexpr void Remove(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Remove from the scheduled queue for its core.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.Remove(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // Remove from the suggested queue for all other cores.
        while (affinity) {
            m_suggested_queue.Remove(priority, GetNextCore(affinity), member);
        }
    }

public:
    constexpr KPriorityQueue() = default;

    // Getters.
    constexpr Member* GetScheduledFront(s32 core) const {
        return m_scheduled_queue.GetFront(core);
    }

    constexpr Member* GetScheduledFront(s32 core, s32 priority) const {
        return m_scheduled_queue.GetFront(priority, core);
    }

    constexpr Member* GetSuggestedFront(s32 core) const {
        return m_suggested_queue.GetFront(core);
    }

    constexpr Member* GetSuggestedFront(s32 core, s32 priority) const {
        return m_suggested_queue.GetFront(priority, core);
    }

    constexpr Member* GetScheduledNext(s32 core, const Member* member) const {
        return m_scheduled_queue.GetNext(core, member);
    }

    constexpr Member* GetSuggestedNext(s32 core, const Member* member) const {
        return m_suggested_queue.GetNext(core, member);
    }

    constexpr Member* GetSamePriorityNext(s32 core, const Member* member) const {
        return member->GetPriorityQueueEntry(core).GetNext();
    }

    // Mutators.
    constexpr void PushBack(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        this->PushBack(member->GetPriority(), member);
    }

    constexpr void Remove(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        this->Remove(member->GetPriority(), member);
    }

    constexpr void MoveToScheduledFront(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        m_scheduled_queue.MoveToFront(member->GetPriority(), member->GetActiveCore(), member);
    }

    constexpr KThread* MoveToScheduledBack(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return {};
        }

        return m_scheduled_queue.MoveToBack(member->GetPriority(), member->GetActiveCore(), member);
    }

    // First class fancy operations.
    constexpr void ChangePriority(s32 prev_priority, bool is_running, Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        ASSERT(IsValidPriority(prev_priority));

        // Remove the member from the queues.
        const s32 new_priority = member->GetPriority();
        this->Remove(prev_priority, member);

        // And enqueue. If the member is running, we want to keep it running.
        if (is_running) {
            this->PushFront(new_priority, member);
        } else {
            this->PushBack(new_priority, member);
        }
    }

    constexpr void ChangeAffinityMask(s32 prev_core, const AffinityMaskType& prev_affinity,
                                      Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        // Get the new information.
        const s32 priority = member->GetPriority();
        const AffinityMaskType& new_affinity = member->GetAffinityMask();
        const s32 new_core = member->GetActiveCore();

        // Remove the member from all queues it was in before.
        for (s32 core = 0; core < static_cast<s32>(NumCores); core++) {
            if (prev_affinity.GetAffinity(core)) {
                if (core == prev_core) {
                    m_scheduled_queue.Remove(priority, core, member);
                } else {
                    m_suggested_queue.Remove(priority, core, member);
                }
            }
        }

        // And add the member to all queues it should be in now.
        for (s32 core = 0; core < static_cast<s32>(NumCores); core++) {
            if (new_affinity.GetAffinity(core)) {
                if (core == new_core) {
                    m_scheduled_queue.PushBack(priority, core, member);
                } else {
                    m_suggested_queue.PushBack(priority, core, member);
                }
            }
        }
    }

    constexpr void ChangeCore(s32 prev_core, Member* member, bool to_front = false) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        // Get the new information.
        const s32 new_core = member->GetActiveCore();
        const s32 priority = member->GetPriority();

        // We don't need to do anything if the core is the same.
        if (prev_core != new_core) {
            // Remove from the scheduled queue for the previous core.
            if (prev_core >= 0) {
                m_scheduled_queue.Remove(priority, prev_core, member);
            }

            // Remove from the suggested queue and add to the scheduled queue for the new core.
            if (new_core >= 0) {
                m_suggested_queue.Remove(priority, new_core, member);
                if (to_front) {
                    m_scheduled_queue.PushFront(priority, new_core, member);
                } else {
                    m_scheduled_queue.PushBack(priority, new_core, member);
                }
            }

            // Add to the suggested queue for the previous core.
            if (prev_core >= 0) {
                m_suggested_queue.PushBack(priority, prev_core, member);
            }
        }
    }
};

} // namespace Kernel