code:
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later

#include <array>
#include <vector>
#include "common/assert.h"
#include "common/scope_exit.h"
#include "video_core/dirty_flags.h"
#include "video_core/engines/draw_manager.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/macro/macro.h"
#include "video_core/macro/macro_hle.h"
#include "video_core/memory_manager.h"
#include "video_core/rasterizer_interface.h"

namespace Tegra {

using Maxwell3D = Engines::Maxwell3D;

namespace {

bool IsTopologySafe(Maxwell3D::Regs::PrimitiveTopology topology) {
    switch (topology) {
    case Maxwell3D::Regs::PrimitiveTopology::Points:
    case Maxwell3D::Regs::PrimitiveTopology::Lines:
    case Maxwell3D::Regs::PrimitiveTopology::LineLoop:
    case Maxwell3D::Regs::PrimitiveTopology::LineStrip:
    case Maxwell3D::Regs::PrimitiveTopology::Triangles:
    case Maxwell3D::Regs::PrimitiveTopology::TriangleStrip:
    case Maxwell3D::Regs::PrimitiveTopology::TriangleFan:
    case Maxwell3D::Regs::PrimitiveTopology::LinesAdjacency:
    case Maxwell3D::Regs::PrimitiveTopology::LineStripAdjacency:
    case Maxwell3D::Regs::PrimitiveTopology::TrianglesAdjacency:
    case Maxwell3D::Regs::PrimitiveTopology::TriangleStripAdjacency:
    case Maxwell3D::Regs::PrimitiveTopology::Patches:
        return true;
    case Maxwell3D::Regs::PrimitiveTopology::Quads:
    case Maxwell3D::Regs::PrimitiveTopology::QuadStrip:
    case Maxwell3D::Regs::PrimitiveTopology::Polygon:
    default:
        return false;
    }
}

class HLEMacroImpl : public CachedMacro {
public:
    explicit HLEMacroImpl(Maxwell3D& maxwell3d_) : maxwell3d{maxwell3d_} {}

protected:
    Maxwell3D& maxwell3d;
};

/*
 * @note: these macros have two versions, a normal and extended version, with the extended version
 * also assigning the base vertex/instance.
 */
template <bool extended>
class HLE_DrawArraysIndirect final : public HLEMacroImpl {
public:
    explicit HLE_DrawArraysIndirect(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[0]);
        if (!maxwell3d.AnyParametersDirty() || !IsTopologySafe(topology)) {
            Fallback(parameters);
            return;
        }

        auto& params = maxwell3d.draw_manager->GetIndirectParams();
        params.is_byte_count = false;
        params.is_indexed = false;
        params.include_count = false;
        params.count_start_address = 0;
        params.indirect_start_address = maxwell3d.GetMacroAddress(1);
        params.buffer_size = 4 * sizeof(u32);
        params.max_draw_counts = 1;
        params.stride = 0;

        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
        }

        maxwell3d.draw_manager->DrawArrayIndirect(topology);

        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::None;
            maxwell3d.replace_table.clear();
        }
    }

private:
    void Fallback(const std::vector<u32>& parameters) {
        SCOPE_EXIT {
            if (extended) {
                maxwell3d.engine_state = Maxwell3D::EngineHint::None;
                maxwell3d.replace_table.clear();
            }
        };
        maxwell3d.RefreshParameters();
        const u32 instance_count = (maxwell3d.GetRegisterValue(0xD1B) & parameters[2]);

        auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[0]);
        const u32 vertex_first = parameters[3];
        const u32 vertex_count = parameters[1];

        if (!IsTopologySafe(topology) &&
            static_cast<size_t>(maxwell3d.GetMaxCurrentVertices()) <
                static_cast<size_t>(vertex_first) + static_cast<size_t>(vertex_count)) {
            ASSERT_MSG(false, "Faulty draw!");
            return;
        }

        const u32 base_instance = parameters[4];
        if constexpr (extended) {
            maxwell3d.regs.global_base_instance_index = base_instance;
            maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
        }

        maxwell3d.draw_manager->DrawArray(topology, vertex_first, vertex_count, base_instance,
                                          instance_count);

        if constexpr (extended) {
            maxwell3d.regs.global_base_instance_index = 0;
            maxwell3d.engine_state = Maxwell3D::EngineHint::None;
            maxwell3d.replace_table.clear();
        }
    }
};

/*
 * @note: these macros have two versions, a normal and extended version, with the extended version
 * also assigning the base vertex/instance.
 */
template <bool extended>
class HLE_DrawIndexedIndirect final : public HLEMacroImpl {
public:
    explicit HLE_DrawIndexedIndirect(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[0]);
        if (!maxwell3d.AnyParametersDirty() || !IsTopologySafe(topology)) {
            Fallback(parameters);
            return;
        }

        const u32 estimate = static_cast<u32>(maxwell3d.EstimateIndexBufferSize());
        const u32 element_base = parameters[4];
        const u32 base_instance = parameters[5];
        maxwell3d.regs.vertex_id_base = element_base;
        maxwell3d.regs.global_base_vertex_index = element_base;
        maxwell3d.regs.global_base_instance_index = base_instance;
        maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseVertex);
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x644, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
        }
        auto& params = maxwell3d.draw_manager->GetIndirectParams();
        params.is_byte_count = false;
        params.is_indexed = true;
        params.include_count = false;
        params.count_start_address = 0;
        params.indirect_start_address = maxwell3d.GetMacroAddress(1);
        params.buffer_size = 5 * sizeof(u32);
        params.max_draw_counts = 1;
        params.stride = 0;
        maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
        maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate);
        maxwell3d.regs.vertex_id_base = 0x0;
        maxwell3d.regs.global_base_vertex_index = 0x0;
        maxwell3d.regs.global_base_instance_index = 0x0;
        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::None;
            maxwell3d.replace_table.clear();
        }
    }

private:
    void Fallback(const std::vector<u32>& parameters) {
        maxwell3d.RefreshParameters();
        const u32 instance_count = (maxwell3d.GetRegisterValue(0xD1B) & parameters[2]);
        const u32 element_base = parameters[4];
        const u32 base_instance = parameters[5];
        maxwell3d.regs.vertex_id_base = element_base;
        maxwell3d.regs.global_base_vertex_index = element_base;
        maxwell3d.regs.global_base_instance_index = base_instance;
        maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseVertex);
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x644, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
        }

        maxwell3d.draw_manager->DrawIndex(
            static_cast<Tegra::Maxwell3D::Regs::PrimitiveTopology>(parameters[0]), parameters[3],
            parameters[1], element_base, base_instance, instance_count);

        maxwell3d.regs.vertex_id_base = 0x0;
        maxwell3d.regs.global_base_vertex_index = 0x0;
        maxwell3d.regs.global_base_instance_index = 0x0;
        if constexpr (extended) {
            maxwell3d.engine_state = Maxwell3D::EngineHint::None;
            maxwell3d.replace_table.clear();
        }
    }
};

class HLE_MultiLayerClear final : public HLEMacroImpl {
public:
    explicit HLE_MultiLayerClear(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        ASSERT(parameters.size() == 1);

        const Maxwell3D::Regs::ClearSurface clear_params{parameters[0]};
        const u32 rt_index = clear_params.RT;
        const u32 num_layers = maxwell3d.regs.rt[rt_index].depth;
        ASSERT(clear_params.layer == 0);

        maxwell3d.regs.clear_surface.raw = clear_params.raw;
        maxwell3d.draw_manager->Clear(num_layers);
    }
};

class HLE_MultiDrawIndexedIndirectCount final : public HLEMacroImpl {
public:
    explicit HLE_MultiDrawIndexedIndirectCount(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        const auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[2]);
        if (!IsTopologySafe(topology)) {
            Fallback(parameters);
            return;
        }

        const u32 start_indirect = parameters[0];
        const u32 end_indirect = parameters[1];
        if (start_indirect >= end_indirect) {
            // Nothing to do.
            return;
        }

        const u32 padding = parameters[3]; // padding is in words

        // size of each indirect segment
        const u32 indirect_words = 5 + padding;
        const u32 stride = indirect_words * sizeof(u32);
        const std::size_t draw_count = end_indirect - start_indirect;
        const u32 estimate = static_cast<u32>(maxwell3d.EstimateIndexBufferSize());
        maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
        auto& params = maxwell3d.draw_manager->GetIndirectParams();
        params.is_byte_count = false;
        params.is_indexed = true;
        params.include_count = true;
        params.count_start_address = maxwell3d.GetMacroAddress(4);
        params.indirect_start_address = maxwell3d.GetMacroAddress(5);
        params.buffer_size = stride * draw_count;
        params.max_draw_counts = draw_count;
        params.stride = stride;
        maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
        maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
        maxwell3d.SetHLEReplacementAttributeType(
            0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseVertex);
        maxwell3d.SetHLEReplacementAttributeType(
            0, 0x644, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
        maxwell3d.SetHLEReplacementAttributeType(0, 0x648,
                                                 Maxwell3D::HLEReplacementAttributeType::DrawID);
        maxwell3d.draw_manager->DrawIndexedIndirect(topology, 0, estimate);
        maxwell3d.engine_state = Maxwell3D::EngineHint::None;
        maxwell3d.replace_table.clear();
    }

private:
    void Fallback(const std::vector<u32>& parameters) {
        SCOPE_EXIT {
            // Clean everything.
            maxwell3d.regs.vertex_id_base = 0x0;
            maxwell3d.engine_state = Maxwell3D::EngineHint::None;
            maxwell3d.replace_table.clear();
        };
        maxwell3d.RefreshParameters();
        const u32 start_indirect = parameters[0];
        const u32 end_indirect = parameters[1];
        if (start_indirect >= end_indirect) {
            // Nothing to do.
            return;
        }
        const auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[2]);
        const u32 padding = parameters[3];
        const std::size_t max_draws = parameters[4];

        const u32 indirect_words = 5 + padding;
        const std::size_t first_draw = start_indirect;
        const std::size_t effective_draws = end_indirect - start_indirect;
        const std::size_t last_draw = start_indirect + std::min(effective_draws, max_draws);

        for (std::size_t index = first_draw; index < last_draw; index++) {
            const std::size_t base = index * indirect_words + 5;
            const u32 base_vertex = parameters[base + 3];
            const u32 base_instance = parameters[base + 4];
            maxwell3d.regs.vertex_id_base = base_vertex;
            maxwell3d.engine_state = Maxwell3D::EngineHint::OnHLEMacro;
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x640, Maxwell3D::HLEReplacementAttributeType::BaseVertex);
            maxwell3d.SetHLEReplacementAttributeType(
                0, 0x644, Maxwell3D::HLEReplacementAttributeType::BaseInstance);
            maxwell3d.CallMethod(0x8e3, 0x648, true);
            maxwell3d.CallMethod(0x8e4, static_cast<u32>(index), true);
            maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true;
            maxwell3d.draw_manager->DrawIndex(topology, parameters[base + 2], parameters[base],
                                              base_vertex, base_instance, parameters[base + 1]);
        }
    }
};

class HLE_DrawIndirectByteCount final : public HLEMacroImpl {
public:
    explicit HLE_DrawIndirectByteCount(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        const bool force = maxwell3d.Rasterizer().HasDrawTransformFeedback();

        auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[0] & 0xFFFFU);
        if (!force && (!maxwell3d.AnyParametersDirty() || !IsTopologySafe(topology))) {
            Fallback(parameters);
            return;
        }
        auto& params = maxwell3d.draw_manager->GetIndirectParams();
        params.is_byte_count = true;
        params.is_indexed = false;
        params.include_count = false;
        params.count_start_address = 0;
        params.indirect_start_address = maxwell3d.GetMacroAddress(2);
        params.buffer_size = 4;
        params.max_draw_counts = 1;
        params.stride = parameters[1];
        maxwell3d.regs.draw.begin = parameters[0];
        maxwell3d.regs.draw_auto_stride = parameters[1];
        maxwell3d.regs.draw_auto_byte_count = parameters[2];

        maxwell3d.draw_manager->DrawArrayIndirect(topology);
    }

private:
    void Fallback(const std::vector<u32>& parameters) {
        maxwell3d.RefreshParameters();

        maxwell3d.regs.draw.begin = parameters[0];
        maxwell3d.regs.draw_auto_stride = parameters[1];
        maxwell3d.regs.draw_auto_byte_count = parameters[2];

        maxwell3d.draw_manager->DrawArray(
            maxwell3d.regs.draw.topology, 0,
            maxwell3d.regs.draw_auto_byte_count / maxwell3d.regs.draw_auto_stride, 0, 1);
    }
};

class HLE_C713C83D8F63CCF3 final : public HLEMacroImpl {
public:
    explicit HLE_C713C83D8F63CCF3(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        const u32 offset = (parameters[0] & 0x3FFFFFFF) << 2;
        const u32 address = maxwell3d.regs.shadow_scratch[24];
        auto& const_buffer = maxwell3d.regs.const_buffer;
        const_buffer.size = 0x7000;
        const_buffer.address_high = (address >> 24) & 0xFF;
        const_buffer.address_low = address << 8;
        const_buffer.offset = offset;
    }
};

class HLE_D7333D26E0A93EDE final : public HLEMacroImpl {
public:
    explicit HLE_D7333D26E0A93EDE(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        const size_t index = parameters[0];
        const u32 address = maxwell3d.regs.shadow_scratch[42 + index];
        const u32 size = maxwell3d.regs.shadow_scratch[47 + index];
        auto& const_buffer = maxwell3d.regs.const_buffer;
        const_buffer.size = size;
        const_buffer.address_high = (address >> 24) & 0xFF;
        const_buffer.address_low = address << 8;
    }
};

class HLE_BindShader final : public HLEMacroImpl {
public:
    explicit HLE_BindShader(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        auto& regs = maxwell3d.regs;
        const u32 index = parameters[0];
        if ((parameters[1] - regs.shadow_scratch[28 + index]) == 0) {
            return;
        }

        regs.pipelines[index & 0xF].offset = parameters[2];
        maxwell3d.dirty.flags[VideoCommon::Dirty::Shaders] = true;
        regs.shadow_scratch[28 + index] = parameters[1];
        regs.shadow_scratch[34 + index] = parameters[2];

        const u32 address = parameters[4];
        auto& const_buffer = regs.const_buffer;
        const_buffer.size = 0x10000;
        const_buffer.address_high = (address >> 24) & 0xFF;
        const_buffer.address_low = address << 8;

        const size_t bind_group_id = parameters[3] & 0x7F;
        auto& bind_group = regs.bind_groups[bind_group_id];
        bind_group.raw_config = 0x11;
        maxwell3d.ProcessCBBind(bind_group_id);
    }
};

class HLE_SetRasterBoundingBox final : public HLEMacroImpl {
public:
    explicit HLE_SetRasterBoundingBox(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        const u32 raster_mode = parameters[0];
        auto& regs = maxwell3d.regs;
        const u32 raster_enabled = maxwell3d.regs.conservative_raster_enable;
        const u32 scratch_data = maxwell3d.regs.shadow_scratch[52];
        regs.raster_bounding_box.raw = raster_mode & 0xFFFFF00F;
        regs.raster_bounding_box.pad.Assign(scratch_data & raster_enabled);
    }
};

template <size_t base_size>
class HLE_ClearConstBuffer final : public HLEMacroImpl {
public:
    explicit HLE_ClearConstBuffer(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();
        static constexpr std::array<u32, base_size> zeroes{};
        auto& regs = maxwell3d.regs;
        regs.const_buffer.size = static_cast<u32>(base_size);
        regs.const_buffer.address_high = parameters[0];
        regs.const_buffer.address_low = parameters[1];
        regs.const_buffer.offset = 0;
        maxwell3d.ProcessCBMultiData(zeroes.data(), parameters[2] * 4);
    }
};

class HLE_ClearMemory final : public HLEMacroImpl {
public:
    explicit HLE_ClearMemory(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();

        const u32 needed_memory = parameters[2] / sizeof(u32);
        if (needed_memory > zero_memory.size()) {
            zero_memory.resize(needed_memory, 0);
        }
        auto& regs = maxwell3d.regs;
        regs.upload.line_length_in = parameters[2];
        regs.upload.line_count = 1;
        regs.upload.dest.address_high = parameters[0];
        regs.upload.dest.address_low = parameters[1];
        maxwell3d.CallMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(launch_dma)), 0x1011, true);
        maxwell3d.CallMultiMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(inline_data)),
                                  zero_memory.data(), needed_memory, needed_memory);
    }

private:
    std::vector<u32> zero_memory;
};

class HLE_TransformFeedbackSetup final : public HLEMacroImpl {
public:
    explicit HLE_TransformFeedbackSetup(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}

    void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
        maxwell3d.RefreshParameters();

        auto& regs = maxwell3d.regs;
        regs.transform_feedback_enabled = 1;
        regs.transform_feedback.buffers[0].start_offset = 0;
        regs.transform_feedback.buffers[1].start_offset = 0;
        regs.transform_feedback.buffers[2].start_offset = 0;
        regs.transform_feedback.buffers[3].start_offset = 0;

        regs.upload.line_length_in = 4;
        regs.upload.line_count = 1;
        regs.upload.dest.address_high = parameters[0];
        regs.upload.dest.address_low = parameters[1];
        maxwell3d.CallMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(launch_dma)), 0x1011, true);
        maxwell3d.CallMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(inline_data)),
                             regs.transform_feedback.controls[0].stride, true);

        maxwell3d.Rasterizer().RegisterTransformFeedback(regs.upload.dest.Address());
    }
};

} // Anonymous namespace

HLEMacro::HLEMacro(Maxwell3D& maxwell3d_) : maxwell3d{maxwell3d_} {
    builders.emplace(0x0D61FC9FAAC9FCADULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_DrawArraysIndirect<false>>(maxwell3d__);
                         }));
    builders.emplace(0x8A4D173EB99A8603ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_DrawArraysIndirect<true>>(maxwell3d__);
                         }));
    builders.emplace(0x771BB18C62444DA0ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_DrawIndexedIndirect<false>>(maxwell3d__);
                         }));
    builders.emplace(0x0217920100488FF7ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_DrawIndexedIndirect<true>>(maxwell3d__);
                         }));
    builders.emplace(0x3F5E74B9C9A50164ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_MultiDrawIndexedIndirectCount>(
                                 maxwell3d__);
                         }));
    builders.emplace(0xEAD26C3E2109B06BULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_MultiLayerClear>(maxwell3d__);
                         }));
    builders.emplace(0xC713C83D8F63CCF3ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_C713C83D8F63CCF3>(maxwell3d__);
                         }));
    builders.emplace(0xD7333D26E0A93EDEULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_D7333D26E0A93EDE>(maxwell3d__);
                         }));
    builders.emplace(0xEB29B2A09AA06D38ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_BindShader>(maxwell3d__);
                         }));
    builders.emplace(0xDB1341DBEB4C8AF7ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_SetRasterBoundingBox>(maxwell3d__);
                         }));
    builders.emplace(0x6C97861D891EDf7EULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_ClearConstBuffer<0x5F00>>(maxwell3d__);
                         }));
    builders.emplace(0xD246FDDF3A6173D7ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_ClearConstBuffer<0x7000>>(maxwell3d__);
                         }));
    builders.emplace(0xEE4D0004BEC8ECF4ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_ClearMemory>(maxwell3d__);
                         }));
    builders.emplace(0xFC0CF27F5FFAA661ULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_TransformFeedbackSetup>(maxwell3d__);
                         }));
    builders.emplace(0xB5F74EDB717278ECULL,
                     std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
                         [](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
                             return std::make_unique<HLE_DrawIndirectByteCount>(maxwell3d__);
                         }));
}

HLEMacro::~HLEMacro() = default;

std::unique_ptr<CachedMacro> HLEMacro::GetHLEProgram(u64 hash) const {
    const auto it = builders.find(hash);
    if (it == builders.end()) {
        return nullptr;
    }
    return it->second(maxwell3d);
}

} // namespace Tegra


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <functional>
#include <memory>
#include <unordered_map>

#include "common/common_types.h"

namespace Tegra {

namespace Engines {
class Maxwell3D;
}

class HLEMacro {
public:
    explicit HLEMacro(Engines::Maxwell3D& maxwell3d_);
    ~HLEMacro();

    // Allocates and returns a cached macro if the hash matches a known function.
    // Returns nullptr otherwise.
    [[nodiscard]] std::unique_ptr<CachedMacro> GetHLEProgram(u64 hash) const;

private:
    Engines::Maxwell3D& maxwell3d;
    std::unordered_map<u64, std::function<std::unique_ptr<CachedMacro>(Engines::Maxwell3D&)>>
        builders;
};

} // namespace Tegra

repo_name: litucks/torzu

license: mit


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#include <array>
#include <optional>

#include "common/assert.h"
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/macro/macro_interpreter.h"

MICROPROFILE_DEFINE(MacroInterp, "GPU", "Execute macro interpreter", MP_RGB(128, 128, 192));

namespace Tegra {
namespace {
class MacroInterpreterImpl final : public CachedMacro {
public:
    explicit MacroInterpreterImpl(Engines::Maxwell3D& maxwell3d_, const std::vector<u32>& code_)
        : maxwell3d{maxwell3d_}, code{code_} {}

    void Execute(const std::vector<u32>& params, u32 method) override;

private:
    /// Resets the execution engine state, zeroing registers, etc.
    void Reset();

    /**
     * Executes a single macro instruction located at the current program counter. Returns whether
     * the interpreter should keep running.
     *
     * @param is_delay_slot Whether the current step is being executed due to a delay slot in a
     *                      previous instruction.
     */
    bool Step(bool is_delay_slot);

    /// Calculates the result of an ALU operation. src_a OP src_b;
    u32 GetALUResult(Macro::ALUOperation operation, u32 src_a, u32 src_b);

    /// Performs the result operation on the input result and stores it in the specified register
    /// (if necessary).
    void ProcessResult(Macro::ResultOperation operation, u32 reg, u32 result);

    /// Evaluates the branch condition and returns whether the branch should be taken or not.
    bool EvaluateBranchCondition(Macro::BranchCondition cond, u32 value) const;

    /// Reads an opcode at the current program counter location.
    Macro::Opcode GetOpcode() const;

    /// Returns the specified register's value. Register 0 is hardcoded to always return 0.
    u32 GetRegister(u32 register_id) const;

    /// Sets the register to the input value.
    void SetRegister(u32 register_id, u32 value);

    /// Sets the method address to use for the next Send instruction.
    void SetMethodAddress(u32 address);

    /// Calls a GPU Engine method with the input parameter.
    void Send(u32 value);

    /// Reads a GPU register located at the method address.
    u32 Read(u32 method) const;

    /// Returns the next parameter in the parameter queue.
    u32 FetchParameter();

    Engines::Maxwell3D& maxwell3d;

    /// Current program counter
    u32 pc{};
    /// Program counter to execute at after the delay slot is executed.
    std::optional<u32> delayed_pc;

    /// General purpose macro registers.
    std::array<u32, Macro::NUM_MACRO_REGISTERS> registers = {};

    /// Method address to use for the next Send instruction.
    Macro::MethodAddress method_address = {};

    /// Input parameters of the current macro.
    std::unique_ptr<u32[]> parameters;
    std::size_t num_parameters = 0;
    std::size_t parameters_capacity = 0;
    /// Index of the next parameter that will be fetched by the 'parm' instruction.
    u32 next_parameter_index = 0;

    bool carry_flag = false;
    const std::vector<u32>& code;
};

void MacroInterpreterImpl::Execute(const std::vector<u32>& params, u32 method) {
    MICROPROFILE_SCOPE(MacroInterp);
    Reset();

    registers[1] = params[0];
    num_parameters = params.size();

    if (num_parameters > parameters_capacity) {
        parameters_capacity = num_parameters;
        parameters = std::make_unique<u32[]>(num_parameters);
    }
    std::memcpy(parameters.get(), params.data(), num_parameters * sizeof(u32));

    // Execute the code until we hit an exit condition.
    bool keep_executing = true;
    while (keep_executing) {
        keep_executing = Step(false);
    }

    // Assert the the macro used all the input parameters
    ASSERT(next_parameter_index == num_parameters);
}

void MacroInterpreterImpl::Reset() {
    registers = {};
    pc = 0;
    delayed_pc = {};
    method_address.raw = 0;
    num_parameters = 0;
    // The next parameter index starts at 1, because $r1 already has the value of the first
    // parameter.
    next_parameter_index = 1;
    carry_flag = false;
}

bool MacroInterpreterImpl::Step(bool is_delay_slot) {
    u32 base_address = pc;

    Macro::Opcode opcode = GetOpcode();
    pc += 4;

    // Update the program counter if we were delayed
    if (delayed_pc) {
        ASSERT(is_delay_slot);
        pc = *delayed_pc;
        delayed_pc = {};
    }

    switch (opcode.operation) {
    case Macro::Operation::ALU: {
        u32 result = GetALUResult(opcode.alu_operation, GetRegister(opcode.src_a),
                                  GetRegister(opcode.src_b));
        ProcessResult(opcode.result_operation, opcode.dst, result);
        break;
    }
    case Macro::Operation::AddImmediate: {
        ProcessResult(opcode.result_operation, opcode.dst,
                      GetRegister(opcode.src_a) + opcode.immediate);
        break;
    }
    case Macro::Operation::ExtractInsert: {
        u32 dst = GetRegister(opcode.src_a);
        u32 src = GetRegister(opcode.src_b);

        src = (src >> opcode.bf_src_bit) & opcode.GetBitfieldMask();
        dst &= ~(opcode.GetBitfieldMask() << opcode.bf_dst_bit);
        dst |= src << opcode.bf_dst_bit;
        ProcessResult(opcode.result_operation, opcode.dst, dst);
        break;
    }
    case Macro::Operation::ExtractShiftLeftImmediate: {
        u32 dst = GetRegister(opcode.src_a);
        u32 src = GetRegister(opcode.src_b);

        u32 result = ((src >> dst) & opcode.GetBitfieldMask()) << opcode.bf_dst_bit;

        ProcessResult(opcode.result_operation, opcode.dst, result);
        break;
    }
    case Macro::Operation::ExtractShiftLeftRegister: {
        u32 dst = GetRegister(opcode.src_a);
        u32 src = GetRegister(opcode.src_b);

        u32 result = ((src >> opcode.bf_src_bit) & opcode.GetBitfieldMask()) << dst;

        ProcessResult(opcode.result_operation, opcode.dst, result);
        break;
    }
    case Macro::Operation::Read: {
        u32 result = Read(GetRegister(opcode.src_a) + opcode.immediate);
        ProcessResult(opcode.result_operation, opcode.dst, result);
        break;
    }
    case Macro::Operation::Branch: {
        ASSERT_MSG(!is_delay_slot, "Executing a branch in a delay slot is not valid");
        u32 value = GetRegister(opcode.src_a);
        bool taken = EvaluateBranchCondition(opcode.branch_condition, value);
        if (taken) {
            // Ignore the delay slot if the branch has the annul bit.
            if (opcode.branch_annul) {
                pc = base_address + opcode.GetBranchTarget();
                return true;
            }

            delayed_pc = base_address + opcode.GetBranchTarget();
            // Execute one more instruction due to the delay slot.
            return Step(true);
        }
        break;
    }
    default:
        UNIMPLEMENTED_MSG("Unimplemented macro operation {}", opcode.operation.Value());
        break;
    }

    // An instruction with the Exit flag will not actually
    // cause an exit if it's executed inside a delay slot.
    if (opcode.is_exit && !is_delay_slot) {
        // Exit has a delay slot, execute the next instruction
        Step(true);
        return false;
    }

    return true;
}

u32 MacroInterpreterImpl::GetALUResult(Macro::ALUOperation operation, u32 src_a, u32 src_b) {
    switch (operation) {
    case Macro::ALUOperation::Add: {
        const u64 result{static_cast<u64>(src_a) + src_b};
        carry_flag = result > 0xffffffff;
        return static_cast<u32>(result);
    }
    case Macro::ALUOperation::AddWithCarry: {
        const u64 result{static_cast<u64>(src_a) + src_b + (carry_flag ? 1ULL : 0ULL)};
        carry_flag = result > 0xffffffff;
        return static_cast<u32>(result);
    }
    case Macro::ALUOperation::Subtract: {
        const u64 result{static_cast<u64>(src_a) - src_b};
        carry_flag = result < 0x100000000;
        return static_cast<u32>(result);
    }
    case Macro::ALUOperation::SubtractWithBorrow: {
        const u64 result{static_cast<u64>(src_a) - src_b - (carry_flag ? 0ULL : 1ULL)};
        carry_flag = result < 0x100000000;
        return static_cast<u32>(result);
    }
    case Macro::ALUOperation::Xor:
        return src_a ^ src_b;
    case Macro::ALUOperation::Or:
        return src_a | src_b;
    case Macro::ALUOperation::And:
        return src_a & src_b;
    case Macro::ALUOperation::AndNot:
        return src_a & ~src_b;
    case Macro::ALUOperation::Nand:
        return ~(src_a & src_b);

    default:
        UNIMPLEMENTED_MSG("Unimplemented ALU operation {}", operation);
        return 0;
    }
}

void MacroInterpreterImpl::ProcessResult(Macro::ResultOperation operation, u32 reg, u32 result) {
    switch (operation) {
    case Macro::ResultOperation::IgnoreAndFetch:
        // Fetch parameter and ignore result.
        SetRegister(reg, FetchParameter());
        break;
    case Macro::ResultOperation::Move:
        // Move result.
        SetRegister(reg, result);
        break;
    case Macro::ResultOperation::MoveAndSetMethod:
        // Move result and use as Method Address.
        SetRegister(reg, result);
        SetMethodAddress(result);
        break;
    case Macro::ResultOperation::FetchAndSend:
        // Fetch parameter and send result.
        SetRegister(reg, FetchParameter());
        Send(result);
        break;
    case Macro::ResultOperation::MoveAndSend:
        // Move and send result.
        SetRegister(reg, result);
        Send(result);
        break;
    case Macro::ResultOperation::FetchAndSetMethod:
        // Fetch parameter and use result as Method Address.
        SetRegister(reg, FetchParameter());
        SetMethodAddress(result);
        break;
    case Macro::ResultOperation::MoveAndSetMethodFetchAndSend:
        // Move result and use as Method Address, then fetch and send parameter.
        SetRegister(reg, result);
        SetMethodAddress(result);
        Send(FetchParameter());
        break;
    case Macro::ResultOperation::MoveAndSetMethodSend:
        // Move result and use as Method Address, then send bits 12:17 of result.
        SetRegister(reg, result);
        SetMethodAddress(result);
        Send((result >> 12) & 0b111111);
        break;
    default:
        UNIMPLEMENTED_MSG("Unimplemented result operation {}", operation);
        break;
    }
}

bool MacroInterpreterImpl::EvaluateBranchCondition(Macro::BranchCondition cond, u32 value) const {
    switch (cond) {
    case Macro::BranchCondition::Zero:
        return value == 0;
    case Macro::BranchCondition::NotZero:
        return value != 0;
    }
    UNREACHABLE();
}

Macro::Opcode MacroInterpreterImpl::GetOpcode() const {
    ASSERT((pc % sizeof(u32)) == 0);
    ASSERT(pc < code.size() * sizeof(u32));
    return {code[pc / sizeof(u32)]};
}

u32 MacroInterpreterImpl::GetRegister(u32 register_id) const {
    return registers.at(register_id);
}

void MacroInterpreterImpl::SetRegister(u32 register_id, u32 value) {
    // Register 0 is hardwired as the zero register.
    // Ensure no writes to it actually occur.
    if (register_id == 0) {
        return;
    }

    registers.at(register_id) = value;
}

void MacroInterpreterImpl::SetMethodAddress(u32 address) {
    method_address.raw = address;
}

void MacroInterpreterImpl::Send(u32 value) {
    maxwell3d.CallMethod(method_address.address, value, true);
    // Increment the method address by the method increment.
    method_address.address.Assign(method_address.address.Value() +
                                  method_address.increment.Value());
}

u32 MacroInterpreterImpl::Read(u32 method) const {
    return maxwell3d.GetRegisterValue(method);
}

u32 MacroInterpreterImpl::FetchParameter() {
    ASSERT(next_parameter_index < num_parameters);
    return parameters[next_parameter_index++];
}
} // Anonymous namespace

MacroInterpreter::MacroInterpreter(Engines::Maxwell3D& maxwell3d_)
    : MacroEngine{maxwell3d_}, maxwell3d{maxwell3d_} {}

std::unique_ptr<CachedMacro> MacroInterpreter::Compile(const std::vector<u32>& code) {
    return std::make_unique<MacroInterpreterImpl>(maxwell3d, code);
}

} // namespace Tegra


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <vector>

#include "common/common_types.h"
#include "video_core/macro/macro.h"

namespace Tegra {
namespace Engines {
class Maxwell3D;
}

class MacroInterpreter final : public MacroEngine {
public:
    explicit MacroInterpreter(Engines::Maxwell3D& maxwell3d_);

protected:
    std::unique_ptr<CachedMacro> Compile(const std::vector<u32>& code) override;

private:
    Engines::Maxwell3D& maxwell3d;
};

} // namespace Tegra


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#include <array>
#include <bitset>
#include <optional>

#include <xbyak/xbyak.h>

#include "common/assert.h"
#include "common/bit_field.h"
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "common/x64/xbyak_abi.h"
#include "common/x64/xbyak_util.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/macro/macro_interpreter.h"
#include "video_core/macro/macro_jit_x64.h"

MICROPROFILE_DEFINE(MacroJitCompile, "GPU", "Compile macro JIT", MP_RGB(173, 255, 47));
MICROPROFILE_DEFINE(MacroJitExecute, "GPU", "Execute macro JIT", MP_RGB(255, 255, 0));

namespace Tegra {
namespace {
constexpr Xbyak::Reg64 STATE = Xbyak::util::rbx;
constexpr Xbyak::Reg32 RESULT = Xbyak::util::r10d;
constexpr Xbyak::Reg64 MAX_PARAMETER = Xbyak::util::r11;
constexpr Xbyak::Reg64 PARAMETERS = Xbyak::util::r12;
constexpr Xbyak::Reg32 METHOD_ADDRESS = Xbyak::util::r14d;
constexpr Xbyak::Reg64 BRANCH_HOLDER = Xbyak::util::r15;

constexpr std::bitset<32> PERSISTENT_REGISTERS = Common::X64::BuildRegSet({
    STATE,
    RESULT,
    MAX_PARAMETER,
    PARAMETERS,
    METHOD_ADDRESS,
    BRANCH_HOLDER,
});

// Arbitrarily chosen based on current booting games.
constexpr size_t MAX_CODE_SIZE = 0x10000;

std::bitset<32> PersistentCallerSavedRegs() {
    return PERSISTENT_REGISTERS & Common::X64::ABI_ALL_CALLER_SAVED;
}

class MacroJITx64Impl final : public Xbyak::CodeGenerator, public CachedMacro {
public:
    explicit MacroJITx64Impl(Engines::Maxwell3D& maxwell3d_, const std::vector<u32>& code_)
        : CodeGenerator{MAX_CODE_SIZE}, code{code_}, maxwell3d{maxwell3d_} {
        Compile();
    }

    void Execute(const std::vector<u32>& parameters, u32 method) override;

    void Compile_ALU(Macro::Opcode opcode);
    void Compile_AddImmediate(Macro::Opcode opcode);
    void Compile_ExtractInsert(Macro::Opcode opcode);
    void Compile_ExtractShiftLeftImmediate(Macro::Opcode opcode);
    void Compile_ExtractShiftLeftRegister(Macro::Opcode opcode);
    void Compile_Read(Macro::Opcode opcode);
    void Compile_Branch(Macro::Opcode opcode);

private:
    void Optimizer_ScanFlags();

    void Compile();
    bool Compile_NextInstruction();

    Xbyak::Reg32 Compile_FetchParameter();
    Xbyak::Reg32 Compile_GetRegister(u32 index, Xbyak::Reg32 dst);

    void Compile_ProcessResult(Macro::ResultOperation operation, u32 reg);
    void Compile_Send(Xbyak::Reg32 value);

    Macro::Opcode GetOpCode() const;

    struct JITState {
        Engines::Maxwell3D* maxwell3d{};
        std::array<u32, Macro::NUM_MACRO_REGISTERS> registers{};
        u32 carry_flag{};
    };
    static_assert(offsetof(JITState, maxwell3d) == 0, "Maxwell3D is not at 0x0");
    using ProgramType = void (*)(JITState*, const u32*, const u32*);

    struct OptimizerState {
        bool can_skip_carry{};
        bool has_delayed_pc{};
        bool zero_reg_skip{};
        bool skip_dummy_addimmediate{};
        bool optimize_for_method_move{};
        bool enable_asserts{};
    };
    OptimizerState optimizer{};

    std::optional<Macro::Opcode> next_opcode{};
    ProgramType program{nullptr};

    std::array<Xbyak::Label, MAX_CODE_SIZE> labels;
    std::array<Xbyak::Label, MAX_CODE_SIZE> delay_skip;
    Xbyak::Label end_of_code{};

    bool is_delay_slot{};
    u32 pc{};

    const std::vector<u32>& code;
    Engines::Maxwell3D& maxwell3d;
};

void MacroJITx64Impl::Execute(const std::vector<u32>& parameters, u32 method) {
    MICROPROFILE_SCOPE(MacroJitExecute);
    ASSERT_OR_EXECUTE(program != nullptr, { return; });
    JITState state{};
    state.maxwell3d = &maxwell3d;
    state.registers = {};
    program(&state, parameters.data(), parameters.data() + parameters.size());
}

void MacroJITx64Impl::Compile_ALU(Macro::Opcode opcode) {
    const bool is_a_zero = opcode.src_a == 0;
    const bool is_b_zero = opcode.src_b == 0;
    const bool valid_operation = !is_a_zero && !is_b_zero;
    [[maybe_unused]] const bool is_move_operation = !is_a_zero && is_b_zero;
    const bool has_zero_register = is_a_zero || is_b_zero;
    const bool no_zero_reg_skip = opcode.alu_operation == Macro::ALUOperation::AddWithCarry ||
                                  opcode.alu_operation == Macro::ALUOperation::SubtractWithBorrow;

    Xbyak::Reg32 src_a;
    Xbyak::Reg32 src_b;

    if (!optimizer.zero_reg_skip || no_zero_reg_skip) {
        src_a = Compile_GetRegister(opcode.src_a, RESULT);
        src_b = Compile_GetRegister(opcode.src_b, eax);
    } else {
        if (!is_a_zero) {
            src_a = Compile_GetRegister(opcode.src_a, RESULT);
        }
        if (!is_b_zero) {
            src_b = Compile_GetRegister(opcode.src_b, eax);
        }
    }

    bool has_emitted = false;

    switch (opcode.alu_operation) {
    case Macro::ALUOperation::Add:
        if (optimizer.zero_reg_skip) {
            if (valid_operation) {
                add(src_a, src_b);
            }
        } else {
            add(src_a, src_b);
        }

        if (!optimizer.can_skip_carry) {
            setc(byte[STATE + offsetof(JITState, carry_flag)]);
        }
        break;
    case Macro::ALUOperation::AddWithCarry:
        bt(dword[STATE + offsetof(JITState, carry_flag)], 0);
        adc(src_a, src_b);
        setc(byte[STATE + offsetof(JITState, carry_flag)]);
        break;
    case Macro::ALUOperation::Subtract:
        if (optimizer.zero_reg_skip) {
            if (valid_operation) {
                sub(src_a, src_b);
                has_emitted = true;
            }
        } else {
            sub(src_a, src_b);
            has_emitted = true;
        }
        if (!optimizer.can_skip_carry && has_emitted) {
            setc(byte[STATE + offsetof(JITState, carry_flag)]);
        }
        break;
    case Macro::ALUOperation::SubtractWithBorrow:
        bt(dword[STATE + offsetof(JITState, carry_flag)], 0);
        sbb(src_a, src_b);
        setc(byte[STATE + offsetof(JITState, carry_flag)]);
        break;
    case Macro::ALUOperation::Xor:
        if (optimizer.zero_reg_skip) {
            if (valid_operation) {
                xor_(src_a, src_b);
            }
        } else {
            xor_(src_a, src_b);
        }
        break;
    case Macro::ALUOperation::Or:
        if (optimizer.zero_reg_skip) {
            if (valid_operation) {
                or_(src_a, src_b);
            }
        } else {
            or_(src_a, src_b);
        }
        break;
    case Macro::ALUOperation::And:
        if (optimizer.zero_reg_skip) {
            if (!has_zero_register) {
                and_(src_a, src_b);
            }
        } else {
            and_(src_a, src_b);
        }
        break;
    case Macro::ALUOperation::AndNot:
        if (optimizer.zero_reg_skip) {
            if (!is_a_zero) {
                not_(src_b);
                and_(src_a, src_b);
            }
        } else {
            not_(src_b);
            and_(src_a, src_b);
        }
        break;
    case Macro::ALUOperation::Nand:
        if (optimizer.zero_reg_skip) {
            if (!is_a_zero) {
                and_(src_a, src_b);
                not_(src_a);
            }
        } else {
            and_(src_a, src_b);
            not_(src_a);
        }
        break;
    default:
        UNIMPLEMENTED_MSG("Unimplemented ALU operation {}", opcode.alu_operation.Value());
        break;
    }
    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void MacroJITx64Impl::Compile_AddImmediate(Macro::Opcode opcode) {
    if (optimizer.skip_dummy_addimmediate) {
        // Games tend to use this as an exit instruction placeholder. It's to encode an instruction
        // without doing anything. In our case we can just not emit anything.
        if (opcode.result_operation == Macro::ResultOperation::Move && opcode.dst == 0) {
            return;
        }
    }
    // Check for redundant moves
    if (optimizer.optimize_for_method_move &&
        opcode.result_operation == Macro::ResultOperation::MoveAndSetMethod) {
        if (next_opcode.has_value()) {
            const auto next = *next_opcode;
            if (next.result_operation == Macro::ResultOperation::MoveAndSetMethod &&
                opcode.dst == next.dst) {
                return;
            }
        }
    }
    if (optimizer.zero_reg_skip && opcode.src_a == 0) {
        if (opcode.immediate == 0) {
            xor_(RESULT, RESULT);
        } else {
            mov(RESULT, opcode.immediate);
        }
    } else {
        auto result = Compile_GetRegister(opcode.src_a, RESULT);
        if (opcode.immediate > 2) {
            add(result, opcode.immediate);
        } else if (opcode.immediate == 1) {
            inc(result);
        } else if (opcode.immediate < 0) {
            sub(result, opcode.immediate * -1);
        }
    }
    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void MacroJITx64Impl::Compile_ExtractInsert(Macro::Opcode opcode) {
    auto dst = Compile_GetRegister(opcode.src_a, RESULT);
    auto src = Compile_GetRegister(opcode.src_b, eax);

    const u32 mask = ~(opcode.GetBitfieldMask() << opcode.bf_dst_bit);
    and_(dst, mask);
    shr(src, opcode.bf_src_bit);
    and_(src, opcode.GetBitfieldMask());
    shl(src, opcode.bf_dst_bit);
    or_(dst, src);

    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void MacroJITx64Impl::Compile_ExtractShiftLeftImmediate(Macro::Opcode opcode) {
    const auto dst = Compile_GetRegister(opcode.src_a, ecx);
    const auto src = Compile_GetRegister(opcode.src_b, RESULT);

    shr(src, dst.cvt8());
    and_(src, opcode.GetBitfieldMask());
    shl(src, opcode.bf_dst_bit);

    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void MacroJITx64Impl::Compile_ExtractShiftLeftRegister(Macro::Opcode opcode) {
    const auto dst = Compile_GetRegister(opcode.src_a, ecx);
    const auto src = Compile_GetRegister(opcode.src_b, RESULT);

    shr(src, opcode.bf_src_bit);
    and_(src, opcode.GetBitfieldMask());
    shl(src, dst.cvt8());

    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void MacroJITx64Impl::Compile_Read(Macro::Opcode opcode) {
    if (optimizer.zero_reg_skip && opcode.src_a == 0) {
        if (opcode.immediate == 0) {
            xor_(RESULT, RESULT);
        } else {
            mov(RESULT, opcode.immediate);
        }
    } else {
        auto result = Compile_GetRegister(opcode.src_a, RESULT);
        if (opcode.immediate > 2) {
            add(result, opcode.immediate);
        } else if (opcode.immediate == 1) {
            inc(result);
        } else if (opcode.immediate < 0) {
            sub(result, opcode.immediate * -1);
        }
    }

    // Equivalent to Engines::Maxwell3D::GetRegisterValue:
    if (optimizer.enable_asserts) {
        Xbyak::Label pass_range_check;
        cmp(RESULT, static_cast<u32>(Engines::Maxwell3D::Regs::NUM_REGS));
        jb(pass_range_check);
        int3();
        L(pass_range_check);
    }
    mov(rax, qword[STATE]);
    mov(RESULT,
        dword[rax + offsetof(Engines::Maxwell3D, regs) +
              offsetof(Engines::Maxwell3D::Regs, reg_array) + RESULT.cvt64() * sizeof(u32)]);

    Compile_ProcessResult(opcode.result_operation, opcode.dst);
}

void Send(Engines::Maxwell3D* maxwell3d, Macro::MethodAddress method_address, u32 value) {
    maxwell3d->CallMethod(method_address.address, value, true);
}

void MacroJITx64Impl::Compile_Send(Xbyak::Reg32 value) {
    Common::X64::ABI_PushRegistersAndAdjustStack(*this, PersistentCallerSavedRegs(), 0);
    mov(Common::X64::ABI_PARAM1, qword[STATE]);
    mov(Common::X64::ABI_PARAM2, METHOD_ADDRESS);
    mov(Common::X64::ABI_PARAM3, value);
    Common::X64::CallFarFunction(*this, &Send);
    Common::X64::ABI_PopRegistersAndAdjustStack(*this, PersistentCallerSavedRegs(), 0);

    Xbyak::Label dont_process{};
    // Get increment
    test(METHOD_ADDRESS, 0x3f000);
    // If zero, method address doesn't update
    je(dont_process);

    mov(ecx, METHOD_ADDRESS);
    and_(METHOD_ADDRESS, 0xfff);
    shr(ecx, 12);
    and_(ecx, 0x3f);
    lea(eax, ptr[rcx + METHOD_ADDRESS.cvt64()]);
    sal(ecx, 12);
    or_(eax, ecx);

    mov(METHOD_ADDRESS, eax);

    L(dont_process);
}

void MacroJITx64Impl::Compile_Branch(Macro::Opcode opcode) {
    ASSERT_MSG(!is_delay_slot, "Executing a branch in a delay slot is not valid");
    const s32 jump_address =
        static_cast<s32>(pc) + static_cast<s32>(opcode.GetBranchTarget() / sizeof(s32));

    Xbyak::Label end;
    auto value = Compile_GetRegister(opcode.src_a, eax);
    cmp(value, 0); // test(value, value);
    if (optimizer.has_delayed_pc) {
        switch (opcode.branch_condition) {
        case Macro::BranchCondition::Zero:
            jne(end, T_NEAR);
            break;
        case Macro::BranchCondition::NotZero:
            je(end, T_NEAR);
            break;
        }

        if (opcode.branch_annul) {
            xor_(BRANCH_HOLDER, BRANCH_HOLDER);
            jmp(labels[jump_address], T_NEAR);
        } else {
            Xbyak::Label handle_post_exit{};
            Xbyak::Label skip{};
            jmp(skip, T_NEAR);

            L(handle_post_exit);
            xor_(BRANCH_HOLDER, BRANCH_HOLDER);
            jmp(labels[jump_address], T_NEAR);

            L(skip);
            mov(BRANCH_HOLDER, handle_post_exit);
            jmp(delay_skip[pc], T_NEAR);
        }
    } else {
        switch (opcode.branch_condition) {
        case Macro::BranchCondition::Zero:
            je(labels[jump_address], T_NEAR);
            break;
        case Macro::BranchCondition::NotZero:
            jne(labels[jump_address], T_NEAR);
            break;
        }
    }

    L(end);
}

void MacroJITx64Impl::Optimizer_ScanFlags() {
    optimizer.can_skip_carry = true;
    optimizer.has_delayed_pc = false;
    for (auto raw_op : code) {
        Macro::Opcode op{};
        op.raw = raw_op;

        if (op.operation == Macro::Operation::ALU) {
            // Scan for any ALU operations which actually use the carry flag, if they don't exist in
            // our current code we can skip emitting the carry flag handling operations
            if (op.alu_operation == Macro::ALUOperation::AddWithCarry ||
                op.alu_operation == Macro::ALUOperation::SubtractWithBorrow) {
                optimizer.can_skip_carry = false;
            }
        }

        if (op.operation == Macro::Operation::Branch) {
            if (!op.branch_annul) {
                optimizer.has_delayed_pc = true;
            }
        }
    }
}

void MacroJITx64Impl::Compile() {
    MICROPROFILE_SCOPE(MacroJitCompile);
    labels.fill(Xbyak::Label());

    Common::X64::ABI_PushRegistersAndAdjustStack(*this, Common::X64::ABI_ALL_CALLEE_SAVED, 8);
    // JIT state
    mov(STATE, Common::X64::ABI_PARAM1);
    mov(PARAMETERS, Common::X64::ABI_PARAM2);
    mov(MAX_PARAMETER, Common::X64::ABI_PARAM3);
    xor_(RESULT, RESULT);
    xor_(METHOD_ADDRESS, METHOD_ADDRESS);
    xor_(BRANCH_HOLDER, BRANCH_HOLDER);

    mov(dword[STATE + offsetof(JITState, registers) + 4], Compile_FetchParameter());

    // Track get register for zero registers and mark it as no-op
    optimizer.zero_reg_skip = true;

    // AddImmediate tends to be used as a NOP instruction, if we detect this we can
    // completely skip the entire code path and no emit anything
    optimizer.skip_dummy_addimmediate = true;

    // SMO tends to emit a lot of unnecessary method moves, we can mitigate this by only emitting
    // one if our register isn't "dirty"
    optimizer.optimize_for_method_move = true;

    // Enable run-time assertions in JITted code
    optimizer.enable_asserts = false;

    // Check to see if we can skip emitting certain instructions
    Optimizer_ScanFlags();

    const u32 op_count = static_cast<u32>(code.size());
    for (u32 i = 0; i < op_count; i++) {
        if (i < op_count - 1) {
            pc = i + 1;
            next_opcode = GetOpCode();
        } else {
            next_opcode = {};
        }
        pc = i;
        Compile_NextInstruction();
    }

    L(end_of_code);

    Common::X64::ABI_PopRegistersAndAdjustStack(*this, Common::X64::ABI_ALL_CALLEE_SAVED, 8);
    ret();
    ready();
    program = getCode<ProgramType>();
}

bool MacroJITx64Impl::Compile_NextInstruction() {
    const auto opcode = GetOpCode();
    if (labels[pc].getAddress()) {
        return false;
    }

    L(labels[pc]);

    switch (opcode.operation) {
    case Macro::Operation::ALU:
        Compile_ALU(opcode);
        break;
    case Macro::Operation::AddImmediate:
        Compile_AddImmediate(opcode);
        break;
    case Macro::Operation::ExtractInsert:
        Compile_ExtractInsert(opcode);
        break;
    case Macro::Operation::ExtractShiftLeftImmediate:
        Compile_ExtractShiftLeftImmediate(opcode);
        break;
    case Macro::Operation::ExtractShiftLeftRegister:
        Compile_ExtractShiftLeftRegister(opcode);
        break;
    case Macro::Operation::Read:
        Compile_Read(opcode);
        break;
    case Macro::Operation::Branch:
        Compile_Branch(opcode);
        break;
    default:
        UNIMPLEMENTED_MSG("Unimplemented opcode {}", opcode.operation.Value());
        break;
    }

    if (optimizer.has_delayed_pc) {
        if (opcode.is_exit) {
            mov(rax, end_of_code);
            test(BRANCH_HOLDER, BRANCH_HOLDER);
            cmove(BRANCH_HOLDER, rax);
            // Jump to next instruction to skip delay slot check
            je(labels[pc + 1], T_NEAR);
        } else {
            // TODO(ogniK): Optimize delay slot branching
            Xbyak::Label no_delay_slot{};
            test(BRANCH_HOLDER, BRANCH_HOLDER);
            je(no_delay_slot, T_NEAR);
            mov(rax, BRANCH_HOLDER);
            xor_(BRANCH_HOLDER, BRANCH_HOLDER);
            jmp(rax);
            L(no_delay_slot);
        }
        L(delay_skip[pc]);
        if (opcode.is_exit) {
            return false;
        }
    } else {
        test(BRANCH_HOLDER, BRANCH_HOLDER);
        jne(end_of_code, T_NEAR);
        if (opcode.is_exit) {
            inc(BRANCH_HOLDER);
            return false;
        }
    }
    return true;
}

static void WarnInvalidParameter(uintptr_t parameter, uintptr_t max_parameter) {
    LOG_CRITICAL(HW_GPU,
                 "Macro JIT: invalid parameter access 0x{:x} (0x{:x} is the last parameter)",
                 parameter, max_parameter - sizeof(u32));
}

Xbyak::Reg32 MacroJITx64Impl::Compile_FetchParameter() {
    Xbyak::Label parameter_ok{};
    cmp(PARAMETERS, MAX_PARAMETER);
    jb(parameter_ok, T_NEAR);
    Common::X64::ABI_PushRegistersAndAdjustStack(*this, PersistentCallerSavedRegs(), 0);
    mov(Common::X64::ABI_PARAM1, PARAMETERS);
    mov(Common::X64::ABI_PARAM2, MAX_PARAMETER);
    Common::X64::CallFarFunction(*this, &WarnInvalidParameter);
    Common::X64::ABI_PopRegistersAndAdjustStack(*this, PersistentCallerSavedRegs(), 0);
    L(parameter_ok);
    mov(eax, dword[PARAMETERS]);
    add(PARAMETERS, sizeof(u32));
    return eax;
}

Xbyak::Reg32 MacroJITx64Impl::Compile_GetRegister(u32 index, Xbyak::Reg32 dst) {
    if (index == 0) {
        // Register 0 is always zero
        xor_(dst, dst);
    } else {
        mov(dst, dword[STATE + offsetof(JITState, registers) + index * sizeof(u32)]);
    }

    return dst;
}

void MacroJITx64Impl::Compile_ProcessResult(Macro::ResultOperation operation, u32 reg) {
    const auto SetRegister = [this](u32 reg_index, const Xbyak::Reg32& result) {
        // Register 0 is supposed to always return 0. NOP is implemented as a store to the zero
        // register.
        if (reg_index == 0) {
            return;
        }
        mov(dword[STATE + offsetof(JITState, registers) + reg_index * sizeof(u32)], result);
    };
    const auto SetMethodAddress = [this](const Xbyak::Reg32& reg32) { mov(METHOD_ADDRESS, reg32); };

    switch (operation) {
    case Macro::ResultOperation::IgnoreAndFetch:
        SetRegister(reg, Compile_FetchParameter());
        break;
    case Macro::ResultOperation::Move:
        SetRegister(reg, RESULT);
        break;
    case Macro::ResultOperation::MoveAndSetMethod:
        SetRegister(reg, RESULT);
        SetMethodAddress(RESULT);
        break;
    case Macro::ResultOperation::FetchAndSend:
        // Fetch parameter and send result.
        SetRegister(reg, Compile_FetchParameter());
        Compile_Send(RESULT);
        break;
    case Macro::ResultOperation::MoveAndSend:
        // Move and send result.
        SetRegister(reg, RESULT);
        Compile_Send(RESULT);
        break;
    case Macro::ResultOperation::FetchAndSetMethod:
        // Fetch parameter and use result as Method Address.
        SetRegister(reg, Compile_FetchParameter());
        SetMethodAddress(RESULT);
        break;
    case Macro::ResultOperation::MoveAndSetMethodFetchAndSend:
        // Move result and use as Method Address, then fetch and send parameter.
        SetRegister(reg, RESULT);
        SetMethodAddress(RESULT);
        Compile_Send(Compile_FetchParameter());
        break;
    case Macro::ResultOperation::MoveAndSetMethodSend:
        // Move result and use as Method Address, then send bits 12:17 of result.
        SetRegister(reg, RESULT);
        SetMethodAddress(RESULT);
        shr(RESULT, 12);
        and_(RESULT, 0b111111);
        Compile_Send(RESULT);
        break;
    default:
        UNIMPLEMENTED_MSG("Unimplemented macro operation {}", operation);
        break;
    }
}

Macro::Opcode MacroJITx64Impl::GetOpCode() const {
    ASSERT(pc < code.size());
    return {code[pc]};
}
} // Anonymous namespace

MacroJITx64::MacroJITx64(Engines::Maxwell3D& maxwell3d_)
    : MacroEngine{maxwell3d_}, maxwell3d{maxwell3d_} {}

std::unique_ptr<CachedMacro> MacroJITx64::Compile(const std::vector<u32>& code) {
    return std::make_unique<MacroJITx64Impl>(maxwell3d, code);
}
} // namespace Tegra


code:
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include "common/common_types.h"
#include "video_core/macro/macro.h"

namespace Tegra {

namespace Engines {
class Maxwell3D;
}

class MacroJITx64 final : public MacroEngine {
public:
    explicit MacroJITx64(Engines::Maxwell3D& maxwell3d_);

protected:
    std::unique_ptr<CachedMacro> Compile(const std::vector<u32>& code) override;

private:
    Engines::Maxwell3D& maxwell3d;
};

} // namespace Tegra


code:
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#include <algorithm>

#include "common/alignment.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "core/core.h"
#include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_process.h"
#include "video_core/guest_memory.h"
#include "video_core/host1x/host1x.h"
#include "video_core/invalidation_accumulator.h"
#include "video_core/memory_manager.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/renderer_base.h"

namespace Tegra {
using Tegra::Memory::GuestMemoryFlags;

std::atomic<size_t> MemoryManager::unique_identifier_generator{};

MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager& memory_,
                             u64 address_space_bits_, GPUVAddr split_address_, u64 big_page_bits_,
                             u64 page_bits_)
    : system{system_}, memory{memory_}, address_space_bits{address_space_bits_},
      split_address{split_address_}, page_bits{page_bits_}, big_page_bits{big_page_bits_},
      entries{}, big_entries{}, page_table{address_space_bits, address_space_bits + page_bits - 38,
                                           page_bits != big_page_bits ? page_bits : 0},
      kind_map{PTEKind::INVALID}, unique_identifier{unique_identifier_generator.fetch_add(
                                      1, std::memory_order_acq_rel)},
      accumulator{std::make_unique<VideoCommon::InvalidationAccumulator>()} {
    address_space_size = 1ULL << address_space_bits;
    page_size = 1ULL << page_bits;
    page_mask = page_size - 1ULL;
    big_page_size = 1ULL << big_page_bits;
    big_page_mask = big_page_size - 1ULL;
    const u64 page_table_bits = address_space_bits - page_bits;
    const u64 big_page_table_bits = address_space_bits - big_page_bits;
    const u64 page_table_size = 1ULL << page_table_bits;
    const u64 big_page_table_size = 1ULL << big_page_table_bits;
    page_table_mask = page_table_size - 1;
    big_page_table_mask = big_page_table_size - 1;

    big_entries.resize(big_page_table_size / 32, 0);
    big_page_table_dev.resize(big_page_table_size);
    big_page_continuous.resize(big_page_table_size / continuous_bits, 0);
    entries.resize(page_table_size / 32, 0);
}

MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_,
                             GPUVAddr split_address_, u64 big_page_bits_, u64 page_bits_)
    : MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_,
                    big_page_bits_, page_bits_) {}

MemoryManager::~MemoryManager() = default;

template <bool is_big_page>
MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
    if constexpr (is_big_page) {
        position = position >> big_page_bits;
        const u64 entry_mask = big_entries[position / 32];
        const size_t sub_index = position % 32;
        return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL);
    } else {
        position = position >> page_bits;
        const u64 entry_mask = entries[position / 32];
        const size_t sub_index = position % 32;
        return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL);
    }
}

template <bool is_big_page>
void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) {
    if constexpr (is_big_page) {
        position = position >> big_page_bits;
        const u64 entry_mask = big_entries[position / 32];
        const size_t sub_index = position % 32;
        big_entries[position / 32] =
            (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2);
    } else {
        position = position >> page_bits;
        const u64 entry_mask = entries[position / 32];
        const size_t sub_index = position % 32;
        entries[position / 32] =
            (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2);
    }
}

PTEKind MemoryManager::GetPageKind(GPUVAddr gpu_addr) const {
    std::unique_lock<std::mutex> lock(guard);
    return kind_map.GetValueAt(gpu_addr);
}

inline bool MemoryManager::IsBigPageContinuous(size_t big_page_index) const {
    const u64 entry_mask = big_page_continuous[big_page_index / continuous_bits];
    const size_t sub_index = big_page_index % continuous_bits;
    return ((entry_mask >> sub_index) & 0x1ULL) != 0;
}

inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool value) {
    const u64 continuous_mask = big_page_continuous[big_page_index / continuous_bits];
    const size_t sub_index = big_page_index % continuous_bits;
    big_page_continuous[big_page_index / continuous_bits] =
        (~(1ULL << sub_index) & continuous_mask) | (value ? 1ULL << sub_index : 0);
}

template <MemoryManager::EntryType entry_type>
GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size,
                                    PTEKind kind) {
    [[maybe_unused]] u64 remaining_size{size};
    if constexpr (entry_type == EntryType::Mapped) {
        page_table.ReserveRange(gpu_addr, size);
    }
    for (u64 offset{}; offset < size; offset += page_size) {
        const GPUVAddr current_gpu_addr = gpu_addr + offset;
        [[maybe_unused]] const auto current_entry_type = GetEntry<false>(current_gpu_addr);
        SetEntry<false>(current_gpu_addr, entry_type);
        if (current_entry_type != entry_type) {
            rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, page_size);
        }
        if constexpr (entry_type == EntryType::Mapped) {
            const DAddr current_dev_addr = dev_addr + offset;
            const auto index = PageEntryIndex<false>(current_gpu_addr);
            const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
            page_table[index] = sub_value;
        }
        remaining_size -= page_size;
    }
    kind_map.Map(gpu_addr, gpu_addr + size, kind);
    return gpu_addr;
}

template <MemoryManager::EntryType entry_type>
GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr,
                                       size_t size, PTEKind kind) {
    [[maybe_unused]] u64 remaining_size{size};
    for (u64 offset{}; offset < size; offset += big_page_size) {
        const GPUVAddr current_gpu_addr = gpu_addr + offset;
        [[maybe_unused]] const auto current_entry_type = GetEntry<true>(current_gpu_addr);
        SetEntry<true>(current_gpu_addr, entry_type);
        if (current_entry_type != entry_type) {
            rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, big_page_size);
        }
        if constexpr (entry_type == EntryType::Mapped) {
            const DAddr current_dev_addr = dev_addr + offset;
            const auto index = PageEntryIndex<true>(current_gpu_addr);
            const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
            big_page_table_dev[index] = sub_value;
            const bool is_continuous = ([&] {
                uintptr_t base_ptr{
                    reinterpret_cast<uintptr_t>(memory.GetPointer<u8>(current_dev_addr))};
                if (base_ptr == 0) {
                    return false;
                }
                for (DAddr start_cpu = current_dev_addr + page_size;
                     start_cpu < current_dev_addr + big_page_size; start_cpu += page_size) {
                    base_ptr += page_size;
                    auto next_ptr = reinterpret_cast<uintptr_t>(memory.GetPointer<u8>(start_cpu));
                    if (next_ptr == 0 || base_ptr != next_ptr) {
                        return false;
                    }
                }
                return true;
            })();
            SetBigPageContinuous(index, is_continuous);
        }
        remaining_size -= big_page_size;
    }
    {
        std::unique_lock<std::mutex> lock(guard);
        kind_map.Map(gpu_addr, gpu_addr + size, kind);
    }
    return gpu_addr;
}

void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
    rasterizer = rasterizer_;
}

GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind,
                            bool is_big_pages) {
    if (is_big_pages) [[likely]] {
        return BigPageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind);
    }
    return PageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind);
}

GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) {
    if (is_big_pages) [[likely]] {
        return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID);
    }
    return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID);
}

void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
    if (size == 0) {
        return;
    }
    GetSubmappedRangeImpl<false>(gpu_addr, size, page_stash);

    for (const auto& [map_addr, map_size] : page_stash) {
        rasterizer->UnmapMemory(map_addr, map_size);
    }
    page_stash.clear();

    BigPageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID);
    PageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID);
}

std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const {
    if (!IsWithinGPUAddressRange(gpu_addr)) [[unlikely]] {
        return std::nullopt;
    }
    if (GetEntry<true>(gpu_addr) != EntryType::Mapped) [[unlikely]] {
        if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
            return std::nullopt;
        }

        const DAddr dev_addr_base = static_cast<DAddr>(page_table[PageEntryIndex<false>(gpu_addr)])
                                    << cpu_page_bits;
        return dev_addr_base + (gpu_addr & page_mask);
    }

    const DAddr dev_addr_base =
        static_cast<DAddr>(big_page_table_dev[PageEntryIndex<true>(gpu_addr)]) << cpu_page_bits;
    return dev_addr_base + (gpu_addr & big_page_mask);
}

std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const {
    size_t page_index{addr >> page_bits};
    const size_t page_last{(addr + size + page_size - 1) >> page_bits};
    while (page_index < page_last) {
        const auto page_addr{GpuToCpuAddress(page_index << page_bits)};
        if (page_addr) {
            return page_addr;
        }
        ++page_index;
    }
    return std::nullopt;
}

template <typename T>
T MemoryManager::Read(GPUVAddr addr) const {
    if (auto page_pointer{GetPointer(addr)}; page_pointer) {
        // NOTE: Avoid adding any extra logic to this fast-path block
        T value;
        std::memcpy(&value, page_pointer, sizeof(T));
        return value;
    }

    ASSERT(false);

    return {};
}

template <typename T>
void MemoryManager::Write(GPUVAddr addr, T data) {
    if (auto page_pointer{GetPointer(addr)}; page_pointer) {
        // NOTE: Avoid adding any extra logic to this fast-path block
        std::memcpy(page_pointer, &data, sizeof(T));
        return;
    }

    ASSERT(false);
}

template u8 MemoryManager::Read<u8>(GPUVAddr addr) const;
template u16 MemoryManager::Read<u16>(GPUVAddr addr) const;
template u32 MemoryManager::Read<u32>(GPUVAddr addr) const;
template u64 MemoryManager::Read<u64>(GPUVAddr addr) const;
template void MemoryManager::Write<u8>(GPUVAddr addr, u8 data);
template void MemoryManager::Write<u16>(GPUVAddr addr, u16 data);
template void MemoryManager::Write<u32>(GPUVAddr addr, u32 data);
template void MemoryManager::Write<u64>(GPUVAddr addr, u64 data);

u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) {
    const auto address{GpuToCpuAddress(gpu_addr)};
    if (!address) {
        return {};
    }

    return memory.GetPointer<u8>(*address);
}

const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
    const auto address{GpuToCpuAddress(gpu_addr)};
    if (!address) {
        return {};
    }

    return memory.GetPointer<u8>(*address);
}

#ifdef _MSC_VER // no need for gcc / clang but msvc's compiler is more conservative with inlining.
#pragma inline_recursion(on)
#endif

template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size,
                                           FuncMapped&& func_mapped, FuncReserved&& func_reserved,
                                           FuncUnmapped&& func_unmapped) const {
    using FuncMappedReturn =
        typename std::invoke_result<FuncMapped, std::size_t, std::size_t, std::size_t>::type;
    using FuncReservedReturn =
        typename std::invoke_result<FuncReserved, std::size_t, std::size_t, std::size_t>::type;
    using FuncUnmappedReturn =
        typename std::invoke_result<FuncUnmapped, std::size_t, std::size_t, std::size_t>::type;
    static constexpr bool BOOL_BREAK_MAPPED = std::is_same_v<FuncMappedReturn, bool>;
    static constexpr bool BOOL_BREAK_RESERVED = std::is_same_v<FuncReservedReturn, bool>;
    static constexpr bool BOOL_BREAK_UNMAPPED = std::is_same_v<FuncUnmappedReturn, bool>;
    u64 used_page_size;
    u64 used_page_mask;
    u64 used_page_bits;
    if constexpr (is_big_pages) {
        used_page_size = big_page_size;
        used_page_mask = big_page_mask;
        used_page_bits = big_page_bits;
    } else {
        used_page_size = page_size;
        used_page_mask = page_mask;
        used_page_bits = page_bits;
    }
    std::size_t remaining_size{size};
    std::size_t page_index{gpu_src_addr >> used_page_bits};
    std::size_t page_offset{gpu_src_addr & used_page_mask};
    GPUVAddr current_address = gpu_src_addr;

    while (remaining_size > 0) {
        const std::size_t copy_amount{
            std::min(static_cast<std::size_t>(used_page_size) - page_offset, remaining_size)};
        auto entry = GetEntry<is_big_pages>(current_address);
        if (entry == EntryType::Mapped) [[likely]] {
            if constexpr (BOOL_BREAK_MAPPED) {
                if (func_mapped(page_index, page_offset, copy_amount)) {
                    return;
                }
            } else {
                func_mapped(page_index, page_offset, copy_amount);
            }

        } else if (entry == EntryType::Reserved) {
            if constexpr (BOOL_BREAK_RESERVED) {
                if (func_reserved(page_index, page_offset, copy_amount)) {
                    return;
                }
            } else {
                func_reserved(page_index, page_offset, copy_amount);
            }

        } else [[unlikely]] {
            if constexpr (BOOL_BREAK_UNMAPPED) {
                if (func_unmapped(page_index, page_offset, copy_amount)) {
                    return;
                }
            } else {
                func_unmapped(page_index, page_offset, copy_amount);
            }
        }
        page_index++;
        page_offset = 0;
        remaining_size -= copy_amount;
        current_address += copy_amount;
    }
}

template <bool is_safe>
void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
                                  [[maybe_unused]] VideoCommon::CacheType which) const {
    auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
                           [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
        std::memset(dest_buffer, 0, copy_amount);
        dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
    };
    auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        if constexpr (is_safe) {
            rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
        }
        u8* physical = memory.GetPointer<u8>(dev_addr_base);
        std::memcpy(dest_buffer, physical, copy_amount);
        dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
    };
    auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        if constexpr (is_safe) {
            rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
        }
        if (!IsBigPageContinuous(page_index)) [[unlikely]] {
            memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount);
        } else {
            u8* physical = memory.GetPointer<u8>(dev_addr_base);
            std::memcpy(dest_buffer, physical, copy_amount);
        }
        dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
    };
    auto read_short_pages = [&](std::size_t page_index, std::size_t offset,
                                std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero);
    };
    MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages);
}

void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
                              VideoCommon::CacheType which) const {
    ReadBlockImpl<true>(gpu_src_addr, dest_buffer, size, which);
}

void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer,
                                    const std::size_t size) const {
    ReadBlockImpl<false>(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None);
}

template <bool is_safe>
void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
                                   [[maybe_unused]] VideoCommon::CacheType which) {
    auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
                            [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
        src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
    };
    auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        if constexpr (is_safe) {
            rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
        }
        u8* physical = memory.GetPointer<u8>(dev_addr_base);
        std::memcpy(physical, src_buffer, copy_amount);
        src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
    };
    auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        if constexpr (is_safe) {
            rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
        }
        if (!IsBigPageContinuous(page_index)) [[unlikely]] {
            memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount);
        } else {
            u8* physical = memory.GetPointer<u8>(dev_addr_base);
            std::memcpy(physical, src_buffer, copy_amount);
        }
        src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
    };
    auto write_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance);
    };
    MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages);
}

void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
                               VideoCommon::CacheType which) {
    WriteBlockImpl<true>(gpu_dest_addr, src_buffer, size, which);
}

void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer,
                                     std::size_t size) {
    WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None);
}

void MemoryManager::WriteBlockCached(GPUVAddr gpu_dest_addr, const void* src_buffer,
                                     std::size_t size) {
    WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None);
    accumulator->Add(gpu_dest_addr, size);
}

void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size,
                                VideoCommon::CacheType which) const {
    auto do_nothing = [&]([[maybe_unused]] std::size_t page_index,
                          [[maybe_unused]] std::size_t offset,
                          [[maybe_unused]] std::size_t copy_amount) {};

    auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
    };
    auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
    };
    auto flush_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
    };
    MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, flush_short_pages);
}

bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
                                  VideoCommon::CacheType which) const {
    bool result = false;
    auto do_nothing = [&]([[maybe_unused]] std::size_t page_index,
                          [[maybe_unused]] std::size_t offset,
                          [[maybe_unused]] std::size_t copy_amount) { return false; };

    auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        result |= rasterizer->MustFlushRegion(dev_addr_base, copy_amount, which);
        return result;
    };
    auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        result |= rasterizer->MustFlushRegion(dev_addr_base, copy_amount, which);
        return result;
    };
    auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
        return result;
    };
    MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, check_short_pages);
    return result;
}

size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
    std::optional<DAddr> old_page_addr{};
    size_t range_so_far = 0;
    bool result{false};
    auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
                    std::size_t copy_amount) {
        result = true;
        return true;
    };
    auto short_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr && *old_page_addr != dev_addr_base) {
            result = true;
            return true;
        }
        range_so_far += copy_amount;
        old_page_addr = {dev_addr_base + copy_amount};
        return false;
    };
    auto big_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr && *old_page_addr != dev_addr_base) {
            return true;
        }
        range_so_far += copy_amount;
        old_page_addr = {dev_addr_base + copy_amount};
        return false;
    };
    auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, short_check, fail, fail);
        return result;
    };
    MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages);
    return range_so_far;
}

size_t MemoryManager::GetMemoryLayoutSize(GPUVAddr gpu_addr, size_t max_size) const {
    std::unique_lock<std::mutex> lock(guard);
    return kind_map.GetContinuousSizeFrom(gpu_addr);
}

void MemoryManager::InvalidateRegion(GPUVAddr gpu_addr, size_t size,
                                     VideoCommon::CacheType which) const {
    auto do_nothing = [&]([[maybe_unused]] std::size_t page_index,
                          [[maybe_unused]] std::size_t offset,
                          [[maybe_unused]] std::size_t copy_amount) {};

    auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
    };
    auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
    };
    auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset,
                                      std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
    };
    MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, invalidate_short_pages);
}

void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size,
                              VideoCommon::CacheType which) {
    Tegra::Memory::GpuGuestMemoryScoped<u8, GuestMemoryFlags::SafeReadWrite> data(
        *this, gpu_src_addr, size);
    data.SetAddressAndSize(gpu_dest_addr, size);
    FlushRegion(gpu_dest_addr, size, which);
}

bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
    if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] {
        size_t page_index = gpu_addr >> big_page_bits;
        if (IsBigPageContinuous(page_index)) [[likely]] {
            const std::size_t page{(page_index & big_page_mask) + size};
            return page <= big_page_size;
        }
        const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
        return page <= Core::DEVICE_PAGESIZE;
    }
    if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
        return false;
    }
    const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
    return page <= Core::DEVICE_PAGESIZE;
}

bool MemoryManager::IsContinuousRange(GPUVAddr gpu_addr, std::size_t size) const {
    std::optional<DAddr> old_page_addr{};
    bool result{true};
    auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
                    std::size_t copy_amount) {
        result = false;
        return true;
    };
    auto short_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr && *old_page_addr != dev_addr_base) {
            result = false;
            return true;
        }
        old_page_addr = {dev_addr_base + copy_amount};
        return false;
    };
    auto big_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr && *old_page_addr != dev_addr_base) {
            result = false;
            return true;
        }
        old_page_addr = {dev_addr_base + copy_amount};
        return false;
    };
    auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, short_check, fail, fail);
        return !result;
    };
    MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages);
    return result;
}

bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) const {
    bool result{true};
    auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
                    [[maybe_unused]] std::size_t copy_amount) {
        result = false;
        return true;
    };
    auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
                    [[maybe_unused]] std::size_t copy_amount) { return false; };
    auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
                                 std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, pass, pass, fail);
        return !result;
    };
    MemoryOperation<true>(gpu_addr, size, pass, fail, check_short_pages);
    return result;
}

boost::container::small_vector<std::pair<GPUVAddr, std::size_t>, 32>
MemoryManager::GetSubmappedRange(GPUVAddr gpu_addr, std::size_t size) const {
    boost::container::small_vector<std::pair<GPUVAddr, std::size_t>, 32> result{};
    GetSubmappedRangeImpl<true>(gpu_addr, size, result);
    return result;
}

template <bool is_gpu_address>
void MemoryManager::GetSubmappedRangeImpl(
    GPUVAddr gpu_addr, std::size_t size,
    boost::container::small_vector<
        std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result)
    const {
    std::optional<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>>
        last_segment{};
    std::optional<DAddr> old_page_addr{};
    const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index,
                                                [[maybe_unused]] std::size_t offset,
                                                [[maybe_unused]] std::size_t copy_amount) {
        if (last_segment) {
            result.push_back(*last_segment);
            last_segment = std::nullopt;
        }
    };
    const auto extend_size_big = [this, &split, &old_page_addr,
                                  &last_segment](std::size_t page_index, std::size_t offset,
                                                 std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr) {
            if (*old_page_addr != dev_addr_base) {
                split(0, 0, 0);
            }
        }
        old_page_addr = {dev_addr_base + copy_amount};
        if (!last_segment) {
            if constexpr (is_gpu_address) {
                const GPUVAddr new_base_addr = (page_index << big_page_bits) + offset;
                last_segment = {new_base_addr, copy_amount};
            } else {
                last_segment = {dev_addr_base, copy_amount};
            }
        } else {
            last_segment->second += copy_amount;
        }
    };
    const auto extend_size_short = [this, &split, &old_page_addr,
                                    &last_segment](std::size_t page_index, std::size_t offset,
                                                   std::size_t copy_amount) {
        const DAddr dev_addr_base =
            (static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
        if (old_page_addr) {
            if (*old_page_addr != dev_addr_base) {
                split(0, 0, 0);
            }
        }
        old_page_addr = {dev_addr_base + copy_amount};
        if (!last_segment) {
            if constexpr (is_gpu_address) {
                const GPUVAddr new_base_addr = (page_index << page_bits) + offset;
                last_segment = {new_base_addr, copy_amount};
            } else {
                last_segment = {dev_addr_base, copy_amount};
            }
        } else {
            last_segment->second += copy_amount;
        }
    };
    auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
        GPUVAddr base = (page_index << big_page_bits) + offset;
        MemoryOperation<false>(base, copy_amount, extend_size_short, split, split);
    };
    MemoryOperation<true>(gpu_addr, size, extend_size_big, split, do_short_pages);
    split(0, 0, 0);
}

void MemoryManager::FlushCaching() {
    if (!accumulator->AnyAccumulated()) {
        return;
    }
    accumulator->Callback([this](GPUVAddr addr, size_t size) {
        GetSubmappedRangeImpl<false>(addr, size, page_stash2);
    });
    rasterizer->InnerInvalidation(page_stash2);
    page_stash2.clear();
    accumulator->Clear();
}

const u8* MemoryManager::GetSpan(const GPUVAddr src_addr, const std::size_t size) const {
    if (!IsContinuousRange(src_addr, size)) {
        return nullptr;
    }
    auto dev_addr = GpuToCpuAddress(src_addr);
    if (dev_addr) {
        return memory.GetSpan(*dev_addr, size);
    }
    return nullptr;
}

u8* MemoryManager::GetSpan(const GPUVAddr src_addr, const std::size_t size) {
    if (!IsContinuousRange(src_addr, size)) {
        return nullptr;
    }
    auto dev_addr = GpuToCpuAddress(src_addr);
    if (dev_addr) {
        return memory.GetSpan(*dev_addr, size);
    }
    return nullptr;
}

} // namespace Tegra