code:
/*
 * Copyright (c) 2017-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stdint.h>

#include <kern/init.h>
#include <kern/intr.h>
#include <kern/kmem.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/panic.h>
#include <kern/spinlock.h>
#include <machine/cpu.h>
#include <machine/ioapic.h>
#include <machine/lapic.h>
#include <machine/pic.h>
#include <vm/vm_kmem.h>

#define IOAPIC_REG_VERSION              0x01
#define IOAPIC_REG_IOREDTBL             0x10

#define IOAPIC_VERSION_VERSION_MASK     0x000000ff
#define IOAPIC_VERSION_VERSION_SHIFT    0
#define IOAPIC_VERSION_MAXREDIR_MASK    0x00ff0000
#define IOAPIC_VERSION_MAXREDIR_SHIFT   16

#define IOAPIC_ENTLOW_FIXED_DEST        0x00000
#define IOAPIC_ENTLOW_PHYS_DELIVERY     0x00000
#define IOAPIC_ENTLOW_ACTIVE_LOW        0x02000
#define IOAPIC_ENTLOW_LEVEL             0x08000
#define IOAPIC_ENTLOW_INTRMASK          0x10000

#define IOAPIC_MAX_ENTRIES 24

struct ioapic_map {
    uint8_t regsel;
    uint8_t _reserved0;
    uint8_t _reserved1;
    uint8_t _reserved2;
    uint32_t _reserved3;
    uint32_t _reserved4;
    uint32_t _reserved5;
    uint32_t win;
};

/*
 * Interrupt source override descriptor.
 */
struct ioapic_iso {
    uint32_t gsi;
    uint8_t source;
    bool active_high;
    bool edge_triggered;
};

struct ioapic {
    struct spinlock lock;
    unsigned int id;
    unsigned int apic_id;
    unsigned int version;
    volatile struct ioapic_map *map;
    unsigned int first_gsi;
    unsigned int last_gsi;
};

static unsigned int ioapic_nr_devs;

static struct ioapic_iso ioapic_isos[PIC_MAX_INTR + 1];
static unsigned int ioapic_nr_isos;

static void
ioapic_iso_init(struct ioapic_iso *iso, uint8_t source, uint32_t gsi,
                bool active_high, bool edge_triggered)
{
    iso->source = source;
    iso->gsi = gsi;
    iso->active_high = active_high;
    iso->edge_triggered = edge_triggered;
}

static struct ioapic_iso * __init
ioapic_alloc_iso(void)
{
    struct ioapic_iso *iso;

    if (ioapic_nr_isos >= ARRAY_SIZE(ioapic_isos)) {
        log_err("ioapic: too many interrupt overrides");
        return NULL;
    }

    iso = &ioapic_isos[ioapic_nr_isos];
    ioapic_nr_isos++;
    return iso;
}

static struct ioapic_iso *
ioapic_lookup_iso(unsigned int intr)
{
    struct ioapic_iso *iso;
    unsigned int i;

    for (i = 0; i < ioapic_nr_isos; i++) {
        iso = &ioapic_isos[i];

        if (intr == iso->source) {
            return iso;
        }
    }

    return NULL;
}

static uint32_t
ioapic_read(struct ioapic *ioapic, uint8_t reg)
{
    ioapic->map->regsel = reg;
    return ioapic->map->win;
}

static void
ioapic_write(struct ioapic *ioapic, uint8_t reg, uint32_t value)
{
    ioapic->map->regsel = reg;
    ioapic->map->win = value;
}

static void
ioapic_write_entry_low(struct ioapic *ioapic, unsigned int id, uint32_t value)
{
    assert(id < IOAPIC_MAX_ENTRIES);
    ioapic_write(ioapic, IOAPIC_REG_IOREDTBL + (id * 2), value);
}

static void
ioapic_write_entry_high(struct ioapic *ioapic, unsigned int id, uint32_t value)
{
    assert(id < IOAPIC_MAX_ENTRIES);
    ioapic_write(ioapic, IOAPIC_REG_IOREDTBL + (id * 2) + 1, value);
}

static void
ioapic_intr(unsigned int vector)
{
    intr_handle(vector - CPU_EXC_INTR_FIRST);
}

static struct ioapic * __init
ioapic_create(unsigned int apic_id, uintptr_t addr, unsigned int gsi_base)
{
    struct ioapic *ioapic;
    unsigned int i, nr_gsis;
    uint32_t value;

    ioapic = kmem_alloc(sizeof(*ioapic));

    if (ioapic == NULL) {
        panic("ioapic: unable to allocate memory for controller");
    }

    spinlock_init(&ioapic->lock);
    ioapic->id = ioapic_nr_devs;
    ioapic->apic_id = apic_id;
    ioapic->first_gsi = gsi_base;

    ioapic->map = vm_kmem_map_pa(addr, sizeof(*ioapic->map), NULL, NULL);

    if (ioapic->map == NULL) {
        panic("ioapic: unable to map register window in kernel map");
    }

    value = ioapic_read(ioapic, IOAPIC_REG_VERSION);
    ioapic->version = (value & IOAPIC_VERSION_VERSION_MASK)
                      >> IOAPIC_VERSION_VERSION_SHIFT;
    nr_gsis = ((value & IOAPIC_VERSION_MAXREDIR_MASK)
               >> IOAPIC_VERSION_MAXREDIR_SHIFT) + 1;
    ioapic->last_gsi = ioapic->first_gsi + nr_gsis - 1;

    /* XXX This assumes that interrupts are mapped 1:1 to traps */
    if (ioapic->last_gsi > (CPU_EXC_INTR_LAST - CPU_EXC_INTR_FIRST)) {
        panic("ioapic: invalid interrupt range");
    }

    for (i = ioapic->first_gsi; i < ioapic->last_gsi; i++) {
        cpu_register_intr(CPU_EXC_INTR_FIRST + i, ioapic_intr);
    }

    log_info("ioapic%u: version:%#x gsis:%u-%u", ioapic->id,
             ioapic->version, ioapic->first_gsi, ioapic->last_gsi);

    ioapic_nr_devs++;
    return ioapic;
}

static bool
ioapic_has_gsi(const struct ioapic *ioapic, unsigned int gsi)
{
    return ((gsi >= ioapic->first_gsi) && (gsi <= ioapic->last_gsi));
}

static unsigned int
ioapic_compute_id(const struct ioapic *ioapic, unsigned int gsi)
{
    assert(ioapic_has_gsi(ioapic, gsi));
    return gsi - ioapic->first_gsi;
}

static void
ioapic_compute_entry(uint32_t *highp, uint32_t *lowp,
                     unsigned int apic_id, unsigned int intr,
                     bool active_high, bool edge_triggered)
{
    assert(apic_id < 16);
    assert(intr < (CPU_NR_EXC_VECTORS - CPU_EXC_INTR_FIRST));

    *highp = apic_id << 24;
    *lowp = (!edge_triggered ? IOAPIC_ENTLOW_LEVEL : 0)
            | (!active_high ? IOAPIC_ENTLOW_ACTIVE_LOW : 0)
            | IOAPIC_ENTLOW_PHYS_DELIVERY
            | IOAPIC_ENTLOW_FIXED_DEST
            | (CPU_EXC_INTR_FIRST + intr);
}

static void
ioapic_enable(void *priv, unsigned int intr, unsigned int cpu)
{
    bool active_high, edge_triggered;
    const struct ioapic_iso *iso;
    uint32_t high, low, gsi;
    struct ioapic *ioapic;
    unsigned long flags;
    unsigned int id;

    iso = ioapic_lookup_iso(intr);

    /* XXX These are defaults that should work with architectural devices */
    if (iso == NULL) {
        active_high = true;
        edge_triggered = true;
        gsi = intr;
    } else {
        active_high = iso->active_high;
        edge_triggered = iso->edge_triggered;
        gsi = iso->gsi;
    }

    ioapic = priv;
    id = ioapic_compute_id(ioapic, gsi);
    ioapic_compute_entry(&high, &low, cpu_apic_id(cpu), intr,
                         active_high, edge_triggered);

    spinlock_lock_intr_save(&ioapic->lock, &flags);
    ioapic_write_entry_high(ioapic, id, high);
    ioapic_write_entry_low(ioapic, id, low);
    spinlock_unlock_intr_restore(&ioapic->lock, flags);
}

static void
ioapic_disable(void *priv, unsigned int intr)
{
    struct ioapic *ioapic;
    unsigned long flags;
    unsigned int id;

    ioapic = priv;
    id = ioapic_compute_id(ioapic, intr);

    spinlock_lock_intr_save(&ioapic->lock, &flags);
    ioapic_write_entry_low(ioapic, id, IOAPIC_ENTLOW_INTRMASK);
    spinlock_unlock_intr_restore(&ioapic->lock, flags);
}

static void
ioapic_eoi(void *priv, unsigned int intr)
{
    (void)priv;
    (void)intr;

    lapic_eoi();
}

static const struct intr_ops ioapic_ops = {
    .enable = ioapic_enable,
    .disable = ioapic_disable,
    .eoi = ioapic_eoi,
};

void __init
ioapic_setup(void)
{
    ioapic_nr_devs = 0;
    ioapic_nr_isos = 0;
}

void __init
ioapic_register(unsigned int apic_id, uintptr_t addr, unsigned int gsi_base)
{
    struct ioapic *ioapic;

    ioapic = ioapic_create(apic_id, addr, gsi_base);

    /*
     * XXX This assumes that any interrupt override source is included
     * in the GSI range.
     */
    intr_register_ctl(&ioapic_ops, ioapic, ioapic->first_gsi, ioapic->last_gsi);
}

void __init
ioapic_override(uint8_t source, uint32_t gsi,
                bool active_high, bool edge_triggered)
{
    struct ioapic_iso *iso;

    iso = ioapic_alloc_iso();

    if (iso == NULL) {
        return;
    }

    ioapic_iso_init(iso, source, gsi, active_high, edge_triggered);
}


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef KERN_IOAPIC_H
#define KERN_IOAPIC_H

#include <stdbool.h>
#include <stdint.h>

/*
 * Initialize the ioapic module.
 */
void ioapic_setup(void);

/*
 * Register an I/O APIC controller.
 */
void ioapic_register(unsigned int apic_id, uintptr_t addr,
                     unsigned int gsi_base);

/*
 * Report an interrupt source override.
 */
void ioapic_override(uint8_t source, uint32_t gsi,
                     bool active_high, bool edge_triggered);

#endif /* KERN_IOAPIC_H */

license: mit


code:
/*
 * Copyright (c) 2011-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

#include <kern/clock.h>
#include <kern/init.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/panic.h>
#include <kern/perfmon.h>
#include <machine/cpu.h>
#include <machine/lapic.h>
#include <machine/pmap.h>
#include <vm/vm_kmem.h>

/*
 * Mask used to check that local APICS are internal.
 */
#define LAPIC_VERSION_MASK 0x10

/*
 * Common bits for registers in the local vector table.
 */
#define LAPIC_LVT_DELIVERY_FIXED    0x00000000
#define LAPIC_LVT_DELIVERY_NMI      0x00000400
#define LAPIC_LVT_DELIVERY_EXTINT   0x00000700
#define LAPIC_LVT_MASK_INTR         0x00010000

/*
 * LVT timer register bits.
 */
#define LAPIC_LVT_TIMER_PERIODIC    0x00020000

/*
 * Various values related to the local APIC timer.
 */
#define LAPIC_TIMER_DCR_DIV1    0x0000000b
#define LAPIC_TIMER_COUNT_MAX   0xffffffff

/*
 * Delay used to calibrate the local APIC timer, in microseconds.
 */
#define LAPIC_TIMER_CAL_DELAY   100000

/*
 * Spurious-interrupt vector register bits.
 */
#define LAPIC_SVR_SOFT_EN 0x00000100

/*
 * Interrupt command register (lower word) bits.
 */
#define LAPIC_ICR_VECTOR_MASK           0x000000ff
#define LAPIC_ICR_DELIVERY_INIT         0x00000500
#define LAPIC_ICR_DELIVERY_STARTUP      0x00000600
#define LAPIC_ICR_STATUS_PENDING        0x00001000
#define LAPIC_ICR_LEVEL_ASSERT          0x00004000
#define LAPIC_ICR_TRIGGER_LEVEL         0x00008000
#define LAPIC_ICR_DEST_SELF             0x00040000
#define LAPIC_ICR_DEST_ALL_WITH_SELF    0x00080000
#define LAPIC_ICR_DEST_ALL_EXCEPT_SELF  0x000c0000
#define LAPIC_ICR_DEST_MASK             0x000c0000
#define LAPIC_ICR_RESERVED              0xfff32000

/*
 * ICR destination shift and mask.
 */
#define LAPIC_DEST_SHIFT    24
#define LAPIC_DEST_MASK     0xff000000

/*
 * Local APIC registers are accessed with 32-bits loads/stores aligned on
 * 128 bits.
 */
struct lapic_register {
    uint32_t reg;
    uint32_t reserved0;
    uint32_t reserved1;
    uint32_t reserved2;
};

/*
 * Local APIC register map.
 */
struct lapic_map {
    const struct lapic_register reserved0;
    const struct lapic_register reserved1;

    /*
     * Some processors don't allow writing to this register, and the
     * specification explicitely discourages modifications. Consider it
     * read only.
     */
    const struct lapic_register id;
    const struct lapic_register version;
    const struct lapic_register reserved2;
    const struct lapic_register reserved3;
    const struct lapic_register reserved4;
    const struct lapic_register reserved5;
    struct lapic_register tpr;
    const struct lapic_register reserved6; /* APR */
    const struct lapic_register ppr;
    struct lapic_register eoi;
    const struct lapic_register reserved7; /* RRD */
    struct lapic_register ldr;
    struct lapic_register dfr;
    struct lapic_register svr;
    const struct lapic_register isr0;
    const struct lapic_register isr1;
    const struct lapic_register isr2;
    const struct lapic_register isr3;
    const struct lapic_register isr4;
    const struct lapic_register isr5;
    const struct lapic_register isr6;
    const struct lapic_register isr7;
    const struct lapic_register tmr0;
    const struct lapic_register tmr1;
    const struct lapic_register tmr2;
    const struct lapic_register tmr3;
    const struct lapic_register tmr4;
    const struct lapic_register tmr5;
    const struct lapic_register tmr6;
    const struct lapic_register tmr7;
    const struct lapic_register irr0;
    const struct lapic_register irr1;
    const struct lapic_register irr2;
    const struct lapic_register irr3;
    const struct lapic_register irr4;
    const struct lapic_register irr5;
    const struct lapic_register irr6;
    const struct lapic_register irr7;
    struct lapic_register esr;
    const struct lapic_register reserved8;
    const struct lapic_register reserved9;
    const struct lapic_register reserved10;
    const struct lapic_register reserved11;
    const struct lapic_register reserved12;
    const struct lapic_register reserved13;
    struct lapic_register lvt_cmci;
    struct lapic_register icr_low;
    struct lapic_register icr_high;
    struct lapic_register lvt_timer;
    const struct lapic_register reserved14; /* Thermal sensor register */
    struct lapic_register lvt_pmc; /* Performance counters register */
    struct lapic_register lvt_lint0;
    struct lapic_register lvt_lint1;
    struct lapic_register lvt_error;
    struct lapic_register timer_icr;
    const struct lapic_register timer_ccr;
    const struct lapic_register reserved16;
    const struct lapic_register reserved17;
    const struct lapic_register reserved18;
    const struct lapic_register reserved19;
    struct lapic_register timer_dcr;
    const struct lapic_register reserved20;
};

/*
 * Address where local APIC registers are mapped.
 */
static volatile struct lapic_map *lapic_map __read_mostly;

/*
 * Base frequency of the local APIC timer.
 */
static uint32_t lapic_bus_freq __read_mostly;

static uint32_t
lapic_read(const volatile struct lapic_register *r)
{
    return r->reg;
}

static void
lapic_write(volatile struct lapic_register *r, uint32_t value)
{
    r->reg = value;
}

static void __init
lapic_compute_freq(void)
{
    uint32_t c1, c2;

    lapic_write(&lapic_map->svr, LAPIC_SVR_SOFT_EN | CPU_EXC_LAPIC_SPURIOUS);
    lapic_write(&lapic_map->timer_dcr, LAPIC_TIMER_DCR_DIV1);

    /* The APIC timer counter should never wrap around here */
    lapic_write(&lapic_map->timer_icr, LAPIC_TIMER_COUNT_MAX);
    c1 = lapic_read(&lapic_map->timer_ccr);
    cpu_delay(LAPIC_TIMER_CAL_DELAY);
    c2 = lapic_read(&lapic_map->timer_ccr);
    lapic_bus_freq = (c1 - c2) * (1000000 / LAPIC_TIMER_CAL_DELAY);
    log_info("lapic: bus frequency: %u.%02u MHz",
             (unsigned int)(lapic_bus_freq / 1000000),
             (unsigned int)(lapic_bus_freq % 1000000));
    lapic_write(&lapic_map->timer_icr, lapic_bus_freq / CLOCK_FREQ);
    lapic_write(&lapic_map->svr, 0);
}

void
lapic_eoi(void)
{
    lapic_write(&lapic_map->eoi, 0);
}

#ifdef CONFIG_PERFMON
static void
lapic_pmc_overflow_intr(unsigned int vector)
{
    (void)vector;

    lapic_eoi();

    /* Reset the LVT entry as it is automatically cleared when triggered */
    lapic_write(&lapic_map->lvt_pmc, CPU_EXC_LAPIC_PMC_OF);

    perfmon_overflow_intr();
}
#endif /* CONFIG_PERFMON */

static void
lapic_timer_intr(unsigned int vector)
{
    (void)vector;

    lapic_eoi();
    clock_tick_intr();
}

static void
lapic_error_intr(unsigned int vector)
{
    uint32_t esr;

    (void)vector;

    esr = lapic_read(&lapic_map->esr);
    log_err("lapic: error on cpu%u: esr:%08x", cpu_id(), (unsigned int)esr);
    lapic_write(&lapic_map->esr, 0);
    lapic_eoi();
}

static void
lapic_spurious_intr(unsigned int vector)
{
    (void)vector;
    log_warning("lapic: spurious interrupt");

    /* No EOI for this interrupt */
}

static void __init
lapic_setup_registers(void)
{
    /*
     * LVT mask bits can only be cleared when the local APIC is enabled.
     * They are kept disabled while the local APIC is disabled.
     */
    lapic_write(&lapic_map->svr, LAPIC_SVR_SOFT_EN | CPU_EXC_LAPIC_SPURIOUS);
    lapic_write(&lapic_map->tpr, 0);
    lapic_write(&lapic_map->eoi, 0);
    lapic_write(&lapic_map->esr, 0);
    lapic_write(&lapic_map->lvt_timer, LAPIC_LVT_TIMER_PERIODIC
                                       | CPU_EXC_LAPIC_TIMER);
    lapic_write(&lapic_map->lvt_lint0, LAPIC_LVT_MASK_INTR);
    lapic_write(&lapic_map->lvt_lint1, LAPIC_LVT_MASK_INTR);
    lapic_write(&lapic_map->lvt_error, CPU_EXC_LAPIC_ERROR);
    lapic_write(&lapic_map->timer_dcr, LAPIC_TIMER_DCR_DIV1);
    lapic_write(&lapic_map->timer_icr, lapic_bus_freq / CLOCK_FREQ);
    lapic_write(&lapic_map->lvt_pmc, CPU_EXC_LAPIC_PMC_OF);
}

void __init
lapic_setup(uint32_t map_addr)
{
    uint32_t value;

#ifdef CONFIG_PERFMON
    cpu_register_intr(CPU_EXC_LAPIC_PMC_OF, lapic_pmc_overflow_intr);
#endif
    cpu_register_intr(CPU_EXC_LAPIC_TIMER, lapic_timer_intr);
    cpu_register_intr(CPU_EXC_LAPIC_ERROR, lapic_error_intr);
    cpu_register_intr(CPU_EXC_LAPIC_SPURIOUS, lapic_spurious_intr);

    lapic_map = vm_kmem_map_pa(map_addr, sizeof(*lapic_map), NULL, NULL);

    if (lapic_map == NULL) {
        panic("lapic: unable to map registers in kernel map");
    }

    value = lapic_read(&lapic_map->version);

    if ((value & LAPIC_VERSION_MASK) != LAPIC_VERSION_MASK) {
        panic("lapic: external local APIC not supported");
    }

    lapic_compute_freq();
    lapic_setup_registers();
}

void __init
lapic_ap_setup(void)
{
    lapic_setup_registers();
}

static void
lapic_ipi(uint32_t apic_id, uint32_t icr)
{
    unsigned long flags;

    cpu_intr_save(&flags);

    if ((icr & LAPIC_ICR_DEST_MASK) == 0) {
        lapic_write(&lapic_map->icr_high, apic_id << LAPIC_DEST_SHIFT);
    }

    lapic_write(&lapic_map->icr_low, icr & ~LAPIC_ICR_RESERVED);

    cpu_intr_restore(flags);
}

static void
lapic_ipi_wait(void)
{
    uint32_t value;

    do {
        value = lapic_read(&lapic_map->icr_low);
        cpu_pause();
    } while (value & LAPIC_ICR_STATUS_PENDING);
}

void
lapic_ipi_init_assert(uint32_t apic_id)
{
    lapic_ipi(apic_id, LAPIC_ICR_TRIGGER_LEVEL | LAPIC_ICR_LEVEL_ASSERT
                       | LAPIC_ICR_DELIVERY_INIT);
    lapic_ipi_wait();
}

void
lapic_ipi_init_deassert(uint32_t apic_id)
{
    lapic_ipi(apic_id, LAPIC_ICR_TRIGGER_LEVEL | LAPIC_ICR_DELIVERY_INIT);
    lapic_ipi_wait();
}

void
lapic_ipi_startup(uint32_t apic_id, uint32_t vector)
{
    lapic_ipi(apic_id, LAPIC_ICR_DELIVERY_STARTUP
                       | (vector & LAPIC_ICR_VECTOR_MASK));
    lapic_ipi_wait();
}

void
lapic_ipi_send(uint32_t apic_id, uint32_t vector)
{
    lapic_ipi_wait();
    lapic_ipi(apic_id, vector & LAPIC_ICR_VECTOR_MASK);
}

void
lapic_ipi_broadcast(uint32_t vector)
{
    lapic_ipi_wait();
    lapic_ipi(0, LAPIC_ICR_DEST_ALL_EXCEPT_SELF
                 | (vector & LAPIC_ICR_VECTOR_MASK));
}


code:
/*
 * Copyright (c) 2011-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_LAPIC_H
#define X86_LAPIC_H

#include <stdbool.h>
#include <stdint.h>

/*
 * Send an end-of-interrupt message to the local APIC.
 */
void lapic_eoi(void);

/*
 * Set up the lapic module.
 */
void lapic_setup(uint32_t map_addr);

/*
 * Set up the local APIC for an AP.
 */
void lapic_ap_setup(void);

/*
 * Functions used when initializing an AP.
 */
void lapic_ipi_init_assert(uint32_t apic_id);
void lapic_ipi_init_deassert(uint32_t apic_id);
void lapic_ipi_startup(uint32_t apic_id, uint32_t vector);

/*
 * Fixed/broadcast inter-processor interrupts.
 */
void lapic_ipi_send(uint32_t apic_id, uint32_t vector);
void lapic_ipi_broadcast(uint32_t vector);

#endif /* X86_LAPIC_H */


code:
/*
 * Copyright (c) 2018 Agustina Arzille.
 * Copyright (c) 2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Architecture-specific definitions for local atomic operations.
 */

#ifndef X86_LATOMIC_H
#define X86_LATOMIC_H

#ifndef KERN_LATOMIC_H
#error "don't include <machine/latomic.h> directly, use <kern/latomic.h> instead"
#endif

#include <kern/atomic_types.h>
#include <kern/macros.h>

/*
 * Memory ordering is implemented with compiler barriers on entry, exit,
 * both, or neither, according to the specified ordering.
 */

#define latomic_x86_enter(memorder)                                         \
MACRO_BEGIN                                                                 \
    if ((memorder) != LATOMIC_RELAXED && (memorder) != LATOMIC_RELEASE) {   \
        barrier();                                                          \
    }                                                                       \
MACRO_END

#define latomic_x86_leave(memorder)                                         \
MACRO_BEGIN                                                                 \
    if ((memorder) != LATOMIC_RELAXED && (memorder) != LATOMIC_ACQUIRE) {   \
        barrier();                                                          \
    }                                                                       \
MACRO_END

#define latomic_x86_cas_n(ptr, oval, nval)                                  \
MACRO_BEGIN                                                                 \
    typeof(oval) prev_;                                                     \
                                                                            \
    asm volatile("cmpxchg %3, %1"                                           \
                 : "=a" (prev_), "+m" (*(ptr))                              \
                 : "0" (oval), "r" (nval));                                 \
    prev_;                                                                  \
MACRO_END

static unsigned int
latomic_x86_cas_ui(unsigned int *ptr, unsigned int oval, unsigned int nval)
{
    return latomic_x86_cas_n(ptr, oval, nval);
}

/*
 * 64-bit local atomic operations on i386 are implemented with loops using
 * the cmpxchg8b instruction. This assumes the processor is at least an i586.
 */
static unsigned long long
latomic_x86_cas_ull(unsigned long long *ptr, unsigned long long oval,
                    unsigned long long nval)
{
#ifdef __LP64__
    return latomic_x86_cas_n(ptr, oval, nval);
#else /* __LP64__ */
    asm volatile("cmpxchg8b %0"
                 : "+m" (*ptr), "+A" (oval)
                 : "b" ((unsigned long)nval),
                   "c" ((unsigned long)(nval >> 32)));
    return oval;
#endif /* __LP64__ */
}

/*
 * Helper for operations implemented with a CAS loop.
 */
#define latomic_x86_cas_loop_n(ptr, cas, op, val)                           \
MACRO_BEGIN                                                                 \
    typeof(val) prev_, oval_, nval_;                                        \
                                                                            \
    do {                                                                    \
        oval_ = *(ptr);                                                     \
        nval_ = oval_ op (val);                                             \
        prev_ = cas(ptr, oval_, nval_);                                     \
    } while (prev_ != oval_);                                               \
                                                                            \
    prev_;                                                                  \
MACRO_END

/* latomic_load */

#ifndef __LP64__
static inline unsigned long long
latomic_i386_load_64(union atomic_constptr_64 ptr, int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_ull((unsigned long long *)ptr.ull_ptr, 0ULL, 0ULL);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_load_64 latomic_i386_load_64
#endif /* __LP64__ */

/* latomic_store */

#ifndef __LP64__
static inline void
latomic_i386_store_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                      int memorder)
{
    unsigned long long oval, prev;

    latomic_x86_enter(memorder);

    do {
        oval = *ptr.ull_ptr;
        prev = latomic_x86_cas_ull(ptr.ull_ptr, oval, val.ull);
    } while (prev != oval);

    latomic_x86_leave(memorder);
}
#define latomic_store_64 latomic_i386_store_64
#endif /* __LP64__ */

/* latomic_swap */

/*
 * The swap operation is implemented with the xchg instruction, which
 * implies the lock prefix. As a result, simply reuse the built-in
 * provided by the compiler.
 */

static inline unsigned int
latomic_x86_swap_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                    int memorder)
{
    return __atomic_exchange_n(ptr.ui_ptr, val.ui, memorder);
}
#define latomic_swap_32 latomic_x86_swap_32

#ifdef __LP64__
static inline unsigned long long
latomic_amd64_swap_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                      int memorder)
{
    return __atomic_exchange_n(ptr.ull_ptr, val.ull, memorder);
}
#define latomic_swap_64 latomic_amd64_swap_64
#else /* __LP64__ */
static inline unsigned long long
latomic_i386_swap_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                     int memorder)
{
    unsigned long long oval, prev;

    latomic_x86_enter(memorder);

    do {
        oval = *ptr.ull_ptr;
        prev = latomic_x86_cas_ull(ptr.ull_ptr, oval, val.ull);
    } while (prev != oval);

    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_swap_64 latomic_i386_swap_64
#endif /* __LP64__ */

/* latomic_cas */

static inline unsigned int
latomic_x86_cas_32(union atomic_ptr_32 ptr, union atomic_val_32 oval,
                   union atomic_val_32 nval, int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_ui(ptr.ui_ptr, oval.ui, nval.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_cas_32 latomic_x86_cas_32

static inline unsigned long long
latomic_x86_cas_64(union atomic_ptr_64 ptr, union atomic_val_64 oval,
                   union atomic_val_64 nval, int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_ull(ptr.ull_ptr, oval.ull, nval.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_cas_64 latomic_x86_cas_64

/* latomic_fetch_add */

/*
 * The fetch_add and fetch_sub operations are the only fetch_xxx operations
 * that may not be implemented with a CAS loop, but with the xadd instruction
 * instead.
 */

#define latomic_x86_fetch_add_n(ptr, val)                                   \
MACRO_BEGIN                                                                 \
   typeof(val) prev_;                                                       \
                                                                            \
   asm volatile("xadd %0, %1"                                               \
                : "=r" (prev_), "+m" (*(ptr))                               \
                : "0" (val));                                               \
   prev_;                                                                   \
MACRO_END

static inline unsigned int
latomic_x86_fetch_add_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                         int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_fetch_add_n(ptr.ui_ptr, val.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_add_32 latomic_x86_fetch_add_32

#ifdef __LP64__
static inline unsigned long long
latomic_amd64_fetch_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                           int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_fetch_add_n(ptr.ull_ptr, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_add_64 latomic_amd64_fetch_add_64
#else /* __LP64__ */
static inline unsigned long long
latomic_i386_fetch_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                          int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ull_ptr, latomic_x86_cas_ull, +, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_add_64 latomic_i386_fetch_add_64
#endif /* __LP64__ */

/* latomic_fetch_sub */

static inline unsigned int
latomic_x86_fetch_sub_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                         int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_fetch_add_n(ptr.ui_ptr, -val.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_sub_32 latomic_x86_fetch_sub_32

#ifdef __LP64__
static inline unsigned long long
latomic_amd64_fetch_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                           int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_fetch_add_n(ptr.ull_ptr, -val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_sub_64 latomic_amd64_fetch_sub_64
#else /* __LP64__ */
static inline unsigned long long
latomic_i386_fetch_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                          int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ull_ptr, latomic_x86_cas_ull, -, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_sub_64 latomic_i386_fetch_sub_64
#endif /* __LP64__ */

/* latomic_fetch_and */

static inline unsigned int
latomic_x86_fetch_and_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                         int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ui_ptr, latomic_x86_cas_ui, &, val.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_and_32 latomic_x86_fetch_and_32

static inline unsigned long long
latomic_x86_fetch_and_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                         int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ull_ptr, latomic_x86_cas_ull, &, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_and_64 latomic_x86_fetch_and_64

/* latomic_fetch_or */

static inline unsigned int
latomic_x86_fetch_or_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                        int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ui_ptr, latomic_x86_cas_ui, |, val.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_or_32 latomic_x86_fetch_or_32

static inline unsigned long long
latomic_x86_fetch_or_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                        int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ull_ptr, latomic_x86_cas_ull, |, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_or_64 latomic_x86_fetch_or_64

/* latomic_fetch_xor */

static inline unsigned int
latomic_x86_fetch_xor_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                         int memorder)
{
    unsigned int prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ui_ptr, latomic_x86_cas_ui, ^, val.ui);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_xor_32 latomic_x86_fetch_xor_32

static inline unsigned long long
latomic_x86_fetch_xor_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                         int memorder)
{
    unsigned long long prev;

    latomic_x86_enter(memorder);
    prev = latomic_x86_cas_loop_n(ptr.ull_ptr, latomic_x86_cas_ull, ^, val.ull);
    latomic_x86_leave(memorder);

    return prev;
}
#define latomic_fetch_xor_64 latomic_x86_fetch_xor_64

/* latomic_add */

#define latomic_x86_add_n(ptr, val, suffix)                                 \
MACRO_BEGIN                                                                 \
    asm volatile("add" suffix " %1, %0"                                     \
                 : "+m" (*(ptr))                                            \
                 : "ir" (val));                                             \
MACRO_END

static inline void
latomic_x86_add_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                   int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_add_n(ptr.ui_ptr, val.ui, "l");
    latomic_x86_leave(memorder);
}
#define latomic_add_32 latomic_x86_add_32

#ifdef __LP64__
static inline void
latomic_amd64_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                     int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_add_n(ptr.ull_ptr, val.ull, "q");
    latomic_x86_leave(memorder);
}
#define latomic_add_64 latomic_amd64_add_64
#else /* __LP64__ */
static inline void
latomic_i386_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                    int memorder)
{
    latomic_fetch_add_64(ptr, val, memorder);
}
#define latomic_add_64 latomic_i386_add_64
#endif /* __LP64__ */

/* latomic_sub */

static inline void
latomic_x86_sub_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                   int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_add_n(ptr.ui_ptr, -val.ui, "l");
    latomic_x86_leave(memorder);
}
#define latomic_sub_32 latomic_x86_sub_32

#ifdef __LP64__
static inline void
latomic_amd64_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                     int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_add_n(ptr.ull_ptr, -val.ull, "q");
    latomic_x86_leave(memorder);
}
#define latomic_sub_64 latomic_amd64_sub_64
#else /* __LP64__ */
static inline void
latomic_i386_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                    int memorder)
{
    latomic_fetch_sub_64(ptr, val, memorder);
}
#define latomic_sub_64 latomic_i386_sub_64
#endif /* __LP64__ */

/* latomic_and */

#define latomic_x86_and_n(ptr, val, suffix)                                 \
MACRO_BEGIN                                                                 \
    asm volatile("and" suffix " %1, %0"                                     \
                 : "+m" (*(ptr))                                            \
                 : "ir" (val));                                             \
MACRO_END

static inline void
latomic_x86_and_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                   int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_and_n(ptr.ui_ptr, val.ui, "l");
    latomic_x86_leave(memorder);
}
#define latomic_and_32 latomic_x86_and_32

#ifdef __LP64__
static inline void
latomic_amd64_and_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                     int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_and_n(ptr.ull_ptr, val.ull, "q");
    latomic_x86_leave(memorder);
}
#define latomic_and_64 latomic_amd64_and_64
#else /* __LP64__ */
static inline void
latomic_i386_and_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                    int memorder)
{
    latomic_fetch_and_64(ptr, val, memorder);
}
#define latomic_and_64 latomic_i386_and_64
#endif /* __LP64__ */

/* latomic_or */

#define latomic_x86_or_n(ptr, val, suffix)                                  \
MACRO_BEGIN                                                                 \
    asm volatile("or" suffix " %1, %0"                                      \
                 : "+m" (*(ptr))                                            \
                 : "ir" (val));                                             \
MACRO_END

static inline void
latomic_x86_or_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                  int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_or_n(ptr.ui_ptr, val.ui, "l");
    latomic_x86_leave(memorder);
}
#define latomic_or_32 latomic_x86_or_32

#ifdef __LP64__
static inline void
latomic_amd64_or_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                    int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_or_n(ptr.ull_ptr, val.ull, "q");
    latomic_x86_leave(memorder);
}
#define latomic_or_64 latomic_amd64_or_64
#else /* __LP64__ */
static inline void
latomic_i386_or_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                   int memorder)
{
    latomic_fetch_or_64(ptr, val, memorder);
}
#define latomic_or_64 latomic_i386_or_64
#endif /* __LP64__ */

/* latomic_xor */

#define latomic_x86_xor_n(ptr, val, suffix)                                 \
MACRO_BEGIN                                                                 \
    asm volatile("xor" suffix " %1, %0"                                     \
                 : "+m" (*(ptr))                                            \
                 : "ir" (val));                                             \
MACRO_END

static inline void
latomic_x86_xor_32(union atomic_ptr_32 ptr, union atomic_val_32 val,
                   int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_xor_n(ptr.ui_ptr, val.ui, "l");
    latomic_x86_leave(memorder);
}
#define latomic_xor_32 latomic_x86_xor_32

#ifdef __LP64__
static inline void
latomic_amd64_xor_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                     int memorder)
{
    latomic_x86_enter(memorder);
    latomic_x86_xor_n(ptr.ull_ptr, val.ull, "q");
    latomic_x86_leave(memorder);
}
#define latomic_xor_64 latomic_amd64_xor_64
#else /* __LP64__ */
static inline void
latomic_i386_xor_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
                    int memorder)
{
    latomic_fetch_xor_64(ptr, val, memorder);
}
#define latomic_xor_64 latomic_i386_xor_64
#endif /* __LP64__ */

#endif /* X86_LATOMIC_H */


code:
/*
 * Copyright (c) 2010, 2012 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_MULTIBOOT_H
#define X86_MULTIBOOT_H

/*
 * Magic number provided by the OS to the boot loader.
 */
#define MULTIBOOT_OS_MAGIC 0x1badb002

/*
 * Multiboot flags requesting services from the boot loader.
 */
#define MULTIBOOT_OS_MEMORY_INFO 0x2

#define MULTIBOOT_OS_FLAGS MULTIBOOT_OS_MEMORY_INFO

/*
 * Magic number to identify a multiboot compliant boot loader.
 */
#define MULTIBOOT_LOADER_MAGIC 0x2badb002

/*
 * Multiboot flags set by the boot loader.
 */
#define MULTIBOOT_LOADER_MEMORY     0x01
#define MULTIBOOT_LOADER_CMDLINE    0x04
#define MULTIBOOT_LOADER_MODULES    0x08
#define MULTIBOOT_LOADER_SHDR       0x20
#define MULTIBOOT_LOADER_MMAP       0x40

#ifndef __ASSEMBLER__

#include <stdint.h>

#include <kern/macros.h>

/*
 * A multiboot module.
 */
struct multiboot_raw_module {
    uint32_t mod_start;
    uint32_t mod_end;
    uint32_t string;
    uint32_t reserved;
} __packed;

/*
 * Memory map entry.
 */
struct multiboot_raw_mmap_entry {
    uint32_t size;
    uint64_t base_addr;
    uint64_t length;
    uint32_t type;
} __packed;

/*
 * Multiboot information structure as passed by the boot loader.
 */
struct multiboot_raw_info {
    uint32_t flags;
    uint32_t mem_lower;
    uint32_t mem_upper;
    uint32_t unused0;
    uint32_t cmdline;
    uint32_t mods_count;
    uint32_t mods_addr;
    uint32_t shdr_num;
    uint32_t shdr_size;
    uint32_t shdr_addr;
    uint32_t shdr_strndx;
    uint32_t mmap_length;
    uint32_t mmap_addr;
    uint32_t unused1[9];
} __packed;

/*
 * Versions of the multiboot structures suitable for use with 64-bit pointers.
 */

struct multiboot_module {
    void *mod_start;
    void *mod_end;
    char *string;
};

struct multiboot_info {
    uint32_t flags;
    struct multiboot_module *mods_addr;
    uint32_t mods_count;
};

#endif /* __ASSEMBLER__ */

#endif /* X86_MULTIBOOT_H */


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * This file is a top header in the inclusion hierarchy, and shouldn't include
 * other headers that may cause circular dependencies.
 */

#ifndef X86_PAGE_H
#define X86_PAGE_H

#define PAGE_SHIFT  12
#define PAGE_SIZE   (1 << PAGE_SHIFT)
#define PAGE_MASK   (PAGE_SIZE - 1)

#endif /* X86_PAGE_H */


code:
/*
 * Copyright (c) 2012-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stdint.h>

#include <kern/error.h>
#include <kern/init.h>
#include <kern/intr.h>
#include <kern/panic.h>
#include <machine/cpu.h>
#include <machine/io.h>
#include <machine/lapic.h>
#include <machine/pic.h>

/*
 * I/O ports.
 */
#define PIC_MASTER_CMD      0x20
#define PIC_MASTER_IMR      0x21
#define PIC_SLAVE_CMD       0xa0
#define PIC_SLAVE_IMR       0xa1

/*
 * Register bits.
 */
#define PIC_ICW1_IC4        0x01
#define PIC_ICW1_INIT       0x10
#define PIC_ICW4_8086       0x01
#define PIC_OCW3_ISR        0x0b
#define PIC_EOI             0x20

/*
 * Special interrupts.
 */
#define PIC_SLAVE_INTR      2
#define PIC_SPURIOUS_INTR   7

static unsigned int pic_nr_slave_intrs;

static uint8_t pic_master_mask;
static uint8_t pic_slave_mask;

static uint8_t pic_master_spurious_intr;
static uint8_t pic_slave_spurious_intr;

static bool
pic_is_slave_intr(unsigned int intr)
{
    assert(intr <= PIC_MAX_INTR);
    return (intr >= PIC_NR_INTRS);
}

static void
pic_inc_slave_intrs(void)
{
    if (pic_nr_slave_intrs == 0) {
        pic_master_mask |= 1 << PIC_SLAVE_INTR;
        io_write_byte(PIC_MASTER_IMR, pic_master_mask);
    }

    pic_nr_slave_intrs++;
    assert(pic_nr_slave_intrs != 0);
}

static void
pic_dec_slave_intrs(void)
{
    assert(pic_nr_slave_intrs != 0);

    pic_nr_slave_intrs--;

    if (pic_nr_slave_intrs == 0) {
        pic_master_mask &= ~(1 << PIC_SLAVE_INTR);
        io_write_byte(PIC_MASTER_IMR, pic_master_mask);
    }
}

static void
pic_eoi(unsigned long intr)
{
    if (intr >= PIC_NR_INTRS) {
        io_write_byte(PIC_SLAVE_CMD, PIC_EOI);
    }

    io_write_byte(PIC_MASTER_CMD, PIC_EOI);
}

static uint8_t
pic_read_isr(uint16_t port)
{
    io_write_byte(port, PIC_OCW3_ISR);
    return io_read_byte(port);
}

static void
pic_ops_enable(void *priv, unsigned int intr, unsigned int cpu)
{
    (void)priv;
    (void)cpu;

    if (pic_is_slave_intr(intr)) {
        pic_slave_mask &= ~(1 << (intr - PIC_NR_INTRS));
        io_write_byte(PIC_SLAVE_IMR, pic_slave_mask);
        pic_inc_slave_intrs();
    } else {
        pic_master_mask &= ~(1 << intr);
        io_write_byte(PIC_MASTER_IMR, pic_master_mask);
    }
}

static void
pic_ops_disable(void *priv, unsigned int intr)
{
    (void)priv;

    if (pic_is_slave_intr(intr)) {
        pic_dec_slave_intrs();
        pic_slave_mask |= 1 << (intr - PIC_NR_INTRS);
        io_write_byte(PIC_SLAVE_IMR, pic_slave_mask);
    } else {
        pic_master_mask |= 1 << intr;
        io_write_byte(PIC_MASTER_IMR, pic_master_mask);
    }
}

static void
pic_ops_eoi(void *priv, unsigned int intr)
{
    (void)priv;
    pic_eoi(intr);
}

static const struct intr_ops pic_ops = {
    .enable = pic_ops_enable,
    .disable = pic_ops_disable,
    .eoi = pic_ops_eoi,
};

static void
pic_intr(unsigned int vector)
{
    intr_handle(vector - CPU_EXC_INTR_FIRST);
}

static void __init
pic_register(void)
{
    unsigned int intr;

    intr_register_ctl(&pic_ops, NULL, 0, PIC_MAX_INTR);

    for (intr = 0; intr <= PIC_MAX_INTR; intr++) {
        cpu_register_intr(CPU_EXC_INTR_FIRST + intr, pic_intr);
    }
}

static int
pic_spurious_intr(void *arg)
{
    uint8_t intr, isr;

    intr = *(const uint8_t *)arg;

    if (arg == &pic_master_spurious_intr) {
        isr = pic_read_isr(PIC_MASTER_CMD);

        if (isr & (1 << PIC_SPURIOUS_INTR)) {
            panic("pic: real interrupt %hhu", intr);
        }
    } else {
        isr = pic_read_isr(PIC_SLAVE_CMD);

        if (isr & (1 << PIC_SPURIOUS_INTR)) {
            panic("pic: real interrupt %hhu", intr);
        }

        pic_eoi(PIC_SLAVE_INTR);
    }

    return 0;
}

static void __init
pic_setup_common(bool register_ctl)
{
    int error;

    pic_nr_slave_intrs = 0;
    pic_master_mask = 0xff;
    pic_slave_mask = 0xff;

    /* ICW 1 - State that ICW 4 will be sent */
    io_write_byte(PIC_MASTER_CMD, PIC_ICW1_INIT | PIC_ICW1_IC4);
    io_write_byte(PIC_SLAVE_CMD, PIC_ICW1_INIT | PIC_ICW1_IC4);

    /* ICW 2 */
    io_write_byte(PIC_MASTER_IMR, CPU_EXC_INTR_FIRST);
    io_write_byte(PIC_SLAVE_IMR, CPU_EXC_INTR_FIRST + PIC_NR_INTRS);

    /* ICW 3 - Set up cascading */
    io_write_byte(PIC_MASTER_IMR, 1 << PIC_SLAVE_INTR);
    io_write_byte(PIC_SLAVE_IMR, PIC_SLAVE_INTR);

    /* ICW 4 - Set 8086 mode */
    io_write_byte(PIC_MASTER_IMR, PIC_ICW4_8086);
    io_write_byte(PIC_SLAVE_IMR, PIC_ICW4_8086);

    /* OCW 1 - Mask all interrupts */
    io_write_byte(PIC_MASTER_IMR, pic_master_mask);
    io_write_byte(PIC_SLAVE_IMR, pic_slave_mask);

    if (register_ctl) {
        pic_register();
    }

    pic_master_spurious_intr = PIC_SPURIOUS_INTR;
    error = intr_register(pic_master_spurious_intr, pic_spurious_intr,
                          &pic_master_spurious_intr);
    error_check(error, __func__);

    pic_slave_spurious_intr = PIC_NR_INTRS + PIC_SPURIOUS_INTR;
    error = intr_register(pic_slave_spurious_intr, pic_spurious_intr,
                          &pic_slave_spurious_intr);
    error_check(error, __func__);
}

void __init
pic_setup(void)
{
    pic_setup_common(true);
}

void __init
pic_setup_disabled(void)
{
    pic_setup_common(false);
}


code:
/*
 * Copyright (c) 2012-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_PIC_H
#define X86_PIC_H

/*
 * Interrupts per PIC.
 */
#define PIC_NR_INTRS    8

/*
 * Maximum global interrupt number.
 */
#define PIC_MAX_INTR    ((PIC_NR_INTRS * 2) - 1)

/*
 * Initialize the pic module.
 */
void pic_setup(void);

/*
 * Initialize the pic module in an APIC system.
 *
 * This function is called by the acpi module if ACPI reports the presence
 * of legacy interrupt controllers.
 *
 * Since it doesn't register the legacy PIC as an interrupt controller, the
 * acpi module must have registered I/O APICs before calling this function.
 */
void pic_setup_disabled(void);

#endif /* X86_PIC_H */


code:
/*
 * Copyright (c) 2011-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>

#include <kern/clock.h>
#include <kern/init.h>
#include <kern/intr.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <machine/io.h>
#include <machine/pit.h>

/*
 * I/O ports.
 */
#define PIT_PORT_COUNTER0   0x40
#define PIT_PORT_MODE       0x43

/*
 * Mode control register bits.
 */
#define PIT_MODE_LATCH      0x00
#define PIT_MODE_RATE_GEN   0x04
#define PIT_MODE_RW_LSB     0x10
#define PIT_MODE_RW_MSB     0x20

/*
 * Native timer frequency.
 */
#define PIT_FREQ 1193182

/*
 * Maximum value of a counter.
 */
#define PIT_MAX_COUNT 0xffff

/*
 * Timer interrupt.
 */
#define PIT_INTR 0

static int
pit_intr(void *arg)
{
    (void)arg;

    clock_tick_intr();
    return 0;
}

static void __init
pit_setup_common(uint16_t count)
{
    io_write_byte(PIT_PORT_MODE, PIT_MODE_RATE_GEN | PIT_MODE_RW_LSB
                                 | PIT_MODE_RW_MSB);
    io_write_byte(PIT_PORT_COUNTER0, count & 0xff);
    io_write_byte(PIT_PORT_COUNTER0, count >> 8);
}

void __init
pit_setup_free_running(void)
{
    pit_setup_common(PIT_MAX_COUNT);
}

void __init
pit_setup(void)
{
    int error;

    pit_setup_common(DIV_CEIL(PIT_FREQ, CLOCK_FREQ));
    error = intr_register(PIT_INTR, pit_intr, NULL);

    if (error) {
        log_err("pit: unable to register interrupt handler");
    }
}

static unsigned int
pit_read(void)
{
    unsigned int low, high;

    io_write_byte(PIT_PORT_MODE, PIT_MODE_LATCH);
    low = io_read_byte(PIT_PORT_COUNTER0);
    high = io_read_byte(PIT_PORT_COUNTER0);
    return (high << 8) | low;
}

void
pit_delay(unsigned long usecs)
{
    long total, prev, count, diff;

    assert(usecs != 0);

    /* TODO Avoid 64-bits conversion if result is known not to overflow */
    total = (long)(((long long)usecs * PIT_FREQ + 999999) / 1000000);
    prev = pit_read();

    do {
        count = pit_read();
        diff = prev - count;
        prev = count;

        if (diff < 0) {
            diff += PIT_MAX_COUNT;
        }

        total -= diff;
    } while (total > 0);
}


code:
/*
 * Copyright (c) 2011 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_PIT_H
#define X86_PIT_H

/*
 * Initialize the PIT as a free running counter.
 *
 * This is used during early initialization to measure the frequency of
 * other clocks. The PIT is used despite its lack of precision because
 * it's the only architectural timer with a known frequency.
 */
void pit_setup_free_running(void);

/*
 * Initialize the pit module.
 */
void pit_setup(void);

/*
 * Wait (without sleeping) until the specified amount of time has elapsed.
 */
void pit_delay(unsigned long usecs);

#endif /* X86_PIT_H */


code:
/*
 * Copyright (c) 2010-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * TODO Review locking.
 */

#include <assert.h>
#include <errno.h>
#include <stdalign.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

#include <kern/cpumap.h>
#include <kern/init.h>
#include <kern/kmem.h>
#include <kern/list.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/mutex.h>
#include <kern/panic.h>
#include <kern/percpu.h>
#include <kern/spinlock.h>
#include <kern/syscnt.h>
#include <kern/thread.h>
#include <machine/biosmem.h>
#include <machine/boot.h>
#include <machine/cpu.h>
#include <machine/page.h>
#include <machine/pmap.h>
#include <machine/tcb.h>
#include <machine/types.h>
#include <vm/vm_kmem.h>
#include <vm/vm_page.h>
#include <vm/vm_prot.h>

/*
 * Properties of a page translation level.
 */
struct pmap_pt_level {
    unsigned int skip;
    unsigned int bits;
    unsigned int ptes_per_pt;
    pmap_pte_t mask;
};

/*
 * Table of page translation properties.
 */
static struct pmap_pt_level pmap_pt_levels[] __read_mostly = {
    { PMAP_L0_SKIP, PMAP_L0_BITS, PMAP_L0_PTES_PER_PT, PMAP_L0_MASK },
    { PMAP_L1_SKIP, PMAP_L1_BITS, PMAP_L1_PTES_PER_PT, PMAP_L1_MASK },
#if PMAP_NR_LEVELS > 2
    { PMAP_L2_SKIP, PMAP_L2_BITS, PMAP_L2_PTES_PER_PT, PMAP_L2_MASK },
#if PMAP_NR_LEVELS > 3
    { PMAP_L3_SKIP, PMAP_L3_BITS, PMAP_L3_PTES_PER_PT, PMAP_L3_MASK },
#endif /* PMAP_NR_LEVELS > 3 */
#endif /* PMAP_NR_LEVELS > 2 */
};

/*
 * Per-CPU page tables.
 */
struct pmap_cpu_table {
    struct list node;
    phys_addr_t root_ptp_pa;
};

struct pmap {
    struct pmap_cpu_table *cpu_tables[CONFIG_MAX_CPUS];
};

/*
 * Type for page table walking functions.
 *
 * See pmap_walk_vas().
 */
typedef void (*pmap_walk_fn_t)(phys_addr_t pa, unsigned int index,
                               unsigned int level);

/*
 * The kernel per-CPU page tables are used early enough during bootstrap
 * that using a percpu variable would actually become ugly. This array
 * is rather small anyway.
 */
static struct pmap_cpu_table pmap_kernel_cpu_tables[CONFIG_MAX_CPUS] __read_mostly;

struct pmap pmap_kernel_pmap;

struct pmap *pmap_current_ptr __percpu;

#ifdef CONFIG_X86_PAE

/*
 * Alignment required on page directory pointer tables.
 */
#define PMAP_PDPT_ALIGN 32

/*
 * "Hidden" kernel root page tables for PAE mode.
 */
static alignas(PMAP_PDPT_ALIGN) pmap_pte_t
    pmap_cpu_kpdpts[CONFIG_MAX_CPUS][PMAP_L2_PTES_PER_PT] __read_mostly;

#endif /* CONFIG_X86_PAE */

/*
 * Flags related to page protection.
 */
#define PMAP_PTE_PROT_MASK PMAP_PTE_RW

/*
 * Table used to convert machine independent protection flags to architecture
 * specific PTE bits.
 */
static pmap_pte_t pmap_prot_table[VM_PROT_ALL + 1] __read_mostly;

/*
 * Structures related to inter-processor page table updates.
 */

#define PMAP_UPDATE_OP_ENTER    1
#define PMAP_UPDATE_OP_REMOVE   2
#define PMAP_UPDATE_OP_PROTECT  3

struct pmap_update_enter_args {
    uintptr_t va;
    phys_addr_t pa;
    int prot;
    int flags;
};

struct pmap_update_remove_args {
    uintptr_t start;
    uintptr_t end;
};

struct pmap_update_protect_args {
    uintptr_t start;
    uintptr_t end;
    int prot;
};

struct pmap_update_op {
    struct cpumap cpumap;
    unsigned int operation;

    union {
        struct pmap_update_enter_args enter_args;
        struct pmap_update_remove_args remove_args;
        struct pmap_update_protect_args protect_args;
    };
};

/*
 * Maximum number of operations that can be batched before an implicit
 * update.
 */
#define PMAP_UPDATE_MAX_OPS 32

/*
 * List of update operations.
 *
 * A list of update operations is a container of operations that are pending
 * for a pmap. Updating can be implicit, e.g. when a list has reached its
 * maximum size, or explicit, when pmap_update() is called. Operation lists
 * are thread-local objects.
 *
 * The cpumap is the union of all processors affected by at least one
 * operation.
 */
struct pmap_update_oplist {
    alignas(CPU_L1_SIZE) struct cpumap cpumap;
    struct pmap *pmap;
    unsigned int nr_ops;
    struct pmap_update_op ops[PMAP_UPDATE_MAX_OPS];
};

/*
 * Statically allocated data for the main booter thread.
 */
static struct cpumap pmap_booter_cpumap __initdata;
static struct pmap_update_oplist pmap_booter_oplist __initdata;

/*
 * Each regular thread gets an operation list from this cache.
 */
static struct kmem_cache pmap_update_oplist_cache;

/*
 * Queue holding update requests from remote processors.
 */
struct pmap_update_queue {
    struct spinlock lock;
    struct list requests;
};

/*
 * Syncer thread.
 *
 * There is one such thread per processor. They are the recipients of
 * update requests, providing thread context for the mapping operations
 * they perform.
 */
struct pmap_syncer {
    alignas(CPU_L1_SIZE) struct thread *thread;
    struct pmap_update_queue queue;
    struct syscnt sc_updates;
    struct syscnt sc_update_enters;
    struct syscnt sc_update_removes;
    struct syscnt sc_update_protects;
};

static void pmap_sync(void *arg);

static struct pmap_syncer pmap_syncer __percpu;

/*
 * Maximum number of mappings for which individual TLB invalidations can be
 * performed. Global TLB flushes are done beyond this value.
 */
#define PMAP_UPDATE_MAX_MAPPINGS 64

/*
 * Per processor request, queued on a remote processor.
 *
 * The number of mappings is used to determine whether it's best to flush
 * individual TLB entries or globally flush the TLB.
 */
struct pmap_update_request {
    alignas(CPU_L1_SIZE) struct list node;
    struct spinlock lock;
    struct thread *sender;
    const struct pmap_update_oplist *oplist;
    unsigned int nr_mappings;
    int done;
    int error;
};

/*
 * Per processor array of requests.
 *
 * When an operation list is to be applied, the thread triggering the update
 * acquires the processor-local array of requests and uses it to queue requests
 * on remote processors.
 */
struct pmap_update_request_array {
    struct pmap_update_request requests[CONFIG_MAX_CPUS];
    struct mutex lock;
};

static struct pmap_update_request_array pmap_update_request_array __percpu;

static int pmap_do_remote_updates __read_mostly;

static struct kmem_cache pmap_cache;

#ifdef CONFIG_X86_PAE
static char pmap_panic_no_pae[] __bootdata
    = "pmap: PAE not supported";
#endif /* CONFIG_X86_PAE */
static char pmap_panic_inval_msg[] __bootdata
    = "pmap: invalid physical address";
static char pmap_panic_directmap_msg[] __bootdata
    = "pmap: invalid direct physical mapping";

static __always_inline unsigned long
pmap_pte_index(uintptr_t va, const struct pmap_pt_level *pt_level)
{
    return ((va >> pt_level->skip) & ((1UL << pt_level->bits) - 1));
}

static void __boot
pmap_boot_enter(pmap_pte_t *root_ptp, uintptr_t va, phys_addr_t pa,
                unsigned long pgsize)
{
    const struct pmap_pt_level *pt_level, *pt_levels;
    pmap_pte_t *pt, *ptp, *pte, bits;
    unsigned int level, last_level;

    if (pa != (pa & PMAP_PA_MASK)) {
        boot_panic(pmap_panic_inval_msg);
    }

    switch (pgsize) {
#ifdef __LP64__
    case (1 << PMAP_L2_SKIP):
        bits = PMAP_PTE_PS;
        last_level = 2;
        break;
#endif /* __LP64__ */
    case (1 << PMAP_L1_SKIP):
        bits = PMAP_PTE_PS;
        last_level = 1;
        break;
    default:
        bits = 0;
        last_level = 0;
    }

    pt_levels = (void *)BOOT_VTOP((uintptr_t)pmap_pt_levels);
    pt = root_ptp;

    for (level = PMAP_NR_LEVELS - 1; level != last_level; level--) {
        pt_level = &pt_levels[level];
        pte = &pt[pmap_pte_index(va, pt_level)];

        if (*pte != 0) {
            ptp = (void *)(uintptr_t)(*pte & PMAP_PA_MASK);
        } else {
            ptp = biosmem_bootalloc(1);
            *pte = ((uintptr_t)ptp | PMAP_PTE_RW | PMAP_PTE_P)
                   & pt_level->mask;
        }

        pt = ptp;
    }

    pt_level = &pt_levels[last_level];
    pte = &pt[pmap_pte_index(va, pt_level)];
    *pte = (pa & PMAP_PA_MASK) | PMAP_PTE_RW | PMAP_PTE_P | bits;
}

static unsigned long __boot
pmap_boot_get_pgsize(void)
{
    unsigned int eax, ebx, ecx, edx;

#ifdef __LP64__
    eax = CPU_CPUID_EXT_BIT;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);

    if (eax <= CPU_CPUID_EXT_BIT) {
        goto out;
    }

    eax = (CPU_CPUID_EXT_BIT | 1);
    cpu_cpuid(&eax, &ebx, &ecx, &edx);

    if (edx & CPU_CPUID_EXT1_EDX_1GP) {
        return (1 << PMAP_L2_SKIP);
    }

out:
    return (1 << PMAP_L1_SKIP);
#else /* __LP64__ */
    eax = 0;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);

    if (eax == 0) {
        goto out;
    }

    eax = 1;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);

#ifdef CONFIG_X86_PAE
    if (!(edx & CPU_CPUID_BASIC1_EDX_PAE)) {
        boot_panic(pmap_panic_no_pae);
    }

    return (1 << PMAP_L1_SKIP);
#else /* CONFIG_X86_PAE */
    if (edx & CPU_CPUID_BASIC1_EDX_PSE) {
        return (1 << PMAP_L1_SKIP);
    }
#endif /* CONFIG_X86_PAE */

out:
    return PAGE_SIZE;
#endif /* __LP64__ */
}

#ifdef __LP64__
#define pmap_boot_enable_pgext(pgsize) ((void)(pgsize))
#else /* __LP64__ */
static void __boot
pmap_boot_enable_pgext(unsigned long pgsize)
{
    if (pgsize == PAGE_SIZE) {
        return;
    }

    /*
     * On 64-bits systems, PAE is already enabled.
     *
     * See the boot module.
     */
#ifdef CONFIG_X86_PAE
    cpu_enable_pae();
#else /* CONFIG_X86_PAE */
    cpu_enable_pse();
#endif /* CONFIG_X86_PAE */
}
#endif /* __LP64__ */

pmap_pte_t * __boot
pmap_setup_paging(void)
{
    struct pmap_cpu_table *cpu_table;
    phys_addr_t pa, directmap_end;
    unsigned long i, size, pgsize;
    pmap_pte_t *root_ptp;
    uintptr_t va;

    /* Use large pages for the direct physical mapping when possible */
    pgsize = pmap_boot_get_pgsize();
    pmap_boot_enable_pgext(pgsize);

    /*
     * Create the initial mappings. The first is for the .boot section
     * and acts as the mandatory identity mapping. The second is the
     * direct physical mapping of physical memory.
     */

#ifdef CONFIG_X86_PAE
    root_ptp = (void *)BOOT_VTOP((uintptr_t)pmap_cpu_kpdpts[0]);
#else /* CONFIG_X86_PAE */
    root_ptp = biosmem_bootalloc(1);
#endif /* CONFIG_X86_PAE */

    va = vm_page_trunc((uintptr_t)&_boot);
    pa = va;
    size = vm_page_round((uintptr_t)&_boot_end) - va;

    for (i = 0; i < size; i += PAGE_SIZE) {
        pmap_boot_enter(root_ptp, va, pa, PAGE_SIZE);
        va += PAGE_SIZE;
        pa += PAGE_SIZE;
    }

    directmap_end = biosmem_directmap_end();

    if (directmap_end > (PMAP_END_DIRECTMAP_ADDRESS
                         - PMAP_START_DIRECTMAP_ADDRESS)) {
        boot_panic(pmap_panic_directmap_msg);
    }

    va = PMAP_START_DIRECTMAP_ADDRESS;
    pa = 0;

    for (i = 0; i < directmap_end; i += pgsize) {
        pmap_boot_enter(root_ptp, va, pa, pgsize);
        va += pgsize;
        pa += pgsize;
    }

#ifdef __LP64__
    /*
     * On 64-bits systems, the kernel isn't linked at addresses included
     * in the direct mapping, which requires the creation of an additional
     * mapping for it.
     */
    va = P2ALIGN((uintptr_t)&_init, pgsize);
    pa = BOOT_VTOP(va);
    size = vm_page_round((uintptr_t)&_end) - va;

    for (i = 0; i < size; i += pgsize) {
        pmap_boot_enter(root_ptp, va, pa, pgsize);
        va += pgsize;
        pa += pgsize;
    }
#endif /* __LP64__ */

    cpu_table = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_cpu_tables[0]);
    cpu_table->root_ptp_pa = (uintptr_t)root_ptp;

    return root_ptp;
}

pmap_pte_t * __boot
pmap_ap_setup_paging(unsigned int ap_id)
{
    struct pmap_cpu_table *cpu_table;
    struct pmap *pmap;
    unsigned long pgsize;

    pgsize = pmap_boot_get_pgsize();
    pmap_boot_enable_pgext(pgsize);

    pmap = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_pmap);
    cpu_table = (void *)BOOT_VTOP((uintptr_t)pmap->cpu_tables[ap_id]);

#ifdef CONFIG_X86_PAE
    return (void *)(uint32_t)cpu_table->root_ptp_pa;
#else /* CONFIG_X86_PAE */
    return (void *)cpu_table->root_ptp_pa;
#endif /* CONFIG_X86_PAE */
}

static bool
pmap_range_valid(const struct pmap *pmap, uintptr_t start, uintptr_t end)
{
    return (start < end)
           && ((end <= PMAP_START_DIRECTMAP_ADDRESS)
               || (start >= PMAP_END_DIRECTMAP_ADDRESS))
           && ((pmap == pmap_get_kernel_pmap())
               ? ((start >= PMAP_START_KMEM_ADDRESS)
                  && (end <= PMAP_END_KMEM_ADDRESS))
               : (end <= PMAP_END_ADDRESS));
}

static inline pmap_pte_t *
pmap_ptp_from_pa(phys_addr_t pa)
{
    uintptr_t va;

    va = vm_page_direct_va(pa);
    return (pmap_pte_t *)va;
}

static void
pmap_ptp_clear(pmap_pte_t *ptp)
{
    memset(ptp, 0, PAGE_SIZE);
}

static inline void
pmap_pte_set(pmap_pte_t *pte, phys_addr_t pa, pmap_pte_t pte_bits,
             const struct pmap_pt_level *pt_level)
{
    *pte = ((pa & PMAP_PA_MASK) | PMAP_PTE_P | pte_bits) & pt_level->mask;
}

static inline void
pmap_pte_clear(pmap_pte_t *pte)
{
    *pte = 0;
}

static inline int
pmap_pte_valid(pmap_pte_t pte)
{
    return (pte != 0);
}

static inline int
pmap_pte_large(pmap_pte_t pte)
{
    return ((pte & PMAP_PTE_PS) != 0);
}

static inline pmap_pte_t *
pmap_pte_next(pmap_pte_t pte)
{
    assert(pmap_pte_valid(pte));
    return pmap_ptp_from_pa(pte & PMAP_PA_MASK);
}

/*
 * Helper function for initialization procedures that require post-fixing
 * page properties.
 */
static void __init
pmap_walk_vas(uintptr_t start, uintptr_t end, pmap_walk_fn_t walk_fn)
{
    const struct pmap_pt_level *pt_level;
    phys_addr_t root_ptp_pa, ptp_pa;
    pmap_pte_t *ptp, *pte;
    unsigned int index, level;
    uintptr_t va;

    assert(vm_page_aligned(start));
    assert(start < end);
#ifdef __LP64__
    assert((start < PMAP_END_ADDRESS) || (start >= PMAP_START_KERNEL_ADDRESS));
#endif /* __LP64__ */

    va = start;
    root_ptp_pa = pmap_get_kernel_pmap()->cpu_tables[cpu_id()]->root_ptp_pa;

    do {
#ifdef __LP64__
        /* Handle long mode canonical form */
        if (va == PMAP_END_ADDRESS) {
            va = PMAP_START_KERNEL_ADDRESS;
        }
#endif /* __LP64__ */

        level = PMAP_NR_LEVELS - 1;
        ptp_pa = root_ptp_pa;
        ptp = pmap_ptp_from_pa(ptp_pa);

        for (;;) {
            pt_level = &pmap_pt_levels[level];
            index = pmap_pte_index(va, pt_level);
            pte = &ptp[index];

            if (!pmap_pte_valid(*pte)) {
                break;
            }

            walk_fn(ptp_pa, index, level);

            if ((level == 0) || pmap_pte_large(*pte)) {
                break;
            }

            level--;
            ptp_pa = *pte & PMAP_PA_MASK;
            ptp = pmap_ptp_from_pa(ptp_pa);
        }

        va = P2END(va, 1UL << pt_level->skip);
    } while ((va > start) && (va < end));
}

static void __init
pmap_setup_global_page(phys_addr_t ptp_pa, unsigned int index,
                       unsigned int level)
{
    pmap_pte_t *pte;

    pte = &pmap_ptp_from_pa(ptp_pa)[index];

    if ((level == 0) || pmap_pte_large(*pte)) {
        *pte |= PMAP_PTE_G;
    }
}

static void __init
pmap_setup_global_pages(void)
{
    pmap_walk_vas(PMAP_START_KERNEL_ADDRESS, PMAP_END_KERNEL_ADDRESS,
                  pmap_setup_global_page);
    pmap_pt_levels[0].mask |= PMAP_PTE_G;
    cpu_enable_global_pages();
}

static void
pmap_update_oplist_ctor(void *arg)
{
    struct pmap_update_oplist *oplist;

    oplist = arg;
    cpumap_zero(&oplist->cpumap);
    oplist->pmap = NULL;
    oplist->nr_ops = 0;
}

static int
pmap_update_oplist_create(struct pmap_update_oplist **oplistp)
{
    struct pmap_update_oplist *oplist;

    oplist = kmem_cache_alloc(&pmap_update_oplist_cache);

    if (oplist == NULL) {
        return ENOMEM;
    }

    *oplistp = oplist;
    return 0;
}

static void
pmap_update_oplist_destroy(struct pmap_update_oplist *oplist)
{
    kmem_cache_free(&pmap_update_oplist_cache, oplist);
}

static struct pmap_update_oplist *
pmap_update_oplist_get(void)
{
    struct pmap_update_oplist *oplist;

    oplist = tcb_get_pmap_update_oplist(tcb_current());
    assert(oplist != NULL);
    return oplist;
}

static int
pmap_update_oplist_prepare(struct pmap_update_oplist *oplist,
                           struct pmap *pmap)
{
    int error;

    if (oplist->pmap != pmap) {
        assert(oplist->pmap == NULL);
        oplist->pmap = pmap;
    } else if (oplist->nr_ops == ARRAY_SIZE(oplist->ops)) {
        error = pmap_update(pmap);
        oplist->pmap = pmap;
        return error;
    }

    return 0;
}

static struct pmap_update_op *
pmap_update_oplist_prev_op(struct pmap_update_oplist *oplist)
{
    if (oplist->nr_ops == 0) {
        return NULL;
    }

    return &oplist->ops[oplist->nr_ops - 1];
}

static struct pmap_update_op *
pmap_update_oplist_prepare_op(struct pmap_update_oplist *oplist)
{
    assert(oplist->nr_ops < ARRAY_SIZE(oplist->ops));
    return &oplist->ops[oplist->nr_ops];
}

static void
pmap_update_oplist_finish_op(struct pmap_update_oplist *oplist)
{
    struct pmap_update_op *op;

    assert(oplist->nr_ops < ARRAY_SIZE(oplist->ops));
    op = &oplist->ops[oplist->nr_ops];
    cpumap_or(&oplist->cpumap, &op->cpumap);
    oplist->nr_ops++;
}

static unsigned int
pmap_update_oplist_count_mappings(const struct pmap_update_oplist *oplist,
                                  unsigned int cpu)
{
    const struct pmap_update_op *op;
    unsigned int i, nr_mappings;

    nr_mappings = 0;

    for (i = 0; i < oplist->nr_ops; i++) {
        op = &oplist->ops[i];

        if (!cpumap_test(&op->cpumap, cpu)) {
            continue;
        }

        switch (op->operation) {
        case PMAP_UPDATE_OP_ENTER:
            nr_mappings++;
            break;
        case PMAP_UPDATE_OP_REMOVE:
            nr_mappings += (op->remove_args.end - op->remove_args.start)
                           / PAGE_SIZE;
            break;
        case PMAP_UPDATE_OP_PROTECT:
            nr_mappings += (op->protect_args.end - op->protect_args.start)
                           / PAGE_SIZE;
            break;
        default:
            assert(!"invalid update operation");
        }
    }

    assert(nr_mappings != 0);
    return nr_mappings;
}

static void
pmap_update_request_array_init(struct pmap_update_request_array *array)
{
    struct pmap_update_request *request;
    unsigned int i;

    for (i = 0; i < ARRAY_SIZE(array->requests); i++) {
        request = &array->requests[i];
        spinlock_init(&request->lock);
    }

    mutex_init(&array->lock);
}

static struct pmap_update_request_array *
pmap_update_request_array_acquire(void)
{
    struct pmap_update_request_array *array;

    thread_pin();
    array = cpu_local_ptr(pmap_update_request_array);
    mutex_lock(&array->lock);
    return array;
}

static void
pmap_update_request_array_release(struct pmap_update_request_array *array)
{
    mutex_unlock(&array->lock);
    thread_unpin();
}

static void __init
pmap_syncer_init(struct pmap_syncer *syncer, unsigned int cpu)
{
    char name[SYSCNT_NAME_SIZE];
    struct pmap_update_queue *queue;

    queue = &syncer->queue;
    spinlock_init(&queue->lock);
    list_init(&queue->requests);
    snprintf(name, sizeof(name), "pmap_updates/%u", cpu);
    syscnt_register(&syncer->sc_updates, name);
    snprintf(name, sizeof(name), "pmap_update_enters/%u", cpu);
    syscnt_register(&syncer->sc_update_enters, name);
    snprintf(name, sizeof(name), "pmap_update_removes/%u", cpu);
    syscnt_register(&syncer->sc_update_removes, name);
    snprintf(name, sizeof(name), "pmap_update_protects/%u", cpu);
    syscnt_register(&syncer->sc_update_protects, name);
}

static int __init
pmap_bootstrap(void)
{
    struct pmap_cpu_table *cpu_table;
    unsigned int i;

    for (i = 0; i < ARRAY_SIZE(pmap_get_kernel_pmap()->cpu_tables); i++) {
        cpu_table = &pmap_kernel_cpu_tables[i];
        pmap_get_kernel_pmap()->cpu_tables[i] = cpu_table;
    }

    cpu_local_assign(pmap_current_ptr, pmap_get_kernel_pmap());

    pmap_prot_table[VM_PROT_NONE] = 0;
    pmap_prot_table[VM_PROT_READ] = 0;
    pmap_prot_table[VM_PROT_WRITE] = PMAP_PTE_RW;
    pmap_prot_table[VM_PROT_WRITE | VM_PROT_READ] = PMAP_PTE_RW;
    pmap_prot_table[VM_PROT_EXECUTE] = 0;
    pmap_prot_table[VM_PROT_EXECUTE | VM_PROT_READ] = 0;
    pmap_prot_table[VM_PROT_EXECUTE | VM_PROT_WRITE] = PMAP_PTE_RW;
    pmap_prot_table[VM_PROT_ALL] = PMAP_PTE_RW;

    pmap_update_request_array_init(cpu_local_ptr(pmap_update_request_array));

    pmap_syncer_init(cpu_local_ptr(pmap_syncer), 0);

    pmap_update_oplist_ctor(&pmap_booter_oplist);
    tcb_set_pmap_update_oplist(tcb_current(), &pmap_booter_oplist);

    cpumap_zero(&pmap_booter_cpumap);
    cpumap_set(&pmap_booter_cpumap, 0);

    if (cpu_has_global_pages()) {
        pmap_setup_global_pages();
    }

    return 0;
}

INIT_OP_DEFINE(pmap_bootstrap,
               INIT_OP_DEP(cpu_setup, true),
               INIT_OP_DEP(mutex_setup, true),
               INIT_OP_DEP(spinlock_setup, true),
               INIT_OP_DEP(syscnt_setup, true),
               INIT_OP_DEP(thread_bootstrap, true));

static void __init
pmap_setup_set_ptp_type(phys_addr_t ptp_pa, unsigned int index,
                        unsigned int level)
{
    struct vm_page *page;

    (void)index;

    if (level == 0) {
        return;
    }

    page = vm_page_lookup(ptp_pa);
    assert(page != NULL);

    if (vm_page_type(page) != VM_PAGE_PMAP) {
        assert(vm_page_type(page) == VM_PAGE_RESERVED);
        vm_page_set_type(page, 0, VM_PAGE_PMAP);
    }
}

static void __init
pmap_setup_fix_ptps(void)
{
    pmap_walk_vas(PMAP_START_ADDRESS, PMAP_END_KERNEL_ADDRESS,
                  pmap_setup_set_ptp_type);
}

static int __init
pmap_setup(void)
{
    pmap_setup_fix_ptps();
    kmem_cache_init(&pmap_cache, "pmap", sizeof(struct pmap), 0, NULL, 0);
    kmem_cache_init(&pmap_update_oplist_cache, "pmap_update_oplist",
                    sizeof(struct pmap_update_oplist), CPU_L1_SIZE,
                    pmap_update_oplist_ctor, 0);
    return 0;
}

INIT_OP_DEFINE(pmap_setup,
               INIT_OP_DEP(kmem_setup, true),
               INIT_OP_DEP(log_setup, true),
               INIT_OP_DEP(pmap_bootstrap, true),
               INIT_OP_DEP(vm_page_setup, true));

void __init
pmap_ap_setup(void)
{
    cpu_local_assign(pmap_current_ptr, pmap_get_kernel_pmap());

    if (cpu_has_global_pages()) {
        cpu_enable_global_pages();
    } else {
        cpu_tlb_flush();
    }
}

static void __init
pmap_copy_cpu_table_page(const pmap_pte_t *sptp, unsigned int level,
                         struct vm_page *page)
{
    const struct pmap_pt_level *pt_level;
    pmap_pte_t *dptp;

    pt_level = &pmap_pt_levels[level];
    dptp = vm_page_direct_ptr(page);
    memcpy(dptp, sptp, pt_level->ptes_per_pt * sizeof(pmap_pte_t));
}

static void __init
pmap_copy_cpu_table_recursive(const pmap_pte_t *sptp, unsigned int level,
                              pmap_pte_t *dptp, uintptr_t start_va)
{
    const struct pmap_pt_level *pt_level;
    struct vm_page *page;
    phys_addr_t pa;
    unsigned int i;
    uintptr_t va;

    assert(level != 0);

    pt_level = &pmap_pt_levels[level];
    memset(dptp, 0, pt_level->ptes_per_pt * sizeof(pmap_pte_t));

    for (i = 0, va = start_va;
         i < pt_level->ptes_per_pt;
         i++, va = P2END(va, 1UL << pt_level->skip)) {
#ifdef __LP64__
        /* Handle long mode canonical form */
        if (va == PMAP_END_ADDRESS) {
            va = PMAP_START_KERNEL_ADDRESS;
        }
#endif /* __LP64__ */

        if (!pmap_pte_valid(sptp[i])) {
            continue;
        } else if (pmap_pte_large(sptp[i])) {
            dptp[i] = sptp[i];
            continue;
        }

        page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP);

        if (page == NULL) {
            panic("pmap: unable to allocate page table page copy");
        }

        pa = vm_page_to_pa(page);
        dptp[i] = (sptp[i] & ~PMAP_PA_MASK) | (pa & PMAP_PA_MASK);

        if (((level - 1) == 0) || pmap_pte_large(sptp[i])) {
            pmap_copy_cpu_table_page(pmap_pte_next(sptp[i]), level - 1, page);
        } else {
            pmap_copy_cpu_table_recursive(pmap_pte_next(sptp[i]), level - 1,
                                          vm_page_direct_ptr(page), va);
        }
    }
}

static void __init
pmap_copy_cpu_table(unsigned int cpu)
{
    struct pmap_cpu_table *cpu_table;
    struct pmap *kernel_pmap;
    unsigned int level;
    const pmap_pte_t *sptp;
    pmap_pte_t *dptp;

    assert(cpu != 0);

    kernel_pmap = pmap_get_kernel_pmap();
    assert(cpu < ARRAY_SIZE(kernel_pmap->cpu_tables));
    cpu_table = kernel_pmap->cpu_tables[cpu];
    level = PMAP_NR_LEVELS - 1;
    sptp = pmap_ptp_from_pa(kernel_pmap->cpu_tables[cpu_id()]->root_ptp_pa);

#ifdef CONFIG_X86_PAE
    cpu_table->root_ptp_pa = BOOT_VTOP((uintptr_t)pmap_cpu_kpdpts[cpu]);
    dptp = pmap_ptp_from_pa(cpu_table->root_ptp_pa);
#else /* CONFIG_X86_PAE */
    struct vm_page *page;

    page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP);

    if (page == NULL) {
        panic("pmap: unable to allocate page table root page copy");
    }

    cpu_table->root_ptp_pa = vm_page_to_pa(page);
    dptp = vm_page_direct_ptr(page);
#endif /* CONFIG_X86_PAE */

    pmap_copy_cpu_table_recursive(sptp, level, dptp, PMAP_START_ADDRESS);
}

void __init
pmap_mp_setup(void)
{
    char name[THREAD_NAME_SIZE];
    struct pmap_update_oplist *oplist;
    struct thread_attr attr;
    struct pmap_syncer *syncer;
    struct cpumap *cpumap;
    struct tcb *tcb;
    unsigned int cpu;
    int error;

    error = cpumap_create(&cpumap);

    if (error) {
        panic("pmap: unable to create syncer cpumap");
    }

    for (cpu = 1; cpu < cpu_count(); cpu++) {
        pmap_update_request_array_init(percpu_ptr(pmap_update_request_array,
                                                  cpu));
        pmap_syncer_init(percpu_ptr(pmap_syncer, cpu), cpu);
    }

    for (cpu = 0; cpu < cpu_count(); cpu++) {
        syncer = percpu_ptr(pmap_syncer, cpu);
        snprintf(name, sizeof(name), THREAD_KERNEL_PREFIX "pmap_sync/%u", cpu);
        cpumap_zero(cpumap);
        cpumap_set(cpumap, cpu);
        thread_attr_init(&attr, name);
        thread_attr_set_cpumap(&attr, cpumap);
        thread_attr_set_priority(&attr, THREAD_SCHED_FS_PRIO_MAX);
        error = thread_create(&syncer->thread, &attr, pmap_sync, syncer);

        if (error) {
            panic("pmap: unable to create syncer thread");
        }

        tcb = thread_get_tcb(syncer->thread);
        oplist = tcb_get_pmap_update_oplist(tcb);
        tcb_set_pmap_update_oplist(tcb, NULL);
        kmem_cache_free(&pmap_update_oplist_cache, oplist);
    }

    cpumap_destroy(cpumap);

    for (cpu = 1; cpu < cpu_count(); cpu++) {
        pmap_copy_cpu_table(cpu);
    }

    pmap_do_remote_updates = 1;
}

int
pmap_thread_build(struct thread *thread)
{
    struct pmap_update_oplist *oplist;
    int error;

    error = pmap_update_oplist_create(&oplist);

    if (error) {
        return error;
    }

    tcb_set_pmap_update_oplist(thread_get_tcb(thread), oplist);
    return 0;
}

void
pmap_thread_cleanup(struct thread *thread)
{
    struct pmap_update_oplist *oplist;

    oplist = tcb_get_pmap_update_oplist(thread_get_tcb(thread));

    if (oplist) {
        pmap_update_oplist_destroy(oplist);
    }
}

int
pmap_kextract(uintptr_t va, phys_addr_t *pap)
{
    const struct pmap_pt_level *pt_level;
    struct pmap *kernel_pmap;
    pmap_pte_t *ptp, *pte;
    unsigned int level;

    level = PMAP_NR_LEVELS - 1;
    kernel_pmap = pmap_get_kernel_pmap();
    ptp = pmap_ptp_from_pa(kernel_pmap->cpu_tables[cpu_id()]->root_ptp_pa);

    for (;;) {
        pt_level = &pmap_pt_levels[level];
        pte = &ptp[pmap_pte_index(va, pt_level)];

        if (!pmap_pte_valid(*pte)) {
            return EFAULT;
        }

        if ((level == 0) || pmap_pte_large(*pte)) {
            break;
        }

        level--;
        ptp = pmap_pte_next(*pte);
    }

    *pap = (*pte & PMAP_PA_MASK);
    return 0;
}

int
pmap_create(struct pmap **pmapp)
{
    struct pmap *pmap;
    unsigned int i;

    pmap = kmem_cache_alloc(&pmap_cache);

    if (pmap == NULL) {
        return ENOMEM;
    }

    for (i = 0; i < ARRAY_SIZE(pmap->cpu_tables); i++) {
        pmap->cpu_tables[i] = NULL;
    }

    *pmapp = pmap;
    return 0;
}

static int
pmap_enter_local(struct pmap *pmap, uintptr_t va, phys_addr_t pa,
                 int prot, int flags)
{
    const struct pmap_pt_level *pt_level;
    struct vm_page *page;
    phys_addr_t ptp_pa;
    pmap_pte_t *ptp, *pte, pte_bits;
    unsigned int level;

    /* TODO Page attributes */
    (void)flags;

    pte_bits = PMAP_PTE_RW;

    if (pmap != pmap_get_kernel_pmap()) {
        pte_bits |= PMAP_PTE_US;
    }

    level = PMAP_NR_LEVELS - 1;
    ptp = pmap_ptp_from_pa(pmap->cpu_tables[cpu_id()]->root_ptp_pa);

    for (;;) {
        pt_level = &pmap_pt_levels[level];
        pte = &ptp[pmap_pte_index(va, pt_level)];

        if (level == 0) {
            break;
        }

        if (pmap_pte_valid(*pte)) {
            ptp = pmap_pte_next(*pte);
        } else {
            page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP);

            if (page == NULL) {
                log_warning("pmap: page table page allocation failure");
                return ENOMEM;
            }

            ptp_pa = vm_page_to_pa(page);
            ptp = pmap_ptp_from_pa(ptp_pa);
            pmap_ptp_clear(ptp);
            pmap_pte_set(pte, ptp_pa, pte_bits, pt_level);
        }

        level--;
    }

    assert(!pmap_pte_valid(*pte));
    pte_bits = ((pmap == pmap_get_kernel_pmap()) ? PMAP_PTE_G : PMAP_PTE_US)
               | pmap_prot_table[prot & VM_PROT_ALL];
    pmap_pte_set(pte, pa, pte_bits, pt_level);
    return 0;
}

int
pmap_enter(struct pmap *pmap, uintptr_t va, phys_addr_t pa,
           int prot, int flags)
{
    struct pmap_update_oplist *oplist;
    struct pmap_update_op *op;
    int error;

    va = vm_page_trunc(va);
    pa = vm_page_trunc(pa);
    assert(pmap_range_valid(pmap, va, va + PAGE_SIZE));

    oplist = pmap_update_oplist_get();
    error = pmap_update_oplist_prepare(oplist, pmap);

    if (error) {
        return error;
    }

    op = pmap_update_oplist_prepare_op(oplist);

    if (flags & PMAP_PEF_GLOBAL) {
        cpumap_copy(&op->cpumap, cpumap_all());
    } else {
        cpumap_zero(&op->cpumap);
        cpumap_set(&op->cpumap, cpu_id());
    }

    op->operation = PMAP_UPDATE_OP_ENTER;
    op->enter_args.va = va;
    op->enter_args.pa = pa;
    op->enter_args.prot = prot;
    op->enter_args.flags = flags & ~PMAP_PEF_GLOBAL;
    pmap_update_oplist_finish_op(oplist);
    return 0;
}

static void
pmap_remove_local_single(struct pmap *pmap, uintptr_t va)
{
    const struct pmap_pt_level *pt_level;
    pmap_pte_t *ptp, *pte;
    unsigned int level;

    level = PMAP_NR_LEVELS - 1;
    ptp = pmap_ptp_from_pa(pmap->cpu_tables[cpu_id()]->root_ptp_pa);

    for (;;) {
        pt_level = &pmap_pt_levels[level];
        pte = &ptp[pmap_pte_index(va, pt_level)];

        if (!pmap_pte_valid(*pte)) {
            return;
        }

        if (level == 0) {
            break;
        }

        level--;
        ptp = pmap_pte_next(*pte);
    }

    pmap_pte_clear(pte);
}

static void
pmap_remove_local(struct pmap *pmap, uintptr_t start, uintptr_t end)
{
    while (start < end) {
        pmap_remove_local_single(pmap, start);
        start += PAGE_SIZE;
    }
}

int
pmap_remove(struct pmap *pmap, uintptr_t va, const struct cpumap *cpumap)
{
    struct pmap_update_oplist *oplist;
    struct pmap_update_op *op;
    int error;

    va = vm_page_trunc(va);
    assert(pmap_range_valid(pmap, va, va + PAGE_SIZE));

    oplist = pmap_update_oplist_get();
    error = pmap_update_oplist_prepare(oplist, pmap);

    if (error) {
        return error;
    }

    /* Attempt naive merge with previous operation */
    op = pmap_update_oplist_prev_op(oplist);

    if ((op != NULL)
        && (op->operation == PMAP_UPDATE_OP_REMOVE)
        && (op->remove_args.end == va)
        && (cpumap_cmp(&op->cpumap, cpumap) == 0)) {
        op->remove_args.end = va + PAGE_SIZE;
        return 0;
    }

    op = pmap_update_oplist_prepare_op(oplist);
    cpumap_copy(&op->cpumap, cpumap);
    op->operation = PMAP_UPDATE_OP_REMOVE;
    op->remove_args.start = va;
    op->remove_args.end = va + PAGE_SIZE;
    pmap_update_oplist_finish_op(oplist);
    return 0;
}

static void
pmap_protect_local(struct pmap *pmap, uintptr_t start,
                   uintptr_t end, int prot)
{
    (void)pmap;
    (void)start;
    (void)end;
    (void)prot;

    /* TODO Implement */
    panic("pmap: pmap_protect not implemented");
}

int
pmap_protect(struct pmap *pmap, uintptr_t va, int prot,
             const struct cpumap *cpumap)
{
    struct pmap_update_oplist *oplist;
    struct pmap_update_op *op;
    int error;

    va = vm_page_trunc(va);
    assert(pmap_range_valid(pmap, va, va + PAGE_SIZE));

    oplist = pmap_update_oplist_get();
    error = pmap_update_oplist_prepare(oplist, pmap);

    if (error) {
        return error;
    }

    /* Attempt naive merge with previous operation */
    op = pmap_update_oplist_prev_op(oplist);

    if ((op != NULL)
        && (op->operation == PMAP_UPDATE_OP_PROTECT)
        && (op->protect_args.end == va)
        && (op->protect_args.prot == prot)
        && (cpumap_cmp(&op->cpumap, cpumap) == 0)) {
        op->protect_args.end = va + PAGE_SIZE;
        return 0;
    }

    op = pmap_update_oplist_prepare_op(oplist);
    cpumap_copy(&op->cpumap, cpumap);
    op->operation = PMAP_UPDATE_OP_PROTECT;
    op->protect_args.start = va;
    op->protect_args.end = va + PAGE_SIZE;
    op->protect_args.prot = prot;
    pmap_update_oplist_finish_op(oplist);
    return 0;
}

static void
pmap_flush_tlb(struct pmap *pmap, uintptr_t start, uintptr_t end)
{
    if ((pmap != pmap_current()) && (pmap != pmap_get_kernel_pmap())) {
        return;
    }

    while (start < end) {
        cpu_tlb_flush_va(start);
        start += PAGE_SIZE;
    }
}

static void
pmap_flush_tlb_all(struct pmap *pmap)
{
    if ((pmap != pmap_current()) && (pmap != pmap_get_kernel_pmap())) {
        return;
    }

    if (pmap == pmap_get_kernel_pmap()) {
        cpu_tlb_flush_all();
    } else {
        cpu_tlb_flush();
    }
}

static int
pmap_update_enter(struct pmap *pmap, int flush,
                  const struct pmap_update_enter_args *args)
{
    int error;

    error = pmap_enter_local(pmap, args->va, args->pa, args->prot, args->flags);

    if (error) {
        return error;
    }

    if (flush) {
        pmap_flush_tlb(pmap, args->va, args->va + PAGE_SIZE);
    }

    return 0;
}

static void
pmap_update_remove(struct pmap *pmap, int flush,
                   const struct pmap_update_remove_args *args)
{
    pmap_remove_local(pmap, args->start, args->end);

    if (flush) {
        pmap_flush_tlb(pmap, args->start, args->end);
    }
}

static void
pmap_update_protect(struct pmap *pmap, int flush,
                    const struct pmap_update_protect_args *args)
{
    pmap_protect_local(pmap, args->start, args->end, args->prot);

    if (flush) {
        pmap_flush_tlb(pmap, args->start, args->end);
    }
}

static int
pmap_update_local(const struct pmap_update_oplist *oplist,
                  unsigned int nr_mappings)
{
    const struct pmap_update_op *op;
    struct pmap_syncer *syncer;
    int error, global_tlb_flush;
    unsigned int i;

    syncer = cpu_local_ptr(pmap_syncer);
    syscnt_inc(&syncer->sc_updates);
    global_tlb_flush = (nr_mappings > PMAP_UPDATE_MAX_MAPPINGS);
    error = 0;

    for (i = 0; i < oplist->nr_ops; i++) {
        op = &oplist->ops[i];

        if (!cpumap_test(&op->cpumap, cpu_id())) {
            continue;
        }

        switch (op->operation) {
        case PMAP_UPDATE_OP_ENTER:
            syscnt_inc(&syncer->sc_update_enters);
            error = pmap_update_enter(oplist->pmap, !global_tlb_flush,
                                      &op->enter_args);
            break;
        case PMAP_UPDATE_OP_REMOVE:
            syscnt_inc(&syncer->sc_update_removes);
            pmap_update_remove(oplist->pmap, !global_tlb_flush,
                               &op->remove_args);
            break;
        case PMAP_UPDATE_OP_PROTECT:
            syscnt_inc(&syncer->sc_update_protects);
            pmap_update_protect(oplist->pmap, !global_tlb_flush,
                                &op->protect_args);
            break;
        default:
            assert(!"invalid update operation");
        }

        if (error) {
            return error;
        }
    }

    if (global_tlb_flush) {
        pmap_flush_tlb_all(oplist->pmap);
    }

    return 0;
}

int
pmap_update(struct pmap *pmap)
{
    struct pmap_update_oplist *oplist;
    struct pmap_update_request_array *array;
    struct pmap_update_request *request;
    struct pmap_update_queue *queue;
    struct pmap_syncer *syncer;
    unsigned int nr_mappings;
    int error, cpu;

    oplist = pmap_update_oplist_get();

    if (pmap != oplist->pmap) {
        /* Make sure pmap_update() is called before manipulating another pmap */
        assert(oplist->pmap == NULL);
        return 0;
    }

    assert(oplist->nr_ops != 0);

    if (!pmap_do_remote_updates) {
        nr_mappings = pmap_update_oplist_count_mappings(oplist, cpu_id());
        error = pmap_update_local(oplist, nr_mappings);
        goto out;
    }

    error = 0;

    array = pmap_update_request_array_acquire();

    cpumap_for_each(&oplist->cpumap, cpu) {
        syncer = percpu_ptr(pmap_syncer, cpu);
        queue = &syncer->queue;
        request = &array->requests[cpu];
        request->sender = thread_self();
        request->oplist = oplist;
        request->nr_mappings = pmap_update_oplist_count_mappings(oplist, cpu);
        request->done = 0;
        request->error = 0;

        spinlock_lock(&queue->lock);
        list_insert_tail(&queue->requests, &request->node);
        thread_wakeup(syncer->thread);
        spinlock_unlock(&queue->lock);
    }

    /* TODO Improve scalability */
    cpumap_for_each(&oplist->cpumap, cpu) {
        request = &array->requests[cpu];

        spinlock_lock(&request->lock);

        while (!request->done) {
            thread_sleep(&request->lock, request, "pmaprq");
        }

        if (!error && request->error) {
            error = request->error;
        }

        spinlock_unlock(&request->lock);
    }

    pmap_update_request_array_release(array);

out:
    cpumap_zero(&oplist->cpumap);
    oplist->pmap = NULL;
    oplist->nr_ops = 0;
    return error;
}

static void
pmap_sync(void *arg)
{
    struct pmap_update_queue *queue;
    struct pmap_update_request *request;
    struct pmap_syncer *self;
    int error;

    self = arg;
    queue = &self->queue;

    for (;;) {
        spinlock_lock(&queue->lock);

        while (list_empty(&queue->requests)) {
            thread_sleep(&queue->lock, queue, "pmapq");
        }

        request = list_first_entry(&queue->requests,
                                   struct pmap_update_request, node);
        list_remove(&request->node);

        spinlock_unlock(&queue->lock);

        error = pmap_update_local(request->oplist, request->nr_mappings);

        spinlock_lock(&request->lock);
        request->done = 1;
        request->error = error;
        thread_wakeup(request->sender);
        spinlock_unlock(&request->lock);
    }
}

void
pmap_load(struct pmap *pmap)
{
    struct pmap_cpu_table *cpu_table;

    assert(!cpu_intr_enabled());
    assert(!thread_preempt_enabled());

    if (pmap_current() == pmap) {
        return;
    }

    /* TODO Lazy TLB invalidation */

    cpu_local_assign(pmap_current_ptr, pmap);

    /* TODO Implement per-CPU page tables for non-kernel pmaps */
    cpu_table = pmap->cpu_tables[cpu_id()];

    cpu_set_cr3(cpu_table->root_ptp_pa);
}