code:
/*
 * Copyright (c) 2010-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <stdalign.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

#include <kern/atomic.h>
#include <kern/init.h>
#include <kern/kmem.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/panic.h>
#include <kern/percpu.h>
#include <kern/spinlock.h>
#include <kern/shutdown.h>
#include <kern/thread.h>
#include <kern/xcall.h>
#include <machine/acpi.h>
#include <machine/biosmem.h>
#include <machine/boot.h>
#include <machine/cpu.h>
#include <machine/io.h>
#include <machine/lapic.h>
#include <machine/page.h>
#include <machine/pit.h>
#include <machine/pmap.h>
#include <machine/ssp.h>
#include <machine/strace.h>
#include <vm/vm_page.h>

/*
 * Delay used for frequency measurement, in microseconds.
 */
#define CPU_FREQ_CAL_DELAY  1000000

#define CPU_CPUID_TYPE_MASK         0x00003000
#define CPU_CPUID_TYPE_SHIFT        12
#define CPU_CPUID_FAMILY_MASK       0x00000f00
#define CPU_CPUID_FAMILY_SHIFT      8
#define CPU_CPUID_EXTFAMILY_MASK    0x0ff00000
#define CPU_CPUID_EXTFAMILY_SHIFT   20
#define CPU_CPUID_MODEL_MASK        0x000000f0
#define CPU_CPUID_MODEL_SHIFT       4
#define CPU_CPUID_EXTMODEL_MASK     0x000f0000
#define CPU_CPUID_EXTMODEL_SHIFT    16
#define CPU_CPUID_STEPPING_MASK     0x0000000f
#define CPU_CPUID_STEPPING_SHIFT    0
#define CPU_CPUID_BRAND_MASK        0x000000ff
#define CPU_CPUID_BRAND_SHIFT       0
#define CPU_CPUID_CLFLUSH_MASK      0x0000ff00
#define CPU_CPUID_CLFLUSH_SHIFT     8
#define CPU_CPUID_APIC_ID_MASK      0xff000000
#define CPU_CPUID_APIC_ID_SHIFT     24

#define CPU_INVALID_APIC_ID ((unsigned int)-1)

struct cpu_vendor {
    unsigned int id;
    const char *str;
};

/*
 * IST indexes (0 means no stack switch).
 */
#define CPU_TSS_IST_INTR    1
#define CPU_TSS_IST_DF      2

/*
 * MP related CMOS ports, registers and values.
 */
#define CPU_MP_CMOS_PORT_REG        0x70
#define CPU_MP_CMOS_PORT_DATA       0x71
#define CPU_MP_CMOS_REG_RESET       0x0f
#define CPU_MP_CMOS_DATA_RESET_WARM 0x0a
#define CPU_MP_CMOS_RESET_VECTOR    0x467

/*
 * Priority of the shutdown operations.
 *
 * Last resort, lower than everything else.
 */
#define CPU_SHUTDOWN_PRIORITY 0

struct cpu_gate_desc {
    uint32_t word1;
    uint32_t word2;
#ifdef __LP64__
    uint32_t word3;
    uint32_t word4;
#endif /* __LP64__ */
};

struct cpu_idt {
    alignas(CPU_L1_SIZE) struct cpu_gate_desc descs[CPU_NR_EXC_VECTORS];
};

struct cpu_pseudo_desc {
    uint16_t limit;
    uintptr_t address;
} __packed;

#ifdef __LP64__
#define cpu_exc_frame_attrs
#else /* __LP64__ */
#define cpu_exc_frame_attrs __packed
#endif /* __LP64__ */

struct cpu_exc_frame {
    unsigned long words[CPU_EXC_FRAME_SIZE];
} cpu_exc_frame_attrs;

/*
 * Type for low level exception handlers.
 *
 * Low level exception handlers are directly installed in the IDT and are
 * first run by the processor when an exception occurs. They route execution
 * through either the main exception or interrupt handler.
 */
typedef void (*cpu_ll_exc_fn_t)(void);

typedef void (*cpu_exc_handler_fn_t)(const struct cpu_exc_frame *frame);

struct cpu_exc_handler {
    cpu_exc_handler_fn_t fn;
};

struct cpu_intr_handler {
    cpu_intr_handler_fn_t fn;
};

/*
 * Set the given GDT for the current processor.
 *
 * On i386, the ds, es and ss segment registers are reloaded.
 *
 * The fs and gs segment registers, which point to the percpu and the TLS
 * areas respectively, must be set separately.
 */
void cpu_load_gdt(struct cpu_pseudo_desc *gdtr);

/*
 * Return a pointer to the processor-local interrupt stack.
 *
 * This function is called by the low level exception handling code.
 */
void * cpu_get_intr_stack_ptr(void);

/*
 * Common entry points for exceptions and interrupts.
 */
void cpu_exc_main(const struct cpu_exc_frame *frame);
void cpu_intr_main(const struct cpu_exc_frame *frame);

void *cpu_local_area __percpu;

/*
 * CPU descriptor, one per CPU.
 */
struct cpu cpu_desc __percpu;

/*
 * Number of active processors.
 */
unsigned int cpu_nr_active __read_mostly = 1;

/*
 * Processor frequency, assumed fixed and equal on all processors.
 */
static uint64_t cpu_freq __read_mostly;

static const struct cpu_tls_seg cpu_tls_seg = {
    .ssp_guard_word = SSP_GUARD_WORD,
};

static struct cpu_idt cpu_idt;

/*
 * This table only exists during initialization, and is a way to
 * communicate the list of low level handlers from assembly to C.
 */
extern cpu_ll_exc_fn_t cpu_ll_exc_handler_addrs[CPU_NR_EXC_VECTORS];

static struct cpu_exc_handler cpu_exc_handlers[CPU_NR_EXC_VECTORS]
    __read_mostly;

static struct cpu_intr_handler cpu_intr_handlers[CPU_NR_EXC_VECTORS]
    __read_mostly;

static const struct cpu_vendor cpu_vendors[] = {
    { CPU_VENDOR_INTEL, "GenuineIntel" },
    { CPU_VENDOR_AMD,   "AuthenticAMD" },
};

static const char *cpu_feature_names[] = {
    [CPU_FEATURE_FPU]   = "fpu",
    [CPU_FEATURE_PSE]   = "pse",
    [CPU_FEATURE_PAE]   = "pae",
    [CPU_FEATURE_MSR]   = "msr",
    [CPU_FEATURE_CX8]   = "cx8",
    [CPU_FEATURE_APIC]  = "apic",
    [CPU_FEATURE_PGE]   = "pge",
    [CPU_FEATURE_1GP]   = "1gp",
    [CPU_FEATURE_LM]    = "lm",
};

static void __init
cpu_exc_handler_init(struct cpu_exc_handler *handler, cpu_exc_handler_fn_t fn)
{
    handler->fn = fn;
}

static void
cpu_exc_handler_run(const struct cpu_exc_handler *handler,
                    const struct cpu_exc_frame *frame)
{
    handler->fn(frame);
}

static void __init
cpu_intr_handler_init(struct cpu_intr_handler *handler,
                      cpu_intr_handler_fn_t fn)
{
    handler->fn = fn;
}

static void
cpu_intr_handler_run(const struct cpu_intr_handler *handler,
                     unsigned int vector)
{
    handler->fn(vector);
}

static cpu_ll_exc_fn_t __init
cpu_get_ll_exc_handler(unsigned int vector)
{
    assert(vector < ARRAY_SIZE(cpu_ll_exc_handler_addrs));
    return cpu_ll_exc_handler_addrs[vector];
}

static struct cpu_exc_handler *
cpu_get_exc_handler(unsigned int vector)
{
    assert(vector < ARRAY_SIZE(cpu_exc_handlers));
    return &cpu_exc_handlers[vector];
}

static void __init
cpu_register_exc(unsigned int vector, cpu_exc_handler_fn_t fn)
{
    cpu_exc_handler_init(cpu_get_exc_handler(vector), fn);
}

static struct cpu_intr_handler *
cpu_get_intr_handler(unsigned int vector)
{
    assert(vector < ARRAY_SIZE(cpu_intr_handlers));
    return &cpu_intr_handlers[vector];
}

void __init
cpu_register_intr(unsigned int vector, cpu_intr_handler_fn_t fn)
{
    cpu_intr_handler_init(cpu_get_intr_handler(vector), fn);
}

static void __init
cpu_gate_desc_init_intr(struct cpu_gate_desc *desc, cpu_ll_exc_fn_t fn,
                        unsigned int ist_index)
{
    uintptr_t addr;

    addr = (uintptr_t)fn;
    desc->word1 = (CPU_GDT_SEL_CODE << 16)
                  | (addr & CPU_DESC_GATE_OFFSET_LOW_MASK);
    desc->word2 = (addr & CPU_DESC_GATE_OFFSET_HIGH_MASK)
                  | CPU_DESC_PRESENT | CPU_DESC_TYPE_GATE_INTR;

#ifdef __LP64__
    desc->word2 |= ist_index & CPU_DESC_SEG_IST_MASK;
    desc->word3 = addr >> 32;
    desc->word4 = 0;
#else /* __LP64__ */
    (void)ist_index;
#endif /* __LP64__ */
}

#ifndef __LP64__
static void __init
cpu_gate_desc_init_task(struct cpu_gate_desc *desc, unsigned int tss_seg_sel)
{
    desc->word2 = CPU_DESC_PRESENT | CPU_DESC_TYPE_GATE_TASK;
    desc->word1 = tss_seg_sel << 16;
}
#endif /* __LP64__ */

static struct cpu_gate_desc * __init
cpu_idt_get_desc(struct cpu_idt *idt, unsigned int vector)
{
    assert(vector < ARRAY_SIZE(idt->descs));
    return &idt->descs[vector];
}

static void __init
cpu_idt_set_intr_gate(struct cpu_idt *idt, unsigned int vector,
                      cpu_ll_exc_fn_t fn)
{
    struct cpu_gate_desc *desc;

    desc = cpu_idt_get_desc(idt, vector);
    cpu_gate_desc_init_intr(desc, fn, CPU_TSS_IST_INTR);
}

static void __init
cpu_idt_setup_double_fault(struct cpu_idt *idt)
{
    struct cpu_gate_desc *desc;

    desc = cpu_idt_get_desc(idt, CPU_EXC_DF);

#ifdef __LP64__
    cpu_ll_exc_fn_t fn;

    fn = cpu_get_ll_exc_handler(CPU_EXC_DF);
    cpu_gate_desc_init_intr(desc, fn, CPU_TSS_IST_DF);
#else /* __LP64__ */
    cpu_gate_desc_init_task(desc, CPU_GDT_SEL_DF_TSS);
#endif /* __LP64__ */
}

static void
cpu_idt_load(const struct cpu_idt *idt)
{
    struct cpu_pseudo_desc idtr;

    idtr.address = (uintptr_t)idt->descs;
    idtr.limit = sizeof(idt->descs) - 1;
    asm volatile("lidt %0" : : "m" (idtr));
}

uint64_t
cpu_get_freq(void)
{
    return cpu_freq;
}

static uint64_t
cpu_get_tsc(void)
{
    uint32_t high, low;

    asm volatile("rdtsc" : "=a" (low), "=d" (high));
    return ((uint64_t)high << 32) | low;
}

void
cpu_delay(unsigned long usecs)
{
    int64_t total, prev, count, diff;

    assert(usecs != 0);

    total = DIV_CEIL((int64_t)usecs * cpu_freq, 1000000);
    prev = cpu_get_tsc();

    do {
        count = cpu_get_tsc();
        diff = count - prev;
        prev = count;
        total -= diff;
        cpu_pause();
    } while (total > 0);
}

static void
cpu_show_thread(void)
{
    struct thread *thread;

    thread = thread_self();

    printf("cpu: interrupted thread: %p (%s)\n", thread, thread_name(thread));
}

#ifdef __LP64__

static void
cpu_show_frame(const struct cpu_exc_frame *frame)
{
    printf("cpu: rax: %016lx rbx: %016lx rcx: %016lx\n"
           "cpu: rdx: %016lx rbp: %016lx rsi: %016lx\n"
           "cpu: rdi: %016lx  r8: %016lx  r9: %016lx\n"
           "cpu: r10: %016lx r11: %016lx r12: %016lx\n"
           "cpu: r13: %016lx r14: %016lx r15: %016lx\n"
           "cpu: vector: %lu error: %08lx\n"
           "cpu: rip: %016lx cs: %lu rflags: %016lx\n"
           "cpu: rsp: %016lx ss: %lu\n",
           frame->words[CPU_EXC_FRAME_RAX],
           frame->words[CPU_EXC_FRAME_RBX],
           frame->words[CPU_EXC_FRAME_RCX],
           frame->words[CPU_EXC_FRAME_RDX],
           frame->words[CPU_EXC_FRAME_RBP],
           frame->words[CPU_EXC_FRAME_RSI],
           frame->words[CPU_EXC_FRAME_RDI],
           frame->words[CPU_EXC_FRAME_R8],
           frame->words[CPU_EXC_FRAME_R9],
           frame->words[CPU_EXC_FRAME_R10],
           frame->words[CPU_EXC_FRAME_R11],
           frame->words[CPU_EXC_FRAME_R12],
           frame->words[CPU_EXC_FRAME_R13],
           frame->words[CPU_EXC_FRAME_R14],
           frame->words[CPU_EXC_FRAME_R15],
           frame->words[CPU_EXC_FRAME_VECTOR],
           frame->words[CPU_EXC_FRAME_ERROR],
           frame->words[CPU_EXC_FRAME_RIP],
           frame->words[CPU_EXC_FRAME_CS],
           frame->words[CPU_EXC_FRAME_RFLAGS],
           frame->words[CPU_EXC_FRAME_RSP],
           frame->words[CPU_EXC_FRAME_SS]);

    /* XXX Until the page fault handler is written */
    if (frame->words[CPU_EXC_FRAME_VECTOR] == 14) {
        printf("cpu: cr2: %016lx\n", cpu_get_cr2());
    }
}

#else /* __LP64__ */

static void
cpu_show_frame(const struct cpu_exc_frame *frame)
{
    unsigned long esp, ss;

    if ((frame->words[CPU_EXC_FRAME_CS] & CPU_PL_USER)
        || (frame->words[CPU_EXC_FRAME_VECTOR] == CPU_EXC_DF)) {
        esp = frame->words[CPU_EXC_FRAME_ESP];
        ss = frame->words[CPU_EXC_FRAME_SS];
    } else {
        esp = 0;
        ss = 0;
    }

    printf("cpu: eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n"
           "cpu: ebp: %08lx esi: %08lx edi: %08lx\n"
           "cpu: ds: %lu es: %lu fs: %lu gs: %lu\n"
           "cpu: vector: %lu error: %08lx\n"
           "cpu: eip: %08lx cs: %lu eflags: %08lx\n"
           "cpu: esp: %08lx ss: %lu\n",
           frame->words[CPU_EXC_FRAME_EAX],
           frame->words[CPU_EXC_FRAME_EBX],
           frame->words[CPU_EXC_FRAME_ECX],
           frame->words[CPU_EXC_FRAME_EDX],
           frame->words[CPU_EXC_FRAME_EBP],
           frame->words[CPU_EXC_FRAME_ESI],
           frame->words[CPU_EXC_FRAME_EDI],
           frame->words[CPU_EXC_FRAME_DS],
           frame->words[CPU_EXC_FRAME_ES],
           frame->words[CPU_EXC_FRAME_FS],
           frame->words[CPU_EXC_FRAME_GS],
           frame->words[CPU_EXC_FRAME_VECTOR],
           frame->words[CPU_EXC_FRAME_ERROR],
           frame->words[CPU_EXC_FRAME_EIP],
           frame->words[CPU_EXC_FRAME_CS],
           frame->words[CPU_EXC_FRAME_EFLAGS],
           esp,
           ss);

    /* XXX Until the page fault handler is written */
    if (frame->words[CPU_EXC_FRAME_VECTOR] == 14) {
        printf("cpu: cr2: %08lx\n", cpu_get_cr2());
    }
}

#endif /* __LP64__ */

static void
cpu_show_stack(const struct cpu_exc_frame *frame)
{
    strace_show(frame->words[CPU_EXC_FRAME_PC], frame->words[CPU_EXC_FRAME_FP]);
}

static void
cpu_exc_double_fault(const struct cpu_exc_frame *frame)
{
    cpu_halt_broadcast();

#ifndef __LP64__
    struct cpu_exc_frame frame_store;
    struct cpu *cpu;

    /*
     * Double faults are catched through a task gate, which makes the given
     * frame useless. The interrupted state is automatically saved in the
     * main TSS by the processor. Build a proper exception frame from there.
     */
    cpu = cpu_current();
    frame_store.words[CPU_EXC_FRAME_EAX]    = cpu->tss.eax;
    frame_store.words[CPU_EXC_FRAME_EBX]    = cpu->tss.ebx;
    frame_store.words[CPU_EXC_FRAME_ECX]    = cpu->tss.ecx;
    frame_store.words[CPU_EXC_FRAME_EDX]    = cpu->tss.edx;
    frame_store.words[CPU_EXC_FRAME_EBP]    = cpu->tss.ebp;
    frame_store.words[CPU_EXC_FRAME_ESI]    = cpu->tss.esi;
    frame_store.words[CPU_EXC_FRAME_EDI]    = cpu->tss.edi;
    frame_store.words[CPU_EXC_FRAME_DS]     = cpu->tss.ds;
    frame_store.words[CPU_EXC_FRAME_ES]     = cpu->tss.es;
    frame_store.words[CPU_EXC_FRAME_FS]     = cpu->tss.fs;
    frame_store.words[CPU_EXC_FRAME_GS]     = cpu->tss.gs;
    frame_store.words[CPU_EXC_FRAME_VECTOR] = CPU_EXC_DF;
    frame_store.words[CPU_EXC_FRAME_ERROR]  = 0;
    frame_store.words[CPU_EXC_FRAME_EIP]    = cpu->tss.eip;
    frame_store.words[CPU_EXC_FRAME_CS]     = cpu->tss.cs;
    frame_store.words[CPU_EXC_FRAME_EFLAGS] = cpu->tss.eflags;
    frame_store.words[CPU_EXC_FRAME_ESP]    = cpu->tss.esp;
    frame_store.words[CPU_EXC_FRAME_SS]     = cpu->tss.ss;
    frame = &frame_store;
#endif /* __LP64__ */

    printf("cpu: double fault (cpu%u):\n", cpu_id());
    cpu_show_thread();
    cpu_show_frame(frame);
    cpu_show_stack(frame);
    cpu_halt();
}

void
cpu_exc_main(const struct cpu_exc_frame *frame)
{
    const struct cpu_exc_handler *handler;
    unsigned int vector;

    vector = (unsigned int)frame->words[CPU_EXC_FRAME_VECTOR];
    handler = cpu_get_exc_handler(vector);
    cpu_exc_handler_run(handler, frame);
    assert(!cpu_intr_enabled());
}

void
cpu_intr_main(const struct cpu_exc_frame *frame)
{
    const struct cpu_intr_handler *handler;
    unsigned int vector;

    vector = (unsigned int)frame->words[CPU_EXC_FRAME_VECTOR];
    handler = cpu_get_intr_handler(vector);

    thread_intr_enter();
    cpu_intr_handler_run(handler, vector);
    thread_intr_leave();

    assert(!cpu_intr_enabled());
}

static void
cpu_exc_default(const struct cpu_exc_frame *frame)
{
    cpu_halt_broadcast();
    printf("cpu: unregistered exception (cpu%u):\n", cpu_id());
    cpu_show_thread();
    cpu_show_frame(frame);
    cpu_show_stack(frame);
    cpu_halt();
}

static void
cpu_intr_default(unsigned int vector)
{
    cpu_halt_broadcast();
    printf("cpu: unregistered interrupt %u (cpu%u):\n", vector, cpu_id());
    cpu_show_thread();
    cpu_halt();
}

static void
cpu_xcall_intr(unsigned int vector)
{
    (void)vector;

    lapic_eoi();
    xcall_intr();
}

static void
cpu_thread_schedule_intr(unsigned int vector)
{
    (void)vector;

    lapic_eoi();
    thread_schedule_intr();
}

static void
cpu_halt_intr(unsigned int vector)
{
    (void)vector;

    lapic_eoi();
    cpu_halt();
}

static void __init
cpu_setup_idt(void)
{
    for (size_t i = 0; i < ARRAY_SIZE(cpu_ll_exc_handler_addrs); i++) {
        cpu_idt_set_intr_gate(&cpu_idt, i, cpu_get_ll_exc_handler(i));
    }

    cpu_idt_setup_double_fault(&cpu_idt);
}

static void __init
cpu_setup_intr(void)
{
    cpu_setup_idt();

    for (size_t i = 0; i < ARRAY_SIZE(cpu_exc_handlers); i++) {
        cpu_register_exc(i, cpu_exc_default);
    }

    /* Architecture defined exceptions */
    cpu_register_exc(CPU_EXC_DE, cpu_exc_default);
    cpu_register_exc(CPU_EXC_DB, cpu_exc_default);
    cpu_register_intr(CPU_EXC_NMI, cpu_intr_default);
    cpu_register_exc(CPU_EXC_BP, cpu_exc_default);
    cpu_register_exc(CPU_EXC_OF, cpu_exc_default);
    cpu_register_exc(CPU_EXC_BR, cpu_exc_default);
    cpu_register_exc(CPU_EXC_UD, cpu_exc_default);
    cpu_register_exc(CPU_EXC_NM, cpu_exc_default);
    cpu_register_exc(CPU_EXC_DF, cpu_exc_double_fault);
    cpu_register_exc(CPU_EXC_TS, cpu_exc_default);
    cpu_register_exc(CPU_EXC_NP, cpu_exc_default);
    cpu_register_exc(CPU_EXC_SS, cpu_exc_default);
    cpu_register_exc(CPU_EXC_GP, cpu_exc_default);
    cpu_register_exc(CPU_EXC_PF, cpu_exc_default);
    cpu_register_exc(CPU_EXC_MF, cpu_exc_default);
    cpu_register_exc(CPU_EXC_AC, cpu_exc_default);
    cpu_register_intr(CPU_EXC_MC, cpu_intr_default);
    cpu_register_exc(CPU_EXC_XM, cpu_exc_default);

    /* System defined exceptions */
    cpu_register_intr(CPU_EXC_XCALL, cpu_xcall_intr);
    cpu_register_intr(CPU_EXC_THREAD_SCHEDULE, cpu_thread_schedule_intr);
    cpu_register_intr(CPU_EXC_HALT, cpu_halt_intr);
}

static void __init
cpu_seg_desc_init_null(struct cpu_seg_desc *desc)
{
    desc->high = 0;
    desc->low = 0;
}

static void __init
cpu_seg_desc_init_code(struct cpu_seg_desc *desc)
{
#ifdef __LP64__
    desc->high = CPU_DESC_LONG | CPU_DESC_PRESENT | CPU_DESC_S
                 | CPU_DESC_TYPE_CODE;
    desc->low = 0;
#else /* __LP64__ */
    desc->high = CPU_DESC_GRAN_4KB | CPU_DESC_DB
                 | (0x000fffff & CPU_DESC_SEG_LIMIT_HIGH_MASK)
                 | CPU_DESC_PRESENT | CPU_DESC_S | CPU_DESC_TYPE_CODE;
    desc->low = 0x000fffff & CPU_DESC_SEG_LIMIT_LOW_MASK;
#endif /* __LP64__ */
}

static void __init
cpu_seg_desc_init_data(struct cpu_seg_desc *desc, uintptr_t base)
{
#ifdef __LP64__
    (void)base;

    desc->high = CPU_DESC_DB | CPU_DESC_PRESENT | CPU_DESC_S
                 | CPU_DESC_TYPE_DATA;
    desc->low = 0;
#else /* __LP64__ */
    desc->high = (base & CPU_DESC_SEG_BASE_HIGH_MASK)
                 | CPU_DESC_GRAN_4KB | CPU_DESC_DB
                 | (0x000fffff & CPU_DESC_SEG_LIMIT_HIGH_MASK)
                 | CPU_DESC_PRESENT | CPU_DESC_S | CPU_DESC_TYPE_DATA
                 | ((base & CPU_DESC_SEG_BASE_MID_MASK) >> 16);
    desc->low = ((base & CPU_DESC_SEG_BASE_LOW_MASK) << 16)
                | (0x000fffff & CPU_DESC_SEG_LIMIT_LOW_MASK);
#endif /* __LP64__ */
}

static void __init
cpu_sysseg_desc_init_tss(struct cpu_sysseg_desc *desc,
                         const struct cpu_tss *tss)
{
    uintptr_t base, limit;

    base = (uintptr_t)tss;
    limit = base + sizeof(*tss) - 1;

#ifdef __LP64__
    desc->word4 = 0;
    desc->word3 = (base >> 32);
#endif /* __LP64__ */
    desc->word2 = (base & CPU_DESC_SEG_BASE_HIGH_MASK)
                  | (limit & CPU_DESC_SEG_LIMIT_HIGH_MASK)
                  | CPU_DESC_PRESENT | CPU_DESC_TYPE_TSS
                  | ((base & CPU_DESC_SEG_BASE_MID_MASK) >> 16);
    desc->word1 = ((base & CPU_DESC_SEG_BASE_LOW_MASK) << 16)
                  | (limit & CPU_DESC_SEG_LIMIT_LOW_MASK);
}

static void * __init
cpu_gdt_get_desc(struct cpu_gdt *gdt, unsigned int selector)
{
    assert((selector % sizeof(struct cpu_seg_desc)) == 0);
    assert(selector < sizeof(gdt->descs));
    return gdt->descs + selector;
}

static void __init
cpu_gdt_set_null(struct cpu_gdt *gdt, unsigned int selector)
{
    struct cpu_seg_desc *desc;

    desc = cpu_gdt_get_desc(gdt, selector);
    cpu_seg_desc_init_null(desc);
}

static void __init
cpu_gdt_set_code(struct cpu_gdt *gdt, unsigned int selector)
{
    struct cpu_seg_desc *desc;

    desc = cpu_gdt_get_desc(gdt, selector);
    cpu_seg_desc_init_code(desc);
}

static void __init
cpu_gdt_set_data(struct cpu_gdt *gdt, unsigned int selector, const void *base)
{
    struct cpu_seg_desc *desc;

    desc = cpu_gdt_get_desc(gdt, selector);
    cpu_seg_desc_init_data(desc, (uintptr_t)base);
}

static void __init
cpu_gdt_set_tss(struct cpu_gdt *gdt, unsigned int selector,
                const struct cpu_tss *tss)
{
    struct cpu_sysseg_desc *desc;

    desc = cpu_gdt_get_desc(gdt, selector);
    cpu_sysseg_desc_init_tss(desc, tss);
}

static void __init
cpu_gdt_init(struct cpu_gdt *gdt, const struct cpu_tss *tss,
             const struct cpu_tss *df_tss, void *pcpu_area)
{
    cpu_gdt_set_null(gdt, CPU_GDT_SEL_NULL);
    cpu_gdt_set_code(gdt, CPU_GDT_SEL_CODE);
    cpu_gdt_set_data(gdt, CPU_GDT_SEL_DATA, 0);
    cpu_gdt_set_tss(gdt, CPU_GDT_SEL_TSS, tss);

#ifdef __LP64__
    (void)df_tss;
    (void)pcpu_area;
#else /* __LP64__ */
    cpu_gdt_set_tss(gdt, CPU_GDT_SEL_DF_TSS, df_tss);
    cpu_gdt_set_data(gdt, CPU_GDT_SEL_PERCPU, pcpu_area);
    cpu_gdt_set_data(gdt, CPU_GDT_SEL_TLS, &cpu_tls_seg);
#endif /* __LP64__ */
}

static void __init
cpu_gdt_load(const struct cpu_gdt *gdt)
{
    struct cpu_pseudo_desc gdtr;

    gdtr.address = (uintptr_t)gdt->descs;
    gdtr.limit = sizeof(gdt->descs) - 1;
    cpu_load_gdt(&gdtr);
}

static void __init
cpu_tss_init(struct cpu_tss *tss, const void *intr_stack_top,
             const void *df_stack_top)
{
    memset(tss, 0, sizeof(*tss));

#ifdef __LP64__
    tss->ist[CPU_TSS_IST_INTR] = (uintptr_t)intr_stack_top;
    tss->ist[CPU_TSS_IST_DF] = (uintptr_t)df_stack_top;
#else /* __LP64__ */
    (void)intr_stack_top;
    (void)df_stack_top;
#endif /* __LP64__ */
}

#ifndef __LP64__
static void __init
cpu_tss_init_i386_double_fault(struct cpu_tss *tss, const void *df_stack_top)
{
    memset(tss, 0, sizeof(*tss));
    tss->cr3 = cpu_get_cr3();
    tss->eip = (uintptr_t)cpu_get_ll_exc_handler(CPU_EXC_DF);
    tss->eflags = CPU_EFL_ONE;
    tss->ebp = (uintptr_t)df_stack_top;
    tss->esp = tss->ebp;
    tss->es = CPU_GDT_SEL_DATA;
    tss->cs = CPU_GDT_SEL_CODE;
    tss->ss = CPU_GDT_SEL_DATA;
    tss->ds = CPU_GDT_SEL_DATA;
    tss->fs = CPU_GDT_SEL_PERCPU;
}
#endif /* __LP64__ */

static void __init
cpu_feature_map_init(struct cpu_feature_map *map)
{
    bitmap_zero(map->flags, CPU_NR_FEATURES);
}

static void __init
cpu_feature_map_cset(struct cpu_feature_map *map, unsigned int word,
                     unsigned int mask, enum cpu_feature feature)
{
    if (word & mask) {
        bitmap_set(map->flags, feature);
    }
}

static void __init
cpu_feature_map_basic1_edx(struct cpu_feature_map *map, unsigned int edx)
{
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_FPU, CPU_FEATURE_FPU);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_PSE, CPU_FEATURE_PSE);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_PAE, CPU_FEATURE_PAE);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_MSR, CPU_FEATURE_MSR);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_CX8, CPU_FEATURE_CX8);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_APIC, CPU_FEATURE_APIC);
    cpu_feature_map_cset(map, edx, CPU_CPUID_BASIC1_EDX_PGE, CPU_FEATURE_PGE);
}

static void __init
cpu_feature_map_ext1_edx(struct cpu_feature_map *map, unsigned int edx)
{
    cpu_feature_map_cset(map, edx, CPU_CPUID_EXT1_EDX_1GP, CPU_FEATURE_1GP);
    cpu_feature_map_cset(map, edx, CPU_CPUID_EXT1_EDX_LM, CPU_FEATURE_LM);
}

static struct cpu_tss * __init
cpu_get_tss(struct cpu *cpu)
{
    return &cpu->tss;
}

static void * __init
cpu_get_intr_stack_top(struct cpu *cpu)
{
    return &cpu->intr_stack[sizeof(cpu->intr_stack)];
}

static struct cpu_tss * __init
cpu_get_df_tss(struct cpu *cpu)
{
#ifdef __LP64__
    (void)cpu;
    return NULL;
#else /* __LP64__ */
    return &cpu->df_tss;
#endif /* __LP64__ */
}

static void * __init
cpu_get_df_stack_top(struct cpu *cpu)
{
    return &cpu->df_stack[sizeof(cpu->df_stack)];
}

static void __init
cpu_init(struct cpu *cpu, unsigned int id, unsigned int apic_id)
{
    memset(cpu, 0, sizeof(*cpu));
    cpu->id = id;
    cpu->apic_id = apic_id;
}

static void __init
cpu_load_ldt(void)
{
    asm volatile("lldt %w0" : : "q" (CPU_GDT_SEL_NULL));
}

static void __init
cpu_load_tss(void)
{
    asm volatile("ltr %w0" : : "q" (CPU_GDT_SEL_TSS));
}

static void __init
cpu_set_percpu_area(const struct cpu *cpu, void *area)
{
#ifdef __LP64__
    unsigned long va;

    va = (unsigned long)area;
    cpu_set_msr(CPU_MSR_FSBASE, (uint32_t)(va >> 32), (uint32_t)va);
#else /* __LP64__ */
    asm volatile("mov %0, %%fs" : : "r" (CPU_GDT_SEL_PERCPU));
#endif /* __LP64__ */

    percpu_var(cpu_local_area, cpu->id) = area;
}

static void __init
cpu_set_tls_area(void)
{
#ifdef __LP64__
    uintptr_t va;

    va = (uintptr_t)&cpu_tls_seg;
    cpu_set_msr(CPU_MSR_GSBASE, (uint32_t)(va >> 32), (uint32_t)va);
#else /* __LP64__ */
    asm volatile("mov %0, %%gs" : : "r" (CPU_GDT_SEL_TLS));
#endif /* __LP64__ */
}

static const struct cpu_vendor * __init
cpu_vendor_lookup(const char *str)
{
    for (size_t i = 0; i < ARRAY_SIZE(cpu_vendors); i++) {
        if (strcmp(str, cpu_vendors[i].str) == 0) {
            return &cpu_vendors[i];
        }
    }

    return NULL;
}

static void __init
cpu_init_vendor_id(struct cpu *cpu)
{
    const struct cpu_vendor *vendor;

    vendor = cpu_vendor_lookup(cpu->vendor_str);

    if (vendor == NULL) {
        return;
    }

    cpu->vendor_id = vendor->id;
}

static void __init
cpu_build(struct cpu *cpu)
{
    unsigned int eax, ebx, ecx, edx, max_basic, max_extended;
    void *pcpu_area;

    pcpu_area = percpu_area(cpu->id);

    /*
     * Assume at least an i586 processor.
     */

    cpu_intr_restore(CPU_EFL_ONE);
    cpu_set_cr0(CPU_CR0_PG | CPU_CR0_AM | CPU_CR0_WP | CPU_CR0_NE | CPU_CR0_ET
                | CPU_CR0_TS | CPU_CR0_MP | CPU_CR0_PE);
    cpu_gdt_init(&cpu->gdt, cpu_get_tss(cpu), cpu_get_df_tss(cpu), pcpu_area);
    cpu_gdt_load(&cpu->gdt);
    cpu_load_ldt();
    cpu_tss_init(&cpu->tss, cpu_get_intr_stack_top(cpu),
                 cpu_get_df_stack_top(cpu));
#ifndef __LP64__
    cpu_tss_init_i386_double_fault(&cpu->df_tss, cpu_get_df_stack_top(cpu));
#endif /* __LP64__ */
    cpu_load_tss();
    cpu_idt_load(&cpu_idt);
    cpu_set_percpu_area(cpu, pcpu_area);
    cpu_set_tls_area();

    /*
     * Perform the check after initializing the GDT and the per-CPU area
     * since cpu_id() relies on them to correctly work.
     */
    assert(cpu->id == cpu_id());

    eax = 0;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);
    max_basic = eax;
    cpu->cpuid_max_basic = max_basic;
    memcpy(cpu->vendor_str, &ebx, sizeof(ebx));
    memcpy(cpu->vendor_str + 4, &edx, sizeof(edx));
    memcpy(cpu->vendor_str + 8, &ecx, sizeof(ecx));
    cpu->vendor_str[sizeof(cpu->vendor_str) - 1] = '\0';
    cpu_init_vendor_id(cpu);

    /* Some fields are only initialized if supported by the processor */
    cpu->model_name[0] = '\0';
    cpu->phys_addr_width = 0;
    cpu->virt_addr_width = 0;

    assert(max_basic >= 1);

    eax = 1;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);
    cpu->type = (eax & CPU_CPUID_TYPE_MASK) >> CPU_CPUID_TYPE_SHIFT;
    cpu->family = (eax & CPU_CPUID_FAMILY_MASK) >> CPU_CPUID_FAMILY_SHIFT;

    if (cpu->family == 0xf) {
        cpu->family += (eax & CPU_CPUID_EXTFAMILY_MASK)
                       >> CPU_CPUID_EXTFAMILY_SHIFT;
    }

    cpu->model = (eax & CPU_CPUID_MODEL_MASK) >> CPU_CPUID_MODEL_SHIFT;

    if ((cpu->model == 6) || (cpu->model == 0xf)) {
        cpu->model += (eax & CPU_CPUID_EXTMODEL_MASK)
                      >> CPU_CPUID_EXTMODEL_SHIFT;
    }

    cpu->stepping = (eax & CPU_CPUID_STEPPING_MASK)
                    >> CPU_CPUID_STEPPING_SHIFT;
    cpu->clflush_size = ((ebx & CPU_CPUID_CLFLUSH_MASK)
                         >> CPU_CPUID_CLFLUSH_SHIFT) * 8;
    cpu->initial_apic_id = (ebx & CPU_CPUID_APIC_ID_MASK)
                           >> CPU_CPUID_APIC_ID_SHIFT;
    cpu_feature_map_init(&cpu->feature_map);
    cpu_feature_map_basic1_edx(&cpu->feature_map, edx);

    eax = CPU_CPUID_EXT_BIT;
    cpu_cpuid(&eax, &ebx, &ecx, &edx);

    if (eax <= CPU_CPUID_EXT_BIT) {
        max_extended = 0;
    } else {
        max_extended = eax;
    }

    cpu->cpuid_max_extended = max_extended;

    if (max_extended >= (CPU_CPUID_EXT_BIT | 1)) {
        eax = CPU_CPUID_EXT_BIT | 1;
        cpu_cpuid(&eax, &ebx, &ecx, &edx);
        cpu_feature_map_ext1_edx(&cpu->feature_map, edx);
    }

    if (max_extended >= (CPU_CPUID_EXT_BIT | 4)) {
        eax = CPU_CPUID_EXT_BIT | 2;
        cpu_cpuid(&eax, &ebx, &ecx, &edx);
        memcpy(cpu->model_name, &eax, sizeof(eax));
        memcpy(cpu->model_name + 4, &ebx, sizeof(ebx));
        memcpy(cpu->model_name + 8, &ecx, sizeof(ecx));
        memcpy(cpu->model_name + 12, &edx, sizeof(edx));

        eax = CPU_CPUID_EXT_BIT | 3;
        cpu_cpuid(&eax, &ebx, &ecx, &edx);
        memcpy(cpu->model_name + 16, &eax, sizeof(eax));
        memcpy(cpu->model_name + 20, &ebx, sizeof(ebx));
        memcpy(cpu->model_name + 24, &ecx, sizeof(ecx));
        memcpy(cpu->model_name + 28, &edx, sizeof(edx));

        eax = CPU_CPUID_EXT_BIT | 4;
        cpu_cpuid(&eax, &ebx, &ecx, &edx);
        memcpy(cpu->model_name + 32, &eax, sizeof(eax));
        memcpy(cpu->model_name + 36, &ebx, sizeof(ebx));
        memcpy(cpu->model_name + 40, &ecx, sizeof(ecx));
        memcpy(cpu->model_name + 44, &edx, sizeof(edx));

        cpu->model_name[sizeof(cpu->model_name) - 1] = '\0';
    }

    if (max_extended >= (CPU_CPUID_EXT_BIT | 8)) {
        eax = CPU_CPUID_EXT_BIT | 8;
        cpu_cpuid(&eax, &ebx, &ecx, &edx);
        cpu->phys_addr_width = (unsigned short)eax & 0xff;
        cpu->virt_addr_width = ((unsigned short)eax >> 8) & 0xff;
    }

    atomic_store(&cpu->started, 1, ATOMIC_RELEASE);
}

static void __init
cpu_measure_freq(void)
{
    uint64_t start, end;

    pit_setup_free_running();

    start = cpu_get_tsc();
    pit_delay(CPU_FREQ_CAL_DELAY);
    end = cpu_get_tsc();

    cpu_freq = (end - start) / (1000000 / CPU_FREQ_CAL_DELAY);
}

static int __init
cpu_setup(void)
{
    struct cpu *cpu;

    cpu_setup_intr();

    cpu = percpu_ptr(cpu_desc, 0);
    cpu_init(cpu, 0, CPU_INVALID_APIC_ID);
    cpu_build(cpu);

    cpu_measure_freq();

    return 0;
}

INIT_OP_DEFINE(cpu_setup,
               INIT_OP_DEP(percpu_bootstrap, true));

static void __init
cpu_panic_on_missing_feature(const char *feature)
{
    panic("cpu: %s feature missing", feature);
}

static void __init
cpu_check(const struct cpu *cpu)
{
    if (!cpu_has_feature(cpu, CPU_FEATURE_FPU)) {
        cpu_panic_on_missing_feature("fpu");
    }

    /*
     * The compiler is expected to produce cmpxchg8b instructions to
     * perform 64-bits atomic operations on a 32-bits processor. Clang
     * currently has trouble doing that so 64-bits atomic support is
     * just disabled when building with it.
     */
#if !defined(__LP64__) && !defined(__clang__)
    if (!cpu_has_feature(cpu, CPU_FEATURE_CX8)) {
        cpu_panic_on_missing_feature("cx8");
    }
#endif
}

static int __init
cpu_check_bsp(void)
{
    cpu_check(cpu_current());
    return 0;
}

INIT_OP_DEFINE(cpu_check_bsp,
               INIT_OP_DEP(cpu_setup, true));

void *
cpu_get_intr_stack_ptr(void)
{
    struct cpu *cpu;

    assert(!thread_interrupted());

    cpu = cpu_local_ptr(cpu_desc);
    return cpu_get_intr_stack_top(cpu);
}

void __init
cpu_log_info(const struct cpu *cpu)
{
    char features[60], *ptr;
    size_t size, bytes;

    log_info("cpu%u: %s, type %u, family %u, model %u, stepping %u",
             cpu->id, cpu->vendor_str, cpu->type, cpu->family, cpu->model,
             cpu->stepping);

    if (strlen(cpu->model_name) > 0) {
        log_info("cpu%u: %s", cpu->id, cpu->model_name);
    }

    if ((cpu->phys_addr_width != 0) && (cpu->virt_addr_width != 0)) {
        log_info("cpu%u: address widths: physical: %hu, virtual: %hu",
                 cpu->id, cpu->phys_addr_width, cpu->virt_addr_width);
    }

    log_info("cpu%u: frequency: %llu.%02llu MHz", cpu->id,
             (unsigned long long)cpu_freq / 1000000,
             (unsigned long long)cpu_freq % 1000000);

    ptr = features;
    size = sizeof(features);

    for (size_t i = 0; i < ARRAY_SIZE(cpu_feature_names); i++) {
        if (!cpu_has_feature(cpu, i)) {
            continue;
        }

        assert(strlen(cpu_feature_names[i]) < sizeof(features));
        bytes = snprintf(ptr, size, " %s", cpu_feature_names[i]);

        if (bytes >= size) {
            *ptr = '\0';
            log_info("cpu%u:%s", cpu->id, features);
            ptr = features;
            size = sizeof(features);
            i--;
            continue;
        }

        ptr += bytes;
        size -= bytes;
    }

    log_info("cpu%u:%s", cpu->id, features);
}

void __init
cpu_mp_register_lapic(unsigned int apic_id, bool is_bsp)
{
    struct cpu *cpu;
    int error;

    if (is_bsp) {
        cpu = percpu_ptr(cpu_desc, 0);

        if (cpu->apic_id != CPU_INVALID_APIC_ID) {
            panic("cpu: another processor pretends to be the BSP");
        }

        cpu->apic_id = apic_id;
        return;
    }

    error = percpu_add(cpu_nr_active);

    if (error) {
        return;
    }

    cpu = percpu_ptr(cpu_desc, cpu_nr_active);
    cpu_init(cpu, cpu_nr_active, apic_id);
    cpu_nr_active++;
}

static void
cpu_trigger_double_fault(void)
{
    asm volatile("movl $0xdead, %esp; push $0");
}

static void
cpu_shutdown_reset(void)
{
    /* Generate a triple fault */
    cpu_idt_load(NULL);
    cpu_trigger_double_fault();
}

static struct shutdown_ops cpu_shutdown_ops = {
    .reset = cpu_shutdown_reset,
};

static int __init
cpu_mp_probe(void)
{
    log_info("cpu: %u processor(s) configured", cpu_count());
    return 0;
}

INIT_OP_DEFINE(cpu_mp_probe,
               INIT_OP_DEP(acpi_setup, true),
               INIT_OP_DEP(cpu_setup, true),
               INIT_OP_DEP(log_setup, true));

static int __init
cpu_setup_shutdown(void)
{
    if (cpu_count() == 1) {
        shutdown_register(&cpu_shutdown_ops, CPU_SHUTDOWN_PRIORITY);
    }

    return 0;
}

INIT_OP_DEFINE(cpu_setup_shutdown,
               INIT_OP_DEP(cpu_mp_probe, true),
               INIT_OP_DEP(shutdown_bootstrap, true));

void __init
cpu_mp_setup(void)
{
    uint16_t reset_vector[2];
    unsigned int started;
    struct cpu *cpu;
    void *ptr;

    if (cpu_count() == 1) {
        pmap_mp_setup();
        return;
    }

    assert(BOOT_MP_TRAMPOLINE_ADDR < BIOSMEM_BASE);
    assert(vm_page_aligned(BOOT_MP_TRAMPOLINE_ADDR));
    assert(boot_mp_trampoline_size <= PAGE_SIZE);

    /* Set up the AP trampoline code */
    ptr = (void *)vm_page_direct_va(BOOT_MP_TRAMPOLINE_ADDR);
    memcpy(ptr, boot_mp_trampoline, boot_mp_trampoline_size);

    /* Set up the warm reset vector */
    reset_vector[0] = 0;
    reset_vector[1] = BOOT_MP_TRAMPOLINE_ADDR >> 4;
    ptr = (void *)vm_page_direct_va(CPU_MP_CMOS_RESET_VECTOR);
    memcpy(ptr, reset_vector, sizeof(reset_vector));

    io_write_byte(CPU_MP_CMOS_PORT_REG, CPU_MP_CMOS_REG_RESET);
    io_write_byte(CPU_MP_CMOS_PORT_DATA, CPU_MP_CMOS_DATA_RESET_WARM);

    boot_alloc_ap_stacks();

    /*
     * This function creates per-CPU copies of the page tables. Just in case,
     * call it last to make sure all processors get the same mappings.
     */
    pmap_mp_setup();

    for (unsigned int i = 1; i < cpu_count(); i++) {
        cpu = percpu_ptr(cpu_desc, i);
        boot_set_ap_id(i);

        /* Perform the "Universal Start-up Algorithm" */
        lapic_ipi_init_assert(cpu->apic_id);
        cpu_delay(200);
        lapic_ipi_init_deassert(cpu->apic_id);
        cpu_delay(10000);
        lapic_ipi_startup(cpu->apic_id, BOOT_MP_TRAMPOLINE_ADDR >> 12);
        cpu_delay(200);
        lapic_ipi_startup(cpu->apic_id, BOOT_MP_TRAMPOLINE_ADDR >> 12);
        cpu_delay(200);

        for (;;) {
            started = atomic_load(&cpu->started, ATOMIC_ACQUIRE);

            if (started) {
                break;
            }
        }
    }
}

void __init
cpu_ap_setup(unsigned int ap_id)
{
    struct cpu *cpu;

    cpu = percpu_ptr(cpu_desc, ap_id);
    cpu_build(cpu);
    cpu_check(cpu_current());
    lapic_ap_setup();
}

void
cpu_halt_broadcast(void)
{
    unsigned int nr_cpus;

    assert(!cpu_intr_enabled());

    nr_cpus = cpu_count();

    if (nr_cpus == 1) {
        return;
    }

    lapic_ipi_broadcast(CPU_EXC_HALT);
}


code:
/*
 * Copyright (c) 2010-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_CPU_H
#define X86_CPU_H

#include <limits.h>

#include <machine/page.h>

/*
 * Architecture defined exception vectors.
 */
#define CPU_EXC_DE                  0   /* Divide Error */
#define CPU_EXC_DB                  1   /* Debug */
#define CPU_EXC_NMI                 2   /* NMI Interrupt */
#define CPU_EXC_BP                  3   /* Breakpoint */
#define CPU_EXC_OF                  4   /* Overflow */
#define CPU_EXC_BR                  5   /* BOUND Range Exceeded */
#define CPU_EXC_UD                  6   /* Undefined Opcode */
#define CPU_EXC_NM                  7   /* No Math Coprocessor */
#define CPU_EXC_DF                  8   /* Double Fault */
#define CPU_EXC_TS                  10  /* Invalid TSS */
#define CPU_EXC_NP                  11  /* Segment Not Present */
#define CPU_EXC_SS                  12  /* Stack-Segment Fault */
#define CPU_EXC_GP                  13  /* General Protection */
#define CPU_EXC_PF                  14  /* Page Fault */
#define CPU_EXC_MF                  16  /* Math Fault */
#define CPU_EXC_AC                  17  /* Alignment Check */
#define CPU_EXC_MC                  18  /* Machine Check */
#define CPU_EXC_XM                  19  /* SIMD Floating-Point Exception */

/*
 * Exception vectors used for external interrupts.
 */
#define CPU_EXC_INTR_FIRST          32
#define CPU_EXC_INTR_LAST           223

/*
 * System defined exception vectors.
 *
 * The local APIC assigns one priority every 16 vectors.
 */
#define CPU_EXC_XCALL               238
#define CPU_EXC_THREAD_SCHEDULE     239
#define CPU_EXC_HALT                240
#define CPU_EXC_LAPIC_PMC_OF        252
#define CPU_EXC_LAPIC_TIMER         253
#define CPU_EXC_LAPIC_ERROR         254
#define CPU_EXC_LAPIC_SPURIOUS      255

#define CPU_NR_EXC_VECTORS          256

#define CPU_INTR_STACK_SIZE PAGE_SIZE

#define CPU_VENDOR_STR_SIZE 13
#define CPU_MODEL_NAME_SIZE 49

#define CPU_VENDOR_UNKNOWN  0
#define CPU_VENDOR_INTEL    1
#define CPU_VENDOR_AMD      2

/*
 * L1 cache line size.
 *
 * XXX Use this value until processor selection is available.
 *
 * TODO Add macros to specifically align to the cache line size, and to
 * do so only in SMP configurations.
 */
#define CPU_L1_SIZE 64

/*
 * CPU word size, 4 or 8 bytes.
 */
#define CPU_WORD_SIZE (LONG_BIT / 8)

/*
 * Data alignment.
 *
 * This is used to align regions of memory that can store any type of
 * data, such as stacks and sections. The modern i386 as well as the
 * amd64 System V ABIs both mandate 16 byte data alignment. Kernel
 * software could use smaller alignments, but this one is meant to
 * be linked against static libraries, and in particular libgcc, which
 * are built for standard ABIs.
 */
#define CPU_DATA_ALIGN 16

/*
 * Function alignment.
 *
 * Aligning functions improves instruction fetching.
 *
 * Used for assembly functions only.
 *
 * XXX Use this value until processor selection is available.
 */
#define CPU_TEXT_ALIGN 16

/*
 * Processor privilege levels.
 */
#define CPU_PL_KERNEL                           0
#define CPU_PL_USER                             3

/*
 * Control register 0 flags.
 */
#define CPU_CR0_PE                              0x00000001
#define CPU_CR0_MP                              0x00000002
#define CPU_CR0_TS                              0x00000008
#define CPU_CR0_ET                              0x00000010
#define CPU_CR0_NE                              0x00000020
#define CPU_CR0_WP                              0x00010000
#define CPU_CR0_AM                              0x00040000
#define CPU_CR0_PG                              0x80000000

/*
 * Control register 4 flags.
 */
#define CPU_CR4_PSE                             0x00000010
#define CPU_CR4_PAE                             0x00000020
#define CPU_CR4_PGE                             0x00000080

/*
 * Model specific registers.
 */
#define CPU_MSR_EFER                            0xc0000080
#define CPU_MSR_FSBASE                          0xc0000100
#define CPU_MSR_GSBASE                          0xc0000101

/*
 * EFER MSR flags.
 */
#define CPU_EFER_LME                            0x00000100

/*
 * Bit used to make extended CPUID requests.
 */
#define CPU_CPUID_EXT_BIT                       0x80000000

/*
 * CPU feature flags as returned by CPUID.
 */
#define CPU_CPUID_BASIC1_EDX_FPU                0x00000001
#define CPU_CPUID_BASIC1_EDX_PSE                0x00000008
#define CPU_CPUID_BASIC1_EDX_PAE                0x00000040
#define CPU_CPUID_BASIC1_EDX_MSR                0x00000020
#define CPU_CPUID_BASIC1_EDX_CX8                0x00000100
#define CPU_CPUID_BASIC1_EDX_APIC               0x00000200
#define CPU_CPUID_BASIC1_EDX_PGE                0x00002000
#define CPU_CPUID_EXT1_EDX_1GP                  0x04000000
#define CPU_CPUID_EXT1_EDX_LM                   0x20000000

#ifndef __ASSEMBLER__

enum cpu_feature {
    CPU_FEATURE_FPU,
    CPU_FEATURE_PSE,
    CPU_FEATURE_PAE,
    CPU_FEATURE_MSR,
    CPU_FEATURE_CX8,
    CPU_FEATURE_APIC,
    CPU_FEATURE_PGE,
    CPU_FEATURE_1GP,
    CPU_FEATURE_LM,
    CPU_NR_FEATURES
};

#endif /* __ASSEMBLER__ */

#include <machine/cpu_i.h>

#ifndef __ASSEMBLER__

#include <stdalign.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdnoreturn.h>

#include <kern/init.h>
#include <kern/macros.h>
#include <kern/percpu.h>
#include <machine/lapic.h>
#include <machine/pit.h>
#include <machine/ssp.h>

#define CPU_INTR_TABLE_SIZE (CPU_EXC_INTR_LAST - CPU_EXC_INTR_FIRST)

/*
 * Gate/segment descriptor bits and masks.
 */
#define CPU_DESC_TYPE_DATA                      0x00000200
#define CPU_DESC_TYPE_CODE                      0x00000a00
#define CPU_DESC_TYPE_TSS                       0x00000900
#define CPU_DESC_TYPE_GATE_INTR                 0x00000e00
#define CPU_DESC_TYPE_GATE_TASK                 0x00000500
#define CPU_DESC_S                              0x00001000
#define CPU_DESC_PRESENT                        0x00008000
#define CPU_DESC_LONG                           0x00200000
#define CPU_DESC_DB                             0x00400000
#define CPU_DESC_GRAN_4KB                       0x00800000

#define CPU_DESC_GATE_OFFSET_LOW_MASK           0x0000ffff
#define CPU_DESC_GATE_OFFSET_HIGH_MASK          0xffff0000
#define CPU_DESC_SEG_IST_MASK                   0x00000007
#define CPU_DESC_SEG_BASE_LOW_MASK              0x0000ffff
#define CPU_DESC_SEG_BASE_MID_MASK              0x00ff0000
#define CPU_DESC_SEG_BASE_HIGH_MASK             0xff000000
#define CPU_DESC_SEG_LIMIT_LOW_MASK             0x0000ffff
#define CPU_DESC_SEG_LIMIT_HIGH_MASK            0x000f0000

/*
 * Type for interrupt handler functions.
 */
typedef void (*cpu_intr_handler_fn_t)(unsigned int vector);

/*
 * TLS segment, as expected by the compiler.
 *
 * TLS isn't actually used inside the kernel. The current purpose of this
 * segment is to implement stack protection.
 *
 * This is a public structure, made available to the boot module so that
 * C code that runs early correctly works when built with stack protection.
 */
struct cpu_tls_seg {
    uintptr_t unused[SSP_WORD_TLS_OFFSET];
    uintptr_t ssp_guard_word;
};

/*
 * Code or data segment descriptor.
 *
 * See Intel 64 and IA-32 Architecture Software Developer's Manual,
 * Volume 3 System Programming Guide, 3.4.5 Segment Descriptors.
 */
struct cpu_seg_desc {
    uint32_t low;
    uint32_t high;
};

/*
 * Macro to create functions that read/write control registers.
 *
 * TODO Break down.
 */
#define CPU_DECL_GETSET_CR(name)                                            \
static __always_inline unsigned long                                        \
cpu_get_ ## name(void)                                                      \
{                                                                           \
    unsigned long name;                                                     \
                                                                            \
    asm volatile("mov %%" __QUOTE(name) ", %0" : "=r" (name) : : "memory"); \
    return name;                                                            \
}                                                                           \
                                                                            \
static __always_inline void                                                 \
cpu_set_ ## name(unsigned long value)                                       \
{                                                                           \
    asm volatile("mov %0, %%" __QUOTE(name) : : "r" (value) : "memory");    \
}

/*
 * Access to the processor control registers. CR1 is reserved.
 *
 * The caller should assume that these functions are declared as :
 *  static inline unsigned long cpu_get_crX(void);
 *  static inline void cpu_set_crX(unsigned long);
 *
 * They all imply a compiler barrier.
 */
CPU_DECL_GETSET_CR(cr0)
CPU_DECL_GETSET_CR(cr2)
CPU_DECL_GETSET_CR(cr3)
CPU_DECL_GETSET_CR(cr4)

/*
 * Enable local interrupts.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_intr_enable(void)
{
    asm volatile("sti" : : : "memory");
}

/*
 * Disable local interrupts.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_intr_disable(void)
{
    asm volatile("cli" : : : "memory");
}

/*
 * Restore the content of the EFLAGS register, possibly enabling interrupts.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_intr_restore(unsigned long flags)
{
    asm volatile("push %0\n"
                 "popf\n"
                 : : "r" (flags)
                 : "memory");
}

/*
 * Disable local interrupts, returning the previous content of the EFLAGS
 * register.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_intr_save(unsigned long *flags)
{
    *flags = cpu_get_eflags();
    cpu_intr_disable();
}

/*
 * Return true if interrupts are enabled.
 *
 * Implies a compiler barrier.
 */
static __always_inline int
cpu_intr_enabled(void)
{
    unsigned long eflags;

    eflags = cpu_get_eflags();
    return (eflags & CPU_EFL_IF) ? 1 : 0;
}

/*
 * Spin-wait loop hint.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_pause(void)
{
    asm volatile("pause" : : : "memory");
}

/*
 * Make the CPU idle until the next interrupt.
 *
 * Interrupts are enabled. Besides, they're enabled in a way that doesn't
 * allow the processor handling them before entering the idle state if they
 * were disabled before calling this function.
 *
 * Implies a compiler barrier.
 */
static __always_inline void
cpu_idle(void)
{
    asm volatile("sti; hlt" : : : "memory");
}

/*
 * Halt the CPU.
 *
 * Implies a compiler barrier.
 */
noreturn static __always_inline void
cpu_halt(void)
{
    cpu_intr_disable();

    for (;;) {
        asm volatile("hlt" : : : "memory");
    }
}

/*
 * Halt all other processors.
 *
 * Interrupts must be disabled when calling this function.
 */
void cpu_halt_broadcast(void);

/* Generic percpu accessors */

#define cpu_local_ptr(var)                  \
MACRO_BEGIN                                 \
    typeof(var) *ptr_ = &(var);             \
                                            \
    asm("add %%fs:%1, %0"                   \
                 : "+r" (ptr_)              \
                 : "m" (cpu_local_area));   \
                                            \
    ptr_;                                   \
MACRO_END

#define cpu_local_var(var) (*cpu_local_ptr(var))

/* Generic interrupt-safe percpu accessors */

#define cpu_local_assign(var, val)          \
    asm("mov %0, %%fs:%1"                   \
                 : : "r" (val), "m" (var));

#define cpu_local_read(var)         \
MACRO_BEGIN                         \
    typeof(var) val_;               \
                                    \
    asm("mov %%fs:%1, %0"           \
                 : "=r" (val_)      \
                 : "m" (var));      \
                                    \
    val_;                           \
MACRO_END

static inline struct cpu *
cpu_current(void)
{
    extern struct cpu cpu_desc;
    return cpu_local_ptr(cpu_desc);
}

static inline unsigned int
cpu_id(void)
{
    extern struct cpu cpu_desc;
    return cpu_local_read(cpu_desc.id);
}

static inline unsigned int
cpu_count(void)
{
    extern unsigned int cpu_nr_active;
    return cpu_nr_active;
}

static inline struct cpu *
cpu_from_id(unsigned int cpu)
{
    extern struct cpu cpu_desc;
    return percpu_ptr(cpu_desc, cpu);
}

static inline bool
cpu_has_feature(const struct cpu *cpu, enum cpu_feature feature)
{
    return cpu_feature_map_test(&cpu->feature_map, feature);
}

static __always_inline void
cpu_enable_pse(void)
{
    cpu_set_cr4(cpu_get_cr4() | CPU_CR4_PSE);
}

static __always_inline void
cpu_enable_pae(void)
{
    cpu_set_cr4(cpu_get_cr4() | CPU_CR4_PAE);
}

static inline int
cpu_has_global_pages(void)
{
    return cpu_has_feature(cpu_current(), CPU_FEATURE_PGE);
}

/*
 * Enable the use of global pages in the TLB.
 *
 * As a side effect, this function causes a complete TLB flush if global
 * pages were previously disabled.
 *
 * Implies a full memory barrier.
 * TODO Update barrier description.
 */
static __always_inline void
cpu_enable_global_pages(void)
{
    cpu_set_cr4(cpu_get_cr4() | CPU_CR4_PGE);
}

/*
 * CPUID instruction wrapper.
 *
 * The CPUID instruction is a serializing instruction.
 */
static __always_inline void
cpu_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx,
          unsigned int *edx)
{
    asm volatile("cpuid" : "+a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)
                 : : "memory");
}

static inline void
cpu_get_msr(uint32_t msr, uint32_t *high, uint32_t *low)
{
    asm("rdmsr" : "=a" (*low), "=d" (*high) : "c" (msr));
}

static inline uint64_t
cpu_get_msr64(uint32_t msr)
{
    uint32_t high, low;

    cpu_get_msr(msr, &high, &low);
    return (((uint64_t)high << 32) | low);
}

/*
 * Implies a full memory barrier.
 */
static inline void
cpu_set_msr(uint32_t msr, uint32_t high, uint32_t low)
{
    asm volatile("wrmsr" : : "c" (msr), "a" (low), "d" (high) : "memory");
}

/*
 * Implies a full memory barrier.
 */
static inline void
cpu_set_msr64(uint32_t msr, uint64_t value)
{
    uint32_t low, high;

    low = value & 0xffffffff;
    high = value >> 32;
    cpu_set_msr(msr, high, low);
}

/*
 * Flush non-global TLB entries.
 *
 * Implies a full memory barrier.
 * TODO Update barrier description.
 */
static __always_inline void
cpu_tlb_flush(void)
{
    cpu_set_cr3(cpu_get_cr3());
}

/*
 * Flush all TLB entries, including global ones.
 *
 * Implies a full memory barrier.
 * TODO Update barrier description.
 */
static __always_inline void
cpu_tlb_flush_all(void)
{
    if (!cpu_has_global_pages()) {
        cpu_tlb_flush();
    } else {
        unsigned long cr4;

        cr4 = cpu_get_cr4();

        if (!(cr4 & CPU_CR4_PGE)) {
            cpu_tlb_flush();
        } else {
            cr4 &= ~CPU_CR4_PGE;
            cpu_set_cr4(cr4);
            cr4 |= CPU_CR4_PGE;
            cpu_set_cr4(cr4);
        }
    }
}

/*
 * Flush a single page table entry in the TLB.
 *
 * Implies a full memory barrier.
 * TODO Update barrier description.
 */
static __always_inline void
cpu_tlb_flush_va(unsigned long va)
{
    asm volatile("invlpg (%0)" : : "r" (va) : "memory");
}

static inline unsigned int
cpu_cpuid_max_basic(const struct cpu *cpu)
{
    return cpu->cpuid_max_basic;
}

static inline unsigned int
cpu_vendor_id(const struct cpu *cpu)
{
    return cpu->vendor_id;
}

static inline unsigned int
cpu_family(const struct cpu *cpu)
{
    return cpu->family;
}

static inline unsigned int
cpu_phys_addr_width(const struct cpu *cpu)
{
    return cpu->phys_addr_width;
}

/*
 * Get CPU frequency in Hz.
 */
uint64_t cpu_get_freq(void);

/*
 * Busy-wait for a given amount of time, in microseconds.
 *
 * Implies a compiler barrier.
 */
void cpu_delay(unsigned long usecs);

/*
 * Log processor information.
 */
void cpu_log_info(const struct cpu *cpu);

/*
 * Register a local APIC.
 */
void cpu_mp_register_lapic(unsigned int apic_id, bool is_bsp);

/*
 * Start application processors.
 *
 * The x86 architecture uses per-CPU page tables, which are created as a
 * side effect of this function. In order to synchronize their page tables,
 * processors must be able to communicate very soon after calling this
 * function. They communicate through interrupts and threading facilities.
 * On return, physical mappings must not be altered until inter-processor
 * communication is available.
 */
void cpu_mp_setup(void);

/*
 * CPU initialization on APs.
 */
void cpu_ap_setup(unsigned int ap_id);

static inline unsigned int
cpu_apic_id(unsigned int cpu)
{
    return cpu_from_id(cpu)->apic_id;
}

/*
 * Send a cross-call interrupt to a remote processor.
 */
static inline void
cpu_send_xcall(unsigned int cpu)
{
    lapic_ipi_send(cpu_apic_id(cpu), CPU_EXC_XCALL);
}

/*
 * Send a scheduling interrupt to a remote processor.
 */
static inline void
cpu_send_thread_schedule(unsigned int cpu)
{
    lapic_ipi_send(cpu_apic_id(cpu), CPU_EXC_THREAD_SCHEDULE);
}

/*
 * Register an interrupt handler.
 *
 * This function is only available during system initialization, before the
 * scheduler is started. It is meant for architectural interrupts, including
 * interrupt controllers, and not directly for drivers, which should use
 * the machine-independent intr module instead.
 *
 * Registration is system-wide.
 */
void cpu_register_intr(unsigned int vector, cpu_intr_handler_fn_t fn);

/*
 * This init operation provides :
 *  - initialization of the BSP structure.
 *  - cpu_delay()
 *  - cpu_local_ptr() and cpu_local_var()
 */
INIT_OP_DECLARE(cpu_setup);

/*
 * This init operation provides :
 *  - cpu_count()
 *  - access to percpu variables on all processors
 */
INIT_OP_DECLARE(cpu_mp_probe);

/*
 * This init operation provides :
 *  - cpu shutdown operations registered
 */
INIT_OP_DECLARE(cpu_setup_shutdown);

#endif /* __ASSEMBLER__ */

#endif /* X86_CPU_H */

license: mit


code:
/*
 * Copyright (c) 2011-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <kern/init.h>
#include <machine/asm.h>
#include <machine/cpu.h>
#include <machine/cpu_i.h>

#define CPU_RED_ZONE_SIZE 128

/*
 * Macro used to compute an offset, in bytes, between two indexes in a
 * low level exception frame. The indexes may be given in any order.
 */
#define CPU_LEF_OFFSET(index1, index2) (((index2) - (index1)) * CPU_WORD_SIZE)

.text

#ifdef __LP64__

/*
 * Number of CPU registers on exception.
 *
 * These are the registers automatically pushed by the CPU when an exception
 * is taken, including the error code.
 */
#define CPU_NR_CRE (CPU_EXC_FRAME_SIZE - CPU_EXC_FRAME_ERROR)

/*
 * Remaining size when taking an exception.
 *
 * This is the total size of an exception frame without the registers
 * automatically pushed by the CPU and those saved for stack switching.
 */
#define CPU_LEF_EXC_REM CPU_LEF_OFFSET(CPU_EXC_FRAME_ERROR - 2, \
                                       CPU_EXC_FRAME_SIZE)

/*
 * RSP offset, from the top of the exception frame.
 */
#define CPU_LEF_RSP     CPU_LEF_OFFSET(CPU_EXC_FRAME_SIZE, \
                                       CPU_EXC_FRAME_RSP)

/*
 * Offsets where registers used for stack switching are saved when taking
 * an exception, from the top of the exception frame.
 */
#define CPU_LEF_EXC_RAX CPU_LEF_OFFSET(CPU_EXC_FRAME_SIZE, \
                                       CPU_EXC_FRAME_ERROR - 2)
#define CPU_LEF_EXC_RBX CPU_LEF_OFFSET(CPU_EXC_FRAME_SIZE, \
                                       CPU_EXC_FRAME_ERROR - 1)

/*
 * Size used to store the vector and the error code.
 */
#define CPU_LEF_VEC_ERR CPU_LEF_OFFSET(CPU_EXC_FRAME_VECTOR, \
                                       CPU_EXC_FRAME_ERROR + 1)

.macro cpu_ll_exc_store_registers
    pushq %r15
    pushq %r14
    pushq %r13
    pushq %r12
    pushq %r11
    pushq %r10
    pushq %r9
    pushq %r8
    pushq %rdi
    pushq %rsi
    pushq %rbp
    pushq %rdx
    pushq %rcx
    pushq %rbx
    pushq %rax
.endm

.macro cpu_ll_exc_load_registers
    popq %rax
    popq %rbx
    popq %rcx
    popq %rdx
    popq %rbp
    popq %rsi
    popq %rdi
    popq %r8
    popq %r9
    popq %r10
    popq %r11
    popq %r12
    popq %r13
    popq %r14
    popq %r15
    addq $CPU_LEF_VEC_ERR, %rsp             /* skip vector and error */
.endm

.macro cpu_ll_exc_push_word source offset buffer
    movq -\offset(\source), \buffer
    pushq \buffer
.endm

.macro cpu_ll_exc_push source nr_words buffer
.set offset, CPU_WORD_SIZE
.rept \nr_words
    cpu_ll_exc_push_word \source offset \buffer
.set offset, offset + CPU_WORD_SIZE
.endr
.endm

/*
 * Note that, unlike i386, the CPU aligns the stack frame to 16 bytes,
 * and skipping the red zone keeps the stack pointer correctly aligned.
 */
.macro cpu_ll_exc_enter vector
    pushq %rbx                              /* save registers used for */
    pushq %rax                              /* stack switching */
    leaq CPU_LEF_EXC_REM(%rsp), %rbx        /* point to exception frame top */
    movq CPU_LEF_RSP(%rbx), %rsp            /* point to thread stack */
    subq $CPU_RED_ZONE_SIZE, %rsp           /* skip red zone */
    cpu_ll_exc_push %rbx, CPU_NR_CRE, %rax  /* copy frame start from interrupt
                                               stack to thread stack */
    movq CPU_LEF_EXC_RAX(%rbx), %rax        /* restore registers used */
    movq CPU_LEF_EXC_RBX(%rbx), %rbx        /* for stack switching */
    pushq $\vector                          /* complete exception frame */
    cpu_ll_exc_store_registers
    xorq %rbp, %rbp                         /* block stack tracing */
    movq %rsp, %rbx                         /* save frame */
.endm

.macro cpu_ll_exc_leave
    movq %rbx, %rsp                         /* restore stack */
    call thread_schedule                    /* schedule threads */
    cpu_ll_exc_load_registers
    iretq
.endm

.macro cpu_ll_exc_handle vector
    cpu_ll_exc_enter \vector
    movq %rbx, %rdi
    andq $(~(CPU_DATA_ALIGN - 1)), %rsp
    call cpu_exc_main
    cpu_ll_exc_leave
.endm

.macro cpu_ll_intr_handle vector
    cpu_ll_exc_enter \vector
    call cpu_get_intr_stack_ptr
    movq %rax, %rsp                         /* switch to interrupt stack */
    movq %rbx, %rdi
    call cpu_intr_main
    cpu_ll_exc_leave
.endm

#else /* __LP64__ */

/*
 * Size used to store the segment registers, the vector and the error code.
 */
#define CPU_LEF_SEGS_VEC_ERR CPU_LEF_OFFSET(CPU_EXC_FRAME_DS, \
                                            CPU_EXC_FRAME_ERROR + 1)

.macro cpu_ll_exc_store_registers
    pushl %gs
    pushl %fs
    pushl %es
    pushl %ds
    pushl %edi
    pushl %esi
    pushl %ebp
    pushl %edx
    pushl %ecx
    pushl %ebx
    pushl %eax
.endm

/* XXX Don't load segment registers for now */
.macro cpu_ll_exc_load_registers
    popl %eax
    popl %ebx
    popl %ecx
    popl %edx
    popl %ebp
    popl %esi
    popl %edi
    addl $CPU_LEF_SEGS_VEC_ERR, %esp        /* skip segment registers,
                                               vector and error */
.endm

.macro cpu_ll_exc_enter vector
    pushl $\vector
    cpu_ll_exc_store_registers
    xorl %ebp, %ebp                         /* block stack tracing */
    movl %esp, %ebx                         /* save frame */
.endm

.macro cpu_ll_exc_align_stack nr_args
    andl $(~(CPU_DATA_ALIGN - 1)), %esp
    addl $((\nr_args * CPU_WORD_SIZE) % CPU_DATA_ALIGN), %esp
.endm

.macro cpu_ll_exc_leave
    movl %ebx, %esp                         /* restore stack */
    call thread_schedule                    /* schedule threads */
    cpu_ll_exc_load_registers
    iretl
.endm

.macro cpu_ll_exc_handle vector
    cpu_ll_exc_enter \vector
    cpu_ll_exc_align_stack 1
    pushl %ebx
    call cpu_exc_main
    cpu_ll_exc_leave
.endm

.macro cpu_ll_intr_handle vector
    cpu_ll_exc_enter \vector
    call cpu_get_intr_stack_ptr
    movl %eax, %esp                         /* switch to interrupt stack */
    cpu_ll_exc_align_stack 1
    pushl %ebx
    call cpu_intr_main
    cpu_ll_exc_leave
.endm

#endif /* __LP64__ */

#define CPU_LL_EXC_HANDLER(vector, name)            \
ASM_ENTRY(name)                                     \
    push $0;                                        \
    cpu_ll_exc_handle vector;                       \
ASM_END(name)

#define CPU_LL_EXC_HANDLER_WITH_ERROR(vector, name) \
ASM_ENTRY(name)                                     \
    cpu_ll_exc_handle vector;                       \
ASM_END(name)

#define CPU_LL_INTR_HANDLER(vector, name)           \
ASM_ENTRY(name)                                     \
    push $0;                                        \
    cpu_ll_intr_handle vector;                      \
ASM_END(name)

/*
 * Low level handlers for architectural exceptions.
 */
CPU_LL_EXC_HANDLER(CPU_EXC_DE, cpu_ll_exc_divide_error)
CPU_LL_EXC_HANDLER(CPU_EXC_DB, cpu_ll_exc_debug)
CPU_LL_INTR_HANDLER(CPU_EXC_NMI, cpu_ll_exc_nmi)
CPU_LL_EXC_HANDLER(CPU_EXC_BP, cpu_ll_exc_breakpoint)
CPU_LL_EXC_HANDLER(CPU_EXC_OF, cpu_ll_exc_overflow)
CPU_LL_EXC_HANDLER(CPU_EXC_BR, cpu_ll_exc_bound_range)
CPU_LL_EXC_HANDLER(CPU_EXC_UD, cpu_ll_exc_undefined_opcode)
CPU_LL_EXC_HANDLER(CPU_EXC_NM, cpu_ll_exc_no_math_coprocessor)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_DF, cpu_ll_exc_double_fault)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_TS, cpu_ll_exc_invalid_tss)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_NP, cpu_ll_exc_segment_not_present)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_SS, cpu_ll_exc_stack_segment_fault)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_GP, cpu_ll_exc_general_protection)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_PF, cpu_ll_exc_page_fault)
CPU_LL_EXC_HANDLER(CPU_EXC_MF, cpu_ll_exc_math_fault)
CPU_LL_EXC_HANDLER_WITH_ERROR(CPU_EXC_AC, cpu_ll_exc_alignment_check)
CPU_LL_INTR_HANDLER(CPU_EXC_MC, cpu_ll_exc_machine_check)
CPU_LL_EXC_HANDLER(CPU_EXC_XM, cpu_ll_exc_simd_fp_exception)

/*
 * Low level handlers for reserved exceptions.
 *
 * These exceptions should normally never occur, but have handlers ready just
 * in case.
 */
CPU_LL_EXC_HANDLER(9, cpu_ll_exc_9)
CPU_LL_EXC_HANDLER(15, cpu_ll_exc_15)
CPU_LL_EXC_HANDLER(20, cpu_ll_exc_20)
CPU_LL_EXC_HANDLER(21, cpu_ll_exc_21)
CPU_LL_EXC_HANDLER(22, cpu_ll_exc_22)
CPU_LL_EXC_HANDLER(23, cpu_ll_exc_23)
CPU_LL_EXC_HANDLER(24, cpu_ll_exc_24)
CPU_LL_EXC_HANDLER(25, cpu_ll_exc_25)
CPU_LL_EXC_HANDLER(26, cpu_ll_exc_26)
CPU_LL_EXC_HANDLER(27, cpu_ll_exc_27)
CPU_LL_EXC_HANDLER(28, cpu_ll_exc_28)
CPU_LL_EXC_HANDLER(29, cpu_ll_exc_29)
CPU_LL_EXC_HANDLER(30, cpu_ll_exc_30)
CPU_LL_EXC_HANDLER(31, cpu_ll_exc_31)

/* Generic low level interrupt handlers */
CPU_LL_INTR_HANDLER(32, cpu_ll_exc_32)
CPU_LL_INTR_HANDLER(33, cpu_ll_exc_33)
CPU_LL_INTR_HANDLER(34, cpu_ll_exc_34)
CPU_LL_INTR_HANDLER(35, cpu_ll_exc_35)
CPU_LL_INTR_HANDLER(36, cpu_ll_exc_36)
CPU_LL_INTR_HANDLER(37, cpu_ll_exc_37)
CPU_LL_INTR_HANDLER(38, cpu_ll_exc_38)
CPU_LL_INTR_HANDLER(39, cpu_ll_exc_39)
CPU_LL_INTR_HANDLER(40, cpu_ll_exc_40)
CPU_LL_INTR_HANDLER(41, cpu_ll_exc_41)
CPU_LL_INTR_HANDLER(42, cpu_ll_exc_42)
CPU_LL_INTR_HANDLER(43, cpu_ll_exc_43)
CPU_LL_INTR_HANDLER(44, cpu_ll_exc_44)
CPU_LL_INTR_HANDLER(45, cpu_ll_exc_45)
CPU_LL_INTR_HANDLER(46, cpu_ll_exc_46)
CPU_LL_INTR_HANDLER(47, cpu_ll_exc_47)
CPU_LL_INTR_HANDLER(48, cpu_ll_exc_48)
CPU_LL_INTR_HANDLER(49, cpu_ll_exc_49)
CPU_LL_INTR_HANDLER(50, cpu_ll_exc_50)
CPU_LL_INTR_HANDLER(51, cpu_ll_exc_51)
CPU_LL_INTR_HANDLER(52, cpu_ll_exc_52)
CPU_LL_INTR_HANDLER(53, cpu_ll_exc_53)
CPU_LL_INTR_HANDLER(54, cpu_ll_exc_54)
CPU_LL_INTR_HANDLER(55, cpu_ll_exc_55)
CPU_LL_INTR_HANDLER(56, cpu_ll_exc_56)
CPU_LL_INTR_HANDLER(57, cpu_ll_exc_57)
CPU_LL_INTR_HANDLER(58, cpu_ll_exc_58)
CPU_LL_INTR_HANDLER(59, cpu_ll_exc_59)
CPU_LL_INTR_HANDLER(60, cpu_ll_exc_60)
CPU_LL_INTR_HANDLER(61, cpu_ll_exc_61)
CPU_LL_INTR_HANDLER(62, cpu_ll_exc_62)
CPU_LL_INTR_HANDLER(63, cpu_ll_exc_63)
CPU_LL_INTR_HANDLER(64, cpu_ll_exc_64)
CPU_LL_INTR_HANDLER(65, cpu_ll_exc_65)
CPU_LL_INTR_HANDLER(66, cpu_ll_exc_66)
CPU_LL_INTR_HANDLER(67, cpu_ll_exc_67)
CPU_LL_INTR_HANDLER(68, cpu_ll_exc_68)
CPU_LL_INTR_HANDLER(69, cpu_ll_exc_69)
CPU_LL_INTR_HANDLER(70, cpu_ll_exc_70)
CPU_LL_INTR_HANDLER(71, cpu_ll_exc_71)
CPU_LL_INTR_HANDLER(72, cpu_ll_exc_72)
CPU_LL_INTR_HANDLER(73, cpu_ll_exc_73)
CPU_LL_INTR_HANDLER(74, cpu_ll_exc_74)
CPU_LL_INTR_HANDLER(75, cpu_ll_exc_75)
CPU_LL_INTR_HANDLER(76, cpu_ll_exc_76)
CPU_LL_INTR_HANDLER(77, cpu_ll_exc_77)
CPU_LL_INTR_HANDLER(78, cpu_ll_exc_78)
CPU_LL_INTR_HANDLER(79, cpu_ll_exc_79)
CPU_LL_INTR_HANDLER(80, cpu_ll_exc_80)
CPU_LL_INTR_HANDLER(81, cpu_ll_exc_81)
CPU_LL_INTR_HANDLER(82, cpu_ll_exc_82)
CPU_LL_INTR_HANDLER(83, cpu_ll_exc_83)
CPU_LL_INTR_HANDLER(84, cpu_ll_exc_84)
CPU_LL_INTR_HANDLER(85, cpu_ll_exc_85)
CPU_LL_INTR_HANDLER(86, cpu_ll_exc_86)
CPU_LL_INTR_HANDLER(87, cpu_ll_exc_87)
CPU_LL_INTR_HANDLER(88, cpu_ll_exc_88)
CPU_LL_INTR_HANDLER(89, cpu_ll_exc_89)
CPU_LL_INTR_HANDLER(90, cpu_ll_exc_90)
CPU_LL_INTR_HANDLER(91, cpu_ll_exc_91)
CPU_LL_INTR_HANDLER(92, cpu_ll_exc_92)
CPU_LL_INTR_HANDLER(93, cpu_ll_exc_93)
CPU_LL_INTR_HANDLER(94, cpu_ll_exc_94)
CPU_LL_INTR_HANDLER(95, cpu_ll_exc_95)
CPU_LL_INTR_HANDLER(96, cpu_ll_exc_96)
CPU_LL_INTR_HANDLER(97, cpu_ll_exc_97)
CPU_LL_INTR_HANDLER(98, cpu_ll_exc_98)
CPU_LL_INTR_HANDLER(99, cpu_ll_exc_99)
CPU_LL_INTR_HANDLER(100, cpu_ll_exc_100)
CPU_LL_INTR_HANDLER(101, cpu_ll_exc_101)
CPU_LL_INTR_HANDLER(102, cpu_ll_exc_102)
CPU_LL_INTR_HANDLER(103, cpu_ll_exc_103)
CPU_LL_INTR_HANDLER(104, cpu_ll_exc_104)
CPU_LL_INTR_HANDLER(105, cpu_ll_exc_105)
CPU_LL_INTR_HANDLER(106, cpu_ll_exc_106)
CPU_LL_INTR_HANDLER(107, cpu_ll_exc_107)
CPU_LL_INTR_HANDLER(108, cpu_ll_exc_108)
CPU_LL_INTR_HANDLER(109, cpu_ll_exc_109)
CPU_LL_INTR_HANDLER(110, cpu_ll_exc_110)
CPU_LL_INTR_HANDLER(111, cpu_ll_exc_111)
CPU_LL_INTR_HANDLER(112, cpu_ll_exc_112)
CPU_LL_INTR_HANDLER(113, cpu_ll_exc_113)
CPU_LL_INTR_HANDLER(114, cpu_ll_exc_114)
CPU_LL_INTR_HANDLER(115, cpu_ll_exc_115)
CPU_LL_INTR_HANDLER(116, cpu_ll_exc_116)
CPU_LL_INTR_HANDLER(117, cpu_ll_exc_117)
CPU_LL_INTR_HANDLER(118, cpu_ll_exc_118)
CPU_LL_INTR_HANDLER(119, cpu_ll_exc_119)
CPU_LL_INTR_HANDLER(120, cpu_ll_exc_120)
CPU_LL_INTR_HANDLER(121, cpu_ll_exc_121)
CPU_LL_INTR_HANDLER(122, cpu_ll_exc_122)
CPU_LL_INTR_HANDLER(123, cpu_ll_exc_123)
CPU_LL_INTR_HANDLER(124, cpu_ll_exc_124)
CPU_LL_INTR_HANDLER(125, cpu_ll_exc_125)
CPU_LL_INTR_HANDLER(126, cpu_ll_exc_126)
CPU_LL_INTR_HANDLER(127, cpu_ll_exc_127)
CPU_LL_INTR_HANDLER(128, cpu_ll_exc_128)
CPU_LL_INTR_HANDLER(129, cpu_ll_exc_129)
CPU_LL_INTR_HANDLER(130, cpu_ll_exc_130)
CPU_LL_INTR_HANDLER(131, cpu_ll_exc_131)
CPU_LL_INTR_HANDLER(132, cpu_ll_exc_132)
CPU_LL_INTR_HANDLER(133, cpu_ll_exc_133)
CPU_LL_INTR_HANDLER(134, cpu_ll_exc_134)
CPU_LL_INTR_HANDLER(135, cpu_ll_exc_135)
CPU_LL_INTR_HANDLER(136, cpu_ll_exc_136)
CPU_LL_INTR_HANDLER(137, cpu_ll_exc_137)
CPU_LL_INTR_HANDLER(138, cpu_ll_exc_138)
CPU_LL_INTR_HANDLER(139, cpu_ll_exc_139)
CPU_LL_INTR_HANDLER(140, cpu_ll_exc_140)
CPU_LL_INTR_HANDLER(141, cpu_ll_exc_141)
CPU_LL_INTR_HANDLER(142, cpu_ll_exc_142)
CPU_LL_INTR_HANDLER(143, cpu_ll_exc_143)
CPU_LL_INTR_HANDLER(144, cpu_ll_exc_144)
CPU_LL_INTR_HANDLER(145, cpu_ll_exc_145)
CPU_LL_INTR_HANDLER(146, cpu_ll_exc_146)
CPU_LL_INTR_HANDLER(147, cpu_ll_exc_147)
CPU_LL_INTR_HANDLER(148, cpu_ll_exc_148)
CPU_LL_INTR_HANDLER(149, cpu_ll_exc_149)
CPU_LL_INTR_HANDLER(150, cpu_ll_exc_150)
CPU_LL_INTR_HANDLER(151, cpu_ll_exc_151)
CPU_LL_INTR_HANDLER(152, cpu_ll_exc_152)
CPU_LL_INTR_HANDLER(153, cpu_ll_exc_153)
CPU_LL_INTR_HANDLER(154, cpu_ll_exc_154)
CPU_LL_INTR_HANDLER(155, cpu_ll_exc_155)
CPU_LL_INTR_HANDLER(156, cpu_ll_exc_156)
CPU_LL_INTR_HANDLER(157, cpu_ll_exc_157)
CPU_LL_INTR_HANDLER(158, cpu_ll_exc_158)
CPU_LL_INTR_HANDLER(159, cpu_ll_exc_159)
CPU_LL_INTR_HANDLER(160, cpu_ll_exc_160)
CPU_LL_INTR_HANDLER(161, cpu_ll_exc_161)
CPU_LL_INTR_HANDLER(162, cpu_ll_exc_162)
CPU_LL_INTR_HANDLER(163, cpu_ll_exc_163)
CPU_LL_INTR_HANDLER(164, cpu_ll_exc_164)
CPU_LL_INTR_HANDLER(165, cpu_ll_exc_165)
CPU_LL_INTR_HANDLER(166, cpu_ll_exc_166)
CPU_LL_INTR_HANDLER(167, cpu_ll_exc_167)
CPU_LL_INTR_HANDLER(168, cpu_ll_exc_168)
CPU_LL_INTR_HANDLER(169, cpu_ll_exc_169)
CPU_LL_INTR_HANDLER(170, cpu_ll_exc_170)
CPU_LL_INTR_HANDLER(171, cpu_ll_exc_171)
CPU_LL_INTR_HANDLER(172, cpu_ll_exc_172)
CPU_LL_INTR_HANDLER(173, cpu_ll_exc_173)
CPU_LL_INTR_HANDLER(174, cpu_ll_exc_174)
CPU_LL_INTR_HANDLER(175, cpu_ll_exc_175)
CPU_LL_INTR_HANDLER(176, cpu_ll_exc_176)
CPU_LL_INTR_HANDLER(177, cpu_ll_exc_177)
CPU_LL_INTR_HANDLER(178, cpu_ll_exc_178)
CPU_LL_INTR_HANDLER(179, cpu_ll_exc_179)
CPU_LL_INTR_HANDLER(180, cpu_ll_exc_180)
CPU_LL_INTR_HANDLER(181, cpu_ll_exc_181)
CPU_LL_INTR_HANDLER(182, cpu_ll_exc_182)
CPU_LL_INTR_HANDLER(183, cpu_ll_exc_183)
CPU_LL_INTR_HANDLER(184, cpu_ll_exc_184)
CPU_LL_INTR_HANDLER(185, cpu_ll_exc_185)
CPU_LL_INTR_HANDLER(186, cpu_ll_exc_186)
CPU_LL_INTR_HANDLER(187, cpu_ll_exc_187)
CPU_LL_INTR_HANDLER(188, cpu_ll_exc_188)
CPU_LL_INTR_HANDLER(189, cpu_ll_exc_189)
CPU_LL_INTR_HANDLER(190, cpu_ll_exc_190)
CPU_LL_INTR_HANDLER(191, cpu_ll_exc_191)
CPU_LL_INTR_HANDLER(192, cpu_ll_exc_192)
CPU_LL_INTR_HANDLER(193, cpu_ll_exc_193)
CPU_LL_INTR_HANDLER(194, cpu_ll_exc_194)
CPU_LL_INTR_HANDLER(195, cpu_ll_exc_195)
CPU_LL_INTR_HANDLER(196, cpu_ll_exc_196)
CPU_LL_INTR_HANDLER(197, cpu_ll_exc_197)
CPU_LL_INTR_HANDLER(198, cpu_ll_exc_198)
CPU_LL_INTR_HANDLER(199, cpu_ll_exc_199)
CPU_LL_INTR_HANDLER(200, cpu_ll_exc_200)
CPU_LL_INTR_HANDLER(201, cpu_ll_exc_201)
CPU_LL_INTR_HANDLER(202, cpu_ll_exc_202)
CPU_LL_INTR_HANDLER(203, cpu_ll_exc_203)
CPU_LL_INTR_HANDLER(204, cpu_ll_exc_204)
CPU_LL_INTR_HANDLER(205, cpu_ll_exc_205)
CPU_LL_INTR_HANDLER(206, cpu_ll_exc_206)
CPU_LL_INTR_HANDLER(207, cpu_ll_exc_207)
CPU_LL_INTR_HANDLER(208, cpu_ll_exc_208)
CPU_LL_INTR_HANDLER(209, cpu_ll_exc_209)
CPU_LL_INTR_HANDLER(210, cpu_ll_exc_210)
CPU_LL_INTR_HANDLER(211, cpu_ll_exc_211)
CPU_LL_INTR_HANDLER(212, cpu_ll_exc_212)
CPU_LL_INTR_HANDLER(213, cpu_ll_exc_213)
CPU_LL_INTR_HANDLER(214, cpu_ll_exc_214)
CPU_LL_INTR_HANDLER(215, cpu_ll_exc_215)
CPU_LL_INTR_HANDLER(216, cpu_ll_exc_216)
CPU_LL_INTR_HANDLER(217, cpu_ll_exc_217)
CPU_LL_INTR_HANDLER(218, cpu_ll_exc_218)
CPU_LL_INTR_HANDLER(219, cpu_ll_exc_219)
CPU_LL_INTR_HANDLER(220, cpu_ll_exc_220)
CPU_LL_INTR_HANDLER(221, cpu_ll_exc_221)
CPU_LL_INTR_HANDLER(222, cpu_ll_exc_222)
CPU_LL_INTR_HANDLER(223, cpu_ll_exc_223)
CPU_LL_INTR_HANDLER(224, cpu_ll_exc_224)
CPU_LL_INTR_HANDLER(225, cpu_ll_exc_225)
CPU_LL_INTR_HANDLER(226, cpu_ll_exc_226)
CPU_LL_INTR_HANDLER(227, cpu_ll_exc_227)
CPU_LL_INTR_HANDLER(228, cpu_ll_exc_228)
CPU_LL_INTR_HANDLER(229, cpu_ll_exc_229)
CPU_LL_INTR_HANDLER(230, cpu_ll_exc_230)
CPU_LL_INTR_HANDLER(231, cpu_ll_exc_231)
CPU_LL_INTR_HANDLER(232, cpu_ll_exc_232)
CPU_LL_INTR_HANDLER(233, cpu_ll_exc_233)
CPU_LL_INTR_HANDLER(234, cpu_ll_exc_234)
CPU_LL_INTR_HANDLER(235, cpu_ll_exc_235)
CPU_LL_INTR_HANDLER(236, cpu_ll_exc_236)
CPU_LL_INTR_HANDLER(237, cpu_ll_exc_237)
CPU_LL_INTR_HANDLER(238, cpu_ll_exc_238)
CPU_LL_INTR_HANDLER(239, cpu_ll_exc_239)
CPU_LL_INTR_HANDLER(240, cpu_ll_exc_240)
CPU_LL_INTR_HANDLER(241, cpu_ll_exc_241)
CPU_LL_INTR_HANDLER(242, cpu_ll_exc_242)
CPU_LL_INTR_HANDLER(243, cpu_ll_exc_243)
CPU_LL_INTR_HANDLER(244, cpu_ll_exc_244)
CPU_LL_INTR_HANDLER(245, cpu_ll_exc_245)
CPU_LL_INTR_HANDLER(246, cpu_ll_exc_246)
CPU_LL_INTR_HANDLER(247, cpu_ll_exc_247)
CPU_LL_INTR_HANDLER(248, cpu_ll_exc_248)
CPU_LL_INTR_HANDLER(249, cpu_ll_exc_249)
CPU_LL_INTR_HANDLER(250, cpu_ll_exc_250)
CPU_LL_INTR_HANDLER(251, cpu_ll_exc_251)
CPU_LL_INTR_HANDLER(252, cpu_ll_exc_252)
CPU_LL_INTR_HANDLER(253, cpu_ll_exc_253)
CPU_LL_INTR_HANDLER(254, cpu_ll_exc_254)
CPU_LL_INTR_HANDLER(255, cpu_ll_exc_255)

#ifdef __LP64__
#define CPU_LL_EXC_HANDLER_ADDR(name) .quad name
#else /* __LP64__ */
#define CPU_LL_EXC_HANDLER_ADDR(name) .long name
#endif /* __LP64__ */

.section INIT_DATA_SECTION

/* See the C declaration */
ASM_DATA(cpu_ll_exc_handler_addrs)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_divide_error)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_debug)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_nmi)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_breakpoint)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_overflow)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_bound_range)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_undefined_opcode)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_no_math_coprocessor)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_double_fault)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_9)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_invalid_tss)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_segment_not_present)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_stack_segment_fault)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_general_protection)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_page_fault)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_15)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_math_fault)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_alignment_check)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_machine_check)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_simd_fp_exception)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_20)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_21)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_22)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_23)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_24)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_25)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_26)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_27)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_28)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_29)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_30)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_31)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_32)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_33)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_34)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_35)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_36)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_37)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_38)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_39)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_40)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_41)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_42)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_43)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_44)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_45)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_46)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_47)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_48)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_49)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_50)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_51)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_52)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_53)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_54)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_55)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_56)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_57)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_58)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_59)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_60)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_61)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_62)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_63)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_64)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_65)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_66)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_67)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_68)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_69)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_70)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_71)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_72)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_73)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_74)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_75)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_76)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_77)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_78)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_79)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_80)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_81)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_82)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_83)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_84)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_85)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_86)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_87)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_88)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_89)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_90)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_91)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_92)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_93)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_94)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_95)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_96)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_97)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_98)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_99)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_100)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_101)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_102)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_103)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_104)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_105)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_106)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_107)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_108)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_109)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_110)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_111)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_112)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_113)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_114)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_115)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_116)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_117)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_118)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_119)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_120)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_121)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_122)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_123)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_124)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_125)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_126)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_127)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_128)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_129)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_130)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_131)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_132)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_133)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_134)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_135)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_136)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_137)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_138)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_139)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_140)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_141)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_142)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_143)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_144)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_145)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_146)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_147)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_148)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_149)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_150)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_151)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_152)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_153)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_154)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_155)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_156)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_157)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_158)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_159)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_160)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_161)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_162)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_163)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_164)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_165)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_166)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_167)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_168)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_169)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_170)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_171)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_172)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_173)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_174)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_175)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_176)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_177)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_178)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_179)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_180)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_181)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_182)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_183)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_184)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_185)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_186)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_187)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_188)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_189)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_190)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_191)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_192)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_193)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_194)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_195)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_196)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_197)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_198)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_199)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_200)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_201)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_202)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_203)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_204)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_205)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_206)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_207)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_208)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_209)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_210)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_211)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_212)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_213)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_214)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_215)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_216)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_217)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_218)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_219)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_220)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_221)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_222)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_223)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_224)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_225)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_226)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_227)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_228)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_229)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_230)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_231)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_232)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_233)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_234)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_235)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_236)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_237)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_238)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_239)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_240)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_241)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_242)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_243)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_244)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_245)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_246)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_247)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_248)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_249)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_250)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_251)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_252)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_253)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_254)
    CPU_LL_EXC_HANDLER_ADDR(cpu_ll_exc_255)
ASM_END(cpu_ll_exc_handler_addrs)

ASM_ENTRY(cpu_load_gdt)
#ifdef __LP64__
    lgdt (%rdi)
#else /* __LP64__ */
    movl 4(%esp), %eax
    lgdt (%eax)
#endif /* __LP64__ */

    movl $CPU_GDT_SEL_DATA, %eax
    movl %eax, %ds
    movl %eax, %es
    movl %eax, %ss

    /* Alter the stack to reload the code segment using a far return */
#ifdef __LP64__
    popq %rax
    pushq $CPU_GDT_SEL_CODE
    pushq %rax
    lretq
#else /* __LP64__ */
    popl %eax
    pushl $CPU_GDT_SEL_CODE
    pushl %eax
    lret
#endif /* __LP64__ */
ASM_END(cpu_load_gdt)


code:
/*
 * Copyright (c) 2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_CPU_I_H
#define X86_CPU_I_H

/*
 * Exception frame offets.
 */

#ifdef __LP64__

#define CPU_EXC_FRAME_RAX       0
#define CPU_EXC_FRAME_RBX       1
#define CPU_EXC_FRAME_RCX       2
#define CPU_EXC_FRAME_RDX       3
#define CPU_EXC_FRAME_RBP       4
#define CPU_EXC_FRAME_RSI       5
#define CPU_EXC_FRAME_RDI       6
#define CPU_EXC_FRAME_R8        7
#define CPU_EXC_FRAME_R9        8
#define CPU_EXC_FRAME_R10       9
#define CPU_EXC_FRAME_R11       10
#define CPU_EXC_FRAME_R12       11
#define CPU_EXC_FRAME_R13       12
#define CPU_EXC_FRAME_R14       13
#define CPU_EXC_FRAME_R15       14
#define CPU_EXC_FRAME_VECTOR    15
#define CPU_EXC_FRAME_ERROR     16
#define CPU_EXC_FRAME_RIP       17
#define CPU_EXC_FRAME_CS        18
#define CPU_EXC_FRAME_RFLAGS    19
#define CPU_EXC_FRAME_RSP       20
#define CPU_EXC_FRAME_SS        21
#define CPU_EXC_FRAME_SIZE      22

#define CPU_EXC_FRAME_FP        CPU_EXC_FRAME_RBP
#define CPU_EXC_FRAME_SP        CPU_EXC_FRAME_RSP
#define CPU_EXC_FRAME_PC        CPU_EXC_FRAME_RIP

#else /* __LP64__ */

#define CPU_EXC_FRAME_EAX       0
#define CPU_EXC_FRAME_EBX       1
#define CPU_EXC_FRAME_ECX       2
#define CPU_EXC_FRAME_EDX       3
#define CPU_EXC_FRAME_EBP       4
#define CPU_EXC_FRAME_ESI       5
#define CPU_EXC_FRAME_EDI       6
#define CPU_EXC_FRAME_DS        7
#define CPU_EXC_FRAME_ES        8
#define CPU_EXC_FRAME_FS        9
#define CPU_EXC_FRAME_GS        10
#define CPU_EXC_FRAME_VECTOR    11
#define CPU_EXC_FRAME_ERROR     12
#define CPU_EXC_FRAME_EIP       13
#define CPU_EXC_FRAME_CS        14
#define CPU_EXC_FRAME_EFLAGS    15
#define CPU_EXC_FRAME_ESP       16
#define CPU_EXC_FRAME_SS        17 /* esp and ss are undefined if trapped in kernel */
#define CPU_EXC_FRAME_SIZE      18

#define CPU_EXC_FRAME_FP        CPU_EXC_FRAME_EBP
#define CPU_EXC_FRAME_SP        CPU_EXC_FRAME_ESP
#define CPU_EXC_FRAME_PC        CPU_EXC_FRAME_EIP

#endif /* __LP64__ */

/*
 * EFLAGS register flags.
 */
#define CPU_EFL_ONE 0x00000002  /* Reserved, must be set */
#define CPU_EFL_IF  0x00000200

/*
 * GDT segment selectors.
 *
 * Keep in mind that, on amd64, the size of a GDT entry referred to
 * by a selector depends on the descriptor type.
 */
#define CPU_GDT_SEL_NULL        0
#define CPU_GDT_SEL_CODE        8
#define CPU_GDT_SEL_DATA        16
#define CPU_GDT_SEL_TSS         24

#ifdef __LP64__
#define CPU_GDT_SIZE            40
#else /* __LP64__ */
#define CPU_GDT_SEL_DF_TSS      32
#define CPU_GDT_SEL_PERCPU      40
#define CPU_GDT_SEL_TLS         48
#define CPU_GDT_SIZE            56
#endif /* __LP64__ */

#ifndef __ASSEMBLER__

#include <stdalign.h>
#include <stdbool.h>
#include <stdint.h>

#include <kern/bitmap.h>

struct cpu_tss {
#ifdef __LP64__
    uint32_t reserved0;
    uint64_t rsp0;
    uint64_t rsp1;
    uint64_t rsp2;
    uint64_t ist[8];
    uint64_t reserved1;
    uint16_t reserved2;
#else /* __LP64__ */
    uint32_t link;
    uint32_t esp0;
    uint32_t ss0;
    uint32_t esp1;
    uint32_t ss1;
    uint32_t esp2;
    uint32_t ss2;
    uint32_t cr3;
    uint32_t eip;
    uint32_t eflags;
    uint32_t eax;
    uint32_t ecx;
    uint32_t edx;
    uint32_t ebx;
    uint32_t esp;
    uint32_t ebp;
    uint32_t esi;
    uint32_t edi;
    uint32_t es;
    uint32_t cs;
    uint32_t ss;
    uint32_t ds;
    uint32_t fs;
    uint32_t gs;
    uint32_t ldt;
    uint16_t trap_bit;
#endif /* __LP64__ */
    uint16_t iobp_base;
} __packed;

/*
 * LDT or TSS system segment descriptor.
 */
struct cpu_sysseg_desc {
    uint32_t word1;
    uint32_t word2;
#ifdef __LP64__
    uint32_t word3;
    uint32_t word4;
#endif /* __LP64__ */
};

struct cpu_gdt {
    alignas(CPU_L1_SIZE) char descs[CPU_GDT_SIZE];
};

#define CPU_VENDOR_ID_SIZE  13
#define CPU_MODEL_NAME_SIZE 49

struct cpu_feature_map {
    BITMAP_DECLARE(flags, CPU_NR_FEATURES);
};

struct cpu {
    unsigned int id;
    unsigned int apic_id;
    char vendor_str[CPU_VENDOR_STR_SIZE];
    char model_name[CPU_MODEL_NAME_SIZE];
    unsigned int cpuid_max_basic;
    unsigned int cpuid_max_extended;
    unsigned int vendor_id;
    unsigned int type;
    unsigned int family;
    unsigned int model;
    unsigned int stepping;
    unsigned int clflush_size;
    unsigned int initial_apic_id;
    struct cpu_feature_map feature_map;
    unsigned short phys_addr_width;
    unsigned short virt_addr_width;

    struct cpu_gdt gdt;

    /*
     * TSS segments, one set per CPU.
     *
     * One TSS at least is required per processor to provide the following :
     *  - stacks for double fault handlers, implemented with task switching
     *    on i386, interrupt stack tables on amd64
     *  - stacks for each privilege level
     *  - I/O permission bitmaps
     *
     * See Intel 64 and IA-32 Architecture Software Developer's Manual,
     * Volume 3 System Programming Guide :
     *  - 6.12.2 Interrupt tasks
     *  - 7.3 Task switching
     */
    struct cpu_tss tss;
#ifndef __LP64__
    struct cpu_tss df_tss;
#endif /* __LP64__ */

    unsigned int started;

    alignas(CPU_DATA_ALIGN) char intr_stack[CPU_INTR_STACK_SIZE];
    alignas(CPU_DATA_ALIGN) char df_stack[CPU_INTR_STACK_SIZE];
};

/*
 * This percpu variable contains the address of the percpu area for the local
 * processor. This is normally the same value stored in the percpu module, but
 * it can be directly accessed through a segment register.
 */
extern void *cpu_local_area;

static inline bool
cpu_feature_map_test(const struct cpu_feature_map *map,
                     enum cpu_feature feature)
{
    return bitmap_test(map->flags, feature);
}

/*
 * Return the content of the EFLAGS register.
 *
 * Implies a compiler barrier.
 *
 * TODO Add cpu_flags_t type.
 */
static __always_inline unsigned long
cpu_get_eflags(void)
{
    unsigned long eflags;

    asm volatile("pushf\n"
                 "pop %0\n"
                 : "=r" (eflags)
                 : : "memory");

    return eflags;
}

#endif /* __ASSEMBLER__ */

#endif /* X86_CPU_I_H */


code:
/*
 * Copyright (c) 2013 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_ELF_H
#define X86_ELF_H

#define ELF_SHT_SYMTAB  2
#define ELF_SHT_STRTAB  3

struct elf_shdr {
    unsigned int name;
    unsigned int type;
    unsigned int flags;
    unsigned long addr;
    unsigned long offset;
    unsigned int size;
    unsigned int link;
    unsigned int info;
    unsigned int addralign;
    unsigned int entsize;
};

#ifdef __LP64__

struct elf_sym {
    unsigned int name;
    unsigned char info;
    unsigned char other;
    unsigned short shndx;
    unsigned long value;
    unsigned long size;
};

#else /* __LP64__ */

struct elf_sym {
    unsigned int name;
    unsigned long value;
    unsigned long size;
    unsigned char info;
    unsigned char other;
    unsigned short shndx;
};

#endif /* __LP64__ */

#endif /* X86_ELF_H */


code:
/*
 * Copyright (c) 2010 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_IO_H
#define X86_IO_H

#include <stdint.h>

/*
 * Read a byte from an I/O port.
 */
static inline uint8_t
io_read_byte(uint16_t port)
{
    uint8_t value;

    asm volatile("inb %%dx, %%al" : "=a" (value) : "d" (port));
    return value;
}

/*
 * Write a byte to an I/O port.
 */
static inline void
io_write_byte(uint16_t port, uint8_t value)
{
    asm volatile("outb %%al, %%dx" : : "d" (port), "a" (value));
}

#endif /* X86_IO_H */