code:
/*
 * Copyright (c) 2010-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * TODO Comment.
 */

#ifndef X86_PMAP_H
#define X86_PMAP_H

#include <kern/macros.h>

/*
 * Virtual memory layout.
 */

/*
 * User space boundaries.
 */
#define PMAP_START_ADDRESS              DECL_CONST(0, UL)

#ifdef __LP64__
#define PMAP_END_ADDRESS                DECL_CONST(0x800000000000, UL)
#else /* __LP64__ */
#define PMAP_END_ADDRESS                DECL_CONST(0xc0000000, UL)
#endif/* __LP64__ */

/*
 * Kernel space boundaries.
 */
#ifdef __LP64__
#define PMAP_START_KERNEL_ADDRESS       DECL_CONST(0xffff800000000000, UL)
#define PMAP_END_KERNEL_ADDRESS         DECL_CONST(0xfffffffffffff000, UL)
#else /* __LP64__ */
#define PMAP_START_KERNEL_ADDRESS       PMAP_END_ADDRESS
#define PMAP_END_KERNEL_ADDRESS         DECL_CONST(0xfffff000, UL)
#endif /* __LP64__ */

/*
 * Direct physical mapping boundaries.
 */
#ifdef __LP64__
#define PMAP_START_DIRECTMAP_ADDRESS    PMAP_START_KERNEL_ADDRESS
#define PMAP_END_DIRECTMAP_ADDRESS      DECL_CONST(0xffffc00000000000, UL)
#else /* __LP64__ */
#define PMAP_START_DIRECTMAP_ADDRESS    PMAP_END_ADDRESS
#define PMAP_END_DIRECTMAP_ADDRESS      DECL_CONST(0xf8000000, UL)
#endif /* __LP64__ */

/*
 * Kernel mapping offset.
 *
 * On 32-bits systems, the kernel is linked at addresses included in the
 * direct physical mapping, whereas on 64-bits systems, it is linked at
 * -2 GiB because the "kernel" memory model is used when compiling (see
 * the -mcmodel=kernel gcc option).
 */
#ifdef __LP64__
#define PMAP_KERNEL_OFFSET              DECL_CONST(0xffffffff80000000, UL)
#else /* __LP64__ */
#define PMAP_KERNEL_OFFSET              PMAP_START_DIRECTMAP_ADDRESS
#endif /* __LP64__ */

/*
 * Kernel virtual space boundaries.
 *
 * In addition to the direct physical mapping, the kernel has its own virtual
 * memory space.
 */
#define PMAP_START_KMEM_ADDRESS         PMAP_END_DIRECTMAP_ADDRESS

#ifdef __LP64__
#define PMAP_END_KMEM_ADDRESS           PMAP_KERNEL_OFFSET
#else /* __LP64__ */
#define PMAP_END_KMEM_ADDRESS           PMAP_END_KERNEL_ADDRESS
#endif /* __LP64__ */

/*
 * Page table entry flags.
 */
#define PMAP_PTE_P      0x00000001
#define PMAP_PTE_RW     0x00000002
#define PMAP_PTE_US     0x00000004
#define PMAP_PTE_PWT    0x00000008
#define PMAP_PTE_PCD    0x00000010
#define PMAP_PTE_A      0x00000020
#define PMAP_PTE_D      0x00000040
#define PMAP_PTE_PS     0x00000080
#define PMAP_PTE_G      0x00000100

/*
 * Page translation hierarchy properties.
 */

/*
 * Masks define valid bits at each page translation level.
 *
 * Additional bits such as the global bit can be added at runtime for optional
 * features.
 */
#define PMAP_L0_MASK    (PMAP_PA_MASK | PMAP_PTE_D | PMAP_PTE_A \
                         | PMAP_PTE_PCD | PMAP_PTE_PWT | PMAP_PTE_US \
                         | PMAP_PTE_RW | PMAP_PTE_P)
#define PMAP_L1_MASK    (PMAP_PA_MASK | PMAP_PTE_A | PMAP_PTE_PCD \
                         | PMAP_PTE_PWT | PMAP_PTE_US | PMAP_PTE_RW \
                         | PMAP_PTE_P)

#define PMAP_PAE_L2_MASK    (PMAP_PA_MASK | PMAP_PTE_PCD | PMAP_PTE_PWT \
                             | PMAP_PTE_P)

#ifdef __LP64__
#define PMAP_NR_LEVELS  4
#define PMAP_L0_BITS    9
#define PMAP_L1_BITS    9
#define PMAP_L2_BITS    9
#define PMAP_L3_BITS    9
#define PMAP_VA_MASK    DECL_CONST(0x0000ffffffffffff, UL)
#define PMAP_PA_MASK    DECL_CONST(0x000ffffffffff000, UL)
#define PMAP_L2_MASK    PMAP_L1_MASK
#define PMAP_L3_MASK    PMAP_L1_MASK
#else /* __LP64__ */
#ifdef CONFIG_X86_PAE
#define PMAP_NR_LEVELS  3
#define PMAP_L0_BITS    9
#define PMAP_L1_BITS    9
#define PMAP_L2_BITS    2
#define PMAP_VA_MASK    DECL_CONST(0xffffffff, UL)
#define PMAP_PA_MASK    DECL_CONST(0x000ffffffffff000, ULL)
#define PMAP_L2_MASK    PMAP_PAE_L2_MASK
#else /* CONFIG_X86_PAE */
#define PMAP_NR_LEVELS  2
#define PMAP_L0_BITS    10
#define PMAP_L1_BITS    10
#define PMAP_VA_MASK    DECL_CONST(0xffffffff, UL)
#define PMAP_PA_MASK    DECL_CONST(0xfffff000, UL)
#endif /* CONFIG_X86_PAE */
#endif /* __LP64__ */

#define PMAP_L0_SKIP    12
#define PMAP_L1_SKIP    (PMAP_L0_SKIP + PMAP_L0_BITS)
#define PMAP_L2_SKIP    (PMAP_L1_SKIP + PMAP_L1_BITS)
#define PMAP_L3_SKIP    (PMAP_L2_SKIP + PMAP_L2_BITS)

#define PMAP_L0_PTES_PER_PT     (1 << PMAP_L0_BITS)
#define PMAP_L1_PTES_PER_PT     (1 << PMAP_L1_BITS)
#define PMAP_L2_PTES_PER_PT     (1 << PMAP_L2_BITS)
#define PMAP_L3_PTES_PER_PT     (1 << PMAP_L3_BITS)

#ifndef __ASSEMBLER__

#include <stdint.h>

#include <kern/cpumap.h>
#include <kern/init.h>
#include <kern/list.h>
#include <kern/mutex.h>
#include <kern/thread.h>
#include <machine/cpu.h>
#include <machine/types.h>

/*
 * Mapping creation flags.
 */
#define PMAP_PEF_GLOBAL 0x1 /* Create a mapping on all processors */

typedef phys_addr_t pmap_pte_t;

/*
 * Physical address map.
 */
struct pmap;

static inline struct pmap *
pmap_get_kernel_pmap(void)
{
    extern struct pmap pmap_kernel_pmap;

    return &pmap_kernel_pmap;
}

/*
 * Early initialization of the MMU.
 *
 * This function is called before paging is enabled by the boot module. It
 * maps the kernel at physical and virtual addresses, after which all kernel
 * functions and data can be accessed.
 */
pmap_pte_t * pmap_setup_paging(void);

/*
 * This function is called by the AP bootstrap code before paging is enabled.
 */
pmap_pte_t * pmap_ap_setup_paging(unsigned int ap_id);

/*
 * Initialize the pmap module on APs.
 */
void pmap_ap_setup(void);

/*
 * Set up the pmap module for multiprocessor operations.
 *
 * This function copies the current page tables so that each processor has
 * its own set of page tables. As a result, it must be called right before
 * starting APs to make sure all processors have the same mappings.
 *
 * This function must be called before starting the scheduler whatever the
 * number of processors.
 */
void pmap_mp_setup(void);

/*
 * Build/clean up pmap thread-local data for the given thread.
 */
int pmap_thread_build(struct thread *thread);
void pmap_thread_cleanup(struct thread *thread);

/*
 * Extract a mapping from the kernel map.
 *
 * This function walks the page tables to retrieve the physical address
 * mapped at the given virtual address.
 */
int pmap_kextract(uintptr_t va, phys_addr_t *pap);

/*
 * Create a pmap for a user task.
 */
int pmap_create(struct pmap **pmapp);

/*
 * Create a mapping on a physical map.
 *
 * If protection is VM_PROT_NONE, this function behaves as if it were
 * VM_PROT_READ. There must not be an existing valid mapping for the given
 * virtual address.
 *
 * If the mapping is local, it is the responsibility of the caller to take
 * care of migration.
 *
 * This function may trigger an implicit update.
 */
int pmap_enter(struct pmap *pmap, uintptr_t va, phys_addr_t pa,
               int prot, int flags);

/*
 * Remove a mapping from a physical map.
 *
 * The caller may use this function on non-existent mappings.
 *
 * This function may trigger an implicit update.
 */
int pmap_remove(struct pmap *pmap, uintptr_t va,
                const struct cpumap *cpumap);

/*
 * Set the protection of a mapping in a physical map.
 *
 * This function may trigger an implicit update.
 */
int pmap_protect(struct pmap *pmap, uintptr_t va, int prot,
                 const struct cpumap *cpumap);

/*
 * Force application of pending modifications on a physical map.
 *
 * The functions that may defer physical map modifications are :
 *  - pmap_enter
 *  - pmap_remove
 *  - pmap_protect
 *
 * On return, if successful, all operations previously performed by the
 * calling thread are guaranteed to be applied on their respective
 * processors. Note that this function doesn't guarantee that modifications
 * performed by different threads are applied.
 *
 * If an error occurs, then some or all of the pending modifications
 * could not be applied. This function lacks the knowledge to handle
 * such cases. As a result, the caller is responsible for the complete
 * set of affected mappings and must take appropriate actions to restore
 * physical mappings consistency. Note that multiple errors may occur
 * when calling this function. The caller shouldn't rely on the specific
 * error value, and should consider the whole operation to have failed.
 *
 * Also note that the only operation that may fail is mapping creation.
 * Therefore, if the caller only queues removals or protection changes
 * between two calls to this function, it is guaranteed to succeed.
 */
int pmap_update(struct pmap *pmap);

/*
 * Load the given pmap on the current processor.
 *
 * This function must be called with interrupts and preemption disabled.
 */
void pmap_load(struct pmap *pmap);

/*
 * Return the pmap currently loaded on the processor.
 *
 * Since threads may borrow pmaps, this can be different than the pmap
 * of the caller.
 */
static inline struct pmap *
pmap_current(void)
{
    extern struct pmap *pmap_current_ptr;
    return cpu_local_read(pmap_current_ptr);
}

/*
 * This init operation provides :
 *  - kernel pmap operations
 */
INIT_OP_DECLARE(pmap_bootstrap);

/*
 * This init operation provides :
 *  - user pmap creation
 *  - module fully initialized
 */
INIT_OP_DECLARE(pmap_setup);

#endif /* __ASSEMBLER__ */

#endif /* X86_PMAP_H */


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Physical memory layout.
 *
 * This file is a top header in the inclusion hierarchy, and shouldn't include
 * other headers that may cause circular dependencies.
 */

#ifndef X86_PMEM_H
#define X86_PMEM_H

#include <kern/macros.h>

/*
 * Zone boundaries.
 */

#define PMEM_DMA_LIMIT          DECL_CONST(0x1000000, UL)

#ifdef __LP64__
#define PMEM_MAX_ZONES          4
#define PMEM_DMA32_LIMIT        DECL_CONST(0x100000000, UL)
#define PMEM_DIRECTMAP_LIMIT    DECL_CONST(0x400000000000, UL)
#define PMEM_HIGHMEM_LIMIT      DECL_CONST(0x10000000000000, UL)
#else /* __LP64__ */
#define PMEM_DIRECTMAP_LIMIT    DECL_CONST(0x38000000, ULL)
#ifdef CONFIG_X86_PAE
#define PMEM_MAX_ZONES          3
#define PMEM_HIGHMEM_LIMIT      DECL_CONST(0x10000000000000, ULL)
#else /* CONFIG_X86_PAE */
#define PMEM_MAX_ZONES          3
#define PMEM_HIGHMEM_LIMIT      DECL_CONST(0xfffff000, UL)
#endif /* CONFIG_X86_PAE */
#endif /* __LP64__ */

/*
 * Zone vm_page indexes.
 */

#define PMEM_ZONE_DMA           0
#define PMEM_ZONE_DMA32         1

#ifdef __LP64__
#define PMEM_ZONE_DIRECTMAP     2
#define PMEM_ZONE_HIGHMEM       3
#else /* __LP64__ */
#define PMEM_ZONE_DMA32         1
#define PMEM_ZONE_DIRECTMAP     1   /* Alias for the DMA32 zone */
#define PMEM_ZONE_HIGHMEM       2
#endif /* __LP64__ */

#endif /* X86_PMEM_H */

license: mit


code:
/*
 * Copyright (c) 2014-2018 Remy Noel.
 * Copyright (c) 2014-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <errno.h>
#include <stdint.h>

#include <kern/clock.h>
#include <kern/init.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/perfmon.h>
#include <machine/cpu.h>
#include <machine/pmu_amd.h>

/*
 * AMD raw event IDs.
 */
#define PMU_AMD_RE_CYCLE            0
#define PMU_AMD_RE_INSTRUCTION      1
#define PMU_AMD_RE_CACHE_REF        2
#define PMU_AMD_RE_CACHE_MISS       3
#define PMU_AMD_RE_BRANCH           4
#define PMU_AMD_RE_BRANCH_MISS      5
#define PMU_AMD_RE_DCACHE_REF       6
#define PMU_AMD_RE_DCACHE_MISS      7
#define PMU_AMD_RE_IFETCH_STALL     8
#define PMU_AMD_RE_INVALID          ((unsigned int)-1)

/*
 * PMU MSR addresses
 */
#define PMU_AMD_MSR_PERFEVTSEL0 0xc0010000
#define PMU_AMD_MSR_PERCTR0     0xc0010004

/*
 * Event Select Register addresses
 */
#define PMU_AMD_EVTSEL_USR  0x00010000
#define PMU_AMD_EVTSEL_OS   0x00020000
#define PMU_AMD_EVTSEL_INT  0x00100000
#define PMU_AMD_EVTSEL_EN   0x00400000

/*
 * XXX These properties have the minimum values required by the architecture.
 * TODO Per-family/model event availability database.
 */
#define PMU_AMD_NR_PMCS     4
#define PMU_AMD_PMC_WIDTH   48

/*
 * Global PMU properties.
 *
 * The bitmap is used to implement counter allocation, where each bit denotes
 * whether a counter is available or not.
 */
struct pmu_amd {
    unsigned int pmc_bm;
};

static struct pmu_amd pmu_amd;

struct pmu_amd_event_code {
    unsigned short event_select;
    unsigned short umask;
};

/*
 * TODO Per-family/model event availability database.
 */
static const struct pmu_amd_event_code pmu_amd_event_codes[] = {
    [PMU_AMD_RE_CYCLE]          = { 0x76, 0x00 },
    [PMU_AMD_RE_INSTRUCTION]    = { 0xc0, 0x00 },
    [PMU_AMD_RE_CACHE_REF]      = { 0x80, 0x00 },
    [PMU_AMD_RE_CACHE_MISS]     = { 0x81, 0x00 },
    [PMU_AMD_RE_BRANCH]         = { 0xc2, 0x00 },
    [PMU_AMD_RE_BRANCH_MISS]    = { 0xc3, 0x00 },
    [PMU_AMD_RE_DCACHE_REF]     = { 0x40, 0x00 },
    [PMU_AMD_RE_DCACHE_MISS]    = { 0x41, 0x00 },
    [PMU_AMD_RE_IFETCH_STALL]   = { 0x87, 0x00 },
};

static const unsigned int pmu_amd_generic_events[] = {
    [PERFMON_EV_CYCLE]          = PMU_AMD_RE_CYCLE,
    [PERFMON_EV_REF_CYCLE]      = PMU_AMD_RE_INVALID,
    [PERFMON_EV_INSTRUCTION]    = PMU_AMD_RE_INSTRUCTION,
    [PERFMON_EV_CACHE_REF]      = PMU_AMD_RE_CACHE_REF,
    [PERFMON_EV_CACHE_MISS]     = PMU_AMD_RE_CACHE_MISS,
    [PERFMON_EV_BRANCH]         = PMU_AMD_RE_BRANCH,
    [PERFMON_EV_BRANCH_MISS]    = PMU_AMD_RE_BRANCH_MISS,
};

static struct pmu_amd *
pmu_amd_get(void)
{
    return &pmu_amd;
}

static int
pmu_amd_translate(unsigned int *raw_event_idp, unsigned int event_id)
{
    assert(event_id < ARRAY_SIZE(pmu_amd_generic_events));

    *raw_event_idp = pmu_amd_generic_events[event_id];
    return 0;
}

static int
pmu_amd_alloc(unsigned int *pmc_idp, unsigned int pmc_index,
              unsigned int raw_event_id)
{
    struct pmu_amd *pmu;
    unsigned int pmc_id;

    /* TODO Per-family/model event availability database */

    (void)pmc_index;
    (void)raw_event_id;

    pmu = pmu_amd_get();

    if (pmu->pmc_bm == 0) {
        return EAGAIN;
    }

    pmc_id = __builtin_ffs(pmu->pmc_bm) - 1;
    pmu->pmc_bm &= ~(1U << pmc_id);
    *pmc_idp = pmc_id;

    return 0;
}

static void
pmu_amd_free(unsigned int pmc_id)
{
    struct pmu_amd *pmu;
    unsigned int mask;

    assert(pmc_id < PMU_AMD_NR_PMCS);

    pmu = pmu_amd_get();
    mask = (1U << pmc_id);
    assert(!(pmu->pmc_bm & mask));
    pmu->pmc_bm |= mask;
}

static void
pmu_amd_start(unsigned int pmc_id, unsigned int raw_event_id)
{
    const struct pmu_amd_event_code *code;
    uint32_t high, low;

    assert(pmc_id < PMU_AMD_NR_PMCS);
    assert(raw_event_id < ARRAY_SIZE(pmu_amd_event_codes));

    code = &pmu_amd_event_codes[raw_event_id];

    /* TODO Handle PERFMON_EF_KERN/PERFMON_EF_USER */
    high = code->event_select >> 8;
    low = PMU_AMD_EVTSEL_EN
          | PMU_AMD_EVTSEL_OS
          | PMU_AMD_EVTSEL_USR
          | (code->umask << 8)
          | (code->event_select & 0xff);
    cpu_set_msr(PMU_AMD_MSR_PERFEVTSEL0 + pmc_id, high, low);
}

static void
pmu_amd_stop(unsigned int pmc_id)
{
    assert(pmc_id < PMU_AMD_NR_PMCS);

    cpu_set_msr(PMU_AMD_MSR_PERFEVTSEL0 + pmc_id, 0, 0);
}

static uint64_t
pmu_amd_read(unsigned int pmc_id)
{
    assert(pmc_id < PMU_AMD_NR_PMCS);

    return cpu_get_msr64(PMU_AMD_MSR_PERCTR0 + pmc_id);
}

static const struct perfmon_dev_ops pmu_amd_ops = {
    .translate  = pmu_amd_translate,
    .alloc      = pmu_amd_alloc,
    .free       = pmu_amd_free,
    .start      = pmu_amd_start,
    .stop       = pmu_amd_stop,
    .read       = pmu_amd_read,
};

static struct perfmon_dev pmu_amd_dev __read_mostly;

static int __init
pmu_amd_setup(void)
{
    const struct cpu *cpu;
    struct pmu_amd *pmu;

    cpu = cpu_current();

    if (cpu_vendor_id(cpu) != CPU_VENDOR_AMD) {
        return ENODEV;
    }

    if (cpu_family(cpu) < 0x10) {
        return ENODEV;
    }

    pmu = pmu_amd_get();
    pmu->pmc_bm = (1U << PMU_AMD_NR_PMCS) - 1;

    pmu_amd_dev.ops = &pmu_amd_ops;
    pmu_amd_dev.pmc_width = PMU_AMD_PMC_WIDTH;
    perfmon_register(&pmu_amd_dev);
    log_info("pmu: amd, nr_pmcs:%u pmc_width:%u",
             PMU_AMD_NR_PMCS, PMU_AMD_PMC_WIDTH);
    return 0;
}

INIT_OP_DEFINE(pmu_amd_setup,
               INIT_OP_DEP(cpu_setup, true),
               INIT_OP_DEP(log_setup, true),
               INIT_OP_DEP(perfmon_bootstrap, true));


code:
/*
 * Copyright (c) 2018 Remy Noel.
 * Copyright (c) 2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * PMU driver for AMD processors.
 */

#ifndef X86_PMU_AMD_H
#define X86_PMU_AMD_H

#include <kern/init.h>

/*
 * This init operation provides :
 *  - module fully initialized
 */
INIT_OP_DECLARE(pmu_amd_setup);

#endif /* X86_PMU_AMD_H */


code:
/*
 * Copyright (c) 2014-2018 Remy Noel.
 * Copyright (c) 2014-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <errno.h>
#include <stdint.h>

#include <kern/clock.h>
#include <kern/init.h>
#include <kern/log.h>
#include <kern/perfmon.h>
#include <kern/percpu.h>
#include <machine/cpu.h>
#include <machine/pmu_intel.h>

/*
 * Intel raw event IDs.
 */
#define PMU_INTEL_RE_CYCLE          0
#define PMU_INTEL_RE_REF_CYCLE      1
#define PMU_INTEL_RE_INSTRUCTION    2
#define PMU_INTEL_RE_CACHE_REF      3
#define PMU_INTEL_RE_CACHE_MISS     4
#define PMU_INTEL_RE_BRANCH         5
#define PMU_INTEL_RE_BRANCH_MISS    6

/*
 * PMU MSR addresses
 */
#define PMU_INTEL_MSR_PMC0      0x0c1
#define PMU_INTEL_MSR_EVTSEL0   0x186

/*
 * V2 MSR addresses
 */
#define PMU_INTEL_MSR_GLOBAL_STATUS     0x038e
#define PMU_INTEL_MSR_GLOBAL_CTRL       0x038f
#define PMU_INTEL_MSR_GLOBAL_OVF_CTRL   0x0390

/*
 * Event Select Register addresses
 */
#define PMU_INTEL_EVTSEL_USR    0x00010000
#define PMU_INTEL_EVTSEL_OS     0x00020000
#define PMU_INTEL_EVTSEL_INT    0x00100000
#define PMU_INTEL_EVTSEL_EN     0x00400000

#define PMU_INTEL_ID_VERSION_MASK       0x000000ff
#define PMU_INTEL_ID_NR_PMCS_MASK       0x0000ff00
#define PMU_INTEL_ID_NR_PMCS_OFFSET     8
#define PMU_INTEL_ID_PMC_WIDTH_MASK     0x00ff0000
#define PMU_INTEL_ID_PMC_WIDTH_OFFSET   16
#define PMU_INTEL_ID_EVLEN_MASK         0xff000000
#define PMU_INTEL_ID_EVLEN_OFFSET       24
#define PMU_INTEL_ID_EVLEN_MAX          7

#define PMU_INTEL_MAX_NR_PMCS 8

/*
 * Global PMU properties.
 *
 * The bitmap is used to implement counter allocation, where each bit denotes
 * whether a counter is available or not.
 */
struct pmu_intel {
    unsigned int version;
    unsigned int nr_pmcs;
    unsigned int pmc_bm;
    unsigned int pmc_indexes[PMU_INTEL_MAX_NR_PMCS];
    unsigned int pmc_width;
    unsigned int events;
};

static struct pmu_intel pmu_intel;

/*
 * Intel hardware events.
 */
#define PMU_INTEL_EVENT_CYCLE          0x01
#define PMU_INTEL_EVENT_INSTRUCTION    0x02
#define PMU_INTEL_EVENT_REF_CYCLE      0x04
#define PMU_INTEL_EVENT_CACHE_REF      0x08
#define PMU_INTEL_EVENT_CACHE_MISS     0x10
#define PMU_INTEL_EVENT_BRANCH         0x20
#define PMU_INTEL_EVENT_BRANCH_MISS    0x40

struct pmu_intel_event_code {
    unsigned int hw_event_id;
    unsigned short event_select;
    unsigned short umask;
};

static const unsigned int pmu_intel_raw_events[] = {
    [PERFMON_EV_CYCLE]          = PMU_INTEL_RE_CYCLE,
    [PERFMON_EV_REF_CYCLE]      = PMU_INTEL_RE_REF_CYCLE,
    [PERFMON_EV_INSTRUCTION]    = PMU_INTEL_RE_INSTRUCTION,
    [PERFMON_EV_CACHE_REF]      = PMU_INTEL_RE_CACHE_REF,
    [PERFMON_EV_CACHE_MISS]     = PMU_INTEL_RE_CACHE_MISS,
    [PERFMON_EV_BRANCH]         = PMU_INTEL_RE_BRANCH,
    [PERFMON_EV_BRANCH_MISS]    = PMU_INTEL_RE_BRANCH_MISS,
};

static const struct pmu_intel_event_code pmu_intel_event_codes[] = {
    [PMU_INTEL_RE_CYCLE]        = { PMU_INTEL_EVENT_CYCLE,        0x3c, 0x00 },
    [PMU_INTEL_RE_REF_CYCLE]    = { PMU_INTEL_EVENT_REF_CYCLE,    0x3c, 0x01 },
    [PMU_INTEL_RE_INSTRUCTION]  = { PMU_INTEL_EVENT_INSTRUCTION,  0xc0, 0x00 },
    [PMU_INTEL_RE_CACHE_REF]    = { PMU_INTEL_EVENT_CACHE_REF,    0x2e, 0x4f },
    [PMU_INTEL_RE_CACHE_MISS]   = { PMU_INTEL_EVENT_CACHE_MISS,   0x2e, 0x41 },
    [PMU_INTEL_RE_BRANCH]       = { PMU_INTEL_EVENT_BRANCH,       0xc4, 0x00 },
    [PMU_INTEL_RE_BRANCH_MISS]  = { PMU_INTEL_EVENT_BRANCH_MISS,  0xc5, 0x00 },
};

static struct pmu_intel *
pmu_intel_get(void)
{
    return &pmu_intel;
}

static uint64_t
pmu_intel_get_status(void)
{
    return cpu_get_msr64(PMU_INTEL_MSR_GLOBAL_STATUS);
}

static void
pmu_intel_ack_status(uint64_t status)
{
    return cpu_set_msr64(PMU_INTEL_MSR_GLOBAL_OVF_CTRL, status);
}

static int
pmu_intel_translate(unsigned int *raw_event_idp, unsigned event_id)
{
    if (event_id >= ARRAY_SIZE(pmu_intel_raw_events)) {
        return EINVAL;
    }

    *raw_event_idp = pmu_intel_raw_events[event_id];
    return 0;
}

static int
pmu_intel_alloc(unsigned int *pmc_idp, unsigned int pmc_index,
                unsigned int raw_event_id)
{
    struct pmu_intel *pmu;
    unsigned int pmc_id;
    unsigned int hw_event_id;

    assert(raw_event_id < ARRAY_SIZE(pmu_intel_event_codes));

    pmu = pmu_intel_get();
    hw_event_id = pmu_intel_event_codes[raw_event_id].hw_event_id;

    if (!(pmu->events & hw_event_id)) {
        return EINVAL;
    }

    if (pmu->pmc_bm == 0) {
        return EAGAIN;
    }

    pmc_id = __builtin_ffs(pmu->pmc_bm) - 1;
    assert(pmc_id < ARRAY_SIZE(pmu->pmc_indexes));
    pmu->pmc_indexes[pmc_id] = pmc_index;
    pmu->pmc_bm &= ~(1U << pmc_id);
    *pmc_idp = pmc_id;
    return 0;
}

static void
pmu_intel_free(unsigned int pmc_id)
{
    struct pmu_intel *pmu;
    unsigned int mask;

    pmu = pmu_intel_get();
    mask = (1U << pmc_id);
    assert(!(pmu->pmc_bm & mask));
    pmu->pmc_bm |= mask;
}

static void
pmu_intel_start(unsigned int pmc_id, unsigned int raw_event_id)
{
    const struct pmu_intel_event_code *code;
    struct pmu_intel *pmu;
    uint32_t evtsel;

    assert(raw_event_id < ARRAY_SIZE(pmu_intel_event_codes));

    code = &pmu_intel_event_codes[raw_event_id];
    pmu = pmu_intel_get();

    /* TODO Handle PERFMON_EF_KERN/PERFMON_EF_USER */
    evtsel = PMU_INTEL_EVTSEL_EN
             | PMU_INTEL_EVTSEL_OS
             | PMU_INTEL_EVTSEL_USR
             | (code->umask << 8)
             | code->event_select;

    if (pmu->version >= 2) {
        evtsel |= PMU_INTEL_EVTSEL_INT;
    }

    cpu_set_msr(PMU_INTEL_MSR_EVTSEL0 + pmc_id, 0, evtsel);
}

static void
pmu_intel_stop(unsigned int pmc_id)
{
    cpu_set_msr(PMU_INTEL_MSR_EVTSEL0 + pmc_id, 0, 0);
}

static uint64_t
pmu_intel_read(unsigned int pmc_id)
{
    return cpu_get_msr64(PMU_INTEL_MSR_PMC0 + pmc_id);
}

static int
pmu_intel_consume_bits(uint64_t *bits)
{
    int bit;

    bit = __builtin_ffsll(*bits) - 1;

    if (bit < 0) {
        return bit;
    }

    *bits &= ~(1U << bit);
    return bit;
}

static void
pmu_intel_handle_overflow_intr(void)
{
    struct pmu_intel *pmu;
    unsigned int pmc_index;
    uint64_t status;
    int pmc_id;

    status = pmu_intel_get_status();

    if (status == 0) {
        return;
    }

    pmu_intel_ack_status(status);
    pmu = pmu_intel_get();

    status &= ((1ULL << pmu->pmc_width) - 1);

    for (;;) {
        pmc_id = pmu_intel_consume_bits(&status);

        if (pmc_id < 0) {
            break;
        }

        pmc_index = pmu->pmc_indexes[pmc_id];
        perfmon_report_overflow(pmc_index);
    }
}

static struct perfmon_dev_ops pmu_intel_ops __read_mostly = {
    .translate  = pmu_intel_translate,
    .alloc      = pmu_intel_alloc,
    .free       = pmu_intel_free,
    .start      = pmu_intel_start,
    .stop       = pmu_intel_stop,
    .read       = pmu_intel_read,
};

static struct perfmon_dev pmu_intel_dev __read_mostly;

static void
pmu_intel_percpu_init(void)
{
    const struct pmu_intel *pmu;
    uint64_t pmc_mask;

    pmu = pmu_intel_get();

    pmc_mask = (1U << pmu->nr_pmcs) - 1;
    cpu_set_msr64(PMU_INTEL_MSR_GLOBAL_CTRL, 0x700000000 | pmc_mask);
}

static struct percpu_op pmu_intel_percpu_op = \
    PERCPU_OP_INITIALIZER(pmu_intel_percpu_init);

static int __init
pmu_intel_setup(void)
{
    unsigned int eax, ebx, ecx, edx, ev_len;
    const struct cpu *cpu;
    struct pmu_intel *pmu;

    cpu = cpu_current();
    eax = 0xa;

    if (cpu_vendor_id(cpu) != CPU_VENDOR_INTEL) {
        return 0;
    }

    if (cpu_cpuid_max_basic(cpu) < eax) {
        return ENODEV;
    }

    pmu = pmu_intel_get();
    cpu_cpuid(&eax, &ebx, &ecx, &edx);
    pmu->version = eax & PMU_INTEL_ID_VERSION_MASK;

    if (pmu->version == 0) {
        return ENODEV;
    }

    pmu->nr_pmcs = (eax & PMU_INTEL_ID_NR_PMCS_MASK)
                   >> PMU_INTEL_ID_NR_PMCS_OFFSET;

    if (pmu->nr_pmcs > ARRAY_SIZE(pmu->pmc_indexes)) {
        log_err("pmu: invalid number of PMCs (%u)", pmu->nr_pmcs);
        return ENODEV;
    }

    pmu->pmc_bm = (1U << pmu->nr_pmcs ) - 1;
    pmu->pmc_width = (eax & PMU_INTEL_ID_PMC_WIDTH_MASK)
                     >> PMU_INTEL_ID_PMC_WIDTH_OFFSET;
    ev_len = (eax & PMU_INTEL_ID_EVLEN_MASK) >> PMU_INTEL_ID_EVLEN_OFFSET;

    assert(ev_len <= PMU_INTEL_ID_EVLEN_MAX);

    pmu->events = ~ebx & ((1U << ev_len) - 1);

    pmu_intel_dev.ops = &pmu_intel_ops;
    pmu_intel_dev.pmc_width = pmu->pmc_width;

    if (pmu->version >= 2) {
        percpu_register_op(&pmu_intel_percpu_op);
        pmu_intel_ops.handle_overflow_intr = pmu_intel_handle_overflow_intr;
    }

    perfmon_register(&pmu_intel_dev);
    log_info("pmu: intel v%d, nr_pmcs:%u pmc_width:%u events:%#x nr_events:%u",
             pmu->version, pmu->nr_pmcs, pmu->pmc_width, pmu->events,
             __builtin_popcount(pmu->events));
    return 0;
}

INIT_OP_DEFINE(pmu_intel_setup,
               INIT_OP_DEP(cpu_setup, true),
               INIT_OP_DEP(log_setup, true),
               INIT_OP_DEP(percpu_setup, true),
               INIT_OP_DEP(perfmon_bootstrap, true));


code:
/*
 * Copyright (c) 2018 Remy Noel.
 * Copyright (c) 2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * PMU driver for Intel processors.
 */

#ifndef X86_PMU_INTEL_H
#define X86_PMU_INTEL_H

#include <kern/init.h>

/*
 * This init operation provides :
 *  - module fully initialized
 */
INIT_OP_DECLARE(pmu_intel_setup);

#endif /* X86_PMU_INTEL_H */


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <kern/macros.h>
#include <kern/panic.h>
#include <machine/ssp.h>

void ssp_panic(void);

__used void
ssp_panic(void)
{
    panic("ssp: stack corruption detected");
}

__used void __stack_chk_fail(void) __attribute__((alias("ssp_panic")));


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_SSP_H
#define X86_SSP_H

#ifdef __LP64__
#define SSP_GUARD_WORD 0xdeadd00ddeadd00d
#else
#define SSP_GUARD_WORD 0xdeadd00d
#endif

/*
 * Offset, in words, of the SSP guard word.
 */
#define SSP_WORD_TLS_OFFSET 5

#endif /* X86_SSP_H */


code:
/*
 * Copyright (c) 2012-2014 Richard Braun.
 * Copyright (c) 2018 Agustina Arzille.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <stddef.h>
#include <stdint.h>
#include <stdio.h>

#include <kern/symbol.h>
#include <machine/pmap.h>
#include <machine/strace.h>
#include <machine/types.h>
#include <vm/vm_kmem.h>

#ifdef __LP64__
#define STRACE_ADDR_FORMAT "%#018lx"
#else /* __LP64__ */
#define STRACE_ADDR_FORMAT "%#010lx"
#endif /* __LP64__ */

static void
strace_show_one(unsigned int index, uintptr_t ip)
{
    const struct symbol *symbol;
    uintptr_t offset;

    symbol = symbol_lookup(ip);

    if (!symbol) {
        printf("#%02u [" STRACE_ADDR_FORMAT "]\n", index, (unsigned long)ip);
    } else {
        offset = ip - symbol->addr;
        printf("#%02u [" STRACE_ADDR_FORMAT "] %s+%#lx/%#lx\n",
               index, (unsigned long)ip, symbol->name,
               (unsigned long)offset, (unsigned long)symbol->size);
    }
}

void
strace_show(uintptr_t ip, uintptr_t bp)
{
    phys_addr_t pa;
    void **frame;
    unsigned int i;
    int error;

    strace_show_one(0, ip);

    i = 1;
    frame = (void **)bp;

    for (;;) {
        if (frame == NULL) {
            break;
        }

        error = pmap_kextract((uintptr_t)&frame[1], &pa);

        if (error) {
            printf("strace: unmapped return address at %p\n", &frame[1]);
            break;
        }

        strace_show_one(i, (uintptr_t)frame[1]);
        error = pmap_kextract((uintptr_t)frame, &pa);

        if (error) {
            printf("strace: unmapped frame address at %p\n", frame);
            break;
        }

        i++;
        frame = frame[0];
    }
}


code:
/*
 * Copyright (c) 2012 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Stack tracing.
 *
 * TODO Make it possible to debug without the frame pointer.
 */

#ifndef X86_STRACE_H
#define X86_STRACE_H

#include <stdint.h>

#include <kern/macros.h>

/*
 * Display a call trace.
 *
 * Attempt to resolve the given instruction pointer, then walk the calling
 * chain from the given frame pointer.
 */
void strace_show(uintptr_t ip, uintptr_t bp);

/*
 * Display the current call trace.
 */
static __always_inline void
strace_dump(void)
{
    uintptr_t ip, bp;

    asm volatile("1: mov $1b, %0" : "=r" (ip));
    bp = (uintptr_t)__builtin_frame_address(0);
    strace_show(ip, bp);
}

#endif /* X86_STRACE_H */


code:
/*
 * Copyright (c) 2014 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Despite comparison and scan instructions not having side-effects, the
 * memory clobber is used because the compiler cannot infer dependencies
 * on the memory referenced by the pointers.
 */

#include <stddef.h>
#include <string.h>

#include <kern/macros.h>
#include <machine/string.h>

#ifdef STRING_ARCH_MEMCPY
void *
memcpy(void *dest, const void *src, size_t n)
{
    void *orig_dest;

    orig_dest = dest;
    asm volatile("rep movsb"
                 : "+D" (dest), "+S" (src), "+c" (n)
                 : : "memory");
    return orig_dest;
}
#endif /* STRING_ARCH_MEMCPY */

#ifdef STRING_ARCH_MEMMOVE
void *
memmove(void *dest, const void *src, size_t n)
{
    void *orig_dest;

    orig_dest = dest;

    if (dest <= src) {
        asm volatile("rep movsb"
                     : "+D" (dest), "+S" (src), "+c" (n)
                     : : "memory");
    } else {
        dest += n - 1;
        src += n - 1;
        asm volatile("std; rep movsb; cld"
                     : "+D" (dest), "+S" (src), "+c" (n)
                     : : "memory");
    }

    return orig_dest;
}
#endif /* STRING_ARCH_MEMMOVE */

#ifdef STRING_ARCH_MEMSET
void *
memset(void *s, int c, size_t n)
{
    void *orig_s;

    orig_s = s;
    asm volatile("rep stosb"
                 : "+D" (s), "+c" (n)
                 : "a" (c)
                 : "memory");
    return orig_s;
}
#endif /* STRING_ARCH_MEMSET */

#ifdef STRING_ARCH_MEMCMP
int
memcmp(const void *s1, const void *s2, size_t n)
{
    unsigned char c1, c2;

    if (n == 0) {
        return 0;
    }

    asm volatile("repe cmpsb"
                 : "+D" (s1), "+S" (s2), "+c" (n)
                 : : "memory");
    c1 = *(((const unsigned char *)s1) - 1);
    c2 = *(((const unsigned char *)s2) - 1);
    return (int)c1 - (int)c2;
}
#endif /* STRING_ARCH_MEMCMP */

#ifdef STRING_ARCH_STRLEN
size_t
strlen(const char *s)
{
    size_t n;

    n = (size_t)-1;
    asm volatile("repne scasb"
                 : "+D" (s), "+c" (n)
                 : "a" (0)
                 : "memory");
    return (size_t)-2 - n;
}
#endif /* STRING_ARCH_STRLEN */

#ifdef STRING_ARCH_STRCPY
char *
strcpy(char *dest, const char *src)
{
    char *orig_dest;

    orig_dest = dest;
    asm volatile("1:\n"
                 "lodsb\n"
                 "stosb\n"
                 "testb %%al, %%al\n"
                 "jnz 1b\n"
                 : "+D" (dest), "+S" (src)
                 : : "al", "memory");
    return orig_dest;
}
#endif /* STRING_ARCH_STRCPY */

#ifdef STRING_ARCH_STRCMP
int
strcmp(const char *s1, const char *s2)
{
    unsigned char c1, c2;

    asm volatile("1:\n"
                 "lodsb\n"
                 "scasb\n"
                 "jne 1f\n"
                 "testb %%al, %%al\n"
                 "jnz 1b\n"
                 "1:\n"
                 : "+D" (s1), "+S" (s2)
                 : : "al", "memory");
    c1 = *(((const unsigned char *)s1) - 1);
    c2 = *(((const unsigned char *)s2) - 1);
    return (int)c1 - (int)c2;
}
#endif /* STRING_ARCH_STRCMP */

#ifdef STRING_ARCH_STRNCMP
int
strncmp(const char *s1, const char *s2, size_t n)
{
    unsigned char c1, c2;

    if (unlikely(n == 0)) {
        return 0;
    }

    asm volatile("1:\n"
                 "lodsb\n"
                 "scasb\n"
                 "jne 1f\n"
                 "testb %%al, %%al\n"
                 "jz 1f\n"
                 "dec %2\n"
                 "jnz 1b\n"
                 "1:\n"
                 : "+D" (s1), "+S" (s2), "+c" (n)
                 : : "al", "memory");
    c1 = *(((const unsigned char *)s1) - 1);
    c2 = *(((const unsigned char *)s2) - 1);
    return (int)c1 - (int)c2;
}
#endif /* STRING_ARCH_STRNCMP */

#ifdef STRING_ARCH_STRCHR
char *
strchr(const char *s, int c)
{
    asm volatile("1:\n"
                 "lodsb\n"
                 "cmpb %%al, %1\n"
                 "je 1f\n"
                 "testb %%al, %%al\n"
                 "jnz 1b\n"
                 "mov $1, %0\n"
                 "1:\n"
                 "dec %0\n"
                 : "+S" (s)
                 : "c" ((char)c)
                 : "al", "memory");
    return (char *)s;
}
#endif /* STRING_ARCH_STRCHR */


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_STRING_H
#define X86_STRING_H

/*
 * Provide architecture-specific string functions.
 */
#define STRING_ARCH_MEMCPY
#define STRING_ARCH_MEMMOVE
#define STRING_ARCH_MEMSET
#define STRING_ARCH_MEMCMP
#define STRING_ARCH_STRLEN
#define STRING_ARCH_STRCPY
#define STRING_ARCH_STRCMP
#define STRING_ARCH_STRNCMP
#define STRING_ARCH_STRCHR

#endif /* X86_STRING_H */


code:
/*
 * Copyright (c) 2012-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <stdnoreturn.h>

#include <kern/init.h>
#include <kern/thread.h>
#include <machine/cpu.h>
#include <machine/pmap.h>
#include <machine/strace.h>
#include <machine/tcb.h>

noreturn void tcb_context_load(struct tcb *tcb);
noreturn void tcb_start(void);
void tcb_context_restore(void);

struct tcb *tcb_current_ptr __percpu;

static void
tcb_stack_push(struct tcb *tcb, uintptr_t word)
{
    uintptr_t *ptr;

    ptr = (uintptr_t *)tcb->sp;
    ptr--;
    *ptr = word;
    tcb->sp = (uintptr_t)ptr;
}

#ifdef __LP64__

static void
tcb_stack_forge(struct tcb *tcb, void (*fn)(void *), void *arg)
{
    tcb_stack_push(tcb, (uintptr_t)arg);
    tcb_stack_push(tcb, (uintptr_t)fn);
    tcb_stack_push(tcb, (uintptr_t)tcb_start);  /* Return address */
    tcb_stack_push(tcb, 0);                     /* RBX */
    tcb_stack_push(tcb, 0);                     /* R12 */
    tcb_stack_push(tcb, 0);                     /* R13 */
    tcb_stack_push(tcb, 0);                     /* R14 */
    tcb_stack_push(tcb, 0);                     /* R15 */
}

#else /* __LP64__ */

static void
tcb_stack_forge(struct tcb *tcb, void (*fn)(void *), void *arg)
{
    tcb_stack_push(tcb, (uintptr_t)arg);
    tcb_stack_push(tcb, (uintptr_t)fn);
    tcb_stack_push(tcb, (uintptr_t)tcb_start);  /* Return address */
    tcb_stack_push(tcb, 0);                     /* EBX */
    tcb_stack_push(tcb, 0);                     /* EDI */
    tcb_stack_push(tcb, 0);                     /* ESI */
}

#endif /* __LP64__ */

int
tcb_build(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg)
{
    int error;

    error = pmap_thread_build(thread_from_tcb(tcb));

    if (error) {
        return error;
    }

    tcb->bp = 0;
    tcb->sp = (uintptr_t)stack + TCB_STACK_SIZE;
    tcb_stack_forge(tcb, fn, arg);
    return 0;
}

void
tcb_cleanup(struct tcb *tcb)
{
    pmap_thread_cleanup(thread_from_tcb(tcb));
}

void __init
tcb_load(struct tcb *tcb)
{
    assert(!cpu_intr_enabled());

    tcb_set_current(tcb);
    tcb_context_load(tcb);
}

void
tcb_trace(const struct tcb *tcb)
{
    strace_show((uintptr_t)tcb_context_restore, tcb->bp);
}

static int __init
tcb_setup(void)
{
    return 0;
}

INIT_OP_DEFINE(tcb_setup,
               INIT_OP_DEP(cpu_setup, true));


code:
/*
 * Copyright (c) 2012-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Thread control block.
 */

#ifndef X86_TCB_H
#define X86_TCB_H

#include <assert.h>
#include <stdint.h>
#include <stdnoreturn.h>

#include <kern/init.h>
#include <machine/cpu.h>
#include <machine/page.h>

/*
 * Thread stack size.
 */
#define TCB_STACK_SIZE PAGE_SIZE

/*
 * Forward declaration.
 */
struct pmap_update_oplist;

/*
 * Thread control block.
 */
struct tcb {
    uintptr_t bp;
    uintptr_t sp;
    struct pmap_update_oplist *oplist;
};

/*
 * Build a TCB.
 *
 * Prepare the given stack for execution. The context is defined so that it
 * will call thread_main(fn, arg) when loaded.
 *
 * In addition, initialize any thread-local machine-specific data.
 */
int tcb_build(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg);

/*
 * Release all resources held by a TCB.
 */
void tcb_cleanup(struct tcb *tcb);

static inline struct tcb *
tcb_current(void)
{
    extern struct tcb *tcb_current_ptr;
    return cpu_local_read(tcb_current_ptr);
}

static inline void
tcb_set_current(struct tcb *tcb)
{
    extern struct tcb *tcb_current_ptr;
    cpu_local_assign(tcb_current_ptr, tcb);
}

static inline void
tcb_set_pmap_update_oplist(struct tcb *tcb, struct pmap_update_oplist *oplist)
{
    tcb->oplist = oplist;
}

static inline struct pmap_update_oplist *
tcb_get_pmap_update_oplist(struct tcb *tcb)
{
    return tcb->oplist;
}

/*
 * Load a TCB.
 *
 * Called with interrupts disabled. The caller context is lost.
 */
noreturn void tcb_load(struct tcb *tcb);

/*
 * Context switch.
 *
 * Called with interrupts disabled.
 */
static inline void
tcb_switch(struct tcb *prev, struct tcb *next)
{
    void tcb_context_switch(struct tcb *prev, struct tcb *next);

    assert(!cpu_intr_enabled());

    tcb_set_current(next);
    tcb_context_switch(prev, next);
}

/*
 * Dump the stack trace of a TCB.
 *
 * The thread associated to the TCB must not be running.
 */
void tcb_trace(const struct tcb *tcb);

/*
 * This init operation provides :
 *  - current TCB handling
 */
INIT_OP_DECLARE(tcb_setup);

#endif /* X86_TCB_H */


code:
/*
 * Copyright (c) 2012-2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <kern/init.h>
#include <machine/asm.h>
#include <machine/cpu.h>

.section INIT_SECTION

#ifdef __LP64__

ASM_ENTRY(tcb_context_load)
 movq (%rdi), %rbp          /* load frame pointer from TCB */
 movq 8(%rdi), %rsp         /* load stack pointer from TCB */
 jmp tcb_context_restore
ASM_END(tcb_context_load)

#else /* __LP64__ */

ASM_ENTRY(tcb_context_load)
 movl 4(%esp), %eax         /* load TCB address */
 movl (%eax), %ebp          /* load frame pointer from TCB */
 movl 4(%eax), %esp         /* load stack pointer from TCB */
 jmp tcb_context_restore
ASM_END(tcb_context_load)

#endif /* __LP64__ */

.text

#ifdef __LP64__

ASM_ENTRY(tcb_start)
 popq %rdi              /* load function */
 popq %rsi              /* load argument */

 /*
  * Use the call instruction to start a clean stack trace.
  *
  * Note that, on amd64, the stack must be 16-byte before the call
  * instruction, so that "$(rsp + 8) is a multiple is always a multiple
  * of 16 when control is transferred to the function entry point",
  * which is another reason to use call instead of a bare jump.
  */
 call thread_main

 /* Never reached */
 nop                    /* make the return address point to an instruction
                           inside the function to build a clean stack trace */
ASM_END(tcb_start)

ASM_ENTRY(tcb_context_switch)
 pushq %rbx             /* store registers as required by ABI */
 pushq %r12
 pushq %r13
 pushq %r14
 pushq %r15

 movq %rbp, (%rdi)      /* store frame pointer into prev TCB */
 movq %rsp, 8(%rdi)     /* store stack pointer into prev TCB */
 movq (%rsi), %rbp      /* load frame pointer from next TCB */
 movq 8(%rsi), %rsp     /* load stack pointer from next TCB */

.global tcb_context_restore
tcb_context_restore:
 popq %r15              /* load registers as required by ABI */
 popq %r14
 popq %r13
 popq %r12
 popq %rbx
 ret
ASM_END(tcb_context_switch)

#else /* __LP64__ */

ASM_ENTRY(tcb_start)
 call thread_main       /* the stack already contains the expected arguments
                           in the expected order, use the call instruction to
                           start a clean stack trace */

 /* Never reached */
 nop                    /* make the return address point to an instruction
                           inside the function to build a clean stack trace */
ASM_END(tcb_start)

ASM_ENTRY(tcb_context_switch)
 movl 4(%esp), %eax     /* load prev TCB address */
 movl 8(%esp), %ecx     /* load next TCB address */

 pushl %ebx             /* store registers as required by ABI */
 pushl %edi
 pushl %esi

 movl %ebp, (%eax)      /* store frame pointer into prev TCB */
 movl %esp, 4(%eax)     /* store stack pointer into prev TCB */
 movl (%ecx), %ebp      /* load frame pointer from next TCB */
 movl 4(%ecx), %esp     /* load stack pointer from next TCB */

/*
 * This code is run on context restoration. The frame and stack pointers
 * have already been loaded to their correct values. Load registers which
 * were stored on the stack when the context was saved and return.
 */
.global tcb_context_restore
tcb_context_restore:
 popl %esi
 popl %edi
 popl %ebx
 ret
ASM_END(tcb_context_switch)

#endif /* __LP64__ */


code:
/*
 * Copyright (c) 2012 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef X86_TYPES_H
#define X86_TYPES_H

#ifdef CONFIG_X86_PAE
typedef unsigned long long phys_addr_t;
#else /* CONFIG_X86_PAE */
typedef unsigned long phys_addr_t;
#endif /* CONFIG_X86_PAE */

#endif /* X86_TYPES_H */


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>

#include <kern/arg.h>
#include <kern/console.h>
#include <kern/init.h>
#include <kern/intr.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <machine/biosmem.h>
#include <machine/io.h>
#include <machine/uart.h>

#define UART_BDA_COM1_OFFSET 0

#define UART_REG_DAT            0
#define UART_REG_DLL            0
#define UART_REG_IER            1
#define UART_REG_DLH            1
#define UART_REG_IIR            2
#define UART_REG_LCR            3
#define UART_REG_MCR            4
#define UART_REG_LSR            5
#define UART_REG_MSR            6
#define UART_NR_REGS            7

#define UART_IER_RX             0x1

#define UART_IIR_NOT_PENDING    0x1
#define UART_IIR_SRC_RX         0x4
#define UART_IIR_SRC_MASK       0xe

#define UART_LCR_8BITS          0x03
#define UART_LCR_1S             0x00
#define UART_LCR_NP             0x00
#define UART_LCR_OP             0x08
#define UART_LCR_EP             0x18
#define UART_LCR_BEN            0x40
#define UART_LCR_DLAB           0x80

#define UART_MCR_DTR            0x01
#define UART_MCR_RTS            0x02
#define UART_MCR_AUX2           0x04

#define UART_LSR_DATA_READY     0x01
#define UART_LSR_TX_EMPTY       0x20

#define UART_MAX_DEVS           4

#define UART_SPEED_MAX          115200
#define UART_SPEED_DEFAULT      UART_SPEED_MAX

enum {
    UART_PARITY_NONE,
    UART_PARITY_ODD,
    UART_PARITY_EVEN,
};

#define UART_PARITY_DEFAULT     UART_PARITY_NONE

#define UART_DATA_BITS_DEFAULT  8

struct uart {
    struct console console;
    uint16_t port;
    uint16_t intr;
};

static struct uart uart_devs[UART_MAX_DEVS];

static uint16_t uart_intrs[UART_MAX_DEVS] = { 4, 3, 4, 3 };

static size_t
uart_get_id(const struct uart *uart)
{
    size_t id;

    id = uart - uart_devs;
    assert(id < ARRAY_SIZE(uart_devs));
    return id;
}

static uint16_t
uart_get_addr(const struct uart *uart, uint16_t reg)
{
    assert(reg < UART_NR_REGS);
    return uart->port + reg;
}

static uint8_t
uart_read(struct uart *uart, uint16_t reg)
{
    return io_read_byte(uart_get_addr(uart, reg));
}

static void
uart_write(struct uart *uart, uint16_t reg, uint8_t byte)
{
    io_write_byte(uart_get_addr(uart, reg), byte);
}

static void
uart_set(struct uart *uart, uint16_t reg, uint8_t mask)
{
    uint16_t addr;
    uint8_t byte;

    addr = uart_get_addr(uart, reg);
    byte = io_read_byte(addr);
    byte |= mask;
    io_write_byte(addr, byte);
}

static void
uart_clear(struct uart *uart, uint16_t reg, uint8_t mask)
{
    uint16_t addr;
    uint8_t byte;

    addr = uart_get_addr(uart, reg);
    byte = io_read_byte(addr);
    byte &= ~mask;
    io_write_byte(addr, byte);
}

static void
uart_recv_intr(struct uart *uart)
{
    uint8_t byte;
    char tmp[2];

    tmp[1] = '\0';

    for (;;) {
        byte = uart_read(uart, UART_REG_LSR);

        if (!(byte & UART_LSR_DATA_READY)) {
            break;
        }

        byte = uart_read(uart, UART_REG_DAT);
        tmp[0] = (char)byte;
        console_intr(&uart->console, tmp);
    }
}

static int
uart_intr(void *arg)
{
    struct uart *uart;
    uint8_t byte;

    uart = arg;

    byte = uart_read(uart, UART_REG_IIR);

    if (byte & UART_IIR_NOT_PENDING) {
        return EAGAIN;
    }

    byte &= UART_IIR_SRC_MASK;

    if (byte == UART_IIR_SRC_RX) {
        uart_recv_intr(uart);
    }

    return 0;
}

static void __init
uart_enable_intr(struct uart *uart)
{
    int error;

    error = intr_register(uart->intr, uart_intr, uart);

    if (error) {
        log_err("uart%zu: unable to register interrupt %u",
                 uart_get_id(uart), uart->intr);
        return;
    }

    uart_write(uart, UART_REG_IER, UART_IER_RX);
}

static void
uart_tx_wait(struct uart *uart)
{
    uint8_t byte;

    for (;;) {
        byte = uart_read(uart, UART_REG_LSR);

        if (byte & UART_LSR_TX_EMPTY) {
            break;
        }
    }
}

static void
uart_write_char_common(struct uart *uart, char c)
{
    uart_tx_wait(uart);
    uart_write(uart, UART_REG_DAT, (uint8_t)c);
}

static void
uart_write_char(struct uart *uart, char c)
{
    if (c == '\n') {
        uart_write_char_common(uart, '\r');
    }

    uart_write_char_common(uart, c);
}

static struct uart *
uart_get_dev(size_t i)
{
    assert(i < ARRAY_SIZE(uart_devs));
    return &uart_devs[i];
}

static struct uart *
uart_get_from_console(struct console *console)
{
    return structof(console, struct uart, console);
}

static void
uart_console_putc(struct console *console, char c)
{
    uart_write_char(uart_get_from_console(console), c);
}

static const struct console_ops uart_console_ops = {
    .putc = uart_console_putc,
};

static void __init
uart_init_default(unsigned int *speed, unsigned int *parity,
                  unsigned int *data_bits)
{
    *speed = UART_SPEED_DEFAULT;
    *parity = UART_PARITY_DEFAULT;
    *data_bits = UART_DATA_BITS_DEFAULT;
}

static int __init
uart_init_check_speed(unsigned int speed)
{
    if (speed > UART_SPEED_MAX) {
        return EINVAL;
    }

    return 0;
}

static int __init
uart_init_convert_parity_char(char c, unsigned int *parity)
{
    switch (c) {
    case 'n':
        *parity = UART_PARITY_NONE;
        break;
    case 'o':
        *parity = UART_PARITY_ODD;
        break;
    case 'e':
        *parity = UART_PARITY_EVEN;
        break;
    default:
        return EINVAL;
    }

    return 0;
}

static int __init
uart_init_check_data_bits(unsigned int data_bits)
{
    switch (data_bits) {
    case 5 ... 8:
        break;
    default:
        return EINVAL;
    }

    return 0;
}

static void __init
uart_init_args(const struct uart *uart, const char *arg_str,
               unsigned int *speed, unsigned int *parity,
               unsigned int *data_bits)
{
    char parity_char;
    int ret, error;

    ret = sscanf(arg_str, "%u%c%1u", speed, &parity_char, data_bits);

    if (ret < 1) {
        goto set_defaults;
    }

    error = uart_init_check_speed(*speed);

    if (error) {
        goto set_defaults;
    } else if (ret < 2) {
        return;
    }

    error = uart_init_convert_parity_char(parity_char, parity);

    if (error) {
        goto set_defaults;
    } else if (ret < 3) {
        return;
    }

    error = uart_init_check_data_bits(*data_bits);

    if (error) {
        goto set_defaults;
    }

    return;

set_defaults:
    log_warning("uart%zu: invalid serial configuration, using defaults",
                uart_get_id(uart));
    uart_init_default(speed, parity, data_bits);
}

static void __init
uart_init(struct uart *uart, uint16_t port, uint16_t intr)
{
    unsigned int speed, parity, data_bits;
    const char *arg_str;
    char name[CONSOLE_NAME_SIZE];
    uint16_t divisor;
    uint8_t byte;

    snprintf(name, sizeof(name), "uart%zu", uart_get_id(uart));
    arg_str = arg_value(name);

    uart_init_default(&speed, &parity, &data_bits);

    if (arg_str != NULL) {
        uart_init_args(uart, arg_str, &speed, &parity, &data_bits);
    }

    log_debug("uart%zu: speed:%u parity:%u data_bits:%u",
              uart_get_id(uart), speed, parity, data_bits);

    uart->port = port;
    uart->intr = intr;

    uart_write(uart, UART_REG_IER, 0);

    divisor = UART_SPEED_MAX / speed;

    uart_set(uart, UART_REG_LCR, UART_LCR_DLAB);
    uart_write(uart, UART_REG_DLH, divisor >> 8);
    uart_write(uart, UART_REG_DLL, divisor & 0xff);
    uart_clear(uart, UART_REG_LCR, UART_LCR_DLAB);

    uart_write(uart, UART_REG_MCR, UART_MCR_AUX2 | UART_MCR_RTS | UART_MCR_DTR);

    byte = UART_LCR_1S;

    switch (parity) {
    case UART_PARITY_NONE:
        byte |= UART_LCR_NP;
        break;
    case UART_PARITY_ODD:
        byte |= UART_LCR_OP;
        break;
    case UART_PARITY_EVEN:
        byte |= UART_LCR_EP;
        break;
    }

    byte |= (data_bits - 5);
    uart_write(uart, UART_REG_LCR, byte);

    console_init(&uart->console, name, &uart_console_ops);
    console_register(&uart->console);
}

static void __init
uart_log_info(void)
{
    const struct uart *uart;
    size_t i;

    for (i = 0; i < ARRAY_SIZE(uart_devs); i++) {
        uart = uart_get_dev(i);

        if (uart->port != 0) {
            log_info("uart%zu: port:%#x irq:%u", i, (unsigned int)uart->port,
                     (unsigned int)uart->intr);
        }
    }
}

static int __init
uart_bootstrap(void)
{
    const uint16_t *ptr;
    size_t i;

    ptr = biosmem_get_bda() + UART_BDA_COM1_OFFSET;

    for (i = 0; i < UART_MAX_DEVS; i++) {
        if (ptr[i] == 0) {
            continue;
        }

        uart_init(uart_get_dev(i), ptr[i], uart_intrs[i]);
    }

    uart_log_info();
    return 0;
}

INIT_OP_DEFINE(uart_bootstrap,
               INIT_OP_DEP(arg_setup, true),
               INIT_OP_DEP(console_bootstrap, true),
               INIT_OP_DEP(log_setup, true));

static int __init
uart_setup(void)
{
    struct uart *uart;
    size_t i;

    for (i = 0; i < ARRAY_SIZE(uart_devs); i++) {
        uart = uart_get_dev(i);

        if (uart->port == 0) {
            continue;
        }

        uart_enable_intr(uart);
    }

    return 0;
}

INIT_OP_DEFINE(uart_setup,
               INIT_OP_DEP(intr_setup, true),
               INIT_OP_DEP(uart_bootstrap, true));


code:
/*
 * Copyright (c) 2017 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Tiny 8250 UART driver.
 */

#ifndef X86_UART_H
#define X86_UART_H

#include <kern/init.h>

/*
 * This init operation provides :
 *  - UART output through the console module
 */
INIT_OP_DECLARE(uart_bootstrap);

/*
 * This init operation provides :
 *  - UART input through the console module
 *  - module fully initialized
 */
INIT_OP_DECLARE(uart_setup);

#endif /* X86_UART_H */


code:
#!/bin/sh

# Amount of physical memory
RAM=1024

# Number of processors. Keep this below the number of physical processors
# because the kernel doesn't replace spinning with sleeping from within
# a virtual machine, which causes performance to collapse.
NR_CPUS=4

# Don't change from here unless you know what you're doing

# QEMU system emulator
QEMU_EXE=qemu-system-x86_64

OBJCOPY="$1"
NATIVE_ARCH=$(uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/)

if [ "$NATIVE_ARCH" != x86 ]; then
    KVM=
elif lsmod | grep -q '^kvm'; then
    KVM="-enable-kvm -cpu host"
fi

X15=$PWD/x15
TMPDIR=$(mktemp -d)

$OBJCOPY -O elf32-i386 $X15 $TMPDIR/x15

cd $TMPDIR
$QEMU_EXE $KVM \
          -ctrl-grab \
          -gdb tcp::1234 \
          -m $RAM \
          -smp $NR_CPUS \
          -nographic \
          -kernel x15 \
          -append "console=uart0"

rm -rf $TMPDIR


code:
#ifdef __LP64__
OUTPUT_FORMAT("elf64-x86-64")
OUTPUT_ARCH(i386:x86-64)
#else
OUTPUT_FORMAT("elf32-i386")
OUTPUT_ARCH(i386)
#endif /* __LP64__ */

ENTRY(boot_start)

#include <kern/init.h>
#include <machine/boot.h>
#include <machine/cpu.h>
#include <machine/page.h>
#include <machine/pmap.h>

PHDRS
{
    /* Flags are actually similar to classic Unix permissions */
    boot    PT_LOAD FLAGS(7);
    init    PT_LOAD FLAGS(7);
    percpu  PT_LOAD FLAGS(6);
    text    PT_LOAD FLAGS(5);
    rodata  PT_LOAD FLAGS(4);
    data    PT_LOAD FLAGS(6);

#ifdef CONFIG_SYMTAB
    symbol  PT_LOAD FLAGS(6);
#endif
}

SECTIONS
{
    . = BOOT_OFFSET;
    _boot = .;

    .boot ALIGN(PAGE_SIZE) : {
        *(.boot.hdr)
        *(.boot.text)
        *(.boot.data)
    } : boot

    . = ALIGN(PAGE_SIZE);
    _boot_end = .;

    . += PMAP_KERNEL_OFFSET;
    _init = .;

    .init ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.init))) {
        *(.init.text)
        *(.init.data)

        . = ALIGN(INIT_OP_ALIGN);
        _init_ops = .;
        *(.init.ops)
        _init_ops_end = .;

    } : init

    . = ALIGN(PAGE_SIZE);
    _init_end = .;
    _percpu = .;

    .percpu 0 : AT(BOOT_VTOP(_percpu)) {
        *(.percpu*)
    } : percpu

    . = _percpu + SIZEOF(.percpu);
    . = ALIGN(PAGE_SIZE);
    _percpu_end = .;
    _text = .;

    .text ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.text))) {
        *(.text*)
    } : text

    . = ALIGN(PAGE_SIZE);
    _rodata = .;

    .rodata ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.rodata))) {
        *(.rodata*)
    } : rodata

    . = ALIGN(PAGE_SIZE);
    _data = .;

    .data ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.data))) {
        . = ALIGN(CPU_L1_SIZE);
        *(.data.read_mostly)
        . = ALIGN(CPU_L1_SIZE);
        *(.data*)
    } : data

    .bss ALIGN(CPU_DATA_ALIGN) : AT(BOOT_VTOP(ADDR(.bss))) {
        *(.bss*)
    } : data

#ifdef CONFIG_SYMTAB
    /*
     * Including the embedded symbol table last should reliably prevent
     * the second link from changing the addresses of the kernel symbols.
     *
     * In addition, use a program header different from data to make
     * sure the linker doesn't include the .bss section in the output file.
     */
    .symbol ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.symbol))) {
        *(.symbol*)
    } : symbol
#endif /* CONFIG_SYMTAB */

    . = ALIGN(PAGE_SIZE);
    _end = .;

    /DISCARD/ : {
        *(.eh_frame*)
        *(.note*)
    }
}


code:
ASCIIDOC := asciidoc
A2X := a2x

ASCIIDOC_FLAGS := -a toc -d manpage -a revnumber=$(VERSION)
A2X_FLAGS := -d manpage -a revnumber=$(VERSION)

ASCIIDOC_SOURCES := \
        doc/cenv.9.txt \
        doc/init.9.txt \
        doc/intro.9.txt \
        doc/style.9.txt \
        doc/xbuild.9.txt

ASCIIDOC_MANDOCS := $(patsubst %.9.txt,%.9,$(ASCIIDOC_SOURCES))
ASCIIDOC_HTMLDOCS := $(patsubst %.9.txt,%.9.html,$(ASCIIDOC_SOURCES))

define DOC_HELP
	@printf '* mandocs                  - Build man pages\n'
	@printf '* htmldocs                 - Build HTML man pages\n'
endef

define DOC_INSTALL_HELP
	@printf '  install-mandocs          - Install man pages\n'
	@printf '  install-htmldocs         - Install HTML man pages\n'
endef

# Expose intermediate DocBook files for correct parallel builds.
%.9.xml: %.9.txt doc/asciidoc.conf $(ALL_MAKEFILES)
	$(call xbuild_action_mkdir,$@)
	$(Q)$(ASCIIDOC) $(ASCIIDOC_FLAGS) -b docbook -o $@ $<

%.9: %.9.xml doc/asciidoc.conf $(ALL_MAKEFILES)
	$(call xbuild_action,MAN,$@) \
		$(A2X) $(A2X_FLAGS) -f manpage $<

%.9.html: %.9.txt doc/asciidoc.conf $(ALL_MAKEFILES)
	$(call xbuild_action,HTML,$@) \
		$(ASCIIDOC) $(ASCIIDOC_FLAGS) -b html5 -o $@ $<

.PHONY: mandocs
mandocs: $(ASCIIDOC_MANDOCS)

.PHONY: htmldocs
htmldocs: $(ASCIIDOC_HTMLDOCS)

.PHONY: docs
docs: mandocs htmldocs

define mandocs_install_targets
	$(foreach manpage,$(notdir $(ASCIIDOC_MANDOCS)),$(manpage)_man_install)
endef

%_man_install: doc/%
	install -D -m 644 $< $(DESTDIR)$(PREFIX)/$(DATAROOTDIR)/man/man9/$*

.PHONY: install-mandocs
install-mandocs: $(mandocs_install_targets)

.PHONY: install-htmldocs
install-htmldocs: $(htmldocs_install_targets)

define htmldocs_install_targets
	$(foreach htmlpage,$(notdir $(ASCIIDOC_HTMLDOCS)),$(htmlpage)_html_install)
endef

%_html_install: doc/%
	install -D -m 644 $< $(DESTDIR)$(PREFIX)/$(DATAROOTDIR)/doc/x15/$*

.PHONY: install-docs
install-docs: install-mandocs install-htmldocs

define clean_docs
	$(Q)rm -f $(ASCIIDOC_MANDOCS) \
	$(ASCIIDOC_HTMLDOCS)
endef

.PHONY: clean-docs
clean-docs:
	$(call clean_docs)

.PHONY: distclean-docs
distclean-docs: clean-docs

path: doc/Makefile


code:
[attributes]
max-width=58em
source-highlighter=highlight
x15-operating-system=https://www.sceen.net/x15/[The X15 operating system]
against-priority-inheritance=https://fsmlabs.com/pdfs/Priority_Inheritance.pdf[Against Priority Inheritance]
the-art-of-unix-programming=http://www.catb.org/esr/writings/taoup/html/[The Art of Unix Programming]
the-kconfig-language=https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt

[macros]
(?su)(?P<name>module_mutex):(?P<impl>\w+)=
(?su)(?P<name>module):(?P<component>\w+)/(?P<module>\w+)=
(?su)(?P<name>manpage):(?P<page>\w+)=
(?su)(?P<name>cheader):(?P\w+)=
(?su)(?P<name>cfunction):(?P<function>\w+)=
(?su)(?P<name>cmacro):(?P<macro>\w+)=
(?su)(?P<name>shell):{(?P<command>[^}]+)}=

[module_mutex-inlinemacro]
ifdef::basebackend-html[]
mutex_{impl}
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<ulink url="https://git.sceen.net/rbraun/x15.git/tree/kern/mutex/mutex_{impl}_i.h"><emphasis role="bold">mutex_{impl}</emphasis></ulink>
endif::basebackend-docbook[]

[module-inlinemacro]
ifdef::basebackend-html[]
{module}
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<ulink url="https://git.sceen.net/rbraun/x15.git/tree/{component@arch:arch/x86/machine:{component}}/{module}.h"><emphasis role="bold">{module}</emphasis></ulink>
endif::basebackend-docbook[]

[manpage-inlinemacro]
ifdef::basebackend-html[]
{page}(9)
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<ulink url="{page}.9.html"><emphasis role="bold">{page}</emphasis>(9)</ulink>
endif::basebackend-docbook[]

[cheader-inlinemacro]
ifdef::basebackend-html[]
<{header}.h>
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<{header}.h>
endif::basebackend-docbook[]

[cfunction-inlinemacro]
ifdef::basebackend-html[]
{function}()
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<emphasis role="italic">{function}()</emphasis>
endif::basebackend-docbook[]

[cmacro-inlinemacro]
ifdef::basebackend-html[]
{macro}
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<emphasis role="italic">{macro}</emphasis>
endif::basebackend-docbook[]

[shell-inlinemacro]
ifdef::basebackend-html[]
{command}
endif::basebackend-html[]
ifdef::basebackend-docbook[]
<emphasis role="italic">{command}</emphasis>
endif::basebackend-docbook[]


code:
CENV(9)
========
:doctype:       manpage
:man source:    X15
:man manual:    X15 C environment

NAME
----

cenv - C environment

DESCRIPTION
-----------

This document describes the C environment of the kernel, including
available standard headers and interfaces.

Standard
~~~~~~~~

X15 must be built with a C compiler supporting the ISO/IEC 9899:2011
(C11) specification with some GCC extensions. It can currently be built
with the GCC and Clang compilers.

Environment
~~~~~~~~~~~

The kernel uses the freestanding execution environment. This means the
compiler is expected to provide the following headers :

** cheader:float
** cheader:iso646
** cheader:limits
** cheader:stdalign
** cheader:stdarg
** cheader:stdbool
** cheader:stddef
** cheader:stdint
** cheader:stdnoreturn

X15 augments the environment with a small subset of the functions provided
in hosted environments. The additional headers are :

** cheader:assert
** cheader:errno
** cheader:limits
** cheader:stdio
** cheader:string

Note that these headers do not provide the complete set of interfaces
expected in a hosted environment.

Formatting functions
~~~~~~~~~~~~~~~~~~~~

Of particular interest is the cheader:stdio header which provides a subset
of the printf and scanf family of functions.

The supported functions are :

** cfunction:getchar
** cfunction:putchar
** cfunction:sprintf
** cfunction:snprintf
** cfunction:vsprintf
** cfunction:vsnprintf
** cfunction:sscanf
** cfunction:vsscanf

The *EOF* macro is also provided.

The formatting functions only implement a subset of what the standard
defines, namely :

** cfunction:sprintf :
*** flags: # 0 - ' ' (space) +
*** field width is supported
*** precision is supported
** cfunction:sscanf :
*** flags: *
*** field width is supported
** common to both :
*** modifiers: hh h l ll z t
*** specifiers: d i o u x X c s p n %

Floating point conversions are currently not supported, but this may change
in the future.

SEE
---

manpage:intro

{x15-operating-system}


code:
INIT(9)
========
:doctype:       manpage
:man source:    X15
:man manual:    X15 Kernel initialization

NAME
----

init - kernel initialization

DESCRIPTION
-----------

This document describes kernel initialization, and in particular, the concept
of initialization operation and their dependencies. All related declarations
are provided by the module:kern/init module.

Initialization operations
~~~~~~~~~~~~~~~~~~~~~~~~~

An initialization operation is a function that enables the use of a specific
interface. For example, The cfunction:thread_setup function enables the
creation of threads. Any thread created before calling that function is
invalid and leads to undefined behaviour, potentially being silent and
unnoticed for a long time. This kind of relation is expressed as an
initialization operation dependency. As a result, the data structure for
initialization operations associates a function and its dependencies.

Definition
~~~~~~~~~~

Initialization operations are defined using the cmacro:INIT_OP_DEFINE()
macro. The first argument is the function implementing the operation,
and any additional argument is a dependency.

Dependencies are built with the cmacro:INIT_OP_DEP() macro. They may
be either required or optional. An optional dependency is only used
to infer execution order. If it fails, the initialization operation
function is run regardless.

Declaration
~~~~~~~~~~~

In order to build dependencies, initialization operations must be
declared as part of the public interface of the module providing it.
This is accomplished using the cmacro:INIT_OP_DECLARE() macro in the
public header.

Execution order
~~~~~~~~~~~~~~~

The complete set of initialization operations and their dependencies
must result in a directed acyclic graph, so that a topological sort
can be applied, starting from the roots, i.e. initialization operations
with no dependencies. For convenience, the topological sort is performed
both at build and run time. Sorting at build time is done with the
shell:{make x15.sorted_init_ops} command.

Debugging
~~~~~~~~~

In order to help debugging initialization issues, the module:kern/init
module provides a debugging mode, that may be enabled at build time
by changing an internal macro. In this mode, text buffers record
various information such as the list of roots, operations with dependency
cycles (in case the build time check isn't used), and the list of
operations that completed. These buffers can be dumped using a debugger
such as a JTAG interface. Debugging mode is enabled by setting the
cmacro:INIT_DEBUG macro to a non-zero value.

SEE
---

manpage:intro

{x15-operating-system}


code:
INTRO(9)
========
:doctype:       manpage
:man source:    X15
:man manual:    X15 Kernel Developer{rsquo}s Manual

NAME
----

intro - introduction to kernel interfaces

DESCRIPTION
-----------

X15 is an open source real-time microkernel intended to provide a performant
and scalable environment for cache-coherent multiprocessor machines.

Section 9 of the manual describes kernel interfaces, both internal and
provided to application code hosted in kernel mode.

Among the features provided are :

* <<preemptive_multithreading,Preemptive multithreading>>
* <<generic_development_tools,Generic development tools>>
* <<multiprocessor_support,Multiprocessor support>>
* <<virtual_memoru,Virtual memory>>
* <<real_time,Real-time>>
* <<portability,Portability>>

Modules
~~~~~~~

The module is the functional unit around which kernel sources are organized.
A single module is made up of at least one public header file. There is
usually also an opaque implementation file. An implementation header can
complement the module for inline functions and private structures allowed
to be allocated from the stack. Finally, a type-only header can also be
present in order to avoid circular inclusions.

The files of a module must strictly be named according to the module name.
Here are the name patterns for each file of a module :

*<module>.c*::
  Opaque implementation.
*<module>.h*::
  Public header.
*<module>_i.h*::
  Implementation header.
*<module>_types.h*::
  Type-only header.

Components
~~~~~~~~~~

Modules are grouped into components, with one directory per component.
The main components are :

*arch*::
  Architecture-specific modules, located in arch/<arch>.
*test*::
  Test modules.
*vm*::
  Virtual memory system.
*kern*::
  Machine-independent modules that don't belong in another component.

[[preemptive_multithreading]]
PREEMPTIVE MULTITHREADING
-------------------------

The X15 kernel provides threads which can be preempted at almost any time,
in both kernel and user space. They have an associated scheduling policy
and priority. Currently, the available scheduling policies are :

*Fair scheduling (FS)*::
  A non real-time, proportionally fair policy.
*First-in, first-out fixed priority (FIFO)*::
  A real-time, fixed-priority based FIFO policy.
*Round-robin (RR)*::
  A real-time, fixed-priority based policy with round-robin among same
  priority threads.

In addition, the kernel provides many thread synchronization facilities.
The relevant modules are :

module:kern/condition::
  Condition variable.
module:arch/cpu::
  Architecture-specific processor interface which provides interrupt
  control functions.
module:kern/mutex::
  Mutual exclusion lock.
module:kern/rcu::
  Read-Copy Update (RCU) lockless synchronization.
module:kern/semaphore::
  Semaphore.
module:kern/sleepq::
  Low level sleep queue.
module:kern/thread::
  Preemptive thread scheduling.
module:kern/work::
  Work queue of deferred asynchronous lightweight jobs.

All wait functions on synchronization objects can be time-bounded.
This includes waiting for a mutex lock, a condition variable, or a
semaphore.

Mutex implementations
~~~~~~~~~~~~~~~~~~~~~

In order to best satisfy either overall performance or latencies, the
kernel provides several mutex implementations for the interface provided
by the *mutex* module. Note that, whatever implementation is selected,
the *rtmutex* module is always available. The mutex implementations are :

module_mutex:adaptive::
  Adaptive spinning mutex, spinning instead of sleeping if the owner
  is running, in the hope the critical section is short and the mutex
  will be unlocked soon, to avoid expensive sleep/wakeup operations.
  This implementation should improve overall performance at the cost
  of increased latencies.
module_mutex:pi::
  Real-time mutex with priority inheritance. This implementation is a
  wrapper in front of the *rtmutex* module. It should improve latencies
  at the cost of overall performance.
module_mutex:plain::
  Default mutex, immediately sleeping on contention.

[[generic_development_tools]]
GENERIC DEVELOPMENT TOOLS
-------------------------

Along with the kernel sources are a set of generic data structures and
other development tools :

module:kern/bitmap::
  Arbitrary-length bit array.
module:kern/bulletin::
  Publish-subscribe mechanism.
module:kern/cbuf::
  Circular byte buffer.
module:kern/clock::
  Low resolution clock.
module:kern/error::
  Common errors and error handling functions.
module:kern/hash::
  Hash functions for integers and strings.
module:kern/hlist::
  Doubly-linked list specialized for forward traversals and O(1) removals.
module:kern/kmem::
  Object caching and general purpose memory allocator.
module:kern/list::
  Doubly-linked list.
module:kern/macros::
  Useful generic macros.
module:kern/perfmon::
  Performance monitoring.
module:kern/rbtree::
  Red-black tree.
module:kern/rdxtree::
  Radix tree (with integer keys).
module:kern/slist::
  Singly-linked list.
module:kern/syscnt::
  Generic 64-bits counter.
module:kern/timer::
  Low resolution timer.

X15 doesn't provide a generic queue interface, because the requirements
often vary too much. Similarly, it doesn't provide a hash table interface.
Instead, users can easily build specialized queues, hash tables and ring
buffers on top of the provided facilities.

See manpage:cenv for information about the subset of standard C interfaces
supported.

[[multiprocessor_support]]
MULTIPROCESSOR SUPPORT
----------------------

The X15 kernel is designed to support hardware with multiple processors.
The scheduler should scale well up to 16-32 processors, with one
run queue per processor. Threads can be bound to a specific set of
processors, or temporarily pinned for short durations. Non real-time
threads can be spontaneously migrated between processors in order to
maximize processor utility.

Here are some modules related to multiprocessor support :

module:kern/atomic::
  Inter-processor atomic operations.
module:kern/cpumap::
  Specialized bitmaps representing processor sets.
module:kern/percpu::
  Per-processor data.
module:kern/spinlock::
  Inter-processor spin locks.
module:kern/sref::
  Scalable multiprocessor reference counters.
module:kern/thread::
  Preemptive thread scheduling.
module:kern/xcall::
  Low level inter-processor function calls.

[[virtual_memoru]]
VIRTUAL MEMORY
--------------

TODO Write when the virtual memory system is rewritten.

[[real_time]]
REAL-TIME
---------

X15 complies with all the requirements of a real-time multiprocessor
system. It is a fully preemptible kernel with short, bounded
preemption-based critical sections. It provides real-time scheduling
policies and a complete priority inheritance algorithm. Preemption and
interrupts are clearly decoupled so that interrupts can remain enabled
as much as possible. Multiprocessor synchronization uses rigorously
fair spin locks. The modules related to real-time are :

module:kern/rtmutex::
  Mutual exclusion with priority inheritance.
module:kern/spinlock::
  Inter-processor spin locks.
module:kern/thread::
  Preemptive thread scheduling.
module:arch/trap::
  Interrupt and exception handling.
module:kern/turnstile::
  Low level priority propagation capable sleep queue.

Priority inheritance can also be enabled for regular mutexes. Please read
Victor Yodaiken's report {against-priority-inheritance} in order to fully
understand the implications of relying on priority inheritance.

TODO Separate man page with more description

[[portability]]
PORTABILITY
-----------

Despite the fact that the kernel currently only supports the x86
architecture, which will remain the reference port, the code is already
very portable, thanks to a clear separation between architecture-specific
and machine-independent modules, as well as good programming practice,
in particular regarding type widths, endianness, and memory models.

Ports are located in the arch directory. Here are the modules that
must provide interfaces expected by the machine-independent layer :

module:arch/atomic::
  Architecture-specific support for atomic instructions.
module:arch/cpu::
  Processor interface.
module:arch/pmap::
  Physical mappings, the MMU driver.
module:arch/strace::
  Stack tracing.
module:arch/tcb::
  Thread control block.
module:arch/trap::
  Interrupt and exception handling.

The machine-independent code assumes either an ILP32 or LP64 data model, and
a completely relaxed memory model as allowed by the C11 specification.

X15 currently requires a memory management unit, but that may change in
the future.

SEE
---

manpage:cenv

manpage:init

manpage:style

manpage:xbuild

{x15-operating-system}