<commit_before>#ifndef group_1_instructions_hpp
#define group_1_instructions_hpp

// Group 1 Instructions
// 01fo oooo aaaa bbbb  iiii iiii iiii iiii

	// f:  1 if can affect flags (and instruction type supports it), 0 if
	// flags unchanged.

	// o:  opcode
	// a: rA
	// b: rB
	// i:  16-bit immediate value

#define LIST_OF_INSTR_G1_RA_RB_IMM__COLLECTION_0(ARGS, VARNAME, VALUE) \
/* rA = rB + (zero-extended imm) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Addi) VALUE("addi") \
\
/* rA = rB + (zero-extended imm) + carry_flag */ \
/* Add with carry */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Adci) VALUE("adci") \
\
/* rA = rB - (zero-extended imm) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Subi) VALUE("subi") \
\
/* rA = rB + (~(zero-extended imm)) + carry_flag */ \
/* Subtract with borrow (6502 style) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Sbci) VALUE("sbci") \
\
\
\
/* rA = (zero-extended imm) - rB */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Rsbi) VALUE("rsbi") \
\
/* rA = rB * (zero-extended imm) */ \
ARGS(ra_rb_uimm16) VARNAME(Muli) VALUE("muli") \
\
/* rA = rB & (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Andi) VALUE("andi") \
\
/* rA = rB | (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Ori) VALUE("ori") \
\
\
\
/* rA = rB ^ (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Xori) VALUE("xori") \
\
/* Logical shift left */ \
/* rA = rB << (zero-extended imm) */ \
ARGS(ra_rb_uimm16) VARNAME(Lsli) VALUE("lsli") \
\
/* Logical shift right */ \
ARGS(ra_rb_uimm16) VARNAME(Lsri) VALUE("lsri") \
\
/* Arithmetic shift right */ \
ARGS(ra_rb_uimm16) VARNAME(Asri) VALUE("asri") \
\
\
\
/* Rotate  left by (zero-extended imm) bits, then store result in rA. */ \
ARGS(ra_rb_uimm16) VARNAME(Roli) VALUE("roli") \
\
/* Rotate  right by (zero-extended imm) bits, then store result in rA. */ \
ARGS(ra_rb_uimm16) VARNAME(Rori) VALUE("rori") \

#define LIST_OF_INSTR_G1_IMM__COLLECTION_0(ARGS, VARNAME, VALUE) \
/* Branch always,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bra) VALUE("bra") \
\
/* Branch never (effectively a NOP), */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bnv) VALUE("bnv") \
\
\
\
/* Branch when Z == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bne) VALUE("bne") \
\
/* Branch when Z == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Beq) VALUE("beq") \
\
/* Branch when C == 0 [unsigned less than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bcc) VALUE("bcc") \
\
/* Branch when C == 1 [unsigned greater than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bcs) VALUE("bcs") \
\
\
\
/* Branch when (C == 0 or Z == 1) [unsigned less than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bls) VALUE("bls") \
\
/* Branch when (C == 1 and Z == 0) [unsigned greater than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bhi) VALUE("bhi") \
\
/* Branch when N == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bpl) VALUE("bpl") \
\
/* Branch when N == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bmi) VALUE("bmi") \
\
\
\
/* Branch when V == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bvc) VALUE("bvc") \
\
/* Branch when V == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bvs) VALUE("bvs") \
\
/* Branch when N == V [signed greater than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bge) VALUE("bge") \
\
/* Branch when N != V [signed less than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Blt) VALUE("blt") \
\
\
\
/* Branch when (N == V and Z == 0) [signed greater than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bgt) VALUE("bgt") \
\
/* Branch when (N != V or Z == 1) [signed less than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Ble) VALUE("ble") \

#define LIST_OF_INSTR_G1_RA_RB_SIMM__COLLECTION_1(ARGS, VARNAME, VALUE) \
/* rA = rB ^ (sign-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_simm16) VARNAME(Xorsi) VALUE("xorsi")

/* Reserved for future expansion. */

#endif		// group_1_instructions_hpp
<commit_msg>Stuffs<commit_after>#ifndef group_1_instructions_hpp
#define group_1_instructions_hpp

// Group 1 Instructions
// 01fo oooo aaaa bbbb  iiii iiii iiii iiii

	// f:  1 if can affect flags (and instruction type supports it), 0 if
	// flags unchanged.

	// o:  opcode
	// a: rA
	// b: rB
	// i:  16-bit immediate value

#define LIST_OF_INSTR_G1_RA_RB_UIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \
/* rA = rB + (zero-extended imm) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Addi) VALUE("addi") \
\
/* rA = rB + (zero-extended imm) + carry_flag */ \
/* Add with carry */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Adci) VALUE("adci") \
\
/* rA = rB - (zero-extended imm) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Subi) VALUE("subi") \
\
/* rA = rB + (~(zero-extended imm)) + carry_flag */ \
/* Subtract with borrow (6502 style) */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Sbci) VALUE("sbci") \
\
\
\
/* rA = (zero-extended imm) - rB */ \
/* This instruction can affect N, V, Z, and C flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Rsbi) VALUE("rsbi") \
\
/* rA = rB * (zero-extended imm) */ \
ARGS(ra_rb_uimm16) VARNAME(Muli) VALUE("muli") \
\
/* rA = rB & (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Andi) VALUE("andi") \
\
/* rA = rB | (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Ori) VALUE("ori") \
\
\
\
/* rA = rB ^ (zero-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_uimm16) VARNAME(Xori) VALUE("xori") \
\
/* Logical shift left */ \
/* rA = rB << (zero-extended imm) */ \
ARGS(ra_rb_uimm16) VARNAME(Lsli) VALUE("lsli") \
\
/* Logical shift right */ \
ARGS(ra_rb_uimm16) VARNAME(Lsri) VALUE("lsri") \
\
/* Arithmetic shift right */ \
ARGS(ra_rb_uimm16) VARNAME(Asri) VALUE("asri") \
\
\
\
/* Rotate  left by (zero-extended imm) bits, then store result in rA. */ \
ARGS(ra_rb_uimm16) VARNAME(Roli) VALUE("roli") \
\
/* Rotate  right by (zero-extended imm) bits, then store result in rA. */ \
ARGS(ra_rb_uimm16) VARNAME(Rori) VALUE("rori") \

#define LIST_OF_INSTR_G1_SIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \
/* Branch always,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bra) VALUE("bra") \
\
/* Branch never (effectively a NOP), */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bnv) VALUE("bnv") \
\
\
\
/* Branch when Z == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bne) VALUE("bne") \
\
/* Branch when Z == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Beq) VALUE("beq") \
\
/* Branch when C == 0 [unsigned less than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bcc) VALUE("bcc") \
\
/* Branch when C == 1 [unsigned greater than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bcs) VALUE("bcs") \
\
\
\
/* Branch when (C == 0 or Z == 1) [unsigned less than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bls) VALUE("bls") \
\
/* Branch when (C == 1 and Z == 0) [unsigned greater than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bhi) VALUE("bhi") \
\
/* Branch when N == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bpl) VALUE("bpl") \
\
/* Branch when N == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bmi) VALUE("bmi") \
\
\
\
/* Branch when V == 0,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bvc) VALUE("bvc") \
\
/* Branch when V == 1,  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bvs) VALUE("bvs") \
\
/* Branch when N == V [signed greater than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bge) VALUE("bge") \
\
/* Branch when N != V [signed less than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Blt) VALUE("blt") \
\
\
\
/* Branch when (N == V and Z == 0) [signed greater than],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Bgt) VALUE("bgt") \
\
/* Branch when (N != V or Z == 1) [signed less than or equal],  */ \
/* to ((pc when instruction starts) */ \
/* + (sign-extended 16-bit immediate value)) */ \
ARGS(simm16) VARNAME(Ble) VALUE("ble") \

#define LIST_OF_INSTR_G1_RA_RB_SIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \
/* rA = rB ^ (sign-extended imm) */ \
/* This instruction can affect the N and Z flags. */ \
ARGS(ra_rb_simm16) VARNAME(Xorsi) VALUE("xorsi")

/* Reserved for future expansion. */

#define LIST_OF_GROUP_1_INSTRUCTIONS(ARGS, VARNAME, VALUE) \
LIST_OF_INSTR_G1_RA_RB_UIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \
LIST_OF_INSTR_G1_SIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \
LIST_OF_INSTR_G1_RA_RB_SIMM16__COLLECTION_0(ARGS, VARNAME, VALUE) \

#endif		// group_1_instructions_hpp
<|endoftext|>


<commit_before>
#include <map>
#include <utility>
#include <string.h>

#include <disir/disir.h>
#include <disir/util.h>
#include <disir/plugin.h>

#define MQ_ENQUEUE(q, x)                                                \
    {                                                                   \
        (x)->next = NULL;                                               \
                                                                        \
        if ((q) == NULL) {                                              \
            (q) = (x)->prev = (x);                                      \
        } else {                                                        \
            (q)->prev->next = (x);                                      \
            (x)->prev = (q)->prev;                                      \
            (q)->prev = (x);                                            \
        }                                                               \
    }

typedef enum disir_status (*output_mold)(struct disir_mold **);

#include "basic_keyval.cc"
#include "basic_section.cc"
#include "json_test_mold.cc"
#include "restriction_keyval_numeric_types.cc"
#include "restriction_entries.cc"
#include "restriction_config_parent_keyval_min_entry.cc"
#include "restriction_config_parent_keyval_max_entry.cc"
#include "restriction_config_parent_section_max_entry.cc"
#include "restriction_section_parent_keyval_max_entry.cc"
#include "basic_version_difference.cc"
#include "complex_section.cc"

// Forward declaration
enum disir_status dio_test_config_read (struct disir_instance *disir,
                                        void *storage, const char *entry_id,
                                        struct disir_mold *mold, struct disir_config **config);
enum disir_status dio_test_config_entries (struct disir_instance *disir,
                                           void *storage, struct disir_entry **entries);
enum disir_status dio_test_config_version (struct disir_instance *disir, void *storage,
                                           const char *entry_id, struct semantic_version *semver);
enum disir_status dio_test_mold_read (struct disir_instance *disir, void *storage,
                                      const char *entry_id, struct disir_mold **mold);
enum disir_status dio_test_mold_entries (struct disir_instance *disir,
                                         void *storage, struct disir_entry **entries);

struct cmp_str
{
   bool operator()(char const *a, char const *b)
   {
      return strcmp(a, b) < 0;
   }
};

//! Internal static map to store test molds by id
static std::map<const char *, output_mold, cmp_str> molds = {
    std::make_pair ("basic_keyval", basic_keyval),
    std::make_pair ("basic_section", basic_section),
    std::make_pair ("json_test_mold", json_test_mold),
    std::make_pair ("restriction_keyval_numeric_types", restriction_keyval_numeric_types),
    std::make_pair ("restriction_entries", restriction_entries),
    std::make_pair ("restriction_config_parent_keyval_min_entry",
                    restriction_config_parent_keyval_min_entry),
    std::make_pair ("restriction_config_parent_keyval_max_entry",
                    restriction_config_parent_keyval_max_entry),
    std::make_pair ("restriction_config_parent_section_max_entry",
                    restriction_config_parent_section_max_entry),
    std::make_pair ("restriction_section_parent_keyval_max_entry",
                    restriction_section_parent_keyval_max_entry),
    std::make_pair ("basic_version_difference", basic_version_difference),
    std::make_pair ("complex_section", complex_section),
};

enum disir_status
dio_test_config_read (struct disir_instance *disir, void *storage, const char *entry_id,
                      struct disir_mold *mold, struct disir_config **config)
{
    enum disir_status status;
    output_mold func_mold;

    fprintf (stderr, "TEST_CONFIG_READ: stroage: %p\n", storage);

    func_mold = molds[entry_id];
    if (func_mold == NULL)
        return DISIR_STATUS_INVALID_ARGUMENT;

    status = func_mold (&mold);
    if (status != DISIR_STATUS_OK)
        return status;

    status = disir_generate_config_from_mold (mold, NULL, config);
    // We are finished with our allocated mold
    disir_mold_finished (&mold);
    return status;
}

enum disir_status
dio_test_config_entries (struct disir_instance *disir, void *storage, struct disir_entry **entries)
{
    return dio_test_mold_entries (disir, storage, entries);
}

enum disir_status
dio_test_config_query (struct disir_instance *disir, void *storage, const char *entry_id)
{
    if (molds[entry_id] == NULL)
        return DISIR_STATUS_NOT_EXIST;
    else
        return DISIR_STATUS_EXISTS;
}

enum disir_status
dio_test_mold_read (struct disir_instance *disir, void *storage,
                    const char *entry_id, struct disir_mold **mold)
{
    enum disir_status status;
    output_mold func_mold;

    func_mold = molds[entry_id];
    if (func_mold == NULL)
        return DISIR_STATUS_INVALID_ARGUMENT;

    status = func_mold (mold);
    if (status != DISIR_STATUS_OK)
        return status;

    return DISIR_STATUS_OK;;
}

enum disir_status
dio_test_mold_entries (struct disir_instance *disir, void *storage, struct disir_entry **entries)
{
    struct disir_entry *queue;
    struct disir_entry *entry;

    queue = NULL;

    for (auto i = molds.begin(); i != molds.end(); ++i)
    {
        entry  = (struct disir_entry *) calloc (1, sizeof (struct disir_entry));
        if (entry == NULL)
            continue;

        entry->de_entry_name = strdup (i->first);
        entry->DE_READABLE = 1;
        entry->DE_WRITABLE = 1;
        MQ_ENQUEUE (queue, entry);
    }

    *entries = queue;

    return DISIR_STATUS_OK;
}

enum disir_status
dio_test_mold_query (struct disir_instance *disir, void *storage, const char *entry_id)
{
    if (molds[entry_id] == NULL)
        return DISIR_STATUS_NOT_EXIST;
    else
        return DISIR_STATUS_EXISTS;
}

extern "C" enum disir_status
dio_register_plugin (struct disir_instance *disir, const char *name)
{
    struct disir_plugin plugin;

    plugin.dp_name = (char *) "test";
    plugin.dp_description = (char *) "A collection of various molds to enumerate functionality in libdisir";
    plugin.dp_type = (char *) "test";
    // Storage space unused.
    plugin.dp_storage = NULL;
    plugin.dp_plugin_finished = NULL;

    plugin.dp_config_read = dio_test_config_read;
    plugin.dp_config_write = NULL;
    plugin.dp_config_entries = dio_test_config_entries;
    plugin.dp_config_query = dio_test_config_query;

    plugin.dp_mold_read = dio_test_mold_read;
    plugin.dp_mold_write = NULL;
    plugin.dp_mold_entries = dio_test_mold_entries;
    plugin.dp_mold_query = dio_test_mold_query;

    return disir_plugin_register (disir, &plugin);
}

<commit_msg>plugin test: remove old print<commit_after>
#include <map>
#include <utility>
#include <string.h>

#include <disir/disir.h>
#include <disir/util.h>
#include <disir/plugin.h>

#define MQ_ENQUEUE(q, x)                                                \
    {                                                                   \
        (x)->next = NULL;                                               \
                                                                        \
        if ((q) == NULL) {                                              \
            (q) = (x)->prev = (x);                                      \
        } else {                                                        \
            (q)->prev->next = (x);                                      \
            (x)->prev = (q)->prev;                                      \
            (q)->prev = (x);                                            \
        }                                                               \
    }

typedef enum disir_status (*output_mold)(struct disir_mold **);

#include "basic_keyval.cc"
#include "basic_section.cc"
#include "json_test_mold.cc"
#include "restriction_keyval_numeric_types.cc"
#include "restriction_entries.cc"
#include "restriction_config_parent_keyval_min_entry.cc"
#include "restriction_config_parent_keyval_max_entry.cc"
#include "restriction_config_parent_section_max_entry.cc"
#include "restriction_section_parent_keyval_max_entry.cc"
#include "basic_version_difference.cc"
#include "complex_section.cc"

// Forward declaration
enum disir_status dio_test_config_read (struct disir_instance *disir,
                                        void *storage, const char *entry_id,
                                        struct disir_mold *mold, struct disir_config **config);
enum disir_status dio_test_config_entries (struct disir_instance *disir,
                                           void *storage, struct disir_entry **entries);
enum disir_status dio_test_config_version (struct disir_instance *disir, void *storage,
                                           const char *entry_id, struct semantic_version *semver);
enum disir_status dio_test_mold_read (struct disir_instance *disir, void *storage,
                                      const char *entry_id, struct disir_mold **mold);
enum disir_status dio_test_mold_entries (struct disir_instance *disir,
                                         void *storage, struct disir_entry **entries);

struct cmp_str
{
   bool operator()(char const *a, char const *b)
   {
      return strcmp(a, b) < 0;
   }
};

//! Internal static map to store test molds by id
static std::map<const char *, output_mold, cmp_str> molds = {
    std::make_pair ("basic_keyval", basic_keyval),
    std::make_pair ("basic_section", basic_section),
    std::make_pair ("json_test_mold", json_test_mold),
    std::make_pair ("restriction_keyval_numeric_types", restriction_keyval_numeric_types),
    std::make_pair ("restriction_entries", restriction_entries),
    std::make_pair ("restriction_config_parent_keyval_min_entry",
                    restriction_config_parent_keyval_min_entry),
    std::make_pair ("restriction_config_parent_keyval_max_entry",
                    restriction_config_parent_keyval_max_entry),
    std::make_pair ("restriction_config_parent_section_max_entry",
                    restriction_config_parent_section_max_entry),
    std::make_pair ("restriction_section_parent_keyval_max_entry",
                    restriction_section_parent_keyval_max_entry),
    std::make_pair ("basic_version_difference", basic_version_difference),
    std::make_pair ("complex_section", complex_section),
};

enum disir_status
dio_test_config_read (struct disir_instance *disir, void *storage, const char *entry_id,
                      struct disir_mold *mold, struct disir_config **config)
{
    enum disir_status status;
    output_mold func_mold;

    func_mold = molds[entry_id];
    if (func_mold == NULL)
        return DISIR_STATUS_INVALID_ARGUMENT;

    status = func_mold (&mold);
    if (status != DISIR_STATUS_OK)
        return status;

    status = disir_generate_config_from_mold (mold, NULL, config);
    // We are finished with our allocated mold
    disir_mold_finished (&mold);
    return status;
}

enum disir_status
dio_test_config_entries (struct disir_instance *disir, void *storage, struct disir_entry **entries)
{
    return dio_test_mold_entries (disir, storage, entries);
}

enum disir_status
dio_test_config_query (struct disir_instance *disir, void *storage, const char *entry_id)
{
    if (molds[entry_id] == NULL)
        return DISIR_STATUS_NOT_EXIST;
    else
        return DISIR_STATUS_EXISTS;
}

enum disir_status
dio_test_mold_read (struct disir_instance *disir, void *storage,
                    const char *entry_id, struct disir_mold **mold)
{
    enum disir_status status;
    output_mold func_mold;

    func_mold = molds[entry_id];
    if (func_mold == NULL)
        return DISIR_STATUS_INVALID_ARGUMENT;

    status = func_mold (mold);
    if (status != DISIR_STATUS_OK)
        return status;

    return DISIR_STATUS_OK;;
}

enum disir_status
dio_test_mold_entries (struct disir_instance *disir, void *storage, struct disir_entry **entries)
{
    struct disir_entry *queue;
    struct disir_entry *entry;

    queue = NULL;

    for (auto i = molds.begin(); i != molds.end(); ++i)
    {
        entry  = (struct disir_entry *) calloc (1, sizeof (struct disir_entry));
        if (entry == NULL)
            continue;

        entry->de_entry_name = strdup (i->first);
        entry->DE_READABLE = 1;
        entry->DE_WRITABLE = 1;
        MQ_ENQUEUE (queue, entry);
    }

    *entries = queue;

    return DISIR_STATUS_OK;
}

enum disir_status
dio_test_mold_query (struct disir_instance *disir, void *storage, const char *entry_id)
{
    if (molds[entry_id] == NULL)
        return DISIR_STATUS_NOT_EXIST;
    else
        return DISIR_STATUS_EXISTS;
}

extern "C" enum disir_status
dio_register_plugin (struct disir_instance *disir, const char *name)
{
    struct disir_plugin plugin;

    plugin.dp_name = (char *) "test";
    plugin.dp_description = (char *) "A collection of various molds to enumerate functionality in libdisir";
    plugin.dp_type = (char *) "test";
    // Storage space unused.
    plugin.dp_storage = NULL;
    plugin.dp_plugin_finished = NULL;

    plugin.dp_config_read = dio_test_config_read;
    plugin.dp_config_write = NULL;
    plugin.dp_config_entries = dio_test_config_entries;
    plugin.dp_config_query = dio_test_config_query;

    plugin.dp_mold_read = dio_test_mold_read;
    plugin.dp_mold_write = NULL;
    plugin.dp_mold_entries = dio_test_mold_entries;
    plugin.dp_mold_query = dio_test_mold_query;

    return disir_plugin_register (disir, &plugin);
}

<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 * Time keeping for memcached.
 *
 * Orginally part of memcached.c this module encapsulates some time
 * keeping functions that are exposed via the engine API.
 *
 * This module implements a prviate and non-data path "tick" function
 * which maintains a view of time.
 *
 *  - see mc_time_clock_tick
 *
 * This module provides public time methods that allow performant
 * data-path code access to:
 *
 *  1. A monotonic interval time (great for relative expiry and timed
 *     locks). - mc_time_get_current_time()
 *  2. approximation of system-clock. - mc_time_convert_to_abs_time()
 *    2.1 The following returns system clock time -
 *        mc_time_convert_to_abs_time(mc_time_get_current_time())
 *  3. A method for work with expiry timestamps as per the memcached
 *     protocol - mc_time_convert_to_real_time()
 *
 */
#include <signal.h>

#include "config.h"
#include "memcached.h"
#include "mc_time.h"

#include <atomic>

extern std::atomic<bool> memcached_shutdown;

/*
 * This constant defines the seconds between libevent clock callbacks.
 * This roughly equates to how frequency of gethrtime calls made.
 */
const time_t memcached_clock_tick_seconds = 1;

/*
 * This constant defines the frequency of system clock checks.
 * This equates to an extra gettimeofday every 'n' seconds.
 */
const time_t memcached_check_system_time = 60;

/*
 * This constant defines the maximum relative time (30 days in seconds)
 * time values above this are interpretted as absolute.
 */
const time_t memcached_maximum_relative_time = 60*60*24*30;

static std::atomic<rel_time_t> memcached_uptime(0);
static volatile time_t memcached_epoch = 0;
static volatile uint64_t memcached_monotonic_start = 0;
static struct event_base* main_ev_base = NULL;

static void mc_time_clock_event_handler(evutil_socket_t fd, short which, void *arg);
static void mc_gather_timing_samples(void);

/*
 * Init internal state and start the timer event callback.
 */
void mc_time_init(struct event_base* ev_base) {

    main_ev_base = ev_base;

    mc_time_init_epoch();

    /* tick once to begin procedings */
    mc_time_clock_tick();

    /* Begin the time keeping by registering for a time based callback */
    mc_time_clock_event_handler(0, 0, 0);
}

/*
 * Initisalise our "EPOCH" variables
 * In order to provide a monotonic "uptime" and track system time, we record
 * some points in time.
 */
void mc_time_init_epoch(void) {
    struct timeval t;
    memcached_uptime = 0;
    memcached_monotonic_start = cb_get_monotonic_seconds();
    cb_get_timeofday(&t);
    memcached_epoch = t.tv_sec;
}

/*
 * Return a monotonically increasing value.
 * The value returned represents seconds since memcached started.
 */
rel_time_t mc_time_get_current_time(void) {
    return memcached_uptime;
}

/// @return true if a + b would overflow rel_time_t
static bool would_overflow(rel_time_t a, std::chrono::seconds b) {
    return a > (std::numeric_limits<rel_time_t>::max() - b.count());
}

/**
 * @return true if a + b would overflow int64_t
 */
static bool would_overflow(int64_t a, int64_t b) {
    return a > (std::numeric_limits<int64_t>::max() - b);
}

// The above would_overflow(a. b) assumes rel_time_t is unsigned
static_assert(std::is_unsigned<rel_time_t>::value,
              "would_overflow assumes rel_time_t is unsigned");

rel_time_t mc_time_convert_to_real_time(rel_time_t t, cb::ExpiryLimit limit) {
    rel_time_t rv = 0;

    int64_t epoch{memcached_epoch};
    int64_t uptime{memcached_uptime.load()};

    if (t > memcached_maximum_relative_time) { // t is absolute

        // Ensure overflow is predictable (we stay at max rel_time_t)
        if (would_overflow(epoch, uptime)) {
            return std::numeric_limits<rel_time_t>::max();
        }
        if (limit &&
            would_overflow(gsl::narrow_cast<rel_time_t>(epoch + uptime),
                           limit.get())) {
            return std::numeric_limits<rel_time_t>::max();
        }

        if (limit && t > (epoch + uptime + limit.get().count())) {
            if (would_overflow(gsl::narrow_cast<int64_t>(epoch + uptime),
                               limit.get().count())) {
                t = std::numeric_limits<rel_time_t>::max();
            } else {
                t = gsl::narrow<rel_time_t>((epoch + uptime) +
                                            limit.get().count());
            }
        }

        /* if item expiration is at/before the server started, give it an
           expiration time of 1 second after the server started.
           (because 0 means don't expire).  without this, we'd
           underflow and wrap around to some large value way in the
           future, effectively making items expiring in the past
           really expiring never */
        if (t <= epoch) {
            rv = (rel_time_t)1;
        } else {
            rv = (rel_time_t)(t - epoch);
        }
    } else if (t != 0) { // t is relative
        if (limit && t > limit.get().count()) {
            t = gsl::narrow<rel_time_t>(limit.get().count());
        }

        // Ensure overflow is predictable (we stay at max rel_time_t)
        if (would_overflow(t, uptime)) {
            rv = std::numeric_limits<rel_time_t>::max();
        } else {
            if (limit &&
                would_overflow(gsl::narrow_cast<rel_time_t>(t + uptime),
                               limit.get())) {
                rv = std::numeric_limits<rel_time_t>::max();
            } else {
                rv = (rel_time_t)(t + uptime);
            }
        }
    }

    return rv;
}

/*
 * Convert the relative time to an absolute time (relative to EPOCH ;) )
 */
time_t mc_time_convert_to_abs_time(const rel_time_t rel_time) {
    return memcached_epoch + rel_time;
}

/*
 * clock_handler - libevent call back.
 * This method is called (ticks) every 'memcached_clock_tick_seconds' and
 * primarily keeps time flowing.
 */
static void mc_time_clock_event_handler(evutil_socket_t fd, short which, void *arg) {
    static bool initialized = false;
    static struct event clockevent;
    struct timeval t;

    t.tv_sec = (long)memcached_clock_tick_seconds;
    t.tv_usec = 0;

    if (memcached_shutdown) {
        return ;
    }

    if (initialized) {
        /* only delete the event if it's actually there. */
        evtimer_del(&clockevent);
    } else {
        initialized = true;
    }

    evtimer_set(&clockevent, mc_time_clock_event_handler, 0);
    event_base_set(main_ev_base, &clockevent);
    evtimer_add(&clockevent, &t);

    mc_time_clock_tick();
}

/*
 * Update a number of time keeping variables and account for system
 * clock changes.
 */
void mc_time_clock_tick(void) {
    static uint64_t check_system_time = 0;
    static bool previous_time_valid = false;
    static struct timeval previous_time = {0, 0};

    /* calculate our monotonic uptime */
    memcached_uptime = (rel_time_t)(cb_get_monotonic_seconds() - memcached_monotonic_start + cb_get_uptime_offset());

    /* Collect samples */
    mc_gather_timing_samples();

    /*
      every 'memcached_check_system_time' seconds, keep an eye on the
      system clock.
    */
    if (memcached_uptime >= check_system_time) {
        struct timeval timeofday;
        cb_get_timeofday(&timeofday);
        time_t difference = labs(timeofday.tv_sec - previous_time.tv_sec);
        /* perform a fuzzy check on time, this allows 2 seconds each way. */
        if (previous_time_valid
            && ((difference > memcached_check_system_time + 1)
            || (difference < memcached_check_system_time - 1))) {
            if (settings.extensions.logger) {
                /* log all variables used in time calculations */
                LOG_WARNING(
                        "system clock changed? difference = {}, "
                        "memcached_epoch = {}, "
                        "memcached_uptime = {}, new memcached_epoch = {}, "
                        "next check {}",
                        difference,
                        memcached_epoch,
                        memcached_uptime.load(),
                        (timeofday.tv_sec - memcached_uptime),
                        check_system_time + memcached_check_system_time);
            }
            /* adjust memcached_epoch to ensure correct timeofday can
               be calculated by clients*/
            memcached_epoch = timeofday.tv_sec - memcached_uptime;
        }

        /* move our checksystem time marker to trigger the next check
           at the correct interval*/
        check_system_time += memcached_check_system_time;

        previous_time_valid = true;
        previous_time = timeofday;
    }

    // check on tasks to be made runnable in the future
    executorPool->clockTick();
}

static void mc_gather_timing_samples(void) {
    bucketsForEach([](Bucket& bucket, void *) -> bool {
        bucket.timings.sample(std::chrono::seconds(1));
        return true;
    }, nullptr);
}
<commit_msg>MB-28635: Fix race in memcached_epoch access<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 * Time keeping for memcached.
 *
 * Orginally part of memcached.c this module encapsulates some time
 * keeping functions that are exposed via the engine API.
 *
 * This module implements a prviate and non-data path "tick" function
 * which maintains a view of time.
 *
 *  - see mc_time_clock_tick
 *
 * This module provides public time methods that allow performant
 * data-path code access to:
 *
 *  1. A monotonic interval time (great for relative expiry and timed
 *     locks). - mc_time_get_current_time()
 *  2. approximation of system-clock. - mc_time_convert_to_abs_time()
 *    2.1 The following returns system clock time -
 *        mc_time_convert_to_abs_time(mc_time_get_current_time())
 *  3. A method for work with expiry timestamps as per the memcached
 *     protocol - mc_time_convert_to_real_time()
 *
 */
#include <signal.h>

#include "config.h"
#include "memcached.h"
#include "mc_time.h"

#include <atomic>

extern std::atomic<bool> memcached_shutdown;

/*
 * This constant defines the seconds between libevent clock callbacks.
 * This roughly equates to how frequency of gethrtime calls made.
 */
const time_t memcached_clock_tick_seconds = 1;

/*
 * This constant defines the frequency of system clock checks.
 * This equates to an extra gettimeofday every 'n' seconds.
 */
const time_t memcached_check_system_time = 60;

/*
 * This constant defines the maximum relative time (30 days in seconds)
 * time values above this are interpretted as absolute.
 */
const time_t memcached_maximum_relative_time = 60*60*24*30;

static std::atomic<rel_time_t> memcached_uptime(0);
static std::atomic<time_t> memcached_epoch(0);
static volatile uint64_t memcached_monotonic_start = 0;
static struct event_base* main_ev_base = NULL;

static void mc_time_clock_event_handler(evutil_socket_t fd, short which, void *arg);
static void mc_gather_timing_samples(void);

/*
 * Init internal state and start the timer event callback.
 */
void mc_time_init(struct event_base* ev_base) {

    main_ev_base = ev_base;

    mc_time_init_epoch();

    /* tick once to begin procedings */
    mc_time_clock_tick();

    /* Begin the time keeping by registering for a time based callback */
    mc_time_clock_event_handler(0, 0, 0);
}

/*
 * Initisalise our "EPOCH" variables
 * In order to provide a monotonic "uptime" and track system time, we record
 * some points in time.
 */
void mc_time_init_epoch(void) {
    struct timeval t;
    memcached_uptime = 0;
    memcached_monotonic_start = cb_get_monotonic_seconds();
    cb_get_timeofday(&t);
    memcached_epoch = t.tv_sec;
}

/*
 * Return a monotonically increasing value.
 * The value returned represents seconds since memcached started.
 */
rel_time_t mc_time_get_current_time(void) {
    return memcached_uptime;
}

/// @return true if a + b would overflow rel_time_t
static bool would_overflow(rel_time_t a, std::chrono::seconds b) {
    return a > (std::numeric_limits<rel_time_t>::max() - b.count());
}

/**
 * @return true if a + b would overflow int64_t
 */
static bool would_overflow(int64_t a, int64_t b) {
    return a > (std::numeric_limits<int64_t>::max() - b);
}

// The above would_overflow(a. b) assumes rel_time_t is unsigned
static_assert(std::is_unsigned<rel_time_t>::value,
              "would_overflow assumes rel_time_t is unsigned");

rel_time_t mc_time_convert_to_real_time(rel_time_t t, cb::ExpiryLimit limit) {
    rel_time_t rv = 0;

    int64_t epoch{memcached_epoch.load()};
    int64_t uptime{memcached_uptime.load()};

    if (t > memcached_maximum_relative_time) { // t is absolute

        // Ensure overflow is predictable (we stay at max rel_time_t)
        if (would_overflow(epoch, uptime)) {
            return std::numeric_limits<rel_time_t>::max();
        }
        if (limit &&
            would_overflow(gsl::narrow_cast<rel_time_t>(epoch + uptime),
                           limit.get())) {
            return std::numeric_limits<rel_time_t>::max();
        }

        if (limit && t > (epoch + uptime + limit.get().count())) {
            if (would_overflow(gsl::narrow_cast<int64_t>(epoch + uptime),
                               limit.get().count())) {
                t = std::numeric_limits<rel_time_t>::max();
            } else {
                t = gsl::narrow<rel_time_t>((epoch + uptime) +
                                            limit.get().count());
            }
        }

        /* if item expiration is at/before the server started, give it an
           expiration time of 1 second after the server started.
           (because 0 means don't expire).  without this, we'd
           underflow and wrap around to some large value way in the
           future, effectively making items expiring in the past
           really expiring never */
        if (t <= epoch) {
            rv = (rel_time_t)1;
        } else {
            rv = (rel_time_t)(t - epoch);
        }
    } else if (t != 0) { // t is relative
        if (limit && t > limit.get().count()) {
            t = gsl::narrow<rel_time_t>(limit.get().count());
        }

        // Ensure overflow is predictable (we stay at max rel_time_t)
        if (would_overflow(t, uptime)) {
            rv = std::numeric_limits<rel_time_t>::max();
        } else {
            if (limit &&
                would_overflow(gsl::narrow_cast<rel_time_t>(t + uptime),
                               limit.get())) {
                rv = std::numeric_limits<rel_time_t>::max();
            } else {
                rv = (rel_time_t)(t + uptime);
            }
        }
    }

    return rv;
}

/*
 * Convert the relative time to an absolute time (relative to EPOCH ;) )
 */
time_t mc_time_convert_to_abs_time(const rel_time_t rel_time) {
    return memcached_epoch + rel_time;
}

/*
 * clock_handler - libevent call back.
 * This method is called (ticks) every 'memcached_clock_tick_seconds' and
 * primarily keeps time flowing.
 */
static void mc_time_clock_event_handler(evutil_socket_t fd, short which, void *arg) {
    static bool initialized = false;
    static struct event clockevent;
    struct timeval t;

    t.tv_sec = (long)memcached_clock_tick_seconds;
    t.tv_usec = 0;

    if (memcached_shutdown) {
        return ;
    }

    if (initialized) {
        /* only delete the event if it's actually there. */
        evtimer_del(&clockevent);
    } else {
        initialized = true;
    }

    evtimer_set(&clockevent, mc_time_clock_event_handler, 0);
    event_base_set(main_ev_base, &clockevent);
    evtimer_add(&clockevent, &t);

    mc_time_clock_tick();
}

/*
 * Update a number of time keeping variables and account for system
 * clock changes.
 */
void mc_time_clock_tick(void) {
    static uint64_t check_system_time = 0;
    static bool previous_time_valid = false;
    static struct timeval previous_time = {0, 0};

    /* calculate our monotonic uptime */
    memcached_uptime = (rel_time_t)(cb_get_monotonic_seconds() - memcached_monotonic_start + cb_get_uptime_offset());

    /* Collect samples */
    mc_gather_timing_samples();

    /*
      every 'memcached_check_system_time' seconds, keep an eye on the
      system clock.
    */
    if (memcached_uptime >= check_system_time) {
        struct timeval timeofday;
        cb_get_timeofday(&timeofday);
        time_t difference = labs(timeofday.tv_sec - previous_time.tv_sec);
        /* perform a fuzzy check on time, this allows 2 seconds each way. */
        if (previous_time_valid
            && ((difference > memcached_check_system_time + 1)
            || (difference < memcached_check_system_time - 1))) {
            if (settings.extensions.logger) {
                /* log all variables used in time calculations */
                LOG_WARNING(
                        "system clock changed? difference = {}, "
                        "memcached_epoch = {}, "
                        "memcached_uptime = {}, new memcached_epoch = {}, "
                        "next check {}",
                        difference,
                        memcached_epoch.load(),
                        memcached_uptime.load(),
                        (timeofday.tv_sec - memcached_uptime),
                        check_system_time + memcached_check_system_time);
            }
            /* adjust memcached_epoch to ensure correct timeofday can
               be calculated by clients*/
            memcached_epoch.store(timeofday.tv_sec - memcached_uptime);
        }

        /* move our checksystem time marker to trigger the next check
           at the correct interval*/
        check_system_time += memcached_check_system_time;

        previous_time_valid = true;
        previous_time = timeofday;
    }

    // check on tasks to be made runnable in the future
    executorPool->clockTick();
}

static void mc_gather_timing_samples(void) {
    bucketsForEach([](Bucket& bucket, void *) -> bool {
        bucket.timings.sample(std::chrono::seconds(1));
        return true;
    }, nullptr);
}
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2017 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */
#include "config.h"
#include "memcached.h"
#include "mcaudit.h"
#include "memcached_audit_events.h"
#include "buckets.h"
#include "debug_helpers.h"
#include "runtime.h"

#include <memcached/audit_interface.h>
#include <cJSON.h>
#include <memcached/isotime.h>
#include <relaxed_atomic.h>

static Couchbase::RelaxedAtomic<bool> audit_enabled;

/**
 * Create the typical memcached audit object. It constists of a
 * timestamp, the socket endpoints and the creds. Then each audit event
 * may add event-specific content.
 *
 * @param c the connection object
 * @return the cJSON object containing the basic information
 */
static unique_cJSON_ptr create_memcached_audit_object(const Connection *c) {
    cJSON *root = cJSON_CreateObject();

    std::string timestamp = ISOTime::generatetimestamp();
    cJSON_AddStringToObject(root, "timestamp", timestamp.c_str());

    cJSON_AddStringToObject(root, "peername", c->getPeername().c_str());
    cJSON_AddStringToObject(root, "sockname", c->getSockname().c_str());
    cJSON *source = cJSON_CreateObject();
    cJSON_AddStringToObject(source, "source", "memcached");
    cJSON_AddStringToObject(source, "user", c->getUsername());
    cJSON_AddItemToObject(root, "real_userid", source);

    return unique_cJSON_ptr(root);
}

/**
 * Convert the JSON object to text and send it to the audit framework
 *
 * @param c the connection object requesting the call
 * @param id the audit identifier
 * @param event the payload of the audit description
 * @param warn what to log if we're failing to put the audit event
 */
static void do_audit(const Connection* c,
                     uint32_t id,
                     unique_cJSON_ptr& event,
                     const char* warn) {
    auto text = to_string(event, false);
    auto status = put_audit_event(get_audit_handle(), id, text.data(),
                                  text.length());

    if (status != AUDIT_SUCCESS) {
        LOG_WARNING(c, "%s: %s", warn, text.c_str());
    }
}

void audit_auth_failure(const Connection *c, const char *reason) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "reason", reason);

    do_audit(c, MEMCACHED_AUDIT_AUTHENTICATION_FAILED, root,
             "Failed to send AUTH FAILED audit event");
}

void audit_auth_success(const Connection *c) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    do_audit(c, MEMCACHED_AUDIT_AUTHENTICATION_SUCCEEDED, root,
             "Failed to send AUTH SUCCESS audit event");
}

void audit_bucket_flush(const Connection *c, const char *bucket) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "bucket", bucket);

    do_audit(c, MEMCACHED_AUDIT_BUCKET_FLUSH, root,
             "Failed to send BUCKET_FLUSH audit event");
}

void audit_dcp_open(const Connection *c) {
    if (!audit_enabled) {
        return;
    }
    if (c->isInternal()) {
        LOG_INFO(c, "Open DCP stream with admin credentials");
    } else {
        auto root = create_memcached_audit_object(c);
        cJSON_AddStringToObject(root.get(), "bucket", getBucketName(c));

        do_audit(c, MEMCACHED_AUDIT_OPENED_DCP_CONNECTION, root,
                 "Failed to send DCP open connection "
                 "audit event to audit daemon");
    }
}

void audit_set_privilege_debug_mode(const Connection* c, bool enable) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddBoolToObject(root.get(), "enable", enable);
    do_audit(c, MEMCACHED_AUDIT_PRIVILEGE_DEBUG_CONFIGURED, root,
             "Failed to send modifications in privilege debug state "
             "audit event to audit daemon");
}

void audit_privilege_debug(const Connection* c,
                           const std::string& command,
                           const std::string& bucket,
                           const std::string& privilege,
                           const std::string& context) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "command", command.c_str());
    cJSON_AddStringToObject(root.get(), "bucket", bucket.c_str());
    cJSON_AddStringToObject(root.get(), "privilege", privilege.c_str());
    cJSON_AddStringToObject(root.get(), "context", context.c_str());

    do_audit(c, MEMCACHED_AUDIT_PRIVILEGE_DEBUG, root,
             "Failed to send privilege debug audit event to audit daemon");
}

void audit_command_access_failed(const McbpConnection *c) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    char buffer[256];
    memset(buffer, 0, sizeof(buffer));
    // Deliberately ignore failure of bytes_to_output_string
    // We'll either have a partial string or no string.
    bytes_to_output_string(buffer, sizeof(buffer), c->getId(), true,
                           "Access to command is not allowed:",
                           reinterpret_cast<const char*>(&c->getBinaryHeader()),
                           sizeof(protocol_binary_request_header));
    cJSON_AddStringToObject(root.get(), "packet", buffer);
    do_audit(c, MEMCACHED_AUDIT_COMMAND_ACCESS_FAILURE, root, buffer);
}

void audit_invalid_packet(const McbpConnection *c) {
    if (!audit_enabled) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    char buffer[256];
    memset(buffer, 0, sizeof(buffer));
    // Deliberately ignore failure of bytes_to_output_string
    // We'll either have a partial string or no string.
    bytes_to_output_string(buffer, sizeof(buffer), c->getId(), true,
                           "Invalid Packet:",
                           reinterpret_cast<const char*>(&c->getBinaryHeader()),
                           sizeof(protocol_binary_request_header));
    cJSON_AddStringToObject(root.get(), "packet", buffer);
    do_audit(c, MEMCACHED_AUDIT_INVALID_PACKET, root, buffer);
}

bool mc_audit_event(uint32_t audit_eventid,
                    const void* payload,
                    size_t length) {
    if (!audit_enabled) {
        return true;
    }

    return put_audit_event(get_audit_handle(), audit_eventid,
                           payload, length) == AUDIT_SUCCESS;
}

static void event_state_listener(uint32_t id, bool enabled) {
    // We only care about the global on / off switch
    if (id == 0) {
        LOG_NOTICE(nullptr, "Audit changed from: %s to: %s",
                   audit_enabled ? "enabled" : "disabled",
                   enabled ? "enabled" : "disabled");
        audit_enabled.store(enabled);
    }
}

void initialize_audit() {
/* Start the audit daemon */
    AUDIT_EXTENSION_DATA audit_extension_data;
    memset(&audit_extension_data, 0, sizeof(audit_extension_data));
    audit_extension_data.log_extension = settings.extensions.logger;
    audit_extension_data.notify_io_complete = notify_io_complete;
    audit_extension_data.configfile = settings.getAuditFile().c_str();
    Audit* handle = nullptr;
    if (start_auditdaemon(&audit_extension_data, &handle) != AUDIT_SUCCESS) {
        FATAL_ERROR(EXIT_FAILURE, "FATAL: Failed to start audit daemon");
    }
    set_audit_handle(handle);
    cb::audit::add_event_state_listener(handle, event_state_listener);
    cb::audit::notify_all_event_states(handle);
}
<commit_msg>Add per-event dropping of audit events<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2017 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */
#include "config.h"
#include "memcached.h"
#include "mcaudit.h"
#include "memcached_audit_events.h"
#include "buckets.h"
#include "debug_helpers.h"
#include "runtime.h"

#include <memcached/audit_interface.h>
#include <cJSON.h>
#include <memcached/isotime.h>
#include <relaxed_atomic.h>

static Couchbase::RelaxedAtomic<bool> audit_enabled;

static const int First = MEMCACHED_AUDIT_OPENED_DCP_CONNECTION;
static const int Last = MEMCACHED_AUDIT_PRIVILEGE_DEBUG;

static std::array<Couchbase::RelaxedAtomic<bool>, Last - First + 1> events;

static bool isEnabled(uint32_t id) {
    if (!audit_enabled) {
        // Global switch is off.. All events are disabled
        return false;
    }

    if (id >= First && id <= Last) {
        // This is one of ours
        return events[id - First];
    }

    // we don't have information about this id... let the underlying event
    // framework deal with it
    return true;
}

void setEnabled(uint32_t id, bool enable) {
    if (id == 0) {
        LOG_NOTICE(nullptr, "Audit changed from: %s to: %s",
                   audit_enabled ? "enabled" : "disabled",
                   enable ? "enabled" : "disabled");
        audit_enabled.store(enable);
    }

    if (id >= First && id <= Last) {
        LOG_INFO(nullptr, "Audit descriptor %u changed from: %s to: %s",
                 id, events[id - First] ? "enabled" : "disabled",
                 enable ? "enabled" : "disabled");
        events[id - First] = enable;
    }
}

/**
 * Create the typical memcached audit object. It constists of a
 * timestamp, the socket endpoints and the creds. Then each audit event
 * may add event-specific content.
 *
 * @param c the connection object
 * @return the cJSON object containing the basic information
 */
static unique_cJSON_ptr create_memcached_audit_object(const Connection *c) {
    cJSON *root = cJSON_CreateObject();

    std::string timestamp = ISOTime::generatetimestamp();
    cJSON_AddStringToObject(root, "timestamp", timestamp.c_str());

    cJSON_AddStringToObject(root, "peername", c->getPeername().c_str());
    cJSON_AddStringToObject(root, "sockname", c->getSockname().c_str());
    cJSON *source = cJSON_CreateObject();
    cJSON_AddStringToObject(source, "source", "memcached");
    cJSON_AddStringToObject(source, "user", c->getUsername());
    cJSON_AddItemToObject(root, "real_userid", source);

    return unique_cJSON_ptr(root);
}

/**
 * Convert the JSON object to text and send it to the audit framework
 *
 * @param c the connection object requesting the call
 * @param id the audit identifier
 * @param event the payload of the audit description
 * @param warn what to log if we're failing to put the audit event
 */
static void do_audit(const Connection* c,
                     uint32_t id,
                     unique_cJSON_ptr& event,
                     const char* warn) {
    auto text = to_string(event, false);
    auto status = put_audit_event(get_audit_handle(), id, text.data(),
                                  text.length());

    if (status != AUDIT_SUCCESS) {
        LOG_WARNING(c, "%s: %s", warn, text.c_str());
    }
}

void audit_auth_failure(const Connection *c, const char *reason) {
    if (!isEnabled(MEMCACHED_AUDIT_AUTHENTICATION_FAILED)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "reason", reason);

    do_audit(c, MEMCACHED_AUDIT_AUTHENTICATION_FAILED, root,
             "Failed to send AUTH FAILED audit event");
}

void audit_auth_success(const Connection *c) {
    if (!isEnabled(MEMCACHED_AUDIT_AUTHENTICATION_SUCCEEDED)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    do_audit(c, MEMCACHED_AUDIT_AUTHENTICATION_SUCCEEDED, root,
             "Failed to send AUTH SUCCESS audit event");
}

void audit_bucket_flush(const Connection *c, const char *bucket) {
    if (!isEnabled(MEMCACHED_AUDIT_BUCKET_FLUSH)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "bucket", bucket);

    do_audit(c, MEMCACHED_AUDIT_BUCKET_FLUSH, root,
             "Failed to send BUCKET_FLUSH audit event");
}

void audit_dcp_open(const Connection *c) {
    if (!isEnabled(MEMCACHED_AUDIT_OPENED_DCP_CONNECTION)) {
        return;
    }
    if (c->isInternal()) {
        LOG_INFO(c, "Open DCP stream with admin credentials");
    } else {
        auto root = create_memcached_audit_object(c);
        cJSON_AddStringToObject(root.get(), "bucket", getBucketName(c));

        do_audit(c, MEMCACHED_AUDIT_OPENED_DCP_CONNECTION, root,
                 "Failed to send DCP open connection "
                 "audit event to audit daemon");
    }
}

void audit_set_privilege_debug_mode(const Connection* c, bool enable) {
    if (!isEnabled(MEMCACHED_AUDIT_PRIVILEGE_DEBUG_CONFIGURED)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddBoolToObject(root.get(), "enable", enable);
    do_audit(c, MEMCACHED_AUDIT_PRIVILEGE_DEBUG_CONFIGURED, root,
             "Failed to send modifications in privilege debug state "
             "audit event to audit daemon");
}

void audit_privilege_debug(const Connection* c,
                           const std::string& command,
                           const std::string& bucket,
                           const std::string& privilege,
                           const std::string& context) {
    if (!isEnabled(MEMCACHED_AUDIT_PRIVILEGE_DEBUG)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    cJSON_AddStringToObject(root.get(), "command", command.c_str());
    cJSON_AddStringToObject(root.get(), "bucket", bucket.c_str());
    cJSON_AddStringToObject(root.get(), "privilege", privilege.c_str());
    cJSON_AddStringToObject(root.get(), "context", context.c_str());

    do_audit(c, MEMCACHED_AUDIT_PRIVILEGE_DEBUG, root,
             "Failed to send privilege debug audit event to audit daemon");
}

void audit_command_access_failed(const McbpConnection *c) {
    if (!isEnabled(MEMCACHED_AUDIT_COMMAND_ACCESS_FAILURE)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    char buffer[256];
    memset(buffer, 0, sizeof(buffer));
    // Deliberately ignore failure of bytes_to_output_string
    // We'll either have a partial string or no string.
    bytes_to_output_string(buffer, sizeof(buffer), c->getId(), true,
                           "Access to command is not allowed:",
                           reinterpret_cast<const char*>(&c->getBinaryHeader()),
                           sizeof(protocol_binary_request_header));
    cJSON_AddStringToObject(root.get(), "packet", buffer);
    do_audit(c, MEMCACHED_AUDIT_COMMAND_ACCESS_FAILURE, root, buffer);
}

void audit_invalid_packet(const McbpConnection *c) {
    if (!isEnabled(MEMCACHED_AUDIT_INVALID_PACKET)) {
        return;
    }
    auto root = create_memcached_audit_object(c);
    char buffer[256];
    memset(buffer, 0, sizeof(buffer));
    // Deliberately ignore failure of bytes_to_output_string
    // We'll either have a partial string or no string.
    bytes_to_output_string(buffer, sizeof(buffer), c->getId(), true,
                           "Invalid Packet:",
                           reinterpret_cast<const char*>(&c->getBinaryHeader()),
                           sizeof(protocol_binary_request_header));
    cJSON_AddStringToObject(root.get(), "packet", buffer);
    do_audit(c, MEMCACHED_AUDIT_INVALID_PACKET, root, buffer);
}

bool mc_audit_event(uint32_t audit_eventid,
                    const void* payload,
                    size_t length) {
    if (!audit_enabled) {
        return true;
    }

    return put_audit_event(get_audit_handle(), audit_eventid,
                           payload, length) == AUDIT_SUCCESS;
}

static void event_state_listener(uint32_t id, bool enabled) {
    setEnabled(id, enabled);
}

void initialize_audit() {
/* Start the audit daemon */
    AUDIT_EXTENSION_DATA audit_extension_data;
    memset(&audit_extension_data, 0, sizeof(audit_extension_data));
    audit_extension_data.log_extension = settings.extensions.logger;
    audit_extension_data.notify_io_complete = notify_io_complete;
    audit_extension_data.configfile = settings.getAuditFile().c_str();
    Audit* handle = nullptr;
    if (start_auditdaemon(&audit_extension_data, &handle) != AUDIT_SUCCESS) {
        FATAL_ERROR(EXIT_FAILURE, "FATAL: Failed to start audit daemon");
    }
    set_audit_handle(handle);
    cb::audit::add_event_state_listener(handle, event_state_listener);
    cb::audit::notify_all_event_states(handle);
}
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2015 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include "topkeys.h"
#include "settings.h"

#include <platform/sysinfo.h>

#include <folly/concurrency/CacheLocality.h>
#include <inttypes.h>
#include <nlohmann/json.hpp>

#include <algorithm>
#include <gsl/gsl>
#include <set>
#include <stdexcept>

/*
 * Implementation Details
 *
 * === TopKeys ===
 *
 * The TopKeys class is split into shards for performance reasons. We create one
 * shard per (logical) core to:
 *
 * a) prevent any cache contention
 * b) allow as much concurrent access as possible (each shard is guarded by a
 *    mutex)
 *
 * Topkeys passes on requests to the correct Shard (determined by the core id of
 * the calling thread), and when statistics are requested it aggregates
 * information from each shard. When aggregating information, the TopKeys class
 * has to remove duplicates from the possible pool of top keys because we shard
 * per core for performance. Previously, TopKeys would create 8 shards
 * (regardless of machine size), each with storage for a configurably amount of
 * keys and shard by key hash (which meant that we would not have duplicate keys
 * across shards). Now that we shard per core instead of by key hash, to keep
 * the size of the stat output the same we need a minimum of 8 X N keys per
 * shard (because each shard could be an exact duplicate of the others).
 *
 * === TopKeys::Shard ===
 *
 * This is where the action happens. Each Shard maintains an ordered
 * list of key names and related statistics, orded by when they were
 * last accessed. There is a fixed number of keys each Shard tracks.

 * Given we generally track only a small number of keys per shard (10
 * by default), it's highly likely that any one key access will 'miss'
 * - i.e. not find an existing topkey. Therefore this class is
 * optimized for that case, attempting to minimise memory allocation
 * in steady-state.
 *
 * Internally Shard consists of a vector of topkey_t, storing the
 * current max_keys top_keys, and a list of topkey_t pointers, used to
 * main the current most -> least recently used ordering:

 * max_keys elements. Each element is a tuple of
 * {hash(key), key, topkey_stats}:
 *
 *
 *       vector<topkey_t>
 *   +----------+-------------+---------------+
 *   | size_t   | std::string | topkey_item_t |
 *   +----------+-------------+---------------+
 *   | <hash 1> | <key 1>     | stats 1       | <--- Node1
 *   | <hash 2> | <key 2>     | stats 2       | <--- Node2
 *   | <hash 3> | <key 3>     | stats 3       | <--- Node3
 *   . ....                                   .
 *   | <hash N> | <key N>     | stats N       | <--- NodeN
 *   +----------------------------------------+
 *
 *   Node3 <---> Node1 <---> ... <---> Node2
 *   ^^^                               ^^^
 *   MRU                               LRU
 *
 *
 * Upon a key 'hit', TopKeys::updateKey() is called. That hashes the
 * key, and finds the Shard responsible for that
 * key. TopKeys::Shard::updateKey() is then called.
 *
 * Shard::updateKey() iterates over the vector, searching for a hash
 * match with an existing element, using the actual key string to
 * validate (in case of a hash collision). If it is found, then the
 * stats are simply updated.  If it is not found, then the
 * least-recently-used element is selected as a 'victim' and it's
 * contents is replaced by the incoming key.  Finally the linked-list
 * is updated to move the updated element to the head of the list.
 */
TopKeys::TopKeys(int mkeys)
    : keys_to_return(mkeys * legacy_multiplier), shards(cb::get_cpu_count()) {
    for (auto& shard : shards) {
        shard->setMaxKeys(keys_to_return);
    }
}

TopKeys::~TopKeys() {
}

void TopKeys::updateKey(const void* key,
                        size_t nkey,
                        rel_time_t operation_time) {
    if (Settings::instance().isTopkeysEnabled()) {
        doUpdateKey(key, nkey, operation_time);
    }
}

ENGINE_ERROR_CODE TopKeys::stats(const void* cookie,
                                 rel_time_t current_time,
                                 const AddStatFn& add_stat) {
    if (Settings::instance().isTopkeysEnabled()) {
        return doStats(cookie, current_time, add_stat);
    }

    return ENGINE_SUCCESS;
}

ENGINE_ERROR_CODE TopKeys::json_stats(nlohmann::json& object,
                                      rel_time_t current_time) {
    if (Settings::instance().isTopkeysEnabled()) {
        return do_json_stats(object, current_time);
    }

    return ENGINE_SUCCESS;
}

TopKeys::Shard& TopKeys::getShard() {
    auto stripe =
            folly::AccessSpreader<std::atomic>::cachedCurrent(shards.size());
    return *shards[stripe];
}

TopKeys::topkey_t* TopKeys::Shard::searchForKey(
        size_t key_hash, const cb::const_char_buffer& key) {
    for (auto& topkey : storage) {
        if (topkey.first.hash == key_hash) {
            // Double-check with full compare
            if (topkey.first.key.compare(
                        0, topkey.first.key.size(), key.buf, key.len) == 0) {
                // Match found.
                return &topkey;
            }
        }
    }
    return nullptr;
}

bool TopKeys::Shard::updateKey(const cb::const_char_buffer& key,
                               size_t key_hash,
                               const rel_time_t ct) {
    try {
        std::lock_guard<std::mutex> lock(mutex);

        topkey_t* found_key = searchForKey(key_hash, key);

        if (!found_key) {
            // Key not found.
            if (storage.size() == max_keys) {
                // Re-use the lowest keys' storage.
                found_key = list.back();
                found_key->first.hash = key_hash;
                found_key->first.key.assign(key.buf, key.len);
                found_key->second = topkey_item_t(ct);

                // Move back element to the front, shuffling down the
                // rest.
                list.splice(list.begin(), list, --list.end());

            } else {
                // add a new element to the storage array.

                storage.emplace_back(std::make_pair(
                        KeyId{key_hash, std::string(key.buf, key.len)},
                        topkey_item_t(ct)));
                found_key = &storage.back();

                // Insert the new item to the front of the list
                list.push_front(found_key);
            }
        } else {
            // Found - shuffle to the front.
            auto it = std::find(list.begin(), list.end(), found_key);
            list.splice(list.begin(), list, it);
        }

        // Increment access count.
        found_key->second.ti_access_count++;
        return true;

    } catch (const std::bad_alloc&) {
        // Failed to update.
        return false;
    }
}

void TopKeys::doUpdateKey(const void* key,
                          size_t nkey,
                          rel_time_t operation_time) {
    if (key == nullptr || nkey == 0) {
        throw std::invalid_argument(
                "TopKeys::doUpdateKey: key must be specified");
    }

    try {
        // We store a key hash to make lookup of topkeys faster and because the
        // memory footprint is relatively small.
        cb::const_char_buffer key_buf(static_cast<const char*>(key), nkey);
        std::hash<cb::const_char_buffer> hash_fn;
        const size_t key_hash = hash_fn(key_buf);

        getShard().updateKey(key_buf, key_hash, operation_time);
    } catch (const std::bad_alloc&) {
        // Failed to increment topkeys, continue...
    }
}

struct tk_context {
    using CallbackFn = void (*)(const std::string&,
                                const topkey_item_t&,
                                void*);
    tk_context(const void* c,
               const AddStatFn& a,
               rel_time_t t,
               nlohmann::json* arr,
               CallbackFn callbackFn)
        : cookie(c),
          add_stat(a),
          current_time(t),
          array(arr),
          callbackFunction(callbackFn) {
        // empty
    }

    const void* cookie;
    AddStatFn add_stat;
    rel_time_t current_time;
    nlohmann::json* array;
    CallbackFn callbackFunction;
};

static void tk_iterfunc(const std::string& key,
                        const topkey_item_t& it,
                        void* arg) {
    auto* c = (struct tk_context*)arg;
    char val_str[500];
    /* Note we use accessed time for both 'atime' and 'ctime' below. They have
     * had the same value since the topkeys code was added; but given that
     * clients may expect separate values we print both.
     */
    rel_time_t created_time = c->current_time - it.ti_ctime;
    int vlen = snprintf(val_str,
                        sizeof(val_str) - 1,
                        "get_hits=%d,"
                        "get_misses=0,cmd_set=0,incr_hits=0,incr_misses=0,"
                        "decr_hits=0,decr_misses=0,delete_hits=0,"
                        "delete_misses=0,evictions=0,cas_hits=0,cas_badval=0,"
                        "cas_misses=0,get_replica=0,evict=0,getl=0,unlock=0,"
                        "get_meta=0,set_meta=0,del_meta=0,ctime=%" PRIu32
                        ",atime=%" PRIu32,
                        it.ti_access_count,
                        created_time,
                        created_time);
    if (vlen > 0 && vlen < int(sizeof(val_str) - 1)) {
        c->add_stat(key, val_str, c->cookie);
    }
}

/**
 * Passing in a list of keys, context, and json array will populate that
 * array with an object for each key in the following format:
 * {
 *    "key": "somekey",
 *    "access_count": nnn,
 *    "ctime": ccc,
 *    "atime": aaa
 * }
 */
static void tk_jsonfunc(const std::string& key,
                        const topkey_item_t& it,
                        void* arg) {
    auto* c = (struct tk_context*)arg;
    if (c->array == nullptr) {
        throw std::invalid_argument("tk_jsonfunc: c->array can't be nullptr");
    }

    nlohmann::json obj;
    obj["key"] = key;
    obj["access_count"] = it.ti_access_count;
    obj["ctime"] = c->current_time - it.ti_ctime;

    c->array->push_back(obj);
}

static void tk_aggregate_func(const TopKeys::topkey_t& it, void* arg) {
    auto* map = (std::unordered_map<std::string, topkey_item_t>*)arg;

    auto res = map->insert(std::make_pair(it.first.key, it.second));

    // If insert failed, then we have a duplicate top key. Add the stats
    if (!res.second) {
        res.first->second.ti_access_count += it.second.ti_access_count;
    }
}

ENGINE_ERROR_CODE TopKeys::doStats(const void* cookie,
                                   rel_time_t current_time,
                                   const AddStatFn& add_stat) {
    struct tk_context context(
            cookie, add_stat, current_time, nullptr, &tk_iterfunc);
    doStatsInner(context);

    return ENGINE_SUCCESS;
}

/**
 * Passing a set of topkeys, and relevant context data will
 * return a json object containing an array of topkeys (with each key
 * appearing as in the example above for tk_jsonfunc):
 * {
 *   "topkeys": [
 *      { ... }, ..., { ... }
 *    ]
 * }
 */
ENGINE_ERROR_CODE TopKeys::do_json_stats(nlohmann::json& object,
                                         rel_time_t current_time) {
    nlohmann::json topkeys = nlohmann::json::array();
    struct tk_context context(
            nullptr, nullptr, current_time, &topkeys, &tk_jsonfunc);
    doStatsInner(context);

    auto str = topkeys.dump();
    object["topkeys"] = topkeys;

    return ENGINE_SUCCESS;
}

void TopKeys::Shard::accept_visitor(iterfunc_t visitor_func,
                                    void* visitor_ctx) {
    std::lock_guard<std::mutex> lock(mutex);
    for (const auto* key : list) {
        visitor_func(*key, visitor_ctx);
    }
}

void TopKeys::doStatsInner(const tk_context& stat_context) {
    // 1) Find the unique set of top keys by putting every top key in a map
    std::unordered_map<std::string, topkey_item_t> map =
            std::unordered_map<std::string, topkey_item_t>();

    for (auto& shard : shards) {
        shard->accept_visitor(tk_aggregate_func, &map);
    }

    // Easiest way to sort this by access_count is to drop the contents of the
    // map into a vector and sort that.
    auto items = std::vector<topkey_stat_t>(map.begin(), map.end());
    std::sort(items.begin(),
              items.end(),
              [](const TopKeys::topkey_stat_t& a,
                 const TopKeys::topkey_stat_t& b) {
                  // Sort by number of accesses
                  return a.second.ti_access_count > b.second.ti_access_count;
              });

    // 2) Iterate on this set making the required callback for each key. We only
    // iterate from the start of the container (highest access count) to the
    // number of keys to return.
    std::for_each(items.begin(),
                  std::min(items.begin() + keys_to_return, items.end()),
                  [stat_context](const topkey_stat_t& t) {
                      stat_context.callbackFunction(
                              t.first, t.second, (void*)&stat_context);
                  });
}
<commit_msg>Refactor c-style cast to C++ casting in top keys<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2020 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include "topkeys.h"
#include "settings.h"

#include <platform/sysinfo.h>

#include <folly/concurrency/CacheLocality.h>
#include <inttypes.h>
#include <nlohmann/json.hpp>

#include <algorithm>
#include <gsl/gsl>
#include <set>
#include <stdexcept>

/*
 * Implementation Details
 *
 * === TopKeys ===
 *
 * The TopKeys class is split into shards for performance reasons. We create one
 * shard per (logical) core to:
 *
 * a) prevent any cache contention
 * b) allow as much concurrent access as possible (each shard is guarded by a
 *    mutex)
 *
 * Topkeys passes on requests to the correct Shard (determined by the core id of
 * the calling thread), and when statistics are requested it aggregates
 * information from each shard. When aggregating information, the TopKeys class
 * has to remove duplicates from the possible pool of top keys because we shard
 * per core for performance. Previously, TopKeys would create 8 shards
 * (regardless of machine size), each with storage for a configurably amount of
 * keys and shard by key hash (which meant that we would not have duplicate keys
 * across shards). Now that we shard per core instead of by key hash, to keep
 * the size of the stat output the same we need a minimum of 8 X N keys per
 * shard (because each shard could be an exact duplicate of the others).
 *
 * === TopKeys::Shard ===
 *
 * This is where the action happens. Each Shard maintains an ordered
 * list of key names and related statistics, orded by when they were
 * last accessed. There is a fixed number of keys each Shard tracks.

 * Given we generally track only a small number of keys per shard (10
 * by default), it's highly likely that any one key access will 'miss'
 * - i.e. not find an existing topkey. Therefore this class is
 * optimized for that case, attempting to minimise memory allocation
 * in steady-state.
 *
 * Internally Shard consists of a vector of topkey_t, storing the
 * current max_keys top_keys, and a list of topkey_t pointers, used to
 * main the current most -> least recently used ordering:

 * max_keys elements. Each element is a tuple of
 * {hash(key), key, topkey_stats}:
 *
 *
 *       vector<topkey_t>
 *   +----------+-------------+---------------+
 *   | size_t   | std::string | topkey_item_t |
 *   +----------+-------------+---------------+
 *   | <hash 1> | <key 1>     | stats 1       | <--- Node1
 *   | <hash 2> | <key 2>     | stats 2       | <--- Node2
 *   | <hash 3> | <key 3>     | stats 3       | <--- Node3
 *   . ....                                   .
 *   | <hash N> | <key N>     | stats N       | <--- NodeN
 *   +----------------------------------------+
 *
 *   Node3 <---> Node1 <---> ... <---> Node2
 *   ^^^                               ^^^
 *   MRU                               LRU
 *
 *
 * Upon a key 'hit', TopKeys::updateKey() is called. That hashes the
 * key, and finds the Shard responsible for that
 * key. TopKeys::Shard::updateKey() is then called.
 *
 * Shard::updateKey() iterates over the vector, searching for a hash
 * match with an existing element, using the actual key string to
 * validate (in case of a hash collision). If it is found, then the
 * stats are simply updated.  If it is not found, then the
 * least-recently-used element is selected as a 'victim' and it's
 * contents is replaced by the incoming key.  Finally the linked-list
 * is updated to move the updated element to the head of the list.
 */
TopKeys::TopKeys(int mkeys)
    : keys_to_return(mkeys * legacy_multiplier), shards(cb::get_cpu_count()) {
    for (auto& shard : shards) {
        shard->setMaxKeys(keys_to_return);
    }
}

TopKeys::~TopKeys() {
}

void TopKeys::updateKey(const void* key,
                        size_t nkey,
                        rel_time_t operation_time) {
    if (Settings::instance().isTopkeysEnabled()) {
        doUpdateKey(key, nkey, operation_time);
    }
}

ENGINE_ERROR_CODE TopKeys::stats(const void* cookie,
                                 rel_time_t current_time,
                                 const AddStatFn& add_stat) {
    if (Settings::instance().isTopkeysEnabled()) {
        return doStats(cookie, current_time, add_stat);
    }

    return ENGINE_SUCCESS;
}

ENGINE_ERROR_CODE TopKeys::json_stats(nlohmann::json& object,
                                      rel_time_t current_time) {
    if (Settings::instance().isTopkeysEnabled()) {
        return do_json_stats(object, current_time);
    }

    return ENGINE_SUCCESS;
}

TopKeys::Shard& TopKeys::getShard() {
    auto stripe =
            folly::AccessSpreader<std::atomic>::cachedCurrent(shards.size());
    return *shards[stripe];
}

TopKeys::topkey_t* TopKeys::Shard::searchForKey(
        size_t key_hash, const cb::const_char_buffer& key) {
    for (auto& topkey : storage) {
        if (topkey.first.hash == key_hash) {
            // Double-check with full compare
            if (topkey.first.key.compare(
                        0, topkey.first.key.size(), key.buf, key.len) == 0) {
                // Match found.
                return &topkey;
            }
        }
    }
    return nullptr;
}

bool TopKeys::Shard::updateKey(const cb::const_char_buffer& key,
                               size_t key_hash,
                               const rel_time_t ct) {
    try {
        std::lock_guard<std::mutex> lock(mutex);

        topkey_t* found_key = searchForKey(key_hash, key);

        if (!found_key) {
            // Key not found.
            if (storage.size() == max_keys) {
                // Re-use the lowest keys' storage.
                found_key = list.back();
                found_key->first.hash = key_hash;
                found_key->first.key.assign(key.buf, key.len);
                found_key->second = topkey_item_t(ct);

                // Move back element to the front, shuffling down the
                // rest.
                list.splice(list.begin(), list, --list.end());

            } else {
                // add a new element to the storage array.

                storage.emplace_back(std::make_pair(
                        KeyId{key_hash, std::string(key.buf, key.len)},
                        topkey_item_t(ct)));
                found_key = &storage.back();

                // Insert the new item to the front of the list
                list.push_front(found_key);
            }
        } else {
            // Found - shuffle to the front.
            auto it = std::find(list.begin(), list.end(), found_key);
            list.splice(list.begin(), list, it);
        }

        // Increment access count.
        found_key->second.ti_access_count++;
        return true;

    } catch (const std::bad_alloc&) {
        // Failed to update.
        return false;
    }
}

void TopKeys::doUpdateKey(const void* key,
                          size_t nkey,
                          rel_time_t operation_time) {
    if (key == nullptr || nkey == 0) {
        throw std::invalid_argument(
                "TopKeys::doUpdateKey: key must be specified");
    }

    try {
        // We store a key hash to make lookup of topkeys faster and because the
        // memory footprint is relatively small.
        cb::const_char_buffer key_buf(static_cast<const char*>(key), nkey);
        std::hash<cb::const_char_buffer> hash_fn;
        const size_t key_hash = hash_fn(key_buf);

        getShard().updateKey(key_buf, key_hash, operation_time);
    } catch (const std::bad_alloc&) {
        // Failed to increment topkeys, continue...
    }
}

struct tk_context {
    using CallbackFn = void (*)(const std::string&,
                                const topkey_item_t&,
                                void*);
    tk_context(const void* c,
               const AddStatFn& a,
               rel_time_t t,
               nlohmann::json* arr,
               CallbackFn callbackFn)
        : cookie(c),
          add_stat(a),
          current_time(t),
          array(arr),
          callbackFunction(callbackFn) {
        // empty
    }

    const void* cookie;
    AddStatFn add_stat;
    rel_time_t current_time;
    nlohmann::json* array;
    CallbackFn callbackFunction;
};

static void tk_iterfunc(const std::string& key,
                        const topkey_item_t& it,
                        void* arg) {
    auto* c = static_cast<struct tk_context*>(arg);
    char val_str[500];
    /* Note we use accessed time for both 'atime' and 'ctime' below. They have
     * had the same value since the topkeys code was added; but given that
     * clients may expect separate values we print both.
     */
    rel_time_t created_time = c->current_time - it.ti_ctime;
    int vlen = snprintf(val_str,
                        sizeof(val_str) - 1,
                        "get_hits=%d,"
                        "get_misses=0,cmd_set=0,incr_hits=0,incr_misses=0,"
                        "decr_hits=0,decr_misses=0,delete_hits=0,"
                        "delete_misses=0,evictions=0,cas_hits=0,cas_badval=0,"
                        "cas_misses=0,get_replica=0,evict=0,getl=0,unlock=0,"
                        "get_meta=0,set_meta=0,del_meta=0,ctime=%" PRIu32
                        ",atime=%" PRIu32,
                        it.ti_access_count,
                        created_time,
                        created_time);
    if (vlen > 0 && vlen < int(sizeof(val_str) - 1)) {
        c->add_stat(key, val_str, c->cookie);
    }
}

/**
 * Passing in a list of keys, context, and json array will populate that
 * array with an object for each key in the following format:
 * {
 *    "key": "somekey",
 *    "access_count": nnn,
 *    "ctime": ccc,
 *    "atime": aaa
 * }
 */
static void tk_jsonfunc(const std::string& key,
                        const topkey_item_t& it,
                        void* arg) {
    auto* c = static_cast<struct tk_context*>(arg);
    if (c->array == nullptr) {
        throw std::invalid_argument("tk_jsonfunc: c->array can't be nullptr");
    }

    nlohmann::json obj;
    obj["key"] = key;
    obj["access_count"] = it.ti_access_count;
    obj["ctime"] = c->current_time - it.ti_ctime;

    c->array->push_back(obj);
}

static void tk_aggregate_func(const TopKeys::topkey_t& it, void* arg) {
    auto* map =
            static_cast<std::unordered_map<std::string, topkey_item_t>*>(arg);

    auto res = map->insert(std::make_pair(it.first.key, it.second));

    // If insert failed, then we have a duplicate top key. Add the stats
    if (!res.second) {
        res.first->second.ti_access_count += it.second.ti_access_count;
    }
}

ENGINE_ERROR_CODE TopKeys::doStats(const void* cookie,
                                   rel_time_t current_time,
                                   const AddStatFn& add_stat) {
    struct tk_context context(
            cookie, add_stat, current_time, nullptr, &tk_iterfunc);
    doStatsInner(context);

    return ENGINE_SUCCESS;
}

/**
 * Passing a set of topkeys, and relevant context data will
 * return a json object containing an array of topkeys (with each key
 * appearing as in the example above for tk_jsonfunc):
 * {
 *   "topkeys": [
 *      { ... }, ..., { ... }
 *    ]
 * }
 */
ENGINE_ERROR_CODE TopKeys::do_json_stats(nlohmann::json& object,
                                         rel_time_t current_time) {
    nlohmann::json topkeys = nlohmann::json::array();
    struct tk_context context(
            nullptr, nullptr, current_time, &topkeys, &tk_jsonfunc);
    doStatsInner(context);

    auto str = topkeys.dump();
    object["topkeys"] = topkeys;

    return ENGINE_SUCCESS;
}

void TopKeys::Shard::accept_visitor(iterfunc_t visitor_func,
                                    void* visitor_ctx) {
    std::lock_guard<std::mutex> lock(mutex);
    for (const auto* key : list) {
        visitor_func(*key, visitor_ctx);
    }
}

void TopKeys::doStatsInner(const tk_context& stat_context) {
    // 1) Find the unique set of top keys by putting every top key in a map
    std::unordered_map<std::string, topkey_item_t> map =
            std::unordered_map<std::string, topkey_item_t>();

    for (auto& shard : shards) {
        shard->accept_visitor(tk_aggregate_func, &map);
    }

    // Easiest way to sort this by access_count is to drop the contents of the
    // map into a vector and sort that.
    auto items = std::vector<topkey_stat_t>(map.begin(), map.end());
    std::sort(items.begin(),
              items.end(),
              [](const TopKeys::topkey_stat_t& a,
                 const TopKeys::topkey_stat_t& b) {
                  // Sort by number of accesses
                  return a.second.ti_access_count > b.second.ti_access_count;
              });

    // 2) Iterate on this set making the required callback for each key. We only
    // iterate from the start of the container (highest access count) to the
    // number of keys to return.
    std::for_each(items.begin(),
                  std::min(items.begin() + keys_to_return, items.end()),
                  [stat_context](const topkey_stat_t& t) {
                      stat_context.callbackFunction(
                              t.first, t.second, (void*)&stat_context);
                  });
}
<|endoftext|>


<commit_before><commit_msg>Create 1133 - Rest of a Division.cpp<commit_after><|endoftext|>