<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

#include <getopt.h>             /* getopt_long() */

#include <fcntl.h>              /* low-level i/o */
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/ioctl.h>

#include <linux/videodev2.h>

#include <GLFW/glfw3.h>

#define CLEAR(x) memset(&(x), 0, sizeof(x))

struct buffer {
        void   *start;
        size_t  length;
};

static char            *dev_name;
static int              fd = -1;
struct buffer          *buffers;
static unsigned int     n_buffers;
static int              out_buf;
static int              force_format;
static int              frame_count = 70;

static void errno_exit(const char *s)
{
        fprintf(stderr, "%s error %d, %s\n", s, errno, strerror(errno));
        exit(EXIT_FAILURE);
}

static int xioctl(int fh, int request, void *arg)
{
        int r;

        do {
                r = ioctl(fh, request, arg);
        } while (-1 == r && EINTR == errno);

        return r;
}

#pragma pack(push, 1)
struct z16y8_pixel { uint16_t z; uint8_t y; };
#pragma pack(pop)

static void process_image(const void *p, int size)
{
    uint16_t z[320*240];
    uint8_t y[320*240];

    auto in = reinterpret_cast<const z16y8_pixel *>(p);
    for(int i=0; i<320*240; ++i)
    {
        z[i] = in[i].z;
        y[i] = in[i].y;
    }

    glRasterPos2i(0, 240);
    glDrawPixels(320, 240, GL_LUMINANCE, GL_UNSIGNED_SHORT, z);
    glRasterPos2i(320, 240);
    glDrawPixels(320, 240, GL_LUMINANCE, GL_UNSIGNED_BYTE, y);
    printf("%d %d\n", size, 320*240*3);
}

static int read_frame(void)
{
        struct v4l2_buffer buf;
        unsigned int i;

        CLEAR(buf);

        buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buf.memory = V4L2_MEMORY_MMAP;

        if (-1 == xioctl(fd, VIDIOC_DQBUF, &buf)) {
                switch (errno) {
                case EAGAIN:
                        return 0;

                case EIO:
                        /* Could ignore EIO, see spec. */

                        /* fall through */

                default:
                        errno_exit("VIDIOC_DQBUF");
                }
        }

        assert(buf.index < n_buffers);

        process_image(buffers[buf.index].start, buf.bytesused);

        if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
                errno_exit("VIDIOC_QBUF");

        return 1;
}

static void mainloop(void)
{

    // Open a GLFW window
    glfwInit();
    GLFWwindow * win = glfwCreateWindow(640, 480, "V4L2 test", 0, 0);
    glfwMakeContextCurrent(win);

    while (!glfwWindowShouldClose(win))
    {
        // Wait for new images
        glfwPollEvents();

        int w,h;
        glfwGetFramebufferSize(win, &w, &h);
        glViewport(0, 0, w, h);
        glClear(GL_COLOR_BUFFER_BIT);

        // Draw the images
        glPushMatrix();
        glfwGetWindowSize(win, &w, &h);
        glOrtho(0, w, h, 0, -1, +1);

        for (;;) {
                fd_set fds;
                struct timeval tv;
                int r;

                FD_ZERO(&fds);
                FD_SET(fd, &fds);

                /* Timeout. */
                tv.tv_sec = 2;
                tv.tv_usec = 0;

                r = select(fd + 1, &fds, NULL, NULL, &tv);

                if (-1 == r) {
                        if (EINTR == errno)
                                continue;
                        errno_exit("select");
                }

                if (0 == r) {
                        fprintf(stderr, "select timeout\n");
                        exit(EXIT_FAILURE);
                }

                if (read_frame())
                        break;
                /* EAGAIN - continue select loop. */
        }

        glPopMatrix();
        glfwSwapBuffers(win);
    }

    glfwDestroyWindow(win);
    glfwTerminate();
}

static void stop_capturing(void)
{
    enum v4l2_buf_type type;

    type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (-1 == xioctl(fd, VIDIOC_STREAMOFF, &type))
            errno_exit("VIDIOC_STREAMOFF");
}

static void start_capturing(void)
{
    unsigned int i;
    enum v4l2_buf_type type;

    for (i = 0; i < n_buffers; ++i) {
            struct v4l2_buffer buf;

            CLEAR(buf);
            buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
            buf.memory = V4L2_MEMORY_MMAP;
            buf.index = i;

            if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
                    errno_exit("VIDIOC_QBUF");
    }
    type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (-1 == xioctl(fd, VIDIOC_STREAMON, &type))
            errno_exit("VIDIOC_STREAMON");
}

static void uninit_device(void)
{
    unsigned int i;

    for (i = 0; i < n_buffers; ++i)
            if (-1 == munmap(buffers[i].start, buffers[i].length))
                    errno_exit("munmap");

    free(buffers);
}

static void init_mmap(void)
{
        struct v4l2_requestbuffers req;

        CLEAR(req);

        req.count = 4;
        req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        req.memory = V4L2_MEMORY_MMAP;

        if (-1 == xioctl(fd, VIDIOC_REQBUFS, &req)) {
                if (EINVAL == errno) {
                        fprintf(stderr, "%s does not support "
                                 "memory mapping\n", dev_name);
                        exit(EXIT_FAILURE);
                } else {
                        errno_exit("VIDIOC_REQBUFS");
                }
        }

        if (req.count < 2) {
                fprintf(stderr, "Insufficient buffer memory on %s\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }

        buffers = (buffer *)calloc(req.count, sizeof(*buffers));

        if (!buffers) {
                fprintf(stderr, "Out of memory\n");
                exit(EXIT_FAILURE);
        }

        for (n_buffers = 0; n_buffers < req.count; ++n_buffers) {
                struct v4l2_buffer buf;

                CLEAR(buf);

                buf.type        = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                buf.memory      = V4L2_MEMORY_MMAP;
                buf.index       = n_buffers;

                if (-1 == xioctl(fd, VIDIOC_QUERYBUF, &buf))
                        errno_exit("VIDIOC_QUERYBUF");

                buffers[n_buffers].length = buf.length;
                buffers[n_buffers].start =
                        mmap(NULL /* start anywhere */,
                              buf.length,
                              PROT_READ | PROT_WRITE /* required */,
                              MAP_SHARED /* recommended */,
                              fd, buf.m.offset);

                if (MAP_FAILED == buffers[n_buffers].start)
                        errno_exit("mmap");
        }
}

static void init_device(void)
{
        struct v4l2_capability cap;
        struct v4l2_cropcap cropcap;
        struct v4l2_crop crop;
        struct v4l2_format fmt;
        unsigned int min;

        if (-1 == xioctl(fd, VIDIOC_QUERYCAP, &cap)) {
                if (EINVAL == errno) {
                        fprintf(stderr, "%s is no V4L2 device\n",
                                 dev_name);
                        exit(EXIT_FAILURE);
                } else {
                        errno_exit("VIDIOC_QUERYCAP");
                }
        }

        if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) {
                fprintf(stderr, "%s is no video capture device\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }

        if (!(cap.capabilities & V4L2_CAP_STREAMING)) {
                fprintf(stderr, "%s does not support streaming i/o\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }

        /* Select video input, video standard and tune here. */

        CLEAR(cropcap);

        cropcap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;

        if (0 == xioctl(fd, VIDIOC_CROPCAP, &cropcap)) {
                crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                crop.c = cropcap.defrect; /* reset to default */

                if (-1 == xioctl(fd, VIDIOC_S_CROP, &crop)) {
                        switch (errno) {
                        case EINVAL:
                                /* Cropping not supported. */
                                break;
                        default:
                                /* Errors ignored. */
                                break;
                        }
                }
        } else {
                /* Errors ignored. */
        }

        CLEAR(fmt);

        fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if (force_format) {
                fmt.fmt.pix.width       = 640;
                fmt.fmt.pix.height      = 480;
                fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
                fmt.fmt.pix.field       = V4L2_FIELD_INTERLACED;

                if (-1 == xioctl(fd, VIDIOC_S_FMT, &fmt))
                        errno_exit("VIDIOC_S_FMT");

                /* Note VIDIOC_S_FMT may change width and height. */
        } else {
                /* Preserve original settings as set by v4l2-ctl for example */
                if (-1 == xioctl(fd, VIDIOC_G_FMT, &fmt))
                        errno_exit("VIDIOC_G_FMT");
        }

        /* Buggy driver paranoia. */
        min = fmt.fmt.pix.width * 2;
        if (fmt.fmt.pix.bytesperline < min)
                fmt.fmt.pix.bytesperline = min;
        min = fmt.fmt.pix.bytesperline * fmt.fmt.pix.height;
        if (fmt.fmt.pix.sizeimage < min)
                fmt.fmt.pix.sizeimage = min;

        init_mmap();
}

static void close_device(void)
{
        if (-1 == close(fd))
                errno_exit("close");

        fd = -1;
}

static void open_device(void)
{
        struct stat st;

        if (-1 == stat(dev_name, &st)) {
                fprintf(stderr, "Cannot identify '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }

        if (!S_ISCHR(st.st_mode)) {
                fprintf(stderr, "%s is no device\n", dev_name);
                exit(EXIT_FAILURE);
        }

        fd = open(dev_name, O_RDWR /* required */ | O_NONBLOCK, 0);

        if (-1 == fd) {
                fprintf(stderr, "Cannot open '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }
}

int main(int argc, char **argv)
{
        dev_name = "/dev/video1";
        out_buf = 1;

        open_device();
        init_device();
        start_capturing();
        mainloop();
        stop_capturing();
        uninit_device();
        close_device();
        fprintf(stderr, "\n");
        return 0;
}

<commit_msg>Simplify<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

#include <getopt.h>             /* getopt_long() */

#include <fcntl.h>              /* low-level i/o */
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/ioctl.h>

#include <linux/videodev2.h>

#include <GLFW/glfw3.h>

#define CLEAR(x) memset(&(x), 0, sizeof(x))

struct buffer {
        void   *start;
        size_t  length;
};

static char            *dev_name;
static int              fd = -1;
struct buffer          *buffers;
static unsigned int     n_buffers;
static int              out_buf;
static int              force_format;
static int              frame_count = 70;

static void errno_exit(const char *s)
{
        fprintf(stderr, "%s error %d, %s\n", s, errno, strerror(errno));
        exit(EXIT_FAILURE);
}

static int xioctl(int fh, int request, void *arg)
{
        int r;

        do {
                r = ioctl(fh, request, arg);
        } while (-1 == r && EINTR == errno);

        return r;
}

#pragma pack(push, 1)
struct z16y8_pixel { uint16_t z; uint8_t y; };
#pragma pack(pop)

static void process_image(const void *p, int size)
{
    uint16_t z[320*240];
    uint8_t y[320*240];

    auto in = reinterpret_cast<const z16y8_pixel *>(p);
    for(int i=0; i<320*240; ++i)
    {
        z[i] = in[i].z;
        y[i] = in[i].y;
    }

    glRasterPos2i(0, 240);
    glDrawPixels(320, 240, GL_LUMINANCE, GL_UNSIGNED_SHORT, z);
    glRasterPos2i(320, 240);
    glDrawPixels(320, 240, GL_LUMINANCE, GL_UNSIGNED_BYTE, y);
    printf("%d %d\n", size, 320*240*3);
}

static int read_frame(void)
{
        struct v4l2_buffer buf;
        unsigned int i;

        CLEAR(buf);

        buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buf.memory = V4L2_MEMORY_MMAP;

        if (-1 == xioctl(fd, VIDIOC_DQBUF, &buf)) {
                switch (errno) {
                case EAGAIN:
                        return 0;

                case EIO:
                        /* Could ignore EIO, see spec. */

                        /* fall through */

                default:
                        errno_exit("VIDIOC_DQBUF");
                }
        }

        assert(buf.index < n_buffers);

        process_image(buffers[buf.index].start, buf.bytesused);

        if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
                errno_exit("VIDIOC_QBUF");

        return 1;
}

static void init_mmap(void)
{
        struct v4l2_requestbuffers req;

        CLEAR(req);

        req.count = 4;
        req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        req.memory = V4L2_MEMORY_MMAP;

        if (-1 == xioctl(fd, VIDIOC_REQBUFS, &req)) {
                if (EINVAL == errno) {
                        fprintf(stderr, "%s does not support "
                                 "memory mapping\n", dev_name);
                        exit(EXIT_FAILURE);
                } else {
                        errno_exit("VIDIOC_REQBUFS");
                }
        }

        if (req.count < 2) {
                fprintf(stderr, "Insufficient buffer memory on %s\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }

        buffers = (buffer *)calloc(req.count, sizeof(*buffers));

        if (!buffers) {
                fprintf(stderr, "Out of memory\n");
                exit(EXIT_FAILURE);
        }

        for (n_buffers = 0; n_buffers < req.count; ++n_buffers) {
                struct v4l2_buffer buf;

                CLEAR(buf);

                buf.type        = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                buf.memory      = V4L2_MEMORY_MMAP;
                buf.index       = n_buffers;

                if (-1 == xioctl(fd, VIDIOC_QUERYBUF, &buf))
                        errno_exit("VIDIOC_QUERYBUF");

                buffers[n_buffers].length = buf.length;
                buffers[n_buffers].start =
                        mmap(NULL /* start anywhere */,
                              buf.length,
                              PROT_READ | PROT_WRITE /* required */,
                              MAP_SHARED /* recommended */,
                              fd, buf.m.offset);

                if (MAP_FAILED == buffers[n_buffers].start)
                        errno_exit("mmap");
        }
}

static void open_device(void)
{
        struct stat st;

        if (-1 == stat(dev_name, &st)) {
                fprintf(stderr, "Cannot identify '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }

        if (!S_ISCHR(st.st_mode)) {
                fprintf(stderr, "%s is no device\n", dev_name);
                exit(EXIT_FAILURE);
        }

        fd = open(dev_name, O_RDWR /* required */ | O_NONBLOCK, 0);

        if (-1 == fd) {
                fprintf(stderr, "Cannot open '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }
}

int main(int argc, char **argv)
{
    dev_name = "/dev/video1";
    out_buf = 1;

    open_device();

    v4l2_capability cap = {};
    if(xioctl(fd, VIDIOC_QUERYCAP, &cap) < 0)
    {
        if (EINVAL == errno) {
                fprintf(stderr, "%s is no V4L2 device\n",
                         dev_name);
                exit(EXIT_FAILURE);
        } else {
                errno_exit("VIDIOC_QUERYCAP");
        }
    }

    if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) {
            fprintf(stderr, "%s is no video capture device\n",
                     dev_name);
            exit(EXIT_FAILURE);
    }

    if (!(cap.capabilities & V4L2_CAP_STREAMING)) {
            fprintf(stderr, "%s does not support streaming i/o\n",
                     dev_name);
            exit(EXIT_FAILURE);
    }

    // Select video input, video standard and tune here.
    v4l2_cropcap cropcap = {};
    cropcap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (0 == xioctl(fd, VIDIOC_CROPCAP, &cropcap))
    {
        v4l2_crop crop = {};
        crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        crop.c = cropcap.defrect; // reset to default
        if (xioctl(fd, VIDIOC_S_CROP, &crop) < 0)
        {
            switch (errno)
            {
            case EINVAL: break; // Cropping not supported
            default: break; // Errors ignored
            }
        }
    } else {} // Errors ignored

    v4l2_format fmt = {};
    fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (force_format)
    {
        fmt.fmt.pix.width       = 640;
        fmt.fmt.pix.height      = 480;
        fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
        fmt.fmt.pix.field       = V4L2_FIELD_INTERLACED;
        if(xioctl(fd, VIDIOC_S_FMT, &fmt) < 0) errno_exit("VIDIOC_S_FMT");
        // Note VIDIOC_S_FMT may change width and height
    }
    else
    {
        // Preserve original settings as set by v4l2-ctl for example
        if(xioctl(fd, VIDIOC_G_FMT, &fmt) < 0) errno_exit("VIDIOC_G_FMT");
    }

    // Buggy driver paranoia.
    unsigned int min = fmt.fmt.pix.width * 2;
    if (fmt.fmt.pix.bytesperline < min)
            fmt.fmt.pix.bytesperline = min;
    min = fmt.fmt.pix.bytesperline * fmt.fmt.pix.height;
    if (fmt.fmt.pix.sizeimage < min)
            fmt.fmt.pix.sizeimage = min;

    init_mmap();

    // Start capturing
    for(int i = 0; i < n_buffers; ++i)
    {
        v4l2_buffer buf = {};
        buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buf.memory = V4L2_MEMORY_MMAP;
        buf.index = i;
        if(xioctl(fd, VIDIOC_QBUF, &buf) < 0) errno_exit("VIDIOC_QBUF");
    }

    v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if(xioctl(fd, VIDIOC_STREAMON, &type) < 0) errno_exit("VIDIOC_STREAMON");

    // Open a GLFW window
    glfwInit();
    GLFWwindow * win = glfwCreateWindow(640, 480, "V4L2 test", 0, 0);
    glfwMakeContextCurrent(win);

    // While window is open
    while (!glfwWindowShouldClose(win))
    {
        glfwPollEvents();

        int w,h;
        glfwGetFramebufferSize(win, &w, &h);
        glViewport(0, 0, w, h);
        glClear(GL_COLOR_BUFFER_BIT);

        glPushMatrix();
        glfwGetWindowSize(win, &w, &h);
        glOrtho(0, w, h, 0, -1, +1);
        for (;;)
        {
            fd_set fds;
            struct timeval tv;
            int r;

            FD_ZERO(&fds);
            FD_SET(fd, &fds);

            /* Timeout. */
            tv.tv_sec = 2;
            tv.tv_usec = 0;

            r = select(fd + 1, &fds, NULL, NULL, &tv);

            if (-1 == r) {
                    if (EINTR == errno)
                            continue;
                    errno_exit("select");
            }

            if (0 == r) {
                    fprintf(stderr, "select timeout\n");
                    exit(EXIT_FAILURE);
            }

            if (read_frame())
                    break;
            /* EAGAIN - continue select loop. */
        }

        glPopMatrix();
        glfwSwapBuffers(win);
    }
    glfwDestroyWindow(win);
    glfwTerminate();

    if(xioctl(fd, VIDIOC_STREAMOFF, &type) < 0) errno_exit("VIDIOC_STREAMOFF");

    for(int i = 0; i < n_buffers; ++i)
    {
        if(munmap(buffers[i].start, buffers[i].length) < 0) errno_exit("munmap");
    }
    free(buffers);

    if(close(fd) < 0) errno_exit("close");

    return EXIT_SUCCESS;
}

<|endoftext|>


<commit_before>#include "../src/benchmark_v2.hpp"
#include "../src/random_int_generator.hpp"
#include "../src/sort.hpp"
#include <cmath>
#include <iostream>
#include <memory>

class generator
{
    shino::random_int_generator<int, std::mt19937_64> gen;
public:
    using input_type = std::size_t;

    generator():
            gen(0, 100'000)
    {}

    generator(generator&& other):
            gen(std::move(other.gen))
    {}

    std::tuple<std::vector<int>, std::size_t> operator()(input_type input)
    {
        static std::vector<int> v(input);
        if (v.size() != input)
        {
            v.resize(input);
            gen(v.begin(), v.end());
        }

        return std::make_tuple(v, 0);
    }
};

class different_winsize_generator
{
    shino::random_int_generator<> gen;
    std::size_t vsize;
public:
    using input_type = std::size_t;

    different_winsize_generator(std::size_t size):
            vsize(size)
    {}

    different_winsize_generator(different_winsize_generator&& other):
            gen(std::move(other.gen)),
            vsize(other.vsize)
    {}

    std::tuple<std::vector<int>, std::size_t> operator()(input_type input)
    {
        static std::vector<int> v(vsize);
        static std::size_t window_length = input;
        if (window_length == input) //return cached value
        {
            return std::make_tuple(v, window_length);
        }
        window_length = input;
        std::uniform_int_distribution<std::size_t> dist(1, input - 1);
        for (int i = 0; i < v.size(); ++i)
        {
            v[i] = i;
        }

        for (std::size_t i = 0; i < v.size(); i += window_length)
        {
            std::shuffle(&v[i], &v[std::min(i + window_length, v.size())], gen.get_engine());
        }

        return std::make_tuple(v, window_length);
    }
};

int main()
{
    std::size_t counter = 0;
    auto combsort_bench = [&counter](std::vector<int>& v, std::size_t)
    {
        shino::comb_sort(v.begin(), v.end(), 1.3);
        std::cout << v.front() << ' ' << counter++ << '\n'; //just to tell compiler to not optimize the code away
    };

    auto stdsort_bench = [&counter](std::vector<int>& v, std::size_t)
    {
        std::sort(v.begin(), v.end());
        std::cout << v.front() << ' ' << counter++ << '\n'; //ditto
    };

//    auto simple_benchmark = shino::benchmarker(generator{}, combsort_bench, stdsort_bench);
//
//    for (std::size_t i = 1000; i <= 5'000; i += 5)
//    {
//        simple_benchmark.time(i, 3); //3 means runcount for the same dataset
//    }
//
//    simple_benchmark.save_as<std::chrono::microseconds>("./benchmarks/simple-benchmark/benchmarks.txt",
//                                                        {"./benchmarks/simple-benchmark/comb sort benchmark.txt",
//                                                         "./benchmarks/simple-benchmark/standard sort benchmark.txt"},
//                                                        "vector<int>.size()", "microseconds");
//
//    auto winsize_benchmark = shino::benchmarker(different_winsize_generator(10'000), combsort_bench, stdsort_bench);
//
//    for (std::size_t i = 1000; i <= 5'000; i += 5)
//    {
//        winsize_benchmark.time(i, 3); //3 means runcount for the same dataset
//    }
//
//    winsize_benchmark.save_as<std::chrono::microseconds>("./benchmarks/winsize-benchmark/benchmarks.txt",
//                                                        {"./benchmarks/winsize-benchmark/comb sort benchmark.txt",
//                                                         "./benchmarks/winsize-benchmark/standard sort benchmark.txt"},
//                                                        "window length",
//                                                        "microseconds");

    auto small_winsize_benchmark = shino::benchmarker(different_winsize_generator(50'000),
                                                      combsort_bench,
                                                      stdsort_bench);
    for (std::size_t i = 2; i <= 10; ++i)
    {
        small_winsize_benchmark.time(i, 3);
    }

    small_winsize_benchmark.save_as<std::chrono::microseconds>("./benchmarks/small-winsize-benchmark/benchmarks.txt",
                                                               {"comb sort benchmark.txt",
                                                               "standard sort benchmark.txt"},
                                                               "window length",
                                                               "microseconds");
}

<commit_msg>fixed warning about signed vs unsigned comparison<commit_after>#include "../src/benchmark_v2.hpp"
#include "../src/random_int_generator.hpp"
#include "../src/sort.hpp"
#include <cmath>
#include <iostream>
#include <memory>

class generator
{
    shino::random_int_generator<int, std::mt19937_64> gen;
public:
    using input_type = std::size_t;

    generator():
            gen(0, 100'000)
    {}

    generator(generator&& other):
            gen(std::move(other.gen))
    {}

    std::tuple<std::vector<int>, std::size_t> operator()(input_type input)
    {
        static std::vector<int> v(input);
        if (v.size() != input)
        {
            v.resize(input);
            gen(v.begin(), v.end());
        }

        return std::make_tuple(v, 0);
    }
};

class different_winsize_generator
{
    shino::random_int_generator<> gen;
    std::size_t vsize;
public:
    using input_type = std::size_t;

    different_winsize_generator(std::size_t size):
            vsize(size)
    {}

    different_winsize_generator(different_winsize_generator&& other):
            gen(std::move(other.gen)),
            vsize(other.vsize)
    {}

    std::tuple<std::vector<int>, std::size_t> operator()(input_type input)
    {
        static std::vector<int> v(vsize);
        static std::size_t window_length = input;
        if (window_length == input) //return cached value
        {
            return std::make_tuple(v, window_length);
        }
        window_length = input;
        std::uniform_int_distribution<std::size_t> dist(1, input - 1);
        for (int i = 0; i < (int)v.size(); ++i)
        {
            v[i] = i;
        }

        for (std::size_t i = 0; i < v.size(); i += window_length)
        {
            std::shuffle(&v[i], &v[std::min(i + window_length, v.size())], gen.get_engine());
        }

        return std::make_tuple(v, window_length);
    }
};

int main()
{
    std::size_t counter = 0;
    auto combsort_bench = [&counter](std::vector<int>& v, std::size_t)
    {
        shino::comb_sort(v.begin(), v.end(), 1.3);
        std::cout << v.front() << ' ' << counter++ << '\n'; //just to tell compiler to not optimize the code away
    };

    auto stdsort_bench = [&counter](std::vector<int>& v, std::size_t)
    {
        std::sort(v.begin(), v.end());
        std::cout << v.front() << ' ' << counter++ << '\n'; //ditto
    };

//    auto simple_benchmark = shino::benchmarker(generator{}, combsort_bench, stdsort_bench);
//
//    for (std::size_t i = 1000; i <= 5'000; i += 5)
//    {
//        simple_benchmark.time(i, 3); //3 means runcount for the same dataset
//    }
//
//    simple_benchmark.save_as<std::chrono::microseconds>("./benchmarks/simple-benchmark/benchmarks.txt",
//                                                        {"./benchmarks/simple-benchmark/comb sort benchmark.txt",
//                                                         "./benchmarks/simple-benchmark/standard sort benchmark.txt"},
//                                                        "vector<int>.size()", "microseconds");
//
//    auto winsize_benchmark = shino::benchmarker(different_winsize_generator(10'000), combsort_bench, stdsort_bench);
//
//    for (std::size_t i = 1000; i <= 5'000; i += 5)
//    {
//        winsize_benchmark.time(i, 3); //3 means runcount for the same dataset
//    }
//
//    winsize_benchmark.save_as<std::chrono::microseconds>("./benchmarks/winsize-benchmark/benchmarks.txt",
//                                                        {"./benchmarks/winsize-benchmark/comb sort benchmark.txt",
//                                                         "./benchmarks/winsize-benchmark/standard sort benchmark.txt"},
//                                                        "window length",
//                                                        "microseconds");

    auto small_winsize_benchmark = shino::benchmarker(different_winsize_generator(50'000),
                                                      combsort_bench,
                                                      stdsort_bench);
    for (std::size_t i = 2; i <= 10; ++i)
    {
        small_winsize_benchmark.time(i, 3);
    }

    small_winsize_benchmark.save_as<std::chrono::microseconds>("./benchmarks/small-winsize-benchmark/benchmarks.txt",
                                                               {"comb sort benchmark.txt",
                                                               "standard sort benchmark.txt"},
                                                               "window length",
                                                               "microseconds");
}

<|endoftext|>


<commit_before>/*
 * Caching HTTP responses.  Implementation for the rules described in
 * RFC 2616.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "http_cache_rfc.hxx"
#include "http_cache_document.hxx"
#include "http_cache_internal.hxx"
#include "http_address.hxx"
#include "http_util.hxx"
#include "strref2.h"
#include "strmap.hxx"
#include "date.h"
#include "resource_address.hxx"
#include "cgi_address.hxx"
#include "lhttp_address.hxx"
#include "pool.hxx"

#include <assert.h>
#include <stdlib.h>

static struct strref *
next_item(struct strref *s, struct strref *p)
{
    const char *comma;

    strref_ltrim(s);
    if (strref_is_empty(s))
        return nullptr;

    comma = strref_chr(s, ',');
    if (comma == nullptr) {
        *p = *s;
        strref_clear(s);
    } else {
        strref_set2(p, s->data, comma);
        strref_set2(s, comma + 1, strref_end(s));
    }

    strref_rtrim(p);
    return p;
}

static struct http_cache_info *
http_cache_info_new(struct pool &pool)
{
    return NewFromPool<http_cache_info>(pool);
}

static bool
resource_address_has_query_string(const struct resource_address *address)
{
    switch (address->type) {
    case RESOURCE_ADDRESS_NONE:
    case RESOURCE_ADDRESS_LOCAL:
    case RESOURCE_ADDRESS_NFS:
        return false;

    case RESOURCE_ADDRESS_HTTP:
        return strchr(address->u.http->path, '?') != nullptr;

    case RESOURCE_ADDRESS_LHTTP:
        return strchr(address->u.lhttp->uri, '?') != nullptr;

    case RESOURCE_ADDRESS_PIPE:
        return false;

    case RESOURCE_ADDRESS_CGI:
    case RESOURCE_ADDRESS_FASTCGI:
    case RESOURCE_ADDRESS_WAS:
        return address->u.cgi->query_string == nullptr ||
            *address->u.cgi->query_string == 0;

    case RESOURCE_ADDRESS_AJP:
        return false;
    }

    /* unreachable */
    assert(false);
    return false;
}

/* check whether the request could produce a cacheable response */
struct http_cache_info *
http_cache_request_evaluate(struct pool *pool,
                            http_method_t method,
                            const struct resource_address *address,
                            const struct strmap *headers,
                            struct istream *body)
{
    struct http_cache_info *info = nullptr;
    const char *p;

    if (method != HTTP_METHOD_GET || body != nullptr)
        /* RFC 2616 13.11 "Write-Through Mandatory" */
        return nullptr;

    if (headers != nullptr) {
        p = headers->Get("range");
        if (p != nullptr)
            return nullptr;

        /* RFC 2616 14.8: "When a shared cache receives a request
           containing an Authorization field, it MUST NOT return the
           corresponding response as a reply to any other request
           [...] */
        if (headers->Get("authorization") != nullptr)
            return nullptr;

        p = headers->Get("cache-control");
        if (p != nullptr) {
            struct strref cc, tmp, *s;

            strref_set_c(&cc, p);

            while ((s = next_item(&cc, &tmp)) != nullptr) {
                if (strref_cmp_literal(s, "no-cache") == 0 ||
                    strref_cmp_literal(s, "no-store") == 0)
                    return nullptr;

                if (strref_cmp_literal(s, "only-if-cached") == 0) {
                    if (info == nullptr)
                        info = http_cache_info_new(*pool);
                    info->only_if_cached = true;
                }
            }
        } else {
            p = headers->Get("pragma");
            if (p != nullptr && strcmp(p, "no-cache") == 0)
                return nullptr;
        }
    }

    if (info == nullptr)
        info = http_cache_info_new(*pool);

    info->is_remote = address->type == RESOURCE_ADDRESS_HTTP ||
        address->type == RESOURCE_ADDRESS_AJP;
    info->has_query_string = resource_address_has_query_string(address);

    return info;
}

gcc_pure
bool
http_cache_vary_fits(const struct strmap &vary, const struct strmap *headers)
{
    for (const auto &i : vary) {
        const char *value = strmap_get_checked(headers, i.key);
        if (value == nullptr)
            value = "";

        if (strcmp(i.value, value) != 0)
            /* mismatch in one of the "Vary" request headers */
            return false;
    }

    return true;
}

gcc_pure
bool
http_cache_vary_fits(const struct strmap *vary, const struct strmap *headers)
{
    return vary == nullptr || http_cache_vary_fits(*vary, headers);
}

bool
http_cache_request_invalidate(http_method_t method)
{
    /* RFC 2616 13.10 "Invalidation After Updates or Deletions" */
    return method == HTTP_METHOD_PUT || method == HTTP_METHOD_DELETE ||
        method == HTTP_METHOD_POST;
}

static time_t
parse_translate_time(const char *p, time_t offset)
{
    time_t t;

    if (p == nullptr)
        return (time_t)-1;

    t = http_date_parse(p);
    if (t != (time_t)-1)
        t += offset;

    return t;
}

/**
 * RFC 2616 13.4
 */
static bool
http_status_cacheable(http_status_t status)
{
    return status == HTTP_STATUS_OK ||
        status == HTTP_STATUS_NON_AUTHORITATIVE_INFORMATION ||
        status == HTTP_STATUS_PARTIAL_CONTENT ||
        status == HTTP_STATUS_MULTIPLE_CHOICES ||
        status == HTTP_STATUS_MOVED_PERMANENTLY ||
        status == HTTP_STATUS_GONE;
}

gcc_pure
static const char *
strmap_get_non_empty(const struct strmap *map, const char *key)
{
    const char *value = map->Get(key);
    if (value != nullptr && *value == 0)
        value = nullptr;
    return value;
}

bool
http_cache_response_evaluate(struct http_cache_info *info,
                             http_status_t status, const struct strmap *headers,
                             off_t body_available)
{
    time_t now, offset;
    const char *p;

    if (!http_status_cacheable(status) || headers == nullptr)
        return false;

    if (body_available != (off_t)-1 && body_available > cacheable_size_limit)
        /* too large for the cache */
        return false;

    p = headers->Get("cache-control");
    if (p != nullptr) {
        struct strref cc, tmp, *s;

        strref_set_c(&cc, p);

        while ((s = next_item(&cc, &tmp)) != nullptr) {
            if (strref_starts_with_n(s, "private", 7) ||
                strref_cmp_literal(s, "no-cache") == 0 ||
                strref_cmp_literal(s, "no-store") == 0)
                return false;

            if (strref_starts_with_n(s, "max-age=", 8)) {
                /* RFC 2616 14.9.3 */
                size_t length = s->length - 8;
                char value[16];
                int seconds;

                if (length >= sizeof(value))
                    continue;

                memcpy(value, s->data + 8, length);
                value[length] = 0;

                seconds = atoi(value);
                if (seconds > 0)
                    info->expires = time(nullptr) + seconds;
            }
        }
    }

    now = time(nullptr);

    if (info->is_remote) {
        p = headers->Get("date");
        if (p == nullptr)
            /* we cannot determine whether to cache a resource if the
               server does not provide its system time */
            return false;

        time_t date = http_date_parse(p);
        if (date == (time_t)-1)
            return false;

        offset = now - date;
    } else
        offset = 0;

    if (info->expires == (time_t)-1) {
        /* RFC 2616 14.9.3: "If a response includes both an Expires
           header and a max-age directive, the max-age directive
           overrides the Expires header" */

        info->expires = parse_translate_time(headers->Get("expires"), offset);
        if (info->expires != (time_t)-1 && info->expires < now)
            cache_log(4, "invalid 'expires' header\n");
    }

    if (info->has_query_string && info->expires == (time_t)-1)
        /* RFC 2616 13.9: "since some applications have traditionally
           used GETs and HEADs with query URLs (those containing a "?"
           in the rel_path part) to perform operations with
           significant side effects, caches MUST NOT treat responses
           to such URIs as fresh unless the server provides an
           explicit expiration time" - this is implemented by not
           storing the resource at all */
        return false;

    info->last_modified = headers->Get("last-modified");
    info->etag = headers->Get("etag");

    info->vary = strmap_get_non_empty(headers, "vary");
    if (info->vary != nullptr && strcmp(info->vary, "*") == 0)
        /* RFC 2616 13.6: A Vary header field-value of "*" always
           fails to match and subsequent requests on that resource can
           only be properly interpreted by the origin server. */
        return false;

    return info->expires != (time_t)-1 || info->last_modified != nullptr ||
        info->etag != nullptr;
}

struct strmap *
http_cache_copy_vary(struct pool *pool, const char *vary,
                     const struct strmap *headers)
{
    struct strmap *dest = strmap_new(pool);
    char **list;

    for (list = http_list_split(pool, vary);
         *list != nullptr; ++list) {
        const char *name = *list;
        const char *value = strmap_get_checked(headers, name);
        if (value == nullptr)
            value = "";
        dest->Set(name, p_strdup(pool, value));
    }

    return dest;
}

bool
http_cache_prefer_cached(const struct http_cache_document *document,
                         const struct strmap *response_headers)
{
    const char *etag;

    if (document->info.etag == nullptr)
        return false;

    etag = strmap_get_checked(response_headers, "etag");

    /* if the ETags are the same, then the resource hasn't changed,
       but the server was too lazy to check that properly */
    return etag != nullptr && strcmp(etag, document->info.etag) == 0;
}
<commit_msg>http_cache_rfc: don't duplicate empty strings<commit_after>/*
 * Caching HTTP responses.  Implementation for the rules described in
 * RFC 2616.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "http_cache_rfc.hxx"
#include "http_cache_document.hxx"
#include "http_cache_internal.hxx"
#include "http_address.hxx"
#include "http_util.hxx"
#include "strref2.h"
#include "strmap.hxx"
#include "date.h"
#include "resource_address.hxx"
#include "cgi_address.hxx"
#include "lhttp_address.hxx"
#include "pool.hxx"

#include <assert.h>
#include <stdlib.h>

static struct strref *
next_item(struct strref *s, struct strref *p)
{
    const char *comma;

    strref_ltrim(s);
    if (strref_is_empty(s))
        return nullptr;

    comma = strref_chr(s, ',');
    if (comma == nullptr) {
        *p = *s;
        strref_clear(s);
    } else {
        strref_set2(p, s->data, comma);
        strref_set2(s, comma + 1, strref_end(s));
    }

    strref_rtrim(p);
    return p;
}

static struct http_cache_info *
http_cache_info_new(struct pool &pool)
{
    return NewFromPool<http_cache_info>(pool);
}

static bool
resource_address_has_query_string(const struct resource_address *address)
{
    switch (address->type) {
    case RESOURCE_ADDRESS_NONE:
    case RESOURCE_ADDRESS_LOCAL:
    case RESOURCE_ADDRESS_NFS:
        return false;

    case RESOURCE_ADDRESS_HTTP:
        return strchr(address->u.http->path, '?') != nullptr;

    case RESOURCE_ADDRESS_LHTTP:
        return strchr(address->u.lhttp->uri, '?') != nullptr;

    case RESOURCE_ADDRESS_PIPE:
        return false;

    case RESOURCE_ADDRESS_CGI:
    case RESOURCE_ADDRESS_FASTCGI:
    case RESOURCE_ADDRESS_WAS:
        return address->u.cgi->query_string == nullptr ||
            *address->u.cgi->query_string == 0;

    case RESOURCE_ADDRESS_AJP:
        return false;
    }

    /* unreachable */
    assert(false);
    return false;
}

/* check whether the request could produce a cacheable response */
struct http_cache_info *
http_cache_request_evaluate(struct pool *pool,
                            http_method_t method,
                            const struct resource_address *address,
                            const struct strmap *headers,
                            struct istream *body)
{
    struct http_cache_info *info = nullptr;
    const char *p;

    if (method != HTTP_METHOD_GET || body != nullptr)
        /* RFC 2616 13.11 "Write-Through Mandatory" */
        return nullptr;

    if (headers != nullptr) {
        p = headers->Get("range");
        if (p != nullptr)
            return nullptr;

        /* RFC 2616 14.8: "When a shared cache receives a request
           containing an Authorization field, it MUST NOT return the
           corresponding response as a reply to any other request
           [...] */
        if (headers->Get("authorization") != nullptr)
            return nullptr;

        p = headers->Get("cache-control");
        if (p != nullptr) {
            struct strref cc, tmp, *s;

            strref_set_c(&cc, p);

            while ((s = next_item(&cc, &tmp)) != nullptr) {
                if (strref_cmp_literal(s, "no-cache") == 0 ||
                    strref_cmp_literal(s, "no-store") == 0)
                    return nullptr;

                if (strref_cmp_literal(s, "only-if-cached") == 0) {
                    if (info == nullptr)
                        info = http_cache_info_new(*pool);
                    info->only_if_cached = true;
                }
            }
        } else {
            p = headers->Get("pragma");
            if (p != nullptr && strcmp(p, "no-cache") == 0)
                return nullptr;
        }
    }

    if (info == nullptr)
        info = http_cache_info_new(*pool);

    info->is_remote = address->type == RESOURCE_ADDRESS_HTTP ||
        address->type == RESOURCE_ADDRESS_AJP;
    info->has_query_string = resource_address_has_query_string(address);

    return info;
}

gcc_pure
bool
http_cache_vary_fits(const struct strmap &vary, const struct strmap *headers)
{
    for (const auto &i : vary) {
        const char *value = strmap_get_checked(headers, i.key);
        if (value == nullptr)
            value = "";

        if (strcmp(i.value, value) != 0)
            /* mismatch in one of the "Vary" request headers */
            return false;
    }

    return true;
}

gcc_pure
bool
http_cache_vary_fits(const struct strmap *vary, const struct strmap *headers)
{
    return vary == nullptr || http_cache_vary_fits(*vary, headers);
}

bool
http_cache_request_invalidate(http_method_t method)
{
    /* RFC 2616 13.10 "Invalidation After Updates or Deletions" */
    return method == HTTP_METHOD_PUT || method == HTTP_METHOD_DELETE ||
        method == HTTP_METHOD_POST;
}

static time_t
parse_translate_time(const char *p, time_t offset)
{
    time_t t;

    if (p == nullptr)
        return (time_t)-1;

    t = http_date_parse(p);
    if (t != (time_t)-1)
        t += offset;

    return t;
}

/**
 * RFC 2616 13.4
 */
static bool
http_status_cacheable(http_status_t status)
{
    return status == HTTP_STATUS_OK ||
        status == HTTP_STATUS_NON_AUTHORITATIVE_INFORMATION ||
        status == HTTP_STATUS_PARTIAL_CONTENT ||
        status == HTTP_STATUS_MULTIPLE_CHOICES ||
        status == HTTP_STATUS_MOVED_PERMANENTLY ||
        status == HTTP_STATUS_GONE;
}

gcc_pure
static const char *
strmap_get_non_empty(const struct strmap *map, const char *key)
{
    const char *value = map->Get(key);
    if (value != nullptr && *value == 0)
        value = nullptr;
    return value;
}

bool
http_cache_response_evaluate(struct http_cache_info *info,
                             http_status_t status, const struct strmap *headers,
                             off_t body_available)
{
    time_t now, offset;
    const char *p;

    if (!http_status_cacheable(status) || headers == nullptr)
        return false;

    if (body_available != (off_t)-1 && body_available > cacheable_size_limit)
        /* too large for the cache */
        return false;

    p = headers->Get("cache-control");
    if (p != nullptr) {
        struct strref cc, tmp, *s;

        strref_set_c(&cc, p);

        while ((s = next_item(&cc, &tmp)) != nullptr) {
            if (strref_starts_with_n(s, "private", 7) ||
                strref_cmp_literal(s, "no-cache") == 0 ||
                strref_cmp_literal(s, "no-store") == 0)
                return false;

            if (strref_starts_with_n(s, "max-age=", 8)) {
                /* RFC 2616 14.9.3 */
                size_t length = s->length - 8;
                char value[16];
                int seconds;

                if (length >= sizeof(value))
                    continue;

                memcpy(value, s->data + 8, length);
                value[length] = 0;

                seconds = atoi(value);
                if (seconds > 0)
                    info->expires = time(nullptr) + seconds;
            }
        }
    }

    now = time(nullptr);

    if (info->is_remote) {
        p = headers->Get("date");
        if (p == nullptr)
            /* we cannot determine whether to cache a resource if the
               server does not provide its system time */
            return false;

        time_t date = http_date_parse(p);
        if (date == (time_t)-1)
            return false;

        offset = now - date;
    } else
        offset = 0;

    if (info->expires == (time_t)-1) {
        /* RFC 2616 14.9.3: "If a response includes both an Expires
           header and a max-age directive, the max-age directive
           overrides the Expires header" */

        info->expires = parse_translate_time(headers->Get("expires"), offset);
        if (info->expires != (time_t)-1 && info->expires < now)
            cache_log(4, "invalid 'expires' header\n");
    }

    if (info->has_query_string && info->expires == (time_t)-1)
        /* RFC 2616 13.9: "since some applications have traditionally
           used GETs and HEADs with query URLs (those containing a "?"
           in the rel_path part) to perform operations with
           significant side effects, caches MUST NOT treat responses
           to such URIs as fresh unless the server provides an
           explicit expiration time" - this is implemented by not
           storing the resource at all */
        return false;

    info->last_modified = headers->Get("last-modified");
    info->etag = headers->Get("etag");

    info->vary = strmap_get_non_empty(headers, "vary");
    if (info->vary != nullptr && strcmp(info->vary, "*") == 0)
        /* RFC 2616 13.6: A Vary header field-value of "*" always
           fails to match and subsequent requests on that resource can
           only be properly interpreted by the origin server. */
        return false;

    return info->expires != (time_t)-1 || info->last_modified != nullptr ||
        info->etag != nullptr;
}

struct strmap *
http_cache_copy_vary(struct pool *pool, const char *vary,
                     const struct strmap *headers)
{
    struct strmap *dest = strmap_new(pool);
    char **list;

    for (list = http_list_split(pool, vary);
         *list != nullptr; ++list) {
        const char *name = *list;
        const char *value = strmap_get_checked(headers, name);
        if (value == nullptr)
            value = "";
        else
            value = p_strdup(pool, value);
        dest->Set(name, value);
    }

    return dest;
}

bool
http_cache_prefer_cached(const struct http_cache_document *document,
                         const struct strmap *response_headers)
{
    const char *etag;

    if (document->info.etag == nullptr)
        return false;

    etag = strmap_get_checked(response_headers, "etag");

    /* if the ETags are the same, then the resource hasn't changed,
       but the server was too lazy to check that properly */
    return etag != nullptr && strcmp(etag, document->info.etag) == 0;
}
<|endoftext|>


<commit_before>#define CATCH_CONFIG_MAIN
#include "catch.hpp"

#include <iostream>
#include <vector>
#include <string>

#include "data/filters.h"
#include "yaml-cpp/yaml.h"
#include "scene/sceneLoader.h"
#include "scene/scene.h"

using namespace Tangram;
using YAML::Node;

// shader property overwrite test
// shader extensions test
// shader block test

TEST_CASE( "Style Mixing Test1: propMerge Tests (recursive overWrite properties)", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("Nodes: { \
                                Node1: { \
                                    prop1: { \
                                        subProp1: { \
                                            tag1: value1 } \
                                    }, \
                                    prop2: { \
                                        subProp3: { \
                                            tag2: value2 } \
                                    } \
                                }, \
                                Node2: { \
                                    prop1: { \
                                        subProp1: { \
                                            tag3: value3 } \
                                    }, \
                                    prop2: { \
                                        subProp2: value_scalar, \
                                        subProp3: value_scalar2 \
                                    } \
                                }, \
                                Node3: { \
                                    prop1: { \
                                        subProp1: {\
                                            tag1: value1_3 \
                                        } \
                                    }, \
                                    prop2: { \
                                        subProp2: [v1, v2], \
                                        subProp3: { \
                                            tag4: value4 \
                                        } \
                                   } \
                                }, \
                            }")["Nodes"];

    // NodeMix:
    //      prop1:
    //          subProp1:
    //              tag1: value1_3
    //              tag3: value3
    //      prop2:
    //          subProp2: [v1, v2]
    //          subProp3:
    //              tag4: value4
    //              tag2: value2

    Node mixedNode;

    for (auto& property : {"prop1", "prop2"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].IsMap());
    REQUIRE(mixedNode["prop1"]["subProp1"].IsMap());
    REQUIRE(mixedNode["prop1"]["subProp1"]["tag1"].as<std::string>() ==  "value1_3");
    REQUIRE(mixedNode["prop1"]["subProp1"]["tag3"].as<std::string>() ==  "value3");
    REQUIRE(mixedNode["prop2"].IsMap());
    REQUIRE(mixedNode["prop2"]["subProp2"].IsSequence());
    REQUIRE(mixedNode["prop2"]["subProp2"].size() == 2);
    REQUIRE(mixedNode["prop2"]["subProp3"].IsMap());
    REQUIRE(mixedNode["prop2"]["subProp3"].size() == 2);
    REQUIRE(mixedNode["prop2"]["subProp3"]["tag4"].as<std::string>() == "value4");
    REQUIRE(mixedNode["prop2"]["subProp3"]["tag2"].as<std::string>() == "value2");
}

TEST_CASE( "Style Mixing Test1: propMerge Tests (overWrite properties)", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("Nodes: { \
                                Node1: { \
                                    prop1: value1 }, \
                                Node2: { \
                                    prop1: value1_2, \
                                    prop2: value2 }, \
                                Node3: { \
                                    prop1: value1_3 } \
                                }")["Nodes"];

    // NodeMix:
    //      prop1: value1_3
    //      prop2: value2

    Node mixedNode;
    for (auto& property : {"prop1", "prop2"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].as<std::string>() == "value1_3");
    REQUIRE(mixedNode["prop2"].as<std::string>() == "value2");
}

TEST_CASE( "Style Mixing Test1: propMerge Tests (boolean properties)", "[mixing][core][yaml]") {

    SceneLoader sceneLoader;
    Node node = YAML::Load("Nodes: { \
                                Node1: { \
                                    prop1: false }, \
                                Node2: { \
                                    prop1: true }, \
                                Node3: { \
                                    prop1: false } \
                            }")["Nodes"];

    // NodeMix:
    //      prop1: true

    Node mixedNode;
    for (auto& property : {"prop1"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].as<bool>());
}

<commit_msg>Unit tests for shaderExtMerge & shaderBlockMerge<commit_after>#define CATCH_CONFIG_MAIN
#include "catch.hpp"

#include <iostream>
#include <vector>
#include <string>

#include "data/filters.h"
#include "yaml-cpp/yaml.h"
#include "scene/sceneLoader.h"
#include "scene/scene.h"

using namespace Tangram;
using YAML::Node;

TEST_CASE( "Style Mixing Test1: Shader Extensions Merging ()", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("{ \
                                Node1: { \
                                    shaders: { \
                                        extensions: extension1 \
                                    } \
                                }, \
                                Node2: { \
                                    shaders: { \
                                        extensions: [extension1, extension2, extension3] \
                                    } \
                                }, \
                                Node3: { \
                                    shaders: { \
                                        extensions: extension3 \
                                        } \
                                }, \
                                Node4: { \
                                    shaders: { \
                                        extensions: [extension4] \
                                        } \
                                } \
                            }");

    Node extNode;

    extNode = sceneLoader.shaderExtMerge( { node["Node1"] } );
    // extNode: [extension1]
    REQUIRE(extNode.size() == 1);
    REQUIRE(extNode[0].as<std::string>() == "extension1");

    extNode = sceneLoader.shaderExtMerge( { node["Node1"], node["Node2"], node["Node3"], node["Node4"] } );
    // extNode: [extension1, extension2, extension3, extension4]
    REQUIRE(extNode.size() == 4);
    REQUIRE(extNode[0].as<std::string>() == "extension1");
    REQUIRE(extNode[1].as<std::string>() == "extension2");
    REQUIRE(extNode[2].as<std::string>() == "extension3");
    REQUIRE(extNode[3].as<std::string>() == "extension4");

}

TEST_CASE( "Style Mixing Test1: Shader Blocks Merging ()", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("{ \
                                Node1: { \
                                    shaders: { \
                                        blocks: { \
                                            color: colorBlockA; ,\
                                            normal: normalBlockA; ,\
                                        } \
                                    } \
                                }, \
                                Node2: { \
                                    shaders: { \
                                        blocks: { \
                                            color: colorBlockB; ,\
                                            position: posBlockB; ,\
                                            global: globalBlockB; \
                                        } \
                                    } \
                                }, \
                                Node3: { \
                                    shaders: { \
                                        blocks: { \
                                            global: globalBlockC; ,\
                                            filter: filterBlockC; \
                                        } \
                                    } \
                                }, \
                            }");

    Node shaderBlocksNode = sceneLoader.shaderBlockMerge( { node["Node1"] } );
    // shaderBlocksNode:
    //          color: colorBlockA;
    //          normal: normalBlockA;

    REQUIRE(shaderBlocksNode["color"].as<std::string>() == "colorBlockA;");
    REQUIRE(shaderBlocksNode["normal"].as<std::string>() == "normalBlockA;");
    REQUIRE(!shaderBlocksNode["global"]);

    shaderBlocksNode = sceneLoader.shaderBlockMerge( { node["Node1"], node["Node2"], node["Node3"] } );

    // shaderBlocksNode:
    //          color: colorBlockA;\ncolorBlockB;
    //          normal: normalBlockA;
    //          position: posBlockB;
    //          global: globalBlockB;\nglobalBlockC;
    //          filter: filterBlockC;

    REQUIRE(shaderBlocksNode["color"].as<std::string>() == "colorBlockA;\ncolorBlockB;");
    REQUIRE(shaderBlocksNode["normal"].as<std::string>() == "normalBlockA;");
    REQUIRE(shaderBlocksNode["position"].as<std::string>() == "posBlockB;");
    REQUIRE(shaderBlocksNode["global"].as<std::string>() == "globalBlockB;\nglobalBlockC;");
    REQUIRE(shaderBlocksNode["filter"].as<std::string>() == "filterBlockC;");
}

TEST_CASE( "Style Mixing Test1: propMerge Tests (recursive overWrite properties)", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("{ \
                                Node1: { \
                                    prop1: { \
                                        subProp1: { \
                                            tag1: value1 } \
                                    }, \
                                    prop2: { \
                                        subProp3: { \
                                            tag2: value2 } \
                                    } \
                                }, \
                                Node2: { \
                                    prop1: { \
                                        subProp1: { \
                                            tag3: value3 } \
                                    }, \
                                    prop2: { \
                                        subProp2: value_scalar, \
                                        subProp3: value_scalar2 \
                                    } \
                                }, \
                                Node3: { \
                                    prop1: { \
                                        subProp1: {\
                                            tag1: value1_3 \
                                        } \
                                    }, \
                                    prop2: { \
                                        subProp2: [v1, v2], \
                                        subProp3: { \
                                            tag4: value4 \
                                        } \
                                   } \
                                }, \
                            }");

    // NodeMix:
    //      prop1:
    //          subProp1:
    //              tag1: value1_3
    //              tag3: value3
    //      prop2:
    //          subProp2: [v1, v2]
    //          subProp3:
    //              tag4: value4
    //              tag2: value2

    Node mixedNode;

    for (auto& property : {"prop1", "prop2"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].IsMap());
    REQUIRE(mixedNode["prop1"]["subProp1"].IsMap());
    REQUIRE(mixedNode["prop1"]["subProp1"]["tag1"].as<std::string>() ==  "value1_3");
    REQUIRE(mixedNode["prop1"]["subProp1"]["tag3"].as<std::string>() ==  "value3");
    REQUIRE(mixedNode["prop2"].IsMap());
    REQUIRE(mixedNode["prop2"]["subProp2"].IsSequence());
    REQUIRE(mixedNode["prop2"]["subProp2"].size() == 2);
    REQUIRE(mixedNode["prop2"]["subProp3"].IsMap());
    REQUIRE(mixedNode["prop2"]["subProp3"].size() == 2);
    REQUIRE(mixedNode["prop2"]["subProp3"]["tag4"].as<std::string>() == "value4");
    REQUIRE(mixedNode["prop2"]["subProp3"]["tag2"].as<std::string>() == "value2");
}

TEST_CASE( "Style Mixing Test1: propMerge Tests (overWrite properties)", "[mixing][core][yaml]") {
    SceneLoader sceneLoader;
    Node node = YAML::Load("{ \
                                Node1: { \
                                    prop1: value1 }, \
                                Node2: { \
                                    prop1: value1_2, \
                                    prop2: value2 }, \
                                Node3: { \
                                    prop1: value1_3 } \
                                }");

    // NodeMix:
    //      prop1: value1_3
    //      prop2: value2

    Node mixedNode;
    for (auto& property : {"prop1", "prop2"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].as<std::string>() == "value1_3");
    REQUIRE(mixedNode["prop2"].as<std::string>() == "value2");
}

TEST_CASE( "Style Mixing Test1: propMerge Tests (boolean properties)", "[mixing][core][yaml]") {

    SceneLoader sceneLoader;
    Node node = YAML::Load("{ \
                                Node1: { \
                                    prop1: false }, \
                                Node2: { \
                                    prop1: true }, \
                                Node3: { \
                                    prop1: false } \
                            }");

    // NodeMix:
    //      prop1: true

    Node mixedNode;
    for (auto& property : {"prop1"}) {
        mixedNode[property] = sceneLoader.propMerge(property, { node["Node1"], node["Node2"], node["Node3"] });
    }

    REQUIRE(mixedNode["prop1"].as<bool>());
}

<|endoftext|>


<commit_before>#include <gtest/gtest.h>

#include <stdio.h>
#include <unistd.h>
#include <cstdlib>
#include <iostream>
#include <memory>
#include <string>
#include <utility>

#include "config/config.h"
#include "http/httpclient.h"
#include "logging/logging.h"
#include "primary/sotauptaneclient.h"
#include "storage/invstorage.h"
#include "utilities/utils.h"

std::string address;

bool match_error(const Json::Value& error, const Uptane::Exception& e) {
  if (error["director"]["update"]["err_msg"].asString() == e.what() ||
      error["director"]["targets"][e.getName()]["err_msg"].asString() == e.what() ||
      error["image_repo"]["update"]["err_msg"].asString() == e.what() ||
      error["image_repo"]["targets"][e.getName()]["err_msg"].asString() == e.what()) {
    return true;
  }
  std::cout << "aktualizr failed with unmatched exception " << typeid(e).name() << ": " << e.what() << "\n";
  std::cout << "Expected error: " << error << "\n";
  return false;
}

class UptaneVector : public ::testing::TestWithParam<std::string> {};

/**
 * Check that aktualizr fails on expired metadata.
 * RecordProperty("zephyr_key", "REQ-150,TST-49");
 * Check that aktualizr fails on bad threshold.
 * RecordProperty("zephyr_key", "REQ-153,TST-52");
 */
TEST_P(UptaneVector, Test) {
  const std::string test_name = GetParam();
  std::cout << "Running test vector " << test_name << "\n";

  TemporaryDirectory temp_dir;
  Config config;
  config.provision.primary_ecu_serial = "test_primary_ecu_serial";
  config.provision.primary_ecu_hardware_id = "test_primary_hardware_id";
  config.uptane.director_server = address + test_name + "/director";
  config.uptane.repo_server = address + test_name + "/image_repo";
  config.storage.path = temp_dir.Path();
  config.storage.uptane_metadata_path = BasedPath(temp_dir.Path() / "metadata");
  config.pacman.type = PackageManager::kNone;
  logger_set_threshold(boost::log::trivial::trace);

  auto storage = INvStorage::newStorage(config.storage);
  Uptane::Manifest uptane_manifest{config, storage};
  auto uptane_client = SotaUptaneClient::newDefaultClient(config, storage);
  Uptane::EcuSerial ecu_serial(config.provision.primary_ecu_serial);
  Uptane::HardwareIdentifier hw_id(config.provision.primary_ecu_hardware_id);
  uptane_client->hw_ids.insert(std::make_pair(ecu_serial, hw_id));
  Uptane::Target target("test_filename", {{Uptane::Hash::Type::kSha256, "sha256"}}, 1, "");
  storage->saveInstalledVersion(ecu_serial.ToString(), target, InstalledVersionUpdateMode::kCurrent);

  HttpClient http_client;
  while (true) {
    HttpResponse response = http_client.post(address + test_name + "/step", Json::Value());
    if (response.http_status_code == 204) {
      return;
    }
    const auto vector(response.getJson());
    std::cout << "VECTOR: " << vector;

    bool should_fail = false;
    if (!vector["director"]["update"]["is_success"].asBool() ||
        !vector["image_repo"]["update"]["is_success"].asBool()) {
      should_fail = true;
    } else {
      for (const auto& t : vector["director"]["targets"]) {
        if (!t["is_success"].asBool()) {
          should_fail = true;
          break;
        }
      }
      for (const auto& t : vector["image_repo"]["targets"]) {
        if (!t["is_success"].asBool()) {
          should_fail = true;
          break;
        }
      }
    }

    try {
      /* Fetch metadata from the director.
       * Check metadata from the director.
       * Identify targets for known ECUs.
       * Fetch metadata from the images repo.
       * Check metadata from the images repo. */
      if (!uptane_client->uptaneIteration()) {
        ASSERT_TRUE(should_fail) << "uptaneIteration unexpectedly failed.";
        throw uptane_client->getLastException();
      }
      std::vector<Uptane::Target> updates;
      ASSERT_TRUE(uptane_client->uptaneOfflineIteration(&updates, nullptr))
          << "uptaneOfflineIteration unexpectedly failed.";
      if (updates.size()) {
        /* Download a binary package.
         * Verify a binary package. */
        result::Download result = uptane_client->downloadImages(updates);
        if (result.status != result::DownloadStatus::kSuccess) {
          ASSERT_TRUE(should_fail) << "downloadImages unexpectedly failed.";
          throw uptane_client->getLastException();
        }
      }

    } catch (const Uptane::Exception& e) {
      ASSERT_TRUE(match_error(vector, e)) << "libaktualizr threw a different exception than expected!";
      continue;
    } catch (const std::exception& e) {
      FAIL() << "libaktualizr failed with unrecognized exception " << typeid(e).name() << ": " << e.what();
    }

    if (should_fail) {
      std::cout << "No exceptions occurred, but expected ";
      if (!vector["director"]["update"]["is_success"].asBool()) {
        std::cout << "exception from director: '" << vector["director"]["update"]["err"]
                  << " with message: " << vector["director"]["update"]["err_msg"] << "\n";
      } else if (!vector["image_repo"]["update"]["is_success"].asBool()) {
        std::cout << "exception from image_repo: '" << vector["image_repo"]["update"]["err"]
                  << " with message: " << vector["image_repo"]["update"]["err_msg"] << "\n";
      }
      FAIL();
    }
  }
  FAIL() << "Step sequence unexpectedly aborted.";
}

std::vector<std::string> GetVectors() {
  HttpClient http_client;
  const Json::Value json_vectors = http_client.get(address, HttpInterface::kNoLimit).getJson();
  std::vector<std::string> vectors;
  for (Json::ValueConstIterator it = json_vectors.begin(); it != json_vectors.end(); it++) {
    vectors.emplace_back((*it).asString());
  }
  return vectors;
}

INSTANTIATE_TEST_SUITE_P(UptaneVectorSuite, UptaneVector, ::testing::ValuesIn(GetVectors()));

int main(int argc, char* argv[]) {
  logger_init();
  logger_set_threshold(boost::log::trivial::trace);

  if (argc < 2) {
    std::cerr << "This program is intended to be run from run_vector_tests.sh!\n";
    return 1;
  }

  /* Use ports to distinguish both the server connection and local storage so
   * that parallel runs of this code don't cause problems that are difficult to
   * debug. */
  const std::string port = argv[1];
  address = "http://127.0.0.1:" + port + "/";

  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>Wrap vector json parsing into a new class.<commit_after>#include <gtest/gtest.h>

#include <stdio.h>
#include <unistd.h>
#include <cstdlib>
#include <iostream>
#include <memory>
#include <string>
#include <utility>

#include "config/config.h"
#include "http/httpclient.h"
#include "logging/logging.h"
#include "primary/sotauptaneclient.h"
#include "storage/invstorage.h"
#include "utilities/utils.h"

std::string address;

class VectorWrapper {
 public:
  VectorWrapper(Json::Value vector) : vector_(std::move(vector)) {}

  bool matchError(const Uptane::Exception& e) {
    if (vector_["director"]["update"]["err_msg"].asString() == e.what() ||
        vector_["director"]["targets"][e.getName()]["err_msg"].asString() == e.what() ||
        vector_["image_repo"]["update"]["err_msg"].asString() == e.what() ||
        vector_["image_repo"]["targets"][e.getName()]["err_msg"].asString() == e.what()) {
      return true;
    }
    std::cout << "aktualizr failed with unmatched exception " << typeid(e).name() << ": " << e.what() << "\n";
    std::cout << "Expected error: " << vector_ << "\n";
    return false;
  }

  bool shouldFail() {
    bool should_fail = false;
    if (!vector_["director"]["update"]["is_success"].asBool() ||
        !vector_["image_repo"]["update"]["is_success"].asBool()) {
      should_fail = true;
    } else {
      for (const auto& t : vector_["director"]["targets"]) {
        if (!t["is_success"].asBool()) {
          should_fail = true;
          break;
        }
      }
      for (const auto& t : vector_["image_repo"]["targets"]) {
        if (!t["is_success"].asBool()) {
          should_fail = true;
          break;
        }
      }
    }
    return should_fail;
  }

  void printExpectedFailure() {
    std::cout << "No exceptions occurred, but expected ";
    if (!vector_["director"]["update"]["is_success"].asBool()) {
      std::cout << "exception from director: '" << vector_["director"]["update"]["err"]
                << " with message: " << vector_["director"]["update"]["err_msg"] << "\n";
    } else if (!vector_["image_repo"]["update"]["is_success"].asBool()) {
      std::cout << "exception from image_repo: '" << vector_["image_repo"]["update"]["err"]
                << " with message: " << vector_["image_repo"]["update"]["err_msg"] << "\n";
    }
  }

 private:
  Json::Value vector_;
};

class UptaneVector : public ::testing::TestWithParam<std::string> {};

/**
 * Check that aktualizr fails on expired metadata.
 * RecordProperty("zephyr_key", "REQ-150,TST-49");
 * Check that aktualizr fails on bad threshold.
 * RecordProperty("zephyr_key", "REQ-153,TST-52");
 */
TEST_P(UptaneVector, Test) {
  const std::string test_name = GetParam();
  std::cout << "Running test vector " << test_name << "\n";

  TemporaryDirectory temp_dir;
  Config config;
  config.provision.primary_ecu_serial = "test_primary_ecu_serial";
  config.provision.primary_ecu_hardware_id = "test_primary_hardware_id";
  config.uptane.director_server = address + test_name + "/director";
  config.uptane.repo_server = address + test_name + "/image_repo";
  config.storage.path = temp_dir.Path();
  config.storage.uptane_metadata_path = BasedPath(temp_dir.Path() / "metadata");
  config.pacman.type = PackageManager::kNone;
  logger_set_threshold(boost::log::trivial::trace);

  auto storage = INvStorage::newStorage(config.storage);
  Uptane::Manifest uptane_manifest{config, storage};
  auto uptane_client = SotaUptaneClient::newDefaultClient(config, storage);
  Uptane::EcuSerial ecu_serial(config.provision.primary_ecu_serial);
  Uptane::HardwareIdentifier hw_id(config.provision.primary_ecu_hardware_id);
  uptane_client->hw_ids.insert(std::make_pair(ecu_serial, hw_id));
  Uptane::Target target("test_filename", {{Uptane::Hash::Type::kSha256, "sha256"}}, 1, "");
  storage->saveInstalledVersion(ecu_serial.ToString(), target, InstalledVersionUpdateMode::kCurrent);

  HttpClient http_client;
  while (true) {
    HttpResponse response = http_client.post(address + test_name + "/step", Json::Value());
    if (response.http_status_code == 204) {
      return;
    }
    const auto vector_json(response.getJson());
    std::cout << "VECTOR: " << vector_json;
    VectorWrapper vector(vector_json);

    bool should_fail = vector.shouldFail();

    try {
      /* Fetch metadata from the director.
       * Check metadata from the director.
       * Identify targets for known ECUs.
       * Fetch metadata from the images repo.
       * Check metadata from the images repo. */
      if (!uptane_client->uptaneIteration()) {
        ASSERT_TRUE(should_fail) << "uptaneIteration unexpectedly failed.";
        throw uptane_client->getLastException();
      }
      std::vector<Uptane::Target> updates;
      ASSERT_TRUE(uptane_client->uptaneOfflineIteration(&updates, nullptr))
          << "uptaneOfflineIteration unexpectedly failed.";
      if (updates.size()) {
        /* Download a binary package.
         * Verify a binary package. */
        result::Download result = uptane_client->downloadImages(updates);
        if (result.status != result::DownloadStatus::kSuccess) {
          ASSERT_TRUE(should_fail) << "downloadImages unexpectedly failed.";
          throw uptane_client->getLastException();
        }
      }

    } catch (const Uptane::Exception& e) {
      ASSERT_TRUE(vector.matchError(e)) << "libaktualizr threw a different exception than expected!";
      continue;
    } catch (const std::exception& e) {
      FAIL() << "libaktualizr failed with unrecognized exception " << typeid(e).name() << ": " << e.what();
    }

    if (should_fail) {
      vector.printExpectedFailure();
      FAIL();
    }
  }
  FAIL() << "Step sequence unexpectedly aborted.";
}

std::vector<std::string> GetVectors() {
  HttpClient http_client;
  const Json::Value json_vectors = http_client.get(address, HttpInterface::kNoLimit).getJson();
  std::vector<std::string> vectors;
  for (Json::ValueConstIterator it = json_vectors.begin(); it != json_vectors.end(); it++) {
    vectors.emplace_back((*it).asString());
  }
  return vectors;
}

INSTANTIATE_TEST_SUITE_P(UptaneVectorSuite, UptaneVector, ::testing::ValuesIn(GetVectors()));

int main(int argc, char* argv[]) {
  logger_init();
  logger_set_threshold(boost::log::trivial::trace);

  if (argc < 2) {
    std::cerr << "This program is intended to be run from run_vector_tests.sh!\n";
    return 1;
  }

  /* Use ports to distinguish both the server connection and local storage so
   * that parallel runs of this code don't cause problems that are difficult to
   * debug. */
  const std::string port = argv[1];
  address = "http://127.0.0.1:" + port + "/";

  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>#include <memory>
#include <vector>
#include <stdexcept>
#include <string>
#include <fstream>

#include "edda_reader.h"
#include "dataset/distr_array.h"
#include "dataset/dataset.h"
#include "core/interpolator.h"
#include "../test/bmp_image.h"

#include "edda.h"
#include "distributions/gaussian.h"
#include "distributions/distribution.h"
#include "distributions/variant.h"

#include "distributions/gmm.h"
#include "distributions/gaussian_mixture.h"
#include "distributions/joint_gaussian.h"
#include "distributions/joint_GMM.h"
#include "distributions/histogram.h"
#include "distributions/joint_histogram.h"

using namespace std;
using namespace edda;
using namespace dist;

#define LC_GENERIC_ARRAY

namespace edda {
#ifdef LC_GENERIC_ARRAY
	DistrArray * readMixArray(ifstream & myfile, int n)
	{
		dist::Variant* distArray;
		distArray = new dist::Variant[n];

		float* distData = new float[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

		for (int index = 0; index < n; index++){
			int distrTypeNumber = 1;
			myfile.read((char*)(&distrTypeNumber), sizeof (int));
			if (distrTypeNumber == 1){ //gaussian
				int nGM;
				myfile.read((char*)(&nGM), sizeof (int));

				myfile.read((char*)(distData), sizeof (float)*nGM * 3);

				dist::GMM new_gmm = dist::GMM(nGM);
				for (int m = 0; m<nGM; m++)
				{
					new_gmm.models[m].m = distData[3 * m];
					new_gmm.models[m].v = distData[3 * m + 1];
					new_gmm.models[m].w = distData[3 * m + 2];
				}
				distArray[index] = new_gmm;
			}
			else if (distrTypeNumber == 2){ //histogram
				int nbins;
				myfile.read((char*)(&nbins), sizeof (int));

				myfile.read((char*)(distData), sizeof (float));
				myfile.read((char*)(distData + 1), sizeof (float));
				myfile.read((char*)(distData + 2), sizeof (float)*nbins);

				distArray[index] = Histogram(distData, nbins);
			}
			else if (distrTypeNumber == 3){ //joint gmm
				int nVar, nComp;
				myfile.read((char*)(&nVar), sizeof (int));
				myfile.read((char*)(&nComp), sizeof (int));

				int sizeEachComp = 1 + nVar + (nVar + 1)*nVar / 2;
				myfile.read((char*)(distData), sizeof (float)* sizeEachComp * nComp);

				std::vector<Real> weights(nComp);
				std::vector<JointGaussian> gaus(nComp);
				for (int c = 0; c < nComp; c++)
				{
					weights[c] = distData[sizeEachComp * c];

					ublas_vector mean = ublas_vector(nVar, 0);
					ublas_matrix cov = ublas::zero_matrix<Real>(nVar, nVar);;
					for (int jj = 0; jj < nVar; jj++){
						mean(jj) = distData[sizeEachComp * c + 1 + jj];
					}
					int count = 0;
					for (int jj = 0; jj < nVar; jj++){
						for (int i = jj; i < nVar; i++){
							cov(jj, i) = distData[c*sizeEachComp + 1 + nVar + count];//cov row major
							count++;
						}
					}
					for (int jj = 0; jj < nVar; jj++){
						for (int i = 0; i < jj; i++){
							cov(jj, i) = cov(i, jj);
						}
					}
					gaus[c] = JointGaussian(mean, cov);
				}
				dist::JointGMM new_distr(weights, gaus, nVar, nComp);
				distArray[index] = new_distr;
			}
			else if (distrTypeNumber == 4) { //joint hist

				int* intData = new int[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

				int num_vars;
				myfile.read((char*)(&num_vars), sizeof(int));

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> min_vals(distData, distData + num_vars);

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> max_vals(distData, distData + num_vars);

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> bin_widths(distData, distData + num_vars);

				myfile.read((char*)(intData), sizeof(int)* num_vars);
				std::vector<int> num_bins(intData, intData + num_vars);

				//read pdf
				boost::unordered_map<std::vector<int>, Real> pdf;
				int n_pdf;
				myfile.read((char*)(&n_pdf), sizeof(int));
				int count = 0;
				while (count < n_pdf) {
					int n_vint;
					myfile.read((char*)(&n_vint), sizeof(int));
					myfile.read((char*)(intData), sizeof(int)* n_vint);
					std::vector<int> vint(intData, intData + n_vint);
					float rfloat;
					myfile.read((char*)(&rfloat), sizeof(float));
					Real r = rfloat;
					pdf[vint] = r;
					count++;
				}

				//read mean
				ublas_vector mean(num_vars, 0);
				myfile.read((char*)(distData), sizeof(float)* num_vars);
				for (int jj = 0; jj < num_vars; jj++) {
					mean(jj) = distData[jj];
				}

				//read cov
				ublas_matrix cov = ublas::zero_matrix<Real>(num_vars, num_vars);;
				int sizeHalfCovMat = (num_vars + 1)*num_vars / 2;
				myfile.read((char*)(distData), sizeof(float)* sizeHalfCovMat);
				count = 0;
				for (int jj = 0; jj < num_vars; jj++) {
					for (int i = jj; i < num_vars; i++) {
						cov(jj, i) = distData[count];//cov row major
						count++;
					}
				}
				for (int jj = 0; jj < num_vars; jj++) {
					for (int i = 0; i < jj; i++) {
						cov(jj, i) = cov(i, jj);
					}
				}

				dist::JointHistogram new_distr(num_vars, min_vals, max_vals, bin_widths, num_bins, pdf, mean, cov);
				distArray[index] = new_distr;
			}
			else{
				myfile.close();
				throw NotImplementedException();
			}
		}

		delete[] distData;

		shared_ary<dist::Variant> pArray(new dist::Variant[n], n);
		for (int i = 0; i < n; i++)
		{
			pArray[i] = distArray[i];
		}
		DistrArray * abs_array = new ScalarDistrArray<dist::Variant>(pArray);
		return abs_array;
	}

#elif
	DistrArray * readMixArray(ifstream & myfile, int n)
	{
		if (n == 0) {
			dist::Variant* distArray = new dist::Variant[n];
			shared_ary<dist::Variant> pArray(distArray, n);
			DistrArray * abs_array = new ScalarDistrArray<dist::Variant>(pArray);
			return abs_array;
		}

		int distrTypeNumber = 1;
		myfile.read((char*)(&distrTypeNumber), sizeof(int));

		//seperate the process of univariate and multivariate. currently not support mixing of univariate and multivariate
		if (distrTypeNumber == 1 || distrTypeNumber == 2) {
			dist::Variant* distArray;
			distArray = new dist::Variant[n];

			float* distData = new float[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

			for (int index = 0; index < n; index++) {
				if (index > 0) {
					myfile.read((char*)(&distrTypeNumber), sizeof(int));
				}
				if (distrTypeNumber == 1) { //gaussian
					int nGM;
					myfile.read((char*)(&nGM), sizeof(int));

					myfile.read((char*)(distData), sizeof(float)*nGM * 3);

					dist::GMM new_gmm = dist::GMM(nGM);
					for (int m = 0; m<nGM; m++)
					{
						new_gmm.models[m].m = distData[3 * m];
						new_gmm.models[m].v = distData[3 * m + 1];
						new_gmm.models[m].w = distData[3 * m + 2];
					}
					distArray[index] = new_gmm;
				}
				else if (distrTypeNumber == 2) { //histogram
					int nbins;
					myfile.read((char*)(&nbins), sizeof(int));

					myfile.read((char*)(distData), sizeof(float));
					myfile.read((char*)(distData + 1), sizeof(float));
					myfile.read((char*)(distData + 2), sizeof(float)*nbins);

					distArray[index] = Histogram(distData, nbins);
				}
				else if (distrTypeNumber == 3 || distrTypeNumber == 4) { //joint distr
					//cannot mix univariate and multivariate
					myfile.close();
					throw NotSupportException();
				}
				else {
					myfile.close();
					throw NotImplementedException();
				}
			}

			delete[] distData;

			//shared_ary<dist::Variant> pArray(new dist::Variant[n], n);
			//for (int i = 0; i < n; i++)
			//{
			//	pArray[i] = distArray[i];
			//}
			shared_ary<dist::Variant> pArray(distArray, n);

			DistrArray * abs_array = new ScalarDistrArray<dist::Variant>(pArray);
			return abs_array;
		}
		else if (distrTypeNumber == 3) {
			dist::JointGMM* distArray;
			distArray = new dist::JointGMM[n];

			float* distData = new float[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

			for (int index = 0; index < n; index++) {
				if (index > 0) {
					myfile.read((char*)(&distrTypeNumber), sizeof(int));
				}
				if (distrTypeNumber == 1 || distrTypeNumber == 2 || distrTypeNumber == 4) { //other typesof distr
					//cannot mix other distr with a type of multivariate
					//in the future may support mix of JointGMM and JointHistogram
					myfile.close();
					throw NotSupportException();
				}
				else if (distrTypeNumber == 3) { //joint gmm
					int nVar, nComp;
					myfile.read((char*)(&nVar), sizeof(int));
					myfile.read((char*)(&nComp), sizeof(int));

					int sizeEachComp = 1 + nVar + (nVar + 1)*nVar / 2;
					myfile.read((char*)(distData), sizeof(float)* sizeEachComp * nComp);

					ublas_vector w = ublas_vector(nComp, 0);
					ublas_matrix m(nComp, nVar);
					ublas_matrix covs(nComp* nVar, nVar);
					for (int c = 0; c < nComp; c++)
					{
						w[c] = distData[sizeEachComp * c];
						ublas_vector mean = ublas_vector(nVar, 0);
						ublas_matrix cov = ublas::zero_matrix<Real>(nVar, nVar);;
						for (int jj = 0; jj < nVar; jj++) {
							mean(jj) = distData[sizeEachComp * c + 1 + jj];
						}
						int count = 0;
						for (int jj = 0; jj < nVar; jj++) {
							for (int i = jj; i < nVar; i++) {
								cov(jj, i) = distData[c*sizeEachComp + 1 + nVar + count];//cov row major
								count++;
							}
						}
						for (int jj = 0; jj < nVar; jj++) {
							for (int i = 0; i < jj; i++) {
								cov(jj, i) = cov(i, jj);
							}
						}
						//copy single GM to full GMM
						for (int jj = 0; jj < nVar; jj++) {
							m(c, jj) = mean(jj);
						}
						subrange(covs, c*nVar, (c + 1)*nVar, 0, nVar) = cov;
					}

					dist::JointGMM gmm;
					gmm.setGMM(nVar, nComp, w, m, covs);
					distArray[index] = gmm;
				}
				else {
					myfile.close();
					throw NotImplementedException();
				}
			}

			delete[] distData;

			////method1
			////have error in the future when using the DistrArray * stored in eddaDataset. indexing at 0 will be ok, but larger indexing will break the program
			//dist::Variant* test  = new dist::Variant[n];
			//for (int i = 0; i < n; i++)
			//{
			//	test[i] = distArray[i];
			//}
			//shared_ary<dist::Variant> pArray(test, n);
			//DistrArray * abs_array = new ScalarDistrArray<dist::Variant>(pArray);

			//method2
			//have error at the end of the program when recycling memory
			shared_ary<dist::JointGMM> pArray(distArray, n);
			DistrArray * abs_array = new JointDistrArray<dist::JointGMM>(pArray, 2);

			return abs_array;
		}

	}
#endif

	template <typename T>
	shared_ptr<Dataset<T> > loadEddaDatasetTemplate(const string &edda_file)
	{
		ifstream myfile(edda_file.c_str(), ios::binary);

		streampos begin = myfile.tellg();
		myfile.seekg(0, ios::end);
		streampos end = myfile.tellg();
		int fileByteSize = end - begin;

		myfile.seekg(0, ios::beg);

		char eddaFileMark[4];
		myfile.read(eddaFileMark, sizeof(char)*4);
		if (eddaFileMark[0] != 'E' || eddaFileMark[1] != 'D' || eddaFileMark[2] != 'D' || eddaFileMark[3] != 'A'){
			myfile.close();
			//TODO: throw a proper exception
			exit(0);
		}

		char majorVersion, minorVersion;
		myfile.read(&majorVersion, sizeof(char));
		myfile.read(&minorVersion, sizeof(char));

		if (majorVersion == 0 && minorVersion == 1){
			/*
			//not supported any more
			*/
		}
		else if (majorVersion == 0 && minorVersion == 2){
			int gridTypeNumber;
			myfile.read((char*)(&gridTypeNumber), sizeof(int));
			if (gridTypeNumber == 1){
				int dims[3];
				myfile.read((char*)(&dims), sizeof (int)* 3);
				float spacing[3];
				myfile.read((char*)(&spacing), sizeof (float)* 3);

				int numDistrArray = 1;
				myfile.read((char*)(&numDistrArray), sizeof (int));

				std::vector<DistrArray *> dVec(numDistrArray);
				for (int i = 0; i < numDistrArray; i++){
					dVec[i] = readMixArray(myfile, dims[0] * dims[1] * dims[2]);
				}

				myfile.close();

				return make_shared<Dataset<T>>(new RegularCartesianGrid(dims[0], dims[1], dims[2]), dVec);
			}
			else{
				myfile.close();
				throw NotImplementedException();
			}
		}
		else{
			myfile.close();
			throw NotImplementedException();
		}

		printf("Read file from %s.\n", edda_file.c_str());
	}

	shared_ptr<Dataset<Real> > loadEddaScalarDataset_noneVTK(const string &edda_file)
	{
		return loadEddaScalarDataset(edda_file);
	}

	shared_ptr<Dataset<Real> > loadEddaScalarDataset(const string &edda_file)
	{
		return loadEddaDatasetTemplate<Real>(edda_file);
	}
}<commit_msg>modify a bug on linux of edda_reader<commit_after>#include <memory>
#include <vector>
#include <stdexcept>
#include <string>
#include <fstream>

#include "edda_reader.h"
#include "dataset/distr_array.h"
#include "dataset/dataset.h"
#include "core/interpolator.h"
#include "../test/bmp_image.h"

#include "edda.h"
#include "distributions/gaussian.h"
#include "distributions/distribution.h"
#include "distributions/variant.h"

#include "distributions/gmm.h"
#include "distributions/gaussian_mixture.h"
#include "distributions/joint_gaussian.h"
#include "distributions/joint_GMM.h"
#include "distributions/histogram.h"
#include "distributions/joint_histogram.h"

using namespace std;
using namespace edda;
using namespace dist;

namespace edda {

	DistrArray * readMixArray(ifstream & myfile, int n)
	{
		dist::Variant* distArray;
		distArray = new dist::Variant[n];

		float* distData = new float[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

		for (int index = 0; index < n; index++){
			int distrTypeNumber = 1;
			myfile.read((char*)(&distrTypeNumber), sizeof (int));
			if (distrTypeNumber == 1){ //gaussian
				int nGM;
				myfile.read((char*)(&nGM), sizeof (int));

				myfile.read((char*)(distData), sizeof (float)*nGM * 3);

				dist::GMM new_gmm = dist::GMM(nGM);
				for (int m = 0; m<nGM; m++)
				{
					new_gmm.models[m].m = distData[3 * m];
					new_gmm.models[m].v = distData[3 * m + 1];
					new_gmm.models[m].w = distData[3 * m + 2];
				}
				distArray[index] = new_gmm;
			}
			else if (distrTypeNumber == 2){ //histogram
				int nbins;
				myfile.read((char*)(&nbins), sizeof (int));

				myfile.read((char*)(distData), sizeof (float));
				myfile.read((char*)(distData + 1), sizeof (float));
				myfile.read((char*)(distData + 2), sizeof (float)*nbins);

				distArray[index] = Histogram(distData, nbins);
			}
			else if (distrTypeNumber == 3){ //joint gmm
				int nVar, nComp;
				myfile.read((char*)(&nVar), sizeof (int));
				myfile.read((char*)(&nComp), sizeof (int));

				int sizeEachComp = 1 + nVar + (nVar + 1)*nVar / 2;
				myfile.read((char*)(distData), sizeof (float)* sizeEachComp * nComp);

				std::vector<Real> weights(nComp);
				std::vector<JointGaussian> gaus(nComp);
				for (int c = 0; c < nComp; c++)
				{
					weights[c] = distData[sizeEachComp * c];

					ublas_vector mean = ublas_vector(nVar, 0);
					ublas_matrix cov = ublas::zero_matrix<Real>(nVar, nVar);;
					for (int jj = 0; jj < nVar; jj++){
						mean(jj) = distData[sizeEachComp * c + 1 + jj];
					}
					int count = 0;
					for (int jj = 0; jj < nVar; jj++){
						for (int i = jj; i < nVar; i++){
							cov(jj, i) = distData[c*sizeEachComp + 1 + nVar + count];//cov row major
							count++;
						}
					}
					for (int jj = 0; jj < nVar; jj++){
						for (int i = 0; i < jj; i++){
							cov(jj, i) = cov(i, jj);
						}
					}
					gaus[c] = JointGaussian(mean, cov);
				}
				dist::JointGMM new_distr(weights, gaus, nVar, nComp);
				distArray[index] = new_distr;
			}
			else if (distrTypeNumber == 4) { //joint hist

				int* intData = new int[333 * 3]; // !!! this array is designed to avoid the time cost by multiple memory allocation and deletion. better check if the size is big enough before each time using it !!!

				int num_vars;
				myfile.read((char*)(&num_vars), sizeof(int));

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> min_vals(distData, distData + num_vars);

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> max_vals(distData, distData + num_vars);

				myfile.read((char*)(distData), sizeof(float)* num_vars);
				std::vector<Real> bin_widths(distData, distData + num_vars);

				myfile.read((char*)(intData), sizeof(int)* num_vars);
				std::vector<int> num_bins(intData, intData + num_vars);

				//read pdf
				boost::unordered_map<std::vector<int>, Real> pdf;
				int n_pdf;
				myfile.read((char*)(&n_pdf), sizeof(int));
				int count = 0;
				while (count < n_pdf) {
					int n_vint;
					myfile.read((char*)(&n_vint), sizeof(int));
					myfile.read((char*)(intData), sizeof(int)* n_vint);
					std::vector<int> vint(intData, intData + n_vint);
					float rfloat;
					myfile.read((char*)(&rfloat), sizeof(float));
					Real r = rfloat;
					pdf[vint] = r;
					count++;
				}

				//read mean
				ublas_vector mean(num_vars, 0);
				myfile.read((char*)(distData), sizeof(float)* num_vars);
				for (int jj = 0; jj < num_vars; jj++) {
					mean(jj) = distData[jj];
				}

				//read cov
				ublas_matrix cov = ublas::zero_matrix<Real>(num_vars, num_vars);;
				int sizeHalfCovMat = (num_vars + 1)*num_vars / 2;
				myfile.read((char*)(distData), sizeof(float)* sizeHalfCovMat);
				count = 0;
				for (int jj = 0; jj < num_vars; jj++) {
					for (int i = jj; i < num_vars; i++) {
						cov(jj, i) = distData[count];//cov row major
						count++;
					}
				}
				for (int jj = 0; jj < num_vars; jj++) {
					for (int i = 0; i < jj; i++) {
						cov(jj, i) = cov(i, jj);
					}
				}

				dist::JointHistogram new_distr(num_vars, min_vals, max_vals, bin_widths, num_bins, pdf, mean, cov);
				distArray[index] = new_distr;
			}
			else{
				myfile.close();
				throw NotImplementedException();
			}
		}

		delete[] distData;

		shared_ary<dist::Variant> pArray(new dist::Variant[n], n);
		for (int i = 0; i < n; i++)
		{
			pArray[i] = distArray[i];
		}
		DistrArray * abs_array = new ScalarDistrArray<dist::Variant>(pArray);
		return abs_array;
	}

	template <typename T>
	shared_ptr<Dataset<T> > loadEddaDatasetTemplate(const string &edda_file)
	{
		ifstream myfile(edda_file.c_str(), ios::binary);

		streampos begin = myfile.tellg();
		myfile.seekg(0, ios::end);
		streampos end = myfile.tellg();
		int fileByteSize = end - begin;

		myfile.seekg(0, ios::beg);

		char eddaFileMark[4];
		myfile.read(eddaFileMark, sizeof(char)*4);
		if (eddaFileMark[0] != 'E' || eddaFileMark[1] != 'D' || eddaFileMark[2] != 'D' || eddaFileMark[3] != 'A'){
			myfile.close();
			//TODO: throw a proper exception
			exit(0);
		}

		char majorVersion, minorVersion;
		myfile.read(&majorVersion, sizeof(char));
		myfile.read(&minorVersion, sizeof(char));

		if (majorVersion == 0 && minorVersion == 1){
			/*
			//not supported any more
			*/
		}
		else if (majorVersion == 0 && minorVersion == 2){
			int gridTypeNumber;
			myfile.read((char*)(&gridTypeNumber), sizeof(int));
			if (gridTypeNumber == 1){
				int dims[3];
				myfile.read((char*)(&dims), sizeof (int)* 3);
				float spacing[3];
				myfile.read((char*)(&spacing), sizeof (float)* 3);

				int numDistrArray = 1;
				myfile.read((char*)(&numDistrArray), sizeof (int));

				std::vector<DistrArray *> dVec(numDistrArray);
				for (int i = 0; i < numDistrArray; i++){
					dVec[i] = readMixArray(myfile, dims[0] * dims[1] * dims[2]);
				}

				myfile.close();

				return make_shared<Dataset<T>>(new RegularCartesianGrid(dims[0], dims[1], dims[2]), dVec);
			}
			else{
				myfile.close();
				throw NotImplementedException();
			}
		}
		else{
			myfile.close();
			throw NotImplementedException();
		}

		printf("Read file from %s.\n", edda_file.c_str());
	}

	shared_ptr<Dataset<Real> > loadEddaScalarDataset_noneVTK(const string &edda_file)
	{
		return loadEddaScalarDataset(edda_file);
	}

	shared_ptr<Dataset<Real> > loadEddaScalarDataset(const string &edda_file)
	{
		return loadEddaDatasetTemplate<Real>(edda_file);
	}
}<|endoftext|>