<commit_before>#pragma once
#include "common.hpp"

DEFINE_string(type, "int", "data type int|float|double");
DEFINE_string(op, "sort", "operation sort|reduce");
DEFINE_uint64(num, 1000, "length of vector");

namespace aya
{
namespace bench
{

void ping();
std::string name();

std::string reportFileName()
{
    std::ostringstream os;
    os << name() << "_" << FLAGS_op << "_" << FLAGS_num << "_" << FLAGS_type << ".csv";
    return os.str();
}

template <typename T>
struct BenchmarkBackend
{
    virtual void generate(int) = 0;
    virtual void copy() = 0;
    virtual void sort() = 0;
    virtual void reduce() = 0;
    virtual ~BenchmarkBackend() {}
};

template <typename T>
BenchmarkBackend<T> *init();

int launch(int argc, char **argv)
{
    std::string usage = "Benchmark " + name() + R"HA(
bench --op sort --num 1000 --type int
    --op   string sort|reduce
    --num  uint64 larger than 1000000000 fails on GPU #5
    --type string int|float|double
)HA";

    google::SetUsageMessage(usage);
    gflags::SetVersionString("0.0.1");

    // only show usage when no argument is provided
    if (argc < 2)
    {
        std::cout << google::ProgramUsage();
        return 0;
    }

    gflags::ParseCommandLineFlags(&argc, &argv, true);

    std::cout << "Let's run some benchmark \\w/" << std::endl;
    // ping();
    // sort<int>(FLAGS_num);
    std::cout << reportFileName() << std::endl;
    auto back_end = init<int>();
    back_end->generate(FLAGS_num);
    back_end->copy();
    if (FLAGS_op == "sort") {
        back_end->sort();
    } else if (FLAGS_op == "reduce") {
        back_end->reduce();
    } else {
        std::cout << "unknown operation" << FLAGS_op << std::endl;
    }

    delete back_end;

    google::ShutDownCommandLineFlags();
    return 0;
}

} // bench
} // aya::bench
<commit_msg>[bench] Add flag validator<commit_after>#pragma once
#include "common.hpp"

DEFINE_string(op, "sort", "operation sort|reduce");
DEFINE_uint64(num, 1000, "length of vector");
DEFINE_string(type, "int", "data type int|float|double");

static bool ValidateOp(const char *flagname, const std::string &value)
{
    if (value == "sort")
    {
        return true;
    }
    else if (value == "reduce")
    {
        return true;
    }
    else
    {
        std::cout << "unknown operation " << value << std::endl;
        return false;
    }
}

static bool ValidateType(const char *flagname, const std::string &value)
{
    if (value == "int")
    {
        return true;
    }
    else if (value == "float")
    {
        return true;
    }
    else if (value == "double")
    {
        return true;
    }
    else
    {
        std::cout << "unknown value type " << value << std::endl;
        return false;
    }
}

DEFINE_validator(op, &ValidateOp);
DEFINE_validator(type, &ValidateType);

namespace aya
{
namespace bench
{

void ping();
std::string name();

std::string reportFileName()
{
    std::ostringstream os;
    os << name() << "_" << FLAGS_op << "_" << FLAGS_num << "_" << FLAGS_type << ".csv";
    return os.str();
}

template <typename T>
struct BenchmarkBackend
{
    virtual void generate(int) = 0;
    virtual void copy() = 0;
    virtual void sort() = 0;
    virtual void reduce() = 0;
    virtual ~BenchmarkBackend() {}
};

template <typename T>
BenchmarkBackend<T> *init();

template <typename T>
void run_bench()
{
    auto back_end = init<T>();
    back_end->generate(FLAGS_num);
    back_end->copy();
    if (FLAGS_op == "sort")
    {
        back_end->sort();
    }
    else if (FLAGS_op == "reduce")
    {
        back_end->reduce();
    }
    else
    {
        std::cout << "unknown operation " << FLAGS_op << std::endl;
        // TODO: we should return non-zero, but we also need to copy the clean up code
    }
    delete back_end;
}

int launch(int argc, char **argv)
{
    std::string usage = "Benchmark " + name() + R"HA(
bench --op sort --num 1000 --type int
    --op   string sort|reduce
    --num  uint64 larger than 1000000000 fails on GPU #5
    --type string int|float|double
)HA";

    google::SetUsageMessage(usage);
    gflags::SetVersionString("0.0.1");

    // only show usage when no argument is provided
    if (argc < 2)
    {
        std::cout << google::ProgramUsage();
        return 0;
    }

    gflags::ParseCommandLineFlags(&argc, &argv, true);

    std::cout << "Let's run some benchmark \\w/" << std::endl;
    std::cout << reportFileName() << std::endl;

    // BenchmarkBackend *back_end;

    if (FLAGS_type == "int")
    {
        // back_end = init<int>();
        run_bench<int>();
    }
    else if (FLAGS_type == "float")
    {
        // back_end = init<float>();
        run_bench<float>();
    }
    else if (FLAGS_type == "double")
    {
        // back_end = init<double>();
        run_bench<double>();
    }
    else
    {
        // TODO: this should not be triggered if we defined option validator
        std::cout << "unknown type " << FLAGS_type << std::endl;
        return 1;
    }

    google::ShutDownCommandLineFlags();
    return 0;
}

} // bench
} // aya::bench
<|endoftext|>


<commit_before>#include <iterator>

template<class RandomIt, class Val>
RandomIt binsearch(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst) return lst;

    auto lft = fst, rgt = std::prev(lst);

    while (lft <= rgt) {
        const auto mid = std::next(lft, std::distance(lft, rgt) / 2);

        if (val == *mid)
            return mid;

        if (val < *mid)
            rgt = std::prev(mid);
        else
            lft = std::next(mid);
    }

    return lst; // not found
}

template<class RandomIt, class Val>
RandomIt xlower_bound(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst) return lst;

    auto lft = fst, rgt = std::prev(lst);

    while (lft <= rgt) {
        const auto mid = std::next(lft, std::distance(lft, rgt) / 2);

        if (*mid < val) {
            lft = std::next(mid);
        } else {
            rgt = std::prev(mid);
        }
    }

    return lft; // lower bound (if it exists) or past-the-end
}

template<class RandomIt, class Val>
RandomIt xupper_bound(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst) return lst;

    while (fst < lst) {
        const auto mid = std::next(fst, std::distance(fst, lst) / 2);

        if (*mid <= val)
            fst = mid + 1;
        else
            lst = mid;
    }

    return fst;
}
<commit_msg>Drop extra variables from xlower_bound<commit_after>#include <iterator>

template<class RandomIt, class Val>
RandomIt binsearch(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst) return lst;

    auto lft = fst, rgt = std::prev(lst);

    while (lft <= rgt) {
        const auto mid = std::next(lft, std::distance(lft, rgt) / 2);

        if (val == *mid)
            return mid;

        if (val < *mid)
            rgt = std::prev(mid);
        else
            lft = std::next(mid);
    }

    return lst; // not found
}

template<class RandomIt, class Val>
RandomIt xlower_bound(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst)
        return lst;

    while (fst < lst) {
        const auto mid = std::next(fst, std::distance(fst, lst) / 2);

        if (val <= *mid)
            lst = mid;
        else
            fst = mid + 1;
    }

    return fst; // lower bound (if it exists) or past-the-end
}

template<class RandomIt, class Val>
RandomIt xupper_bound(RandomIt fst, RandomIt lst, const Val& val) {
    if (fst == lst) return lst;

    while (fst < lst) {
        const auto mid = std::next(fst, std::distance(fst, lst) / 2);

        if (*mid <= val)
            fst = mid + 1;
        else
            lst = mid;
    }

    return fst;
}
<|endoftext|>


<commit_before>#include <cstddef>
#include <cstdint>
#include <iomanip>
#include <ostream>
#include <univalue.h>
#include <insertionfinder/termcolor.hpp>
#include "utils.hpp"
using std::int64_t;
using std::size_t;
using Algorithm = InsertionFinder::Algorithm;
using Insertion = InsertionFinder::Insertion;
using Solution = InsertionFinder::Solution;
namespace Details = InsertionFinder::Details;

void Details::print_duration(std::ostream& out, int64_t duration) {
    out << termcolor::bold << "Time usage: " << termcolor::reset
        << std::fixed << std::setprecision(3) << termcolor::yellow;
    if (duration < 1000) {
        out << duration << termcolor::reset << " nanoseconds." << std::endl;
    } else if (duration < 1'000'000) {
        out << duration / 1e3 << termcolor::reset << " microseconds." << std::endl;
    } else if (duration < 1'000'000'000) {
        out << duration / 1e6 << termcolor::reset << " milliseconds." << std::endl;
    } else if (duration < 60 * INT64_C(1'000'000'000)) {
        out << duration / 1e9 << termcolor::reset << " seconds." << std::endl;
    } else if (duration < 60 * 60 * INT64_C(1'000'000'000)) {
        int64_t milliseconds = (duration + 500'000) / 1'000'000;
        out << milliseconds / (60 * 1000)
            << std::right << std::setfill('0')
            << ':' << std::setw(2) << milliseconds / 1000 % 60
            << '.'<< std::setw(3) << milliseconds % 1000
            << termcolor::reset << std::endl;
    } else {
        int64_t milliseconds = (duration + 500'000) / 1'000'000;
        out << milliseconds / (60 * 60 * 1000)
            << std::right << std::setfill('0')
            << ':' << std::setw(2) << milliseconds / (60 * 1000) % 60
            << ':' << std::setw(2) << milliseconds / 1000 % 60
            << ':' << std::setw(3) << milliseconds % 1000
            << termcolor::reset << '.' << std::endl;
    }
}

UniValue Details::create_json_solution(const Algorithm& skeleton, const Solution& solution) {
    UniValue insertion_list(UniValue::VARR);
    for (const Insertion& insertion: solution.insertions) {
        UniValue insertion_map(UniValue::VOBJ);
        insertion_map.pushKV("skeleton", insertion.skeleton.str());
        insertion_map.pushKV("insert_place", static_cast<int>(insertion.insert_place));
        insertion_map.pushKV("insertion", insertion.insertion->str());
        insertion_list.push_back(insertion_map);
    }
    UniValue merged_insertion_list(UniValue::VARR);
    size_t initial_order = 0;
    for (const auto& sub_solution: solution.merge_insertions(skeleton)) {
        UniValue insertion_list(UniValue::VARR);
        for (const auto& [place, insertions]: sub_solution.get_insertions()) {
            UniValue algorithms(UniValue::VARR);
            for (auto [insertion, order]: insertions) {
                UniValue algorithm_object(UniValue::VOBJ);
                algorithm_object.pushKV("algorithm", insertion->str());
                algorithm_object.pushKV("order", static_cast<int>(order));
                algorithms.push_back(algorithm_object);
            }
            UniValue place_object(UniValue::VOBJ);
            place_object.pushKV("insert_place", static_cast<int>(place));
            place_object.pushKV("algorithms", algorithms);
            insertion_list.push_back(place_object);
        }
        UniValue insertion_map(UniValue::VOBJ);
        insertion_map.pushKV("skeleton", sub_solution.skeleton.str());
        insertion_map.pushKV("insertions", insertion_list);
        merged_insertion_list.push_back(insertion_map);
        initial_order += sub_solution.insertions.size();
    }
    UniValue solution_map(UniValue::VOBJ);
    solution_map.pushKV("final_solution", solution.final_solution.str());
    solution_map.pushKV("cancellation", static_cast<int>(solution.cancellation));
    solution_map.pushKV("insertions", insertion_list);
    solution_map.pushKV("merged_insertions", merged_insertion_list);
    return solution_map;
}
<commit_msg>add cumulative insertion order<commit_after>#include <cstddef>
#include <cstdint>
#include <iomanip>
#include <ostream>
#include <univalue.h>
#include <insertionfinder/termcolor.hpp>
#include "utils.hpp"
using std::int64_t;
using std::size_t;
using Algorithm = InsertionFinder::Algorithm;
using Insertion = InsertionFinder::Insertion;
using Solution = InsertionFinder::Solution;
namespace Details = InsertionFinder::Details;

void Details::print_duration(std::ostream& out, int64_t duration) {
    out << termcolor::bold << "Time usage: " << termcolor::reset
        << std::fixed << std::setprecision(3) << termcolor::yellow;
    if (duration < 1000) {
        out << duration << termcolor::reset << " nanoseconds." << std::endl;
    } else if (duration < 1'000'000) {
        out << duration / 1e3 << termcolor::reset << " microseconds." << std::endl;
    } else if (duration < 1'000'000'000) {
        out << duration / 1e6 << termcolor::reset << " milliseconds." << std::endl;
    } else if (duration < 60 * INT64_C(1'000'000'000)) {
        out << duration / 1e9 << termcolor::reset << " seconds." << std::endl;
    } else if (duration < 60 * 60 * INT64_C(1'000'000'000)) {
        int64_t milliseconds = (duration + 500'000) / 1'000'000;
        out << milliseconds / (60 * 1000)
            << std::right << std::setfill('0')
            << ':' << std::setw(2) << milliseconds / 1000 % 60
            << '.'<< std::setw(3) << milliseconds % 1000
            << termcolor::reset << std::endl;
    } else {
        int64_t milliseconds = (duration + 500'000) / 1'000'000;
        out << milliseconds / (60 * 60 * 1000)
            << std::right << std::setfill('0')
            << ':' << std::setw(2) << milliseconds / (60 * 1000) % 60
            << ':' << std::setw(2) << milliseconds / 1000 % 60
            << ':' << std::setw(3) << milliseconds % 1000
            << termcolor::reset << '.' << std::endl;
    }
}

UniValue Details::create_json_solution(const Algorithm& skeleton, const Solution& solution) {
    UniValue insertion_list(UniValue::VARR);
    for (const Insertion& insertion: solution.insertions) {
        UniValue insertion_map(UniValue::VOBJ);
        insertion_map.pushKV("skeleton", insertion.skeleton.str());
        insertion_map.pushKV("insert_place", static_cast<int>(insertion.insert_place));
        insertion_map.pushKV("insertion", insertion.insertion->str());
        insertion_list.push_back(insertion_map);
    }
    UniValue merged_insertion_list(UniValue::VARR);
    size_t initial_order = 0;
    for (const auto& sub_solution: solution.merge_insertions(skeleton)) {
        UniValue insertion_list(UniValue::VARR);
        for (const auto& [place, insertions]: sub_solution.get_insertions()) {
            UniValue algorithms(UniValue::VARR);
            for (auto [insertion, order]: insertions) {
                UniValue algorithm_object(UniValue::VOBJ);
                algorithm_object.pushKV("algorithm", insertion->str());
                algorithm_object.pushKV("order", static_cast<int>(order));
                algorithm_object.pushKV("cumulative_order", static_cast<int>(initial_order + order));
                algorithms.push_back(algorithm_object);
            }
            UniValue place_object(UniValue::VOBJ);
            place_object.pushKV("insert_place", static_cast<int>(place));
            place_object.pushKV("algorithms", algorithms);
            insertion_list.push_back(place_object);
        }
        UniValue insertion_map(UniValue::VOBJ);
        insertion_map.pushKV("skeleton", sub_solution.skeleton.str());
        insertion_map.pushKV("insertions", insertion_list);
        merged_insertion_list.push_back(insertion_map);
        initial_order += sub_solution.insertions.size();
    }
    UniValue solution_map(UniValue::VOBJ);
    solution_map.pushKV("final_solution", solution.final_solution.str());
    solution_map.pushKV("cancellation", static_cast<int>(solution.cancellation));
    solution_map.pushKV("insertions", insertion_list);
    solution_map.pushKV("merged_insertions", merged_insertion_list);
    return solution_map;
}
<|endoftext|>


<commit_before>
/*
    pbrt source code is Copyright(c) 1998-2016
                        Matt Pharr, Greg Humphreys, and Wenzel Jakob.

    This file is part of pbrt.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:

    - Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.

    - Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

// core/film.cpp*
#include "film.h"
#include "paramset.h"
#include "imageio.h"
#include "stats.h"

namespace pbrt {

STAT_MEMORY_COUNTER("Memory/Film pixels", filmPixelMemory);

// Film Method Definitions
Film::Film(const Point2i &resolution, const Bounds2f &cropWindow,
           std::unique_ptr<Filter> filt, Float diagonal,
           const std::string &filename, Float scale, Float maxSampleLuminance)
    : fullResolution(resolution),
      diagonal(diagonal * .001),
      filter(std::move(filt)),
      filename(filename),
      scale(scale),
      maxSampleLuminance(maxSampleLuminance) {
    // Compute film image bounds
    croppedPixelBounds =
        Bounds2i(Point2i(std::ceil(fullResolution.x * cropWindow.pMin.x),
                         std::ceil(fullResolution.y * cropWindow.pMin.y)),
                 Point2i(std::ceil(fullResolution.x * cropWindow.pMax.x),
                         std::ceil(fullResolution.y * cropWindow.pMax.y)));
    LOG(INFO) << "Created film with full resolution " << resolution <<
        ". Crop window of " << cropWindow << " -> croppedPixelBounds " <<
        croppedPixelBounds;

    // Allocate film image storage
    pixels = std::unique_ptr<Pixel[]>(new Pixel[croppedPixelBounds.Area()]);
    filmPixelMemory += croppedPixelBounds.Area() * sizeof(Pixel);

    // Precompute filter weight table
    int offset = 0;
    for (int y = 0; y < filterTableWidth; ++y) {
        for (int x = 0; x < filterTableWidth; ++x, ++offset) {
            Point2f p;
            p.x = (x + 0.5f) * filter->radius.x / filterTableWidth;
            p.y = (y + 0.5f) * filter->radius.y / filterTableWidth;
            filterTable[offset] = filter->Evaluate(p);
        }
    }
}

Bounds2i Film::GetSampleBounds() const {
    Bounds2f floatBounds(Floor(Point2f(croppedPixelBounds.pMin) +
                               Vector2f(0.5f, 0.5f) - filter->radius),
                         Ceil(Point2f(croppedPixelBounds.pMax) -
                              Vector2f(0.5f, 0.5f) + filter->radius));
    return (Bounds2i)floatBounds;
}

Bounds2f Film::GetPhysicalExtent() const {
    Float aspect = (Float)fullResolution.y / (Float)fullResolution.x;
    Float x = std::sqrt(diagonal * diagonal / (1 + aspect * aspect));
    Float y = aspect * x;
    return Bounds2f(Point2f(-x / 2, -y / 2), Point2f(x / 2, y / 2));
}

std::unique_ptr<FilmTile> Film::GetFilmTile(const Bounds2i &sampleBounds) {
    // Bound image pixels that samples in _sampleBounds_ contribute to
    Vector2f halfPixel = Vector2f(0.5f, 0.5f);
    Bounds2f floatBounds = (Bounds2f)sampleBounds;
    Point2i p0 = (Point2i)Ceil(floatBounds.pMin - halfPixel - filter->radius);
    Point2i p1 = (Point2i)Floor(floatBounds.pMax - halfPixel + filter->radius) +
                 Point2i(1, 1);
    Bounds2i tilePixelBounds = Intersect(Bounds2i(p0, p1), croppedPixelBounds);
    return std::unique_ptr<FilmTile>(new FilmTile(
        tilePixelBounds, filter->radius, filterTable, filterTableWidth,
        maxSampleLuminance));
}

void Film::Clear() {
    for (Point2i p : croppedPixelBounds) {
        Pixel &pixel = GetPixel(p);
        for (int c = 0; c < 3; ++c)
            pixel.splatXYZ[c] = pixel.xyz[c] = 0;
        pixel.filterWeightSum = 0;
    }
}

void Film::MergeFilmTile(std::unique_ptr<FilmTile> tile) {
    ProfilePhase p(Prof::MergeFilmTile);
    VLOG(1) << "Merging film tile " << tile->pixelBounds;
    std::lock_guard<std::mutex> lock(mutex);
    for (Point2i pixel : tile->GetPixelBounds()) {
        // Merge _pixel_ into _Film::pixels_
        const FilmTilePixel &tilePixel = tile->GetPixel(pixel);
        Pixel &mergePixel = GetPixel(pixel);
        Float xyz[3];
        tilePixel.contribSum.ToXYZ(xyz);
        for (int i = 0; i < 3; ++i) mergePixel.xyz[i] += xyz[i];
        mergePixel.filterWeightSum += tilePixel.filterWeightSum;
    }
}

void Film::SetImage(const Spectrum *img) const {
    int nPixels = croppedPixelBounds.Area();
    for (int i = 0; i < nPixels; ++i) {
        Pixel &p = pixels[i];
        img[i].ToXYZ(p.xyz);
        p.filterWeightSum = 1;
        p.splatXYZ[0] = p.splatXYZ[1] = p.splatXYZ[2] = 0;
    }
}

void Film::AddSplat(const Point2f &p, Spectrum v) {
    ProfilePhase pp(Prof::SplatFilm);

    if (v.HasNaNs()) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with NaN values "
                                   "at (%f, %f)", p.x, p.y);
        return;
    } else if (v.y() < 0.) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with negative "
                                   "luminance %f at (%f, %f)", v.y(), p.x, p.y);
        return;
    } else if (std::isinf(v.y())) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with infinite "
                                   "luminance at (%f, %f)", p.x, p.y);
        return;
    }

    if (!InsideExclusive((Point2i)p, croppedPixelBounds)) return;
    if (v.y() > maxSampleLuminance)
        v *= maxSampleLuminance / v.y();
    Float xyz[3];
    v.ToXYZ(xyz);
    Pixel &pixel = GetPixel((Point2i)p);
    for (int i = 0; i < 3; ++i) pixel.splatXYZ[i].Add(xyz[i]);
}

void Film::WriteImage(Float splatScale) {
    // Convert image to RGB and compute final pixel values
    LOG(INFO) <<
        "Converting image to RGB and computing final weighted pixel values";
    std::unique_ptr<Float[]> rgb(new Float[3 * croppedPixelBounds.Area()]);
    int offset = 0;
    for (Point2i p : croppedPixelBounds) {
        // Convert pixel XYZ color to RGB
        Pixel &pixel = GetPixel(p);
        XYZToRGB(pixel.xyz, &rgb[3 * offset]);

        // Normalize pixel with weight sum
        Float filterWeightSum = pixel.filterWeightSum;
        if (filterWeightSum != 0) {
            Float invWt = (Float)1 / filterWeightSum;
            rgb[3 * offset] = std::max((Float)0, rgb[3 * offset] * invWt);
            rgb[3 * offset + 1] =
                std::max((Float)0, rgb[3 * offset + 1] * invWt);
            rgb[3 * offset + 2] =
                std::max((Float)0, rgb[3 * offset + 2] * invWt);
        }

        // Add splat value at pixel
        Float splatRGB[3];
        Float splatXYZ[3] = {pixel.splatXYZ[0], pixel.splatXYZ[1],
                             pixel.splatXYZ[2]};
        XYZToRGB(splatXYZ, splatRGB);
        rgb[3 * offset] += splatScale * splatRGB[0];
        rgb[3 * offset + 1] += splatScale * splatRGB[1];
        rgb[3 * offset + 2] += splatScale * splatRGB[2];

        // Scale pixel value by _scale_
        rgb[3 * offset] *= scale;
        rgb[3 * offset + 1] *= scale;
        rgb[3 * offset + 2] *= scale;
        ++offset;
    }

    // Write RGB image
    LOG(INFO) << "Writing image " << filename << " with bounds " <<
        croppedPixelBounds;
    pbrt::WriteImage(filename, &rgb[0], croppedPixelBounds, fullResolution);
}

Film *CreateFilm(const ParamSet &params, std::unique_ptr<Filter> filter) {
    // Intentionally use FindOneString() rather than FindOneFilename() here
    // so that the rendered image is left in the working directory, rather
    // than the directory the scene file lives in.
    std::string filename = params.FindOneString("filename", "");
    if (PbrtOptions.imageFile != "") {
        if (filename != "") {
            Warning(
                "Output filename supplied on command line, \"%s\", ignored "
                "due to filename provided in scene description file, \"%s\".",
                PbrtOptions.imageFile.c_str(), filename.c_str());
        } else
            filename = PbrtOptions.imageFile;
    }
    if (filename == "") filename = "pbrt.exr";

    int xres = params.FindOneInt("xresolution", 1280);
    int yres = params.FindOneInt("yresolution", 720);
    if (PbrtOptions.quickRender) xres = std::max(1, xres / 4);
    if (PbrtOptions.quickRender) yres = std::max(1, yres / 4);
    Bounds2f crop(Point2f(0, 0), Point2f(1, 1));
    int cwi;
    const Float *cr = params.FindFloat("cropwindow", &cwi);
    if (cr && cwi == 4) {
        crop.pMin.x = Clamp(std::min(cr[0], cr[1]), 0.f, 1.f);
        crop.pMax.x = Clamp(std::max(cr[0], cr[1]), 0.f, 1.f);
        crop.pMin.y = Clamp(std::min(cr[2], cr[3]), 0.f, 1.f);
        crop.pMax.y = Clamp(std::max(cr[2], cr[3]), 0.f, 1.f);
    } else if (cr)
        Error("%d values supplied for \"cropwindow\". Expected 4.", cwi);

    Float scale = params.FindOneFloat("scale", 1.);
    Float diagonal = params.FindOneFloat("diagonal", 35.);
    Float maxSampleLuminance = params.FindOneFloat("maxsampleluminance",
                                                   Infinity);
    return new Film(Point2i(xres, yres), crop, std::move(filter), diagonal,
                    filename, scale, maxSampleLuminance);
}

}  // namespace pbrt
<commit_msg>Increase priority of --outfile w.r.t. Film parameters<commit_after>
/*
    pbrt source code is Copyright(c) 1998-2016
                        Matt Pharr, Greg Humphreys, and Wenzel Jakob.

    This file is part of pbrt.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:

    - Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.

    - Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

// core/film.cpp*
#include "film.h"
#include "paramset.h"
#include "imageio.h"
#include "stats.h"

namespace pbrt {

STAT_MEMORY_COUNTER("Memory/Film pixels", filmPixelMemory);

// Film Method Definitions
Film::Film(const Point2i &resolution, const Bounds2f &cropWindow,
           std::unique_ptr<Filter> filt, Float diagonal,
           const std::string &filename, Float scale, Float maxSampleLuminance)
    : fullResolution(resolution),
      diagonal(diagonal * .001),
      filter(std::move(filt)),
      filename(filename),
      scale(scale),
      maxSampleLuminance(maxSampleLuminance) {
    // Compute film image bounds
    croppedPixelBounds =
        Bounds2i(Point2i(std::ceil(fullResolution.x * cropWindow.pMin.x),
                         std::ceil(fullResolution.y * cropWindow.pMin.y)),
                 Point2i(std::ceil(fullResolution.x * cropWindow.pMax.x),
                         std::ceil(fullResolution.y * cropWindow.pMax.y)));
    LOG(INFO) << "Created film with full resolution " << resolution <<
        ". Crop window of " << cropWindow << " -> croppedPixelBounds " <<
        croppedPixelBounds;

    // Allocate film image storage
    pixels = std::unique_ptr<Pixel[]>(new Pixel[croppedPixelBounds.Area()]);
    filmPixelMemory += croppedPixelBounds.Area() * sizeof(Pixel);

    // Precompute filter weight table
    int offset = 0;
    for (int y = 0; y < filterTableWidth; ++y) {
        for (int x = 0; x < filterTableWidth; ++x, ++offset) {
            Point2f p;
            p.x = (x + 0.5f) * filter->radius.x / filterTableWidth;
            p.y = (y + 0.5f) * filter->radius.y / filterTableWidth;
            filterTable[offset] = filter->Evaluate(p);
        }
    }
}

Bounds2i Film::GetSampleBounds() const {
    Bounds2f floatBounds(Floor(Point2f(croppedPixelBounds.pMin) +
                               Vector2f(0.5f, 0.5f) - filter->radius),
                         Ceil(Point2f(croppedPixelBounds.pMax) -
                              Vector2f(0.5f, 0.5f) + filter->radius));
    return (Bounds2i)floatBounds;
}

Bounds2f Film::GetPhysicalExtent() const {
    Float aspect = (Float)fullResolution.y / (Float)fullResolution.x;
    Float x = std::sqrt(diagonal * diagonal / (1 + aspect * aspect));
    Float y = aspect * x;
    return Bounds2f(Point2f(-x / 2, -y / 2), Point2f(x / 2, y / 2));
}

std::unique_ptr<FilmTile> Film::GetFilmTile(const Bounds2i &sampleBounds) {
    // Bound image pixels that samples in _sampleBounds_ contribute to
    Vector2f halfPixel = Vector2f(0.5f, 0.5f);
    Bounds2f floatBounds = (Bounds2f)sampleBounds;
    Point2i p0 = (Point2i)Ceil(floatBounds.pMin - halfPixel - filter->radius);
    Point2i p1 = (Point2i)Floor(floatBounds.pMax - halfPixel + filter->radius) +
                 Point2i(1, 1);
    Bounds2i tilePixelBounds = Intersect(Bounds2i(p0, p1), croppedPixelBounds);
    return std::unique_ptr<FilmTile>(new FilmTile(
        tilePixelBounds, filter->radius, filterTable, filterTableWidth,
        maxSampleLuminance));
}

void Film::Clear() {
    for (Point2i p : croppedPixelBounds) {
        Pixel &pixel = GetPixel(p);
        for (int c = 0; c < 3; ++c)
            pixel.splatXYZ[c] = pixel.xyz[c] = 0;
        pixel.filterWeightSum = 0;
    }
}

void Film::MergeFilmTile(std::unique_ptr<FilmTile> tile) {
    ProfilePhase p(Prof::MergeFilmTile);
    VLOG(1) << "Merging film tile " << tile->pixelBounds;
    std::lock_guard<std::mutex> lock(mutex);
    for (Point2i pixel : tile->GetPixelBounds()) {
        // Merge _pixel_ into _Film::pixels_
        const FilmTilePixel &tilePixel = tile->GetPixel(pixel);
        Pixel &mergePixel = GetPixel(pixel);
        Float xyz[3];
        tilePixel.contribSum.ToXYZ(xyz);
        for (int i = 0; i < 3; ++i) mergePixel.xyz[i] += xyz[i];
        mergePixel.filterWeightSum += tilePixel.filterWeightSum;
    }
}

void Film::SetImage(const Spectrum *img) const {
    int nPixels = croppedPixelBounds.Area();
    for (int i = 0; i < nPixels; ++i) {
        Pixel &p = pixels[i];
        img[i].ToXYZ(p.xyz);
        p.filterWeightSum = 1;
        p.splatXYZ[0] = p.splatXYZ[1] = p.splatXYZ[2] = 0;
    }
}

void Film::AddSplat(const Point2f &p, Spectrum v) {
    ProfilePhase pp(Prof::SplatFilm);

    if (v.HasNaNs()) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with NaN values "
                                   "at (%f, %f)", p.x, p.y);
        return;
    } else if (v.y() < 0.) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with negative "
                                   "luminance %f at (%f, %f)", v.y(), p.x, p.y);
        return;
    } else if (std::isinf(v.y())) {
        LOG(ERROR) << StringPrintf("Ignoring splatted spectrum with infinite "
                                   "luminance at (%f, %f)", p.x, p.y);
        return;
    }

    if (!InsideExclusive((Point2i)p, croppedPixelBounds)) return;
    if (v.y() > maxSampleLuminance)
        v *= maxSampleLuminance / v.y();
    Float xyz[3];
    v.ToXYZ(xyz);
    Pixel &pixel = GetPixel((Point2i)p);
    for (int i = 0; i < 3; ++i) pixel.splatXYZ[i].Add(xyz[i]);
}

void Film::WriteImage(Float splatScale) {
    // Convert image to RGB and compute final pixel values
    LOG(INFO) <<
        "Converting image to RGB and computing final weighted pixel values";
    std::unique_ptr<Float[]> rgb(new Float[3 * croppedPixelBounds.Area()]);
    int offset = 0;
    for (Point2i p : croppedPixelBounds) {
        // Convert pixel XYZ color to RGB
        Pixel &pixel = GetPixel(p);
        XYZToRGB(pixel.xyz, &rgb[3 * offset]);

        // Normalize pixel with weight sum
        Float filterWeightSum = pixel.filterWeightSum;
        if (filterWeightSum != 0) {
            Float invWt = (Float)1 / filterWeightSum;
            rgb[3 * offset] = std::max((Float)0, rgb[3 * offset] * invWt);
            rgb[3 * offset + 1] =
                std::max((Float)0, rgb[3 * offset + 1] * invWt);
            rgb[3 * offset + 2] =
                std::max((Float)0, rgb[3 * offset + 2] * invWt);
        }

        // Add splat value at pixel
        Float splatRGB[3];
        Float splatXYZ[3] = {pixel.splatXYZ[0], pixel.splatXYZ[1],
                             pixel.splatXYZ[2]};
        XYZToRGB(splatXYZ, splatRGB);
        rgb[3 * offset] += splatScale * splatRGB[0];
        rgb[3 * offset + 1] += splatScale * splatRGB[1];
        rgb[3 * offset + 2] += splatScale * splatRGB[2];

        // Scale pixel value by _scale_
        rgb[3 * offset] *= scale;
        rgb[3 * offset + 1] *= scale;
        rgb[3 * offset + 2] *= scale;
        ++offset;
    }

    // Write RGB image
    LOG(INFO) << "Writing image " << filename << " with bounds " <<
        croppedPixelBounds;
    pbrt::WriteImage(filename, &rgb[0], croppedPixelBounds, fullResolution);
}

Film *CreateFilm(const ParamSet &params, std::unique_ptr<Filter> filter) {
    std::string filename;
    if (PbrtOptions.imageFile != "") {
        filename = PbrtOptions.imageFile;
        std::string paramsFilename = params.FindOneString("filename", "");
        if (paramsFilename != "")
            Warning(
                "Output filename supplied on command line, \"%s\" is overriding "
                "filename provided in scene description file, \"%s\".",
                PbrtOptions.imageFile.c_str(), paramsFilename.c_str());
    } else
        filename = params.FindOneString("filename", "pbrt.exr");

    int xres = params.FindOneInt("xresolution", 1280);
    int yres = params.FindOneInt("yresolution", 720);
    if (PbrtOptions.quickRender) xres = std::max(1, xres / 4);
    if (PbrtOptions.quickRender) yres = std::max(1, yres / 4);
    Bounds2f crop(Point2f(0, 0), Point2f(1, 1));
    int cwi;
    const Float *cr = params.FindFloat("cropwindow", &cwi);
    if (cr && cwi == 4) {
        crop.pMin.x = Clamp(std::min(cr[0], cr[1]), 0.f, 1.f);
        crop.pMax.x = Clamp(std::max(cr[0], cr[1]), 0.f, 1.f);
        crop.pMin.y = Clamp(std::min(cr[2], cr[3]), 0.f, 1.f);
        crop.pMax.y = Clamp(std::max(cr[2], cr[3]), 0.f, 1.f);
    } else if (cr)
        Error("%d values supplied for \"cropwindow\". Expected 4.", cwi);

    Float scale = params.FindOneFloat("scale", 1.);
    Float diagonal = params.FindOneFloat("diagonal", 35.);
    Float maxSampleLuminance = params.FindOneFloat("maxsampleluminance",
                                                   Infinity);
    return new Film(Point2i(xres, yres), crop, std::move(filter), diagonal,
                    filename, scale, maxSampleLuminance);
}

}  // namespace pbrt
<|endoftext|>


<commit_before>#include "cvm.h"
#include "../cshared/disasm.h"

void DabVM_debug::print_registers()
{
    fprintf(stderr, "IP = %p (%d)\n", (void *)vm.ip(), (int)vm.ip());
}

void DabVM_debug::print_ssa_registers()
{
    auto err_stream = $VM->options.output;
    fprintf(err_stream, "Registers:\n");
    size_t index = 0;
    for (const auto &reg : vm._registers)
    {
        fprintf(err_stream, "R%zu: ", index);
        reg.dump(err_stream);
        fprintf(err_stream, "\n");
    }
}

void DabVM_debug::print_classes()
{
    for (const auto &it : vm.classes)
    {
        fprintf(stderr, " - 0x%04x %s (super = 0x%04x)\n", it.first, it.second.name.c_str(),
                it.second.superclass_index);
        for (const auto &fin : it.second.static_functions)
        {
            fprintf(stderr, "   ::%s [%s]\n", $VM->get_symbol(fin.first).c_str(),
                    fin.second.regular ? "Dab" : "C");
        }
        for (const auto &fin : it.second.functions)
        {
            fprintf(stderr, "    .%s [%s]\n", $VM->get_symbol(fin.first).c_str(),
                    fin.second.regular ? "Dab" : "C");
        }
    }
}

void DabVM_debug::print_functions()
{
    for (auto it : vm.functions)
    {
        auto &fun = it.second;
        fprintf(stderr, " - %s: %s at %p\n", fun.name.c_str(), fun.regular ? "Dab" : "C",
                (void *)fun.address);
    }
}

void DabVM_debug::print_constants()
{
    auto output = $VM->options.output;
    fprintf(output, "symbols:\n");
    size_t i = 0;
    for (const auto &symbol : $VM->symbols)
    {
        fprintf(output, "%d: %s\n", (int)i, symbol.c_str());
        i++;
    }
}

void DabVM_debug::print_stack()
{
}

void DabVM_debug::print_code(bool current_only)
{
    prepare_disasm();

    auto err_stream = $VM->options.output;

    auto ip = vm.ip();

    fprintf(err_stream, "IP = %d\n", (int)ip);

    auto it = find_if(disasm.begin(), disasm.end(),
                      [ip](const std::pair<size_t, std::string> &obj) { return obj.first == ip; });

    if (it == disasm.end())
        return;

    auto index = std::distance(disasm.begin(), it);

    long start = current_only ? (index - 2) : 0;
    long end   = current_only ? (index + 3) : disasm.size();

    for (long i = start; i < end; i++)
    {
        if (i < 0 || i >= (int)disasm.size())
            continue;

        const auto &line = disasm[i];
        fprintf(err_stream, "%c %8" PRIu64 ": %s\n", line.first == ip ? '>' : ' ', line.first,
                line.second.c_str());
    }
}

struct InstructionsReader : public BaseReader
{
    DabVM &vm;

    const byte *_data;
    uint64_t    _length;

    uint64_t start_position = 0;

    InstructionsReader(DabVM &vm, uint64_t &position) : BaseReader(position), vm(vm)
    {
        _data   = vm.instructions.raw_base_data();
        _length = vm.instructions.raw_base_length();

        BinSection code_section;
        bool       has_code = false;

        for (auto &section : vm.sections)
        {
            if (std::string(section.name) == "code")
            {
                code_section = section;
                has_code     = true;
                break;
            }
        }

        if (!has_code)
            throw "no code?";

        start_position = code_section.pos;

        _data += start_position;
        _length = code_section.length;
    }

    virtual uint64_t raw_read(void *buffer, uint64_t size) override
    {
        auto data_ptr   = _data;
        auto length     = _length;
        auto offset     = position();
        auto max_length = std::min(size, length - offset);

        memcpy(buffer, data_ptr + offset, (size_t)max_length);

        return max_length;
    }

    bool feof()
    {
        return false;
    }
};

void DabVM_debug::prepare_disasm()
{
    if (has_disasm)
        return;

    has_disasm = true;

    uint64_t                            position = 0;
    InstructionsReader                  reader(vm, position);
    DisasmProcessor<InstructionsReader> processor(reader);

    processor.go([this, reader](uint64_t pos, std::string info) {
        disasm.push_back(std::make_pair(pos + reader.start_position, info));
    });
}

void DabVM::execute_debug(Stream &input)
{
    auto err_stream = options.output;

    DabVM_debug debug(*this);
    while (!input.eof())
    {
        char rawcmd[2048];
        fprintf(stderr, "> ");
        fgets(rawcmd, sizeof(rawcmd), stdin);
        std::string cmd = rawcmd;

        cmd = cmd.substr(0, cmd.length() - 1);

        if (cmd == "help")
        {
            fprintf(err_stream, "Help:\n");
            fprintf(err_stream, "help - print this\n");
            fprintf(err_stream, "[s]tep - run single instruction\n");
            fprintf(err_stream, "[r]egisters - show registers\n");
            fprintf(err_stream, "classes - print classes\n");
            fprintf(err_stream, "functions - print functions\n");
            fprintf(err_stream, "constants - dump constants\n");
            fprintf(err_stream, "stack - dump stack\n");
            fprintf(err_stream, "code - show current code\n");
            fprintf(err_stream, "allcode - show all code\n");
            fprintf(err_stream, "run - run remaining instructions\n");
            fprintf(err_stream, "break [ip] - break at defined IP\n");
            fprintf(err_stream, "quit - quit\n");
        }
        else if (cmd == "step" || cmd == "s")
        {
            execute_single(input);
        }
        else if (cmd == "registers" || cmd == "r")
        {
            debug.print_registers();
        }
        else if (cmd == "classes")
        {
            debug.print_classes();
        }
        else if (cmd == "functions")
        {
            debug.print_functions();
        }
        else if (cmd == "constants")
        {
            debug.print_constants();
        }
        else if (cmd == "stack")
        {
            debug.print_stack();
        }
        else if (cmd == "ip")
        {
            fprintf(err_stream, "IP = %" PRIu64 "\n", ip());
        }
        else if (cmd == "code")
        {
            debug.print_code(true);
        }
        else if (cmd == "allcode")
        {
            debug.print_code(false);
        }
        else if (cmd == "quit")
        {
            exit(0);
        }
        else if (cmd == "run")
        {
            execute(input);
        }
        else if (cmd.substr(0, 6) == "break ")
        {
            int ip  = 0;
            int ret = sscanf(cmd.c_str(), "break %d", &ip);
            assert(ret == 1);
            fprintf(err_stream, "debug: break at %d.\n", ip);
            breakpoints.insert(ip);
        }
        else if (cmd == "ssaregs")
        {
            debug.print_ssa_registers();
        }
        else
        {
            fprintf(stderr, "Unknown command, type <help> to get available commands.\n");
        }
    }
}
<commit_msg>vm: fix some size_t/uint64_t issues (17)<commit_after>#include "cvm.h"
#include "../cshared/disasm.h"

void DabVM_debug::print_registers()
{
    fprintf(stderr, "IP = %p (%d)\n", (void *)vm.ip(), (int)vm.ip());
}

void DabVM_debug::print_ssa_registers()
{
    auto err_stream = $VM->options.output;
    fprintf(err_stream, "Registers:\n");
    size_t index = 0;
    for (const auto &reg : vm._registers)
    {
        fprintf(err_stream, "R%zu: ", index);
        reg.dump(err_stream);
        fprintf(err_stream, "\n");
    }
}

void DabVM_debug::print_classes()
{
    for (const auto &it : vm.classes)
    {
        fprintf(stderr, " - 0x%04x %s (super = 0x%04x)\n", it.first, it.second.name.c_str(),
                it.second.superclass_index);
        for (const auto &fin : it.second.static_functions)
        {
            fprintf(stderr, "   ::%s [%s]\n", $VM->get_symbol(fin.first).c_str(),
                    fin.second.regular ? "Dab" : "C");
        }
        for (const auto &fin : it.second.functions)
        {
            fprintf(stderr, "    .%s [%s]\n", $VM->get_symbol(fin.first).c_str(),
                    fin.second.regular ? "Dab" : "C");
        }
    }
}

void DabVM_debug::print_functions()
{
    for (auto it : vm.functions)
    {
        auto &fun = it.second;
        fprintf(stderr, " - %s: %s at %p\n", fun.name.c_str(), fun.regular ? "Dab" : "C",
                (void *)fun.address);
    }
}

void DabVM_debug::print_constants()
{
    auto output = $VM->options.output;
    fprintf(output, "symbols:\n");
    size_t i = 0;
    for (const auto &symbol : $VM->symbols)
    {
        fprintf(output, "%d: %s\n", (int)i, symbol.c_str());
        i++;
    }
}

void DabVM_debug::print_stack()
{
}

void DabVM_debug::print_code(bool current_only)
{
    prepare_disasm();

    auto err_stream = $VM->options.output;

    auto ip = vm.ip();

    fprintf(err_stream, "IP = %d\n", (int)ip);

    auto it =
        find_if(disasm.begin(), disasm.end(),
                [ip](const std::pair<uint64_t, std::string> &obj) { return obj.first == ip; });

    if (it == disasm.end())
        return;

    auto index = std::distance(disasm.begin(), it);

    long start = current_only ? (index - 2) : 0;
    long end   = current_only ? (index + 3) : disasm.size();

    for (long i = start; i < end; i++)
    {
        if (i < 0 || i >= (int)disasm.size())
            continue;

        const auto &line = disasm[i];
        fprintf(err_stream, "%c %8" PRIu64 ": %s\n", line.first == ip ? '>' : ' ', line.first,
                line.second.c_str());
    }
}

struct InstructionsReader : public BaseReader
{
    DabVM &vm;

    const byte *_data;
    uint64_t    _length;

    uint64_t start_position = 0;

    InstructionsReader(DabVM &vm, uint64_t &position) : BaseReader(position), vm(vm)
    {
        _data   = vm.instructions.raw_base_data();
        _length = vm.instructions.raw_base_length();

        BinSection code_section;
        bool       has_code = false;

        for (auto &section : vm.sections)
        {
            if (std::string(section.name) == "code")
            {
                code_section = section;
                has_code     = true;
                break;
            }
        }

        if (!has_code)
            throw "no code?";

        start_position = code_section.pos;

        _data += start_position;
        _length = code_section.length;
    }

    virtual uint64_t raw_read(void *buffer, uint64_t size) override
    {
        auto data_ptr   = _data;
        auto length     = _length;
        auto offset     = position();
        auto max_length = std::min(size, length - offset);

        memcpy(buffer, data_ptr + offset, (size_t)max_length);

        return max_length;
    }

    bool feof()
    {
        return false;
    }
};

void DabVM_debug::prepare_disasm()
{
    if (has_disasm)
        return;

    has_disasm = true;

    uint64_t                            position = 0;
    InstructionsReader                  reader(vm, position);
    DisasmProcessor<InstructionsReader> processor(reader);

    processor.go([this, reader](uint64_t pos, std::string info) {
        disasm.push_back(std::make_pair(pos + reader.start_position, info));
    });
}

void DabVM::execute_debug(Stream &input)
{
    auto err_stream = options.output;

    DabVM_debug debug(*this);
    while (!input.eof())
    {
        char rawcmd[2048];
        fprintf(stderr, "> ");
        fgets(rawcmd, sizeof(rawcmd), stdin);
        std::string cmd = rawcmd;

        cmd = cmd.substr(0, cmd.length() - 1);

        if (cmd == "help")
        {
            fprintf(err_stream, "Help:\n");
            fprintf(err_stream, "help - print this\n");
            fprintf(err_stream, "[s]tep - run single instruction\n");
            fprintf(err_stream, "[r]egisters - show registers\n");
            fprintf(err_stream, "classes - print classes\n");
            fprintf(err_stream, "functions - print functions\n");
            fprintf(err_stream, "constants - dump constants\n");
            fprintf(err_stream, "stack - dump stack\n");
            fprintf(err_stream, "code - show current code\n");
            fprintf(err_stream, "allcode - show all code\n");
            fprintf(err_stream, "run - run remaining instructions\n");
            fprintf(err_stream, "break [ip] - break at defined IP\n");
            fprintf(err_stream, "quit - quit\n");
        }
        else if (cmd == "step" || cmd == "s")
        {
            execute_single(input);
        }
        else if (cmd == "registers" || cmd == "r")
        {
            debug.print_registers();
        }
        else if (cmd == "classes")
        {
            debug.print_classes();
        }
        else if (cmd == "functions")
        {
            debug.print_functions();
        }
        else if (cmd == "constants")
        {
            debug.print_constants();
        }
        else if (cmd == "stack")
        {
            debug.print_stack();
        }
        else if (cmd == "ip")
        {
            fprintf(err_stream, "IP = %" PRIu64 "\n", ip());
        }
        else if (cmd == "code")
        {
            debug.print_code(true);
        }
        else if (cmd == "allcode")
        {
            debug.print_code(false);
        }
        else if (cmd == "quit")
        {
            exit(0);
        }
        else if (cmd == "run")
        {
            execute(input);
        }
        else if (cmd.substr(0, 6) == "break ")
        {
            int ip  = 0;
            int ret = sscanf(cmd.c_str(), "break %d", &ip);
            assert(ret == 1);
            fprintf(err_stream, "debug: break at %d.\n", ip);
            breakpoints.insert(ip);
        }
        else if (cmd == "ssaregs")
        {
            debug.print_ssa_registers();
        }
        else
        {
            fprintf(stderr, "Unknown command, type <help> to get available commands.\n");
        }
    }
}
<|endoftext|>


<commit_before>#include "db_single.h"

namespace LeviDB {
    DBSingle::DBSingle(std::string name, Options options, SeqGenerator * seq_gen)
            : DB(std::move(name), options), _seq_gen(seq_gen) {
        std::string prefix = _name + '/' + _name;
        _file_lock.build(prefix + ".lock");

        if (IOEnv::fileExists(_name)) {
            if (_options.error_if_exists) {
                throw Exception::invalidArgumentException("DB already exists");
            }
            // 打开现有数据库
            std::string data_fname = prefix + ".data";
            if (!IOEnv::fileExists(data_fname)) {
                throw Exception::notFoundException("data file missing", data_fname);
            }
            std::string index_fname = prefix + ".index";
            std::string keeper_fname = prefix + ".keeper";
            if (!IOEnv::fileExists(index_fname) || !IOEnv::fileExists(keeper_fname)) {
                simpleRepair();
                return;
            }
            _meta.build(std::move(prefix));
            _af.build(data_fname);
            _rf.build(std::move(data_fname));
            _index.build(std::move(index_fname), _meta->immut_value()._offset, _seq_gen, _rf.get());
            _writer.build(_af.get());
        } else {
            if (!_options.create_if_missing) {
                throw Exception::notFoundException("DB not found");
            }
            // 新建数据库
            IOEnv::createDir(_name);
            _af.build(prefix + ".data");
            _rf.build(prefix + ".data");
            _index.build(prefix + ".index", _seq_gen, _rf.get());
            _writer.build(_af.get());
            _meta.build(std::move(prefix), DBSingleWeakMeta{}, ""); // WeakKeeper will add ".keeper" automatically
        }
    }

    void DBSingle::put(const WriteOptions & options,
                       const Slice & key,
                       const Slice & value) {
        RWLockWriteGuard write_guard(_rwlock);

        uint32_t pos = _writer->calcWritePos();
        std::vector<uint8_t> bin = LogWriter::makeRecord(key, value);
        _writer->addRecord({bin.data(), bin.size()});
        _index->insert(key, OffsetToData{pos});

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    void DBSingle::remove(const WriteOptions & options,
                          const Slice & key) {
        RWLockWriteGuard write_guard(_rwlock);

        std::vector<uint8_t> bin = LogWriter::makeRecord(key, {});
        _writer->addDelRecord({bin.data(), bin.size()});
        _index->remove(key);

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    void DBSingle::write(const WriteOptions & options,
                         const std::vector<std::pair<Slice, Slice>> & kvs) {
        RWLockWriteGuard write_guard(_rwlock);

        if (options.compress) {
            assert(options.uncompress_size != 0);
            uint32_t pos = _writer->calcWritePos();
            std::vector<uint8_t> bin = LogWriter::makeCompressRecord(kvs);
            if (bin.size() <= options.uncompress_size / 8 * 7) { // worth
                _writer->addCompressRecord({bin.data(), bin.size()});
                for (const auto & kv:kvs) {
                    _index->insert(kv.first, OffsetToData{pos});
                }

                _index->tryApplyPending();
                if (options.sync) _af->sync();
                return;
            }
        }

        std::vector<std::vector<uint8_t>> group;
        group.reserve(kvs.size());
        std::transform(kvs.cbegin(), kvs.cend(), group.begin(), [](const std::pair<Slice, Slice> & kv) noexcept {
            return LogWriter::makeRecord(kv.first, kv.second);
        });

        std::vector<Slice> bkvs;
        bkvs.reserve(group.size());
        std::transform(group.cbegin(), group.cend(), bkvs.begin(),
                       [](const std::vector<uint8_t> & bkv) noexcept -> Slice {
                           return {bkv.data(), bkv.size()};
                       });

        std::vector<uint32_t> addrs = _writer->addRecords(bkvs);
        assert(kvs.size() == addrs.size());
        for (int i = 0; i < kvs.size(); ++i) {
            _index->insert(kvs[i].first, OffsetToData{addrs[i]});
        }

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    std::pair<std::string, bool>
    DBSingle::get(const ReadOptions & options, const Slice & key) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->find(key, options.sequence_number);
    };

    std::unique_ptr<Snapshot>
    DBSingle::makeSnapshot() {
        RWLockWriteGuard write_guard(_rwlock);
        _index->tryApplyPending();
        return _seq_gen->makeSnapshot();
    };

    std::unique_ptr<Iterator<Slice, std::string>>
    DBSingle::makeIterator(std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeIterator(std::move(snapshot));
    };

    std::unique_ptr<SimpleIterator<std::pair<Slice, std::string>>>
    DBSingle::makeRegexIterator(std::shared_ptr<Regex::R> regex,
                                std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeRegexIterator(std::move(regex), std::move(snapshot));
    };

    std::unique_ptr<SimpleIterator<std::pair<Slice, std::string>>>
    DBSingle::makeRegexReversedIterator(std::shared_ptr<Regex::R> regex,
                                        std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeRegexReversedIterator(std::move(regex), std::move(snapshot));
    };

    void DBSingle::tryApplyPending() {
        RWLockWriteGuard write_guard(_rwlock);
        _index->tryApplyPending();
    }

    uint64_t DBSingle::indexFileSize() const noexcept {
        return _index->immut_dst().immut_length();
    };

    uint64_t DBSingle::dataFileSize() const noexcept {
        return _af->immut_length();
    };

    void DBSingle::explicitRemove(const WriteOptions & options, const Slice & key) {
        RWLockWriteGuard write_guard(_rwlock);

        uint32_t pos = _writer->calcWritePos();
        std::vector<uint8_t> bin = LogWriter::makeRecord(key, {});
        _writer->addDelRecord({bin.data(), bin.size()});
        _index->insert(key, OffsetToData{pos});

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    }

    void DBSingle::simpleRepair() noexcept {

    }

    Slice DBSingle::largestKey() const noexcept {

    };

    Slice DBSingle::smallestKey() const noexcept {

    };

    void DBSingle::updateKeyRange(const Slice & key) noexcept {

    };

    bool repairDBSingle(const std::string & db_single_name) noexcept {

    };
}
<commit_msg>updateKeyRange<commit_after>#include "db_single.h"

namespace LeviDB {
    DBSingle::DBSingle(std::string name, Options options, SeqGenerator * seq_gen)
            : DB(std::move(name), options), _seq_gen(seq_gen) {
        std::string prefix = _name + '/' + _name;
        _file_lock.build(prefix + ".lock");

        if (IOEnv::fileExists(_name)) {
            if (_options.error_if_exists) {
                throw Exception::invalidArgumentException("DB already exists");
            }
            // 打开现有数据库
            std::string data_fname = prefix + ".data";
            if (!IOEnv::fileExists(data_fname)) {
                throw Exception::notFoundException("data file missing", data_fname);
            }
            std::string index_fname = prefix + ".index";
            std::string keeper_fname = prefix + ".keeper";
            if (!IOEnv::fileExists(index_fname) || !IOEnv::fileExists(keeper_fname)) {
                simpleRepair();
                return;
            }
            _meta.build(std::move(prefix));
            _af.build(data_fname);
            _rf.build(std::move(data_fname));
            _index.build(std::move(index_fname), _meta->immut_value()._offset, _seq_gen, _rf.get());
            _writer.build(_af.get());
        } else {
            if (!_options.create_if_missing) {
                throw Exception::notFoundException("DB not found");
            }
            // 新建数据库
            IOEnv::createDir(_name);
            _af.build(prefix + ".data");
            _rf.build(prefix + ".data");
            _index.build(prefix + ".index", _seq_gen, _rf.get());
            _writer.build(_af.get());
            _meta.build(std::move(prefix), DBSingleWeakMeta{}, ""); // WeakKeeper will add ".keeper" automatically
        }
    }

    void DBSingle::put(const WriteOptions & options,
                       const Slice & key,
                       const Slice & value) {
        RWLockWriteGuard write_guard(_rwlock);

        uint32_t pos = _writer->calcWritePos();
        std::vector<uint8_t> bin = LogWriter::makeRecord(key, value);
        _writer->addRecord({bin.data(), bin.size()});
        updateKeyRange(key);
        _index->insert(key, OffsetToData{pos});

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    void DBSingle::remove(const WriteOptions & options,
                          const Slice & key) {
        RWLockWriteGuard write_guard(_rwlock);

        std::vector<uint8_t> bin = LogWriter::makeRecord(key, {});
        _writer->addDelRecord({bin.data(), bin.size()});
        _index->remove(key);

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    void DBSingle::write(const WriteOptions & options,
                         const std::vector<std::pair<Slice, Slice>> & kvs) {
        RWLockWriteGuard write_guard(_rwlock);

        if (options.compress) {
            assert(options.uncompress_size != 0);
            uint32_t pos = _writer->calcWritePos();
            std::vector<uint8_t> bin = LogWriter::makeCompressRecord(kvs);
            if (bin.size() <= options.uncompress_size / 8 * 7) { // worth
                _writer->addCompressRecord({bin.data(), bin.size()});
                for (const auto & kv:kvs) {
                    updateKeyRange(kv.first);
                    _index->insert(kv.first, OffsetToData{pos});
                }

                _index->tryApplyPending();
                if (options.sync) _af->sync();
                return;
            }
        }

        std::vector<std::vector<uint8_t>> group;
        group.reserve(kvs.size());
        std::transform(kvs.cbegin(), kvs.cend(), group.begin(), [](const std::pair<Slice, Slice> & kv) noexcept {
            return LogWriter::makeRecord(kv.first, kv.second);
        });

        std::vector<Slice> bkvs;
        bkvs.reserve(group.size());
        std::transform(group.cbegin(), group.cend(), bkvs.begin(),
                       [](const std::vector<uint8_t> & bkv) noexcept -> Slice {
                           return {bkv.data(), bkv.size()};
                       });

        std::vector<uint32_t> addrs = _writer->addRecords(bkvs);
        assert(kvs.size() == addrs.size());
        for (int i = 0; i < kvs.size(); ++i) {
            updateKeyRange(kvs[i].first);
            _index->insert(kvs[i].first, OffsetToData{addrs[i]});
        }

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    };

    std::pair<std::string, bool>
    DBSingle::get(const ReadOptions & options, const Slice & key) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->find(key, options.sequence_number);
    };

    std::unique_ptr<Snapshot>
    DBSingle::makeSnapshot() {
        RWLockWriteGuard write_guard(_rwlock);
        _index->tryApplyPending();
        return _seq_gen->makeSnapshot();
    };

    std::unique_ptr<Iterator<Slice, std::string>>
    DBSingle::makeIterator(std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeIterator(std::move(snapshot));
    };

    std::unique_ptr<SimpleIterator<std::pair<Slice, std::string>>>
    DBSingle::makeRegexIterator(std::shared_ptr<Regex::R> regex,
                                std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeRegexIterator(std::move(regex), std::move(snapshot));
    };

    std::unique_ptr<SimpleIterator<std::pair<Slice, std::string>>>
    DBSingle::makeRegexReversedIterator(std::shared_ptr<Regex::R> regex,
                                        std::unique_ptr<Snapshot> && snapshot) const {
        RWLockReadGuard read_guard(_rwlock);
        return _index->makeRegexReversedIterator(std::move(regex), std::move(snapshot));
    };

    void DBSingle::tryApplyPending() {
        RWLockWriteGuard write_guard(_rwlock);
        _index->tryApplyPending();
    }

    uint64_t DBSingle::indexFileSize() const noexcept {
        return _index->immut_dst().immut_length();
    };

    uint64_t DBSingle::dataFileSize() const noexcept {
        return _af->immut_length();
    };

    void DBSingle::explicitRemove(const WriteOptions & options, const Slice & key) {
        RWLockWriteGuard write_guard(_rwlock);

        uint32_t pos = _writer->calcWritePos();
        std::vector<uint8_t> bin = LogWriter::makeRecord(key, {});
        _writer->addDelRecord({bin.data(), bin.size()});
        _index->insert(key, OffsetToData{pos});

        _index->tryApplyPending();
        if (options.sync) _af->sync();
    }

    void DBSingle::simpleRepair() noexcept {

    }

    Slice DBSingle::largestKey() const noexcept {
        const std::string & trailing = _meta->immut_trailing();
        uint32_t from_k_len = _meta->immut_value().from_k_len;
        uint32_t to_k_len = _meta->immut_value().to_k_len;
        return {trailing.data() + from_k_len, to_k_len};
    };

    Slice DBSingle::smallestKey() const noexcept {
        const std::string & trailing = _meta->immut_trailing();
        uint32_t from_k_len = _meta->immut_value().from_k_len;
        return {trailing.data(), from_k_len};
    };

    void DBSingle::updateKeyRange(const Slice & key) noexcept {
        if (SliceComparator{}(key, smallestKey())) { // find smaller
            std::string & trailing = _meta->mut_trailing();
            uint32_t from_k_len = _meta->immut_value().from_k_len;
            trailing.replace(trailing.begin(), trailing.begin() + from_k_len, key.data(), key.data() + key.size());
            _meta->mut_value().from_k_len = static_cast<uint32_t>(key.size());
        } else if (SliceComparator{}(largestKey(), key)) { // find larger
            std::string & trailing = _meta->mut_trailing();
            uint32_t from_k_len = _meta->immut_value().from_k_len;
            uint32_t to_k_len = _meta->immut_value().to_k_len;
            trailing.replace(trailing.begin() + from_k_len, trailing.begin() + from_k_len + to_k_len,
                             key.data(), key.data() + key.size());
            _meta->mut_value().to_k_len = static_cast<uint32_t>(key.size());
        }
    };

    bool repairDBSingle(const std::string & db_single_name) noexcept {

    };
}
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <memory>

#include "dll/dbn.hpp"
#include "dll/test.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

namespace {

template<typename DBN, typename P>
void test_all(DBN& dbn, std::vector<vector<double>>& training_images, const std::vector<uint8_t>& training_labels, P&& predictor){
    auto test_images = mnist::read_test_images<std::vector, vector, double>();
    auto test_labels = mnist::read_test_labels<std::vector>();

    if(test_images.empty() || test_labels.empty()){
        std::cout << "Impossible to read test set" << std::endl;
        return;
    }

    std::cout << "Start testing" << std::endl;

    std::cout << "Training Set" << std::endl;
    auto error_rate = dll::test_set(dbn, training_images, training_labels, predictor);
    std::cout << "\tError rate (normal): " << 100.0 * error_rate << std::endl;

    std::cout << "Test Set" << std::endl;
    error_rate =  dll::test_set(dbn, test_images, test_labels, predictor);
    std::cout << "\tError rate (normal): " << 100.0 * error_rate << std::endl;
}

} //end of anonymous namespace

int main(int argc, char* argv[]){
    auto simple = false;
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "simple"){
            simple = true;
        }

        if(command == "load"){
            load = true;
        }
    }

    auto dataset = mnist::read_dataset<std::vector, vector, double>();

    if(dataset.training_images.empty() || dataset.training_labels.empty()){
        return 1;
    }

    mnist::binarize_dataset(dataset);

    if(simple){
        typedef dll::dbn<
            dll::layer<28 * 28, 100, dll::in_dbn, dll::batch_size<50>, dll::init_weights, dll::momentum, dll::weight_decay<dll::decay_type::L2>>,
            dll::layer<100, 100, dll::in_dbn, dll::batch_size<50>, dll::momentum, dll::weight_decay<dll::decay_type::L2>>,
            dll::layer<110, 200, dll::in_dbn, dll::batch_size<50>, dll::momentum, dll::weight_decay<dll::decay_type::L2>>> dbn_simple_t;

        auto dbn = std::make_shared<dbn_simple_t>();

        dbn->train_with_labels(dataset.training_images, dataset.training_labels, 10, 5);

        test_all(dbn, dataset.training_images, dataset.training_labels, dll::label_predictor());
    } else {
        typedef dll::dbn<
            dll::layer<28 * 28, 100, dll::in_dbn, dll::momentum, dll::batch_size<50>, dll::init_weights>,
            dll::layer<100, 200, dll::in_dbn, dll::momentum, dll::batch_size<50>>,
            dll::layer<200, 10, dll::in_dbn, dll::momentum, dll::batch_size<50>, dll::hidden<dll::unit_type::EXP>>> dbn_t;

        auto dbn = make_unique<dbn_t>();

        dbn->display();

        if(load){
            std::cout << "Load from file" << std::endl;

            std::ifstream is("dbn.dat", std::ifstream::binary);
            dbn->load(is);
        } else {
            std::cout << "Start pretraining" << std::endl;
            dbn->pretrain(dataset.training_images, 5);

            std::cout << "Start fine-tuning" << std::endl;
            dbn->fine_tune(dataset.training_images, dataset.training_labels, 5, 1000);

            std::ofstream os("dbn.dat", std::ofstream::binary);
            dbn->store(os);
        }

        test_all(dbn, dataset.training_images, dataset.training_labels, dll::predictor());
    }

    return 0;
}
<commit_msg>Clena<commit_after>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <memory>

#include "dll/dbn.hpp"
#include "dll/test.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

namespace {

template<typename DBN, typename P>
void test_all(DBN& dbn, std::vector<vector<double>>& training_images, const std::vector<uint8_t>& training_labels, P&& predictor){
    auto test_images = mnist::read_test_images<std::vector, vector, double>();
    auto test_labels = mnist::read_test_labels<std::vector>();

    if(test_images.empty() || test_labels.empty()){
        std::cout << "Impossible to read test set" << std::endl;
        return;
    }

    std::cout << "Start testing" << std::endl;

    std::cout << "Training Set" << std::endl;
    auto error_rate = dll::test_set(dbn, training_images, training_labels, predictor);
    std::cout << "\tError rate (normal): " << 100.0 * error_rate << std::endl;

    std::cout << "Test Set" << std::endl;
    error_rate =  dll::test_set(dbn, test_images, test_labels, predictor);
    std::cout << "\tError rate (normal): " << 100.0 * error_rate << std::endl;
}

} //end of anonymous namespace

int main(int argc, char* argv[]){
    auto simple = false;
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "simple"){
            simple = true;
        }

        if(command == "load"){
            load = true;
        }
    }

    auto dataset = mnist::read_dataset<std::vector, vector, double>();

    if(dataset.training_images.empty() || dataset.training_labels.empty()){
        return 1;
    }

    mnist::binarize_dataset(dataset);

    if(simple){
        typedef dll::dbn_desc<
            dll::dbn_layers<
                dll::rbm_desc<28 * 28, 100, dll::in_dbn, dll::batch_size<50>, dll::init_weights, dll::momentum, dll::weight_decay<dll::decay_type::L2>>::rbm_t,
                dll::rbm_desc<100, 100, dll::in_dbn, dll::batch_size<50>, dll::momentum, dll::weight_decay<dll::decay_type::L2>>::rbm_t,
                dll::rbm_desc<110, 200, dll::in_dbn, dll::batch_size<50>, dll::momentum, dll::weight_decay<dll::decay_type::L2>>::rbm_t,
            >> dbn_simple_t;

        auto dbn = std::make_shared<dbn_simple_t>();

        dbn->train_with_labels(dataset.training_images, dataset.training_labels, 10, 5);

        test_all(dbn, dataset.training_images, dataset.training_labels, dll::label_predictor());
    } else {
        typedef dll::dbn<
            dll::layer<28 * 28, 100, dll::in_dbn, dll::momentum, dll::batch_size<50>, dll::init_weights>,
            dll::layer<100, 200, dll::in_dbn, dll::momentum, dll::batch_size<50>>,
            dll::layer<200, 10, dll::in_dbn, dll::momentum, dll::batch_size<50>, dll::hidden<dll::unit_type::EXP>>> dbn_t;

        auto dbn = make_unique<dbn_t>();

        dbn->display();

        if(load){
            std::cout << "Load from file" << std::endl;

            std::ifstream is("dbn.dat", std::ifstream::binary);
            dbn->load(is);
        } else {
            std::cout << "Start pretraining" << std::endl;
            dbn->pretrain(dataset.training_images, 5);

            std::cout << "Start fine-tuning" << std::endl;
            dbn->fine_tune(dataset.training_images, dataset.training_labels, 5, 1000);

            std::ofstream os("dbn.dat", std::ofstream::binary);
            dbn->store(os);
        }

        test_all(dbn, dataset.training_images, dataset.training_labels, dll::predictor());
    }

    return 0;
}
<|endoftext|>


<commit_before>// Copyright (c) 2009-2014 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "ecwrapper.h"

#include "serialize.h"
#include "uint256.h"

#include <openssl/bn.h>
#include <openssl/ecdsa.h>
#include <openssl/obj_mac.h>

namespace {

/**
 * Perform ECDSA key recovery (see SEC1 4.1.6) for curves over (mod p)-fields
 * recid selects which key is recovered
 * if check is non-zero, additional checks are performed
 */
int ECDSA_SIG_recover_key_GFp(EC_KEY *eckey, ECDSA_SIG *ecsig, const unsigned char *msg, int msglen, int recid, int check)
{
    if (!eckey) return 0;

    int ret = 0;
    BN_CTX *ctx = NULL;

    BIGNUM *x = NULL;
    BIGNUM *e = NULL;
    BIGNUM *order = NULL;
    BIGNUM *sor = NULL;
    BIGNUM *eor = NULL;
    BIGNUM *field = NULL;
    EC_POINT *R = NULL;
    EC_POINT *O = NULL;
    EC_POINT *Q = NULL;
    BIGNUM *rr = NULL;
    BIGNUM *zero = NULL;
    int n = 0;
    int i = recid / 2;

    const EC_GROUP *group = EC_KEY_get0_group(eckey);
    if ((ctx = BN_CTX_new()) == NULL) { ret = -1; goto err; }
    BN_CTX_start(ctx);
    order = BN_CTX_get(ctx);
    if (!EC_GROUP_get_order(group, order, ctx)) { ret = -2; goto err; }
    x = BN_CTX_get(ctx);
    if (!BN_copy(x, order)) { ret=-1; goto err; }
    if (!BN_mul_word(x, i)) { ret=-1; goto err; }
    if (!BN_add(x, x, ecsig->r)) { ret=-1; goto err; }
    field = BN_CTX_get(ctx);
    if (!EC_GROUP_get_curve_GFp(group, field, NULL, NULL, ctx)) { ret=-2; goto err; }
    if (BN_cmp(x, field) >= 0) { ret=0; goto err; }
    if ((R = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
    if (!EC_POINT_set_compressed_coordinates_GFp(group, R, x, recid % 2, ctx)) { ret=0; goto err; }
    if (check)
    {
        if ((O = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
        if (!EC_POINT_mul(group, O, NULL, R, order, ctx)) { ret=-2; goto err; }
        if (!EC_POINT_is_at_infinity(group, O)) { ret = 0; goto err; }
    }
    if ((Q = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
    n = EC_GROUP_get_degree(group);
    e = BN_CTX_get(ctx);
    if (!BN_bin2bn(msg, msglen, e)) { ret=-1; goto err; }
    if (8*msglen > n) BN_rshift(e, e, 8-(n & 7));
    zero = BN_CTX_get(ctx);
    if (!BN_zero(zero)) { ret=-1; goto err; }
    if (!BN_mod_sub(e, zero, e, order, ctx)) { ret=-1; goto err; }
    rr = BN_CTX_get(ctx);
    if (!BN_mod_inverse(rr, ecsig->r, order, ctx)) { ret=-1; goto err; }
    sor = BN_CTX_get(ctx);
    if (!BN_mod_mul(sor, ecsig->s, rr, order, ctx)) { ret=-1; goto err; }
    eor = BN_CTX_get(ctx);
    if (!BN_mod_mul(eor, e, rr, order, ctx)) { ret=-1; goto err; }
    if (!EC_POINT_mul(group, Q, eor, R, sor, ctx)) { ret=-2; goto err; }
    if (!EC_KEY_set_public_key(eckey, Q)) { ret=-2; goto err; }

    ret = 1;

err:
    if (ctx) {
        BN_CTX_end(ctx);
        BN_CTX_free(ctx);
    }
    if (R != NULL) EC_POINT_free(R);
    if (O != NULL) EC_POINT_free(O);
    if (Q != NULL) EC_POINT_free(Q);
    return ret;
}

} // anon namespace

CECKey::CECKey() {
    pkey = EC_KEY_new_by_curve_name(NID_secp256k1);
    assert(pkey != NULL);
}

CECKey::~CECKey() {
    EC_KEY_free(pkey);
}

void CECKey::GetPubKey(std::vector<unsigned char> &pubkey, bool fCompressed) {
    EC_KEY_set_conv_form(pkey, fCompressed ? POINT_CONVERSION_COMPRESSED : POINT_CONVERSION_UNCOMPRESSED);
    int nSize = i2o_ECPublicKey(pkey, NULL);
    assert(nSize);
    assert(nSize <= 65);
    pubkey.clear();
    pubkey.resize(nSize);
    unsigned char *pbegin(begin_ptr(pubkey));
    int nSize2 = i2o_ECPublicKey(pkey, &pbegin);
    assert(nSize == nSize2);
}

bool CECKey::SetPubKey(const unsigned char* pubkey, size_t size) {
    return o2i_ECPublicKey(&pkey, &pubkey, size) != NULL;
}

bool CECKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchSig) {
    // -1 = error, 0 = bad sig, 1 = good
    if (ECDSA_verify(0, (unsigned char*)&hash, sizeof(hash), &vchSig[0], vchSig.size(), pkey) != 1)
        return false;
    return true;
}

bool CECKey::Recover(const uint256 &hash, const unsigned char *p64, int rec)
{
    if (rec<0 || rec>=3)
        return false;
    ECDSA_SIG *sig = ECDSA_SIG_new();
    BN_bin2bn(&p64[0],  32, sig->r);
    BN_bin2bn(&p64[32], 32, sig->s);
    bool ret = ECDSA_SIG_recover_key_GFp(pkey, sig, (unsigned char*)&hash, sizeof(hash), rec, 0) == 1;
    ECDSA_SIG_free(sig);
    return ret;
}

bool CECKey::TweakPublic(const unsigned char vchTweak[32]) {
    bool ret = true;
    BN_CTX *ctx = BN_CTX_new();
    BN_CTX_start(ctx);
    BIGNUM *bnTweak = BN_CTX_get(ctx);
    BIGNUM *bnOrder = BN_CTX_get(ctx);
    BIGNUM *bnOne = BN_CTX_get(ctx);
    const EC_GROUP *group = EC_KEY_get0_group(pkey);
    EC_GROUP_get_order(group, bnOrder, ctx); // what a grossly inefficient way to get the (constant) group order...
    BN_bin2bn(vchTweak, 32, bnTweak);
    if (BN_cmp(bnTweak, bnOrder) >= 0)
        ret = false; // extremely unlikely
    EC_POINT *point = EC_POINT_dup(EC_KEY_get0_public_key(pkey), group);
    BN_one(bnOne);
    EC_POINT_mul(group, point, bnTweak, point, bnOne, ctx);
    if (EC_POINT_is_at_infinity(group, point))
        ret = false; // ridiculously unlikely
    EC_KEY_set_public_key(pkey, point);
    EC_POINT_free(point);
    BN_CTX_end(ctx);
    BN_CTX_free(ctx);
    return ret;
}

bool CECKey::SanityCheck()
{
    EC_KEY *pkey = EC_KEY_new_by_curve_name(NID_secp256k1);
    if(pkey == NULL)
        return false;
    EC_KEY_free(pkey);

    // TODO Is there more EC functionality that could be missing?
    return true;
}
<commit_msg>consensus: guard against openssl's new strict DER checks<commit_after>// Copyright (c) 2009-2014 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "ecwrapper.h"

#include "serialize.h"
#include "uint256.h"

#include <openssl/bn.h>
#include <openssl/ecdsa.h>
#include <openssl/obj_mac.h>

namespace {

/**
 * Perform ECDSA key recovery (see SEC1 4.1.6) for curves over (mod p)-fields
 * recid selects which key is recovered
 * if check is non-zero, additional checks are performed
 */
int ECDSA_SIG_recover_key_GFp(EC_KEY *eckey, ECDSA_SIG *ecsig, const unsigned char *msg, int msglen, int recid, int check)
{
    if (!eckey) return 0;

    int ret = 0;
    BN_CTX *ctx = NULL;

    BIGNUM *x = NULL;
    BIGNUM *e = NULL;
    BIGNUM *order = NULL;
    BIGNUM *sor = NULL;
    BIGNUM *eor = NULL;
    BIGNUM *field = NULL;
    EC_POINT *R = NULL;
    EC_POINT *O = NULL;
    EC_POINT *Q = NULL;
    BIGNUM *rr = NULL;
    BIGNUM *zero = NULL;
    int n = 0;
    int i = recid / 2;

    const EC_GROUP *group = EC_KEY_get0_group(eckey);
    if ((ctx = BN_CTX_new()) == NULL) { ret = -1; goto err; }
    BN_CTX_start(ctx);
    order = BN_CTX_get(ctx);
    if (!EC_GROUP_get_order(group, order, ctx)) { ret = -2; goto err; }
    x = BN_CTX_get(ctx);
    if (!BN_copy(x, order)) { ret=-1; goto err; }
    if (!BN_mul_word(x, i)) { ret=-1; goto err; }
    if (!BN_add(x, x, ecsig->r)) { ret=-1; goto err; }
    field = BN_CTX_get(ctx);
    if (!EC_GROUP_get_curve_GFp(group, field, NULL, NULL, ctx)) { ret=-2; goto err; }
    if (BN_cmp(x, field) >= 0) { ret=0; goto err; }
    if ((R = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
    if (!EC_POINT_set_compressed_coordinates_GFp(group, R, x, recid % 2, ctx)) { ret=0; goto err; }
    if (check)
    {
        if ((O = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
        if (!EC_POINT_mul(group, O, NULL, R, order, ctx)) { ret=-2; goto err; }
        if (!EC_POINT_is_at_infinity(group, O)) { ret = 0; goto err; }
    }
    if ((Q = EC_POINT_new(group)) == NULL) { ret = -2; goto err; }
    n = EC_GROUP_get_degree(group);
    e = BN_CTX_get(ctx);
    if (!BN_bin2bn(msg, msglen, e)) { ret=-1; goto err; }
    if (8*msglen > n) BN_rshift(e, e, 8-(n & 7));
    zero = BN_CTX_get(ctx);
    if (!BN_zero(zero)) { ret=-1; goto err; }
    if (!BN_mod_sub(e, zero, e, order, ctx)) { ret=-1; goto err; }
    rr = BN_CTX_get(ctx);
    if (!BN_mod_inverse(rr, ecsig->r, order, ctx)) { ret=-1; goto err; }
    sor = BN_CTX_get(ctx);
    if (!BN_mod_mul(sor, ecsig->s, rr, order, ctx)) { ret=-1; goto err; }
    eor = BN_CTX_get(ctx);
    if (!BN_mod_mul(eor, e, rr, order, ctx)) { ret=-1; goto err; }
    if (!EC_POINT_mul(group, Q, eor, R, sor, ctx)) { ret=-2; goto err; }
    if (!EC_KEY_set_public_key(eckey, Q)) { ret=-2; goto err; }

    ret = 1;

err:
    if (ctx) {
        BN_CTX_end(ctx);
        BN_CTX_free(ctx);
    }
    if (R != NULL) EC_POINT_free(R);
    if (O != NULL) EC_POINT_free(O);
    if (Q != NULL) EC_POINT_free(Q);
    return ret;
}

} // anon namespace

CECKey::CECKey() {
    pkey = EC_KEY_new_by_curve_name(NID_secp256k1);
    assert(pkey != NULL);
}

CECKey::~CECKey() {
    EC_KEY_free(pkey);
}

void CECKey::GetPubKey(std::vector<unsigned char> &pubkey, bool fCompressed) {
    EC_KEY_set_conv_form(pkey, fCompressed ? POINT_CONVERSION_COMPRESSED : POINT_CONVERSION_UNCOMPRESSED);
    int nSize = i2o_ECPublicKey(pkey, NULL);
    assert(nSize);
    assert(nSize <= 65);
    pubkey.clear();
    pubkey.resize(nSize);
    unsigned char *pbegin(begin_ptr(pubkey));
    int nSize2 = i2o_ECPublicKey(pkey, &pbegin);
    assert(nSize == nSize2);
}

bool CECKey::SetPubKey(const unsigned char* pubkey, size_t size) {
    return o2i_ECPublicKey(&pkey, &pubkey, size) != NULL;
}

bool CECKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchSig) {
    // New versions of OpenSSL will reject non-canonical DER signatures. de/re-serialize first.
    unsigned char *norm_der = NULL;
    ECDSA_SIG *norm_sig = ECDSA_SIG_new();
    const unsigned char* sigptr = &vchSig[0];
    d2i_ECDSA_SIG(&norm_sig, &sigptr, vchSig.size());
    int derlen = i2d_ECDSA_SIG(norm_sig, &norm_der);
    ECDSA_SIG_free(norm_sig);
    if (derlen <= 0)
        return false;

    // -1 = error, 0 = bad sig, 1 = good
    bool ret = ECDSA_verify(0, (unsigned char*)&hash, sizeof(hash), norm_der, derlen, pkey) == 1;
    OPENSSL_free(norm_der);
    return ret;
}

bool CECKey::Recover(const uint256 &hash, const unsigned char *p64, int rec)
{
    if (rec<0 || rec>=3)
        return false;
    ECDSA_SIG *sig = ECDSA_SIG_new();
    BN_bin2bn(&p64[0],  32, sig->r);
    BN_bin2bn(&p64[32], 32, sig->s);
    bool ret = ECDSA_SIG_recover_key_GFp(pkey, sig, (unsigned char*)&hash, sizeof(hash), rec, 0) == 1;
    ECDSA_SIG_free(sig);
    return ret;
}

bool CECKey::TweakPublic(const unsigned char vchTweak[32]) {
    bool ret = true;
    BN_CTX *ctx = BN_CTX_new();
    BN_CTX_start(ctx);
    BIGNUM *bnTweak = BN_CTX_get(ctx);
    BIGNUM *bnOrder = BN_CTX_get(ctx);
    BIGNUM *bnOne = BN_CTX_get(ctx);
    const EC_GROUP *group = EC_KEY_get0_group(pkey);
    EC_GROUP_get_order(group, bnOrder, ctx); // what a grossly inefficient way to get the (constant) group order...
    BN_bin2bn(vchTweak, 32, bnTweak);
    if (BN_cmp(bnTweak, bnOrder) >= 0)
        ret = false; // extremely unlikely
    EC_POINT *point = EC_POINT_dup(EC_KEY_get0_public_key(pkey), group);
    BN_one(bnOne);
    EC_POINT_mul(group, point, bnTweak, point, bnOne, ctx);
    if (EC_POINT_is_at_infinity(group, point))
        ret = false; // ridiculously unlikely
    EC_KEY_set_public_key(pkey, point);
    EC_POINT_free(point);
    BN_CTX_end(ctx);
    BN_CTX_free(ctx);
    return ret;
}

bool CECKey::SanityCheck()
{
    EC_KEY *pkey = EC_KEY_new_by_curve_name(NID_secp256k1);
    if(pkey == NULL)
        return false;
    EC_KEY_free(pkey);

    // TODO Is there more EC functionality that could be missing?
    return true;
}
<|endoftext|>