<commit_before>//---------------------------------------------------------
// Copyright 2016 Ontario Institute for Cancer Research
// Written by Matei David (matei.david@oicr.on.ca)
//---------------------------------------------------------
//
#include "nanopolish_extract.h"

#include <iostream>
#include <sstream>
#include <getopt.h>

#include "nanopolish_common.h"
#include "fs_support.hpp"
#include "logger.hpp"
#include "alg.hpp"
#include <fast5.hpp>

//
// Getopt
//
#define SUBPROGRAM "extract"
#define LOG_FACILITY SUBPROGRAM

static const char *EXTRACT_VERSION_MESSAGE =
SUBPROGRAM " Version " PACKAGE_VERSION "\n"
"Written by Jared Simpson.\n"
"\n"
"Copyright 2015 Ontario Institute for Cancer Research\n";

static const char *EXTRACT_USAGE_MESSAGE =
"Usage: " PACKAGE_NAME " " SUBPROGRAM " [OPTIONS] <fast5|dir>...\n"
"Extract reads in fasta format\n"
"\n"
"      --help                           display this help and exit\n"
"      --version                        display version\n"
"  -v, --verbose                        display verbose output\n"
"  -r, --recurse                        recurse into subdirectories\n"
"  -t, --type=TYPE                      read type: template, complement, 2d, 2d-or-template, all\n"
"                                         (default: 2d-or-template)\n"
"  -o, --output=FILE                    write output to FILE (default: stdout)\n"
"\nReport bugs to " PACKAGE_BUGREPORT "\n\n";

namespace opt
{
    static unsigned int verbose = 0;
    static bool recurse = false;
    static std::string read_type = "2d-or-template";
    static std::string output_file;
    static std::deque< std::string > paths;
}
static std::ostream* os_p;

std::vector< std::pair< unsigned, std::string > >
get_preferred_basecall_groups(const fast5::File& f, const std::string& read_type)
{
    bool have_2d = false;
    std::vector< std::pair< unsigned, std::string > > res;
    // check 2d
    if (read_type == "all"
        or read_type == "2d"
        or read_type == "2d-or-template")
    {
        const auto& gr_l = f.get_basecall_strand_group_list(2);
        for (const auto& gr : gr_l)
        {
            if (f.have_basecall_fastq(2, gr)
                and f.have_basecall_events(0, gr)
                and f.have_basecall_events(1, gr))
            {
                have_2d = true;
                res.push_back(std::make_pair(2, gr));
                if (read_type != "all")
                {
                    break;
                }
            }
        }
    }
    // check 1d
    for (unsigned st = 0; st < 2; ++st)
    {
        if (opt::read_type == "all"
            or (st == 0
                and (opt::read_type == "template"
                     or (not have_2d and opt::read_type == "2d-or-template")))
            or (st == 1
                and opt::read_type == "complement"))
        {
            const auto& gr_l = f.get_basecall_strand_group_list(st);
            for (const auto& gr : gr_l)
            {
                if (f.have_basecall_fastq(st, gr)
                    and f.have_basecall_events(st, gr))
                {
                    res.push_back(std::make_pair(st, gr));
                    if (read_type != "all")
                    {
                        break;
                    }
                }
            }
        }
    }
    LOG(debug)
        << "preferred_groups: "
        << alg::os_join(res, ", ", [] (const decltype(res)::value_type& p) {
                std::ostringstream oss;
                oss << "(" << p.first << ":" << p.second << ")";
                return oss.str();
            })
        << "\n";
    return res;
} // get_preferred_basecall_groups

void process_file(const std::string& fn)
{
    LOG(debug) << "processing_file: " << fn << "\n";
    auto pos = fn.find_last_of('/');
    std::string base_fn = (pos != std::string::npos? fn.substr(pos + 1) : fn);
    if (base_fn.substr(base_fn.size() - 6) == ".fast5")
    {
        base_fn.resize(base_fn.size() - 6);
    }
    fast5::File f;
    do
    {
        try
        {
            // open file
            f.open(fn);
            // get preferred basecall groups
            auto l = get_preferred_basecall_groups(f, opt::read_type);
            if (l.empty()) return;
            const auto& p = l.front();
            // get and parse fastq
            auto fq = f.get_basecall_fastq(p.first, p.second);
            auto fq_a = f.split_fq(fq);
            // construct name
            auto pos = fq_a[0].find_first_of('_');
            std::string name = fq_a[0].substr(0, pos) + ":" + p.second + ":";
            if (p.first == 0)
            {
                name += "template";
            }
            else if (p.first == 1)
            {
                name += "complement";
            }
            else
            {
                name += "2d";
            }
            (*os_p) << ">" << name << " " << base_fn << " " << fn << "\n"
                    << fq_a[1] << "\n";
        }
        catch (hdf5_tools::Exception& e)
        {
            LOG(warning) << fn << ": HDF5 error: " << e.what() << "\n";
        }
    } while (false);
} // process_file

void process_path(const std::string& path)
{
    LOG(info) << "processing_path: " << path << "\n";
    if (is_directory(path))
    {
        auto dir_list = list_directory(path);
        for (const auto& fn : dir_list)
        {
            if (fn == "." or fn == "..") continue;
            std::string full_fn = path + "/" + fn;
            if (is_directory(full_fn))
            {
                if (opt::recurse)
                {
                    opt::paths.push_back(full_fn);
                }
                else
                {
                    LOG(info) << "ignoring_subdir: " << full_fn << "\n";
                }
            }
            else if (fast5::File::is_valid_file(full_fn))
            {
                process_file(full_fn);
            }
            else
            {
                LOG(info) << "ignoring_file: " << full_fn << "\n";
            }
        }
    }
    else // not is_directory; must be a fast5 file
    {
        if (fast5::File::is_valid_file(path))
        {
            process_file(path);
        }
        else
        {
            LOG(error) << "path [" << path << "] is neither a fast5 file nor a directory\n";
            exit(EXIT_FAILURE);
        }
    }
} // process_path

static const char* shortopts = "vrt:o:";

enum {
    OPT_HELP = 1,
    OPT_VERSION,
    OPT_LOG_LEVEL,
};

static const struct option longopts[] = {
    { "help",               no_argument,       NULL, OPT_HELP },
    { "version",            no_argument,       NULL, OPT_VERSION },
    { "log-level",          required_argument, NULL, OPT_LOG_LEVEL },
    { "verbose",            no_argument,       NULL, 'v' },
    { "recurse",            no_argument,       NULL, 'r' },
    { "type",               required_argument, NULL, 't' },
    { "output",             required_argument, NULL, 'o' },
    { NULL, 0, NULL, 0 }
};

void parse_extract_options(int argc, char** argv)
{
    bool die = false;
    for (char c; (c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1;) {
        std::istringstream arg(optarg != NULL ? optarg : "");
        switch (c) {
            case OPT_HELP:
                std::cout << EXTRACT_USAGE_MESSAGE;
                exit(EXIT_SUCCESS);
            case OPT_VERSION:
                std::cout << EXTRACT_VERSION_MESSAGE;
                exit(EXIT_SUCCESS);
            case OPT_LOG_LEVEL:
                Logger::set_level_from_option(arg.str());
                break;
            case 'v': opt::verbose++; break;
            case 'r': opt::recurse = true; break;
            case 't': arg >> opt::read_type; break;
            case 'o': arg >> opt::output_file; break;
        }
    }
    // parse paths to process
    while (optind < argc)
    {
        std::string path = argv[optind++];
        while (path.size() > 1 and path[path.size() - 1] == '/')
        {
            path.resize(path.size() - 1);
        }
        opt::paths.push_back(path);
    }
    if (opt::paths.empty())
    {
        std::cerr << SUBPROGRAM ": no paths to process\n";
        die = true;
    }
    // check read type
    if (not (opt::read_type == "template"
             or opt::read_type == "complement"
             or opt::read_type == "2d"
             or opt::read_type == "2d-or-template"
             or opt::read_type == "all"))
    {
        std::cerr << SUBPROGRAM ": invalid read type: " << opt::read_type << "\n";
        die = true;
    }
    // die if errors
    if (die)
    {
        std::cerr << "\n" << EXTRACT_USAGE_MESSAGE;
        exit(EXIT_FAILURE);
    }
    LOG(info) << "paths: " << alg::os_join(opt::paths, " ") << "\n";
    LOG(info) << "recurse: " << (opt::recurse? "yes" : "no") << "\n";
    LOG(info) << "read_type: " << opt::read_type << "\n";
}

int extract_main(int argc, char** argv)
{
    parse_extract_options(argc, argv);
    std::ofstream ofs;
    if (not opt::output_file.empty())
    {
        ofs.open(opt::output_file);
        os_p = &ofs;
    }
    else
    {
        os_p = &std::cout;
    }
    for (unsigned i = 0; i < opt::paths.size(); ++i)
    {
        process_path(opt::paths[i]);
    }
    return 0;
}
<commit_msg>added -q|--fastq option to extract fastq<commit_after>//---------------------------------------------------------
// Copyright 2016 Ontario Institute for Cancer Research
// Written by Matei David (matei.david@oicr.on.ca)
//---------------------------------------------------------
//
#include "nanopolish_extract.h"

#include <iostream>
#include <sstream>
#include <getopt.h>

#include "nanopolish_common.h"
#include "fs_support.hpp"
#include "logger.hpp"
#include "alg.hpp"
#include <fast5.hpp>

//
// Getopt
//
#define SUBPROGRAM "extract"
#define LOG_FACILITY SUBPROGRAM

static const char *EXTRACT_VERSION_MESSAGE =
SUBPROGRAM " Version " PACKAGE_VERSION "\n"
"Written by Jared Simpson.\n"
"\n"
"Copyright 2015 Ontario Institute for Cancer Research\n";

static const char *EXTRACT_USAGE_MESSAGE =
"Usage: " PACKAGE_NAME " " SUBPROGRAM " [OPTIONS] <fast5|dir>...\n"
"Extract reads in fasta format\n"
"\n"
"      --help                           display this help and exit\n"
"      --version                        display version\n"
"  -v, --verbose                        display verbose output\n"
"  -r, --recurse                        recurse into subdirectories\n"
"  -q, --fastq                          extract fastq (default: fasta)\n"
"  -t, --type=TYPE                      read type: template, complement, 2d, 2d-or-template, all\n"
"                                         (default: 2d-or-template)\n"
"  -o, --output=FILE                    write output to FILE (default: stdout)\n"
"\nReport bugs to " PACKAGE_BUGREPORT "\n\n";

namespace opt
{
    static unsigned int verbose = 0;
    static bool recurse = false;
    static std::string read_type = "2d-or-template";
    static bool fastq = false;
    static std::string output_file;
    static std::deque< std::string > paths;
}
static std::ostream* os_p;

std::vector< std::pair< unsigned, std::string > >
get_preferred_basecall_groups(const fast5::File& f, const std::string& read_type)
{
    bool have_2d = false;
    std::vector< std::pair< unsigned, std::string > > res;
    // check 2d
    if (read_type == "all"
        or read_type == "2d"
        or read_type == "2d-or-template")
    {
        const auto& gr_l = f.get_basecall_strand_group_list(2);
        for (const auto& gr : gr_l)
        {
            if (f.have_basecall_fastq(2, gr)
                and f.have_basecall_events(0, gr)
                and f.have_basecall_events(1, gr))
            {
                have_2d = true;
                res.push_back(std::make_pair(2, gr));
                if (read_type != "all")
                {
                    break;
                }
            }
        }
    }
    // check 1d
    for (unsigned st = 0; st < 2; ++st)
    {
        if (opt::read_type == "all"
            or (st == 0
                and (opt::read_type == "template"
                     or (not have_2d and opt::read_type == "2d-or-template")))
            or (st == 1
                and opt::read_type == "complement"))
        {
            const auto& gr_l = f.get_basecall_strand_group_list(st);
            for (const auto& gr : gr_l)
            {
                if (f.have_basecall_fastq(st, gr)
                    and f.have_basecall_events(st, gr))
                {
                    res.push_back(std::make_pair(st, gr));
                    if (read_type != "all")
                    {
                        break;
                    }
                }
            }
        }
    }
    LOG(debug)
        << "preferred_groups: "
        << alg::os_join(res, ", ", [] (const decltype(res)::value_type& p) {
                std::ostringstream oss;
                oss << "(" << p.first << ":" << p.second << ")";
                return oss.str();
            })
        << "\n";
    return res;
} // get_preferred_basecall_groups

void process_file(const std::string& fn)
{
    LOG(debug) << "processing_file: " << fn << "\n";
    auto pos = fn.find_last_of('/');
    std::string base_fn = (pos != std::string::npos? fn.substr(pos + 1) : fn);
    if (base_fn.substr(base_fn.size() - 6) == ".fast5")
    {
        base_fn.resize(base_fn.size() - 6);
    }
    fast5::File f;
    do
    {
        try
        {
            // open file
            f.open(fn);
            // get preferred basecall groups
            auto l = get_preferred_basecall_groups(f, opt::read_type);
            if (l.empty()) return;
            const auto& p = l.front();
            // get and parse fastq
            auto fq = f.get_basecall_fastq(p.first, p.second);
            auto fq_a = f.split_fq(fq);
            // construct name
            auto pos = fq_a[0].find_first_of('_');
            std::string name = fq_a[0].substr(0, pos) + ":" + p.second + ":";
            if (p.first == 0)
            {
                name += "template";
            }
            else if (p.first == 1)
            {
                name += "complement";
            }
            else
            {
                name += "2d";
            }
            if (not opt::fastq)
            {
                (*os_p)
                    << ">" << name << " " << base_fn << " " << fn << "\n"
                    << fq_a[1] << "\n";
            }
            else
            {
                (*os_p)
                    << "@" << name << " " << base_fn << " " << fn << "\n"
                    << fq_a[1] << "\n"
                    << "+" << fq_a[2] << "\n"
                    << fq_a[3] << "\n";
            }
        }
        catch (hdf5_tools::Exception& e)
        {
            LOG(warning) << fn << ": HDF5 error: " << e.what() << "\n";
        }
    } while (false);
} // process_file

void process_path(const std::string& path)
{
    LOG(info) << "processing_path: " << path << "\n";
    if (is_directory(path))
    {
        auto dir_list = list_directory(path);
        for (const auto& fn : dir_list)
        {
            if (fn == "." or fn == "..") continue;
            std::string full_fn = path + "/" + fn;
            if (is_directory(full_fn))
            {
                if (opt::recurse)
                {
                    opt::paths.push_back(full_fn);
                }
                else
                {
                    LOG(info) << "ignoring_subdir: " << full_fn << "\n";
                }
            }
            else if (fast5::File::is_valid_file(full_fn))
            {
                process_file(full_fn);
            }
            else
            {
                LOG(info) << "ignoring_file: " << full_fn << "\n";
            }
        }
    }
    else // not is_directory; must be a fast5 file
    {
        if (fast5::File::is_valid_file(path))
        {
            process_file(path);
        }
        else
        {
            LOG(error) << "path [" << path << "] is neither a fast5 file nor a directory\n";
            exit(EXIT_FAILURE);
        }
    }
} // process_path

static const char* shortopts = "vrqt:o:";

enum {
    OPT_HELP = 1,
    OPT_VERSION,
    OPT_LOG_LEVEL,
};

static const struct option longopts[] = {
    { "help",               no_argument,       NULL, OPT_HELP },
    { "version",            no_argument,       NULL, OPT_VERSION },
    { "log-level",          required_argument, NULL, OPT_LOG_LEVEL },
    { "verbose",            no_argument,       NULL, 'v' },
    { "recurse",            no_argument,       NULL, 'r' },
    { "fastq",              no_argument,       NULL, 'q' },
    { "type",               required_argument, NULL, 't' },
    { "output",             required_argument, NULL, 'o' },
    { NULL, 0, NULL, 0 }
};

void parse_extract_options(int argc, char** argv)
{
    bool die = false;
    for (char c; (c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1;) {
        std::istringstream arg(optarg != NULL ? optarg : "");
        switch (c) {
            case OPT_HELP:
                std::cout << EXTRACT_USAGE_MESSAGE;
                exit(EXIT_SUCCESS);
            case OPT_VERSION:
                std::cout << EXTRACT_VERSION_MESSAGE;
                exit(EXIT_SUCCESS);
            case OPT_LOG_LEVEL:
                Logger::set_level_from_option(arg.str());
                break;
            case 'v': opt::verbose++; break;
            case 'r': opt::recurse = true; break;
            case 'q': opt::fastq = true; break;
            case 't': arg >> opt::read_type; break;
            case 'o': arg >> opt::output_file; break;
        }
    }
    // parse paths to process
    while (optind < argc)
    {
        std::string path = argv[optind++];
        while (path.size() > 1 and path[path.size() - 1] == '/')
        {
            path.resize(path.size() - 1);
        }
        opt::paths.push_back(path);
    }
    if (opt::paths.empty())
    {
        std::cerr << SUBPROGRAM ": no paths to process\n";
        die = true;
    }
    // check read type
    if (not (opt::read_type == "template"
             or opt::read_type == "complement"
             or opt::read_type == "2d"
             or opt::read_type == "2d-or-template"
             or opt::read_type == "all"))
    {
        std::cerr << SUBPROGRAM ": invalid read type: " << opt::read_type << "\n";
        die = true;
    }
    // die if errors
    if (die)
    {
        std::cerr << "\n" << EXTRACT_USAGE_MESSAGE;
        exit(EXIT_FAILURE);
    }
    LOG(info) << "paths: " << alg::os_join(opt::paths, " ") << "\n";
    LOG(info) << "recurse: " << (opt::recurse? "yes" : "no") << "\n";
    LOG(info) << "read_type: " << opt::read_type << "\n";
}

int extract_main(int argc, char** argv)
{
    parse_extract_options(argc, argv);
    std::ofstream ofs;
    if (not opt::output_file.empty())
    {
        ofs.open(opt::output_file);
        os_p = &ofs;
    }
    else
    {
        os_p = &std::cout;
    }
    for (unsigned i = 0; i < opt::paths.size(); ++i)
    {
        process_path(opt::paths[i]);
    }
    return 0;
}
<|endoftext|>


<commit_before>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//#define DEBUG
#include <info>
#define MYINFO(X,...) INFO("DHCPv4",X,##__VA_ARGS__)

#include <net/dhcp/dh4client.hpp>
#include <net/dhcp/dhcp4.hpp>
#include <os> // OS::micros_since_boot()
#include <debug>

namespace net
{
  inline dhcp_option_t* conv_option(uint8_t* option)
  {
    return (dhcp_option_t*) option;
  }

  DHClient::DHClient(Stack& inet)
    : stack(inet), xid(0), console_spam(true), in_progress(false)
  {
    this->on_config(
    [this] (bool timeout)
    {
      if (console_spam) {
        if (timeout)
          INFO("DHCPv4", "Negotiation timed out");
        else
          INFO("DHCPv4", "Config complete");
      }
    });
  }

  void DHClient::on_config(config_func handler)
  {
    assert(handler);
    config_handlers_.push_back(handler);
  }

  void DHClient::negotiate(uint32_t timeout_secs)
  {
    // Allow multiple calls to negotiate without restarting the process
    if (in_progress) return;
    in_progress = true;

    // set timeout handler
    using namespace std::chrono;
    this->timeout = Timers::oneshot(seconds(timeout_secs),
    [this] (int) {
      // reset session ID
      this->xid = 0;
      this->in_progress = false;

      // call on_config with timeout = true
      for(auto handler : this->config_handlers_)
          handler(true);
    });

    // create a random session ID
    this->xid = OS::micros_since_boot() & 0xFFFFFFFF;
    if (console_spam)
      MYINFO("Negotiating IP-address (xid=%u)", xid);

    // create DHCP discover packet
    const size_t packetlen = sizeof(dhcp_packet_t);
    char packet[packetlen];

    dhcp_packet_t* dhcp = (dhcp_packet_t*) packet;
    dhcp->op    = BOOTREQUEST;
    dhcp->htype = HTYPE_ETHER;
    dhcp->hlen  = ETH_ALEN;
    dhcp->hops  = 0;
    dhcp->xid   = htonl(this->xid);
    dhcp->secs  = 0;
    dhcp->flags = htons(BOOTP_BROADCAST);
    dhcp->ciaddr = IP4::ADDR_ANY;
    dhcp->yiaddr = IP4::ADDR_ANY;
    dhcp->siaddr = IP4::ADDR_ANY;
    dhcp->giaddr = IP4::ADDR_ANY;

    Ethernet::addr link_addr = stack.link_addr();

    // copy our hardware address to chaddr field
    memset(dhcp->chaddr, 0, dhcp_packet_t::CHADDR_LEN);
    memcpy(dhcp->chaddr, &link_addr, ETH_ALEN);
    // zero server, file and options
    memset(dhcp->sname, 0, dhcp_packet_t::SNAME_LEN + dhcp_packet_t::FILE_LEN);

    dhcp->magic[0] =  99;
    dhcp->magic[1] = 130;
    dhcp->magic[2] =  83;
    dhcp->magic[3] =  99;

    dhcp_option_t* opt = conv_option(dhcp->options + 0);
    // DHCP discover
    opt->code   = DHO_DHCP_MESSAGE_TYPE;
    opt->length = 1;
    opt->val[0] = DHCPDISCOVER;
    // DHCP client identifier
    opt = conv_option(dhcp->options + 3);
    opt->code   = DHO_DHCP_CLIENT_IDENTIFIER;
    opt->length = 7;
    opt->val[0] = HTYPE_ETHER;
    memcpy(&opt->val[1], &link_addr, ETH_ALEN);
    // DHCP Parameter Request Field
    opt = conv_option(dhcp->options + 12);
    opt->code   = DHO_DHCP_PARAMETER_REQUEST_LIST;
    opt->length = 3;
    opt->val[0] = DHO_ROUTERS;
    opt->val[1] = DHO_SUBNET_MASK;
    opt->val[2] = DHO_DOMAIN_NAME_SERVERS;
    // END
    opt = conv_option(dhcp->options + 17);
    opt->code   = DHO_END;
    opt->length = 0;

    ////////////////////////////////////////////////////////
    auto& socket = stack.udp().bind(DHCP_CLIENT_PORT);
    /// broadcast our DHCP plea as 0.0.0.0:67
    socket.bcast(IP4::ADDR_ANY, DHCP_SERVER_PORT, packet, packetlen);

    socket.on_read(
    [this, &socket] (IP4::addr, UDP::port_t port,
                     const char* data, size_t len)
    {
      if (port == DHCP_SERVER_PORT)
      {
        // we have got a DHCP Offer
        debug("Received possible DHCP OFFER from %s:%d\n",
              addr.str().c_str(), DHCP_SERVER_PORT);
        this->offer(socket, data, len);
      }
    });
  }

  const dhcp_option_t* get_option(const uint8_t* options, uint8_t code)
  {
    const dhcp_option_t* opt = (const dhcp_option_t*) options;
    while (opt->code != code && opt->code != DHO_END)
      {
        // go to next option
        opt = (const dhcp_option_t*) (((const uint8_t*) opt) + 2 + opt->length);
      }
    return opt;
  }

  void DHClient::offer(UDPSocket& sock, const char* data, size_t)
  {
    const dhcp_packet_t* dhcp = (const dhcp_packet_t*) data;

    uint32_t xid = htonl(dhcp->xid);
    // silently ignore transactions not our own
    if (xid != this->xid) return;

    // check if the BOOTP message is a DHCP OFFER
    const dhcp_option_t* opt;
    const dhcp_option_t* server_id {nullptr};
    opt = get_option(dhcp->options, DHO_DHCP_MESSAGE_TYPE);

    if (opt->code == DHO_DHCP_MESSAGE_TYPE)
    {
      // verify that the type is indeed DHCPOFFER
      debug("Found DHCP message type %d  (DHCP Offer = %d)\n",
            opt->val[0], DHCPOFFER);

      // ignore when not a DHCP Offer
      if (opt->val[0] != DHCPOFFER) return;
    }
    // ignore message when DHCP message type is missing
    else return;

    // the offered IP address:
    this->ipaddr = dhcp->yiaddr;
    if (console_spam)
      MYINFO("IP ADDRESS: \t%s", this->ipaddr.str().c_str());

    opt = get_option(dhcp->options, DHO_SUBNET_MASK);
    if (opt->code == DHO_SUBNET_MASK)
    {
      memcpy(&this->netmask, opt->val, sizeof(IP4::addr));
      if (console_spam)
        MYINFO("SUBNET MASK: \t%s", this->netmask.str().c_str());
    }

    opt = get_option(dhcp->options, DHO_DHCP_LEASE_TIME);
    if (opt->code == DHO_DHCP_LEASE_TIME)
    {
      this->lease_time = (((uint32_t) opt->val[0]) << 24 |
        ((uint32_t) opt->val[1]) << 16 |
        ((uint32_t) opt->val[2]) <<  8 |
        ((uint32_t) opt->val[3]) <<  0);

      if (console_spam)
        MYINFO("LEASE TIME: \t%u mins", this->lease_time / 60);
    }

    // Preserve DHCP server address
    opt = get_option(dhcp->options, DHO_DHCP_SERVER_IDENTIFIER);
    if (opt->code == DHO_DHCP_SERVER_IDENTIFIER)
      {
        server_id = opt;
      }

    // now validate the offer, checking for minimum information
    opt = get_option(dhcp->options, DHO_ROUTERS);
    if (opt->code == DHO_ROUTERS)
    {
      memcpy(&this->router, opt->val, sizeof(IP4::addr));
      if (console_spam)
        MYINFO("GATEWAY: \t%s", this->router.str().c_str());
    }
    // assume that the server we received the request from is the gateway
    else
    {
      if (server_id)
      {
        memcpy(&this->router, server_id->val, sizeof(IP4::addr));
        if (console_spam)
          MYINFO("GATEWAY: \t%s", this->router.str().c_str());
      }
      // silently ignore when both ROUTER and SERVER_ID is missing
      else return;
    }

    opt = get_option(dhcp->options, DHO_DOMAIN_NAME_SERVERS);
    if (opt->code == DHO_DOMAIN_NAME_SERVERS)
    {
      memcpy(&this->dns_server, opt->val, sizeof(IP4::addr));
    }
    else
    { // just try using ROUTER as DNS server
      this->dns_server = this->router;
    }
    if (console_spam)
      MYINFO("DNS SERVER: \t%s", this->dns_server.str().c_str());

    // we can accept the offer now by requesting the IP!
    this->request(sock, server_id);
  }

  void DHClient::request(UDPSocket& sock, const dhcp_option_t* server_id)
  {
    // form a response
    const size_t packetlen = sizeof(dhcp_packet_t);
    char packet[packetlen];

    dhcp_packet_t* resp = (dhcp_packet_t*) packet;
    resp->op    = BOOTREQUEST;
    resp->htype = HTYPE_ETHER;
    resp->hlen  = ETH_ALEN;
    resp->hops  = 0;
    resp->xid   = htonl(this->xid);
    resp->secs  = 0;
    resp->flags = htons(BOOTP_BROADCAST);

    resp->ciaddr = IP4::ADDR_ANY;
    resp->yiaddr = IP4::ADDR_ANY;
    resp->siaddr = IP4::ADDR_ANY;
    resp->giaddr = IP4::ADDR_ANY;

    Ethernet::addr link_addr = stack.link_addr();

    // copy our hardware address to chaddr field
    memset(resp->chaddr, 0, dhcp_packet_t::CHADDR_LEN);
    memcpy(resp->chaddr, &link_addr, ETH_ALEN);
    // zero server, file and options
    memset(resp->sname, 0, dhcp_packet_t::SNAME_LEN + dhcp_packet_t::FILE_LEN);
    // magic DHCP bootp values
    resp->magic[0] =  99;
    resp->magic[1] = 130;
    resp->magic[2] =  83;
    resp->magic[3] =  99;

    dhcp_option_t* opt = conv_option(resp->options + 0);
    // DHCP Request
    opt->code   = DHO_DHCP_MESSAGE_TYPE;
    opt->length = 1;
    opt->val[0] = DHCPREQUEST;
    // DHCP client identifier
    opt = conv_option(resp->options + 3);
    opt->code   = DHO_DHCP_CLIENT_IDENTIFIER;
    opt->length = 7;
    opt->val[0] = HTYPE_ETHER;
    memcpy(&opt->val[1], &link_addr, ETH_ALEN);
    // DHCP server identifier
    opt = conv_option(resp->options + 12);
    opt->code   = DHO_DHCP_SERVER_IDENTIFIER;
    opt->length = 4;
    if (server_id) {
      memcpy(&opt->val[0], &server_id->val[0], sizeof(IP4::addr));
      MYINFO("Server IP set to %s", ((IP4::addr*)&opt->val[0])->to_string().c_str());
    } else {
      memcpy(&opt->val[0], &this->router, sizeof(IP4::addr));
      MYINFO("Server IP set to gateway (%s)", ((IP4::addr*)&opt->val[0])->to_string().c_str());
    }
    // DHCP Requested Address
    opt = conv_option(resp->options + 18);
    opt->code   = DHO_DHCP_REQUESTED_ADDRESS;
    opt->length = 4;
    memcpy(&opt->val[0], &this->ipaddr, sizeof(IP4::addr));
    // DHCP Parameter Request Field
    opt = conv_option(resp->options + 24);
    opt->code   = DHO_DHCP_PARAMETER_REQUEST_LIST;
    opt->length = 3;
    opt->val[0] = DHO_ROUTERS;
    opt->val[1] = DHO_SUBNET_MASK;
    opt->val[2] = DHO_DOMAIN_NAME_SERVERS;
    // END
    opt = conv_option(resp->options + 29);
    opt->code   = DHO_END;
    opt->length = 0;

    // set our onRead function to point to a hopeful DHCP ACK!
    sock.on_read(
    [this] (IP4::addr, UDP::port_t port,
            const char* data, size_t len)
    {
      if (port == DHCP_SERVER_PORT)
      {
        // we have hopefully got a DHCP Ack
        debug("\tReceived DHCP ACK from %s:%d\n",
              addr.str().c_str(), DHCP_SERVER_PORT);
        this->acknowledge(data, len);
      }
    });

    // send our DHCP Request
    sock.bcast(IP4::ADDR_ANY, DHCP_SERVER_PORT, packet, packetlen);
  }

  void DHClient::acknowledge(const char* data, size_t)
  {
    const dhcp_packet_t* dhcp = (const dhcp_packet_t*) data;

    uint32_t xid = htonl(dhcp->xid);
    // silently ignore transactions not our own
    if (xid != this->xid) return;

    // check if the BOOTP message is a DHCP OFFER
    const dhcp_option_t* opt;
    opt = get_option(dhcp->options, DHO_DHCP_MESSAGE_TYPE);

    if (opt->code == DHO_DHCP_MESSAGE_TYPE)
    {
      // verify that the type is indeed DHCPOFFER
      debug("\tFound DHCP message type %d  (DHCP Ack = %d)\n",
            opt->val[0], DHCPACK);
      // ignore when not a DHCP Offer
      if (opt->val[0] != DHCPACK) return;
    }
    // ignore message when DHCP message type is missing
    else return;

    if (console_spam)
      MYINFO("Server acknowledged our request!");

    // configure our network stack
    stack.network_config(this->ipaddr, this->netmask,
                         this->router, this->dns_server);
    // stop timeout from happening
    Timers::stop(timeout);

    in_progress = false;

    // run some post-DHCP event to release the hounds
    for (auto handler : config_handlers_)
      handler(false);
  }
}
<commit_msg>dh4client: xid (random session id) now based on uniform_int_distribution, not time<commit_after>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//#define DEBUG
#include <info>
#define MYINFO(X,...) INFO("DHCPv4",X,##__VA_ARGS__)

#include <net/dhcp/dh4client.hpp>
#include <net/dhcp/dhcp4.hpp>
#include <os> // OS::micros_since_boot()
#include <debug>

#include <random>

namespace net
{
  inline dhcp_option_t* conv_option(uint8_t* option)
  {
    return (dhcp_option_t*) option;
  }

  DHClient::DHClient(Stack& inet)
    : stack(inet), xid(0), console_spam(true), in_progress(false)
  {
    this->on_config(
    [this] (bool timeout)
    {
      if (console_spam) {
        if (timeout)
          INFO("DHCPv4", "Negotiation timed out");
        else
          INFO("DHCPv4", "Config complete");
      }
    });
  }

  void DHClient::on_config(config_func handler)
  {
    assert(handler);
    config_handlers_.push_back(handler);
  }

  void DHClient::negotiate(uint32_t timeout_secs)
  {
    // Allow multiple calls to negotiate without restarting the process
    if (in_progress) return;
    in_progress = true;

    // set timeout handler
    using namespace std::chrono;
    this->timeout = Timers::oneshot(seconds(timeout_secs),
    [this] (int) {
      // reset session ID
      this->xid = 0;
      this->in_progress = false;

      // call on_config with timeout = true
      for(auto handler : this->config_handlers_)
          handler(true);
    });

    // create a random session ID
    std::random_device rd;  // Will be used to obtain a seed for the random number engine
    std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd()
    std::uniform_int_distribution<> dis(std::numeric_limits<uint32_t>::min(), std::numeric_limits<uint32_t>::max());
    this->xid = dis(gen);

    if (console_spam)
      MYINFO("Negotiating IP-address (xid=%u)", xid);

    // create DHCP discover packet
    const size_t packetlen = sizeof(dhcp_packet_t);
    char packet[packetlen];

    dhcp_packet_t* dhcp = (dhcp_packet_t*) packet;
    dhcp->op    = BOOTREQUEST;
    dhcp->htype = HTYPE_ETHER;
    dhcp->hlen  = ETH_ALEN;
    dhcp->hops  = 0;
    dhcp->xid   = htonl(this->xid);
    dhcp->secs  = 0;
    dhcp->flags = htons(BOOTP_BROADCAST);
    dhcp->ciaddr = IP4::ADDR_ANY;
    dhcp->yiaddr = IP4::ADDR_ANY;
    dhcp->siaddr = IP4::ADDR_ANY;
    dhcp->giaddr = IP4::ADDR_ANY;

    Ethernet::addr link_addr = stack.link_addr();

    // copy our hardware address to chaddr field
    memset(dhcp->chaddr, 0, dhcp_packet_t::CHADDR_LEN);
    memcpy(dhcp->chaddr, &link_addr, ETH_ALEN);
    // zero server, file and options
    memset(dhcp->sname, 0, dhcp_packet_t::SNAME_LEN + dhcp_packet_t::FILE_LEN);

    dhcp->magic[0] =  99;
    dhcp->magic[1] = 130;
    dhcp->magic[2] =  83;
    dhcp->magic[3] =  99;

    dhcp_option_t* opt = conv_option(dhcp->options + 0);
    // DHCP discover
    opt->code   = DHO_DHCP_MESSAGE_TYPE;
    opt->length = 1;
    opt->val[0] = DHCPDISCOVER;
    // DHCP client identifier
    opt = conv_option(dhcp->options + 3);
    opt->code   = DHO_DHCP_CLIENT_IDENTIFIER;
    opt->length = 7;
    opt->val[0] = HTYPE_ETHER;
    memcpy(&opt->val[1], &link_addr, ETH_ALEN);
    // DHCP Parameter Request Field
    opt = conv_option(dhcp->options + 12);
    opt->code   = DHO_DHCP_PARAMETER_REQUEST_LIST;
    opt->length = 3;
    opt->val[0] = DHO_ROUTERS;
    opt->val[1] = DHO_SUBNET_MASK;
    opt->val[2] = DHO_DOMAIN_NAME_SERVERS;
    // END
    opt = conv_option(dhcp->options + 17);
    opt->code   = DHO_END;
    opt->length = 0;

    ////////////////////////////////////////////////////////
    auto& socket = stack.udp().bind(DHCP_CLIENT_PORT);
    /// broadcast our DHCP plea as 0.0.0.0:67
    socket.bcast(IP4::ADDR_ANY, DHCP_SERVER_PORT, packet, packetlen);

    socket.on_read(
    [this, &socket] (IP4::addr, UDP::port_t port,
                     const char* data, size_t len)
    {
      if (port == DHCP_SERVER_PORT)
      {
        // we have got a DHCP Offer
        debug("Received possible DHCP OFFER from %s:%d\n",
              addr.str().c_str(), DHCP_SERVER_PORT);
        this->offer(socket, data, len);
      }
    });
  }

  const dhcp_option_t* get_option(const uint8_t* options, uint8_t code)
  {
    const dhcp_option_t* opt = (const dhcp_option_t*) options;
    while (opt->code != code && opt->code != DHO_END)
      {
        // go to next option
        opt = (const dhcp_option_t*) (((const uint8_t*) opt) + 2 + opt->length);
      }
    return opt;
  }

  void DHClient::offer(UDPSocket& sock, const char* data, size_t)
  {
    const dhcp_packet_t* dhcp = (const dhcp_packet_t*) data;

    uint32_t xid = htonl(dhcp->xid);
    // silently ignore transactions not our own
    if (xid != this->xid) return;

    // check if the BOOTP message is a DHCP OFFER
    const dhcp_option_t* opt;
    const dhcp_option_t* server_id {nullptr};
    opt = get_option(dhcp->options, DHO_DHCP_MESSAGE_TYPE);

    if (opt->code == DHO_DHCP_MESSAGE_TYPE)
    {
      // verify that the type is indeed DHCPOFFER
      debug("Found DHCP message type %d  (DHCP Offer = %d)\n",
            opt->val[0], DHCPOFFER);

      // ignore when not a DHCP Offer
      if (opt->val[0] != DHCPOFFER) return;
    }
    // ignore message when DHCP message type is missing
    else return;

    // the offered IP address:
    this->ipaddr = dhcp->yiaddr;
    if (console_spam)
      MYINFO("IP ADDRESS: \t%s", this->ipaddr.str().c_str());

    opt = get_option(dhcp->options, DHO_SUBNET_MASK);
    if (opt->code == DHO_SUBNET_MASK)
    {
      memcpy(&this->netmask, opt->val, sizeof(IP4::addr));
      if (console_spam)
        MYINFO("SUBNET MASK: \t%s", this->netmask.str().c_str());
    }

    opt = get_option(dhcp->options, DHO_DHCP_LEASE_TIME);
    if (opt->code == DHO_DHCP_LEASE_TIME)
    {
      this->lease_time = (((uint32_t) opt->val[0]) << 24 |
        ((uint32_t) opt->val[1]) << 16 |
        ((uint32_t) opt->val[2]) <<  8 |
        ((uint32_t) opt->val[3]) <<  0);

      if (console_spam)
        MYINFO("LEASE TIME: \t%u mins", this->lease_time / 60);
    }

    // Preserve DHCP server address
    opt = get_option(dhcp->options, DHO_DHCP_SERVER_IDENTIFIER);
    if (opt->code == DHO_DHCP_SERVER_IDENTIFIER)
      {
        server_id = opt;
      }

    // now validate the offer, checking for minimum information
    opt = get_option(dhcp->options, DHO_ROUTERS);
    if (opt->code == DHO_ROUTERS)
    {
      memcpy(&this->router, opt->val, sizeof(IP4::addr));
      if (console_spam)
        MYINFO("GATEWAY: \t%s", this->router.str().c_str());
    }
    // assume that the server we received the request from is the gateway
    else
    {
      if (server_id)
      {
        memcpy(&this->router, server_id->val, sizeof(IP4::addr));
        if (console_spam)
          MYINFO("GATEWAY: \t%s", this->router.str().c_str());
      }
      // silently ignore when both ROUTER and SERVER_ID is missing
      else return;
    }

    opt = get_option(dhcp->options, DHO_DOMAIN_NAME_SERVERS);
    if (opt->code == DHO_DOMAIN_NAME_SERVERS)
    {
      memcpy(&this->dns_server, opt->val, sizeof(IP4::addr));
    }
    else
    { // just try using ROUTER as DNS server
      this->dns_server = this->router;
    }
    if (console_spam)
      MYINFO("DNS SERVER: \t%s", this->dns_server.str().c_str());

    // we can accept the offer now by requesting the IP!
    this->request(sock, server_id);
  }

  void DHClient::request(UDPSocket& sock, const dhcp_option_t* server_id)
  {
    // form a response
    const size_t packetlen = sizeof(dhcp_packet_t);
    char packet[packetlen];

    dhcp_packet_t* resp = (dhcp_packet_t*) packet;
    resp->op    = BOOTREQUEST;
    resp->htype = HTYPE_ETHER;
    resp->hlen  = ETH_ALEN;
    resp->hops  = 0;
    resp->xid   = htonl(this->xid);
    resp->secs  = 0;
    resp->flags = htons(BOOTP_BROADCAST);

    resp->ciaddr = IP4::ADDR_ANY;
    resp->yiaddr = IP4::ADDR_ANY;
    resp->siaddr = IP4::ADDR_ANY;
    resp->giaddr = IP4::ADDR_ANY;

    Ethernet::addr link_addr = stack.link_addr();

    // copy our hardware address to chaddr field
    memset(resp->chaddr, 0, dhcp_packet_t::CHADDR_LEN);
    memcpy(resp->chaddr, &link_addr, ETH_ALEN);
    // zero server, file and options
    memset(resp->sname, 0, dhcp_packet_t::SNAME_LEN + dhcp_packet_t::FILE_LEN);
    // magic DHCP bootp values
    resp->magic[0] =  99;
    resp->magic[1] = 130;
    resp->magic[2] =  83;
    resp->magic[3] =  99;

    dhcp_option_t* opt = conv_option(resp->options + 0);
    // DHCP Request
    opt->code   = DHO_DHCP_MESSAGE_TYPE;
    opt->length = 1;
    opt->val[0] = DHCPREQUEST;
    // DHCP client identifier
    opt = conv_option(resp->options + 3);
    opt->code   = DHO_DHCP_CLIENT_IDENTIFIER;
    opt->length = 7;
    opt->val[0] = HTYPE_ETHER;
    memcpy(&opt->val[1], &link_addr, ETH_ALEN);
    // DHCP server identifier
    opt = conv_option(resp->options + 12);
    opt->code   = DHO_DHCP_SERVER_IDENTIFIER;
    opt->length = 4;
    if (server_id) {
      memcpy(&opt->val[0], &server_id->val[0], sizeof(IP4::addr));
      MYINFO("Server IP set to %s", ((IP4::addr*)&opt->val[0])->to_string().c_str());
    } else {
      memcpy(&opt->val[0], &this->router, sizeof(IP4::addr));
      MYINFO("Server IP set to gateway (%s)", ((IP4::addr*)&opt->val[0])->to_string().c_str());
    }
    // DHCP Requested Address
    opt = conv_option(resp->options + 18);
    opt->code   = DHO_DHCP_REQUESTED_ADDRESS;
    opt->length = 4;
    memcpy(&opt->val[0], &this->ipaddr, sizeof(IP4::addr));
    // DHCP Parameter Request Field
    opt = conv_option(resp->options + 24);
    opt->code   = DHO_DHCP_PARAMETER_REQUEST_LIST;
    opt->length = 3;
    opt->val[0] = DHO_ROUTERS;
    opt->val[1] = DHO_SUBNET_MASK;
    opt->val[2] = DHO_DOMAIN_NAME_SERVERS;
    // END
    opt = conv_option(resp->options + 29);
    opt->code   = DHO_END;
    opt->length = 0;

    // set our onRead function to point to a hopeful DHCP ACK!
    sock.on_read(
    [this] (IP4::addr, UDP::port_t port,
            const char* data, size_t len)
    {
      if (port == DHCP_SERVER_PORT)
      {
        // we have hopefully got a DHCP Ack
        debug("\tReceived DHCP ACK from %s:%d\n",
              addr.str().c_str(), DHCP_SERVER_PORT);
        this->acknowledge(data, len);
      }
    });

    // send our DHCP Request
    sock.bcast(IP4::ADDR_ANY, DHCP_SERVER_PORT, packet, packetlen);
  }

  void DHClient::acknowledge(const char* data, size_t)
  {
    const dhcp_packet_t* dhcp = (const dhcp_packet_t*) data;

    uint32_t xid = htonl(dhcp->xid);
    // silently ignore transactions not our own
    if (xid != this->xid) return;

    // check if the BOOTP message is a DHCP OFFER
    const dhcp_option_t* opt;
    opt = get_option(dhcp->options, DHO_DHCP_MESSAGE_TYPE);

    if (opt->code == DHO_DHCP_MESSAGE_TYPE)
    {
      // verify that the type is indeed DHCPOFFER
      debug("\tFound DHCP message type %d  (DHCP Ack = %d)\n",
            opt->val[0], DHCPACK);
      // ignore when not a DHCP Offer
      if (opt->val[0] != DHCPACK) return;
    }
    // ignore message when DHCP message type is missing
    else return;

    if (console_spam)
      MYINFO("Server acknowledged our request!");

    // configure our network stack
    stack.network_config(this->ipaddr, this->netmask,
                         this->router, this->dns_server);
    // stop timeout from happening
    Timers::stop(timeout);

    in_progress = false;

    // run some post-DHCP event to release the hounds
    for (auto handler : config_handlers_)
      handler(false);
  }
}
<|endoftext|>


<commit_before>// Copyright NVIDIA Corporation 2007 -- Ignacio Castano <icastano@nvidia.com>
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.

#include "CompressorRGB.h"
#include "CompressionOptions.h"
#include "OutputOptions.h"

#include <nvimage/Image.h>
#include <nvimage/FloatImage.h>
#include <nvimage/PixelFormat.h>

#include <nvmath/Color.h>
#include <nvmath/Half.h>

#include <nvcore/Debug.h>

using namespace nv;
using namespace nvtt;

namespace
{

	inline uint computePitch(uint w, uint bitsize)
	{
		uint p = w * ((bitsize + 7) / 8);

		// Align to 32 bits.
		return ((p + 3) / 4) * 4;
	}

	inline void convert_to_a8r8g8b8(const void * src, void * dst, uint w)
	{
		memcpy(dst, src, 4 * w);
	}

	inline void convert_to_x8r8g8b8(const void * src, void * dst, uint w)
	{
		memcpy(dst, src, 4 * w);
	}

} // namespace

void PixelFormatConverter::compress(nvtt::InputFormat inputFormat, nvtt::AlphaMode alphaMode, uint w, uint h, const void * data, const nvtt::CompressionOptions::Private & compressionOptions, const nvtt::OutputOptions::Private & outputOptions)
{
	uint bitCount;
	uint rmask, rshift, rsize;
	uint gmask, gshift, gsize;
	uint bmask, bshift, bsize;
	uint amask, ashift, asize;

    if (compressionOptions.pixelType == nvtt::PixelType_Float)
    {
	    rsize = compressionOptions.rsize;
	    gsize = compressionOptions.gsize;
	    bsize = compressionOptions.bsize;
	    asize = compressionOptions.asize;

	    nvCheck(rsize == 0 || rsize == 16 || rsize == 32);
	    nvCheck(gsize == 0 || gsize == 16 || gsize == 32);
	    nvCheck(bsize == 0 || bsize == 16 || bsize == 32);
	    nvCheck(asize == 0 || asize == 16 || asize == 32);

	    bitCount = rsize + gsize + bsize + asize;
    }
    else
    {
	    if (compressionOptions.bitcount != 0)
	    {
		    bitCount = compressionOptions.bitcount;
		    nvCheck(bitCount == 8 || bitCount == 16 || bitCount == 24 || bitCount == 32);

		    rmask = compressionOptions.rmask;
		    gmask = compressionOptions.gmask;
		    bmask = compressionOptions.bmask;
		    amask = compressionOptions.amask;

		    PixelFormat::maskShiftAndSize(rmask, &rshift, &rsize);
		    PixelFormat::maskShiftAndSize(gmask, &gshift, &gsize);
		    PixelFormat::maskShiftAndSize(bmask, &bshift, &bsize);
		    PixelFormat::maskShiftAndSize(amask, &ashift, &asize);
	    }
	    else
	    {
		    rsize = compressionOptions.rsize;
		    gsize = compressionOptions.gsize;
		    bsize = compressionOptions.bsize;
		    asize = compressionOptions.asize;

		    bitCount = rsize + gsize + bsize + asize;
		    nvCheck(bitCount <= 32);

		    ashift = 0;
		    bshift = ashift + asize;
		    gshift = bshift + bsize;
		    rshift = gshift + gsize;

		    rmask = ((1 << rsize) - 1) << rshift;
		    gmask = ((1 << gsize) - 1) << gshift;
		    bmask = ((1 << bsize) - 1) << bshift;
		    amask = ((1 << asize) - 1) << ashift;
	    }
    }

	uint byteCount = (bitCount + 7) / 8;
    uint pitch = computePitch(w, bitCount);

    uint srcPitchDiv4 = w;
    if (inputFormat == nvtt::InputFormat_RGBA_32F) srcPitchDiv4 = w * 4;

    // Allocate output scanline.
	uint8 * dst = (uint8 *)mem::malloc(pitch + 4);

	for (uint y = 0; y < h; y++)
	{
        const uint * src = (const uint *)data + srcPitchDiv4;

		for (uint x = 0; x < w; x++)
		{
            float r, g, b, a;

            if (inputFormat == nvtt::InputFormat_BGRA_8UB) {
                Color32 c = Color32(src[x]);
                r = float(c.r) / 255.0f;
                g = float(c.g) / 255.0f;
                b = float(c.b) / 255.0f;
                a = float(c.a) / 255.0f;
            }
            else /*if (inputFormat == nvtt::InputFormat_RGBA_32F)*/ {
			    r = src[4 * x + 0];
			    g = src[4 * x + 1];
			    b = src[4 * x + 2];
			    a = src[4 * x + 3];
            }

            if (compressionOptions.pixelType == nvtt::PixelType_Float)
            {

            }

        }

		if (outputOptions.outputHandler != NULL)
		{
			outputOptions.outputHandler->writeData(dst, pitch);
		}
    }

	mem::free(dst);
}

// Pixel format converter.
void compressRGB(const Image * image, const OutputOptions::Private & outputOptions, const CompressionOptions::Private & compressionOptions)
{
	nvCheck(image != NULL);

	const uint w = image->width();
	const uint h = image->height();

	uint bitCount;
	uint rmask, rshift, rsize;
	uint gmask, gshift, gsize;
	uint bmask, bshift, bsize;
	uint amask, ashift, asize;

	if (compressionOptions.bitcount != 0)
	{
		bitCount = compressionOptions.bitcount;
		nvCheck(bitCount == 8 || bitCount == 16 || bitCount == 24 || bitCount == 32);

		rmask = compressionOptions.rmask;
		gmask = compressionOptions.gmask;
		bmask = compressionOptions.bmask;
		amask = compressionOptions.amask;

		PixelFormat::maskShiftAndSize(rmask, &rshift, &rsize);
		PixelFormat::maskShiftAndSize(gmask, &gshift, &gsize);
		PixelFormat::maskShiftAndSize(bmask, &bshift, &bsize);
		PixelFormat::maskShiftAndSize(amask, &ashift, &asize);
	}
	else
	{
		rsize = compressionOptions.rsize;
		gsize = compressionOptions.gsize;
		bsize = compressionOptions.bsize;
		asize = compressionOptions.asize;

		bitCount = rsize + gsize + bsize + asize;
		nvCheck(bitCount <= 32);

		ashift = 0;
		bshift = ashift + asize;
		gshift = bshift + bsize;
		rshift = gshift + gsize;

		rmask = ((1 << rsize) - 1) << rshift;
		gmask = ((1 << gsize) - 1) << gshift;
		bmask = ((1 << bsize) - 1) << bshift;
		amask = ((1 << asize) - 1) << ashift;
	}

	const uint byteCount = bitCount / 8;

	// Determine pitch.
	uint pitch = computePitch(w, bitCount);

	uint8 * dst = (uint8 *)mem::malloc(pitch + 4);

	for (uint y = 0; y < h; y++)
	{
		const Color32 * src = image->scanline(y);

		if (bitCount == 32 && rmask == 0xFF0000 && gmask == 0xFF00 && bmask == 0xFF && amask == 0xFF000000)
		{
			convert_to_a8r8g8b8(src, dst, w);
		}
		else if (bitCount == 32 && rmask == 0xFF0000 && gmask == 0xFF00 && bmask == 0xFF && amask == 0)
		{
			convert_to_x8r8g8b8(src, dst, w);
		}
		else
		{
			// Generic pixel format conversion.
			for (uint x = 0; x < w; x++)
			{
				uint c = 0;
				c |= PixelFormat::convert(src[x].r, 8, rsize) << rshift;
				c |= PixelFormat::convert(src[x].g, 8, gsize) << gshift;
				c |= PixelFormat::convert(src[x].b, 8, bsize) << bshift;
				c |= PixelFormat::convert(src[x].a, 8, asize) << ashift;

				// Output one byte at a time.
				for (uint i = 0; i < byteCount; i++)
				{
					*(dst + x * byteCount + i) = (c >> (i * 8)) & 0xFF;
				}
			}

			// Zero padding.
			for (uint x = w * byteCount; x < pitch; x++)
			{
				*(dst + x) = 0;
			}
		}

		if (outputOptions.outputHandler != NULL)
		{
			outputOptions.outputHandler->writeData(dst, pitch);
		}
	}

	mem::free(dst);
}

void compressRGB(const FloatImage * image, const OutputOptions::Private & outputOptions, const CompressionOptions::Private & compressionOptions)
{
	nvCheck(image != NULL);

	const uint w = image->width();
	const uint h = image->height();

	const uint rsize = compressionOptions.rsize;
	const uint gsize = compressionOptions.gsize;
	const uint bsize = compressionOptions.bsize;
	const uint asize = compressionOptions.asize;

	nvCheck(rsize == 0 || rsize == 16 || rsize == 32);
	nvCheck(gsize == 0 || gsize == 16 || gsize == 32);
	nvCheck(bsize == 0 || bsize == 16 || bsize == 32);
	nvCheck(asize == 0 || asize == 16 || asize == 32);

	const uint bitCount = rsize + gsize + bsize + asize;
	const uint byteCount = bitCount / 8;
	const uint pitch = w * byteCount;

	uint8 * dst = (uint8 *)mem::malloc(pitch);

	for (uint y = 0; y < h; y++)
	{
		const float * rchannel = image->scanline(y, 0);
		const float * gchannel = image->scanline(y, 1);
		const float * bchannel = image->scanline(y, 2);
		const float * achannel = image->scanline(y, 3);

		union FLOAT
		{
			float f;
			uint32 u;
		};

		uint8 * ptr = dst;

		for (uint x = 0; x < w; x++)
		{
			FLOAT r, g, b, a;
			r.f = rchannel[x];
			g.f = gchannel[x];
			b.f = bchannel[x];
			a.f = achannel[x];

			if (rsize == 32) *((uint32 *)ptr) = r.u;
			else if (rsize == 16) *((uint16 *)ptr) = half_from_float(r.u);
			ptr += rsize / 8;

			if (gsize == 32) *((uint32 *)ptr) = g.u;
			else if (gsize == 16) *((uint16 *)ptr) = half_from_float(g.u);
			ptr += gsize / 8;

			if (bsize == 32) *((uint32 *)ptr) = b.u;
			else if (bsize == 16) *((uint16 *)ptr) = half_from_float(b.u);
			ptr += bsize / 8;

			if (asize == 32) *((uint32 *)ptr) = a.u;
			else if (asize == 16) *((uint16 *)ptr) = half_from_float(a.u);
			ptr += asize / 8;
		}

		if (outputOptions.outputHandler != NULL)
		{
			outputOptions.outputHandler->writeData(dst, pitch);
		}
	}

	mem::free(dst);
}
<commit_msg>Finish pixel format converter.<commit_after>// Copyright NVIDIA Corporation 2007 -- Ignacio Castano <icastano@nvidia.com>
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.

#include "CompressorRGB.h"
#include "CompressionOptions.h"
#include "OutputOptions.h"

#include <nvimage/Image.h>
#include <nvimage/FloatImage.h>
#include <nvimage/PixelFormat.h>

#include <nvmath/Color.h>
#include <nvmath/Half.h>

#include <nvcore/Debug.h>

using namespace nv;
using namespace nvtt;

namespace
{

	inline uint computePitch(uint w, uint bitsize)
	{
		uint p = w * ((bitsize + 7) / 8);

		// Align to 32 bits.
		return ((p + 3) / 4) * 4;
	}

	inline void convert_to_a8r8g8b8(const void * src, void * dst, uint w)
	{
		memcpy(dst, src, 4 * w);
	}

	inline void convert_to_x8r8g8b8(const void * src, void * dst, uint w)
	{
		memcpy(dst, src, 4 * w);
	}

    static uint16 to_half(float f)
    {
	    union { float f; uint32 u; } c;
        c.f = f;
        return half_from_float(c.u);
    }

} // namespace

void PixelFormatConverter::compress(nvtt::InputFormat inputFormat, nvtt::AlphaMode alphaMode, uint w, uint h, const void * data, const nvtt::CompressionOptions::Private & compressionOptions, const nvtt::OutputOptions::Private & outputOptions)
{
	uint bitCount;
	uint rmask, rshift, rsize;
	uint gmask, gshift, gsize;
	uint bmask, bshift, bsize;
	uint amask, ashift, asize;

    if (compressionOptions.pixelType == nvtt::PixelType_Float)
    {
	    rsize = compressionOptions.rsize;
	    gsize = compressionOptions.gsize;
	    bsize = compressionOptions.bsize;
	    asize = compressionOptions.asize;

	    nvCheck(rsize == 0 || rsize == 16 || rsize == 32);
	    nvCheck(gsize == 0 || gsize == 16 || gsize == 32);
	    nvCheck(bsize == 0 || bsize == 16 || bsize == 32);
	    nvCheck(asize == 0 || asize == 16 || asize == 32);

	    bitCount = rsize + gsize + bsize + asize;
    }
    else
    {
	    if (compressionOptions.bitcount != 0)
	    {
		    bitCount = compressionOptions.bitcount;
		    nvCheck(bitCount == 8 || bitCount == 16 || bitCount == 24 || bitCount == 32);

		    rmask = compressionOptions.rmask;
		    gmask = compressionOptions.gmask;
		    bmask = compressionOptions.bmask;
		    amask = compressionOptions.amask;

		    PixelFormat::maskShiftAndSize(rmask, &rshift, &rsize);
		    PixelFormat::maskShiftAndSize(gmask, &gshift, &gsize);
		    PixelFormat::maskShiftAndSize(bmask, &bshift, &bsize);
		    PixelFormat::maskShiftAndSize(amask, &ashift, &asize);
	    }
	    else
	    {
		    rsize = compressionOptions.rsize;
		    gsize = compressionOptions.gsize;
		    bsize = compressionOptions.bsize;
		    asize = compressionOptions.asize;

		    bitCount = rsize + gsize + bsize + asize;
		    nvCheck(bitCount <= 32);

		    ashift = 0;
		    bshift = ashift + asize;
		    gshift = bshift + bsize;
		    rshift = gshift + gsize;

		    rmask = ((1 << rsize) - 1) << rshift;
		    gmask = ((1 << gsize) - 1) << gshift;
		    bmask = ((1 << bsize) - 1) << bshift;
		    amask = ((1 << asize) - 1) << ashift;
	    }
    }

	uint byteCount = (bitCount + 7) / 8;
    uint pitch = computePitch(w, bitCount);

    uint srcPitch = w;
    uint srcPlane = w * h;

    // Allocate output scanline.
	uint8 * dst = (uint8 *)mem::malloc(pitch + 4);

	for (uint y = 0; y < h; y++)
	{
        const uint * src = (const uint *)data + y * srcPitch;
        const float * fsrc = (const float *)data + y * srcPitch;

        uint8 * ptr = dst;

		for (uint x = 0; x < w; x++)
		{
            float r, g, b, a;

            if (inputFormat == nvtt::InputFormat_BGRA_8UB) {
                Color32 c = Color32(src[x]);
                r = float(c.r) / 255.0f;
                g = float(c.g) / 255.0f;
                b = float(c.b) / 255.0f;
                a = float(c.a) / 255.0f;
            }
            else {
                nvDebugCheck (inputFormat == nvtt::InputFormat_RGBA_32F);

			    //r = ((float *)src)[4 * x + 0]; // Color components not interleaved.
			    //g = ((float *)src)[4 * x + 1];
			    //b = ((float *)src)[4 * x + 2];
			    //a = ((float *)src)[4 * x + 3];
			    r = fsrc[x + 0 * srcPlane];
			    g = fsrc[x + 1 * srcPlane];
			    b = fsrc[x + 2 * srcPlane];
			    a = fsrc[x + 3 * srcPlane];
            }

            if (compressionOptions.pixelType == nvtt::PixelType_Float)
            {
			    if (rsize == 32) *((float *)ptr) = r;
			    else if (rsize == 16) *((uint16 *)ptr) = to_half(r);
			    ptr += rsize / 8;

			    if (gsize == 32) *((float *)ptr) = g;
			    else if (gsize == 16) *((uint16 *)ptr) = to_half(g);
			    ptr += gsize / 8;

			    if (bsize == 32) *((float *)ptr) = b;
			    else if (bsize == 16) *((uint16 *)ptr) = to_half(b);
			    ptr += bsize / 8;

			    if (asize == 32) *((float *)ptr) = a;
			    else if (asize == 16) *((uint16 *)ptr) = to_half(a);
			    ptr += asize / 8;
            }
            else
            {
                Color32 c;
                if (compressionOptions.pixelType == nvtt::PixelType_UnsignedNorm) {
                    c.r = uint8(clamp(r * 255, 0.0f, 255.0f));
                    c.g = uint8(clamp(g * 255, 0.0f, 255.0f));
                    c.b = uint8(clamp(b * 255, 0.0f, 255.0f));
                    c.a = uint8(clamp(a * 255, 0.0f, 255.0f));
                }
                // @@ Add support for nvtt::PixelType_SignedInt, nvtt::PixelType_SignedNorm, nvtt::PixelType_UnsignedInt

				uint p = 0;
				p |= PixelFormat::convert(c.r, 8, rsize) << rshift;
				p |= PixelFormat::convert(c.g, 8, gsize) << gshift;
				p |= PixelFormat::convert(c.b, 8, bsize) << bshift;
				p |= PixelFormat::convert(c.a, 8, asize) << ashift;

				// Output one byte at a time.
				for (uint i = 0; i < byteCount; i++)
				{
					*(dst + x * byteCount + i) = (p >> (i * 8)) & 0xFF;
				}
            }
        }

		// Zero padding.
		for (uint x = w * byteCount; x < pitch; x++)
		{
			*(dst + x) = 0;
		}

		if (outputOptions.outputHandler != NULL)
		{
			outputOptions.outputHandler->writeData(dst, pitch);
		}
    }

	mem::free(dst);
}
<|endoftext|>


<commit_before>#include "omnicore/wallettxs.h"

#include "omnicore/log.h"
#include "omnicore/omnicore.h"
#include "omnicore/rules.h"
#include "omnicore/script.h"
#include "omnicore/utilsbitcoin.h"

#include "amount.h"
#include "base58.h"
#include "coincontrol.h"
#include "init.h"
#include "main.h"
#include "pubkey.h"
#include "script/standard.h"
#include "sync.h"
#include "txmempool.h"
#include "uint256.h"
#include "utilstrencodings.h"
#ifdef ENABLE_WALLET
#include "script/ismine.h"
#include "wallet/wallet.h"
#endif

#include <stdint.h>
#include <map>
#include <string>

namespace mastercore
{
/**
 * Retrieves a public key from the wallet, or converts a hex-string to a public key.
 */
bool AddressToPubKey(const std::string& key, CPubKey& pubKey)
{
#ifdef ENABLE_WALLET
    // Case 1: Bitcoin address and the key is in the wallet
    CBitcoinAddress address(key);
    if (pwalletMain && address.IsValid()) {
        CKeyID keyID;
        if (!address.GetKeyID(keyID)) {
            PrintToLog("%s() ERROR: redemption address %s does not refer to a public key\n", __func__, key);
            return false;
        }
        if (!pwalletMain->GetPubKey(keyID, pubKey)) {
            PrintToLog("%s() ERROR: no public key in wallet for redemption address %s\n", __func__, key);
            return false;
        }
    }
    // Case 2: Hex-encoded public key
    else
#endif
    if (IsHex(key)) {
        pubKey = CPubKey(ParseHex(key));
    }

    if (!pubKey.IsFullyValid()) {
        PrintToLog("%s() ERROR: invalid redemption key %s\n", __func__, key);
        return false;
    }

    return true;
}

/**
 * Checks, whether enough spendable outputs are available to pay for transaction fees.
 */
bool CheckFee(const std::string& fromAddress, size_t nDataSize)
{
    int64_t minFee = 0;
    int64_t feePerKB = 0;
    int64_t inputTotal = 0;
#ifdef ENABLE_WALLET
    bool fUseClassC = UseEncodingClassC(nDataSize);

    CCoinControl coinControl;
    inputTotal = SelectCoins(fromAddress, coinControl, 0);

    // calculate the estimated fee per KB based on the currently set confirm target
    CFeeRate feeRate = mempool.estimateFee(nTxConfirmTarget);

    // if there is not enough data (and zero is estimated) then base minimum on a fairly high/safe 50,000 satoshi fee per KB
    if (feeRate == CFeeRate(0)) {
        feePerKB = 50000;
    } else {
        feePerKB = feeRate.GetFeePerK();
    }

    // we do not know the size of the transaction at this point.  Warning calculation employs some guesswork
    if (!fUseClassC) {
        // Calculation based on a 3KB transaction due to:
        //   - the average size across all Class B transactions ever sent to date is 909 bytes.
        //   - under 2% of Class B transactions are over 2KB, under 0.6% of transactions are over 3KB.
        // Thus if created transaction will be over 3KB (rare as per above) warning may not be sufficient.
        minFee = feePerKB * 3;
    } else {
        // Averages for Class C transactions are not yet available, Calculation based on a 2KB transaction due to:
        //   - Class B values but considering Class C removes outputs for both data and Exodus (reduces size).
        minFee = feePerKB * 2;
    }
#endif
    return inputTotal >= minFee;
}

/**
 * Checks, whether the output qualifies as input for a transaction.
 */
bool CheckInput(const CTxOut& txOut, int nHeight, CTxDestination& dest)
{
    txnouttype whichType;

    if (!GetOutputType(txOut.scriptPubKey, whichType)) {
        return false;
    }
    if (!IsAllowedInputType(whichType, nHeight)) {
        return false;
    }
    if (!ExtractDestination(txOut.scriptPubKey, dest)) {
        return false;
    }

    return true;
}

/**
 * Retrieves the label, used by the UI, for an address from the wallet.
 */
std::string GetAddressLabel(const std::string& address)
{
    std::string addressLabel;
#ifdef ENABLE_WALLET
    if (pwalletMain) {
        LOCK(pwalletMain->cs_wallet);

        CBitcoinAddress addressParsed(address);
        std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwalletMain->mapAddressBook.find(addressParsed.Get());
        if (mi != pwalletMain->mapAddressBook.end()) {
            addressLabel = mi->second.name;
        }
    }
#endif
    return addressLabel;
}

/**
 * IsMine wrapper to determine whether the address is in the wallet.
 */
int IsMyAddress(const std::string& address)
{
#ifdef ENABLE_WALLET
    if (pwalletMain) {
        // TODO: resolve deadlock caused cs_tally, cs_wallet
        // LOCK(pwalletMain->cs_wallet);
        CBitcoinAddress parsedAddress(address);
        isminetype isMine = IsMine(*pwalletMain, parsedAddress.Get());

        return static_cast<int>(isMine);
    }
#endif
    return 0;
}

/**
 * Selects spendable outputs to create a transaction.
 */
int64_t SelectCoins(const std::string& fromAddress, CCoinControl& coinControl, int64_t additional)
{
    // total output funds collected
    int64_t nTotal = 0;

#ifdef ENABLE_WALLET
    if (NULL == pwalletMain) {
        return 0;
    }

    // assume 20 KB max. transaction size at 0.0001 per kilobyte
    int64_t nMax = (COIN * (20 * (0.0001)));

    // if referenceamount is set it is needed to be accounted for here too
    if (0 < additional) nMax += additional;

    int nHeight = GetHeight();
    LOCK2(cs_main, pwalletMain->cs_wallet);

    // iterate over the wallet
    for (std::map<uint256, CWalletTx>::const_iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) {
        const uint256& txid = it->first;
        const CWalletTx& wtx = it->second;

        if (!wtx.IsTrusted()) {
            continue;
        }
        if (!wtx.GetAvailableCredit()) {
            continue;
        }

        for (unsigned int n = 0; n < wtx.vout.size(); n++) {
            const CTxOut& txOut = wtx.vout[n];

            CTxDestination dest;
            if (!CheckInput(txOut, nHeight, dest)) {
                continue;
            }
            if (!IsMine(*pwalletMain, dest)) {
                continue;
            }
            if (pwalletMain->IsSpent(txid, n)) {
                continue;
            }

            std::string sAddress = CBitcoinAddress(dest).ToString();
            if (msc_debug_tokens)
                PrintToLog("%s(): sender: %s, outpoint: %s:%d, value: %d\n", __func__, sAddress, txid.GetHex(), n, txOut.nValue);

            // only use funds from the sender's address
            if (fromAddress == sAddress) {
                COutPoint outpoint(txid, n);
                coinControl.Select(outpoint);

                nTotal += txOut.nValue;

                if (nMax <= nTotal) break;
            }
        }

        if (nMax <= nTotal) break;
    }
#endif

    return nTotal;
}

} // namespace mastercore
<commit_msg>Use estimated fees to filter and create Omni wallet transactions<commit_after>#include "omnicore/wallettxs.h"

#include "omnicore/log.h"
#include "omnicore/omnicore.h"
#include "omnicore/rules.h"
#include "omnicore/script.h"
#include "omnicore/utilsbitcoin.h"

#include "amount.h"
#include "base58.h"
#include "coincontrol.h"
#include "init.h"
#include "main.h"
#include "pubkey.h"
#include "script/standard.h"
#include "sync.h"
#include "txmempool.h"
#include "uint256.h"
#include "utilstrencodings.h"
#ifdef ENABLE_WALLET
#include "script/ismine.h"
#include "wallet/wallet.h"
#endif

#include <stdint.h>
#include <algorithm>
#include <map>
#include <string>

namespace mastercore
{
/**
 * Retrieves a public key from the wallet, or converts a hex-string to a public key.
 */
bool AddressToPubKey(const std::string& key, CPubKey& pubKey)
{
#ifdef ENABLE_WALLET
    // Case 1: Bitcoin address and the key is in the wallet
    CBitcoinAddress address(key);
    if (pwalletMain && address.IsValid()) {
        CKeyID keyID;
        if (!address.GetKeyID(keyID)) {
            PrintToLog("%s() ERROR: redemption address %s does not refer to a public key\n", __func__, key);
            return false;
        }
        if (!pwalletMain->GetPubKey(keyID, pubKey)) {
            PrintToLog("%s() ERROR: no public key in wallet for redemption address %s\n", __func__, key);
            return false;
        }
    }
    // Case 2: Hex-encoded public key
    else
#endif
    if (IsHex(key)) {
        pubKey = CPubKey(ParseHex(key));
    }

    if (!pubKey.IsFullyValid()) {
        PrintToLog("%s() ERROR: invalid redemption key %s\n", __func__, key);
        return false;
    }

    return true;
}

/**
 * Checks, whether enough spendable outputs are available to pay for transaction fees.
 */
bool CheckFee(const std::string& fromAddress, size_t nDataSize)
{
    int64_t minFee = 0;
    int64_t feePerKB = 0;
    int64_t inputTotal = 0;
#ifdef ENABLE_WALLET
    bool fUseClassC = UseEncodingClassC(nDataSize);

    CCoinControl coinControl;
    inputTotal = SelectCoins(fromAddress, coinControl, 0);

    // calculate the estimated fee per KB based on the currently set confirm target
    CFeeRate feeRate = mempool.estimateFee(nTxConfirmTarget);

    // if there is not enough data (and zero is estimated) then base minimum on a fairly high/safe 50,000 satoshi fee per KB
    if (feeRate == CFeeRate(0)) {
        feePerKB = 50000;
    } else {
        feePerKB = feeRate.GetFeePerK();
    }

    // we do not know the size of the transaction at this point.  Warning calculation employs some guesswork
    if (!fUseClassC) {
        // Calculation based on a 3KB transaction due to:
        //   - the average size across all Class B transactions ever sent to date is 909 bytes.
        //   - under 2% of Class B transactions are over 2KB, under 0.6% of transactions are over 3KB.
        // Thus if created transaction will be over 3KB (rare as per above) warning may not be sufficient.
        minFee = feePerKB * 3;
    } else {
        // Averages for Class C transactions are not yet available, Calculation based on a 2KB transaction due to:
        //   - Class B values but considering Class C removes outputs for both data and Exodus (reduces size).
        minFee = feePerKB * 2;
    }
#endif
    return inputTotal >= minFee;
}

/**
 * Checks, whether the output qualifies as input for a transaction.
 */
bool CheckInput(const CTxOut& txOut, int nHeight, CTxDestination& dest)
{
    txnouttype whichType;

    if (!GetOutputType(txOut.scriptPubKey, whichType)) {
        return false;
    }
    if (!IsAllowedInputType(whichType, nHeight)) {
        return false;
    }
    if (!ExtractDestination(txOut.scriptPubKey, dest)) {
        return false;
    }

    return true;
}

/**
 * Retrieves the label, used by the UI, for an address from the wallet.
 */
std::string GetAddressLabel(const std::string& address)
{
    std::string addressLabel;
#ifdef ENABLE_WALLET
    if (pwalletMain) {
        LOCK(pwalletMain->cs_wallet);

        CBitcoinAddress addressParsed(address);
        std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwalletMain->mapAddressBook.find(addressParsed.Get());
        if (mi != pwalletMain->mapAddressBook.end()) {
            addressLabel = mi->second.name;
        }
    }
#endif
    return addressLabel;
}

/**
 * IsMine wrapper to determine whether the address is in the wallet.
 */
int IsMyAddress(const std::string& address)
{
#ifdef ENABLE_WALLET
    if (pwalletMain) {
        // TODO: resolve deadlock caused cs_tally, cs_wallet
        // LOCK(pwalletMain->cs_wallet);
        CBitcoinAddress parsedAddress(address);
        isminetype isMine = IsMine(*pwalletMain, parsedAddress.Get());

        return static_cast<int>(isMine);
    }
#endif
    return 0;
}

/**
 * Estimate the minimum fee considering user set parameters and the required fee.
 *
 * @return The estimated fee per 1000 byte, or 50000 satoshi as per default
 */
static int64_t GetEstimatedFeePerKb()
{
    int64_t nFee = 50000; // 0.0005 BTC;

#ifdef ENABLE_WALLET
    if (pwalletMain) {
        nFee = pwalletMain->GetMinimumFee(1000, nTxConfirmTarget, mempool);
    }
#endif

    return nFee;
}

/**
 * Output values below this value are considered uneconomic, because it would
 * require more fees to pay than the output is worth.
 *
 * @param txOut[in]  The transaction output to check
 * @return The minimum value an output to spent should have
 */
static int64_t GetEconomicThreshold(const CTxOut& txOut)
{
    // Minimum value needed to relay the transaction
    int64_t nThresholdDust = txOut.GetDustThreshold(minRelayTxFee);

    // Use the estimated fee that is also used to contruct transactions.
    // We use the absolute minimum, so we divide by 3, to get rid of the
    // safety margin used for the dust threshold used for relaying.
    CFeeRate estimatedFeeRate(GetEstimatedFeePerKb());
    int64_t nThresholdFees = txOut.GetDustThreshold(estimatedFeeRate) / 3;

    return std::max(nThresholdDust, nThresholdFees);
}

/**
 * Selects spendable outputs to create a transaction.
 */
int64_t SelectCoins(const std::string& fromAddress, CCoinControl& coinControl, int64_t additional)
{
    // total output funds collected
    int64_t nTotal = 0;

#ifdef ENABLE_WALLET
    if (NULL == pwalletMain) {
        return 0;
    }

    // select coins to cover up to 20 kB max. transaction size
    int64_t nMax = 20 * GetEstimatedFeePerKb();

    // if referenceamount is set it is needed to be accounted for here too
    if (0 < additional) nMax += additional;

    int nHeight = GetHeight();
    LOCK2(cs_main, pwalletMain->cs_wallet);

    // iterate over the wallet
    for (std::map<uint256, CWalletTx>::const_iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) {
        const uint256& txid = it->first;
        const CWalletTx& wtx = it->second;

        if (!wtx.IsTrusted()) {
            continue;
        }
        if (!wtx.GetAvailableCredit()) {
            continue;
        }

        for (unsigned int n = 0; n < wtx.vout.size(); n++) {
            const CTxOut& txOut = wtx.vout[n];

            CTxDestination dest;
            if (!CheckInput(txOut, nHeight, dest)) {
                continue;
            }
            if (!IsMine(*pwalletMain, dest)) {
                continue;
            }
            if (pwalletMain->IsSpent(txid, n)) {
                continue;
            }
            if (txOut.nValue < GetEconomicThreshold(txOut)) {
                if (msc_debug_tokens)
                    PrintToLog("%s: output value below economic threshold: %s:%d, value: %d\n",
                            __func__, txid.GetHex(), n, txOut.nValue);
                continue;
            }

            std::string sAddress = CBitcoinAddress(dest).ToString();
            if (msc_debug_tokens)
                PrintToLog("%s: sender: %s, outpoint: %s:%d, value: %d\n", __func__, sAddress, txid.GetHex(), n, txOut.nValue);

            // only use funds from the sender's address
            if (fromAddress == sAddress) {
                COutPoint outpoint(txid, n);
                coinControl.Select(outpoint);

                nTotal += txOut.nValue;

                if (nMax <= nTotal) break;
            }
        }

        if (nMax <= nTotal) break;
    }
#endif

    return nTotal;
}

} // namespace mastercore
<|endoftext|>


<commit_before>/** \file
 *
 *  Copyright (c) 2015 by Travis Gockel. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author Travis Gockel (travis@gockelhut.com)
**/
#include "core.hpp"

#include <jsonv/all.hpp>

#include <chrono>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <map>
#include <random>
#include <sstream>

using namespace jsonv;

struct generated_json_settings
{
    std::uniform_int_distribution<int> kind_distribution{0, 6};

    template <typename TRng>
    jsonv::kind kind(TRng& rng, std::size_t current_depth)
    {
        if (current_depth < 2)
            return jsonv::kind::object;

        auto k = static_cast<jsonv::kind>(kind_distribution(rng));
        if (current_depth > 5 && (k == jsonv::kind::array || k == jsonv::kind::object))
            return kind(rng, current_depth);
        else
            return k;
    }

    std::uniform_int_distribution<std::size_t> array_length_distribution{0, 100};
    template <typename TRng>
    std::size_t array_length(TRng& rng)
    {
        return array_length_distribution(rng);
    }

    std::uniform_int_distribution<std::size_t> object_size_distribution{5, 25};
    template <typename TRng>
    std::size_t object_size(TRng& rng)
    {
        return object_size_distribution(rng);
    }

    std::uniform_int_distribution<std::int64_t> string_length_distribution{0, 100};
    template <typename TRng>
    std::size_t string_length(TRng& rng)
    {
        return string_length_distribution(rng);
    }

    std::normal_distribution<> decimal_distribution{1.0, 40.0};
    template <typename TRng>
    double decimal(TRng& rng)
    {
        return decimal_distribution(rng);
    }

    std::uniform_int_distribution<std::int64_t> integer_distribution{~0};
    template <typename TRng>
    std::int64_t integer(TRng& rng)
    {
        return integer_distribution(rng);
    }
};

template <typename TRng, typename TSettings>
jsonv::value generate_json(TRng& rng, TSettings& settings, std::size_t current_depth = 0)
{
    switch (settings.kind(rng, current_depth))
    {
        case kind::array:
        {
            value out = array();
            std::size_t len = settings.array_length(rng);
            for (std::size_t idx = 0; idx < len; ++idx)
                out.push_back(generate_json(rng, settings, current_depth + 1));
            return out;
        }
        case kind::boolean:
            return !(rng() & 1);
        case kind::decimal:
            return settings.decimal(rng);
        case kind::integer:
            return settings.integer(rng);
        case kind::string:
            // TODO: Improve this
            return std::string(settings.string_length(rng), 'a');
        case kind::object:
        {
            value out = object();
            std::size_t len = settings.object_size(rng);
            for (std::size_t idx = 1; idx <= len; ++idx)
            {
                // TODO: Improve
                std::string key(idx, 'a');
                value val = generate_json(rng, settings, current_depth + 1);
                out.insert({ std::move(key), std::move(val) });
            }
            return out;
        }
        case kind::null:
        default:
            return null;
    }
}

class stopwatch
{
public:
    using clock      = std::chrono::steady_clock;
    using time_point = clock::time_point;
    using duration   = std::chrono::nanoseconds;

    struct ticker
    {
        stopwatch* owner;
        time_point start_time;

        ticker(stopwatch* o) :
                owner(o),
                start_time(clock::now())
        { }

        ticker(ticker&& src) :
                owner(src.owner),
                start_time(src.start_time)
        {
            src.owner = nullptr;
        }

        ~ticker()
        {
            if (owner)
                owner->add_time(clock::now() - start_time);
        }
    };

public:
    stopwatch() :
            tick_count(0),
            total_time(duration(0))
    { }

    ticker start()
    {
        return ticker(this);
    }

    void add_time(duration dur)
    {
        ++tick_count;
        total_time += dur;
    }

public:
    std::size_t tick_count;
    duration    total_time;
};

static std::string get_encoded_json()
{
    std::ifstream in("temp.json");
    if (in.good())
    {
        std::stringstream buff;
        buff << in.rdbuf();
        return buff.str();
    }

    std::ostringstream encoded_stream;
    ostream_pretty_encoder out(encoded_stream);
    std::mt19937_64 rng{std::random_device()()};
    generated_json_settings settings;
    value val = generate_json(rng, settings);
    out.encode(val);
    std::string encoded = encoded_stream.str();

    // save the string to a file in case we want to re-use it
    std::ofstream file("temp.json", std::ofstream::out | std::ofstream::trunc);
    file << encoded;
    return encoded;
}

int main(int argc, char** argv)
{
    using namespace json_benchmark;

    std::string filter;
    if (argc == 2)
        filter = argv[1];

    std::string encoded = get_encoded_json();
    static const int LOOP_COUNT = 10;

    for (const benchmark_suite* suite : benchmark_suite::all())
    {
        if (!filter.empty() && filter != suite->name())
            continue;

        std::cout << suite->name() << "..." << std::endl;
        stopwatch watch;
        for (int idx = 1; idx <= LOOP_COUNT; ++idx)
        {
            std::cout << "  " << idx << '/' << LOOP_COUNT << std::endl;
            auto ticker = watch.start();
            suite->parse_test(encoded);
        }

        auto average = std::chrono::duration_cast<std::chrono::microseconds>(watch.total_time) / watch.tick_count;
        std::cout << suite->name() << '\t' << average.count() << "us" << std::endl;
    }
}
<commit_msg>Benchmarking: Adds the ability to control the loop count in testing.<commit_after>/** \file
 *
 *  Copyright (c) 2015 by Travis Gockel. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author Travis Gockel (travis@gockelhut.com)
**/
#include "core.hpp"

#include <jsonv/all.hpp>

#include <chrono>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <map>
#include <random>
#include <sstream>

#include <boost/lexical_cast.hpp>

using namespace jsonv;

struct generated_json_settings
{
    std::uniform_int_distribution<int> kind_distribution{0, 6};

    template <typename TRng>
    jsonv::kind kind(TRng& rng, std::size_t current_depth)
    {
        if (current_depth < 2)
            return jsonv::kind::object;

        auto k = static_cast<jsonv::kind>(kind_distribution(rng));
        if (current_depth > 5 && (k == jsonv::kind::array || k == jsonv::kind::object))
            return kind(rng, current_depth);
        else
            return k;
    }

    std::uniform_int_distribution<std::size_t> array_length_distribution{0, 100};
    template <typename TRng>
    std::size_t array_length(TRng& rng)
    {
        return array_length_distribution(rng);
    }

    std::uniform_int_distribution<std::size_t> object_size_distribution{5, 25};
    template <typename TRng>
    std::size_t object_size(TRng& rng)
    {
        return object_size_distribution(rng);
    }

    std::uniform_int_distribution<std::int64_t> string_length_distribution{0, 100};
    template <typename TRng>
    std::size_t string_length(TRng& rng)
    {
        return string_length_distribution(rng);
    }

    std::normal_distribution<> decimal_distribution{1.0, 40.0};
    template <typename TRng>
    double decimal(TRng& rng)
    {
        return decimal_distribution(rng);
    }

    std::uniform_int_distribution<std::int64_t> integer_distribution{~0};
    template <typename TRng>
    std::int64_t integer(TRng& rng)
    {
        return integer_distribution(rng);
    }
};

template <typename TRng, typename TSettings>
jsonv::value generate_json(TRng& rng, TSettings& settings, std::size_t current_depth = 0)
{
    switch (settings.kind(rng, current_depth))
    {
        case kind::array:
        {
            value out = array();
            std::size_t len = settings.array_length(rng);
            for (std::size_t idx = 0; idx < len; ++idx)
                out.push_back(generate_json(rng, settings, current_depth + 1));
            return out;
        }
        case kind::boolean:
            return !(rng() & 1);
        case kind::decimal:
            return settings.decimal(rng);
        case kind::integer:
            return settings.integer(rng);
        case kind::string:
            // TODO: Improve this
            return std::string(settings.string_length(rng), 'a');
        case kind::object:
        {
            value out = object();
            std::size_t len = settings.object_size(rng);
            for (std::size_t idx = 1; idx <= len; ++idx)
            {
                // TODO: Improve
                std::string key(idx, 'a');
                value val = generate_json(rng, settings, current_depth + 1);
                out.insert({ std::move(key), std::move(val) });
            }
            return out;
        }
        case kind::null:
        default:
            return null;
    }
}

class stopwatch
{
public:
    using clock      = std::chrono::steady_clock;
    using time_point = clock::time_point;
    using duration   = std::chrono::nanoseconds;

    struct ticker
    {
        stopwatch* owner;
        time_point start_time;

        ticker(stopwatch* o) :
                owner(o),
                start_time(clock::now())
        { }

        ticker(ticker&& src) :
                owner(src.owner),
                start_time(src.start_time)
        {
            src.owner = nullptr;
        }

        ~ticker()
        {
            if (owner)
                owner->add_time(clock::now() - start_time);
        }
    };

public:
    stopwatch() :
            tick_count(0),
            total_time(duration(0))
    { }

    ticker start()
    {
        return ticker(this);
    }

    void add_time(duration dur)
    {
        ++tick_count;
        total_time += dur;
    }

public:
    std::size_t tick_count;
    duration    total_time;
};

static std::string get_encoded_json()
{
    std::ifstream in("temp.json");
    if (in.good())
    {
        std::stringstream buff;
        buff << in.rdbuf();
        return buff.str();
    }

    std::ostringstream encoded_stream;
    ostream_pretty_encoder out(encoded_stream);
    std::mt19937_64 rng{std::random_device()()};
    generated_json_settings settings;
    value val = generate_json(rng, settings);
    out.encode(val);
    std::string encoded = encoded_stream.str();

    // save the string to a file in case we want to re-use it
    std::ofstream file("temp.json", std::ofstream::out | std::ofstream::trunc);
    file << encoded;
    return encoded;
}

int main(int argc, char** argv)
{
    using namespace json_benchmark;

    std::string filter;
    if (argc >= 2)
        filter = argv[1];

    int loop_count = 10;
    if (argc >= 3)
        loop_count = boost::lexical_cast<int>(argv[2]);

    std::string encoded = get_encoded_json();

    for (const benchmark_suite* suite : benchmark_suite::all())
    {
        if (!filter.empty() && filter != suite->name())
            continue;

        std::cout << suite->name() << "..." << std::endl;
        stopwatch watch;
        for (int idx = 1; idx <= loop_count; ++idx)
        {
            std::cout << "  " << idx << '/' << loop_count << std::endl;
            auto ticker = watch.start();
            suite->parse_test(encoded);
        }

        auto average = std::chrono::duration_cast<std::chrono::microseconds>(watch.total_time) / watch.tick_count;
        std::cout << suite->name() << '\t' << average.count() << "us" << std::endl;
    }
}
<|endoftext|>


<commit_before>/*!
 * Copyright (c) 2015 by Contributors
 * \file upsampling_nearest.cc
 * \brief
 * \author Bing Xu
*/

#include "./deconvolution-inl.h"
#include "./upsampling-inl.h"

namespace mxnet {
namespace op {
template<>
Operator *CreateOp<cpu>(UpSamplingParam param) {
  if (param.sample_type == up_enum::kNearest) {
    return new UpSamplingNearestOp<cpu>(param);
  } else if (param.sample_type == up_enum::kBilinear) {
    DeconvolutionParam p;
    int kernel = 2 * param.scale - param.scale % 2;
    int stride = param.scale;
    int pad = static_cast<int>(ceil((param.scale - 1) / 2.));
    p.num_group = param.num_filter;
    p.num_filter = param.num_filter;
    p.no_bias =  true;
    int shape[] = {1, 1};
    shape[0] = shape[1] = kernel;
    p.kernel = TShape(shape, shape + 2);
    shape[0] = shape[1] = stride;
    p.stride = TShape(shape, shape + 2);
    shape[0] = shape[1] = pad;
    p.pad = TShape(shape, shape + 2);
    return new DeconvolutionOp<cpu>(p);
  } else {
    LOG(FATAL) << "Unknown sample type";
    return NULL;
  }
}

Operator* UpSamplingProp::CreateOperator(Context ctx) const {
  DO_BIND_DISPATCH(CreateOp, param_);
}

DMLC_REGISTER_PARAMETER(UpSamplingParam);

MXNET_REGISTER_OP_PROPERTY(UpSampling, UpSamplingProp)
.describe("Perform nearest neighboor/bilinear up sampling to inputs")
.add_arguments(UpSamplingParam::__FIELDS__())
.set_key_var_num_args("num_args");
}  // namespace op
}  // namespace mxnet
<commit_msg>fix OpInfo for upsamplingOp<commit_after>/*!
 * Copyright (c) 2015 by Contributors
 * \file upsampling_nearest.cc
 * \brief
 * \author Bing Xu
*/

#include "./deconvolution-inl.h"
#include "./upsampling-inl.h"

namespace mxnet {
namespace op {
template<>
Operator *CreateOp<cpu>(UpSamplingParam param) {
  if (param.sample_type == up_enum::kNearest) {
    return new UpSamplingNearestOp<cpu>(param);
  } else if (param.sample_type == up_enum::kBilinear) {
    DeconvolutionParam p;
    int kernel = 2 * param.scale - param.scale % 2;
    int stride = param.scale;
    int pad = static_cast<int>(ceil((param.scale - 1) / 2.));
    p.num_group = param.num_filter;
    p.num_filter = param.num_filter;
    p.no_bias =  true;
    int shape[] = {1, 1};
    shape[0] = shape[1] = kernel;
    p.kernel = TShape(shape, shape + 2);
    shape[0] = shape[1] = stride;
    p.stride = TShape(shape, shape + 2);
    shape[0] = shape[1] = pad;
    p.pad = TShape(shape, shape + 2);
    return new DeconvolutionOp<cpu>(p);
  } else {
    LOG(FATAL) << "Unknown sample type";
    return NULL;
  }
}

Operator* UpSamplingProp::CreateOperator(Context ctx) const {
  DO_BIND_DISPATCH(CreateOp, param_);
}

DMLC_REGISTER_PARAMETER(UpSamplingParam);

MXNET_REGISTER_OP_PROPERTY(UpSampling, UpSamplingProp)
.describe("Perform nearest neighboor/bilinear up sampling to inputs")
.add_argument("data", "Symbol[]", "Array of tensors to upsample")
.add_arguments(UpSamplingParam::__FIELDS__())
.set_key_var_num_args("num_args");
}  // namespace op
}  // namespace mxnet
<|endoftext|>