<commit_before>#include "acmacs-base/rjson-v3.hh"
#include "acmacs-base/timeit.hh"
#include "acmacs-map-draw/mapi-settings.hh"
#include "acmacs-map-draw/draw.hh"

// ----------------------------------------------------------------------

bool acmacs::mapi::v1::Settings::apply_built_in(std::string_view name) // returns true if built-in command with that name found and applied
{
    using namespace std::string_view_literals;
    AD_LOG(acmacs::log::settings, "mapi::apply_built_in \"{}\"", name);
    AD_LOG_INDENT;
    try {
        // printenv();
        if (name == "antigens"sv)
            return apply_antigens();
        else if (name == "sera"sv)
            return apply_sera();
        else if (name == "vaccine"sv)
            return apply_vaccine();
        else if (name == "circle"sv)
            return apply_circle();
        else if (name == "path"sv)
            return apply_path();
        else if (name == "rotate"sv)
            return apply_rotate();
        else if (name == "flip"sv)
            return apply_flip();
        else if (name == "viewport"sv)
            return apply_viewport();
        else if (name == "background"sv)
            return apply_background();
        else if (name == "border"sv)
            return apply_border();
        else if (name == "grid"sv)
            return apply_grid();
        else if (name == "point-scale"sv || name == "point_scale"sv)
            return apply_point_scale();
        else if (name == "connection_lines"sv || name == "connection-lines"sv)
            return apply_connection_lines();
        else if (name == "error_lines"sv || name == "error-lines"sv)
            return apply_error_lines();
        else if (name == "legend"sv)
            return apply_legend();
        else if (name == "title"sv)
            return apply_title();
        else if (name == "serum-circles"sv || name == "serum-circle"sv || name == "serum_circles"sv || name == "serum_circle"sv)
            return apply_serum_circles();
        else if (name == "serum-coverage"sv || name == "serum_coverage"sv)
            return apply_serum_coverage();
        else if (name == "procrustes-arrows"sv || name == "procrustes_arrows"sv)
            return apply_procrustes();
        else if (name == "remove-procrustes-arrows"sv || name == "remove_procrustes_arrows"sv)
            return apply_remove_procrustes();
        else if (name == "move"sv)
            return apply_move();
        else if (name == "reset"sv)
            return apply_reset();
        else if (name == "export"sv)
            return apply_export();
        else if (name == "pdf"sv)
            return apply_pdf();
        else if (name == "relax"sv)
            return apply_relax();
        else if (name == "compare-sequences"sv || name == "compare_sequences"sv)
            return apply_compare_sequences();
        else if (name == "time-series"sv || name == "time_series"sv)
            return apply_time_series();
        // else if (name == ""sv)
        //     return apply_();
        return acmacs::settings::v3::Data::apply_built_in(name);
    }
    catch (std::exception& err) {
        throw error{fmt::format("cannot apply mapi built in \"{}\": {}", name, err)};
    }

} // acmacs::mapi::v1::Settings::apply_built_in

// ----------------------------------------------------------------------

void acmacs::mapi::v1::Settings::update_env(env_put_antigen_serum_names epasn)
{
    using namespace std::string_view_literals;
    const auto& chart = chart_draw().chart(0).chart();
    setenv("name"sv, chart.make_name());

    auto info{chart.info()};
    setenv("virus"sv, info->virus());

    const std::string virus_type{info->virus_type()};
    setenv("virus-type"sv, virus_type);

    const std::string lineage{chart.lineage()};
    const auto subset{info->subset(acmacs::chart::Info::Compute::Yes)};
    setenv("lineage"sv, lineage);
    setenv("lineage-cap"sv, ::string::capitalize(lineage));
    setenv("subset"sv, subset);
    setenv("subset-up"sv, ::string::upper(subset));
    if (virus_type == "A(H1N1)"sv) {
        setenv("virus-type/lineage"sv, virus_type);
        setenv("virus-type/lineage-subset"sv, virus_type + subset);
        if (subset == "2009pdm"sv)
            setenv("virus-type-lineage-subset-short-low"sv, "h1pdm"sv);
        else
            setenv("virus-type-lineage-subset-short-low"sv, "h1"sv);
    }
    else if (virus_type == "A(H3N2)"sv) {
        setenv("virus-type/lineage"sv, virus_type);
        setenv("virus-type/lineage-subset"sv, virus_type);
        setenv("virus-type-lineage-subset-short-low"sv, "h3"sv);
    }
    else if (virus_type == "B"sv) {
        const auto vtl = fmt::format("{}/{}", virus_type, ::string::capitalize(lineage.substr(0, 3)));
        setenv("virus-type/lineage"sv, vtl);
        setenv("virus-type/lineage-subset"sv, vtl);
        setenv("virus-type-lineage-subset-short-low"sv, fmt::format("b{}", ::string::lower(lineage.substr(0, 3))));
    }
    else {
        std::string vtl{virus_type};
        if (!lineage.empty())
            vtl = fmt::format("{}/{}", virus_type, ::string::capitalize(lineage.substr(0, 3)));
        setenv("virus-type/lineage"sv, vtl);
        setenv("virus-type/lineage-subset"sv, fmt::format("{}{}", vtl, subset));
        setenv("virus-type-lineage-subset-short-low"sv, ::string::lower(virus_type));
    }

    const auto assay{info->assay()};
    setenv("assay-full"sv, *assay);
    const std::string assay_cap{assay.HI_or_Neut()};
    setenv("assay-cap"sv, assay_cap);
    const std::string assay_neut{assay.hi_or_neut()};
    setenv("assay-low"sv, assay_neut);
    if (assay_neut == "hi" && virus_type != "A(H3N2)") {
        setenv("assay-no-hi-low"sv, ""sv);
        setenv("assay-no-hi-cap"sv, ""sv);
    }
    else {
        setenv("assay-no-hi-low"sv, assay_neut);
        setenv("assay-no-hi-cap"sv, assay_cap);
    }

    auto lab{info->lab()};
    if (lab == "MELB"sv || lab == "MELB+VIDRL"sv)
        lab = Lab{"VIDRL"};
    setenv("lab"sv, lab);
    setenv("lab-low"sv, ::string::lower(lab));

    setenv("rbc"sv, info->rbc_species());
    if (assay_neut == "hi" && !info->rbc_species().empty())
        setenv("assay-rbc"sv, fmt::format("{}-{}", assay_neut, ::string::lower(info->rbc_species())));
    else
        setenv("assay-rbc"sv, assay_neut);

    setenv("assay-low"sv, assay_neut);

    setenv("table-date"sv, info->date(acmacs::chart::Info::Compute::Yes));

    setenv("num-ag"sv, fmt::format("{}", chart.number_of_antigens()));
    setenv("num-sr"sv, fmt::format("{}", chart.number_of_sera()));

    auto titers{chart.titers()};
    setenv("num-layers"sv, fmt::format("{}", titers->number_of_layers()));

    const auto& projection = chart_draw().chart(0).modified_projection();
    const auto minimum_column_basis = static_cast<std::string>(projection.minimum_column_basis());
    setenv("minimum-column-basis"sv, minimum_column_basis);
    setenv("mcb"sv, minimum_column_basis);

    if (epasn == env_put_antigen_serum_names::yes) {
        auto antigens = chart.antigens();
        for (auto [no, antigen] : acmacs::enumerate(*antigens)) {
            setenv(fmt::format("ag-{}-full-name", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-full-name-without-passage", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-full-name-with-passage", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-abbreviated-name", no), antigen->format("{name_abbreviated}"));
            setenv(fmt::format("ag-{}-designation", no), antigen->format("{designation}"));
        }

        auto sera = chart.sera();
        for (auto [no, serum] : acmacs::enumerate(*sera)) {
            setenv(fmt::format("sr-{}-full-name", no), serum->format("{name_full}"));
            setenv(fmt::format("sr-{}-full-name-without-passage", no), serum->format("{name_full}"));
            setenv(fmt::format("sr-{}-full-name-with-passage", no), serum->format("{name_full_passage}"));
            setenv(fmt::format("sr-{}-abbreviated-name", no), serum->format("{name_abbreviated}"));
            setenv(fmt::format("sr-{}-abbreviated-name-with-serum-id", no), serum->format("{name_full}{ }{serum_id}", acmacs::chart::collapse_spaces_t::yes));
            setenv(fmt::format("sr-{}-designation", no), serum->format("{designation}"));
            setenv(fmt::format("sr-{}-designation-without-serum-id", no), serum->format("{designation_without_serum_id}"));
            setenv(fmt::format("sr-{}-clades", no), serum->format("{clades}"));
            setenv(fmt::format("sr-{}-clade", no), serum->format("{clade}")); // longest clade name
        }
    }

    update_env_upon_projection_change();

} // acmacs::mapi::v1::Settings::update_env

// ----------------------------------------------------------------------

void acmacs::mapi::v1::Settings::update_env_upon_projection_change()
{
    using namespace std::string_view_literals;
    const auto& projection = chart_draw().chart(0).modified_projection();

    setenv("stress"sv, fmt::format("{:.4f}", projection.stress()), settings::v3::replace::yes);
    setenv("stress-full"sv, fmt::format("{}", projection.stress()), settings::v3::replace::yes);

    // AD_DEBUG("env stress: {}", getenv("stress"sv));

} // acmacs::mapi::v1::Settings::update_env_upon_projection_change

// ----------------------------------------------------------------------
<commit_msg>"{clade}" -> longest clade name<commit_after>#include "acmacs-base/rjson-v3.hh"
#include "acmacs-base/timeit.hh"
#include "acmacs-map-draw/mapi-settings.hh"
#include "acmacs-map-draw/draw.hh"

// ----------------------------------------------------------------------

bool acmacs::mapi::v1::Settings::apply_built_in(std::string_view name) // returns true if built-in command with that name found and applied
{
    using namespace std::string_view_literals;
    AD_LOG(acmacs::log::settings, "mapi::apply_built_in \"{}\"", name);
    AD_LOG_INDENT;
    try {
        // printenv();
        if (name == "antigens"sv)
            return apply_antigens();
        else if (name == "sera"sv)
            return apply_sera();
        else if (name == "vaccine"sv)
            return apply_vaccine();
        else if (name == "circle"sv)
            return apply_circle();
        else if (name == "path"sv)
            return apply_path();
        else if (name == "rotate"sv)
            return apply_rotate();
        else if (name == "flip"sv)
            return apply_flip();
        else if (name == "viewport"sv)
            return apply_viewport();
        else if (name == "background"sv)
            return apply_background();
        else if (name == "border"sv)
            return apply_border();
        else if (name == "grid"sv)
            return apply_grid();
        else if (name == "point-scale"sv || name == "point_scale"sv)
            return apply_point_scale();
        else if (name == "connection_lines"sv || name == "connection-lines"sv)
            return apply_connection_lines();
        else if (name == "error_lines"sv || name == "error-lines"sv)
            return apply_error_lines();
        else if (name == "legend"sv)
            return apply_legend();
        else if (name == "title"sv)
            return apply_title();
        else if (name == "serum-circles"sv || name == "serum-circle"sv || name == "serum_circles"sv || name == "serum_circle"sv)
            return apply_serum_circles();
        else if (name == "serum-coverage"sv || name == "serum_coverage"sv)
            return apply_serum_coverage();
        else if (name == "procrustes-arrows"sv || name == "procrustes_arrows"sv)
            return apply_procrustes();
        else if (name == "remove-procrustes-arrows"sv || name == "remove_procrustes_arrows"sv)
            return apply_remove_procrustes();
        else if (name == "move"sv)
            return apply_move();
        else if (name == "reset"sv)
            return apply_reset();
        else if (name == "export"sv)
            return apply_export();
        else if (name == "pdf"sv)
            return apply_pdf();
        else if (name == "relax"sv)
            return apply_relax();
        else if (name == "compare-sequences"sv || name == "compare_sequences"sv)
            return apply_compare_sequences();
        else if (name == "time-series"sv || name == "time_series"sv)
            return apply_time_series();
        // else if (name == ""sv)
        //     return apply_();
        return acmacs::settings::v3::Data::apply_built_in(name);
    }
    catch (std::exception& err) {
        throw error{fmt::format("cannot apply mapi built in \"{}\": {}", name, err)};
    }

} // acmacs::mapi::v1::Settings::apply_built_in

// ----------------------------------------------------------------------

void acmacs::mapi::v1::Settings::update_env(env_put_antigen_serum_names epasn)
{
    using namespace std::string_view_literals;
    const auto& chart = chart_draw().chart(0).chart();
    setenv("name"sv, chart.make_name());

    auto info{chart.info()};
    setenv("virus"sv, info->virus());

    const std::string virus_type{info->virus_type()};
    setenv("virus-type"sv, virus_type);

    const std::string lineage{chart.lineage()};
    const auto subset{info->subset(acmacs::chart::Info::Compute::Yes)};
    setenv("lineage"sv, lineage);
    setenv("lineage-cap"sv, ::string::capitalize(lineage));
    setenv("subset"sv, subset);
    setenv("subset-up"sv, ::string::upper(subset));
    if (virus_type == "A(H1N1)"sv) {
        setenv("virus-type/lineage"sv, virus_type);
        setenv("virus-type/lineage-subset"sv, virus_type + subset);
        if (subset == "2009pdm"sv)
            setenv("virus-type-lineage-subset-short-low"sv, "h1pdm"sv);
        else
            setenv("virus-type-lineage-subset-short-low"sv, "h1"sv);
    }
    else if (virus_type == "A(H3N2)"sv) {
        setenv("virus-type/lineage"sv, virus_type);
        setenv("virus-type/lineage-subset"sv, virus_type);
        setenv("virus-type-lineage-subset-short-low"sv, "h3"sv);
    }
    else if (virus_type == "B"sv) {
        const auto vtl = fmt::format("{}/{}", virus_type, ::string::capitalize(lineage.substr(0, 3)));
        setenv("virus-type/lineage"sv, vtl);
        setenv("virus-type/lineage-subset"sv, vtl);
        setenv("virus-type-lineage-subset-short-low"sv, fmt::format("b{}", ::string::lower(lineage.substr(0, 3))));
    }
    else {
        std::string vtl{virus_type};
        if (!lineage.empty())
            vtl = fmt::format("{}/{}", virus_type, ::string::capitalize(lineage.substr(0, 3)));
        setenv("virus-type/lineage"sv, vtl);
        setenv("virus-type/lineage-subset"sv, fmt::format("{}{}", vtl, subset));
        setenv("virus-type-lineage-subset-short-low"sv, ::string::lower(virus_type));
    }

    const auto assay{info->assay()};
    setenv("assay-full"sv, *assay);
    const std::string assay_cap{assay.HI_or_Neut()};
    setenv("assay-cap"sv, assay_cap);
    const std::string assay_neut{assay.hi_or_neut()};
    setenv("assay-low"sv, assay_neut);
    if (assay_neut == "hi" && virus_type != "A(H3N2)") {
        setenv("assay-no-hi-low"sv, ""sv);
        setenv("assay-no-hi-cap"sv, ""sv);
    }
    else {
        setenv("assay-no-hi-low"sv, assay_neut);
        setenv("assay-no-hi-cap"sv, assay_cap);
    }

    auto lab{info->lab()};
    if (lab == "MELB"sv || lab == "MELB+VIDRL"sv)
        lab = Lab{"VIDRL"};
    setenv("lab"sv, lab);
    setenv("lab-low"sv, ::string::lower(lab));

    setenv("rbc"sv, info->rbc_species());
    if (assay_neut == "hi" && !info->rbc_species().empty())
        setenv("assay-rbc"sv, fmt::format("{}-{}", assay_neut, ::string::lower(info->rbc_species())));
    else
        setenv("assay-rbc"sv, assay_neut);

    setenv("assay-low"sv, assay_neut);

    setenv("table-date"sv, info->date(acmacs::chart::Info::Compute::Yes));

    setenv("num-ag"sv, fmt::format("{}", chart.number_of_antigens()));
    setenv("num-sr"sv, fmt::format("{}", chart.number_of_sera()));

    auto titers{chart.titers()};
    setenv("num-layers"sv, fmt::format("{}", titers->number_of_layers()));

    const auto& projection = chart_draw().chart(0).modified_projection();
    const auto minimum_column_basis = static_cast<std::string>(projection.minimum_column_basis());
    setenv("minimum-column-basis"sv, minimum_column_basis);
    setenv("mcb"sv, minimum_column_basis);

    if (epasn == env_put_antigen_serum_names::yes) {
        auto antigens = chart.antigens();
        for (auto [no, antigen] : acmacs::enumerate(*antigens)) {
            setenv(fmt::format("ag-{}-full-name", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-full-name-without-passage", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-full-name-with-passage", no), antigen->format("{name_full}"));
            setenv(fmt::format("ag-{}-abbreviated-name", no), antigen->format("{name_abbreviated}"));
            setenv(fmt::format("ag-{}-designation", no), antigen->format("{designation}"));
            setenv(fmt::format("ag-{}-clades", no), antigen->format("{clades}"));
            setenv(fmt::format("ag-{}-clade", no), antigen->format("{clade}")); // longest clade name
        }

        auto sera = chart.sera();
        for (auto [no, serum] : acmacs::enumerate(*sera)) {
            setenv(fmt::format("sr-{}-full-name", no), serum->format("{name_full}"));
            setenv(fmt::format("sr-{}-full-name-without-passage", no), serum->format("{name_full}"));
            setenv(fmt::format("sr-{}-full-name-with-passage", no), serum->format("{name_full_passage}"));
            setenv(fmt::format("sr-{}-abbreviated-name", no), serum->format("{name_abbreviated}"));
            setenv(fmt::format("sr-{}-abbreviated-name-with-serum-id", no), serum->format("{name_full}{ }{serum_id}", acmacs::chart::collapse_spaces_t::yes));
            setenv(fmt::format("sr-{}-designation", no), serum->format("{designation}"));
            setenv(fmt::format("sr-{}-designation-without-serum-id", no), serum->format("{designation_without_serum_id}"));
            setenv(fmt::format("sr-{}-clades", no), serum->format("{clades}"));
            setenv(fmt::format("sr-{}-clade", no), serum->format("{clade}")); // longest clade name
        }
    }

    update_env_upon_projection_change();

} // acmacs::mapi::v1::Settings::update_env

// ----------------------------------------------------------------------

void acmacs::mapi::v1::Settings::update_env_upon_projection_change()
{
    using namespace std::string_view_literals;
    const auto& projection = chart_draw().chart(0).modified_projection();

    setenv("stress"sv, fmt::format("{:.4f}", projection.stress()), settings::v3::replace::yes);
    setenv("stress-full"sv, fmt::format("{}", projection.stress()), settings::v3::replace::yes);

    // AD_DEBUG("env stress: {}", getenv("stress"sv));

} // acmacs::mapi::v1::Settings::update_env_upon_projection_change

// ----------------------------------------------------------------------
<|endoftext|>


<commit_before>// Used for testing, do not use it as an example
#include <iostream>

#include "qpp.h"

int main() {
    /////////// testing ///////////
    using namespace qpp;

    QCircuit qc{4, 4};
    qc.measureV(gt.H, 3, 3);
    qc.QFT();
    qc.measureZ(0, 0);
    qc.measureZ(1, 1);

    std::cout << qc << std::endl << std::endl;

    QEngine q_engine{qc};
    std::cout << q_engine << std::endl;

    std::cout << qc.get_gate_depth("CTRL-R2") << std::endl;
    std::cout << qc.get_gate_depth() << std::endl;
    std::cout << qc.get_gate_count("H") << std::endl;
    std::cout << qc.get_gate_depth("H") << std::endl;
}
<commit_msg>commit<commit_after>// Used for testing, do not use it as an example
#include <iostream>

#include "qpp.h"

int main() {
    /////////// testing ///////////
    using namespace qpp;

    const idx N = 100; // qubits
    QCircuit qc{N};

    std::cout << "filling the circuit...\n";
    for (idx i = 0; i < 1000000 * N; ++i) {
        idx ctrl = randidx(0, N - 1);
        idx target = randidx(0, N - 1);
        while (target == ctrl)
            target = randidx(0, N - 1);
        qc.gate(gt.CNOT, ctrl, target);
    }

    std::cout << "gate count: " << qc.get_gate_count() << std::endl;
    std::cout << "gate depth: " << qc.get_gate_depth() << std::endl;
}
<|endoftext|>


<commit_before>// kmmsginfo.cpp

#include "kmmsginfo.h"
#include "kmmessage.h"
#include "kmmsgpart.h" // for encode

#include <string.h>
#include <stdio.h>
#include <assert.h>
#include <mimelib/datetime.h>

// Unused, right ? (David)
//static QString result;

//-----------------------------------------------------------------------------
KMMsgInfo::KMMsgInfo(KMFolder* p):
  KMMsgInfoInherited(p), mSubject(), mFromStrip(), mToStrip()
{
}

//-----------------------------------------------------------------------------
KMMsgInfo::~KMMsgInfo()
{
}

//-----------------------------------------------------------------------------
KMMsgInfo& KMMsgInfo::operator=(const KMMsgInfo& other)
{
  KMMsgInfoInherited::assign(&other);
  mSubject = other.mSubject.copy();
  mFromStrip = other.mFromStrip.copy();
  mToStrip = other.mToStrip.copy();
  mReplyToIdMD5 = other.replyToIdMD5().copy();
  mMsgIdMD5 = other.msgIdMD5().copy();
  return *this;
}

//-----------------------------------------------------------------------------
KMMsgInfo& KMMsgInfo::operator=(const KMMessage& msg)
{
  KMMsgInfoInherited::assign(&msg);
  mSubject = msg.subject().copy();
  mFromStrip = msg.fromStrip().copy();
  mToStrip = msg.toStrip().copy();
  mReplyToIdMD5 = msg.replyToIdMD5().copy();
  mMsgIdMD5 = msg.msgIdMD5().copy();
  return *this;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::init(const QString& aSubject, const QString& aFrom,
                     const QString& aTo, time_t aDate,
		     KMMsgStatus aStatus, const QString& aXMark,
		     const QString& replyToId, const QString& msgId,
		     unsigned long aFolderOffset, unsigned long aMsgSize)
{
  mSubject   = decodeRFC2047String(aSubject).copy();
  mFromStrip = KMMessage::stripEmailAddr( decodeRFC2047String(aFrom) );
  mToStrip   = KMMessage::stripEmailAddr( decodeRFC2047String(aTo) );
  mDate      = aDate;
  mXMark     = aXMark;
  //  mReplyToIdMD5 =  KMMessagePart::encodeBase64( decodeRFC2047String(replyToId) );
  mReplyToIdMD5 =  KMMessagePart::encodeBase64( replyToId );
  //  mMsgIdMD5  = KMMessagePart::encodeBase64( decodeRFC2047String(msgId) );
  mMsgIdMD5  = KMMessagePart::encodeBase64( msgId );
  mStatus    = aStatus;
  mMsgSize   = aMsgSize;
  mFolderOffset = aFolderOffset;
  mDirty     = FALSE;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::subject(void) const
{
  return mSubject;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::fromStrip(void) const
{
  return mFromStrip;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::toStrip(void) const
{
  return mToStrip;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::xmark(void) const
{
  return mXMark;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::replyToIdMD5(void) const
{
  return mReplyToIdMD5;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::msgIdMD5(void) const
{
  return mMsgIdMD5;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setSubject(const QString& aSubject)
{
  mSubject = aSubject.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setXMark(const QString& aXMark)
{
  mXMark = aXMark.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setReplyToIdMD5(const QString& aReplyToIdMD5)
{
  mReplyToIdMD5 = aReplyToIdMD5.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setMsgIdMD5(const QString& aMsgIdMD5)
{
  mMsgIdMD5 = aMsgIdMD5.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::fromIndexString(const QString& str, bool toUtf8)
{
  char statusCh;
  unsigned long ldate;

  sscanf(str.data(), "%c %9lu %9lu %10lu", &statusCh,
	 &mFolderOffset, &mMsgSize, &ldate);

  mDate    = (time_t)ldate;
  mStatus  = (KMMsgStatus)statusCh;
  mXMark   = str.mid(33, 3).stripWhiteSpace();
  if (toUtf8)
  {
    mSubject = str.mid(37, 100).stripWhiteSpace();
    mFromStrip = str.mid(138, 50).stripWhiteSpace();
    mToStrip = str.mid(189, 50).stripWhiteSpace();
  } else {
    mSubject = QString::fromUtf8(str.mid(37, 100).stripWhiteSpace());
    mFromStrip = QString::fromUtf8(str.mid(138, 50).stripWhiteSpace());
    mToStrip = QString::fromUtf8(str.mid(189, 50).stripWhiteSpace());
  }
  mReplyToIdMD5 = str.mid(240, 22).stripWhiteSpace();
  mMsgIdMD5 = str.mid(263, 22).stripWhiteSpace();
  mDirty = FALSE;
}
<commit_msg>Patch from Michael Teske<commit_after>// kmmsginfo.cpp

#include "kmmsginfo.h"
#include "kmmessage.h"
#include "kmmsgpart.h" // for encode

#include <string.h>
#include <stdio.h>
#include <assert.h>
#include <mimelib/datetime.h>

// Unused, right ? (David)
//static QString result;

//-----------------------------------------------------------------------------
KMMsgInfo::KMMsgInfo(KMFolder* p):
  KMMsgInfoInherited(p), mSubject(), mFromStrip(), mToStrip()
{
}

//-----------------------------------------------------------------------------
KMMsgInfo::~KMMsgInfo()
{
}

//-----------------------------------------------------------------------------
KMMsgInfo& KMMsgInfo::operator=(const KMMsgInfo& other)
{
  KMMsgInfoInherited::assign(&other);
  mSubject = other.mSubject.copy();
  mFromStrip = other.mFromStrip.copy();
  mToStrip = other.mToStrip.copy();
  mReplyToIdMD5 = other.replyToIdMD5().copy();
  mMsgIdMD5 = other.msgIdMD5().copy();
  mXMark = other.xmark();
  return *this;
}

//-----------------------------------------------------------------------------
KMMsgInfo& KMMsgInfo::operator=(const KMMessage& msg)
{
  KMMsgInfoInherited::assign(&msg);
  mSubject = msg.subject().copy();
  mFromStrip = msg.fromStrip().copy();
  mToStrip = msg.toStrip().copy();
  mReplyToIdMD5 = msg.replyToIdMD5().copy();
  mMsgIdMD5 = msg.msgIdMD5().copy();
  mXMark = msg.xmark();
  return *this;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::init(const QString& aSubject, const QString& aFrom,
                     const QString& aTo, time_t aDate,
		     KMMsgStatus aStatus, const QString& aXMark,
		     const QString& replyToId, const QString& msgId,
		     unsigned long aFolderOffset, unsigned long aMsgSize)
{
  mSubject   = decodeRFC2047String(aSubject).copy();
  mFromStrip = KMMessage::stripEmailAddr( decodeRFC2047String(aFrom) );
  mToStrip   = KMMessage::stripEmailAddr( decodeRFC2047String(aTo) );
  mDate      = aDate;
  mXMark     = aXMark;
  //  mReplyToIdMD5 =  KMMessagePart::encodeBase64( decodeRFC2047String(replyToId) );
  mReplyToIdMD5 =  KMMessagePart::encodeBase64( replyToId );
  //  mMsgIdMD5  = KMMessagePart::encodeBase64( decodeRFC2047String(msgId) );
  mMsgIdMD5  = KMMessagePart::encodeBase64( msgId );
  mStatus    = aStatus;
  mMsgSize   = aMsgSize;
  mFolderOffset = aFolderOffset;
  mDirty     = FALSE;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::subject(void) const
{
  return mSubject;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::fromStrip(void) const
{
  return mFromStrip;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::toStrip(void) const
{
  return mToStrip;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::xmark(void) const
{
  return mXMark;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::replyToIdMD5(void) const
{
  return mReplyToIdMD5;
}

//-----------------------------------------------------------------------------
const QString KMMsgInfo::msgIdMD5(void) const
{
  return mMsgIdMD5;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setSubject(const QString& aSubject)
{
  mSubject = aSubject.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setXMark(const QString& aXMark)
{
  mXMark = aXMark.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setReplyToIdMD5(const QString& aReplyToIdMD5)
{
  mReplyToIdMD5 = aReplyToIdMD5.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::setMsgIdMD5(const QString& aMsgIdMD5)
{
  mMsgIdMD5 = aMsgIdMD5.copy();
  mDirty = TRUE;
}

//-----------------------------------------------------------------------------
void KMMsgInfo::fromIndexString(const QString& str, bool toUtf8)
{
  char statusCh;
  unsigned long ldate;

  sscanf(str.data(), "%c %9lu %9lu %10lu", &statusCh,
	 &mFolderOffset, &mMsgSize, &ldate);

  mDate    = (time_t)ldate;
  mStatus  = (KMMsgStatus)statusCh;
  mXMark   = str.mid(33, 3).stripWhiteSpace();
  if (toUtf8)
  {
    mSubject = str.mid(37, 100).stripWhiteSpace();
    mFromStrip = str.mid(138, 50).stripWhiteSpace();
    mToStrip = str.mid(189, 50).stripWhiteSpace();
  } else {
    mSubject = QString::fromUtf8(str.mid(37, 100).stripWhiteSpace());
    mFromStrip = QString::fromUtf8(str.mid(138, 50).stripWhiteSpace());
    mToStrip = QString::fromUtf8(str.mid(189, 50).stripWhiteSpace());
  }
  mReplyToIdMD5 = str.mid(240, 22).stripWhiteSpace();
  mMsgIdMD5 = str.mid(263, 22).stripWhiteSpace();
  mDirty = FALSE;
}
<|endoftext|>


<commit_before>// arrayT.cc

#include <fstream>
#include <iostream>
#include <sstream>

using std::ofstream;
using std::ios;
using std::cerr;
using std::endl;
using std::ostringstream;

#include <DataDDS.h>
#include <Sequence.h>
#include <Int32.h>
#include <Str.h>

using namespace ::libdap;

#include <BESDataHandlerInterface.h>
#include <BESDataNames.h>
#include <BESDebug.h>

#include "test_config.h"
#include "test_send_data.h"

class MySequence: public Sequence {
public:
    MySequence(const string &n, const string &d) :
            Sequence(n, d)
    {
    }
    MySequence(const MySequence &rhs) :
            Sequence(rhs)
    {
    }
    virtual ~MySequence()
    {
    }

    MySequence &operator=(const MySequence &rhs)
    {
        if (this == &rhs)
            return *this;

        dynamic_cast<Sequence &>(*this) = rhs; // run Sequence assignment

        return *this;
    }
    virtual BaseType *ptr_duplicate()
    {
        return new MySequence(*this);
    }
    virtual bool read()
    {
        set_read_p(true);
        return false;
    }
};

int main(int argc, char **argv)
{
    bool debug = false;
    if (argc > 1) {
        for (int i = 0; i < argc; i++) {
            string arg = argv[i];
            if (arg == "debug") {
                debug = true;
            }
        }
    }

    string bes_conf = (string) "BES_CONF=" + TEST_BUILD_DIR + "/bes.conf";
    putenv((char *) bes_conf.c_str());
    if (debug)
        BESDebug::SetUp("cerr,fonc");

    try {
        // build a DataDDS of simple types and set values for each of the
        // simple types.
        DataDDS *dds = new DataDDS(NULL, "virtual");
        MySequence s("people", "");
        Str name("name");
        s.add_var(&name);
        Int32 age("age");
        s.add_var(&age);
        dds->add_var(&s);

        MySequence *sp = dynamic_cast<MySequence *>(dds->var("people"));
        BaseTypeRow *row1 = new BaseTypeRow;
        Str *pcw = new Str("name");
        pcw->set_value("Patrick West");
        row1->push_back(pcw);
        Int32 *pcw_age = new Int32("age");
        pcw_age->set_value(41);
        row1->push_back(pcw_age);

        BaseTypeRow *row2 = new BaseTypeRow;
        Str *crw = new Str("name");
        crw->set_value("Christopher West");
        row2->push_back(crw);
        Int32 *crw_age = new Int32("age");
        crw_age->set_value(10);
        row2->push_back(crw_age);

        SequenceValues values;
        values.push_back(row1);
        values.push_back(row2);

        sp->set_value(values);
        sp->set_read_p(true);

        // transform the DataDDS into a netcdf file. The dhi only needs the
        // output stream and the post constraint. Test no constraints and
        // then some different constraints (1 var, 2 var)

        // The resulting netcdf file is streamed back. Write this file to a
        // test file locally

        BESDataHandlerInterface dhi;
        ofstream fstrm("./seqT.nc", ios::out | ios::trunc);
        dhi.set_output_stream(&fstrm);
        dhi.data[POST_CONSTRAINT] = "";

        ConstraintEvaluator eval;
        send_data(dds, eval, dhi);

        fstrm.close();

        delete dds;
    }
    catch (BESError &e) {
        cerr << e.get_message() << endl;
        return 1;
    }

    return 0;
}

<commit_msg>Edits to match the new, sane, semantics for BaseType::read(). True now means done/eof and flase means more data to read (more instances for a sequence). All the tests pass.<commit_after>// arrayT.cc

#include <fstream>
#include <iostream>
#include <sstream>

using std::ofstream;
using std::ios;
using std::cerr;
using std::endl;
using std::ostringstream;

#include <DataDDS.h>
#include <Sequence.h>
#include <Int32.h>
#include <Str.h>

using namespace ::libdap;

#include <BESDataHandlerInterface.h>
#include <BESDataNames.h>
#include <BESDebug.h>

#include "test_config.h"
#include "test_send_data.h"

class MySequence: public Sequence {
public:
    MySequence(const string &n, const string &d) :
            Sequence(n, d)
    {
    }
    MySequence(const MySequence &rhs) :
            Sequence(rhs)
    {
    }
    virtual ~MySequence()
    {
    }

    MySequence &operator=(const MySequence &rhs)
    {
        if (this == &rhs)
            return *this;

        dynamic_cast<Sequence &>(*this) = rhs; // run Sequence assignment

        return *this;
    }
    virtual BaseType *ptr_duplicate()
    {
        return new MySequence(*this);
    }
    virtual bool read()
    {
        set_read_p(true);
        return true;
    }
};

int main(int argc, char **argv)
{
    bool debug = false;
    if (argc > 1) {
        for (int i = 0; i < argc; i++) {
            string arg = argv[i];
            if (arg == "debug") {
                debug = true;
            }
        }
    }

    string bes_conf = (string) "BES_CONF=" + TEST_BUILD_DIR + "/bes.conf";
    putenv((char *) bes_conf.c_str());
    if (debug)
        BESDebug::SetUp("cerr,fonc");

    try {
        // build a DataDDS of simple types and set values for each of the
        // simple types.
        DataDDS *dds = new DataDDS(NULL, "virtual");
        MySequence s("people", "");
        Str name("name");
        s.add_var(&name);
        Int32 age("age");
        s.add_var(&age);
        dds->add_var(&s);

        MySequence *sp = dynamic_cast<MySequence *>(dds->var("people"));
        BaseTypeRow *row1 = new BaseTypeRow;
        Str *pcw = new Str("name");
        pcw->set_value("Patrick West");
        row1->push_back(pcw);
        Int32 *pcw_age = new Int32("age");
        pcw_age->set_value(41);
        row1->push_back(pcw_age);

        BaseTypeRow *row2 = new BaseTypeRow;
        Str *crw = new Str("name");
        crw->set_value("Christopher West");
        row2->push_back(crw);
        Int32 *crw_age = new Int32("age");
        crw_age->set_value(10);
        row2->push_back(crw_age);

        SequenceValues values;
        values.push_back(row1);
        values.push_back(row2);

        sp->set_value(values);
        sp->set_read_p(true);

        // transform the DataDDS into a netcdf file. The dhi only needs the
        // output stream and the post constraint. Test no constraints and
        // then some different constraints (1 var, 2 var)

        // The resulting netcdf file is streamed back. Write this file to a
        // test file locally

        BESDataHandlerInterface dhi;
        ofstream fstrm("./seqT.nc", ios::out | ios::trunc);
        dhi.set_output_stream(&fstrm);
        dhi.data[POST_CONSTRAINT] = "";

        ConstraintEvaluator eval;
        send_data(dds, eval, dhi);

        fstrm.close();

        delete dds;
    }
    catch (BESError &e) {
        cerr << e.get_message() << endl;
        return 1;
    }

    return 0;
}

<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 The Syscoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "test/test_syscoin_services.h"
#include "utiltime.h"
#include "util.h"
#include "rpc/server.h"
#include "alias.h"
#include "cert.h"
#include "asset.h"
#include "base58.h"
#include "chainparams.h"
#include <boost/test/unit_test.hpp>
#include <boost/lexical_cast.hpp>
#include <iterator>
#include <chrono>
#include "ranges.h"
using namespace boost::chrono;
using namespace std;
BOOST_GLOBAL_FIXTURE( SyscoinTestingSetup );

BOOST_FIXTURE_TEST_SUITE(syscoin_asset_allocation_tests, BasicSyscoinTestingSetup)
BOOST_AUTO_TEST_CASE(generate_asset_allocation_send)
{
	UniValue r;
	printf("Running generate_asset_allocation_send...\n");
	GenerateBlocks(5);
	AliasNew("node1", "jagassetallocationsend1", "data");
	AliasNew("node2", "jagassetallocationsend2", "data");
	string guid = AssetNew("node1", "usd", "jagassetallocationsend1", "data", "8", "false", "1", "-1");

	AssetSend("node1", guid, "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend1\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend1 false"));
	UniValue balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 1 * COIN);

	string txid0 = AssetAllocationTransfer(false, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false),0.11 * COIN);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	// send using zdag
	string txid1 = AssetAllocationTransfer(true, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.12}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 0.23 * COIN);

	// non zdag cannot be found since it was already mined, but ends up briefly in conflict state because sender is conflicted
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// first tx should have to wait 1 sec for good status
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid +  " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// check just sender
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// wait for 1 second as required by unit test
	MilliSleep(1000);

	// second send
	string txid2 = AssetAllocationTransfer(true, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.13}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 0.36 * COIN);

	// sender is conflicted so txid0 is conflicted by extension even if its not found
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// first ones now OK because it was found explicitely
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// second one hasn't waited enough time yet
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// check just sender
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// wait for 1.5 second to clear minor warning status
	MilliSleep(1500);
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// check just sender as well
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// after pow, should get not found on all of them
	GenerateBlocks(1);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	// check just sender as well
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

}
BOOST_AUTO_TEST_CASE(generate_assetallocationpruning)
{
	UniValue r;
	// makes sure services expire in 100 blocks instead of 1 year of blocks for testing purposes
	printf("Running generate_assetallocationpruning...\n");
	AliasNew("node1", "jagprunealias2", "changeddata1");
	MilliSleep(5000);
	AliasNew("node2", "jagprunealias2node2", "data");
	// stop node2 create a service,  mine some blocks to expire the service, when we restart the node the service data won't be synced with node2
	StopNode("node2");
	string guid = AssetNew("node1", "bcf", "jagprunealias2", "pubdata");
	AssetSend("node1", guid, "\"[{\\\"aliasto\\\":\\\"jagprunealias2\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	AssetAllocationTransfer(false, "node1", guid, "jagprunealias2", "\"[{\\\"aliasto\\\":\\\"jagprunealias2node2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	// we can find it as normal first
	BOOST_CHECK_NO_THROW(CallRPC("node1", "assetinfo " + guid + " false"));
	// make sure our alias doesn't expire
	string hex_str = AliasUpdate("node1", "jagprunealias2");
	BOOST_CHECK(hex_str.empty());
	GenerateBlocks(5, "node1");
	ExpireAlias("jagprunealias2");
	StartNode("node2");
	GenerateBlocks(5, "node2");

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid + " false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);

	// asset allocation is not expired but cannot claim interest because asset is expired
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), false);

	BOOST_CHECK_THROW(r = CallRPC("node2", "assetallocationcollectinterest " + guid + " jagprunealias2node2 ''"), runtime_error);

	// should be pruned
	BOOST_CHECK_THROW(CallRPC("node2", "assetinfo " + guid + " false"), runtime_error);

	// expire asset allocation alias and now asset allocation is also expired
	ExpireAlias("jagprunealias2node2");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);

	// stop node3
	StopNode("node3");
	// should fail: already expired alias
	BOOST_CHECK_THROW(CallRPC("node1", "assetupdate " + guid + " jagprunealias2 assets 0 0 ''"), runtime_error);
	GenerateBlocks(5, "node1");

	// stop and start node1
	StopNode("node1");
	StartNode("node1");
	GenerateBlocks(5, "node1");

	BOOST_CHECK_THROW(CallRPC("node1", "assetinfo " + guid + " false"), runtime_error);

	// create a new service
	AliasNew("node1", "jagprunealias2", "temp");
	AliasNew("node2", "jagprunealias2node2", "temp");
	string guid1 = AssetNew("node1", "bcf", "jagprunealias2", "pubdata");
	// ensure you can still update before expiry
	AssetUpdate("node1", guid1);
	AssetSend("node1", guid, "\"[{\\\"aliasto\\\":\\\"jagprunealias2\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	AssetAllocationTransfer(false, "node1", guid1, "jagprunealias2", "\"[{\\\"aliasto\\\":\\\"jagprunealias2node2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	// you can search it still on node1/node2
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	// make sure our alias doesn't expire
	hex_str = AliasUpdate("node1", "jagprunealias2");
	BOOST_CHECK(hex_str.empty());
	GenerateBlocks(5, "node1");
	ExpireAlias("jagprunealias2node2");
	// now it should be expired
	BOOST_CHECK_THROW(CallRPC("node1", "assetupdate " + guid1 + " jagprunealias2 assets 0 0 ''"), runtime_error);
	BOOST_CHECK_THROW(CallRPC("node2", "assetallocationcollectinterest " + guid + " jagprunealias2node2 ''"), runtime_error);
	GenerateBlocks(5, "node1");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);
	// and it should say its expired
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);
	GenerateBlocks(5, "node1");
	StartNode("node3");
	GenerateBlocks(5, "node3");
	// node3 shouldn't find the service at all (meaning node3 doesn't sync the data)
	BOOST_CHECK_THROW(CallRPC("node3", "assetinfo " + guid1 + " false"), runtime_error);
	BOOST_CHECK_THROW(CallRPC("node3", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"), runtime_error);
}
BOOST_AUTO_TEST_SUITE_END ()
<commit_msg>fix allocation test<commit_after>// Copyright (c) 2016-2017 The Syscoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "test/test_syscoin_services.h"
#include "utiltime.h"
#include "util.h"
#include "rpc/server.h"
#include "alias.h"
#include "cert.h"
#include "asset.h"
#include "base58.h"
#include "chainparams.h"
#include <boost/test/unit_test.hpp>
#include <boost/lexical_cast.hpp>
#include <iterator>
#include <chrono>
#include "ranges.h"
using namespace boost::chrono;
using namespace std;
BOOST_GLOBAL_FIXTURE( SyscoinTestingSetup );

BOOST_FIXTURE_TEST_SUITE(syscoin_asset_allocation_tests, BasicSyscoinTestingSetup)
BOOST_AUTO_TEST_CASE(generate_asset_allocation_send)
{
	UniValue r;
	printf("Running generate_asset_allocation_send...\n");
	GenerateBlocks(5);
	AliasNew("node1", "jagassetallocationsend1", "data");
	AliasNew("node2", "jagassetallocationsend2", "data");
	string guid = AssetNew("node1", "usd", "jagassetallocationsend1", "data", "8", "false", "1", "-1");

	AssetSend("node1", guid, "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend1\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend1 false"));
	UniValue balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 1 * COIN);

	string txid0 = AssetAllocationTransfer(false, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false),0.11 * COIN);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	// send using zdag
	string txid1 = AssetAllocationTransfer(true, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.12}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 0.23 * COIN);

	// non zdag cannot be found since it was already mined, but ends up briefly in conflict state because sender is conflicted
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// first tx should have to wait 1 sec for good status
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid +  " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// check just sender
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// wait for 1 second as required by unit test
	MilliSleep(1000);

	// second send
	string txid2 = AssetAllocationTransfer(true, "node1", guid, "jagassetallocationsend1", "\"[{\\\"aliasto\\\":\\\"jagassetallocationsend2\\\",\\\"amount\\\":0.13}]\"", "allocationsendmemo");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagassetallocationsend2 false"));
	balance = find_value(r.get_obj(), "balance");
	BOOST_CHECK_EQUAL(AssetAmountFromValue(balance, 8, false), 0.36 * COIN);

	// sender is conflicted so txid0 is conflicted by extension even if its not found
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// first ones now OK because it was found explicitely
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// second one hasn't waited enough time yet
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// check just sender
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_MINOR_CONFLICT_OK);

	// wait for 1.5 second to clear minor warning status
	MilliSleep(1500);
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// check just sender as well
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_STATUS_OK);

	// after pow, should get not found on all of them
	GenerateBlocks(1);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid0));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid1));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 " + txid2));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

	// check just sender as well
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationsenderstatus " + guid + " jagassetallocationsend1 ''"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "status").get_int(), ZDAG_NOT_FOUND);

}
BOOST_AUTO_TEST_CASE(generate_assetallocationpruning)
{
	UniValue r;
	// makes sure services expire in 100 blocks instead of 1 year of blocks for testing purposes
	printf("Running generate_assetallocationpruning...\n");
	AliasNew("node1", "jagprunealias2", "changeddata1");
	MilliSleep(5000);
	AliasNew("node2", "jagprunealias2node2", "data");
	// stop node2 create a service,  mine some blocks to expire the service, when we restart the node the service data won't be synced with node2
	StopNode("node2");
	string guid = AssetNew("node1", "bcf", "jagprunealias2", "pubdata");
	AssetSend("node1", guid, "\"[{\\\"aliasto\\\":\\\"jagprunealias2\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	AssetAllocationTransfer(false, "node1", guid, "jagprunealias2", "\"[{\\\"aliasto\\\":\\\"jagprunealias2node2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	// we can find it as normal first
	BOOST_CHECK_NO_THROW(CallRPC("node1", "assetinfo " + guid + " false"));
	// make sure our alias doesn't expire
	string hex_str = AliasUpdate("node1", "jagprunealias2");
	BOOST_CHECK(hex_str.empty());
	GenerateBlocks(5, "node1");
	ExpireAlias("jagprunealias2");
	StartNode("node2");
	GenerateBlocks(5, "node2");

	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid + " false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);

	// asset allocation is not expired but cannot claim interest because asset is expired
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), false);

	BOOST_CHECK_THROW(r = CallRPC("node2", "assetallocationcollectinterest " + guid + " jagprunealias2node2 ''"), runtime_error);

	// should be pruned
	BOOST_CHECK_THROW(CallRPC("node2", "assetinfo " + guid + " false"), runtime_error);

	// expire asset allocation alias and now asset allocation is also expired
	ExpireAlias("jagprunealias2node2");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);

	// stop node3
	StopNode("node3");
	// should fail: already expired alias
	BOOST_CHECK_THROW(CallRPC("node1", "assetupdate " + guid + " jagprunealias2 assets 0 0 ''"), runtime_error);
	GenerateBlocks(5, "node1");

	// stop and start node1
	StopNode("node1");
	StartNode("node1");
	GenerateBlocks(5, "node1");

	BOOST_CHECK_THROW(CallRPC("node1", "assetinfo " + guid + " false"), runtime_error);

	// create a new service
	AliasNew("node1", "jagprunealias2", "temp");
	AliasNew("node2", "jagprunealias2node2", "temp");
	string guid1 = AssetNew("node1", "bcf", "jagprunealias2", "pubdata");
	// ensure you can still update before expiry
	AssetUpdate("node1", guid1);
	AssetSend("node1", guid1, "\"[{\\\"aliasto\\\":\\\"jagprunealias2\\\",\\\"amount\\\":1}]\"", "assetallocationsend");
	AssetAllocationTransfer(false, "node1", guid1, "jagprunealias2", "\"[{\\\"aliasto\\\":\\\"jagprunealias2node2\\\",\\\"amount\\\":0.11}]\"", "allocationsendmemo");
	// you can search it still on node1/node2
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	// make sure our alias doesn't expire
	hex_str = AliasUpdate("node1", "jagprunealias2");
	BOOST_CHECK(hex_str.empty());
	GenerateBlocks(5, "node1");
	ExpireAlias("jagprunealias2node2");
	// now it should be expired
	BOOST_CHECK_THROW(CallRPC("node1", "assetupdate " + guid1 + " jagprunealias2 assets 0 0 ''"), runtime_error);
	BOOST_CHECK_THROW(CallRPC("node2", "assetallocationcollectinterest " + guid1 + " jagprunealias2node2 ''"), runtime_error);
	GenerateBlocks(5, "node1");
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node1", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);
	// and it should say its expired
	BOOST_CHECK_NO_THROW(r = CallRPC("node2", "assetinfo " + guid1 + " false"));
	BOOST_CHECK_EQUAL(find_value(r.get_obj(), "expired").get_bool(), true);
	GenerateBlocks(5, "node1");
	StartNode("node3");
	GenerateBlocks(5, "node3");
	// node3 shouldn't find the service at all (meaning node3 doesn't sync the data)
	BOOST_CHECK_THROW(CallRPC("node3", "assetinfo " + guid1 + " false"), runtime_error);
	BOOST_CHECK_THROW(CallRPC("node3", "assetallocationinfo " + guid1 + " jagprunealias2node2 false"), runtime_error);
}
BOOST_AUTO_TEST_SUITE_END ()
<|endoftext|>


<commit_before>#include "training/graph_group_multinode_sync.h"
#include "functional/functional.h"
#include "tensors/tensor_operators.h"

namespace marian {

/**
 * Set given scheduler to register training observers on the shard optimizers.
 */
void MultiNodeGraphGroupSync::setScheduler(Ptr<Scheduler> scheduler) {
  scheduler_ = scheduler;
  // optimizer has to be registered last to see a change of learning rate
  scheduler_->registerTrainingObserver(scheduler_);

  scheduler_->registerTrainingObserver(syncOptimizer_);

}

/**
 * Allocate new tensor on given GPU and store allocator.
 */
Tensor MultiNodeGraphGroupSync::newTensor(int size, Ptr<Backend> backend) {
  Tensor t;
  Ptr<TensorAllocator> allocator = New<TensorAllocator>(backend);
  allocator->reserveExact(size * sizeof(float));
  allocator->allocate(t, {1, size});
  allocators_.push_back(allocator);
  return t;
}

/**
 * Setup training environment and launch server thread and (if enabled) client
 * communication overlap threads.
 * Includes setting up MPI, node and shard sizes, clients, server shards and
 * communication overlap stuff.
 */
void MultiNodeGraphGroupSync::init(Ptr<data::Batch> batch) {
  // Setup clients and shards
  setupClients(batch);

  // setup sync sgd storage, We keep the summed gradient on Node 0
  sumGradientBuffer = newTensor(clientGraphs_[0]->params()->vals()->size(), clientGraphs_[0]->getBackend());
  accGradientsSync = newTensor(clientGraphs_[0]->params()->vals()->size(), clientGraphs_[0]->getBackend());
  accGradientsSync->set(0);
}

/**
 * Initialize the CPU arrays, with pinned memory for faster CudaMemCpy operations.
 * Requires the graph to be initialized first so we know its size
 */
void MultiNodeGraphGroupSync::initCPUArrays() {
  accGradientsSync_cpu = std::vector<float>(clientGraphs_[0]->params()->vals()->size());
  receiveBuffer_cpu = std::vector<float>(clientGraphs_[0]->params()->vals()->size());
}

/**
 * Setup MPI world size and rank of this node.
 */
void MultiNodeGraphGroupSync::setupMPI() {
#if MPI_FOUND
  MPI_Comm_size(MPI_COMM_WORLD, &mpi_comm_world_size_);
  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_my_rank_);
#endif
}

/**
 * Setup clients that will compute gradients and communicate them with the
 * server shards.
 * There is one client per GPU.
 */
void MultiNodeGraphGroupSync::setupClients(Ptr<data::Batch> batch) {
  runBatchThroughClientGraphs(batch);
  initCPUArrays();

  clientThreadPool_ = new marian::ThreadPool(devices_.size(), devices_.size());
}

/**
 * Initialize the graphs (models) of all clients on this node with the given
 * batch.
 */
void MultiNodeGraphGroupSync::runBatchThroughClientGraphs(Ptr<data::Batch> batch) {
  for(int i = 0; i < devices_.size(); i++) {
    THREAD_GUARD(clientBuilders_[i]->build(clientGraphs_[i], batch);
                 clientGraphs_[i]->forward();
                 clientGraphs_[i]->getBackend()->synchronize(););
  }
}

/**
 * Initialize variables required for overlapping client computations and
 * communication.
 * Includes summed and committed word counts, buffer flags, mutexes and
 * condition variables.
 */
void MultiNodeGraphGroupSync::sumGRAD(Tensor gradient) {
  std::lock_guard<std::mutex> guard(sumGradientMutex_);
  sumGradientBuffer->copyFrom(gradient);
  using namespace functional; //@TODO makes more sense to do that on the CPU i think
  Element(_1 += _2, accGradientsSync, sumGradientBuffer);
}

/**
 * If it's rank 0, it's a local update, if it's rank one it's remote
 * send and receive. Make sure you only call from device 0.
 */

void MultiNodeGraphGroupSync::sendReceiveUpdateSync() {
  #if MPI_FOUND
  // Copy the data to the CPU
  CUDA_CHECK(cudaMemcpy(accGradientsSync_cpu.data(),
                 accGradientsSync->data(),
                 accGradientsSync->size(),
                 cudaMemcpyDeviceToHost));
  // Wait until all nodes are ready
  MPI_Barrier(MPI_COMM_WORLD);
  int reduce_result = MPI_Reduce(accGradientsSync_cpu.data(), //CPU buffers
              receiveBuffer_cpu.data(),
              accGradientsSync_cpu.size(),
              MPI_FLOAT,
              MPI_SUM,
              0, //Rank of the process with the data. In this case Node 0
              MPI_COMM_WORLD);
  if (reduce_result != MPI_SUCCESS) {
    LOG(critical, "Error: MPI_REDUCE failed with error {}.", reduce_result);
    std::abort();
  }
  if (mpi_my_rank_ == 0) {
    // Copy the data back to the GPU and do optimizer update
    CUDA_CHECK(cudaMemcpy(accGradientsSync->data(),
                 receiveBuffer_cpu.data(),
                 accGradientsSync->size(),
                 cudaMemcpyHostToDevice));

    // Perform optimizer step
    syncOptimizer_->update(clientGraphs_[0]->params()->vals(),
                         accGradientsSync);

    // Copy the data back to the host
    CUDA_CHECK(cudaMemcpy(accGradientsSync_cpu.data(), //This is now the updated params
                   clientGraphs_[0]->params()->vals()->data(),
                   accGradientsSync->size(),
                   cudaMemcpyDeviceToHost));
  }

  int bcast_result = MPI_Bcast(accGradientsSync_cpu.data(), //This is now the updated params.
            accGradientsSync_cpu.size(),
            MPI_FLOAT,
            0, //Root process
            MPI_COMM_WORLD);
  if (bcast_result != MPI_SUCCESS) {
    LOG(critical, "Error: MPI_REDUCE failed with error {}.", bcast_result);
    std::abort();
  }

  if (mpi_my_rank_ != 0) {
    //Copy the data to the GPU
    CUDA_CHECK(cudaMemcpy(clientGraphs_[0]->params()->vals()->data(),
                   accGradientsSync_cpu.data(),
                   accGradientsSync->size(),
                   cudaMemcpyHostToDevice));
  }

  //Distribute the graph to the rest of the devices
  std::vector<std::thread> threads;
  for(int idx = 1; idx < devices_.size(); idx++) {
    threads.emplace_back(std::thread(
        [=](int idx) {
          //If NVLINK is not available it's faster to do this from the CPU
          //Because we don't have to go Device->Host->device
          CUDA_CHECK(cudaMemcpy(clientGraphs_[idx]->params()->vals()->data(),
                   accGradientsSync_cpu.data(),
                   accGradientsSync->size(),
                   cudaMemcpyHostToDevice));
        },
        idx));
  }
  for(auto&& t : threads) {
    t.join();
  }
  //set the accumulating buffers to zero;
  accGradientsSync->set(0);
  std::fill(accGradientsSync_cpu.begin(), accGradientsSync_cpu.end(), 0);
  std::fill(receiveBuffer_cpu.begin(), receiveBuffer_cpu.end(), 0);
  #endif
}

/**
 * Execute given batch on this node, pushing/pulling the resulting
 * gradients/parameters to/from the server shards
 * or -- if comm. overlap enabled -- to/from the communication buffers, summing
 * gradients locally if the communication thread is busy
 *
 * @param batch Batch on which to perform forward and backward passes.
 */
void MultiNodeGraphGroupSync::execute(Ptr<data::Batch> fullBatch) {
  if(!initialized_) {
    init(fullBatch);
    initialized_ = true;
  }

  std::vector<Ptr<data::Batch>> batches = fullBatch->split(devices_.size());

  static int t = 0;

  static float cost = 0;
  static size_t num_seen_words = 0;
  static size_t num_seen_sentences = 0;

  {
    auto task = [this, batches](int my_id) {
      auto batch = batches[my_id];
      auto graph = clientGraphs_[my_id];
      auto builder = clientBuilders_[my_id];

      auto costNode = builder->build(graph, batch);

      if (t == 0) {
        if (my_id != 0)
          graph->params()->vals()->copyFrom(clientGraphs_[0]->params()->vals());
      }

      graph->forward();
      {
        std::lock_guard<std::mutex> guard(sumCostMutex_);
        cost += costNode->scalar();
        num_seen_words += batch->words();
        num_seen_sentences += batch->size();
      }
      graph->backward();

      graph->getBackend()->synchronize(); //@Alham do you know why we need this here?

      sumGRAD(graph->params()->grads());
    };

    ThreadPool pool(devices_.size(), devices_.size());
    for(int idx = 0; idx < devices_.size(); ++idx)
      pool.enqueue(task, idx);
  }

  if (t % tau_ == 0)
    sendReceiveUpdateSync();

  t++;

  // Run scheduler (if enabled)
  if(t % tau_ == 0 && scheduler_) {
    if (options_->get<std::string>("cost-type") != "ce-sum")
      cost /= (tau_ * devices_.size());

    if (tau_ > 1) {
      std::vector<size_t> fakeLength = {1, 1};
      auto fb = data::CorpusBatch::fakeBatch(fakeLength,
                                        num_seen_sentences,
                                        NULL);
      fb->front()->setWords(num_seen_words);
      scheduler_->update(cost, fb);
    } else {
      scheduler_->update(cost, fullBatch);
    }

    num_seen_words = 0;
    num_seen_sentences = 0;
    cost = 0;

    if((scheduler_->saving() || scheduler_->validating())) {
      #if MPI_FOUND
      //wait until other nodes are ready
      MPI_Barrier(MPI_COMM_WORLD);

      // TODO: Saving is broken
      //if(mpi_my_rank_ == 0 && scheduler_->saving())
      //  this->save(graph);

      if(mpi_my_rank_ == 0 && scheduler_->validating())
        scheduler_->validate(clientGraphs_);

      // inform other nodes to continue
      MPI_Barrier(MPI_COMM_WORLD);
      #endif
    }
    }

}
}
<commit_msg>Cudamemcpy also needs to be smaller<commit_after>#include "training/graph_group_multinode_sync.h"
#include "functional/functional.h"
#include "tensors/tensor_operators.h"

namespace marian {

/**
 * Set given scheduler to register training observers on the shard optimizers.
 */
void MultiNodeGraphGroupSync::setScheduler(Ptr<Scheduler> scheduler) {
  scheduler_ = scheduler;
  // optimizer has to be registered last to see a change of learning rate
  scheduler_->registerTrainingObserver(scheduler_);

  scheduler_->registerTrainingObserver(syncOptimizer_);

}

/**
 * Allocate new tensor on given GPU and store allocator.
 */
Tensor MultiNodeGraphGroupSync::newTensor(int size, Ptr<Backend> backend) {
  Tensor t;
  Ptr<TensorAllocator> allocator = New<TensorAllocator>(backend);
  allocator->reserveExact(size * sizeof(float));
  allocator->allocate(t, {1, size});
  allocators_.push_back(allocator);
  return t;
}

/**
 * Setup training environment and launch server thread and (if enabled) client
 * communication overlap threads.
 * Includes setting up MPI, node and shard sizes, clients, server shards and
 * communication overlap stuff.
 */
void MultiNodeGraphGroupSync::init(Ptr<data::Batch> batch) {
  // Setup clients and shards
  setupClients(batch);

  // setup sync sgd storage, We keep the summed gradient on Node 0
  sumGradientBuffer = newTensor(clientGraphs_[0]->params()->vals()->size(), clientGraphs_[0]->getBackend());
  accGradientsSync = newTensor(clientGraphs_[0]->params()->vals()->size(), clientGraphs_[0]->getBackend());
  accGradientsSync->set(0);
}

/**
 * Initialize the CPU arrays, with pinned memory for faster CudaMemCpy operations.
 * Requires the graph to be initialized first so we know its size
 */
void MultiNodeGraphGroupSync::initCPUArrays() {
  accGradientsSync_cpu = std::vector<float>(clientGraphs_[0]->params()->vals()->size());
  receiveBuffer_cpu = std::vector<float>(clientGraphs_[0]->params()->vals()->size());
}

/**
 * Setup MPI world size and rank of this node.
 */
void MultiNodeGraphGroupSync::setupMPI() {
#if MPI_FOUND
  MPI_Comm_size(MPI_COMM_WORLD, &mpi_comm_world_size_);
  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_my_rank_);
#endif
}

/**
 * Setup clients that will compute gradients and communicate them with the
 * server shards.
 * There is one client per GPU.
 */
void MultiNodeGraphGroupSync::setupClients(Ptr<data::Batch> batch) {
  runBatchThroughClientGraphs(batch);
  initCPUArrays();

  clientThreadPool_ = new marian::ThreadPool(devices_.size(), devices_.size());
}

/**
 * Initialize the graphs (models) of all clients on this node with the given
 * batch.
 */
void MultiNodeGraphGroupSync::runBatchThroughClientGraphs(Ptr<data::Batch> batch) {
  for(int i = 0; i < devices_.size(); i++) {
    THREAD_GUARD(clientBuilders_[i]->build(clientGraphs_[i], batch);
                 clientGraphs_[i]->forward();
                 clientGraphs_[i]->getBackend()->synchronize(););
  }
}

/**
 * Initialize variables required for overlapping client computations and
 * communication.
 * Includes summed and committed word counts, buffer flags, mutexes and
 * condition variables.
 */
void MultiNodeGraphGroupSync::sumGRAD(Tensor gradient) {
  std::lock_guard<std::mutex> guard(sumGradientMutex_);
  sumGradientBuffer->copyFrom(gradient);
  using namespace functional; //@TODO makes more sense to do that on the CPU i think
  Element(_1 += _2, accGradientsSync, sumGradientBuffer);
}

/**
 * If it's rank 0, it's a local update, if it's rank one it's remote
 * send and receive. Make sure you only call from device 0.
 */

void MultiNodeGraphGroupSync::sendReceiveUpdateSync() {
  #if MPI_FOUND
  // Copy the data to the CPU
  CUDA_CHECK(cudaMemcpy(accGradientsSync_cpu.data(),
                 accGradientsSync->data(),
                 accGradientsSync->size()*sizeof(float),
                 cudaMemcpyDeviceToHost));
  // Wait until all nodes are ready
  MPI_Barrier(MPI_COMM_WORLD);
  int reduce_result = MPI_Reduce(accGradientsSync_cpu.data(), //CPU buffers
              receiveBuffer_cpu.data(),
              accGradientsSync_cpu.size(),
              MPI_FLOAT,
              MPI_SUM,
              0, //Rank of the process with the data. In this case Node 0
              MPI_COMM_WORLD);
  if (reduce_result != MPI_SUCCESS) {
    LOG(critical, "Error: MPI_REDUCE failed with error {}.", reduce_result);
    std::abort();
  }
  if (mpi_my_rank_ == 0) {
    // Copy the data back to the GPU and do optimizer update
    CUDA_CHECK(cudaMemcpy(accGradientsSync->data(),
                 receiveBuffer_cpu.data(),
                 accGradientsSync->size()*sizeof(float),
                 cudaMemcpyHostToDevice));

    // Perform optimizer step
    syncOptimizer_->update(clientGraphs_[0]->params()->vals(),
                         accGradientsSync);

    // Copy the data back to the host
    CUDA_CHECK(cudaMemcpy(accGradientsSync_cpu.data(), //This is now the updated params
                   clientGraphs_[0]->params()->vals()->data(),
                   accGradientsSync->size()*sizeof(float),
                   cudaMemcpyDeviceToHost));
  }

  int bcast_result = MPI_Bcast(accGradientsSync_cpu.data(), //This is now the updated params.
            accGradientsSync_cpu.size(),
            MPI_FLOAT,
            0, //Root process
            MPI_COMM_WORLD);
  if (bcast_result != MPI_SUCCESS) {
    LOG(critical, "Error: MPI_REDUCE failed with error {}.", bcast_result);
    std::abort();
  }

  if (mpi_my_rank_ != 0) {
    //Copy the data to the GPU
    CUDA_CHECK(cudaMemcpy(clientGraphs_[0]->params()->vals()->data(),
                   accGradientsSync_cpu.data(),
                   accGradientsSync->size()*sizeof(float),
                   cudaMemcpyHostToDevice));
  }

  //Distribute the graph to the rest of the devices
  std::vector<std::thread> threads;
  for(int idx = 1; idx < devices_.size(); idx++) {
    threads.emplace_back(std::thread(
        [=](int idx) {
          //If NVLINK is not available it's faster to do this from the CPU
          //Because we don't have to go Device->Host->device
          CUDA_CHECK(cudaMemcpy(clientGraphs_[idx]->params()->vals()->data(),
                   accGradientsSync_cpu.data(),
                   accGradientsSync->size()*sizeof(float),
                   cudaMemcpyHostToDevice));
        },
        idx));
  }
  for(auto&& t : threads) {
    t.join();
  }
  //set the accumulating buffers to zero;
  accGradientsSync->set(0);
  std::fill(accGradientsSync_cpu.begin(), accGradientsSync_cpu.end(), 0);
  std::fill(receiveBuffer_cpu.begin(), receiveBuffer_cpu.end(), 0);
  #endif
}

/**
 * Execute given batch on this node, pushing/pulling the resulting
 * gradients/parameters to/from the server shards
 * or -- if comm. overlap enabled -- to/from the communication buffers, summing
 * gradients locally if the communication thread is busy
 *
 * @param batch Batch on which to perform forward and backward passes.
 */
void MultiNodeGraphGroupSync::execute(Ptr<data::Batch> fullBatch) {
  if(!initialized_) {
    init(fullBatch);
    initialized_ = true;
  }

  std::vector<Ptr<data::Batch>> batches = fullBatch->split(devices_.size());

  static int t = 0;

  static float cost = 0;
  static size_t num_seen_words = 0;
  static size_t num_seen_sentences = 0;

  {
    auto task = [this, batches](int my_id) {
      auto batch = batches[my_id];
      auto graph = clientGraphs_[my_id];
      auto builder = clientBuilders_[my_id];

      auto costNode = builder->build(graph, batch);

      if (t == 0) {
        if (my_id != 0)
          graph->params()->vals()->copyFrom(clientGraphs_[0]->params()->vals());
      }

      graph->forward();
      {
        std::lock_guard<std::mutex> guard(sumCostMutex_);
        cost += costNode->scalar();
        num_seen_words += batch->words();
        num_seen_sentences += batch->size();
      }
      graph->backward();

      graph->getBackend()->synchronize(); //@Alham do you know why we need this here?

      sumGRAD(graph->params()->grads());
    };

    ThreadPool pool(devices_.size(), devices_.size());
    for(int idx = 0; idx < devices_.size(); ++idx)
      pool.enqueue(task, idx);
  }

  if (t % tau_ == 0)
    sendReceiveUpdateSync();

  t++;

  // Run scheduler (if enabled)
  if(t % tau_ == 0 && scheduler_) {
    if (options_->get<std::string>("cost-type") != "ce-sum")
      cost /= (tau_ * devices_.size());

    if (tau_ > 1) {
      std::vector<size_t> fakeLength = {1, 1};
      auto fb = data::CorpusBatch::fakeBatch(fakeLength,
                                        num_seen_sentences,
                                        NULL);
      fb->front()->setWords(num_seen_words);
      scheduler_->update(cost, fb);
    } else {
      scheduler_->update(cost, fullBatch);
    }

    num_seen_words = 0;
    num_seen_sentences = 0;
    cost = 0;

    if((scheduler_->saving() || scheduler_->validating())) {
      #if MPI_FOUND
      //wait until other nodes are ready
      MPI_Barrier(MPI_COMM_WORLD);

      // TODO: Saving is broken
      //if(mpi_my_rank_ == 0 && scheduler_->saving())
      //  this->save(graph);

      if(mpi_my_rank_ == 0 && scheduler_->validating())
        scheduler_->validate(clientGraphs_);

      // inform other nodes to continue
      MPI_Barrier(MPI_COMM_WORLD);
      #endif
    }
    }

}
}
<|endoftext|>


<commit_before>#include <ctime>
#include <iostream>
#include <vector>
#include "../common/util.h"
#include "../common/json.h"
#include "localenv.h"

bool LocalEnv::setupHome() {
	std::ofstream hometest;
	hometest.open(m_homedir + "/.jobhome", std::ofstream::out);
	if (hometest.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	hometest.close();

	std::ofstream tab;
	tab.open(m_homedir + "/.inittab");
	if (tab.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	tab.close();

	std::ofstream lock;
	lock.open(m_homedir + "/MXLOCK");
	if (lock.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	lock << m_jobid;
	lock.close();

	//TODO: check if directories already exist
	mkdir((m_homedir + "/run").c_str(), 0700);
	mkdir((m_homedir + "/tmp").c_str(), 0700);
	return true;
}

bool LocalEnv::setupEnv() {
	std::ifstream environ;
	environ.open(m_homedir + "/.env", std::ofstream::in);
	if (environ.fail()) {
		environ.clear();
		return true;
	}

	std::string line;
	while (std::getline(environ, line)) {
		size_t pos = line.find("=");
		auto key = line.substr(0, pos);
		auto value = line.substr(pos + 1);

		setenv(key.c_str(), value.c_str(), 1);
	}

	environ.close();
	return true;
}

std::string LocalEnv::packageMain() {
	std::ifstream package;
	package.open(m_homedir + "/package.json", std::ofstream::in);
	if (package.fail()) {
		return "";
	}

	std::string str((std::istreambuf_iterator<char>(package)),
		std::istreambuf_iterator<char>());

	auto object = nlohmann::json::parse(str);
	if (!object.is_object())
		return "";

	return object["meta"]["main"].get<std::string>();
}

std::string LocalEnv::packageInvoke() {
	std::ifstream package;
	package.open(m_homedir + "/package.json", std::ofstream::in);
	if (package.fail()) {
		return "";
	}

	std::string str((std::istreambuf_iterator<char>(package)),
		std::istreambuf_iterator<char>());

	auto object = nlohmann::json::parse(str);
	if (!object.is_object())
		return "";

	return object["meta"]["invoke"].get<std::string>();
}

bool LocalEnv::teardown(double runtime) {
	std::ofstream lastrun, lastjobid, lastruntime;

	time_t t = time(NULL);
	struct tm *now = localtime(&t);

	lastrun.open(m_homedir + "/lastrun", std::ofstream::out);
	lastjobid.open(m_homedir + "/lastjobid", std::ofstream::out);
	lastruntime.open(m_homedir + "/lastruntime", std::ofstream::out);

	lastrun << (now->tm_year + 1900) << '-'
		 << (now->tm_mon + 1) << '-'
		 << now->tm_mday << ' '
		 << now->tm_hour << '-'
		 << now->tm_min << '-'
		 << now->tm_sec
		 << std::endl;

	lastjobid << m_jobid << std::endl;

	lastruntime << runtime << std::endl;

	lastruntime.close();
	lastjobid.close();
	lastrun.close();
	return true;
}
<commit_msg>Remove lock<commit_after>#include <ctime>
#include <iostream>
#include <vector>
#include <unistd.h>
#include "../common/util.h"
#include "../common/json.h"
#include "localenv.h"

bool LocalEnv::setupHome() {
	std::ofstream hometest;
	hometest.open(m_homedir + "/.jobhome", std::ofstream::out);
	if (hometest.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	hometest.close();

	std::ofstream tab;
	tab.open(m_homedir + "/.inittab");
	if (tab.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	tab.close();

	std::ofstream lock;
	lock.open(m_homedir + "/MXLOCK");
	if (lock.fail()) {
		std::cerr << m_homedir << " not writable" << std::endl;
		return false;
	}
	lock << m_jobid;
	lock.close();

	//TODO: check if directories already exist
	mkdir((m_homedir + "/run").c_str(), 0700);
	mkdir((m_homedir + "/tmp").c_str(), 0700);
	return true;
}

bool LocalEnv::setupEnv() {
	std::ifstream environ;
	environ.open(m_homedir + "/.env", std::ofstream::in);
	if (environ.fail()) {
		environ.clear();
		return true;
	}

	std::string line;
	while (std::getline(environ, line)) {
		size_t pos = line.find("=");
		auto key = line.substr(0, pos);
		auto value = line.substr(pos + 1);

		setenv(key.c_str(), value.c_str(), 1);
	}

	environ.close();
	return true;
}

std::string LocalEnv::packageMain() {
	std::ifstream package;
	package.open(m_homedir + "/package.json", std::ofstream::in);
	if (package.fail()) {
		return "";
	}

	std::string str((std::istreambuf_iterator<char>(package)),
		std::istreambuf_iterator<char>());

	auto object = nlohmann::json::parse(str);
	if (!object.is_object())
		return "";

	return object["meta"]["main"].get<std::string>();
}

std::string LocalEnv::packageInvoke() {
	std::ifstream package;
	package.open(m_homedir + "/package.json", std::ofstream::in);
	if (package.fail()) {
		return "";
	}

	std::string str((std::istreambuf_iterator<char>(package)),
		std::istreambuf_iterator<char>());

	auto object = nlohmann::json::parse(str);
	if (!object.is_object())
		return "";

	return object["meta"]["invoke"].get<std::string>();
}

bool LocalEnv::teardown(double runtime) {
	std::ofstream lastrun, lastjobid, lastruntime;

	time_t t = time(NULL);
	struct tm *now = localtime(&t);

	lastrun.open(m_homedir + "/lastrun", std::ofstream::out);
	lastjobid.open(m_homedir + "/lastjobid", std::ofstream::out);
	lastruntime.open(m_homedir + "/lastruntime", std::ofstream::out);

	lastrun << (now->tm_year + 1900) << '-'
		 << (now->tm_mon + 1) << '-'
		 << now->tm_mday << ' '
		 << now->tm_hour << '-'
		 << now->tm_min << '-'
		 << now->tm_sec
		 << std::endl;

	lastjobid << m_jobid << std::endl;

	lastruntime << runtime << std::endl;

	unlink((m_homedir + "/MXLOCK").c_str());

	lastruntime.close();
	lastjobid.close();
	lastrun.close();
	return true;
}
<|endoftext|>


<commit_before>/*
 * SyncedMMPLDProvider.cpp
 *
 * Copyright (C) 2019 by Universitaet Stuttgart (VISUS).
 * Alle Rechte vorbehalten.
 */

#include "stdafx.h"
#include "SyncedMMPLDProvider.h"

namespace megamol {
namespace stdplugin {
namespace datatools {

SyncedMMPLDProvider::SyncedMMPLDProvider()
    : StaticMMPLDProvider(), getSyncSlot("getSync", "Slot for synchronization messages.") {

    this->getSyncSlot.SetCallback(core::cluster::SyncDataSourcesCall::ClassName(),
        core::cluster::SyncDataSourcesCall::FunctionName(0), &SyncedMMPLDProvider::checkDirtyCallback);
    this->getSyncSlot.SetCallback(core::cluster::SyncDataSourcesCall::ClassName(),
        core::cluster::SyncDataSourcesCall::FunctionName(1), &SyncedMMPLDProvider::setDirtyCallback);
    MakeSlotAvailable(&getSyncSlot);
}

bool SyncedMMPLDProvider::setDirtyCallback(core::Call& c) {
    this->filenamesSlot.ForceSetDirty();

    return true;
}

bool SyncedMMPLDProvider::checkDirtyCallback(core::Call& c) {
    this->filenamesSlot.ResetDirty();
    auto ss = dynamic_cast<core::cluster::SyncDataSourcesCall*>(&c);

    ss->setFilenameDirty();

    return true;
}

} // namespace datatools
} // namespace stdplugin
} // namespace megamol<commit_msg>erst gruebeln, dann duebeln(TM)<commit_after>/*
 * SyncedMMPLDProvider.cpp
 *
 * Copyright (C) 2019 by Universitaet Stuttgart (VISUS).
 * Alle Rechte vorbehalten.
 */

#include "stdafx.h"
#include "SyncedMMPLDProvider.h"

namespace megamol {
namespace stdplugin {
namespace datatools {

SyncedMMPLDProvider::SyncedMMPLDProvider()
    : StaticMMPLDProvider(), getSyncSlot("getSync", "Slot for synchronization messages.") {

    this->getSyncSlot.SetCallback(core::cluster::SyncDataSourcesCall::ClassName(),
        core::cluster::SyncDataSourcesCall::FunctionName(0), &SyncedMMPLDProvider::checkDirtyCallback);
    this->getSyncSlot.SetCallback(core::cluster::SyncDataSourcesCall::ClassName(),
        core::cluster::SyncDataSourcesCall::FunctionName(1), &SyncedMMPLDProvider::setDirtyCallback);
    MakeSlotAvailable(&getSyncSlot);
}

bool SyncedMMPLDProvider::setDirtyCallback(core::Call& c) {
    this->filenamesSlot.ForceSetDirty();

    return true;
}

bool SyncedMMPLDProvider::checkDirtyCallback(core::Call& c) {
    auto ss = dynamic_cast<core::cluster::SyncDataSourcesCall*>(&c);

    if (this->filenamesSlot.IsDirty()) {
        ss->setFilenameDirty();
    }

    this->filenamesSlot.ResetDirty();

    return true;
}

} // namespace datatools
} // namespace stdplugin
} // namespace megamol<|endoftext|>


<commit_before>/*****************************************************************************

    (c) 2013 Hobu, Inc. hobu.inc@gmail.com

    Author: Uday Verma uday dot karan at gmail dot com

    This is free software; you can redistribute and/or modify it under the
    terms of the GNU Lesser General Licence as published by the Free Software
    Foundation. See the COPYING file for more information.

    This software is distributed WITHOUT ANY WARRANTY and without even the
    implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

*****************************************************************************/

#ifndef __HEXER_LAS_H__
#define __HEXER_LAS_H__

#include <boost/noncopyable.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/algorithm/string.hpp>

#include <algorithm>
#include <stdexcept>

class las_file : public boost::noncopyable {
public:
	las_file() : is_open_(false) {
    }

	~las_file() {
		close();
	}

	void open(const std::string& filename, int offset = 0, int count = -1) {
		using namespace boost::interprocess;

        start_offset_ = offset;
        count_ = count;

		pmapping_.reset(new file_mapping(filename.c_str(), read_only));
		pregion_.reset(new mapped_region(*pmapping_, read_only));

		void *addr = pregion_->get_address();

		std::string magic((char *)addr, (char *)addr + 4);
		if (!boost::iequals(magic, "LASF")) {
			throw std::runtime_error("Not a las file");
		}

		char versionMajor = readAs<char>(24);
		char versionMinor = readAs<char>(25);

		if ((int)(versionMajor * 10 + versionMinor) >= 13) {
			throw std::runtime_error("Only version 1.0-1.2 files are supported");
		}

		points_offset_ = readAs<unsigned int>(32*3);
		points_format_id_ = readAs<unsigned char>(32*3 + 8);
		points_struct_size_ = readAs<unsigned short>(32*3 + 8 + 1);
		points_count_ = readAs<unsigned int>(32*3 + 11);

		std::cerr << "points count: " << points_count_ << std::endl;

		size_t start = 32*3 + 35;
		readN(start, scale_, 3); start += sizeof(double) * 3;
		readN(start, offset_, 3); start += sizeof(double) * 3;

		maxs_[0] = readAs<double>(start); mins_[0] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);
		maxs_[1] = readAs<double>(start); mins_[1] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);
		maxs_[2] = readAs<double>(start); mins_[2] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);

		std::cerr << "region size: " << pregion_->get_size() << std::endl;
		std::cerr << "points offset: " << points_offset_ << std::endl;

		uint64_t diff = pregion_->get_size() - points_offset_;

		if (diff % stride() != 0)
			throw std::runtime_error("Point record data size is inconsistent");

		if (diff / stride() != points_count_)
			throw std::runtime_error("Point record count is inconsistent with computed point records size");

        updateMinsMaxes();

        is_open_ = true;
	}

	long size() {
		return pregion_->get_size();
	}

	void *points_offset() {
		return (char *)pregion_->get_address() + points_offset_ + start_offset_;
	}

	void close() {
		pregion_.reset();
		pmapping_.reset();
        is_open_ = false;
	}

	size_t stride() {
		if (points_struct_size_ != 0)
			return points_struct_size_;

		switch(points_format_id_) {
		case 2:
			return 26;
		case 3:
			return 26+2;
		default:
			break;
		}
		throw std::runtime_error("Unknown point format");
	}

	size_t points_count() {
        if (count_ == -1)
            return points_count_;

        return std::min(points_count_, (unsigned int)count_);
	}

	double* minimums() { return mins_; }
	double* maximums() { return maxs_; }

	double *scale() { return scale_; }
	double *offset() { return offset_; }

    bool is_open() { return is_open_; }

    inline double getX(size_t point)
    {
        char *position = (char*)points_offset() + stride()* point;

        int *xi = (int *)position;

        double x = *xi * scale_[0] + offset_[0];

        return x;
    }

    inline double getY(size_t point)
    {
        char *position = (char *)points_offset() + stride() * point + sizeof(int);

        int *yi = (int *)position;

        double y = *yi * scale_[1] + offset_[1];

        return y;
    }

private:
    void updateMinsMaxes() {
        if (start_offset_ == 0 && count_ == -1)
            return; // no update required if no subrange is requested

        int largest = std::numeric_limits<int>::max();
        int smallest = std::numeric_limits<int>::min();

        int n[3] = { largest, largest, largest };
        int x[3] = { smallest, smallest, smallest };

        char *ip = (char *)points_offset();
        for (size_t i = 0 ; i < points_count() ; i ++) {
            int *p = (int *)ip;
            for (int j = 0 ; j < 3 ; j ++) {
                n[j] = std::min(n[j], p[j]);
                x[j] = std::max(x[j], p[j]);
            }

            ip += stride();
        }

        for (int i = 0 ; i < 3 ; i++) {
            mins_[i] = n[i] * scale_[i] + offset_[i];
            maxs_[i] = x[i] * scale_[i] + offset_[i];
        }
    }

	template<typename T>
	T readAs(size_t offset) {
		return *((T*)((char*)pregion_->get_address() + offset));
	}

	template<typename T>
	void readN(size_t offset, T* dest, size_t n) {
		char *buf = (char *)pregion_->get_address() + offset;
		for(size_t i = 0 ; i < n ; i ++) {
			dest[i] = *((T*)(buf + sizeof(T) * i));
		}
	}

private:
	boost::shared_ptr<boost::interprocess::file_mapping> pmapping_;
	boost::shared_ptr<boost::interprocess::mapped_region> pregion_;

    unsigned int start_offset_;
	int count_;
	unsigned int points_offset_;
	unsigned char points_format_id_;
	unsigned int points_count_;
	unsigned short points_struct_size_;

	double scale_[3], offset_[3], mins_[3], maxs_[3];

    bool is_open_;
};

#endif // __HEXER_LAS_H__
<commit_msg>mark count method const<commit_after>/*****************************************************************************

    (c) 2013 Hobu, Inc. hobu.inc@gmail.com

    Author: Uday Verma uday dot karan at gmail dot com

    This is free software; you can redistribute and/or modify it under the
    terms of the GNU Lesser General Licence as published by the Free Software
    Foundation. See the COPYING file for more information.

    This software is distributed WITHOUT ANY WARRANTY and without even the
    implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

*****************************************************************************/

#ifndef __HEXER_LAS_H__
#define __HEXER_LAS_H__

#include <boost/noncopyable.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/algorithm/string.hpp>

#include <algorithm>
#include <stdexcept>

class las_file : public boost::noncopyable {
public:
	las_file() : is_open_(false) {
    }

	~las_file() {
		close();
	}

	void open(const std::string& filename, int offset = 0, int count = -1) {
		using namespace boost::interprocess;

        start_offset_ = offset;
        count_ = count;

		pmapping_.reset(new file_mapping(filename.c_str(), read_only));
		pregion_.reset(new mapped_region(*pmapping_, read_only));

		void *addr = pregion_->get_address();

		std::string magic((char *)addr, (char *)addr + 4);
		if (!boost::iequals(magic, "LASF")) {
			throw std::runtime_error("Not a las file");
		}

		char versionMajor = readAs<char>(24);
		char versionMinor = readAs<char>(25);

		if ((int)(versionMajor * 10 + versionMinor) >= 13) {
			throw std::runtime_error("Only version 1.0-1.2 files are supported");
		}

		points_offset_ = readAs<unsigned int>(32*3);
		points_format_id_ = readAs<unsigned char>(32*3 + 8);
		points_struct_size_ = readAs<unsigned short>(32*3 + 8 + 1);
		points_count_ = readAs<unsigned int>(32*3 + 11);

		std::cerr << "points count: " << points_count_ << std::endl;

		size_t start = 32*3 + 35;
		readN(start, scale_, 3); start += sizeof(double) * 3;
		readN(start, offset_, 3); start += sizeof(double) * 3;

		maxs_[0] = readAs<double>(start); mins_[0] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);
		maxs_[1] = readAs<double>(start); mins_[1] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);
		maxs_[2] = readAs<double>(start); mins_[2] = readAs<double>(start + sizeof(double)); start += 2*sizeof(double);

		std::cerr << "region size: " << pregion_->get_size() << std::endl;
		std::cerr << "points offset: " << points_offset_ << std::endl;

		uint64_t diff = pregion_->get_size() - points_offset_;

		if (diff % stride() != 0)
			throw std::runtime_error("Point record data size is inconsistent");

		if (diff / stride() != points_count_)
			throw std::runtime_error("Point record count is inconsistent with computed point records size");

        updateMinsMaxes();

        is_open_ = true;
	}

	long size() {
		return pregion_->get_size();
	}

	void *points_offset() {
		return (char *)pregion_->get_address() + points_offset_ + start_offset_;
	}

	void close() {
		pregion_.reset();
		pmapping_.reset();
        is_open_ = false;
	}

	size_t stride() {
		if (points_struct_size_ != 0)
			return points_struct_size_;

		switch(points_format_id_) {
		case 2:
			return 26;
		case 3:
			return 26+2;
		default:
			break;
		}
		throw std::runtime_error("Unknown point format");
	}

	size_t points_count() const {
        if (count_ == -1)
            return points_count_;

        return std::min(points_count_, (unsigned int)count_);
	}

	double* minimums() { return mins_; }
	double* maximums() { return maxs_; }

	double *scale() { return scale_; }
	double *offset() { return offset_; }

    bool is_open() { return is_open_; }

    inline double getX(size_t point)
    {
        char *position = (char*)points_offset() + stride()* point;

        int *xi = (int *)position;

        double x = *xi * scale_[0] + offset_[0];

        return x;
    }

    inline double getY(size_t point)
    {
        char *position = (char *)points_offset() + stride() * point + sizeof(int);

        int *yi = (int *)position;

        double y = *yi * scale_[1] + offset_[1];

        return y;
    }

private:
    void updateMinsMaxes() {
        if (start_offset_ == 0 && count_ == -1)
            return; // no update required if no subrange is requested

        int largest = std::numeric_limits<int>::max();
        int smallest = std::numeric_limits<int>::min();

        int n[3] = { largest, largest, largest };
        int x[3] = { smallest, smallest, smallest };

        char *ip = (char *)points_offset();
        for (size_t i = 0 ; i < points_count() ; i ++) {
            int *p = (int *)ip;
            for (int j = 0 ; j < 3 ; j ++) {
                n[j] = std::min(n[j], p[j]);
                x[j] = std::max(x[j], p[j]);
            }

            ip += stride();
        }

        for (int i = 0 ; i < 3 ; i++) {
            mins_[i] = n[i] * scale_[i] + offset_[i];
            maxs_[i] = x[i] * scale_[i] + offset_[i];
        }
    }

	template<typename T>
	T readAs(size_t offset) {
		return *((T*)((char*)pregion_->get_address() + offset));
	}

	template<typename T>
	void readN(size_t offset, T* dest, size_t n) {
		char *buf = (char *)pregion_->get_address() + offset;
		for(size_t i = 0 ; i < n ; i ++) {
			dest[i] = *((T*)(buf + sizeof(T) * i));
		}
	}

private:
	boost::shared_ptr<boost::interprocess::file_mapping> pmapping_;
	boost::shared_ptr<boost::interprocess::mapped_region> pregion_;

    unsigned int start_offset_;
	int count_;
	unsigned int points_offset_;
	unsigned char points_format_id_;
	unsigned int points_count_;
	unsigned short points_struct_size_;

	double scale_[3], offset_[3], mins_[3], maxs_[3];

    bool is_open_;
};

#endif // __HEXER_LAS_H__
<|endoftext|>


<commit_before><commit_msg>Fix a crash reported in ChromeFrame on the crash server, which occurs because the VARIANT representing the URL has a NULL bstrVal. <commit_after><|endoftext|>