<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: drawsh.hxx,v $
 *
 *  $Revision: 1.8 $
 *
 *  last change: $Author: rt $ $Date: 2007-07-06 12:28:04 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef SC_DRAWSH_HXX
#define SC_DRAWSH_HXX

#ifndef _SFX_SHELL_HXX //autogen
#include <sfx2/shell.hxx>
#endif
#include "shellids.hxx"
#ifndef _SFXMODULE_HXX //autogen
#include <sfx2/module.hxx>
#endif

#ifndef _SVDMARK_HXX //autogen
#include <svx/svdmark.hxx>
#endif

#ifndef _LINK_HXX
#include <tools/link.hxx>
#endif

class AbstractSvxNameDialog; //CHINA001 class SvxNameDialog;
class ScViewData;
class ScDrawView;

class ScDrawShell : public SfxShell
{
    ScViewData* pViewData;

    DECL_LINK( NameObjectHdl, AbstractSvxNameDialog* );

protected:
    ScViewData* GetViewData()   { return pViewData; }

public:
    TYPEINFO();
    SFX_DECL_INTERFACE(SCID_DRAW_SHELL)

                    ScDrawShell(ScViewData* pData);
                    ~ScDrawShell();

    void    StateDisableItems( SfxItemSet &rSet );

    void    ExecDrawAttr(SfxRequest& rReq);
    void    GetDrawAttrState(SfxItemSet &rSet);
    void    GetAttrFuncState(SfxItemSet &rSet);

    void    ExecDrawFunc(SfxRequest& rReq);
    void    GetDrawFuncState(SfxItemSet &rSet);
    void    GetState(SfxItemSet &rSet);

    void    ExecFormText(SfxRequest& rReq);     // StarFontWork
    void    GetFormTextState(SfxItemSet& rSet);

    void    ExecuteHLink(SfxRequest& rReq);     // Hyperlink
    void    GetHLinkState(SfxItemSet& rSet);

    void    ExecFormatPaintbrush(SfxRequest& rReq);
    void    StateFormatPaintbrush(SfxItemSet& rSet);

    void    ExecuteLineDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );
    void    ExecuteAreaDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );
    void    ExecuteTextAttrDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );

    ScDrawView* GetDrawView();

    BOOL    AreAllObjectsOnLayer(USHORT nLayerNo,const SdrMarkList& rMark);
};

#endif

<commit_msg>INTEGRATION: CWS dr55 (1.7.84); FILE MERGED 2007/06/28 13:58:02 dr 1.7.84.3: #i66550# disabled 2007/06/15 15:41:21 dr 1.7.84.2: #i66550# add hyperlinks to shapes, patch by npower, reviewed by dr 2007/06/13 14:35:56 dr 1.7.84.1: #i59082# assign macros to drawing objects and pictures, patch by npower, reviewed by dr<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: drawsh.hxx,v $
 *
 *  $Revision: 1.9 $
 *
 *  last change: $Author: rt $ $Date: 2007-07-06 12:44:12 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef SC_DRAWSH_HXX
#define SC_DRAWSH_HXX

#ifndef _SFX_SHELL_HXX //autogen
#include <sfx2/shell.hxx>
#endif
#include "shellids.hxx"
#ifndef _SFXMODULE_HXX //autogen
#include <sfx2/module.hxx>
#endif

#ifndef _SVDMARK_HXX //autogen
#include <svx/svdmark.hxx>
#endif

#ifndef _LINK_HXX
#include <tools/link.hxx>
#endif

class AbstractSvxNameDialog; //CHINA001 class SvxNameDialog;
class ScViewData;
class ScDrawView;

class ScDrawShell : public SfxShell
{
    ScViewData* pViewData;

    DECL_LINK( NameObjectHdl, AbstractSvxNameDialog* );

#ifdef ISSUE66550_HLINK_FOR_SHAPES
    void SetHlinkForObject( SdrObject* pObj, const rtl::OUString& rHlnk );
#endif

protected:
    ScViewData* GetViewData()   { return pViewData; }

public:
    TYPEINFO();
    SFX_DECL_INTERFACE(SCID_DRAW_SHELL)

                    ScDrawShell(ScViewData* pData);
                    ~ScDrawShell();

    void    StateDisableItems( SfxItemSet &rSet );

    void    ExecDrawAttr(SfxRequest& rReq);
    void    GetDrawAttrState(SfxItemSet &rSet);
    void    GetAttrFuncState(SfxItemSet &rSet);

    void    ExecDrawFunc(SfxRequest& rReq);
    void    GetDrawFuncState(SfxItemSet &rSet);
    void    GetState(SfxItemSet &rSet);

    void    ExecFormText(SfxRequest& rReq);     // StarFontWork
    void    GetFormTextState(SfxItemSet& rSet);

    void    ExecuteHLink(SfxRequest& rReq);     // Hyperlink
    void    GetHLinkState(SfxItemSet& rSet);

    void    ExecFormatPaintbrush(SfxRequest& rReq);
    void    StateFormatPaintbrush(SfxItemSet& rSet);

    void    ExecuteMacroAssign( SdrObject* pObj, Window* pWin );
    void    ExecuteLineDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );
    void    ExecuteAreaDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );
    void    ExecuteTextAttrDlg( SfxRequest& rReq, USHORT nTabPage = 0xffff );

    ScDrawView* GetDrawView();

    BOOL    AreAllObjectsOnLayer(USHORT nLayerNo,const SdrMarkList& rMark);
};

#endif

<|endoftext|>


<commit_before>/*
 * Provide entropy for the GLib PRNG.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "random.hxx"
#include "fd_util.h"
#include "fd-util.h"

#include <daemon/log.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>

static unsigned
read_some_entropy(const char *path, guint32 *dest, unsigned max)
{
    int fd = open_cloexec(path, O_RDONLY, 0);
    if (fd < 0) {
        daemon_log(2, "Failed to open %s: %s\n", path, strerror(errno));
        return 0;
    }

    fd_set_nonblock(fd, true);

    ssize_t nbytes = read(fd, dest, max * sizeof(dest[0]));
    if (nbytes < 0) {
        if (errno != EAGAIN)
            daemon_log(2, "Failed to read from %s: %s\n",
                       path, strerror(errno));
        close(fd);
        return 0;
    }

    close(fd);
    return nbytes / sizeof(dest[0]);
}

static size_t
obtain_entropy(uint32_t *p, size_t size)
{
    /* read from /dev/random for strong entropy */

    unsigned n = read_some_entropy("/dev/random", p, size);

    /* fill the rest up with /dev/urandom */

    if (n < size)
        n += read_some_entropy("/dev/urandom", p + n, size - n);

    return n;
}

void
obtain_entropy(GRand *r)
{
    guint32 seed[64];
    auto n = obtain_entropy(seed, G_N_ELEMENTS(seed));
    if (n < 8) {
        daemon_log(2, "Not enough entropy found\n");
        return;
    }

    g_rand_set_seed_array(r, seed, n);
}
<commit_msg>random: convert read_some_entropy() to template<commit_after>/*
 * Provide entropy for the GLib PRNG.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "random.hxx"
#include "fd_util.h"
#include "fd-util.h"

#include <daemon/log.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>

template<typename T>
static unsigned
read_some_entropy(const char *path, T *dest, unsigned max)
{
    int fd = open_cloexec(path, O_RDONLY, 0);
    if (fd < 0) {
        daemon_log(2, "Failed to open %s: %s\n", path, strerror(errno));
        return 0;
    }

    fd_set_nonblock(fd, true);

    ssize_t nbytes = read(fd, dest, max * sizeof(dest[0]));
    if (nbytes < 0) {
        if (errno != EAGAIN)
            daemon_log(2, "Failed to read from %s: %s\n",
                       path, strerror(errno));
        close(fd);
        return 0;
    }

    close(fd);
    return nbytes / sizeof(dest[0]);
}

static size_t
obtain_entropy(uint32_t *p, size_t size)
{
    /* read from /dev/random for strong entropy */

    unsigned n = read_some_entropy("/dev/random", p, size);

    /* fill the rest up with /dev/urandom */

    if (n < size)
        n += read_some_entropy("/dev/urandom", p + n, size - n);

    return n;
}

void
obtain_entropy(GRand *r)
{
    guint32 seed[64];
    auto n = obtain_entropy(seed, G_N_ELEMENTS(seed));
    if (n < 8) {
        daemon_log(2, "Not enough entropy found\n");
        return;
    }

    g_rand_set_seed_array(r, seed, n);
}
<|endoftext|>


<commit_before>/*
 * PrivateCommand.cpp
 *
 * Copyright (C) 2009-17 by RStudio, Inc.
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/terminal/PrivateCommand.hpp>

#include <core/system/System.hpp>

namespace rstudio {
namespace core {
namespace terminal {

namespace {

boost::posix_time::ptime now()
{
   return boost::posix_time::microsec_clock::universal_time();
}

const std::string kEol = "\r\n";

} // anonymous namespace

/*
 * Implementation note: private command output is detected by echoing start and end
 * marker (uuids) around the output, e.g. the command "ls -l" would be issued as:
 *
 * echo BOM && ls -l && EOM
 *
 * where BOM and EOM would be actual uuids. This assumes our shell supports && for
 * issuing sequential commands.
 *
 * Command is prefixed with a space, to take advantage of
 * HISTCONTROL=ignorespace (or ignoreboth) as a way to prevent command from
 * showing up in shell history.
 */

PrivateCommand::PrivateCommand(const std::string& command,
                               int privateCommandDelayMs,
                               int waitAfterCommandDelayMs,
                               int privateCommandTimeoutMs,
                               bool oncePerUserCommand)
   :
     command_(command),
     oncePerUserCommand_(oncePerUserCommand),
     privateCommandLoop_(false),
     lastPrivateCommand_(boost::posix_time::not_a_date_time),
     lastEnterTime_(boost::posix_time::not_a_date_time),
     pendingCommand_(false),
     privateCommandDelay_(privateCommandDelayMs),
     waitForCommandDelay_(waitAfterCommandDelayMs),
     privateCommandTimeout_(privateCommandTimeoutMs),
     firstCRLF_(std::string::npos),
     outputStart_(std::string::npos),
     timeout_(false)
{
   outputBOM_ = core::system::generateShortenedUuid();
   outputEOM_ = core::system::generateShortenedUuid();

   std::string commandBefore_ = " echo ";
   commandBefore_ += outputBOM_;
   std::string commandAfter_ = "echo ";
   commandAfter_ += outputEOM_;

   fullCommand_ = commandBefore_ + " && " + command + " && " + commandAfter_ + "\n";
}

bool PrivateCommand::onTryCapture(core::system::ProcessOperations& ops, bool hasChildProcs)
{
   boost::posix_time::ptime currentTime = now();
   if (privateCommandLoop_)
   {
      if (!output_.empty())
      {
         privateCommandLoop_ = false;
         return false; // all done
      }

      if (currentTime - lastPrivateCommand_ > privateCommandTimeout_)
      {
         terminateCapture();
         ops.ptyInterrupt();
         timeout_ = true;
         return false;
      }

      return true; // still waiting for output
   }
   else
   {
      if (hasChildProcs)
         return false;

      if (pendingCommand_ || (oncePerUserCommand_ && lastEnterTime_.is_not_a_date_time()))
      {
         // We don't start a private command if something is being typed, or a command has never
         // been run.
         lastPrivateCommand_ = currentTime;
         return false;
      }

      if (!lastEnterTime_.is_not_a_date_time() &&
          (currentTime - waitForCommandDelay_ <= lastEnterTime_))
      {
         // not enough time has elapsed since last command was submitted
         return false;
      }

      if (!lastPrivateCommand_.is_not_a_date_time() &&
          currentTime - privateCommandDelay_ <= lastPrivateCommand_)
      {
         // not enough time has elapsed since last private command ran
         return false;
      }

      if (oncePerUserCommand_ &&
          !lastPrivateCommand_.is_not_a_date_time() &&
          lastPrivateCommand_ > lastEnterTime_)
      {
         // Hasn't been a new command executed since our last private command, no need
         // to run it.
         return false;
      }

      lastPrivateCommand_ = currentTime;
      privateCommandOutput_.clear();
      privateCommandLoop_ = true;
      timeout_ = false;

      // send the command
      Error error = ops.writeToStdin(fullCommand_, false);
      if (error)
      {
         LOG_ERROR(error);
         privateCommandLoop_ = false;
         lastPrivateCommand_ = boost::posix_time::pos_infin; // disable private commands
         return false;
      }
      return true;
   }
   return false;
}

bool PrivateCommand::hasCaptured() const
{
   return privateCommandLoop_;
}

void PrivateCommand::userInput(const std::string& input)
{
   if (!input.empty() && (*input.rbegin() == '\r' || *input.rbegin() == '\n'))
   {
      lastEnterTime_ = now();
      pendingCommand_ = false;
   }
   else
   {
      pendingCommand_ = true;
   }
}

bool PrivateCommand::output(const std::string& output)
{
   if (!hasCaptured())
   {
      return false;
   }

   privateCommandOutput_.append(output);

   // find the end of command line
   if (firstCRLF_ == std::string::npos)
   {
      firstCRLF_ = privateCommandOutput_.find(kEol, 0);
      if (firstCRLF_ == std::string::npos)
      {
         return true;
      }
      firstCRLF_ += kEol.length();
   }

   // find the start of output
   if (outputStart_ == std::string::npos)
   {
      std::string bomLine = outputBOM_ + kEol;
      outputStart_ = privateCommandOutput_.find(bomLine, firstCRLF_);
      if (outputStart_ == std::string::npos)
      {
         return true;
      }
      outputStart_ += bomLine.length();
   }

   // find the end of output
   std::string eomLine = outputEOM_ + kEol;
   size_t outputEnd = privateCommandOutput_.find(eomLine, outputStart_);
   if (outputEnd == std::string::npos)
   {
      return true;
   }

   // extract the command output
   output_ = privateCommandOutput_.substr(outputStart_, outputEnd - outputStart_);

   resetParse();
   return true;
}

std::string PrivateCommand::getPrivateOutput()
{
   if (hasCaptured()) // still in-process
      return "";

   std::string result = output_;
   output_.clear();
   return result;
}

void PrivateCommand::resetParse()
{
   firstCRLF_ = std::string::npos;
   outputStart_ = std::string::npos;
   privateCommandOutput_.clear();
}

void PrivateCommand::terminateCapture()
{
   resetParse();
   privateCommandLoop_ = false;
   output_.clear();
}

bool PrivateCommand::timeout()
{
   bool didTimeout = timeout_;
   timeout_ = false;
   return didTimeout;
}

} // namespace terminal
} // namespace core
} // namespace rstudio
<commit_msg>Log warning when private command times out<commit_after>/*
 * PrivateCommand.cpp
 *
 * Copyright (C) 2009-17 by RStudio, Inc.
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/terminal/PrivateCommand.hpp>

#include <core/system/System.hpp>

namespace rstudio {
namespace core {
namespace terminal {

namespace {

boost::posix_time::ptime now()
{
   return boost::posix_time::microsec_clock::universal_time();
}

const std::string kEol = "\r\n";

} // anonymous namespace

/*
 * Implementation note: private command output is detected by echoing start and end
 * marker (uuids) around the output, e.g. the command "ls -l" would be issued as:
 *
 * echo BOM && ls -l && EOM
 *
 * where BOM and EOM would be actual uuids. This assumes our shell supports && for
 * issuing sequential commands.
 *
 * Command is prefixed with a space, to take advantage of
 * HISTCONTROL=ignorespace (or ignoreboth) as a way to prevent command from
 * showing up in shell history.
 */

PrivateCommand::PrivateCommand(const std::string& command,
                               int privateCommandDelayMs,
                               int waitAfterCommandDelayMs,
                               int privateCommandTimeoutMs,
                               bool oncePerUserCommand)
   :
     command_(command),
     oncePerUserCommand_(oncePerUserCommand),
     privateCommandLoop_(false),
     lastPrivateCommand_(boost::posix_time::not_a_date_time),
     lastEnterTime_(boost::posix_time::not_a_date_time),
     pendingCommand_(false),
     privateCommandDelay_(privateCommandDelayMs),
     waitForCommandDelay_(waitAfterCommandDelayMs),
     privateCommandTimeout_(privateCommandTimeoutMs),
     firstCRLF_(std::string::npos),
     outputStart_(std::string::npos),
     timeout_(false)
{
   outputBOM_ = core::system::generateShortenedUuid();
   outputEOM_ = core::system::generateShortenedUuid();

   std::string commandBefore_ = " echo ";
   commandBefore_ += outputBOM_;
   std::string commandAfter_ = "echo ";
   commandAfter_ += outputEOM_;

   fullCommand_ = commandBefore_ + " && " + command + " && " + commandAfter_ + "\n";
}

bool PrivateCommand::onTryCapture(core::system::ProcessOperations& ops, bool hasChildProcs)
{
   boost::posix_time::ptime currentTime = now();
   if (privateCommandLoop_)
   {
      if (!output_.empty())
      {
         privateCommandLoop_ = false;
         return false; // all done
      }

      if (currentTime - lastPrivateCommand_ > privateCommandTimeout_)
      {
         LOG_WARNING_MESSAGE("PrivateCommand timeout");
         terminateCapture();
         ops.ptyInterrupt();
         timeout_ = true;
         return false;
      }

      return true; // still waiting for output
   }
   else
   {
      if (hasChildProcs)
         return false;

      if (pendingCommand_ || (oncePerUserCommand_ && lastEnterTime_.is_not_a_date_time()))
      {
         // We don't start a private command if something is being typed, or a command has never
         // been run.
         lastPrivateCommand_ = currentTime;
         return false;
      }

      if (!lastEnterTime_.is_not_a_date_time() &&
          (currentTime - waitForCommandDelay_ <= lastEnterTime_))
      {
         // not enough time has elapsed since last command was submitted
         return false;
      }

      if (!lastPrivateCommand_.is_not_a_date_time() &&
          currentTime - privateCommandDelay_ <= lastPrivateCommand_)
      {
         // not enough time has elapsed since last private command ran
         return false;
      }

      if (oncePerUserCommand_ &&
          !lastPrivateCommand_.is_not_a_date_time() &&
          lastPrivateCommand_ > lastEnterTime_)
      {
         // Hasn't been a new command executed since our last private command, no need
         // to run it.
         return false;
      }

      lastPrivateCommand_ = currentTime;
      privateCommandOutput_.clear();
      privateCommandLoop_ = true;
      timeout_ = false;

      // send the command
      Error error = ops.writeToStdin(fullCommand_, false);
      if (error)
      {
         LOG_ERROR(error);
         privateCommandLoop_ = false;
         lastPrivateCommand_ = boost::posix_time::pos_infin; // disable private commands
         return false;
      }
      return true;
   }
   return false;
}

bool PrivateCommand::hasCaptured() const
{
   return privateCommandLoop_;
}

void PrivateCommand::userInput(const std::string& input)
{
   if (!input.empty() && (*input.rbegin() == '\r' || *input.rbegin() == '\n'))
   {
      lastEnterTime_ = now();
      pendingCommand_ = false;
   }
   else
   {
      pendingCommand_ = true;
   }
}

bool PrivateCommand::output(const std::string& output)
{
   if (!hasCaptured())
   {
      return false;
   }

   privateCommandOutput_.append(output);

   // find the end of command line
   if (firstCRLF_ == std::string::npos)
   {
      firstCRLF_ = privateCommandOutput_.find(kEol, 0);
      if (firstCRLF_ == std::string::npos)
      {
         return true;
      }
      firstCRLF_ += kEol.length();
   }

   // find the start of output
   if (outputStart_ == std::string::npos)
   {
      std::string bomLine = outputBOM_ + kEol;
      outputStart_ = privateCommandOutput_.find(bomLine, firstCRLF_);
      if (outputStart_ == std::string::npos)
      {
         return true;
      }
      outputStart_ += bomLine.length();
   }

   // find the end of output
   std::string eomLine = outputEOM_ + kEol;
   size_t outputEnd = privateCommandOutput_.find(eomLine, outputStart_);
   if (outputEnd == std::string::npos)
   {
      return true;
   }

   // extract the command output
   output_ = privateCommandOutput_.substr(outputStart_, outputEnd - outputStart_);

   resetParse();
   return true;
}

std::string PrivateCommand::getPrivateOutput()
{
   if (hasCaptured()) // still in-process
      return "";

   std::string result = output_;
   output_.clear();
   return result;
}

void PrivateCommand::resetParse()
{
   firstCRLF_ = std::string::npos;
   outputStart_ = std::string::npos;
   privateCommandOutput_.clear();
}

void PrivateCommand::terminateCapture()
{
   resetParse();
   privateCommandLoop_ = false;
   output_.clear();
}

bool PrivateCommand::timeout()
{
   bool didTimeout = timeout_;
   timeout_ = false;
   return didTimeout;
}

} // namespace terminal
} // namespace core
} // namespace rstudio
<|endoftext|>


<commit_before>/*
 * replay.cpp
 *
 * eXploration module for Autonomous Robotic Embedded Systems
 *
 * xares replay : replay dump from xares (and xares-genom)
 *
 * author:  Cyril Robin <cyril.robin@laas.fr>
 * created: 2013-09-19
 * license: BSD
 */

#include <ostream>
#include <fstream>
#include <sys/time.h>

#include "xares/xares.hpp"
#include "gladys/point.hpp"
#include "gladys/weight_map.hpp"

int replay_genom_dump( const std::string& path_to_dump_file) {//{{{

  std::ifstream dump_file;
  dump_file.open (path_to_dump_file) ;

  if ( dump_file.is_open () ) {
    // init
    struct timeval tv0, tv1;

    std::string line, name ;
    gladys::points_t r_pos;
    int n, max_nf ;
    double min_size, x_origin, y_origin, height_max, width_max ;
    double x, y, yaw, min_dist, max_dist ;

    //read data
    getline (dump_file,line) ;
    std::istringstream iss1(line);
    iss1 >> name >> n ;
    std::cerr << "[Xares replay] r_pos ["<< n << "] =";
    while ( n-- > 0) {
      iss1 >> x >> y ;
      r_pos.push_back( gladys::point_xy_t {x,y} );
      std::cerr << " ("<< x << "," << y << ")";
    }
    std::cerr << std::endl;

    getline (dump_file,line) ;
    std::istringstream iss2(line);
    iss2 >> name >> yaw ;
    std::cerr << "[Xares replay] yaw = " << yaw << std::endl;

    // load and display internal params
    getline (dump_file,line) ;
    std::istringstream iss3(line);
    iss3 >> name >> max_nf >> min_size >> min_dist >> max_dist ;
    std::cerr << "[Xares replay] internal params "
              << "(max_nf,min_size,min_dist,max_dist) = ("
              << max_nf << "," << min_size << "," << min_dist << "," << max_dist
              << ")" << std::endl;

    std::cerr << "[Xares replay] max nbr of frontiers = " << max_nf << std::endl;
    getline (dump_file,line) ;
    gladys::weight_map wm ( line );

    // weightmap : origin, size, scale
    std::cerr << "[Xares repay] weight map: (x0,y0,xSc,ySc,W,H) = ("
              << wm.get_utm_pose_x()<< ","
              << wm.get_utm_pose_y()<< ","
              << wm.get_scale_x()   << ","
              << wm.get_scale_y()   << ","
              << wm.get_height()    << ","
              << wm.get_width()     << ")"
              << std::endl;

    // bounded area to explore
    getline (dump_file,line) ;
    std::istringstream iss4(line);
    iss4 >> name >> x_origin >> y_origin >> height_max >> width_max ;
    std::cerr << "[Xares replay] bounded area : (x0,y0,W,H) "
              << x_origin << "," << y_origin << ","
              << height_max << "," << width_max
              << ")" << std::endl;

    dump_file.close();

    // load the planner
    gettimeofday(&tv0, NULL);
    xares::xares xp( wm, x_origin, y_origin, height_max, width_max );
    gettimeofday(&tv1, NULL);

    std::cerr << "[Xares replay] planner loaded ("
              << (tv1.tv_sec -  tv0.tv_sec) * 1000 +
              (tv1.tv_usec - tv0.tv_usec) / 1000 << " ms)." << std::endl;

    // Set internal parameters for xares
    xp.set_params( max_nf, min_size, min_dist, max_dist );
    std::cerr << "[Xares replay] parameters settled. Let's plan !" << std::endl;

    //plan (=replay !)
    gettimeofday(&tv0, NULL);
    xares::xares::return_value rv = xp.plan( r_pos, yaw );
    gettimeofday(&tv1, NULL);

    // parse results
    if ( rv == xares::xares::XARES_FAILURE ) {
      std::cerr << "[Xares replay] #EEE# xares : unexpected data ; unable to compute frontiers :'( " << std::endl;
      return EXIT_SUCCESS;
    }

    if ( rv == xares::xares::XARES_NO_FRONTIER ) {
      std::cerr << "[Xares replay] #EEE# xares : no valuable frontier detected." << std::endl;
      return EXIT_SUCCESS;
    }

    // From here, there should be a valid plan
    std::cerr << "[Xares replay] Plan computed ("
              << (tv1.tv_sec -  tv0.tv_sec) * 1000 +
              (tv1.tv_usec - tv0.tv_usec) / 1000 << " ms)" << std::endl;

    //get planned path
    const gladys::path_t& path = xp.get_goal().path ;

    for (unsigned int i = 0 ; i < path.size() ; i++)
      std::cerr   << "[Xares replay ] ----waypoint #" << i
                  << " = (" << path[i][0]<< "," << path[i][1]
                  <<")"<<std::endl;

    // get poster path
    gladys::path_t::const_iterator it;
    double dist = 0.0;

    gladys::point_xy_t curr = path.front() ;

    unsigned int pt = 0 ;
    for ( it = path.begin(); it != path.end() ; ++it)
    {
      dist += gladys::distance( curr, *it );
      curr = *it;

      if ( dist > 6.0 ) {
        std::cerr   << "[Xares replay ] ---- poster waypoint #" << pt++
                    << " = (" << (*it)[0]<< "," << (*it)[1]
                    <<")"<<std::endl;
      }
    }

    std::cerr   << "[Xares replay ] ---- poster waypoint #" << pt
                << " = (" << curr[0]<< "," << curr[1]
                <<")"<<std::endl;

    // ASCII plot of weightmap, seed & goal
    std::cerr   << "[Xares replay ] ASCII weight map : " << std::endl;
    size_t index0 = wm.index_utm( gladys::point_xy_t { 0.0 , 0.0 } ) ; //origin
    size_t indexS = wm.index_utm( r_pos[0] ) ; // seed (robot position)
    size_t indexG = wm.index_utm( curr ) ;  // goal
    size_t index_curr ;

    // foreach from (origin) to (dim+origin)
    for ( double j = wm.get_utm_pose_y(); j < wm.get_width()*wm.get_scale_y() + wm.get_utm_pose_y(); j+= wm.get_scale_y() )
    {
      if ( j < y_origin || j > width_max ) continue; //cropping
      for (double i = wm.get_utm_pose_x(); i < wm.get_height()*wm.get_scale_x() + wm.get_utm_pose_x(); i+= wm.get_scale_x() )
      {
        if ( i < x_origin || i > height_max ) continue; //cropping

        index_curr = wm.index_utm( gladys::point_xy_t {(double)i,(double)j} );

        // Special points
        // O = origin
        // S = Seed
        // G = Goal
        if ( index_curr == index0 ) { // origin
            std::cerr << "O"; continue; }
        if ( index_curr == indexS ) { // seed
            std::cerr << "S"; continue; }
        if ( index_curr == indexG ) { // goal
            std::cerr << "G"; continue;}

        // Common points
        // u = unknown
        // . = known
        // # = obstacle
        double w = wm.get_weight_band()[ index_curr ];
        if        (w < 0)     std::cerr << "u"; //unknonwn
        else if   (w > 100 )  std::cerr << "#"; //obstacle
        else                  std::cerr << ".";

      }
      std::cerr << std::endl;
    }
  }
  else {
    std::cerr << "[Xares replay] Cannot open provided dump file. :-(" << std::endl;
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}//}}}

int replay_lib_dump(char dumpFile[]) {//{{{

  std::cerr << "Not yet implemented..." << std::endl;

  return EXIT_SUCCESS;

}//}}}

int main(int argc, char **argv) {//{{{

  int return_value ;
  return_value = replay_genom_dump( argv[1] ) ;

  std::cerr << "[Xares replay] Done." << std::endl;

  return return_value;

}//}}}

<commit_msg>[replay] Add colored output (enhance the display)<commit_after>/*
 * replay.cpp
 *
 * eXploration module for Autonomous Robotic Embedded Systems
 *
 * xares replay : replay dump from xares (and xares-genom)
 *
 * author:  Cyril Robin <cyril.robin@laas.fr>
 * created: 2013-09-19
 * license: BSD
 */

#include <ostream>
#include <fstream>
#include <sys/time.h>

#include "xares/xares.hpp"
#include "gladys/point.hpp"
#include "gladys/weight_map.hpp"

// Colours (output)
#define colour_bg_green  "\33[37;42m"
#define colour_bg_yellow "\33[30;43m"
#define colour_bg_blue   "\33[37;44m"
#define colour_bg_red    "\33[37;41m"
#define colour_fg_green  "\33[32;40m"
#define colour_fg_blue   "\33[34;40m"
#define colour_fg_red    "\33[31;40m"
#define colour_reset     "\33[0m"

int replay_genom_dump( const std::string& path_to_dump_file) {//{{{

  std::ifstream dump_file;
  dump_file.open (path_to_dump_file) ;

  if ( dump_file.is_open () ) {
    // init
    struct timeval tv0, tv1;

    std::string line, name ;
    gladys::points_t r_pos;
    int n, max_nf ;
    double min_size, x_origin, y_origin, height_max, width_max ;
    double x, y, yaw, min_dist, max_dist ;

    //read data
    getline (dump_file,line) ;
    std::istringstream iss1(line);
    iss1 >> name >> n ;
    std::cerr << "[Xares replay] r_pos ["<< n << "] =";
    while ( n-- > 0) {
      iss1 >> x >> y ;
      r_pos.push_back( gladys::point_xy_t {x,y} );
      std::cerr << " ("<< x << "," << y << ")";
    }
    std::cerr << std::endl;

    getline (dump_file,line) ;
    std::istringstream iss2(line);
    iss2 >> name >> yaw ;
    std::cerr << "[Xares replay] yaw = " << yaw << std::endl;

    // load and display internal params
    getline (dump_file,line) ;
    std::istringstream iss3(line);
    iss3 >> name >> max_nf >> min_size >> min_dist >> max_dist ;
    std::cerr << "[Xares replay] internal params "
              << "(max_nf,min_size,min_dist,max_dist) = ("
              << max_nf << "," << min_size << "," << min_dist << "," << max_dist
              << ")" << std::endl;

    std::cerr << "[Xares replay] max nbr of frontiers = " << max_nf << std::endl;
    getline (dump_file,line) ;
    gladys::weight_map wm ( line );

    // weightmap : origin, size, scale
    std::cerr << "[Xares repay] weight map: (x0,y0,xSc,ySc,W,H) = ("
              << wm.get_utm_pose_x()<< ","
              << wm.get_utm_pose_y()<< ","
              << wm.get_scale_x()   << ","
              << wm.get_scale_y()   << ","
              << wm.get_height()    << ","
              << wm.get_width()     << ")"
              << std::endl;

    // bounded area to explore
    getline (dump_file,line) ;
    std::istringstream iss4(line);
    iss4 >> name >> x_origin >> y_origin >> height_max >> width_max ;
    std::cerr << "[Xares replay] bounded area : (x0,y0,W,H) "
              << x_origin << "," << y_origin << ","
              << height_max << "," << width_max
              << ")" << std::endl;

    dump_file.close();

    // load the planner
    gettimeofday(&tv0, NULL);
    xares::xares xp( wm, x_origin, y_origin, height_max, width_max );
    gettimeofday(&tv1, NULL);

    std::cerr << "[Xares replay] planner loaded ("
              << (tv1.tv_sec -  tv0.tv_sec) * 1000 +
              (tv1.tv_usec - tv0.tv_usec) / 1000 << " ms)." << std::endl;

    // Set internal parameters for xares
    xp.set_params( max_nf, min_size, min_dist, max_dist );
    std::cerr << "[Xares replay] parameters settled. Let's plan !" << std::endl;

    //plan (=replay !)
    gettimeofday(&tv0, NULL);
    xares::xares::return_value rv = xp.plan( r_pos, yaw );
    gettimeofday(&tv1, NULL);

    // parse results
    if ( rv == xares::xares::XARES_FAILURE ) {
      std::cerr << "[Xares replay] #EEE# xares : unexpected data ; unable to compute frontiers :'( " << std::endl;
      return EXIT_SUCCESS;
    }

    if ( rv == xares::xares::XARES_NO_FRONTIER ) {
      std::cerr << "[Xares replay] #EEE# xares : no valuable frontier detected." << std::endl;
      return EXIT_SUCCESS;
    }

    // From here, there should be a valid plan
    std::cerr << "[Xares replay] Plan computed ("
              << (tv1.tv_sec -  tv0.tv_sec) * 1000 +
              (tv1.tv_usec - tv0.tv_usec) / 1000 << " ms)" << std::endl;

    //get planned path
    const gladys::path_t& path = xp.get_goal().path ;

    for (unsigned int i = 0 ; i < path.size() ; i++)
      std::cerr   << "[Xares replay ] ----waypoint #" << i
                  << " = (" << path[i][0]<< "," << path[i][1]
                  <<")"<<std::endl;

    // get poster path
    gladys::path_t::const_iterator it;
    double dist = 0.0;

    gladys::point_xy_t curr = path.front() ;

    unsigned int pt = 0 ;
    for ( it = path.begin(); it != path.end() ; ++it)
    {
      dist += gladys::distance( curr, *it );
      curr = *it;

      if ( dist > 6.0 ) {
        std::cerr   << "[Xares replay ] ---- poster waypoint #" << pt++
                    << " = (" << (*it)[0]<< "," << (*it)[1]
                    <<")"<<std::endl;
      }
    }

    std::cerr   << "[Xares replay ] ---- poster waypoint #" << pt
                << " = (" << curr[0]<< "," << curr[1]
                <<")"<<std::endl;

    // ASCII plot of weightmap, seed & goal
    std::cerr   << "[Xares replay ] ASCII weight map : " << std::endl;
    size_t index0 = wm.index_utm( gladys::point_xy_t { 0.0 , 0.0 } ) ; //origin
    size_t indexS = wm.index_utm( r_pos[0] ) ; // seed (robot position)
    size_t indexG = wm.index_utm( curr ) ;  // goal
    size_t index_curr ;

    //std::cerr << index0 << " (0,0)" << std::endl;
    //std::cerr << indexS << " (" << r_pos[0][0] << "," << r_pos[0][1] << ")" << std::endl;
    //std::cerr << indexG << " (" << curr[0] << "," << curr[1] << ")" << std::endl;

    // foreach from (origin) to (dim+origin)
    for ( double j = wm.get_utm_pose_y(); j < wm.get_width()*wm.get_scale_y() + wm.get_utm_pose_y(); j+= wm.get_scale_y() )
    {
      if ( j < y_origin || j > width_max ) continue; //cropping
      for (double i = wm.get_utm_pose_x(); i < wm.get_height()*wm.get_scale_x() + wm.get_utm_pose_x(); i+= wm.get_scale_x() )
      {
        if ( i < x_origin || i > height_max ) continue; //cropping

        index_curr = wm.index_utm( gladys::point_xy_t {(double)i,(double)j} );

        // Special points
        // O = origin
        // S = Seed
        // G = Goal
        if ( index_curr == index0 ) { // origin
            std::cerr << colour_bg_yellow << "O"; continue; }
        if ( index_curr == indexS ) { // seed
            std::cerr << colour_bg_green  << "S"; continue; }
        if ( index_curr == indexG ) { // goal
            std::cerr << colour_bg_red << "G"; continue;}

        // Common points
        // u = unknown
        // . = known
        // # = obstacle
        double w = wm.get_weight_band()[ index_curr ];
        if        (w < 0)     std::cerr << colour_fg_blue  << "u"; //unknonwn
        else if   (w > 100 )  std::cerr << colour_fg_red   << "#"; //obstacle
        else                  std::cerr << colour_fg_green << ".";

      }
      std::cerr << std::endl;
    }

    std::cerr << colour_reset << std::endl;
  }
  else {
    std::cerr << "[Xares replay] Cannot open provided dump file. :-(" << std::endl;
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}//}}}

int replay_lib_dump(char dumpFile[]) {//{{{

  std::cerr << "Not yet implemented..." << std::endl;

  return EXIT_SUCCESS;

}//}}}

int main(int argc, char **argv) {//{{{

  int return_value ;
  return_value = replay_genom_dump( argv[1] ) ;

  std::cerr << "[Xares replay] Done." << std::endl;

  return return_value;

}//}}}

<|endoftext|>


<commit_before>// Copyright (c)2008-2011, Preferred Infrastructure Inc.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//
//     * Neither the name of Preferred Infrastructure nor the names of other
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "sparse_matrix.h"

#include <cstring>
#include <string>
#include <algorithm>
#include <sstream>
#include <iostream>

#include "../../system/file.h"
#include "../../system/sysstat.h"

using namespace std;
using namespace pfi::data::code;
using namespace pfi::system::file;

namespace pfi {
namespace data {
namespace sparse_matrix {

  ////////////////////////////////////////////////////////////////
  //
  // MatrixOffsets
  //

  enum MATRIX_IO_MODE {
    MATRIX_IO_NONE,   // undefined
    MATRIX_IO_READ,   // "r"
    MATRIX_IO_WRITE,  // "w"
  };

  struct RowInfo {
    int num,size;
  };

  class matrix_offsets
  {
  public:
    matrix_offsets();
    int open(string fnOffsets, MATRIX_IO_MODE mode);
    void close();
    int flush() const;
    int offsets_num() const;

    int num(int index) const;
    int size(int index) const;
    int offset(int index) const;

    void append_row(int size, int num);
  private:
    string fnOffsets;
    vector<RowInfo> rowInfos;
    vector<int> offsets;
    MATRIX_IO_MODE ioMode;
  };

  matrix_offsets::matrix_offsets()
    :ioMode(MATRIX_IO_NONE)
  {
  }

  int matrix_offsets::open(string fn, MATRIX_IO_MODE mode)
  {
    close();

    ioMode=mode;
    fnOffsets=fn;

    if (mode==MATRIX_IO_READ) {
      ifstream ifs(fnOffsets.c_str());
      if (!ifs) return -1;

      int rowNum;
      ifs.read((char*)&rowNum,sizeof(int));
      rowInfos=vector<RowInfo>(rowNum);
      offsets=vector<int>(rowNum);
      ifs.read((char*)&rowInfos[0],rowNum*sizeof(RowInfo));
      if (offsets.size()>0) {
        offsets[0]=0;
        for (int i=1;i<(int)offsets.size();++i) offsets[i]=offsets[i-1]+rowInfos[i-1].size;
      }
    }
    return 0;
  }

  void matrix_offsets::close() {
    fnOffsets="";
    rowInfos.clear();
    offsets.clear();
    ioMode=MATRIX_IO_NONE;
  }

  int matrix_offsets::flush() const
  {
    if (fnOffsets.size()==0) return -1;

    if (ioMode==MATRIX_IO_WRITE) {
      ofstream ofs(fnOffsets.c_str());
      if (!ofs) {
        fprintf(stderr,"matrix_offsets::flush: cannot open %s\n",fnOffsets.c_str());
        return -1;
      }
      int rowNum=rowInfos.size();
      ofs.write((char*)&rowNum,sizeof(int));
      if (rowNum>0) ofs.write((char*)&rowInfos[0],rowNum*sizeof(RowInfo));
    }
    return 0;
  }

  int matrix_offsets::offsets_num() const
  {
    return rowInfos.size();
  }

  int matrix_offsets::num(int index) const
  {
    if (index>=(int)rowInfos.size()){
      fprintf(stderr,"matrix_offsets::Num: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return rowInfos[index].num;
  }

  int matrix_offsets::size(int index) const
  {
    if (index>=(int)rowInfos.size()){
      fprintf(stderr,"matrix_offsets::Size: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return rowInfos[index].size;
  }

  int matrix_offsets::offset(int index) const
  {
    if (index>=(int)offsets.size()){
      fprintf(stderr,"matrix_offsets::Offset: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return offsets[index];
  }

  void matrix_offsets::append_row(int size, int num)
  {
    RowInfo rowInfo;
    rowInfo.size=size;
    rowInfo.num=num;
    rowInfos.push_back(rowInfo);

    int offset=offsets.size()==0?0:offsets.back()+size;
    offsets.push_back(offset);
  }

  ////////////////////////////////////////////////////////////////
  //
  // sparse_matrix_writer
  //

  sparse_matrix_writer::sparse_matrix_writer()
    :offsets(NULL)
  {
  }

  sparse_matrix_writer::~sparse_matrix_writer()
  {
    if (offsets) delete offsets;
  }

  int sparse_matrix_writer::open(const string& fnMat)
  {
    close();

    ofs.open(fnMat.c_str());
    if (!ofs) {
      fprintf(stderr,"sparse_matrix_writer::open: cannot open %s\n",fnMat.c_str());
      return -1;
    }

    offsets=new matrix_offsets;
    if (offsets->open(fnMat+".offset",MATRIX_IO_WRITE)<0) {
      fprintf(stderr,"sparse_matrix_writer::open: cannot open %s.offset\n",fnMat.c_str());
      ofs.close();
      return -1;
    }
    return 0;
  }

  void sparse_matrix_writer::close()
  {
    ofs.close();
    if (offsets) {
      offsets->flush();
      delete offsets;
      offsets=NULL;
    }
  }

  void sparse_matrix_writer::append_row(const std::map<int,unsigned char>& row)
  {
    if (!ofs.is_open()||!offsets) return;

    encoder ec;

    int termNum=0;
    int prevTerm=-1;
    std::map<int,unsigned char>::const_iterator it;
    for (it=row.begin();it!=row.end();++it) {
      ec.prefix_code(it->first-prevTerm);
      ec.byte((unsigned char)it->second);
      prevTerm=it->first;
      ++termNum;
    }
    int columnSize=ec.flush(ofs);
    offsets->append_row(columnSize,termNum);
  }

  void sparse_matrix_writer::append_row(const vector<pair<int,unsigned char> >& row) {
    if (!ofs.is_open()||!offsets) return;

    encoder ec;
    int termNum=0;
    int prevTerm=-1;
    for (int i=0;i<(int)row.size();++i) {
      ec.prefix_code(row[i].first-prevTerm);
      ec.byte(row[i].second);
      prevTerm=row[i].first;
      ++termNum;
    }
    int columnSize=ec.flush(ofs);
    offsets->append_row(columnSize,termNum);
  }

  int sparse_matrix_writer::row_num() {
    return offsets->offsets_num();
  }

  ////////////////////////////////////////////////////////////////
  //
  // sparse_matrix_reader
  //

  sparse_matrix_reader::sparse_matrix_reader()
    :offsets(NULL)
  {

  }

  sparse_matrix_reader::~sparse_matrix_reader()
  {
    close();
    //    if (buf) delete[] buf;
    //    if (offsets) delete offsets;
  }

  int sparse_matrix_reader::open(const string& fnMat)
  {
    close();
    dc.attach(fnMat);

    offsets=new matrix_offsets;
    if (offsets->open(fnMat+".offset",MATRIX_IO_READ)<0){
      fprintf(stderr,"sparse_matrix_reader::open: cannot open %s.offset\n",fnMat.c_str());
      return -1;
    }
    return 0;
  }

  void sparse_matrix_reader::close()
  {
    dc.detach();
    if (offsets) {
      delete offsets;
      offsets=NULL;
    }
  }

  int sparse_matrix_reader::row_num() const
  {
    if (!offsets) return 0;
    return offsets->offsets_num();
  }

  int sparse_matrix_reader::get_row_size(int row) const
  {
    if (!offsets) return 0;
    return offsets->num(row);
  }

  void sparse_matrix_reader::get_row(int row, vector<pair<int,unsigned char> >& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    data=vector<pair<int,unsigned char> >(num);
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[i].first=col;
      data[i].second=dc.byte();
    }
  }

  void sparse_matrix_reader::get_row(int row, std::map<int,unsigned char>& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[col]=dc.byte();
    }
  }

  void sparse_matrix_reader::get_row(int row, pfi::data::unordered_map<int,unsigned char>& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[col]=dc.byte();
    }
  }

  int sparse_matrix_reader::get_row_nonzero_value_num(int row) const
  {
    if (!offsets) return 0;
    return offsets->num(row);
  }

  ////////////////////////////////////////////////////////////////
  //
  // MatrixTranspose
  //

  int matrix_transpose(const string& fnMat, const string& fnMatT)
  {
    fputs("transpose start\n",stderr);
    vector<std::map<int,unsigned char> > hash;

    sparse_matrix_reader index;
    fprintf(stderr,"index.open(%s)\n",fnMat.c_str());
    if (index.open(fnMat)<0){
      fprintf(stderr,"MatrixTranspose cannot open %s\n",fnMat.c_str());
      return -1;
    }
    sparse_matrix_writer indexT;
    fputs("indexT.open()\n",stderr);
    if (indexT.open(fnMatT)<0){
      fprintf(stderr,"MatrixTranspose cannot open %s\n",fnMatT.c_str());
      return -1;
    }
    int docNum=index.row_num();
    fprintf(stderr,"index.row_num() = %d\n",docNum);

    // count how much number of element not equal to zero contains for each column
    vector<int> termCount;
    for (int i=0;i<docNum;++i) {
      vector<pair<int,unsigned char> > docs;
      index.get_row(i,docs);
      for (int j=0;j<(int)docs.size();++j) {
        int term=docs[j].first;
        if (term>=(int)termCount.size()) termCount.resize(term+1);
        ++termCount[term];
      }
    }
    int termNum=termCount.size();

    fputs("term count finished\n",stderr);

    // for memory usage, split columns to small pieces
    uint64_t sum=0;
    vector<int> boundary;
    pfi::system::sysstat::sysstat_ret stat;
    int stat_ret = pfi::system::sysstat::get_sysstat(stat);
    if (stat_ret<0){
      fprintf(stderr,"get_sysstat failed\n");
      return -1;
    }
    uint64_t limit=stat.free_memory/20;

    for (int i=0;i<(int)termCount.size();++i) {
      sum+=termCount[i];
      if (sum>=limit) {
        fprintf(stderr,"boundary: %d\n",i);
        boundary.push_back(i+1);
        sum=0;
      }
    }
    if (sum>0) {
      fprintf(stderr,"boundary: %d\n",termNum);
      boundary.push_back(termNum);
    }

    fputs("split rows finished\n",stderr);

    // transpose for each column sets
    int st=0;
    for (int i=0;i<(int)boundary.size();++i) {
      fprintf(stderr,"transpose: %d / %d\n",i,(int)boundary.size());

      int en=boundary[i];
      vector<std::map<int,unsigned char> > hash(en-st);

      for (int j=0;j<docNum;++j) {
        vector<pair<int,unsigned char> > docs;
        index.get_row(j,docs);
        for (int k=0;k<(int)docs.size();++k)
          if (st<=(int)docs[k].first&&(int)docs[k].first<en)
            hash[docs[k].first-st][j]=docs[k].second;
      }
      for (int i=st;i<en;++i) indexT.append_row(hash[i-st]);
      st=en;
    }

    index.close();
    indexT.close();
    return 0;
  }

} // sparse_matrix
} // data
} // pfi
<commit_msg>Removed operator[] access to an empty vector.<commit_after>// Copyright (c)2008-2011, Preferred Infrastructure Inc.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//
//     * Neither the name of Preferred Infrastructure nor the names of other
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "sparse_matrix.h"

#include <cstring>
#include <string>
#include <algorithm>
#include <sstream>
#include <iostream>

#include "../../system/file.h"
#include "../../system/sysstat.h"

using namespace std;
using namespace pfi::data::code;
using namespace pfi::system::file;

namespace pfi {
namespace data {
namespace sparse_matrix {

  ////////////////////////////////////////////////////////////////
  //
  // MatrixOffsets
  //

  enum MATRIX_IO_MODE {
    MATRIX_IO_NONE,   // undefined
    MATRIX_IO_READ,   // "r"
    MATRIX_IO_WRITE,  // "w"
  };

  struct RowInfo {
    int num,size;
  };

  class matrix_offsets
  {
  public:
    matrix_offsets();
    int open(string fnOffsets, MATRIX_IO_MODE mode);
    void close();
    int flush() const;
    int offsets_num() const;

    int num(int index) const;
    int size(int index) const;
    int offset(int index) const;

    void append_row(int size, int num);
  private:
    string fnOffsets;
    vector<RowInfo> rowInfos;
    vector<int> offsets;
    MATRIX_IO_MODE ioMode;
  };

  matrix_offsets::matrix_offsets()
    :ioMode(MATRIX_IO_NONE)
  {
  }

  int matrix_offsets::open(string fn, MATRIX_IO_MODE mode)
  {
    close();

    ioMode=mode;
    fnOffsets=fn;

    if (mode==MATRIX_IO_READ) {
      ifstream ifs(fnOffsets.c_str());
      if (!ifs) return -1;

      int rowNum;
      ifs.read((char*)&rowNum,sizeof(int));
      rowInfos=vector<RowInfo>(rowNum);
      offsets=vector<int>(rowNum);
      if (rowNum > 0)
        ifs.read((char*)&rowInfos[0],rowNum*sizeof(RowInfo));
      if (offsets.size()>0) {
        offsets[0]=0;
        for (int i=1;i<(int)offsets.size();++i) offsets[i]=offsets[i-1]+rowInfos[i-1].size;
      }
    }
    return 0;
  }

  void matrix_offsets::close() {
    fnOffsets="";
    rowInfos.clear();
    offsets.clear();
    ioMode=MATRIX_IO_NONE;
  }

  int matrix_offsets::flush() const
  {
    if (fnOffsets.size()==0) return -1;

    if (ioMode==MATRIX_IO_WRITE) {
      ofstream ofs(fnOffsets.c_str());
      if (!ofs) {
        fprintf(stderr,"matrix_offsets::flush: cannot open %s\n",fnOffsets.c_str());
        return -1;
      }
      int rowNum=rowInfos.size();
      ofs.write((char*)&rowNum,sizeof(int));
      if (rowNum>0) ofs.write((char*)&rowInfos[0],rowNum*sizeof(RowInfo));
    }
    return 0;
  }

  int matrix_offsets::offsets_num() const
  {
    return rowInfos.size();
  }

  int matrix_offsets::num(int index) const
  {
    if (index>=(int)rowInfos.size()){
      fprintf(stderr,"matrix_offsets::Num: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return rowInfos[index].num;
  }

  int matrix_offsets::size(int index) const
  {
    if (index>=(int)rowInfos.size()){
      fprintf(stderr,"matrix_offsets::Size: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return rowInfos[index].size;
  }

  int matrix_offsets::offset(int index) const
  {
    if (index>=(int)offsets.size()){
      fprintf(stderr,"matrix_offsets::Offset: Try to access out of bounds of row index\n");
      fprintf(stderr,"Row size: %d, but you try to access to %d\n",(int)rowInfos.size(),index);
      return -1;
    }
    return offsets[index];
  }

  void matrix_offsets::append_row(int size, int num)
  {
    RowInfo rowInfo;
    rowInfo.size=size;
    rowInfo.num=num;
    rowInfos.push_back(rowInfo);

    int offset=offsets.size()==0?0:offsets.back()+size;
    offsets.push_back(offset);
  }

  ////////////////////////////////////////////////////////////////
  //
  // sparse_matrix_writer
  //

  sparse_matrix_writer::sparse_matrix_writer()
    :offsets(NULL)
  {
  }

  sparse_matrix_writer::~sparse_matrix_writer()
  {
    if (offsets) delete offsets;
  }

  int sparse_matrix_writer::open(const string& fnMat)
  {
    close();

    ofs.open(fnMat.c_str());
    if (!ofs) {
      fprintf(stderr,"sparse_matrix_writer::open: cannot open %s\n",fnMat.c_str());
      return -1;
    }

    offsets=new matrix_offsets;
    if (offsets->open(fnMat+".offset",MATRIX_IO_WRITE)<0) {
      fprintf(stderr,"sparse_matrix_writer::open: cannot open %s.offset\n",fnMat.c_str());
      ofs.close();
      return -1;
    }
    return 0;
  }

  void sparse_matrix_writer::close()
  {
    ofs.close();
    if (offsets) {
      offsets->flush();
      delete offsets;
      offsets=NULL;
    }
  }

  void sparse_matrix_writer::append_row(const std::map<int,unsigned char>& row)
  {
    if (!ofs.is_open()||!offsets) return;

    encoder ec;

    int termNum=0;
    int prevTerm=-1;
    std::map<int,unsigned char>::const_iterator it;
    for (it=row.begin();it!=row.end();++it) {
      ec.prefix_code(it->first-prevTerm);
      ec.byte((unsigned char)it->second);
      prevTerm=it->first;
      ++termNum;
    }
    int columnSize=ec.flush(ofs);
    offsets->append_row(columnSize,termNum);
  }

  void sparse_matrix_writer::append_row(const vector<pair<int,unsigned char> >& row) {
    if (!ofs.is_open()||!offsets) return;

    encoder ec;
    int termNum=0;
    int prevTerm=-1;
    for (int i=0;i<(int)row.size();++i) {
      ec.prefix_code(row[i].first-prevTerm);
      ec.byte(row[i].second);
      prevTerm=row[i].first;
      ++termNum;
    }
    int columnSize=ec.flush(ofs);
    offsets->append_row(columnSize,termNum);
  }

  int sparse_matrix_writer::row_num() {
    return offsets->offsets_num();
  }

  ////////////////////////////////////////////////////////////////
  //
  // sparse_matrix_reader
  //

  sparse_matrix_reader::sparse_matrix_reader()
    :offsets(NULL)
  {

  }

  sparse_matrix_reader::~sparse_matrix_reader()
  {
    close();
    //    if (buf) delete[] buf;
    //    if (offsets) delete offsets;
  }

  int sparse_matrix_reader::open(const string& fnMat)
  {
    close();
    dc.attach(fnMat);

    offsets=new matrix_offsets;
    if (offsets->open(fnMat+".offset",MATRIX_IO_READ)<0){
      fprintf(stderr,"sparse_matrix_reader::open: cannot open %s.offset\n",fnMat.c_str());
      return -1;
    }
    return 0;
  }

  void sparse_matrix_reader::close()
  {
    dc.detach();
    if (offsets) {
      delete offsets;
      offsets=NULL;
    }
  }

  int sparse_matrix_reader::row_num() const
  {
    if (!offsets) return 0;
    return offsets->offsets_num();
  }

  int sparse_matrix_reader::get_row_size(int row) const
  {
    if (!offsets) return 0;
    return offsets->num(row);
  }

  void sparse_matrix_reader::get_row(int row, vector<pair<int,unsigned char> >& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    data=vector<pair<int,unsigned char> >(num);
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[i].first=col;
      data[i].second=dc.byte();
    }
  }

  void sparse_matrix_reader::get_row(int row, std::map<int,unsigned char>& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[col]=dc.byte();
    }
  }

  void sparse_matrix_reader::get_row(int row, pfi::data::unordered_map<int,unsigned char>& data)
  {
    if (!dc.is_open()||!offsets) return;

    int num=offsets->num(row);
    if (num<0){
      fprintf(stderr,"sparse_matrix_reader::get_row: offsets.Num error\n");
      return;
    }
    dc.seek(offsets->offset(row));

    int col=-1;
    for (int i=0;i<num;++i) {
      col+=dc.prefix_code();
      data[col]=dc.byte();
    }
  }

  int sparse_matrix_reader::get_row_nonzero_value_num(int row) const
  {
    if (!offsets) return 0;
    return offsets->num(row);
  }

  ////////////////////////////////////////////////////////////////
  //
  // MatrixTranspose
  //

  int matrix_transpose(const string& fnMat, const string& fnMatT)
  {
    fputs("transpose start\n",stderr);
    vector<std::map<int,unsigned char> > hash;

    sparse_matrix_reader index;
    fprintf(stderr,"index.open(%s)\n",fnMat.c_str());
    if (index.open(fnMat)<0){
      fprintf(stderr,"MatrixTranspose cannot open %s\n",fnMat.c_str());
      return -1;
    }
    sparse_matrix_writer indexT;
    fputs("indexT.open()\n",stderr);
    if (indexT.open(fnMatT)<0){
      fprintf(stderr,"MatrixTranspose cannot open %s\n",fnMatT.c_str());
      return -1;
    }
    int docNum=index.row_num();
    fprintf(stderr,"index.row_num() = %d\n",docNum);

    // count how much number of element not equal to zero contains for each column
    vector<int> termCount;
    for (int i=0;i<docNum;++i) {
      vector<pair<int,unsigned char> > docs;
      index.get_row(i,docs);
      for (int j=0;j<(int)docs.size();++j) {
        int term=docs[j].first;
        if (term>=(int)termCount.size()) termCount.resize(term+1);
        ++termCount[term];
      }
    }
    int termNum=termCount.size();

    fputs("term count finished\n",stderr);

    // for memory usage, split columns to small pieces
    uint64_t sum=0;
    vector<int> boundary;
    pfi::system::sysstat::sysstat_ret stat;
    int stat_ret = pfi::system::sysstat::get_sysstat(stat);
    if (stat_ret<0){
      fprintf(stderr,"get_sysstat failed\n");
      return -1;
    }
    uint64_t limit=stat.free_memory/20;

    for (int i=0;i<(int)termCount.size();++i) {
      sum+=termCount[i];
      if (sum>=limit) {
        fprintf(stderr,"boundary: %d\n",i);
        boundary.push_back(i+1);
        sum=0;
      }
    }
    if (sum>0) {
      fprintf(stderr,"boundary: %d\n",termNum);
      boundary.push_back(termNum);
    }

    fputs("split rows finished\n",stderr);

    // transpose for each column sets
    int st=0;
    for (int i=0;i<(int)boundary.size();++i) {
      fprintf(stderr,"transpose: %d / %d\n",i,(int)boundary.size());

      int en=boundary[i];
      vector<std::map<int,unsigned char> > hash(en-st);

      for (int j=0;j<docNum;++j) {
        vector<pair<int,unsigned char> > docs;
        index.get_row(j,docs);
        for (int k=0;k<(int)docs.size();++k)
          if (st<=(int)docs[k].first&&(int)docs[k].first<en)
            hash[docs[k].first-st][j]=docs[k].second;
      }
      for (int i=st;i<en;++i) indexT.append_row(hash[i-st]);
      st=en;
    }

    index.close();
    indexT.close();
    return 0;
  }

} // sparse_matrix
} // data
} // pfi
<|endoftext|>


<commit_before>/* slave_banker.cc
   Jeremy Barnes, 8 November 2012
   Copyright (c) 2012 Datacratic Inc.  All rights reserved.

   Slave banker implementation.
*/

#include "slave_banker.h"
#include "soa/service/http_header.h"
#include "jml/utils/vector_utils.h"

using namespace std;

namespace RTBKIT {

/*****************************************************************************/
/* SLAVE BUDGET CONTROLLER                                                   */
/*****************************************************************************/

SlaveBudgetController::
SlaveBudgetController()
{
}

void
SlaveBudgetController::
addAccount(const AccountKey & account,
           const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "POST", "/v1/accounts",
         { {"accountName", account.toString()},
           { "accountType", "budget" } });
}

void
SlaveBudgetController::
topupTransfer(const AccountKey & account,
              CurrencyPool amount,
              const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "PUT", "/v1/accounts/" + account.toString() + "/balance",
         { { "accountType", "budget"} },
         amount.toJson().toString());
}

void
SlaveBudgetController::
setBudget(const std::string & topLevelAccount,
          CurrencyPool amount,
          const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "PUT", "/v1/accounts/" + topLevelAccount + "/budget",
         { /* {"amount", amount.toString()}*/ },
         amount.toJson().toString());
}

void
SlaveBudgetController::
addBudget(const std::string & topLevelAccount,
          CurrencyPool amount,
          const OnBudgetResult & onResult)
{
    throw ML::Exception("addBudget no good any more");
}

void
SlaveBudgetController::
getAccountList(const AccountKey & account,
               int depth,
               std::function<void (std::exception_ptr,
                                   std::vector<AccountKey> &&)>)
{
    throw ML::Exception("getAccountList not needed anymore");
}

void
SlaveBudgetController::
getAccountSummary(const AccountKey & account,
                  int depth,
                  std::function<void (std::exception_ptr,
                                      AccountSummary &&)> onResult)
{
    push([=] (std::exception_ptr ptr, int resultCode, string body)
         {
             AccountSummary summary;
             if (ptr)
                 onResult(ptr, std::move(summary));
             else if (resultCode < 200 || resultCode >= 300)
                 onResult(std::make_exception_ptr(ML::Exception("getAccountSummary returned code %d: %s", resultCode, body.c_str())),
                          std::move(summary));
             else {
                 try {
                     Json::Value result = Json::parse(body);
                     summary = AccountSummary::fromJson(result);
                     onResult(nullptr, std::move(summary));
                 } catch (...) {
                     onResult(std::current_exception(), std::move(summary));
                 }
             }
         },
         "GET", "/v1/accounts/" + account.toString() + "/summary",
         { {"depth", to_string(depth)} },
         "");
}

void
SlaveBudgetController::
getAccount(const AccountKey & accountKey,
           std::function<void (std::exception_ptr,
                               Account &&)> onResult)
{
    push([=] (std::exception_ptr ptr, int resultCode, string body)
         {
             Account account;
             if (ptr)
                 onResult(ptr, std::move(account));
             else if (resultCode < 200 || resultCode >= 300)
                 onResult(std::make_exception_ptr(ML::Exception("getAccount returned code %d: %s", resultCode, body.c_str())),
                          std::move(account));
             else {
                 try {
                     Json::Value result = Json::parse(body);
                     account = Account::fromJson(result);
                     onResult(nullptr, std::move(account));
                 } catch (...) {
                     onResult(std::current_exception(), std::move(account));
                 }
             }
         },
         "GET", "/v1/accounts/" + accountKey.toString());
}

SlaveBudgetController::OnDone
SlaveBudgetController::
budgetResultCallback(const OnBudgetResult & onResult)
{
    return [=] (std::exception_ptr ptr, int resultCode, string body)
        {
            //cerr << "got budget result callback with resultCode "
            //     << resultCode << " body " << body << endl;
            onResult(ptr);
        };
}

/*****************************************************************************/
/* SLAVE BANKER                                                              */
/*****************************************************************************/

SlaveBanker::
SlaveBanker(std::shared_ptr<zmq::context_t> context)
    : RestProxy(context), createdAccounts(128)
{
}

SlaveBanker::
SlaveBanker(std::shared_ptr<zmq::context_t> context,
            std::shared_ptr<ConfigurationService> config,
            const std::string & accountSuffix,
            const std::string & bankerServiceName)
    : RestProxy(context), createdAccounts(128)
{
    init(config, accountSuffix, bankerServiceName);
}

void
SlaveBanker::
init(std::shared_ptr<ConfigurationService> config,
     const std::string & accountSuffix,
     const std::string & bankerServiceName)
{
    // When our account manager creates an account, it will call this
    // function.  We can't do anything from it (because the lock could
    // be held), but we *can* push a message asynchronously to be
    // handled later...
    accounts.onNewAccount = [=] (const AccountKey & accountKey)
        {
            //cerr << "((((1)))) new account " << accountKey << endl;
            createdAccounts.push(accountKey);
        };

    // ... here.  Now we know that no lock is held and so we can
    // perform the work we need to synchronize the account with
    // the server.
    createdAccounts.onEvent = [=] (const AccountKey & accountKey)
        {
            //cerr << "((((2)))) new account " << accountKey << endl;

            auto onDone = [=] (std::exception_ptr exc,
                               ShadowAccount && account)
            {
#if 0
                cerr << "((((3)))) new account " << accountKey << endl;

                cerr << "got back shadow account " << account
                << " for " << accountKey << endl;

                cerr << "current status is " << accounts.getAccount(accountKey)
                << endl;
#endif
            };

            addSpendAccount(accountKey, USD(0), onDone);
        };

    addSource("SlaveBanker::createdAccounts", createdAccounts);

    this->accountSuffix = accountSuffix;

    // Connect to the master banker
    RestProxy::initServiceClass(config, bankerServiceName, "zeromq");

    addPeriodic("SlaveBanker::reportSpend", 1.0,
                std::bind(&SlaveBanker::reportSpend,
                          this,
                          std::placeholders::_1),
                true /* single threaded */);
    addPeriodic("SlaveBanker::reauthorizeBudget", 1.0,
                std::bind(&SlaveBanker::reauthorizeBudget,
                          this,
                          std::placeholders::_1),
                true /* single threaded */);
}

ShadowAccount
SlaveBanker::
syncAccountSync(const AccountKey & account)
{
    BankerSyncResult<ShadowAccount> result;
    syncAccount(account, result);
    return result.get();
}

void
SlaveBanker::
onSyncResult(const AccountKey & accountKey,
               std::function<void (std::exception_ptr,
                                   ShadowAccount &&)> onDone,
               std::exception_ptr exc,
               Account&& masterAccount)
{
    ShadowAccount result;

    try {
        if (exc) {
            onDone(exc, std::move(result));
            return;
        }

        //cerr << "got result from master for " << accountKey
        //     << " which is "
        //     << masterAccount << endl;

        result = accounts.syncFromMaster(accountKey, masterAccount);
    } catch (...) {
        onDone(std::current_exception(), std::move(result));
    }

    try {
        onDone(nullptr, std::move(result));
    } catch (...) {
        cerr << "warning: onDone handler threw" << endl;
    }
}

void
SlaveBanker::
onInitializeResult(const AccountKey & accountKey,
                   std::function<void (std::exception_ptr,
                                       ShadowAccount &&)> onDone,
                   std::exception_ptr exc,
                   Account&& masterAccount)
{
    ShadowAccount result;

    try {
        if (exc) {
            onDone(exc, std::move(result));
            return;
        }

        result = accounts.initializeAndMergeState(accountKey, masterAccount);
    } catch (...) {
        onDone(std::current_exception(), std::move(result));
    }

    try {
        onDone(nullptr, std::move(result));
    } catch (...) {
        cerr << "warning: onDone handler threw" << endl;
    }
}

void
SlaveBanker::
syncAccount(const AccountKey & accountKey,
            std::function<void (std::exception_ptr,
                                ShadowAccount &&)> onDone)
{
    auto onDone2
        = std::bind(&SlaveBanker::onSyncResult,
                    this,
                    accountKey,
                    onDone,
                    std::placeholders::_1,
                    std::placeholders::_2);

    //cerr << "syncing account " << accountKey << ": "
    //     << accounts.getAccount(accountKey) << endl;

    push(makeRestResponseJsonDecoder<Account>("syncAccount", onDone2),
         "PUT",
         "/v1/accounts/" + getShadowAccountStr(accountKey) + "/shadow",
         {},
         accounts.getAccount(accountKey).toJson().toString());
}

void
SlaveBanker::
syncAllSync()
{
    BankerSyncResult<void> result;
    syncAll(result);
    result.get();
}

void
SlaveBanker::
syncAll(std::function<void (std::exception_ptr)> onDone)
{
    auto allKeys = accounts.getAccountKeys();

    vector<AccountKey> filteredKeys;
    for (auto k: allKeys)
    	if (accounts.isInitialized(k))
    		filteredKeys.push_back(k);

    allKeys.swap(filteredKeys);

    if (allKeys.empty()) {
        if (onDone)
            onDone(nullptr);
        return;
    }

    struct Aggregator {

        Aggregator(int numTotal,
                   std::function<void (std::exception_ptr)> onDone)
            : itl(new Itl())
        {
            itl->numTotal = numTotal;
            itl->numFinished = 0;
            itl->exc = nullptr;
            itl->onDone = onDone;
        }

        struct Itl {
            int numTotal;
            int numFinished;
            std::exception_ptr exc;
            std::function<void (std::exception_ptr)> onDone;
        };

        std::shared_ptr<Itl> itl;

        void operator () (std::exception_ptr exc, ShadowAccount && account)
        {
            if (exc)
                itl->exc = exc;
            int nowDone = __sync_add_and_fetch(&itl->numFinished, 1);
            if (nowDone == itl->numTotal) {
                if (itl->onDone)
                    itl->onDone(itl->exc);
                else {
                    if (itl->exc)
                        cerr << "warning: async callback aggregator ate "
                             << "exception" << endl;
                }
            }
        }
    };

    Aggregator aggregator(allKeys.size(), onDone);

    //cerr << "syncing " << allKeys.size() << " keys" << endl;

    for (auto & key: allKeys) {
        // We take its parent since syncAccount assumes nothing was added
    	if (accounts.isInitialized(key))
    		syncAccount(key, aggregator);
    }
}

void
SlaveBanker::
addSpendAccount(const AccountKey & accountKey,
                CurrencyPool accountFloat,
                std::function<void (std::exception_ptr, ShadowAccount&&)> onDone)
{
	bool first = accounts.createAccountAtomic(accountKey);
	if(!first) {
		// already done
		if (onDone) {
			auto account = accounts.getAccount(accountKey);
			onDone(nullptr, std::move(account));
		}
    }
	else {
		// TODO: record float
		//accountFloats[accountKey] = accountFloat;

		// Now kick off the initial synchronization step
		auto onDone2
			= std::bind(&SlaveBanker::onInitializeResult,
						this,
						accountKey,
						onDone,
						std::placeholders::_1,
						std::placeholders::_2);

		cerr << "********* calling addSpendAccount for " << accountKey
			 << " for SlaveBanker " << accountSuffix << endl;

		push(makeRestResponseJsonDecoder<Account>("addSpendAccount", onDone2),
			 "POST",
			 "/v1/accounts",
			 { { "accountName", getShadowAccountStr(accountKey) },
			   { "accountType", "spend" } },
			 "");
	}
}

void
SlaveBanker::
reportSpend(uint64_t numTimeoutsExpired)
{
    if (numTimeoutsExpired > 1) {
        cerr << "warning: slave banker missed " << numTimeoutsExpired
             << " timeouts" << endl;
    }

    if (reportSpendSent != Date())
        cerr << "warning: report spend still in progress" << endl;

    //cerr << "started report spend" << endl;

    auto onDone = [=] (std::exception_ptr exc)
        {
            //cerr << "finished report spend" << endl;
            reportSpendSent = Date();
            if (exc)
                cerr << "reportSpend got exception" << endl;
        };

    syncAll(onDone);
}

void
SlaveBanker::
reauthorizeBudget(uint64_t numTimeoutsExpired)
{
    if (numTimeoutsExpired > 1) {
        cerr << "warning: slave banker missed " << numTimeoutsExpired
             << " timeouts" << endl;
    }

    //std::unique_lock<Lock> guard(lock);
    if (reauthorizeBudgetSent != Date()) {
        cerr << "warning: reauthorize budget still in progress" << endl;
    }

    int numDone = 0;

    // For each of our accounts, we report back what has been spent
    // and re-up to our desired float
    auto onAccount = [&] (const AccountKey & key,
                          const ShadowAccount & account)
        {
            RestRequest request;
            request.verb = "POST";
            request.resource
                = "/v1/accounts/"
                + getShadowAccountStr(key)
                + "/balance";
            request.params = { { "accountType", "spend" } };

            Json::Value payload = CurrencyPool(USD(0.10)).toJson();
            request.payload = payload.toString();

            //cerr << "sending out request " << request << endl;
            ++numDone;

            // Finally, send it out
            push(request, std::bind(&SlaveBanker::onReauthorizeBudgetMessage,
                                    this,
                                    key,
                                    std::placeholders::_1,
                                    std::placeholders::_2,
                                    std::placeholders::_3));
        };

    accounts.forEachInitializedAccount(onAccount);

    if (numDone != 0)
        reauthorizeBudgetSent = Date::now();
}

void
SlaveBanker::
onReauthorizeBudgetMessage(const AccountKey & accountKey,
                           std::exception_ptr exc,
                           int responseCode,
                           const std::string & payload)
{
    //cerr << "finished reauthorize budget" << endl;

    if (exc) {
        cerr << "reauthorize budget got exception" << payload << endl;
        cerr << "accountKey = " << accountKey << endl;
        abort();  // for now...
        return;
    }

    Account masterAccount = Account::fromJson(Json::parse(payload));
    accounts.syncFromMaster(accountKey, masterAccount);
    reauthorizeBudgetSent = Date();
}

} // namespace RTBKIT
<commit_msg>ensure that "accountSuffix" and "bankerServiceName" are not empty<commit_after>/* slave_banker.cc
   Jeremy Barnes, 8 November 2012
   Copyright (c) 2012 Datacratic Inc.  All rights reserved.

   Slave banker implementation.
*/

#include "slave_banker.h"
#include "soa/service/http_header.h"
#include "jml/utils/vector_utils.h"

using namespace std;

namespace RTBKIT {

/*****************************************************************************/
/* SLAVE BUDGET CONTROLLER                                                   */
/*****************************************************************************/

SlaveBudgetController::
SlaveBudgetController()
{
}

void
SlaveBudgetController::
addAccount(const AccountKey & account,
           const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "POST", "/v1/accounts",
         { {"accountName", account.toString()},
           { "accountType", "budget" } });
}

void
SlaveBudgetController::
topupTransfer(const AccountKey & account,
              CurrencyPool amount,
              const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "PUT", "/v1/accounts/" + account.toString() + "/balance",
         { { "accountType", "budget"} },
         amount.toJson().toString());
}

void
SlaveBudgetController::
setBudget(const std::string & topLevelAccount,
          CurrencyPool amount,
          const OnBudgetResult & onResult)
{
    push(budgetResultCallback(onResult),
         "PUT", "/v1/accounts/" + topLevelAccount + "/budget",
         { /* {"amount", amount.toString()}*/ },
         amount.toJson().toString());
}

void
SlaveBudgetController::
addBudget(const std::string & topLevelAccount,
          CurrencyPool amount,
          const OnBudgetResult & onResult)
{
    throw ML::Exception("addBudget no good any more");
}

void
SlaveBudgetController::
getAccountList(const AccountKey & account,
               int depth,
               std::function<void (std::exception_ptr,
                                   std::vector<AccountKey> &&)>)
{
    throw ML::Exception("getAccountList not needed anymore");
}

void
SlaveBudgetController::
getAccountSummary(const AccountKey & account,
                  int depth,
                  std::function<void (std::exception_ptr,
                                      AccountSummary &&)> onResult)
{
    push([=] (std::exception_ptr ptr, int resultCode, string body)
         {
             AccountSummary summary;
             if (ptr)
                 onResult(ptr, std::move(summary));
             else if (resultCode < 200 || resultCode >= 300)
                 onResult(std::make_exception_ptr(ML::Exception("getAccountSummary returned code %d: %s", resultCode, body.c_str())),
                          std::move(summary));
             else {
                 try {
                     Json::Value result = Json::parse(body);
                     summary = AccountSummary::fromJson(result);
                     onResult(nullptr, std::move(summary));
                 } catch (...) {
                     onResult(std::current_exception(), std::move(summary));
                 }
             }
         },
         "GET", "/v1/accounts/" + account.toString() + "/summary",
         { {"depth", to_string(depth)} },
         "");
}

void
SlaveBudgetController::
getAccount(const AccountKey & accountKey,
           std::function<void (std::exception_ptr,
                               Account &&)> onResult)
{
    push([=] (std::exception_ptr ptr, int resultCode, string body)
         {
             Account account;
             if (ptr)
                 onResult(ptr, std::move(account));
             else if (resultCode < 200 || resultCode >= 300)
                 onResult(std::make_exception_ptr(ML::Exception("getAccount returned code %d: %s", resultCode, body.c_str())),
                          std::move(account));
             else {
                 try {
                     Json::Value result = Json::parse(body);
                     account = Account::fromJson(result);
                     onResult(nullptr, std::move(account));
                 } catch (...) {
                     onResult(std::current_exception(), std::move(account));
                 }
             }
         },
         "GET", "/v1/accounts/" + accountKey.toString());
}

SlaveBudgetController::OnDone
SlaveBudgetController::
budgetResultCallback(const OnBudgetResult & onResult)
{
    return [=] (std::exception_ptr ptr, int resultCode, string body)
        {
            //cerr << "got budget result callback with resultCode "
            //     << resultCode << " body " << body << endl;
            onResult(ptr);
        };
}

/*****************************************************************************/
/* SLAVE BANKER                                                              */
/*****************************************************************************/

SlaveBanker::
SlaveBanker(std::shared_ptr<zmq::context_t> context)
    : RestProxy(context), createdAccounts(128)
{
}

SlaveBanker::
SlaveBanker(std::shared_ptr<zmq::context_t> context,
            std::shared_ptr<ConfigurationService> config,
            const std::string & accountSuffix,
            const std::string & bankerServiceName)
    : RestProxy(context), createdAccounts(128)
{
    init(config, accountSuffix, bankerServiceName);
}

void
SlaveBanker::
init(std::shared_ptr<ConfigurationService> config,
     const std::string & accountSuffix,
     const std::string & bankerServiceName)
{
    if (accountSuffix.empty()) {
        throw ML::Exception("'accountSuffix' cannot be empty");
    }
    if (bankerServiceName.empty()) {
        throw ML::Exception("'bankerServiceName' cannot be empty");
    }

    // When our account manager creates an account, it will call this
    // function.  We can't do anything from it (because the lock could
    // be held), but we *can* push a message asynchronously to be
    // handled later...
    accounts.onNewAccount = [=] (const AccountKey & accountKey)
        {
            //cerr << "((((1)))) new account " << accountKey << endl;
            createdAccounts.push(accountKey);
        };

    // ... here.  Now we know that no lock is held and so we can
    // perform the work we need to synchronize the account with
    // the server.
    createdAccounts.onEvent = [=] (const AccountKey & accountKey)
        {
            //cerr << "((((2)))) new account " << accountKey << endl;

            auto onDone = [=] (std::exception_ptr exc,
                               ShadowAccount && account)
            {
#if 0
                cerr << "((((3)))) new account " << accountKey << endl;

                cerr << "got back shadow account " << account
                << " for " << accountKey << endl;

                cerr << "current status is " << accounts.getAccount(accountKey)
                << endl;
#endif
            };

            addSpendAccount(accountKey, USD(0), onDone);
        };

    addSource("SlaveBanker::createdAccounts", createdAccounts);

    this->accountSuffix = accountSuffix;

    // Connect to the master banker
    RestProxy::initServiceClass(config, bankerServiceName, "zeromq");

    addPeriodic("SlaveBanker::reportSpend", 1.0,
                std::bind(&SlaveBanker::reportSpend,
                          this,
                          std::placeholders::_1),
                true /* single threaded */);
    addPeriodic("SlaveBanker::reauthorizeBudget", 1.0,
                std::bind(&SlaveBanker::reauthorizeBudget,
                          this,
                          std::placeholders::_1),
                true /* single threaded */);
}

ShadowAccount
SlaveBanker::
syncAccountSync(const AccountKey & account)
{
    BankerSyncResult<ShadowAccount> result;
    syncAccount(account, result);
    return result.get();
}

void
SlaveBanker::
onSyncResult(const AccountKey & accountKey,
               std::function<void (std::exception_ptr,
                                   ShadowAccount &&)> onDone,
               std::exception_ptr exc,
               Account&& masterAccount)
{
    ShadowAccount result;

    try {
        if (exc) {
            onDone(exc, std::move(result));
            return;
        }

        //cerr << "got result from master for " << accountKey
        //     << " which is "
        //     << masterAccount << endl;

        result = accounts.syncFromMaster(accountKey, masterAccount);
    } catch (...) {
        onDone(std::current_exception(), std::move(result));
    }

    try {
        onDone(nullptr, std::move(result));
    } catch (...) {
        cerr << "warning: onDone handler threw" << endl;
    }
}

void
SlaveBanker::
onInitializeResult(const AccountKey & accountKey,
                   std::function<void (std::exception_ptr,
                                       ShadowAccount &&)> onDone,
                   std::exception_ptr exc,
                   Account&& masterAccount)
{
    ShadowAccount result;

    try {
        if (exc) {
            onDone(exc, std::move(result));
            return;
        }

        result = accounts.initializeAndMergeState(accountKey, masterAccount);
    } catch (...) {
        onDone(std::current_exception(), std::move(result));
    }

    try {
        onDone(nullptr, std::move(result));
    } catch (...) {
        cerr << "warning: onDone handler threw" << endl;
    }
}

void
SlaveBanker::
syncAccount(const AccountKey & accountKey,
            std::function<void (std::exception_ptr,
                                ShadowAccount &&)> onDone)
{
    auto onDone2
        = std::bind(&SlaveBanker::onSyncResult,
                    this,
                    accountKey,
                    onDone,
                    std::placeholders::_1,
                    std::placeholders::_2);

    //cerr << "syncing account " << accountKey << ": "
    //     << accounts.getAccount(accountKey) << endl;

    push(makeRestResponseJsonDecoder<Account>("syncAccount", onDone2),
         "PUT",
         "/v1/accounts/" + getShadowAccountStr(accountKey) + "/shadow",
         {},
         accounts.getAccount(accountKey).toJson().toString());
}

void
SlaveBanker::
syncAllSync()
{
    BankerSyncResult<void> result;
    syncAll(result);
    result.get();
}

void
SlaveBanker::
syncAll(std::function<void (std::exception_ptr)> onDone)
{
    auto allKeys = accounts.getAccountKeys();

    vector<AccountKey> filteredKeys;
    for (auto k: allKeys)
    	if (accounts.isInitialized(k))
    		filteredKeys.push_back(k);

    allKeys.swap(filteredKeys);

    if (allKeys.empty()) {
        if (onDone)
            onDone(nullptr);
        return;
    }

    struct Aggregator {

        Aggregator(int numTotal,
                   std::function<void (std::exception_ptr)> onDone)
            : itl(new Itl())
        {
            itl->numTotal = numTotal;
            itl->numFinished = 0;
            itl->exc = nullptr;
            itl->onDone = onDone;
        }

        struct Itl {
            int numTotal;
            int numFinished;
            std::exception_ptr exc;
            std::function<void (std::exception_ptr)> onDone;
        };

        std::shared_ptr<Itl> itl;

        void operator () (std::exception_ptr exc, ShadowAccount && account)
        {
            if (exc)
                itl->exc = exc;
            int nowDone = __sync_add_and_fetch(&itl->numFinished, 1);
            if (nowDone == itl->numTotal) {
                if (itl->onDone)
                    itl->onDone(itl->exc);
                else {
                    if (itl->exc)
                        cerr << "warning: async callback aggregator ate "
                             << "exception" << endl;
                }
            }
        }
    };

    Aggregator aggregator(allKeys.size(), onDone);

    //cerr << "syncing " << allKeys.size() << " keys" << endl;

    for (auto & key: allKeys) {
        // We take its parent since syncAccount assumes nothing was added
    	if (accounts.isInitialized(key))
    		syncAccount(key, aggregator);
    }
}

void
SlaveBanker::
addSpendAccount(const AccountKey & accountKey,
                CurrencyPool accountFloat,
                std::function<void (std::exception_ptr, ShadowAccount&&)> onDone)
{
	bool first = accounts.createAccountAtomic(accountKey);
	if(!first) {
		// already done
		if (onDone) {
			auto account = accounts.getAccount(accountKey);
			onDone(nullptr, std::move(account));
		}
    }
	else {
		// TODO: record float
		//accountFloats[accountKey] = accountFloat;

		// Now kick off the initial synchronization step
		auto onDone2
			= std::bind(&SlaveBanker::onInitializeResult,
						this,
						accountKey,
						onDone,
						std::placeholders::_1,
						std::placeholders::_2);

		cerr << "********* calling addSpendAccount for " << accountKey
			 << " for SlaveBanker " << accountSuffix << endl;

		push(makeRestResponseJsonDecoder<Account>("addSpendAccount", onDone2),
			 "POST",
			 "/v1/accounts",
			 { { "accountName", getShadowAccountStr(accountKey) },
			   { "accountType", "spend" } },
			 "");
	}
}

void
SlaveBanker::
reportSpend(uint64_t numTimeoutsExpired)
{
    if (numTimeoutsExpired > 1) {
        cerr << "warning: slave banker missed " << numTimeoutsExpired
             << " timeouts" << endl;
    }

    if (reportSpendSent != Date())
        cerr << "warning: report spend still in progress" << endl;

    //cerr << "started report spend" << endl;

    auto onDone = [=] (std::exception_ptr exc)
        {
            //cerr << "finished report spend" << endl;
            reportSpendSent = Date();
            if (exc)
                cerr << "reportSpend got exception" << endl;
        };

    syncAll(onDone);
}

void
SlaveBanker::
reauthorizeBudget(uint64_t numTimeoutsExpired)
{
    if (numTimeoutsExpired > 1) {
        cerr << "warning: slave banker missed " << numTimeoutsExpired
             << " timeouts" << endl;
    }

    //std::unique_lock<Lock> guard(lock);
    if (reauthorizeBudgetSent != Date()) {
        cerr << "warning: reauthorize budget still in progress" << endl;
    }

    int numDone = 0;

    // For each of our accounts, we report back what has been spent
    // and re-up to our desired float
    auto onAccount = [&] (const AccountKey & key,
                          const ShadowAccount & account)
        {
            RestRequest request;
            request.verb = "POST";
            request.resource
                = "/v1/accounts/"
                + getShadowAccountStr(key)
                + "/balance";
            request.params = { { "accountType", "spend" } };

            Json::Value payload = CurrencyPool(USD(0.10)).toJson();
            request.payload = payload.toString();

            //cerr << "sending out request " << request << endl;
            ++numDone;

            // Finally, send it out
            push(request, std::bind(&SlaveBanker::onReauthorizeBudgetMessage,
                                    this,
                                    key,
                                    std::placeholders::_1,
                                    std::placeholders::_2,
                                    std::placeholders::_3));
        };

    accounts.forEachInitializedAccount(onAccount);

    if (numDone != 0)
        reauthorizeBudgetSent = Date::now();
}

void
SlaveBanker::
onReauthorizeBudgetMessage(const AccountKey & accountKey,
                           std::exception_ptr exc,
                           int responseCode,
                           const std::string & payload)
{
    //cerr << "finished reauthorize budget" << endl;

    if (exc) {
        cerr << "reauthorize budget got exception" << payload << endl;
        cerr << "accountKey = " << accountKey << endl;
        abort();  // for now...
        return;
    }

    Account masterAccount = Account::fromJson(Json::parse(payload));
    accounts.syncFromMaster(accountKey, masterAccount);
    reauthorizeBudgetSent = Date();
}

} // namespace RTBKIT
<|endoftext|>