<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "file_function.h"

#include <sstream>

#include <zorba/empty_sequence.h>
#include <zorba/diagnostic_list.h>
#include <zorba/serializer.h>
#include <zorba/store_consts.h>
#include <zorba/user_exception.h>
#include <zorba/util/fs_util.h>
#include <zorba/xquery_functions.h>
#include <zorba/singleton_item_sequence.h>
#include <zorba/zorba.h>

#include "file_module.h"

#include <cassert>

#include <stdlib.h>
#include <stdio.h>
#ifdef WIN32
#include <Windows.h>
#include <direct.h>
#else
#include <unistd.h>
#endif

namespace zorba { namespace filemodule {

FileFunction::FileFunction(const FileModule* aModule)
  : theModule(aModule)
{
}

FileFunction::~FileFunction()
{
}

int
FileFunction::raiseFileError(
  char const *aQName,
  char const *aMessage,
  String const &aPath ) const
{
  Item const lQName(
    theModule->getItemFactory()->createQName( getURI(), "file", aQName )
  );
  std::ostringstream lErrorMessage;
  lErrorMessage << '"' << aPath << "\": " << aMessage;
  throw USER_EXCEPTION( lQName, lErrorMessage.str() );
}

String
FileFunction::getURI() const
{
  return theModule->getURI();
}

String
FileFunction::getFilePathString(
  const ExternalFunction::Arguments_t& aArgs,
  unsigned int aPos) const
{
  String lFileArg;

  Item lItem;
  Iterator_t args_iter = aArgs[aPos]->getIterator();
  args_iter->open();
  if (args_iter->next(lItem)) {
    lFileArg = lItem.getStringValue();
  }
  args_iter->close();

  try {
    return fs::normalize_path(lFileArg, fs::curdir());
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), lFileArg );
  }
}

String
FileFunction::pathToFullOSPath(const String& aPath) const {
  try {
    return fs::normalize_path( aPath );
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), aPath );
  }
}

String
FileFunction::getEncodingArg(
  const ExternalFunction::Arguments_t& aArgs,
  unsigned int aPos) const
{
  // the default file encoding
  zorba::String lEncoding("UTF-8");
  if (aArgs.size() > aPos) {
    Item lEncodingItem;
    Iterator_t arg_iter = aArgs[aPos]->getIterator();
    arg_iter->open();
    if (arg_iter->next(lEncodingItem)) {
      lEncoding = fn::upper_case( lEncodingItem.getStringValue() );
    }
    arg_iter->close();
  }

  return lEncoding;
}

String
FileFunction::pathToOSPath(const String& aPath) const {
  try {
    return fs::normalize_path( aPath );
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), aPath );
  }
}

String
FileFunction::pathToUriString(const String& aPath) const {

  if ( fn::starts_with( aPath,"file://" ) ) {
    std::stringstream lErrorMessage;
    lErrorMessage << "Please provide a path, not a URI";
    Item lQName = theModule->getItemFactory()->createQName(
        "http://www.w3.org/2005/xqt-errors",
        "err",
        "XPTY0004");
    throw USER_EXCEPTION( lQName, lErrorMessage.str() );
  }

  String uri( aPath );

  return uri;
}

#ifdef WIN32
bool
FileFunction::isValidDriveSegment(
    String& aString)
{
  aString = fn::upper_case( aString );
  // the drive segment has one of the forms: "C:", "C%3A"
  if ((aString.length() != 2 && aString.length() != 4) ||
      (aString.length() == 2 && !fn::ends_with(aString,":")) ||
      (aString.length() == 4 && !fn::ends_with(aString,"%3A"))) {
    return false;
  }

  char const lDrive = aString[0];
  // the string is already upper case
  return lDrive >= 'A' && lDrive <= 'Z';
}
#endif

//*****************************************************************************

StreamableFileFunction::StreamableFileFunction(const FileModule* aModule)
  : FileFunction(aModule)
{
}

StreamableFileFunction::~StreamableFileFunction()
{
}

bool
StreamableFileFunction::StreamableItemSequence::InternalIterator::next(Item& aResult)
{
  assert(theIsOpen);

  if (theHasNext) {
    aResult = theItemSequence->theItem;
    theHasNext = false;
    return !aResult.isNull();
  }
  return false;
}

//*****************************************************************************

WriterFileFunction::WriterFileFunction(const FileModule* aModule)
  : FileFunction(aModule)
{
}

WriterFileFunction::~WriterFileFunction()
{
}

ItemSequence_t
WriterFileFunction::evaluate(
  const ExternalFunction::Arguments_t& aArgs,
  const StaticContext*,
  const DynamicContext* ) const
{
  String const lFileStr( getFilePathString(aArgs, 0) );

  if ( fs::get_type( lFileStr ) != fs::file )
    raiseFileError( "FOFL0004", "not a plain file", lFileStr );

  bool const lBinary = isBinary();

  std::ios_base::openmode mode = std::ios_base::out
    | (isAppend() ? std::ios_base::app : std::ios_base::trunc);
  if ( lBinary )
    mode |= std::ios_base::binary;

  std::ofstream lOutStream( lFileStr.c_str(), mode );
  if ( !lOutStream ) {
    std::ostringstream oss;
    oss << '"' << lFileStr << "\": can not open file for writing";
    raiseFileError( "FOFL9999", oss.str().c_str(), lFileStr );
  }

  // if this is a binary write
  if (lBinary)
  {
    Item lBinaryItem;
    Iterator_t lContentSeq = aArgs[1]->getIterator();
    lContentSeq->open();
    while (lContentSeq->next(lBinaryItem))
    {
      if (lBinaryItem.isStreamable() && !lBinaryItem.isEncoded())
      {
        lOutStream << lBinaryItem.getStream().rdbuf();
      }
      else
      {
        Zorba_SerializerOptions lOptions;
        lOptions.ser_method = ZORBA_SERIALIZATION_METHOD_BINARY;
        Serializer_t lSerializer = Serializer::createSerializer(lOptions);
        SingletonItemSequence lSeq(lBinaryItem);
        lSerializer->serialize(&lSeq, lOutStream);
      }

    }
  }
  // if we only write text
  else
  {
    Item lStringItem;
    Iterator_t lContentSeq = aArgs[1]->getIterator();
    lContentSeq->open();
    // for each item (string or base64Binary) in the content sequence
    while (lContentSeq->next(lStringItem)) {
      // if the item is streamable make use of the stream
      if (lStringItem.isStreamable()) {
        std::istream& lInStream = lStringItem.getStream();
        char lBuf[1024];
        while (!lInStream.eof()) {
          lInStream.read(lBuf, 1024);
          lOutStream.write(lBuf, lInStream.gcount());
        }
      }
      // else write the string value
      else {
        zorba::String lString = lStringItem.getStringValue();
        lOutStream.write(lString.data(), lString.size());
      }
    }
    lContentSeq->close();
  }

  // close the file stream
  lOutStream.close();

  return ItemSequence_t(new EmptySequence());
}

} // namespace filemodule
} // namespace zorba
/* vim:set et sw=2 ts=2: */
<commit_msg>Fixed WriterFileFunction.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "file_function.h"

#include <sstream>

#include <zorba/empty_sequence.h>
#include <zorba/diagnostic_list.h>
#include <zorba/serializer.h>
#include <zorba/store_consts.h>
#include <zorba/user_exception.h>
#include <zorba/util/fs_util.h>
#include <zorba/xquery_functions.h>
#include <zorba/singleton_item_sequence.h>
#include <zorba/zorba.h>

#include "file_module.h"

#include <cassert>

#include <stdlib.h>
#include <stdio.h>
#ifdef WIN32
#include <Windows.h>
#include <direct.h>
#else
#include <unistd.h>
#endif

namespace zorba { namespace filemodule {

FileFunction::FileFunction(const FileModule* aModule)
  : theModule(aModule)
{
}

FileFunction::~FileFunction()
{
}

int
FileFunction::raiseFileError(
  char const *aQName,
  char const *aMessage,
  String const &aPath ) const
{
  Item const lQName(
    theModule->getItemFactory()->createQName( getURI(), "file", aQName )
  );
  std::ostringstream lErrorMessage;
  lErrorMessage << '"' << aPath << "\": " << aMessage;
  throw USER_EXCEPTION( lQName, lErrorMessage.str() );
}

String
FileFunction::getURI() const
{
  return theModule->getURI();
}

String
FileFunction::getFilePathString(
  const ExternalFunction::Arguments_t& aArgs,
  unsigned int aPos) const
{
  String lFileArg;

  Item lItem;
  Iterator_t args_iter = aArgs[aPos]->getIterator();
  args_iter->open();
  if (args_iter->next(lItem)) {
    lFileArg = lItem.getStringValue();
  }
  args_iter->close();

  try {
    return fs::normalize_path(lFileArg, fs::curdir());
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), lFileArg );
  }
}

String
FileFunction::pathToFullOSPath(const String& aPath) const {
  try {
    return fs::normalize_path( aPath );
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), aPath );
  }
}

String
FileFunction::getEncodingArg(
  const ExternalFunction::Arguments_t& aArgs,
  unsigned int aPos) const
{
  // the default file encoding
  zorba::String lEncoding("UTF-8");
  if (aArgs.size() > aPos) {
    Item lEncodingItem;
    Iterator_t arg_iter = aArgs[aPos]->getIterator();
    arg_iter->open();
    if (arg_iter->next(lEncodingItem)) {
      lEncoding = fn::upper_case( lEncodingItem.getStringValue() );
    }
    arg_iter->close();
  }

  return lEncoding;
}

String
FileFunction::pathToOSPath(const String& aPath) const {
  try {
    return fs::normalize_path( aPath );
  }
  catch ( std::invalid_argument const &e ) {
    throw raiseFileError( "FOFL9999", e.what(), aPath );
  }
}

String
FileFunction::pathToUriString(const String& aPath) const {

  if ( fn::starts_with( aPath,"file://" ) ) {
    std::stringstream lErrorMessage;
    lErrorMessage << "Please provide a path, not a URI";
    Item lQName = theModule->getItemFactory()->createQName(
        "http://www.w3.org/2005/xqt-errors",
        "err",
        "XPTY0004");
    throw USER_EXCEPTION( lQName, lErrorMessage.str() );
  }

  String uri( aPath );

  return uri;
}

#ifdef WIN32
bool
FileFunction::isValidDriveSegment(
    String& aString)
{
  aString = fn::upper_case( aString );
  // the drive segment has one of the forms: "C:", "C%3A"
  if ((aString.length() != 2 && aString.length() != 4) ||
      (aString.length() == 2 && !fn::ends_with(aString,":")) ||
      (aString.length() == 4 && !fn::ends_with(aString,"%3A"))) {
    return false;
  }

  char const lDrive = aString[0];
  // the string is already upper case
  return lDrive >= 'A' && lDrive <= 'Z';
}
#endif

//*****************************************************************************

StreamableFileFunction::StreamableFileFunction(const FileModule* aModule)
  : FileFunction(aModule)
{
}

StreamableFileFunction::~StreamableFileFunction()
{
}

bool
StreamableFileFunction::StreamableItemSequence::InternalIterator::next(Item& aResult)
{
  assert(theIsOpen);

  if (theHasNext) {
    aResult = theItemSequence->theItem;
    theHasNext = false;
    return !aResult.isNull();
  }
  return false;
}

//*****************************************************************************

WriterFileFunction::WriterFileFunction(const FileModule* aModule)
  : FileFunction(aModule)
{
}

WriterFileFunction::~WriterFileFunction()
{
}

ItemSequence_t
WriterFileFunction::evaluate(
  const ExternalFunction::Arguments_t& aArgs,
  const StaticContext*,
  const DynamicContext* ) const
{
  String const lFileStr( getFilePathString(aArgs, 0) );

  fs::type const fs_type = fs::get_type( lFileStr );
  if ( fs_type && fs_type != fs::file )
    raiseFileError( "FOFL0004", "not a plain file", lFileStr );

  bool const lBinary = isBinary();

  std::ios_base::openmode mode = std::ios_base::out
    | (isAppend() ? std::ios_base::app : std::ios_base::trunc);
  if ( lBinary )
    mode |= std::ios_base::binary;

  std::ofstream lOutStream( lFileStr.c_str(), mode );
  if ( !lOutStream ) {
    std::ostringstream oss;
    oss << '"' << lFileStr << "\": can not open file for writing";
    raiseFileError( "FOFL9999", oss.str().c_str(), lFileStr );
  }

  // if this is a binary write
  if (lBinary)
  {
    Item lBinaryItem;
    Iterator_t lContentSeq = aArgs[1]->getIterator();
    lContentSeq->open();
    while (lContentSeq->next(lBinaryItem))
    {
      if (lBinaryItem.isStreamable() && !lBinaryItem.isEncoded())
      {
        lOutStream << lBinaryItem.getStream().rdbuf();
      }
      else
      {
        Zorba_SerializerOptions lOptions;
        lOptions.ser_method = ZORBA_SERIALIZATION_METHOD_BINARY;
        Serializer_t lSerializer = Serializer::createSerializer(lOptions);
        SingletonItemSequence lSeq(lBinaryItem);
        lSerializer->serialize(&lSeq, lOutStream);
      }

    }
  }
  // if we only write text
  else
  {
    Item lStringItem;
    Iterator_t lContentSeq = aArgs[1]->getIterator();
    lContentSeq->open();
    // for each item (string or base64Binary) in the content sequence
    while (lContentSeq->next(lStringItem)) {
      // if the item is streamable make use of the stream
      if (lStringItem.isStreamable()) {
        std::istream& lInStream = lStringItem.getStream();
        char lBuf[1024];
        while (!lInStream.eof()) {
          lInStream.read(lBuf, 1024);
          lOutStream.write(lBuf, lInStream.gcount());
        }
      }
      // else write the string value
      else {
        zorba::String lString = lStringItem.getStringValue();
        lOutStream.write(lString.data(), lString.size());
      }
    }
    lContentSeq->close();
  }

  // close the file stream
  lOutStream.close();

  return ItemSequence_t(new EmptySequence());
}

} // namespace filemodule
} // namespace zorba
/* vim:set et sw=2 ts=2: */
<|endoftext|>


<commit_before>#include "logger.h"
#include "convolution.h"
#include "math/random.hpp"
#include "math/numeric.hpp"
#include "tensor/numeric.hpp"
#include "text/to_string.hpp"
#include "text/from_params.hpp"
#include "tensor/serialize.hpp"

namespace nano
{
        template <typename timatrix, typename tsize, typename tomatrix>
        static void make_convo(const timatrix& imat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                tomatrix&& omat)
        {
                for (tsize kr = 0; kr < krows; ++ kr)
                {
                        for (tsize kc = 0; kc < kcols; ++ kc)
                        {
                                for (tsize r = 0; r < orows; ++ r)
                                {
                                        for (tsize c = 0; c < ocols; ++ c)
                                        {
                                                omat(kr * kcols + kc, r * ocols + c) =
                                                imat(r * drows + kr, c * dcols + kc);
                                        }
                                }
                        }
                }
        }

        template <typename timatrix, typename tsize, typename tkmatrix>
        static void make_convk(const timatrix& imat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                tkmatrix&& kmat)
        {
                for (tsize kr = 0; kr < krows; ++ kr)
                {
                        for (tsize kc = 0; kc < kcols; ++ kc)
                        {
                                for (tsize r = 0; r < orows; ++ r)
                                {
                                        for (tsize c = 0; c < ocols; ++ c)
                                        {
                                                kmat(r * ocols + c, kr * kcols + kc) =
                                                imat(r * drows + kr, c * dcols + kc);
                                        }
                                }
                        }
                }
        }

        template <typename tomatrix, typename tsize, typename timatrix>
        static void make_corr(const tomatrix& omat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                const tsize icols,
                timatrix& imat)
        {
                imat.setZero();
                for (tsize kr = 0; kr < krows; ++ kr)
                {
                        for (tsize kc = 0; kc < kcols; ++ kc)
                        {
                                for (tsize r = 0; r < orows; ++ r)
                                {
                                        for (tsize c = 0; c < ocols; ++ c)
                                        {
                                                imat(kr * kcols + kc, (r * drows + kr) * icols + c * dcols + kc) +=
                                                omat(r, c);
                                        }
                                }
                        }
                }
        }

        convolution_layer_t::convolution_layer_t(const string_t& parameters) :
                layer_t(parameters),
                m_kconn(1), m_drows(1), m_dcols(1)
        {
        }

        tensor_size_t convolution_layer_t::resize(const tensor3d_t& tensor)
        {
                const auto idims = tensor.size<0>();
                const auto irows = tensor.size<1>();
                const auto icols = tensor.size<2>();

                const auto odims = clamp(from_params<tensor_size_t>(config(), "dims"), 1, 256);
                const auto krows = clamp(from_params<tensor_size_t>(config(), "rows"), 1, 32);
                const auto kcols = clamp(from_params<tensor_size_t>(config(), "cols"), 1, 32);
                const auto kconn = clamp(from_params<tensor_size_t>(config(), "conn"), 1, 16);
                const auto drows = clamp(from_params<tensor_size_t>(config(), "drow"), 1, 8);
                const auto dcols = clamp(from_params<tensor_size_t>(config(), "dcol"), 1, 8);

                const auto orows = (irows - krows + 1) / drows;
                const auto ocols = (icols - kcols + 1) / dcols;

                // check convolution size
                if (orows < 1 || ocols < 1)
                {
                        log_error() << "convolution layer: invalid size (" << idims << "x" << irows << "x" << icols
                                    << ") -> (" << odims << "x" << krows << "x" << kcols << ")!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // check input connectivity factor
                if ((idims % kconn) || (odims % kconn))
                {
                        log_error() << "convolution layer: invalid input connectivity factor!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // check stride
                if (2 * drows >= krows || 2 * dcols >= kcols)
                {
                        log_error() << "convolution layer: invalid stride - it should be less than half the kernel size!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // resize buffers
                m_idata.resize(idims, irows, icols);
                m_odata.resize(odims, orows, ocols);
                m_kdata.resize(odims, idims / kconn, krows, kcols);
                m_bdata.resize(odims);
                m_kconn = kconn;
                m_drows = drows;
                m_dcols = dcols;

                m_toe_oidata.resize(krows * kcols, orows * ocols);
                m_toe_okdata.resize(odims / kconn, krows * kcols);
                m_toe_oodata.resize(odims / kconn, orows * ocols);

                m_toe_iodata.resize(krows * kcols, irows * icols);
                m_toe_ikdata.resize(idims / kconn, krows * kcols);
                m_toe_iidata.resize(idims / kconn, irows * icols);

                m_toe_kidata.resize(orows * ocols, krows * kcols);
                m_toe_kodata.resize(odims / kconn, orows * ocols);
                m_toe_kkdata.resize(odims / kconn, krows * kcols);

                return psize();
        }

        void convolution_layer_t::zero_params()
        {
                tensor::set_zero(m_kdata, m_bdata);
        }

        void convolution_layer_t::random_params(scalar_t min, scalar_t max)
        {
                tensor::set_random(random_t<scalar_t>(min, max), m_kdata, m_bdata);
        }

        scalar_t* convolution_layer_t::save_params(scalar_t* params) const
        {
                return tensor::to_array(params, m_kdata, m_bdata);
        }

        const scalar_t* convolution_layer_t::load_params(const scalar_t* params)
        {
                return tensor::from_array(params, m_kdata, m_bdata);
        }

        const tensor3d_t& convolution_layer_t::output(const tensor3d_t& input)
        {
                assert(idims() == input.size<0>());
                assert(irows() == input.size<1>());
                assert(icols() == input.size<2>());

                m_idata = input;

                // convolution
                m_odata.setZero();

                for (tensor_size_t i = 0; i < idims(); ++ i)
                {
                        make_convo(m_idata.matrix(i), orows(), ocols(), krows(), kcols(), drows(), dcols(), m_toe_oidata);
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_toe_okdata.row(ok) = m_kdata.vector(o, i / kconn());
                        }

                        m_toe_oodata.noalias() = m_toe_okdata * m_toe_oidata;

                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_odata.vector(o) += m_toe_oodata.row(ok);
                        }
                }

                // +bias
                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        m_odata.vector(o).array() += m_bdata(o);
                }

                return m_odata;
        }

        const tensor3d_t& convolution_layer_t::ginput(const tensor3d_t& output)
        {
                assert(odims() == output.size<0>());
                assert(orows() == output.size<1>());
                assert(ocols() == output.size<2>());

                m_odata = output;

                // correlation
                m_idata.setZero();

                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        make_corr(m_odata.matrix(o), orows(), ocols(), krows(), kcols(), drows(), dcols(), icols(), m_toe_iodata);
                        for (tensor_size_t i = (o % kconn()), ik = 0; i < idims(); ++ ik, i += kconn())
                        {
                                m_toe_ikdata.row(ik) = m_kdata.vector(o, ik);
                        }

                        m_toe_iidata.noalias() = m_toe_ikdata * m_toe_iodata;

                        for (tensor_size_t i = (o % kconn()), ik = 0; i < idims(); ++ ik, i += kconn())
                        {
                                m_idata.vector(i) += m_toe_iidata.row(ik);
                        }
                }

                return m_idata;
        }

        void convolution_layer_t::gparam(const tensor3d_t& output, scalar_t* gradient)
        {
                assert(odims() == output.size<0>());
                assert(orows() == output.size<1>());
                assert(ocols() == output.size<2>());

                m_odata = output;

                // wrt convolution
                auto gkdata = tensor::map_tensor(gradient, m_kdata.size<0>(), m_kdata.size<1>(), krows(), kcols());
                gkdata.setZero();

                for (tensor_size_t i = 0; i < idims(); ++ i)
                {
                        make_convk(m_idata.matrix(i), orows(), ocols(), krows(), kcols(), drows(), dcols(), m_toe_kidata);
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_toe_kodata.row(ok) = m_odata.vector(o);
                        }

                        m_toe_kkdata.noalias() = m_toe_kodata * m_toe_kidata;
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                gkdata.vector(o, i / kconn()) += m_toe_kkdata.row(ok);
                        }
                }

                // wrt bias
                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        gradient[m_kdata.size() + o] = m_odata.vector(o).sum();
                }
        }
}

<commit_msg>faster construction of toeplitz-like matrices<commit_after>#include "logger.h"
#include "convolution.h"
#include "math/random.hpp"
#include "math/numeric.hpp"
#include "tensor/numeric.hpp"
#include "text/to_string.hpp"
#include "text/from_params.hpp"
#include "tensor/serialize.hpp"

namespace nano
{
        template <typename timatrix, typename tsize, typename tomatrix>
        static void make_convo(const timatrix& imat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                tomatrix&& omat)
        {
                assert(omat.rows() == krows * kcols);
                assert(omat.cols() == orows * ocols);

                for (tsize kr = 0; kr < krows; ++ kr)
                {
                        for (tsize kc = 0; kc < kcols; ++ kc)
                        {
                                for (tsize r = 0; r < orows; ++ r)
                                {
                                        for (tsize c = 0; c < ocols; ++ c)
                                        {
                                                omat(kr * kcols + kc, r * ocols + c) =
                                                imat(r * drows + kr, c * dcols + kc);
                                        }
                                }
                        }
                }
        }

        template <typename timatrix, typename tsize, typename tkmatrix>
        static void make_convk(const timatrix& imat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                tkmatrix&& kmat)
        {
                assert(kmat.rows() == orows * ocols);
                assert(kmat.cols() == krows * kcols);

                for (tsize r = 0; r < orows; ++ r)
                {
                        for (tsize c = 0; c < ocols; ++ c)
                        {
                                for (tsize kr = 0; kr < krows; ++ kr)
                                {
                                        for (tsize kc = 0; kc < kcols; ++ kc)
                                        {
                                                kmat(r * ocols + c, kr * kcols + kc) =
                                                imat(r * drows + kr, c * dcols + kc);
                                        }
                                }
                        }
                }
        }

        template <typename tomatrix, typename tsize, typename timatrix>
        static void make_corr(const tomatrix& omat,
                const tsize orows, const tsize ocols,
                const tsize krows, const tsize kcols,
                const tsize drows, const tsize dcols,
                const tsize irows, const tsize icols,
                timatrix& imat)
        {
                assert(imat.rows() == krows * kcols);
                assert(imat.cols() == irows * icols);

                NANO_UNUSED1_RELEASE(irows);

                imat.setZero();
                for (tsize kr = 0; kr < krows; ++ kr)
                {
                        for (tsize kc = 0; kc < kcols; ++ kc)
                        {
                                for (tsize r = 0; r < orows; ++ r)
                                {
                                        for (tsize c = 0; c < ocols; ++ c)
                                        {
                                                imat(kr * kcols + kc, (r * drows + kr) * icols + c * dcols + kc) +=
                                                omat(r, c);
                                        }
                                }
                        }
                }
        }

        convolution_layer_t::convolution_layer_t(const string_t& parameters) :
                layer_t(parameters),
                m_kconn(1), m_drows(1), m_dcols(1)
        {
        }

        tensor_size_t convolution_layer_t::resize(const tensor3d_t& tensor)
        {
                const auto idims = tensor.size<0>();
                const auto irows = tensor.size<1>();
                const auto icols = tensor.size<2>();

                const auto odims = clamp(from_params<tensor_size_t>(config(), "dims"), 1, 256);
                const auto krows = clamp(from_params<tensor_size_t>(config(), "rows"), 1, 32);
                const auto kcols = clamp(from_params<tensor_size_t>(config(), "cols"), 1, 32);
                const auto kconn = clamp(from_params<tensor_size_t>(config(), "conn"), 1, 16);
                const auto drows = clamp(from_params<tensor_size_t>(config(), "drow"), 1, 8);
                const auto dcols = clamp(from_params<tensor_size_t>(config(), "dcol"), 1, 8);

                const auto orows = (irows - krows + 1) / drows;
                const auto ocols = (icols - kcols + 1) / dcols;

                // check convolution size
                if (orows < 1 || ocols < 1)
                {
                        log_error() << "convolution layer: invalid size (" << idims << "x" << irows << "x" << icols
                                    << ") -> (" << odims << "x" << krows << "x" << kcols << ")!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // check input connectivity factor
                if ((idims % kconn) || (odims % kconn))
                {
                        log_error() << "convolution layer: invalid input connectivity factor!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // check stride
                if (2 * drows >= krows || 2 * dcols >= kcols)
                {
                        log_error() << "convolution layer: invalid stride - it should be less than half the kernel size!";
                        throw std::invalid_argument("invalid configuration for the convolution layer");
                }

                // resize buffers
                m_idata.resize(idims, irows, icols);
                m_odata.resize(odims, orows, ocols);
                m_kdata.resize(odims, idims / kconn, krows, kcols);
                m_bdata.resize(odims);
                m_kconn = kconn;
                m_drows = drows;
                m_dcols = dcols;

                m_toe_oidata.resize(krows * kcols, orows * ocols);
                m_toe_okdata.resize(odims / kconn, krows * kcols);
                m_toe_oodata.resize(odims / kconn, orows * ocols);

                m_toe_iodata.resize(krows * kcols, irows * icols);
                m_toe_ikdata.resize(idims / kconn, krows * kcols);
                m_toe_iidata.resize(idims / kconn, irows * icols);

                m_toe_kidata.resize(orows * ocols, krows * kcols);
                m_toe_kodata.resize(odims / kconn, orows * ocols);
                m_toe_kkdata.resize(odims / kconn, krows * kcols);

                return psize();
        }

        void convolution_layer_t::zero_params()
        {
                tensor::set_zero(m_kdata, m_bdata);
        }

        void convolution_layer_t::random_params(scalar_t min, scalar_t max)
        {
                tensor::set_random(random_t<scalar_t>(min, max), m_kdata, m_bdata);
        }

        scalar_t* convolution_layer_t::save_params(scalar_t* params) const
        {
                return tensor::to_array(params, m_kdata, m_bdata);
        }

        const scalar_t* convolution_layer_t::load_params(const scalar_t* params)
        {
                return tensor::from_array(params, m_kdata, m_bdata);
        }

        const tensor3d_t& convolution_layer_t::output(const tensor3d_t& input)
        {
                assert(idims() == input.size<0>());
                assert(irows() == input.size<1>());
                assert(icols() == input.size<2>());

                m_idata = input;

                // convolution
                m_odata.setZero();

                for (tensor_size_t i = 0; i < idims(); ++ i)
                {
                        make_convo(m_idata.matrix(i), orows(), ocols(), krows(), kcols(), drows(), dcols(), m_toe_oidata);
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_toe_okdata.row(ok) = m_kdata.vector(o, i / kconn());
                        }

                        m_toe_oodata.noalias() = m_toe_okdata * m_toe_oidata;

                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_odata.vector(o) += m_toe_oodata.row(ok);
                        }
                }

                // +bias
                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        m_odata.vector(o).array() += m_bdata(o);
                }

                return m_odata;
        }

        const tensor3d_t& convolution_layer_t::ginput(const tensor3d_t& output)
        {
                assert(odims() == output.size<0>());
                assert(orows() == output.size<1>());
                assert(ocols() == output.size<2>());

                m_odata = output;

                // correlation
                m_idata.setZero();

                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        make_corr(m_odata.matrix(o), orows(), ocols(), krows(), kcols(), drows(), dcols(), irows(), icols(), m_toe_iodata);
                        for (tensor_size_t i = (o % kconn()), ik = 0; i < idims(); ++ ik, i += kconn())
                        {
                                m_toe_ikdata.row(ik) = m_kdata.vector(o, ik);
                        }

                        m_toe_iidata.noalias() = m_toe_ikdata * m_toe_iodata;

                        for (tensor_size_t i = (o % kconn()), ik = 0; i < idims(); ++ ik, i += kconn())
                        {
                                m_idata.vector(i) += m_toe_iidata.row(ik);
                        }
                }

                return m_idata;
        }

        void convolution_layer_t::gparam(const tensor3d_t& output, scalar_t* gradient)
        {
                assert(odims() == output.size<0>());
                assert(orows() == output.size<1>());
                assert(ocols() == output.size<2>());

                m_odata = output;

                // wrt convolution
                auto gkdata = tensor::map_tensor(gradient, m_kdata.size<0>(), m_kdata.size<1>(), krows(), kcols());
                gkdata.setZero();

                for (tensor_size_t i = 0; i < idims(); ++ i)
                {
                        make_convk(m_idata.matrix(i), orows(), ocols(), krows(), kcols(), drows(), dcols(), m_toe_kidata);
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                m_toe_kodata.row(ok) = m_odata.vector(o);
                        }

                        m_toe_kkdata.noalias() = m_toe_kodata * m_toe_kidata;
                        for (tensor_size_t o = (i % kconn()), ok = 0; o < odims(); ++ ok, o += kconn())
                        {
                                gkdata.vector(o, i / kconn()) += m_toe_kkdata.row(ok);
                        }
                }

                // wrt bias
                for (tensor_size_t o = 0; o < odims(); ++ o)
                {
                        gradient[m_kdata.size() + o] = m_odata.vector(o).sum();
                }
        }
}

<|endoftext|>


<commit_before>#include <QtTest/QtTest>

#include <QtNetwork/QSslSocket>

#include "../src/serverapi.h"

class ServerApiTest : public QObject
{
    Q_OBJECT

private:
    struct Response {
        QVariantMap data;
        bool ok;
    };

    Response sendRequest(const QString &path, const QJsonObject &reqData, bool secure,
                         int callbackExpireTime = 1000, int updateInterval = 500);

private slots:
    void initTestCase();
    void cleanupTestCase();

    void testUserCreateRequest();
    void testTownsRequest();
    void testCashpointRequest();
};

void ServerApiTest::initTestCase()
{
}

void ServerApiTest::cleanupTestCase()
{
}

void ServerApiTest::testUserCreateRequest()
{
    const QString path = "/user";
    QJsonObject req
    {
        {"login", "testUser"},
        {"password", "testPassword"}
    };

    const Response res = sendRequest(path, req, false);

    qDebug() << res.ok << "\n" << res.data;
}

void ServerApiTest::testTownsRequest()
{
    const int townId = 32;
    const QString path = "/town/" + QString::number(townId);

    const Response res = sendRequest(path, QJsonObject(), false);

    QCOMPARE(res.ok, true);
    {
        QCOMPARE(res.data["id"].toInt(),         townId);
        QCOMPARE(res.data["name"].toString(),    QString("Волоколамск"));
        QCOMPARE(res.data["name_tr"].toString(), QString("Volokolamsk"));
        QCOMPARE(res.data["zoom"].toInt(),       12);
    }
}

void ServerApiTest::testCashpointRequest()
{
    const int cpId = 6271738;
    const QString path = "/cashpoint/" + QString::number(cpId);

    const Response res = sendRequest(path, QJsonObject(), false);

    QCOMPARE(res.ok, true);
    {
        QCOMPARE(res.data["id"].toInt(),      cpId);
        QCOMPARE(res.data["type"].toString(), QString("atm"));
        QCOMPARE(res.data["bank_id"].toInt(), 3425);
        QCOMPARE(res.data["town_id"].toInt(), 4);
//        QVERIFY(qFuzzyCompare(res.data["longitude"].toDouble(), 37.699253076057));
//        QVERIFY(qFuzzyCompare(res.data["latitude"].toDouble(), 55.7949921030773));
//        QCOMPARE(res.data["address"].toString(), QString("г.Москва, ул. Стромынка, д. 21 корп. 1"));
        QCOMPARE(res.data["address_comment"].toString(), QString("Управление социальной защиты населения района «Преображенский»"));
        QCOMPARE(res.data["metro_name"].toString(), QString("Преображенская площадь"));
        QCOMPARE(res.data["free_access"].toInt(), 1);
        QCOMPARE(res.data["main_office"].toInt(), 0);
        QCOMPARE(res.data["without_weekend"].toInt(), 1);
        QCOMPARE(res.data["round_the_clock"].toInt(), 0);
        QCOMPARE(res.data["works_as_shop"].toInt(), 1);
        QCOMPARE(res.data["rub"].toInt(), 1);
        QCOMPARE(res.data["usd"].toInt(), 0);
        QCOMPARE(res.data["eur"].toInt(), 0);
        QCOMPARE(res.data["cash_in"].toInt(), 0);
    }
}

/// ===============================================================

ServerApiTest::Response ServerApiTest::sendRequest(const QString &path, const QJsonObject &reqData, bool secure,
                                                   int callbackExpireTime, int updateInterval)
{
    /// TODO: secure connection
    Q_UNUSED(secure)
//    QFile certFile(":/data/cert.pem");
//    QVERIFY2(certFile.open(QIODevice::ReadOnly), "Cannot open cert file");
//    QSslSocket::addDefaultCaCertificate(QSslCertificate(&certFile, QSsl::Pem));

    Response response;

    ServerApi api("127.0.0.1", 8080);
//    ServerApi api("127.0.0.1", 8081, &certFile);
    {
        QEventLoop loop;
        connect(&api, SIGNAL(requestTimedout(qint64)), &loop, SLOT(quit()));
        connect(&api, SIGNAL(responseReceived(qint64)), &loop, SLOT(quit()));

        QTimer *apiUpdateTimer = new QTimer(&api);
        connect(apiUpdateTimer, SIGNAL(timeout()), &api, SLOT(update()));

        api.setCallbacksExpireTime(callbackExpireTime);
        api.sendRequest(path, reqData, [&](const QByteArray &data, bool timeOut)
        {
            if (!timeOut) {
                response.ok = true;
                QJsonDocument jsonDoc = QJsonDocument::fromJson(data);
                response.data = jsonDoc.object().toVariantMap();
            } else {
                qWarning() << "Request timed out";
                response.ok = false;
            }
            loop.quit();
        });

        apiUpdateTimer->start(updateInterval);

        loop.exec();
    }

    return response;
}

QTEST_MAIN(ServerApiTest)
#include "ts_serverapi.moc"
<commit_msg>[C|Client] Tests: recovered coord tests for town req (impl floatCompare)<commit_after>#include <QtTest/QtTest>

#include <QtNetwork/QSslSocket>

#include "../src/serverapi.h"

bool floatCompare(float a, float b)
{
    return qAbs(a - b) < 0.0001f;
}

class ServerApiTest : public QObject
{
    Q_OBJECT

private:
    struct Response {
        QVariantMap data;
        bool ok;
    };

    Response sendRequest(const QString &path, const QJsonObject &reqData, bool secure,
                         int callbackExpireTime = 1000, int updateInterval = 500);

private slots:
    void initTestCase();
    void cleanupTestCase();

    void testUserCreateRequest();
    void testTownsRequest();
    void testCashpointRequest();
};

void ServerApiTest::initTestCase()
{
}

void ServerApiTest::cleanupTestCase()
{
}

void ServerApiTest::testUserCreateRequest()
{
    const QString path = "/user";
    QJsonObject req
    {
        {"login", "testUser"},
        {"password", "testPassword"}
    };

    const Response res = sendRequest(path, req, false);

    qDebug() << res.ok << "\n" << res.data;
}

void ServerApiTest::testTownsRequest()
{
    const int townId = 32;
    const QString path = "/town/" + QString::number(townId);

    const Response res = sendRequest(path, QJsonObject(), false);

    QCOMPARE(res.ok, true);
    {
        QCOMPARE(res.data["id"].toInt(),         townId);
        QCOMPARE(res.data["name"].toString(),    QString("Волоколамск"));
        QCOMPARE(res.data["name_tr"].toString(), QString("Volokolamsk"));
        QCOMPARE(res.data["zoom"].toInt(),       12);
    }
}

void ServerApiTest::testCashpointRequest()
{
    const int cpId = 6271738;
    const QString path = "/cashpoint/" + QString::number(cpId);

    const Response res = sendRequest(path, QJsonObject(), false);

    QCOMPARE(res.ok, true);
    {
        QCOMPARE(res.data["id"].toInt(),      cpId);
        QCOMPARE(res.data["type"].toString(), QString("atm"));
        QCOMPARE(res.data["bank_id"].toInt(), 3425);
        QCOMPARE(res.data["town_id"].toInt(), 4);
        QVERIFY(floatCompare(res.data["longitude"].toDouble(), 37.699253076057));
        QVERIFY(floatCompare(res.data["latitude"].toDouble(), 55.7949921030773));
//        QCOMPARE(res.data["address"].toString(), QString("г.Москва, ул. Стромынка, д. 21 корп. 1"));
        QCOMPARE(res.data["address_comment"].toString(), QString("Управление социальной защиты населения района «Преображенский»"));
        QCOMPARE(res.data["metro_name"].toString(), QString("Преображенская площадь"));
        QCOMPARE(res.data["free_access"].toInt(), 1);
        QCOMPARE(res.data["main_office"].toInt(), 0);
        QCOMPARE(res.data["without_weekend"].toInt(), 1);
        QCOMPARE(res.data["round_the_clock"].toInt(), 0);
        QCOMPARE(res.data["works_as_shop"].toInt(), 1);
        QCOMPARE(res.data["rub"].toInt(), 1);
        QCOMPARE(res.data["usd"].toInt(), 0);
        QCOMPARE(res.data["eur"].toInt(), 0);
        QCOMPARE(res.data["cash_in"].toInt(), 0);
    }
}

/// ===============================================================

ServerApiTest::Response ServerApiTest::sendRequest(const QString &path, const QJsonObject &reqData, bool secure,
                                                   int callbackExpireTime, int updateInterval)
{
    /// TODO: secure connection
    Q_UNUSED(secure)
//    QFile certFile(":/data/cert.pem");
//    QVERIFY2(certFile.open(QIODevice::ReadOnly), "Cannot open cert file");
//    QSslSocket::addDefaultCaCertificate(QSslCertificate(&certFile, QSsl::Pem));

    Response response;

    ServerApi api("127.0.0.1", 8080);
//    ServerApi api("127.0.0.1", 8081, &certFile);
    {
        QEventLoop loop;
        connect(&api, SIGNAL(requestTimedout(qint64)), &loop, SLOT(quit()));
        connect(&api, SIGNAL(responseReceived(qint64)), &loop, SLOT(quit()));

        QTimer *apiUpdateTimer = new QTimer(&api);
        connect(apiUpdateTimer, SIGNAL(timeout()), &api, SLOT(update()));

        api.setCallbacksExpireTime(callbackExpireTime);
        api.sendRequest(path, reqData, [&](const QByteArray &data, bool timeOut)
        {
            if (!timeOut) {
                response.ok = true;
                QJsonDocument jsonDoc = QJsonDocument::fromJson(data);
                response.data = jsonDoc.object().toVariantMap();
            } else {
                qWarning() << "Request timed out";
                response.ok = false;
            }
            loop.quit();
        });

        apiUpdateTimer->start(updateInterval);

        loop.exec();
    }

    return response;
}

QTEST_MAIN(ServerApiTest)
#include "ts_serverapi.moc"
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "ports/mojo_system/message_pipe_dispatcher.h"

#include "ports/mojo_system/core.h"

namespace mojo {
namespace edk {

MessagePipeDispatcher::MessagePipeDispatcher(Node* node,
                                             const ports::PortName& port_name)
    : node_(node), port_name_(port_name) {
  node_->SetPortObserver(port_name_, this);
}

Dispatcher::Type MessagePipeDispatcher::GetType() const {
  return Type::MESSAGE_PIPE;
}

MessagePipeDispatcher::~MessagePipeDispatcher() {
  node_->SetPortObserver(port_name_, nullptr);
}

void MessagePipeDispatcher::CloseImplNoLock() {
  lock().AssertAcquired();
  DCHECK(is_closed());
  if (!port_transferred_)
    node_->ClosePort(port_name_);
}

MojoResult MessagePipeDispatcher::WriteMessageImplNoLock(
    const void* bytes,
    uint32_t num_bytes,
    const DispatcherInTransit* dispatchers,
    uint32_t num_dispatchers,
    MojoWriteMessageFlags flags) {
  lock().AssertAcquired();

  ports::ScopedMessage message(ports::AllocMessage(num_bytes, num_dispatchers));
  memcpy(message->bytes, bytes, num_bytes);
  for (size_t i = 0; i < num_dispatchers; ++i) {
    Dispatcher* d = dispatchers[i].dispatcher.get();

    // TODO: support transferring other types of handles
    CHECK_EQ(d->GetType(), Type::MESSAGE_PIPE);

    MessagePipeDispatcher* mpd = static_cast<MessagePipeDispatcher*>(d);
    message->ports[i].name = mpd->GetPortName();
  }

  int rv = node_->SendMessage(port_name_, std::move(message));

  // TODO: More detailed result code on failure
  if (rv != ports::OK)
    return MOJO_RESULT_INVALID_ARGUMENT;

  return MOJO_RESULT_OK;
}

MojoResult MessagePipeDispatcher::ReadMessageImplNoLock(
    void* bytes,
    uint32_t* num_bytes,
    MojoHandle* handles,
    uint32_t* num_handles,
    MojoReadMessageFlags flags) {
  lock().AssertAcquired();

  bool no_space = false;

  ports::ScopedMessage message;
  int rv = node_->GetMessageIf(
      port_name_,
      [num_bytes, num_handles, &no_space](const ports::Message& next_message) {
        size_t bytes_to_read = 0;
        if (num_bytes) {
          bytes_to_read = std::min(static_cast<size_t>(*num_bytes),
                                   next_message.num_bytes);
          *num_bytes = next_message.num_bytes;
        }

        size_t handles_to_read = 0;
        if (num_handles) {
          handles_to_read = std::min(static_cast<size_t>(*num_handles),
                                     next_message.num_ports);
          *num_handles = next_message.num_ports;
        }

        if (bytes_to_read < next_message.num_bytes ||
            handles_to_read < next_message.num_ports) {
          no_space = true;
          return false;
        }

        return true;
      },
      &message);

  if (rv != ports::OK) {
    if (rv == ports::ERROR_PORT_UNKNOWN)
      return MOJO_RESULT_INVALID_ARGUMENT;
    if (rv == ports::ERROR_PORT_PEER_CLOSED) {
      peer_closed_ = true;
      awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
      return MOJO_RESULT_FAILED_PRECONDITION;
    }
    return MOJO_RESULT_UNKNOWN;  // TODO: Add a better error code here?
  }

  if (no_space)
    return MOJO_RESULT_RESOURCE_EXHAUSTED;

  if (!message)
    return MOJO_RESULT_SHOULD_WAIT;

  if (!node_->core()->AddDispatchersForReceivedPorts(*message, handles)) {
    // TODO: Close all of the received ports.
    return MOJO_RESULT_UNKNOWN;  // TODO: Add a better error code here?
  }

  memcpy(bytes, message->bytes, message->num_bytes);

  // Check to see if the queue is completely drained so we can unset the
  // READABLE signal on the handle.
  //
  // TODO: maybe there's a better way to do this?
  bool still_readable = false;
  rv = node_->GetMessageIf(
      port_name_, [&still_readable](const ports::Message&) {
        still_readable = true;
        return false;
      }, &message);
  if (!still_readable ) {
    port_readable_ = false;
    awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
  }

  return MOJO_RESULT_OK;
}

HandleSignalsState
MessagePipeDispatcher::GetHandleSignalsStateImplNoLock() const {
  lock().AssertAcquired();

  HandleSignalsState rv;
  if (port_readable_) {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_READABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_READABLE;
  }
  if (!peer_closed_) {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_WRITABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_READABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_WRITABLE;
  } else {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_PEER_CLOSED;
  }
  rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_PEER_CLOSED;
  return rv;
}

MojoResult MessagePipeDispatcher::AddAwakableImplNoLock(
    Awakable* awakable,
    MojoHandleSignals signals,
    uintptr_t context,
    HandleSignalsState* signals_state) {
  lock().AssertAcquired();
  HandleSignalsState state = GetHandleSignalsStateImplNoLock();
  if (state.satisfies(signals)) {
    if (signals_state)
      *signals_state = state;
    return MOJO_RESULT_ALREADY_EXISTS;
  }
  if (!state.can_satisfy(signals)) {
    if (signals_state)
      *signals_state = state;
    return MOJO_RESULT_FAILED_PRECONDITION;
  }

  awakables_.Add(awakable, signals, context);
  return MOJO_RESULT_OK;
}

void MessagePipeDispatcher::RemoveAwakableImplNoLock(
    Awakable* awakable,
    HandleSignalsState* signals_state) {
  lock().AssertAcquired();
  awakables_.Remove(awakable);
}

bool MessagePipeDispatcher::BeginTransitImplNoLock() {
  return true;
}

void MessagePipeDispatcher::EndTransitImplNoLock(bool canceled) {
  if (!canceled) {
    // port_name_ has been closed by virtue of having been transferred.
    // This dispatcher needs to be closed as well.
    port_transferred_ = true;
    CloseNoLock();

    // TODO: Need to implement CancelAllAwakablesNoLock.
  }
}

void MessagePipeDispatcher::OnMessagesAvailable() {
  LOG(ERROR) << "OnMessagesAvailable";

  base::AutoLock dispatcher_lock(lock());
  port_readable_ = true;
  awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
}

}  // namespace edk
}  // namespace mojo
<commit_msg>make sure the port really has messages before signaling<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "ports/mojo_system/message_pipe_dispatcher.h"

#include "ports/mojo_system/core.h"

namespace mojo {
namespace edk {

MessagePipeDispatcher::MessagePipeDispatcher(Node* node,
                                             const ports::PortName& port_name)
    : node_(node), port_name_(port_name) {
  node_->SetPortObserver(port_name_, this);
}

Dispatcher::Type MessagePipeDispatcher::GetType() const {
  return Type::MESSAGE_PIPE;
}

MessagePipeDispatcher::~MessagePipeDispatcher() {
  node_->SetPortObserver(port_name_, nullptr);
}

void MessagePipeDispatcher::CloseImplNoLock() {
  lock().AssertAcquired();
  DCHECK(is_closed());
  if (!port_transferred_)
    node_->ClosePort(port_name_);
}

MojoResult MessagePipeDispatcher::WriteMessageImplNoLock(
    const void* bytes,
    uint32_t num_bytes,
    const DispatcherInTransit* dispatchers,
    uint32_t num_dispatchers,
    MojoWriteMessageFlags flags) {
  lock().AssertAcquired();

  ports::ScopedMessage message(ports::AllocMessage(num_bytes, num_dispatchers));
  memcpy(message->bytes, bytes, num_bytes);
  for (size_t i = 0; i < num_dispatchers; ++i) {
    Dispatcher* d = dispatchers[i].dispatcher.get();

    // TODO: support transferring other types of handles
    CHECK_EQ(d->GetType(), Type::MESSAGE_PIPE);

    MessagePipeDispatcher* mpd = static_cast<MessagePipeDispatcher*>(d);
    message->ports[i].name = mpd->GetPortName();
  }

  int rv = node_->SendMessage(port_name_, std::move(message));

  // TODO: More detailed result code on failure
  if (rv != ports::OK)
    return MOJO_RESULT_INVALID_ARGUMENT;

  return MOJO_RESULT_OK;
}

MojoResult MessagePipeDispatcher::ReadMessageImplNoLock(
    void* bytes,
    uint32_t* num_bytes,
    MojoHandle* handles,
    uint32_t* num_handles,
    MojoReadMessageFlags flags) {
  lock().AssertAcquired();

  bool no_space = false;

  ports::ScopedMessage message;
  int rv = node_->GetMessageIf(
      port_name_,
      [num_bytes, num_handles, &no_space](const ports::Message& next_message) {
        size_t bytes_to_read = 0;
        if (num_bytes) {
          bytes_to_read = std::min(static_cast<size_t>(*num_bytes),
                                   next_message.num_bytes);
          *num_bytes = next_message.num_bytes;
        }

        size_t handles_to_read = 0;
        if (num_handles) {
          handles_to_read = std::min(static_cast<size_t>(*num_handles),
                                     next_message.num_ports);
          *num_handles = next_message.num_ports;
        }

        if (bytes_to_read < next_message.num_bytes ||
            handles_to_read < next_message.num_ports) {
          no_space = true;
          return false;
        }

        return true;
      },
      &message);

  if (rv != ports::OK) {
    if (rv == ports::ERROR_PORT_UNKNOWN)
      return MOJO_RESULT_INVALID_ARGUMENT;
    if (rv == ports::ERROR_PORT_PEER_CLOSED) {
      peer_closed_ = true;
      awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
      return MOJO_RESULT_FAILED_PRECONDITION;
    }
    return MOJO_RESULT_UNKNOWN;  // TODO: Add a better error code here?
  }

  if (no_space)
    return MOJO_RESULT_RESOURCE_EXHAUSTED;

  if (!message)
    return MOJO_RESULT_SHOULD_WAIT;

  if (!node_->core()->AddDispatchersForReceivedPorts(*message, handles)) {
    // TODO: Close all of the received ports.
    return MOJO_RESULT_UNKNOWN;  // TODO: Add a better error code here?
  }

  memcpy(bytes, message->bytes, message->num_bytes);

  // Check to see if the queue is completely drained so we can unset the
  // READABLE signal on the handle.
  //
  // TODO: maybe there's a better way to do this?
  bool still_readable = false;
  rv = node_->GetMessageIf(
      port_name_, [&still_readable](const ports::Message&) {
        still_readable = true;
        return false;
      }, &message);
  if (!still_readable ) {
    port_readable_ = false;
    awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
  }

  return MOJO_RESULT_OK;
}

HandleSignalsState
MessagePipeDispatcher::GetHandleSignalsStateImplNoLock() const {
  lock().AssertAcquired();

  HandleSignalsState rv;
  if (port_readable_) {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_READABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_READABLE;
  }
  if (!peer_closed_) {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_WRITABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_READABLE;
    rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_WRITABLE;
  } else {
    rv.satisfied_signals |= MOJO_HANDLE_SIGNAL_PEER_CLOSED;
  }
  rv.satisfiable_signals |= MOJO_HANDLE_SIGNAL_PEER_CLOSED;
  return rv;
}

MojoResult MessagePipeDispatcher::AddAwakableImplNoLock(
    Awakable* awakable,
    MojoHandleSignals signals,
    uintptr_t context,
    HandleSignalsState* signals_state) {
  lock().AssertAcquired();
  HandleSignalsState state = GetHandleSignalsStateImplNoLock();
  if (state.satisfies(signals)) {
    if (signals_state)
      *signals_state = state;
    return MOJO_RESULT_ALREADY_EXISTS;
  }
  if (!state.can_satisfy(signals)) {
    if (signals_state)
      *signals_state = state;
    return MOJO_RESULT_FAILED_PRECONDITION;
  }

  awakables_.Add(awakable, signals, context);
  return MOJO_RESULT_OK;
}

void MessagePipeDispatcher::RemoveAwakableImplNoLock(
    Awakable* awakable,
    HandleSignalsState* signals_state) {
  lock().AssertAcquired();
  awakables_.Remove(awakable);
}

bool MessagePipeDispatcher::BeginTransitImplNoLock() {
  return true;
}

void MessagePipeDispatcher::EndTransitImplNoLock(bool canceled) {
  if (!canceled) {
    // port_name_ has been closed by virtue of having been transferred.
    // This dispatcher needs to be closed as well.
    port_transferred_ = true;
    CloseNoLock();

    // TODO: Need to implement CancelAllAwakablesNoLock.
  }
}

void MessagePipeDispatcher::OnMessagesAvailable() {
  LOG(ERROR) << "OnMessagesAvailable";

  base::AutoLock dispatcher_lock(lock());

  // Check to see if the queue is actually readable before signaling.
  //
  // We can be notified of messages being available when in fact its peer has
  // just been closed.
  //
  // TODO: something better
  bool is_port_readable = false;
  ports::ScopedMessage message;
  int rv = node_->GetMessageIf(
      port_name_, [&is_port_readable](const ports::Message&) {
        is_port_readable = true;
        return false;
      }, &message);

  DCHECK(rv == ports::OK || ports::ERROR_PORT_PEER_CLOSED);

  if (!port_readable_ && is_port_readable) {
    port_readable_ = true;
    awakables_.AwakeForStateChange(GetHandleSignalsStateImplNoLock());
  }
}

}  // namespace edk
}  // namespace mojo
<|endoftext|>


<commit_before>/******************************************************************************

  This source file is part of the MoleQueue project.

  Copyright 2012 Kitware, Inc.

  This source code is released under the New BSD License, (the "License").

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

******************************************************************************/

#include <QtTest>

#include "filesystemtools.h"

#include "molequeuetestconfig.h"
#include "testserver.h" // for getRandomSocketName

#include <QtCore/QProcess>
#include <QtCore/QProcessEnvironment>

// Define ENABLE_ZMQ_TESTS if both zeromq and python are available
#ifndef MoleQueue_PYTHON_EXECUTABLE
#ifndef MoleQueue_HAS_ZMQ
#define ENABLE_ZMQ_TESTS
#endif // MoleQueue_HAS_ZMQ
#endif // MoleQueue_PYTHON_EXECUTABLE

class ClientServerTest : public QObject
{
  Q_OBJECT
public:
  ClientServerTest()
    : QObject(NULL),
      m_numClients(10),
      m_workDir(MoleQueue_BINARY_DIR "/testworkdir"),
      m_moleQueueExecutable(MoleQueue_BINARY_DIR "/bin/molequeue"),
      m_serverProcess(NULL)
  {
#ifdef __APPLE__
    m_moleQueueExecutable = MoleQueue_BINARY_DIR "/bin/molequeue.app/Contents/"
        "MacOS/molequeue";
#endif // __APPLE__
    randomizeSocketName();
  }

private:
  int m_numClients;
  QString m_workDir;
  QString m_socketName;
  QString m_moleQueueExecutable;
  QStringList m_moleQueueDefaultArgs;
  QProcess *m_serverProcess;
  QList<QProcess*> m_clientProcesses;

  /// Delete the testing workdir and initialize it with the directory at
  /// @a sourcePath.
  bool resetWorkDir(const QString &sourcePath);

  /// Create a new randomized socket name, stored in m_socketName;
  void randomizeSocketName();

  /// Create the server process (m_serverProcess) and reset
  /// m_moleQueueDefaultArgs to set the workdir, socketname, and enable rpcKill.
  bool setupServerProcess();

  /// Create a Cxx client process
  QProcess *addClientProcess();

#ifdef ENABLE_ZMQ_TESTS
  /// Create a client process initialized for python. The process is returned
  /// and added to m_clientProcesses.
  QProcess *addPythonClientProcess();
#endif // ENABLE_ZMQ_TESTS

private slots:
  /// Called before the first test function is executed.
  void initTestCase();
  /// Called after the last test function is executed.
  void cleanupTestCase();
  /// Called before each test function is executed.
  void init();
  /// Called after every test function.
  void cleanup();

  // Python client tests:
#ifdef ENABLE_ZMQ_TESTS
  void submitOnePy();
  void submit200Py();
  void submit200FromManyClientsPy();
#endif // ENABLE_ZMQ_TESTS
};

bool ClientServerTest::resetWorkDir(const QString &sourcePath)
{
  // Initialize working directory
  if (QFileInfo(m_workDir).exists()) {
    if (!MoleQueue::FileSystemTools::recursiveRemoveDirectory(m_workDir)) {
      qWarning() << "Could not remove old working directory" << m_workDir;
      return false;
    }
  }
  if (!MoleQueue::FileSystemTools::recursiveCopyDirectory(sourcePath,
                                                          m_workDir)) {
    qWarning() << "Could not initialize working directory" << m_workDir
               << "from" << sourcePath;
    return false;
  }
  return true;
}

void ClientServerTest::randomizeSocketName()
{
  m_socketName = TestServer::getRandomSocketName();
}

bool ClientServerTest::setupServerProcess()
{
  m_moleQueueDefaultArgs.clear();
  m_moleQueueDefaultArgs
      << "--workdir" << m_workDir
      << "--socketname" << m_socketName
      << "--rpc-kill";

  if (m_serverProcess) {
    delete m_serverProcess;
    m_serverProcess = NULL;
  }
  m_serverProcess = new QProcess(this);
  m_serverProcess->setProcessChannelMode(QProcess::ForwardedChannels);
  return true;
}

QProcess *ClientServerTest::addClientProcess()
{
  QProcess *clientProcess = new QProcess(this);
  clientProcess->setProcessChannelMode(::QProcess::ForwardedChannels);
  m_clientProcesses.append(clientProcess);
  return clientProcess;
}

#ifdef ENABLE_ZMQ_TESTS
QProcess *ClientServerTest::addPythonClientProcess()
{
  QProcess *clientProcess = addClientProcess();
  QProcessEnvironment env = clientProcess->processEnvironment();
  env.insert("PYTHONPATH", (env.value("PYTHONPATH").isEmpty()
                            ? QString()
                            : env.value("PYTHONPATH") + ':') +
             MoleQueue_SOURCE_DIR "/python");
  clientProcess->setProcessEnvironment(env);
  return clientProcess;
}
#endif // ENABLE_ZMQ_TESTS

void ClientServerTest::initTestCase()
{
  QVERIFY2(resetWorkDir(MoleQueue_TESTDATA_DIR "/testworkdir_unix"),
           "Failed to reset working directory for test.");

  // Setup server process
  QVERIFY(setupServerProcess());

  // Start server
  qDebug() << "Starting server:" << m_moleQueueExecutable
           << m_moleQueueDefaultArgs.join(" ");
  m_serverProcess->start(m_moleQueueExecutable, m_moleQueueDefaultArgs);
  QVERIFY(m_serverProcess->waitForStarted(10*1000));
  QTest::qSleep(1 * 1000); // Wait one second for server to start
}

void ClientServerTest::cleanupTestCase()
{
  // send killRpc message
  QProcess *clientProcess = addClientProcess();
  QString clientCommand = MoleQueue_TESTEXEC_DIR "sendRpcKill";
  QStringList clientArguments;
  clientArguments << "-s" << m_socketName;

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");
  clientProcess->start(clientCommand, clientArguments);

  // Wait for client to finish
  QVERIFY2(clientProcess->waitForFinished(300*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);

  // Wait for server to finish
  QVERIFY2(m_serverProcess->waitForFinished(5*1000), "Server timed out.");
  QCOMPARE(m_serverProcess->exitCode(), 0);

  // In case the rpcKill call fails, kill the process
  if (m_serverProcess->state() != QProcess::NotRunning)
    m_serverProcess->kill();
  m_serverProcess->deleteLater();
  m_serverProcess = NULL;

  // Clean up the sendRpcKill client.
  cleanup();
}

void ClientServerTest::init()
{
}

void ClientServerTest::cleanup()
{
  foreach (QProcess *proc, m_clientProcesses) {
    if (proc->state() != QProcess::NotRunning)
      proc->kill();
  }
  qDeleteAll(m_clientProcesses);
  m_clientProcesses.clear();
}

#ifdef ENABLE_ZMQ_TESTS
void ClientServerTest::submitOnePy()
{
  // Setup client process
  QProcess *clientProcess = addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(1);

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");

  clientProcess->start(clientCommand, QStringList(clientArguments));

  // Wait 5 seconds for client to start
  QVERIFY2(clientProcess->waitForStarted(5*1000), "Client did not start.");

  // Wait 10 seconds for client to finish
  QVERIFY2(clientProcess->waitForFinished(10*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);
}

void ClientServerTest::submit200Py()
{
  // Setup client process
  QProcess *clientProcess = addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(200);

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");

  clientProcess->start(clientCommand, QStringList(clientArguments));

  // Wait 5 seconds for client to start
  QVERIFY2(clientProcess->waitForStarted(5*1000), "Client did not start.");

  // Wait one minute for client to finish
  QVERIFY2(clientProcess->waitForFinished(60*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);
}

void ClientServerTest::submit200FromManyClientsPy()
{
  // Setup client processes
  while (m_clientProcesses.size() < m_numClients)
    addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(200);

  qDebug() << "Starting" << m_numClients << "clients:" << clientCommand
           << clientArguments.join(" ");

  int clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    cliProc->start(clientCommand,
                   QStringList(clientArguments)
                   << "-c" << QString::number(++clientId));
  }

  // Wait 5 seconds for each client to start
  clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    ++clientId;
    QVERIFY2(cliProc->waitForStarted(5*1000),
             (QByteArray("Client ") + QByteArray::number(clientId) +
              QByteArray(" failed to start.")).constData());
  }

  // Wait two minutes for all clients to finish
  clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    ++clientId;
    QVERIFY2(cliProc->waitForFinished(2*60*1000),
             (QByteArray("Client ") + QByteArray::number(clientId) +
              QByteArray(" timed out.")).constData());
    QCOMPARE(cliProc->exitCode(), 0);
  }
}
#endif // ENABLE_ZMQ_TESTS
QTEST_MAIN(ClientServerTest)

#include "clientservertest.moc"
<commit_msg>Actually run the zeromq client-server test when possible.<commit_after>/******************************************************************************

  This source file is part of the MoleQueue project.

  Copyright 2012 Kitware, Inc.

  This source code is released under the New BSD License, (the "License").

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

******************************************************************************/

#include <QtTest>

#include "filesystemtools.h"

#include "molequeuetestconfig.h"
#include "testserver.h" // for getRandomSocketName

#include <QtCore/QProcess>
#include <QtCore/QProcessEnvironment>

// Define ENABLE_ZMQ_TESTS if both zeromq and python are available
#ifdef MoleQueue_PYTHON_EXECUTABLE
#ifdef MoleQueue_HAS_ZMQ
#define ENABLE_ZMQ_TESTS
#endif // MoleQueue_HAS_ZMQ
#endif // MoleQueue_PYTHON_EXECUTABLE

class ClientServerTest : public QObject
{
  Q_OBJECT
public:
  ClientServerTest()
    : QObject(NULL),
      m_numClients(10),
      m_workDir(MoleQueue_BINARY_DIR "/testworkdir"),
      m_moleQueueExecutable(MoleQueue_BINARY_DIR "/bin/molequeue"),
      m_serverProcess(NULL)
  {
#ifdef __APPLE__
    m_moleQueueExecutable = MoleQueue_BINARY_DIR "/bin/molequeue.app/Contents/"
        "MacOS/molequeue";
#endif // __APPLE__
    randomizeSocketName();
  }

private:
  int m_numClients;
  QString m_workDir;
  QString m_socketName;
  QString m_moleQueueExecutable;
  QStringList m_moleQueueDefaultArgs;
  QProcess *m_serverProcess;
  QList<QProcess*> m_clientProcesses;

  /// Delete the testing workdir and initialize it with the directory at
  /// @a sourcePath.
  bool resetWorkDir(const QString &sourcePath);

  /// Create a new randomized socket name, stored in m_socketName;
  void randomizeSocketName();

  /// Create the server process (m_serverProcess) and reset
  /// m_moleQueueDefaultArgs to set the workdir, socketname, and enable rpcKill.
  bool setupServerProcess();

  /// Create a Cxx client process
  QProcess *addClientProcess();

#ifdef ENABLE_ZMQ_TESTS
  /// Create a client process initialized for python. The process is returned
  /// and added to m_clientProcesses.
  QProcess *addPythonClientProcess();
#endif // ENABLE_ZMQ_TESTS

private slots:
  /// Called before the first test function is executed.
  void initTestCase();
  /// Called after the last test function is executed.
  void cleanupTestCase();
  /// Called before each test function is executed.
  void init();
  /// Called after every test function.
  void cleanup();

  // Python client tests:
#ifdef ENABLE_ZMQ_TESTS
  void submitOnePy();
  void submit200Py();
  void submit200FromManyClientsPy();
#endif // ENABLE_ZMQ_TESTS
};

bool ClientServerTest::resetWorkDir(const QString &sourcePath)
{
  // Initialize working directory
  if (QFileInfo(m_workDir).exists()) {
    if (!MoleQueue::FileSystemTools::recursiveRemoveDirectory(m_workDir)) {
      qWarning() << "Could not remove old working directory" << m_workDir;
      return false;
    }
  }
  if (!MoleQueue::FileSystemTools::recursiveCopyDirectory(sourcePath,
                                                          m_workDir)) {
    qWarning() << "Could not initialize working directory" << m_workDir
               << "from" << sourcePath;
    return false;
  }
  return true;
}

void ClientServerTest::randomizeSocketName()
{
  m_socketName = TestServer::getRandomSocketName();
}

bool ClientServerTest::setupServerProcess()
{
  m_moleQueueDefaultArgs.clear();
  m_moleQueueDefaultArgs
      << "--workdir" << m_workDir
      << "--socketname" << m_socketName
      << "--rpc-kill";

  if (m_serverProcess) {
    delete m_serverProcess;
    m_serverProcess = NULL;
  }
  m_serverProcess = new QProcess(this);
  m_serverProcess->setProcessChannelMode(QProcess::ForwardedChannels);
  return true;
}

QProcess *ClientServerTest::addClientProcess()
{
  QProcess *clientProcess = new QProcess(this);
  clientProcess->setProcessChannelMode(::QProcess::ForwardedChannels);
  m_clientProcesses.append(clientProcess);
  return clientProcess;
}

#ifdef ENABLE_ZMQ_TESTS
QProcess *ClientServerTest::addPythonClientProcess()
{
  QProcess *clientProcess = addClientProcess();
  QProcessEnvironment env = clientProcess->processEnvironment();
  env.insert("PYTHONPATH", (env.value("PYTHONPATH").isEmpty()
                            ? QString()
                            : env.value("PYTHONPATH") + ':') +
             MoleQueue_SOURCE_DIR "/python");
  clientProcess->setProcessEnvironment(env);
  return clientProcess;
}
#endif // ENABLE_ZMQ_TESTS

void ClientServerTest::initTestCase()
{
  QVERIFY2(resetWorkDir(MoleQueue_TESTDATA_DIR "/testworkdir_unix"),
           "Failed to reset working directory for test.");

  // Setup server process
  QVERIFY(setupServerProcess());

  // Start server
  qDebug() << "Starting server:" << m_moleQueueExecutable
           << m_moleQueueDefaultArgs.join(" ");
  m_serverProcess->start(m_moleQueueExecutable, m_moleQueueDefaultArgs);
  QVERIFY(m_serverProcess->waitForStarted(10*1000));
  QTest::qSleep(1 * 1000); // Wait one second for server to start
}

void ClientServerTest::cleanupTestCase()
{
  // send killRpc message
  QProcess *clientProcess = addClientProcess();
  QString clientCommand = MoleQueue_TESTEXEC_DIR "sendRpcKill";
  QStringList clientArguments;
  clientArguments << "-s" << m_socketName;

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");
  clientProcess->start(clientCommand, clientArguments);

  // Wait for client to finish
  QVERIFY2(clientProcess->waitForFinished(300*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);

  // Wait for server to finish
  QVERIFY2(m_serverProcess->waitForFinished(5*1000), "Server timed out.");
  QCOMPARE(m_serverProcess->exitCode(), 0);

  // In case the rpcKill call fails, kill the process
  if (m_serverProcess->state() != QProcess::NotRunning)
    m_serverProcess->kill();
  m_serverProcess->deleteLater();
  m_serverProcess = NULL;

  // Clean up the sendRpcKill client.
  cleanup();
}

void ClientServerTest::init()
{
}

void ClientServerTest::cleanup()
{
  foreach (QProcess *proc, m_clientProcesses) {
    if (proc->state() != QProcess::NotRunning)
      proc->kill();
  }
  qDeleteAll(m_clientProcesses);
  m_clientProcesses.clear();
}

#ifdef ENABLE_ZMQ_TESTS
void ClientServerTest::submitOnePy()
{
  // Setup client process
  QProcess *clientProcess = addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(1);

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");

  clientProcess->start(clientCommand, QStringList(clientArguments));

  // Wait 5 seconds for client to start
  QVERIFY2(clientProcess->waitForStarted(5*1000), "Client did not start.");

  // Wait 10 seconds for client to finish
  QVERIFY2(clientProcess->waitForFinished(10*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);
}

void ClientServerTest::submit200Py()
{
  // Setup client process
  QProcess *clientProcess = addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(200);

  qDebug() << "Starting client:" << clientCommand
           << clientArguments.join(" ");

  clientProcess->start(clientCommand, QStringList(clientArguments));

  // Wait 5 seconds for client to start
  QVERIFY2(clientProcess->waitForStarted(5*1000), "Client did not start.");

  // Wait one minute for client to finish
  QVERIFY2(clientProcess->waitForFinished(60*1000), "Client timed out.");
  QCOMPARE(clientProcess->exitCode(), 0);
}

void ClientServerTest::submit200FromManyClientsPy()
{
  // Setup client processes
  while (m_clientProcesses.size() < m_numClients)
    addPythonClientProcess();
  QString clientCommand = MoleQueue_PYTHON_EXECUTABLE;
  QStringList clientArguments;
  clientArguments
      << MoleQueue_TESTSCRIPT_DIR "/submitJob.py"
      << "-s" << m_socketName
      << "-n" << QString::number(200);

  qDebug() << "Starting" << m_numClients << "clients:" << clientCommand
           << clientArguments.join(" ");

  int clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    cliProc->start(clientCommand,
                   QStringList(clientArguments)
                   << "-c" << QString::number(++clientId));
  }

  // Wait 5 seconds for each client to start
  clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    ++clientId;
    QVERIFY2(cliProc->waitForStarted(5*1000),
             (QByteArray("Client ") + QByteArray::number(clientId) +
              QByteArray(" failed to start.")).constData());
  }

  // Wait two minutes for all clients to finish
  clientId = 0;
  foreach (QProcess *cliProc, m_clientProcesses) {
    ++clientId;
    QVERIFY2(cliProc->waitForFinished(2*60*1000),
             (QByteArray("Client ") + QByteArray::number(clientId) +
              QByteArray(" timed out.")).constData());
    QCOMPARE(cliProc->exitCode(), 0);
  }
}
#endif // ENABLE_ZMQ_TESTS
QTEST_MAIN(ClientServerTest)

#include "clientservertest.moc"
<|endoftext|>


<commit_before>#include <libpq-fe.h>
#include <node.h>
#include <node_events.h>
#include <assert.h>
#include <stdlib.h>

#define LOG(msg) printf("%s\n",msg)
#define TRACE(msg) //printf("%s\n", msg);

#define THROW(msg) return ThrowException(Exception::Error(String::New(msg)));

using namespace v8;
using namespace node;

static Persistent<String> connect_symbol;
static Persistent<String> error_symbol;
static Persistent<String> ready_symbol;

class Connection : public EventEmitter {

public:

  //creates the V8 objects & attaches them to the module (target)
  static void
  Init (Handle<Object> target)
  {
    HandleScope scope;
    Local<FunctionTemplate> t = FunctionTemplate::New(New);

    t->Inherit(EventEmitter::constructor_template);
    t->InstanceTemplate()->SetInternalFieldCount(1);
    t->SetClassName(String::NewSymbol("Connection"));

    connect_symbol = NODE_PSYMBOL("connect");
    error_symbol = NODE_PSYMBOL("error");
    ready_symbol = NODE_PSYMBOL("readyForQuery");

    NODE_SET_PROTOTYPE_METHOD(t, "connect", Connect);
    NODE_SET_PROTOTYPE_METHOD(t, "_sendQuery", SendQuery);
    NODE_SET_PROTOTYPE_METHOD(t, "end", End);

    target->Set(String::NewSymbol("Connection"), t->GetFunction());
    TRACE("created class");
  }

  //static function called by libev as callback entrypoint
  static void
  io_event(EV_P_ ev_io *w, int revents)
  {
    TRACE("Received IO event");
    Connection *connection = static_cast<Connection*>(w->data);
    connection->HandleIOEvent(revents);
  }

  //v8 entry point into Connection#connect
  static Handle<Value>
  Connect(const Arguments& args)
  {
    HandleScope scope;
    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());
    if(args.Length() == 0 || !args[0]->IsString()) {
      THROW("Must include connection string as only argument to connect");
    }

    String::Utf8Value conninfo(args[0]->ToString());

    self->Connect(*conninfo);

    return Undefined();
  }

  //v8 entry point into Connection#_sendQuery
  static Handle<Value>
  SendQuery(const Arguments& args)
  {
    HandleScope scope;
    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());
    String::Utf8Value queryText(args[0]->ToString());

    int result = self->Send(*queryText);
    if(result == 0) {
      THROW("PQsendQuery returned error code");
    }
    //TODO should we flush before throw?
    self->Flush();
    return Undefined();
  }

  //v8 entry point into Connection#end
  static Handle<Value>
  End(const Arguments& args)
  {
    HandleScope scope;

    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());

    self->End();
    return Undefined();
  }

  ev_io read_watcher_;
  ev_io write_watcher_;
  PGconn *connection_;
  bool connecting_;
  Connection () : EventEmitter ()
  {
    connection_ = NULL;
    connecting_ = false;

    TRACE("Initializing ev watchers");
    ev_init(&read_watcher_, io_event);
    read_watcher_.data = this;
    ev_init(&write_watcher_, io_event);
    write_watcher_.data = this;
  }

  ~Connection ()
  {
  }

protected:

  //v8 entry point to constructor
  static Handle<Value>
  New (const Arguments& args)
  {
    HandleScope scope;
    Connection *connection = new Connection();
    connection->Wrap(args.This());

    return args.This();
  }

  int Send(const char *queryText)
  {
    return PQsendQuery(connection_, queryText);
  }

  //flushes socket
  void Flush()
  {
    if(PQflush(connection_) == 1) {
      TRACE("Flushing");
      ev_io_start(EV_DEFAULT_ &write_watcher_);
    }
  }

  //initializes initial async connection to postgres via libpq
  //and hands off control to libev
  bool Connect(const char* conninfo)
  {
    connection_ = PQconnectStart(conninfo);

    if (!connection_) {
      LOG("Connection couldn't be created");
    } else {
      TRACE("Connect created");
    }

    if (PQsetnonblocking(connection_, 1) == -1) {
      LOG("Unable to set connection to non-blocking");
      PQfinish(connection_);
      connection_ = NULL;
    }

    ConnStatusType status = PQstatus(connection_);

    if(CONNECTION_BAD == status) {
      PQfinish(connection_);
      LOG("Bad connection status");
      connection_ = NULL;
    }

    int fd = PQsocket(connection_);
    if(fd < 0) {
      LOG("socket fd was negative. error");
      return false;
    }

    assert(PQisnonblocking(connection_));

    TRACE("Setting watchers to socket");
    ev_io_set(&read_watcher_, fd, EV_READ);
    ev_io_set(&write_watcher_, fd, EV_WRITE);

    connecting_ = true;
    StartWrite();

    Ref();
    return true;
  }

  //called to process io_events from libev
  void HandleIOEvent(int revents)
  {
    if(revents & EV_ERROR) {
      LOG("Connection error.");
      return;
    }

    if(connecting_) {
      HandleConnectionIO();
      return;
    }

    if(revents & EV_READ) {
      TRACE("revents & EV_READ");
      if(PQconsumeInput(connection_) == 0) {
        LOG("Something happened, consume input is 0");
        return;
      }

      if (PQisBusy(connection_) == 0) {
        PGresult *result;
        while ((result = PQgetResult(connection_))) {
          LOG("Got result");
          //EmitResult(result);
          PQclear(result);
        }
        Emit(ready_symbol, 0, NULL);
      } else {
        LOG("PQisBusy true");
      }

      //TODO look at this later
      PGnotify *notify;
      while ((notify = PQnotifies(connection_))) {
        LOG("Unhandled (not implemented) Notification received....");
        PQfreemem(notify);
      }

    }

    if(revents & EV_WRITE) {
      TRACE("revents & EV_WRITE");
      if (PQflush(connection_) == 0) {
        StopWrite();
      }
    }
  }

  void End()
  {
    PQfinish(connection_);
  }

private:
  void HandleConnectionIO()
  {
    PostgresPollingStatusType status = PQconnectPoll(connection_);
    switch(status) {
    case PGRES_POLLING_READING:
      TRACE("Polled: PGRES_POLLING_READING");
      StopWrite();
      StartRead();
      break;
    case PGRES_POLLING_WRITING:
      TRACE("Polled: PGRES_POLLING_WRITING");
      StopRead();
      StartWrite();
      break;
    case PGRES_POLLING_FAILED:
      StopRead();
      StopWrite();
      TRACE("Polled: PGRES_POLLING_FAILED");
      EmitLastError();
      EmitError("Something happened...polling error");
      break;
    case PGRES_POLLING_OK:
      TRACE("Polled: PGRES_POLLING_OK");
      connecting_ = false;
      StartRead();
      Emit(connect_symbol, 0, NULL);
    default:
      //printf("Unknown polling status: %d\n", status);
      break;
    }
  }

  void EmitError(const char *message)
  {
    Local<Value> exception = Exception::Error(String::New(message));
    Emit(error_symbol, 1, &exception);
  }

  void EmitLastError()
  {
    EmitError(PQerrorMessage(connection_));
  }

  void StopWrite()
  {
    TRACE("Stoping write watcher");
    ev_io_stop(EV_DEFAULT_ &write_watcher_);
  }

  void StartWrite()
  {
    TRACE("Starting write watcher");
    ev_io_start(EV_DEFAULT_ &write_watcher_);
  }

  void StopRead()
  {
    TRACE("Stoping read watcher");
    ev_io_stop(EV_DEFAULT_ &read_watcher_);
  }

  void StartRead()
  {
    TRACE("Starting read watcher");
    ev_io_start(EV_DEFAULT_ &read_watcher_);
  }

};

extern "C" void
init (Handle<Object> target)
{
  HandleScope scope;
  Connection::Init(target);
}
<commit_msg>passed first existing integration test!<commit_after>#include <libpq-fe.h>
#include <node.h>
#include <node_events.h>
#include <assert.h>
#include <stdlib.h>

#define LOG(msg) printf("%s\n",msg)
#define TRACE(msg) //printf("%s\n", msg);

#define THROW(msg) return ThrowException(Exception::Error(String::New(msg)));

using namespace v8;
using namespace node;

static Persistent<String> connect_symbol;
static Persistent<String> error_symbol;
static Persistent<String> ready_symbol;
static Persistent<String> row_symbol;

class Connection : public EventEmitter {

public:

  //creates the V8 objects & attaches them to the module (target)
  static void
  Init (Handle<Object> target)
  {
    HandleScope scope;
    Local<FunctionTemplate> t = FunctionTemplate::New(New);

    t->Inherit(EventEmitter::constructor_template);
    t->InstanceTemplate()->SetInternalFieldCount(1);
    t->SetClassName(String::NewSymbol("Connection"));

    connect_symbol = NODE_PSYMBOL("connect");
    error_symbol = NODE_PSYMBOL("error");
    ready_symbol = NODE_PSYMBOL("readyForQuery");
    row_symbol = NODE_PSYMBOL("row");

    NODE_SET_PROTOTYPE_METHOD(t, "connect", Connect);
    NODE_SET_PROTOTYPE_METHOD(t, "_sendQuery", SendQuery);
    NODE_SET_PROTOTYPE_METHOD(t, "end", End);

    target->Set(String::NewSymbol("Connection"), t->GetFunction());
    TRACE("created class");
  }

  //static function called by libev as callback entrypoint
  static void
  io_event(EV_P_ ev_io *w, int revents)
  {
    TRACE("Received IO event");
    Connection *connection = static_cast<Connection*>(w->data);
    connection->HandleIOEvent(revents);
  }

  //v8 entry point into Connection#connect
  static Handle<Value>
  Connect(const Arguments& args)
  {
    HandleScope scope;
    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());
    if(args.Length() == 0 || !args[0]->IsString()) {
      THROW("Must include connection string as only argument to connect");
    }

    String::Utf8Value conninfo(args[0]->ToString());

    self->Connect(*conninfo);

    return Undefined();
  }

  //v8 entry point into Connection#_sendQuery
  static Handle<Value>
  SendQuery(const Arguments& args)
  {
    HandleScope scope;
    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());
    String::Utf8Value queryText(args[0]->ToString());

    int result = self->Send(*queryText);
    if(result == 0) {
      THROW("PQsendQuery returned error code");
    }
    //TODO should we flush before throw?
    self->Flush();
    return Undefined();
  }

  //v8 entry point into Connection#end
  static Handle<Value>
  End(const Arguments& args)
  {
    HandleScope scope;

    Connection *self = ObjectWrap::Unwrap<Connection>(args.This());

    self->End();
    return Undefined();
  }

  ev_io read_watcher_;
  ev_io write_watcher_;
  PGconn *connection_;
  bool connecting_;
  Connection () : EventEmitter ()
  {
    connection_ = NULL;
    connecting_ = false;

    TRACE("Initializing ev watchers");
    ev_init(&read_watcher_, io_event);
    read_watcher_.data = this;
    ev_init(&write_watcher_, io_event);
    write_watcher_.data = this;
  }

  ~Connection ()
  {
  }

protected:

  //v8 entry point to constructor
  static Handle<Value>
  New (const Arguments& args)
  {
    HandleScope scope;
    Connection *connection = new Connection();
    connection->Wrap(args.This());

    return args.This();
  }

  int Send(const char *queryText)
  {
    return PQsendQuery(connection_, queryText);
  }

  //flushes socket
  void Flush()
  {
    if(PQflush(connection_) == 1) {
      TRACE("Flushing");
      ev_io_start(EV_DEFAULT_ &write_watcher_);
    }
  }

  //initializes initial async connection to postgres via libpq
  //and hands off control to libev
  bool Connect(const char* conninfo)
  {
    connection_ = PQconnectStart(conninfo);

    if (!connection_) {
      LOG("Connection couldn't be created");
    } else {
      TRACE("Connect created");
    }

    if (PQsetnonblocking(connection_, 1) == -1) {
      LOG("Unable to set connection to non-blocking");
      PQfinish(connection_);
      connection_ = NULL;
    }

    ConnStatusType status = PQstatus(connection_);

    if(CONNECTION_BAD == status) {
      PQfinish(connection_);
      LOG("Bad connection status");
      connection_ = NULL;
    }

    int fd = PQsocket(connection_);
    if(fd < 0) {
      LOG("socket fd was negative. error");
      return false;
    }

    assert(PQisnonblocking(connection_));

    TRACE("Setting watchers to socket");
    ev_io_set(&read_watcher_, fd, EV_READ);
    ev_io_set(&write_watcher_, fd, EV_WRITE);

    connecting_ = true;
    StartWrite();

    Ref();
    return true;
  }

  //called to process io_events from libev
  void HandleIOEvent(int revents)
  {
    //declare handlescope as this method is entered via a libev callback
    //and not part of the public v8 interface
    HandleScope scope;

    if(revents & EV_ERROR) {
      LOG("Connection error.");
      return;
    }

    if(connecting_) {
      HandleConnectionIO();
      return;
    }

    if(revents & EV_READ) {
      TRACE("revents & EV_READ");
      if(PQconsumeInput(connection_) == 0) {
        LOG("Something happened, consume input is 0");
        return;
      }

      if (PQisBusy(connection_) == 0) {
        PGresult *result;
        while ((result = PQgetResult(connection_))) {
          int rowCount = PQntuples(result);
          for(int rowNumber = 0; rowNumber < rowCount; rowNumber++) {
            //create result object for this row
            Local<Object> row = Object::New();
            int fieldCount = PQnfields(result);
            for(int fieldNumber = 0; fieldNumber < fieldCount; fieldNumber++) {
              char* fieldName = PQfname(result, fieldNumber);
              char* fieldValue = PQgetvalue(result, rowNumber, fieldNumber);
              row->Set(String::New(fieldName), String::New(fieldValue));
            }

            //not sure about what to dealloc or scope#Close here
            Handle<Value> e = (Handle<Value>)row;
            Emit(row_symbol, 1, &e);
          }
          PQclear(result);
        }
        Emit(ready_symbol, 0, NULL);
      } else {
        LOG("PQisBusy true");
      }

      //TODO look at this later
      PGnotify *notify;
      while ((notify = PQnotifies(connection_))) {
        LOG("Unhandled (not implemented) Notification received....");
        PQfreemem(notify);
      }

    }

    if(revents & EV_WRITE) {
      TRACE("revents & EV_WRITE");
      if (PQflush(connection_) == 0) {
        StopWrite();
      }
    }
  }

  void End()
  {
    StopRead();
    StopWrite();
    PQfinish(connection_);
  }

private:
  void HandleConnectionIO()
  {
    PostgresPollingStatusType status = PQconnectPoll(connection_);
    switch(status) {
    case PGRES_POLLING_READING:
      TRACE("Polled: PGRES_POLLING_READING");
      StopWrite();
      StartRead();
      break;
    case PGRES_POLLING_WRITING:
      TRACE("Polled: PGRES_POLLING_WRITING");
      StopRead();
      StartWrite();
      break;
    case PGRES_POLLING_FAILED:
      StopRead();
      StopWrite();
      TRACE("Polled: PGRES_POLLING_FAILED");
      EmitLastError();
      EmitError("Something happened...polling error");
      break;
    case PGRES_POLLING_OK:
      TRACE("Polled: PGRES_POLLING_OK");
      connecting_ = false;
      StartRead();
      Emit(connect_symbol, 0, NULL);
    default:
      //printf("Unknown polling status: %d\n", status);
      break;
    }
  }

  void EmitError(const char *message)
  {
    Local<Value> exception = Exception::Error(String::New(message));
    Emit(error_symbol, 1, &exception);
  }

  void EmitLastError()
  {
    EmitError(PQerrorMessage(connection_));
  }

  void StopWrite()
  {
    TRACE("Stoping write watcher");
    ev_io_stop(EV_DEFAULT_ &write_watcher_);
  }

  void StartWrite()
  {
    TRACE("Starting write watcher");
    ev_io_start(EV_DEFAULT_ &write_watcher_);
  }

  void StopRead()
  {
    TRACE("Stoping read watcher");
    ev_io_stop(EV_DEFAULT_ &read_watcher_);
  }

  void StartRead()
  {
    TRACE("Starting read watcher");
    ev_io_start(EV_DEFAULT_ &read_watcher_);
  }

};

extern "C" void
init (Handle<Object> target)
{
  HandleScope scope;
  Connection::Init(target);
}
<|endoftext|>