<commit_before>//== BasicObjCFoundationChecks.cpp - Simple Apple-Foundation checks -*- C++ -*--
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines BasicObjCFoundationChecks, a class that encapsulates
//  a set of simple checks to run on Objective-C code using Apple's Foundation
//  classes.
//
//===----------------------------------------------------------------------===//

#include "BasicObjCFoundationChecks.h"

#include "clang/Analysis/PathSensitive/ExplodedGraph.h"
#include "clang/Analysis/PathSensitive/GRExprEngine.h"
#include "clang/Analysis/PathSensitive/GRSimpleAPICheck.h"
#include "clang/Analysis/PathSensitive/ValueState.h"
#include "clang/Analysis/PathSensitive/BugReporter.h"
#include "clang/Analysis/PathDiagnostic.h"
#include "clang/AST/Expr.h"
#include "clang/AST/ASTContext.h"
#include "llvm/Support/Compiler.h"

#include <vector>
#include <sstream>

using namespace clang;

static ObjCInterfaceType* GetReceiverType(ObjCMessageExpr* ME) {
  Expr* Receiver = ME->getReceiver();

  if (!Receiver)
    return NULL;

  assert (Receiver->getType()->isPointerType());

  const PointerType* T = Receiver->getType()->getAsPointerType();

  return dyn_cast<ObjCInterfaceType>(T->getPointeeType().getTypePtr());
}

static const char* GetReceiverNameType(ObjCMessageExpr* ME) {
  ObjCInterfaceType* ReceiverType = GetReceiverType(ME);
  return ReceiverType ? ReceiverType->getDecl()->getIdentifier()->getName()
                      : NULL;
}

namespace {

class VISIBILITY_HIDDEN NilArg : public BugDescription {
  std::string Msg;
  const char* s;
  SourceRange R;
public:
  NilArg(ObjCMessageExpr* ME, unsigned Arg);
  virtual ~NilArg() {}

  virtual const char* getName() const {
    return "nil argument";
  }

  virtual const char* getDescription() const {
    return s;
  }

  virtual void getRanges(const SourceRange*& beg,
                         const SourceRange*& end) const {
    beg = &R;
    end = beg+1;
  }

};

NilArg::NilArg(ObjCMessageExpr* ME, unsigned Arg) : s(NULL) {

  Expr* E = ME->getArg(Arg);
  R = E->getSourceRange();

  std::ostringstream os;

  os << "Argument to '" << GetReceiverNameType(ME) << "' method '"
     << ME->getSelector().getName() << "' cannot be nil.";

  Msg = os.str();
  s = Msg.c_str();
}

class VISIBILITY_HIDDEN BasicObjCFoundationChecks : public GRSimpleAPICheck {

  ASTContext &Ctx;
  ValueStateManager* VMgr;

  typedef std::vector<std::pair<NodeTy*,BugDescription*> > ErrorsTy;
  ErrorsTy Errors;

  RVal GetRVal(ValueState* St, Expr* E) { return VMgr->GetRVal(St, E); }

  bool isNSString(ObjCInterfaceType* T, const char* suffix);
  bool AuditNSString(NodeTy* N, ObjCMessageExpr* ME);

  void Warn(NodeTy* N, Expr* E, const std::string& s);
  void WarnNilArg(NodeTy* N, Expr* E);

  bool CheckNilArg(NodeTy* N, unsigned Arg);

public:
  BasicObjCFoundationChecks(ASTContext& ctx, ValueStateManager* vmgr)
    : Ctx(ctx), VMgr(vmgr) {}

  virtual ~BasicObjCFoundationChecks() {
    for (ErrorsTy::iterator I = Errors.begin(), E = Errors.end(); I!=E; ++I)
      delete I->second;
  }

  virtual bool Audit(ExplodedNode<ValueState>* N);

  virtual void ReportResults(Diagnostic& Diag, PathDiagnosticClient* PD,
                             ASTContext& Ctx, BugReporter& BR,
                             ExplodedGraph<GRExprEngine>& G);

private:

  void AddError(NodeTy* N, BugDescription* D) {
    Errors.push_back(std::make_pair(N, D));
  }

  void WarnNilArg(NodeTy* N, ObjCMessageExpr* ME, unsigned Arg) {
    AddError(N, new NilArg(ME, Arg));
  }
};

} // end anonymous namespace

GRSimpleAPICheck*
clang::CreateBasicObjCFoundationChecks(ASTContext& Ctx,
                                       ValueStateManager* VMgr) {

  return new BasicObjCFoundationChecks(Ctx, VMgr);
}

bool BasicObjCFoundationChecks::Audit(ExplodedNode<ValueState>* N) {

  ObjCMessageExpr* ME =
    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());

  ObjCInterfaceType* ReceiverType = GetReceiverType(ME);

  if (!ReceiverType)
    return NULL;

  const char* name = ReceiverType->getDecl()->getIdentifier()->getName();

  if (!name)
    return false;

  if (name[0] != 'N' || name[1] != 'S')
    return false;

  name += 2;

  // FIXME: Make all of this faster.

  if (isNSString(ReceiverType, name))
    return AuditNSString(N, ME);

  return false;
}

static inline bool isNil(RVal X) {
  return isa<lval::ConcreteInt>(X);
}

//===----------------------------------------------------------------------===//
// Error reporting.
//===----------------------------------------------------------------------===//

void BasicObjCFoundationChecks::ReportResults(Diagnostic& Diag,
                                              PathDiagnosticClient* PD,
                                              ASTContext& Ctx, BugReporter& BR,
                                              ExplodedGraph<GRExprEngine>& G) {

  for (ErrorsTy::iterator I=Errors.begin(), E=Errors.end(); I!=E; ++I)
    BR.EmitPathWarning(Diag, PD, Ctx, *I->second, G, I->first);
}

bool BasicObjCFoundationChecks::CheckNilArg(NodeTy* N, unsigned Arg) {
  ObjCMessageExpr* ME =
    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());

  Expr * E = ME->getArg(Arg);

  if (isNil(GetRVal(N->getState(), E))) {
    WarnNilArg(N, ME, Arg);
    return true;
  }

  return false;
}

//===----------------------------------------------------------------------===//
// NSString checking.
//===----------------------------------------------------------------------===//

bool BasicObjCFoundationChecks::isNSString(ObjCInterfaceType* T,
                                           const char* suffix) {

  return !strcmp("String", suffix) || !strcmp("MutableString", suffix);
}

bool BasicObjCFoundationChecks::AuditNSString(NodeTy* N,
                                              ObjCMessageExpr* ME) {

  Selector S = ME->getSelector();

  if (S.isUnarySelector())
    return false;

  // FIXME: This is going to be really slow doing these checks with
  //  lexical comparisons.

  std::string name = S.getName();
  assert (!name.empty());
  const char* cstr = &name[0];
  unsigned len = name.size();

  switch (len) {
    default:
      break;
    case 8:
      if (!strcmp(cstr, "compare:"))
        return CheckNilArg(N, 0);

      break;

    case 15:
      // FIXME: Checking for initWithFormat: will not work in most cases
      //  yet because [NSString alloc] returns id, not NSString*.  We will
      //  need support for tracking expected-type information in the analyzer
      //  to find these errors.
      if (!strcmp(cstr, "initWithFormat:"))
        return CheckNilArg(N, 0);

      break;

    case 16:
      if (!strcmp(cstr, "compare:options:"))
        return CheckNilArg(N, 0);

      break;

    case 22:
      if (!strcmp(cstr, "compare:options:range:"))
        return CheckNilArg(N, 0);

      break;

    case 23:

      if (!strcmp(cstr, "caseInsensitiveCompare:"))
        return CheckNilArg(N, 0);

      break;

    case 29:
      if (!strcmp(cstr, "compare:options:range:locale:"))
        return CheckNilArg(N, 0);

      break;

    case 37:
    if (!strcmp(cstr, "componentsSeparatedByCharactersInSet:"))
      return CheckNilArg(N, 0);

    break;
  }

  return false;
}
<commit_msg>Added investigate patch for an occasionally failing assertion (heisenbug?)<commit_after>//== BasicObjCFoundationChecks.cpp - Simple Apple-Foundation checks -*- C++ -*--
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//  This file defines BasicObjCFoundationChecks, a class that encapsulates
//  a set of simple checks to run on Objective-C code using Apple's Foundation
//  classes.
//
//===----------------------------------------------------------------------===//

#include "BasicObjCFoundationChecks.h"

#include "clang/Analysis/PathSensitive/ExplodedGraph.h"
#include "clang/Analysis/PathSensitive/GRExprEngine.h"
#include "clang/Analysis/PathSensitive/GRSimpleAPICheck.h"
#include "clang/Analysis/PathSensitive/ValueState.h"
#include "clang/Analysis/PathSensitive/BugReporter.h"
#include "clang/Analysis/PathDiagnostic.h"
#include "clang/AST/Expr.h"
#include "clang/AST/ASTContext.h"
#include "llvm/Support/Compiler.h"

#include <vector>
#include <sstream>

using namespace clang;

static ObjCInterfaceType* GetReceiverType(ObjCMessageExpr* ME) {
  Expr* Receiver = ME->getReceiver();

  if (!Receiver)
    return NULL;

  // FIXME: Cleanup
  QualType X = Receiver->getType();
  Type* TP = X.getTypePtr();
  assert (TP->isPointerType());

  const PointerType* T = TP->getAsPointerType();

  return dyn_cast<ObjCInterfaceType>(T->getPointeeType().getTypePtr());
}

static const char* GetReceiverNameType(ObjCMessageExpr* ME) {
  ObjCInterfaceType* ReceiverType = GetReceiverType(ME);
  return ReceiverType ? ReceiverType->getDecl()->getIdentifier()->getName()
                      : NULL;
}

namespace {

class VISIBILITY_HIDDEN NilArg : public BugDescription {
  std::string Msg;
  const char* s;
  SourceRange R;
public:
  NilArg(ObjCMessageExpr* ME, unsigned Arg);
  virtual ~NilArg() {}

  virtual const char* getName() const {
    return "nil argument";
  }

  virtual const char* getDescription() const {
    return s;
  }

  virtual void getRanges(const SourceRange*& beg,
                         const SourceRange*& end) const {
    beg = &R;
    end = beg+1;
  }

};

NilArg::NilArg(ObjCMessageExpr* ME, unsigned Arg) : s(NULL) {

  Expr* E = ME->getArg(Arg);
  R = E->getSourceRange();

  std::ostringstream os;

  os << "Argument to '" << GetReceiverNameType(ME) << "' method '"
     << ME->getSelector().getName() << "' cannot be nil.";

  Msg = os.str();
  s = Msg.c_str();
}

class VISIBILITY_HIDDEN BasicObjCFoundationChecks : public GRSimpleAPICheck {

  ASTContext &Ctx;
  ValueStateManager* VMgr;

  typedef std::vector<std::pair<NodeTy*,BugDescription*> > ErrorsTy;
  ErrorsTy Errors;

  RVal GetRVal(ValueState* St, Expr* E) { return VMgr->GetRVal(St, E); }

  bool isNSString(ObjCInterfaceType* T, const char* suffix);
  bool AuditNSString(NodeTy* N, ObjCMessageExpr* ME);

  void Warn(NodeTy* N, Expr* E, const std::string& s);
  void WarnNilArg(NodeTy* N, Expr* E);

  bool CheckNilArg(NodeTy* N, unsigned Arg);

public:
  BasicObjCFoundationChecks(ASTContext& ctx, ValueStateManager* vmgr)
    : Ctx(ctx), VMgr(vmgr) {}

  virtual ~BasicObjCFoundationChecks() {
    for (ErrorsTy::iterator I = Errors.begin(), E = Errors.end(); I!=E; ++I)
      delete I->second;
  }

  virtual bool Audit(ExplodedNode<ValueState>* N);

  virtual void ReportResults(Diagnostic& Diag, PathDiagnosticClient* PD,
                             ASTContext& Ctx, BugReporter& BR,
                             ExplodedGraph<GRExprEngine>& G);

private:

  void AddError(NodeTy* N, BugDescription* D) {
    Errors.push_back(std::make_pair(N, D));
  }

  void WarnNilArg(NodeTy* N, ObjCMessageExpr* ME, unsigned Arg) {
    AddError(N, new NilArg(ME, Arg));
  }
};

} // end anonymous namespace

GRSimpleAPICheck*
clang::CreateBasicObjCFoundationChecks(ASTContext& Ctx,
                                       ValueStateManager* VMgr) {

  return new BasicObjCFoundationChecks(Ctx, VMgr);
}

bool BasicObjCFoundationChecks::Audit(ExplodedNode<ValueState>* N) {

  ObjCMessageExpr* ME =
    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());

  ObjCInterfaceType* ReceiverType = GetReceiverType(ME);

  if (!ReceiverType)
    return NULL;

  const char* name = ReceiverType->getDecl()->getIdentifier()->getName();

  if (!name)
    return false;

  if (name[0] != 'N' || name[1] != 'S')
    return false;

  name += 2;

  // FIXME: Make all of this faster.

  if (isNSString(ReceiverType, name))
    return AuditNSString(N, ME);

  return false;
}

static inline bool isNil(RVal X) {
  return isa<lval::ConcreteInt>(X);
}

//===----------------------------------------------------------------------===//
// Error reporting.
//===----------------------------------------------------------------------===//

void BasicObjCFoundationChecks::ReportResults(Diagnostic& Diag,
                                              PathDiagnosticClient* PD,
                                              ASTContext& Ctx, BugReporter& BR,
                                              ExplodedGraph<GRExprEngine>& G) {

  for (ErrorsTy::iterator I=Errors.begin(), E=Errors.end(); I!=E; ++I)
    BR.EmitPathWarning(Diag, PD, Ctx, *I->second, G, I->first);
}

bool BasicObjCFoundationChecks::CheckNilArg(NodeTy* N, unsigned Arg) {
  ObjCMessageExpr* ME =
    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());

  Expr * E = ME->getArg(Arg);

  if (isNil(GetRVal(N->getState(), E))) {
    WarnNilArg(N, ME, Arg);
    return true;
  }

  return false;
}

//===----------------------------------------------------------------------===//
// NSString checking.
//===----------------------------------------------------------------------===//

bool BasicObjCFoundationChecks::isNSString(ObjCInterfaceType* T,
                                           const char* suffix) {

  return !strcmp("String", suffix) || !strcmp("MutableString", suffix);
}

bool BasicObjCFoundationChecks::AuditNSString(NodeTy* N,
                                              ObjCMessageExpr* ME) {

  Selector S = ME->getSelector();

  if (S.isUnarySelector())
    return false;

  // FIXME: This is going to be really slow doing these checks with
  //  lexical comparisons.

  std::string name = S.getName();
  assert (!name.empty());
  const char* cstr = &name[0];
  unsigned len = name.size();

  switch (len) {
    default:
      break;
    case 8:
      if (!strcmp(cstr, "compare:"))
        return CheckNilArg(N, 0);

      break;

    case 15:
      // FIXME: Checking for initWithFormat: will not work in most cases
      //  yet because [NSString alloc] returns id, not NSString*.  We will
      //  need support for tracking expected-type information in the analyzer
      //  to find these errors.
      if (!strcmp(cstr, "initWithFormat:"))
        return CheckNilArg(N, 0);

      break;

    case 16:
      if (!strcmp(cstr, "compare:options:"))
        return CheckNilArg(N, 0);

      break;

    case 22:
      if (!strcmp(cstr, "compare:options:range:"))
        return CheckNilArg(N, 0);

      break;

    case 23:

      if (!strcmp(cstr, "caseInsensitiveCompare:"))
        return CheckNilArg(N, 0);

      break;

    case 29:
      if (!strcmp(cstr, "compare:options:range:locale:"))
        return CheckNilArg(N, 0);

      break;

    case 37:
    if (!strcmp(cstr, "componentsSeparatedByCharactersInSet:"))
      return CheckNilArg(N, 0);

    break;
  }

  return false;
}
<|endoftext|>


<commit_before>#include <iostream>

#include "stress-application.h"

#include "vtrc-rpc-lowlevel.pb.h"

#include "vtrc-common/vtrc-pool-pair.h"
#include "vtrc-common/vtrc-rpc-service-wrapper.h"
#include "vtrc-common/vtrc-connection-iface.h"
#include "vtrc-common/vtrc-delayed-call.h"

#include "vtrc-common/vtrc-connection-list.h"

#include "vtrc-server/vtrc-application.h"
#include "vtrc-server/vtrc-listener-tcp.h"
#include "vtrc-server/vtrc-listener-local.h"
#include "vtrc-server/vtrc-listener.h"

#include "stress-service-impl.h"

#include "boost/program_options.hpp"
#include "boost/algorithm/string.hpp"

#include "vtrc-bind.h"
#include "vtrc-ref.h"
#include "vtrc-mutex.h"
#include "vtrc-atomic.h"
#include "vtrc-thread.h"

#include "boost/asio.hpp"
#include "boost/asio/error.hpp"

#include "google/protobuf/service.h"

#include "vtrc-common/vtrc-delayed-call.h"
#include "vtrc-common/vtrc-thread-pool.h"
#include "vtrc-common/vtrc-mutex-typedefs.h"
#include "vtrc-common/vtrc-closure.h"

namespace stress {

namespace {
void precall_closure( vtrc::common::connection_iface &,
                      const google::protobuf::MethodDescriptor *m,
                      vtrc::rpc::lowlevel_unit & )
{
    std::cout << "Precall " << m->full_name( ) << "\n";
}

void postcall_closure( vtrc::common::connection_iface &,
                       vtrc::rpc::lowlevel_unit & )
{
    std::cout << "Postcall " << "\n";
}

}

    using namespace vtrc;
    namespace po = boost::program_options;
    namespace gpb = google::protobuf;
    namespace basio = boost::asio;
    namespace bsys  = boost::system;

    typedef vtrc::common::delayed_call            delayed_call;
    typedef vtrc::shared_ptr<delayed_call>        delayed_call_sptr;

    typedef std::pair<
            delayed_call_sptr,
            application::timer_callback > call_cb_pair;

    typedef std::map<unsigned, call_cb_pair> delayed_map;

    namespace {

        server::listener_sptr create_from_string( const std::string &name,
                                  server::application &app,
                                  const rpc::session_options &opts,
                                  bool tcp_nodelay)
        {
            /// result endpoint
            server::listener_sptr result;

            std::vector<std::string> params;

            size_t delim_pos = name.find_last_of( ':' );
            if( delim_pos == std::string::npos ) {

                /// local: <localname>
                params.push_back( name );

            } else {

                /// tcp: <addres>:<port>
                params.push_back( std::string( name.begin( ),
                                               name.begin( ) + delim_pos ) );

                params.push_back( std::string( name.begin( ) + delim_pos + 1,
                                               name.end( ) ) );
            }

            if( params.size( ) == 1 ) {         /// local endpoint
#ifndef _WIN32
                ::unlink( params[0].c_str( ) ); /// unlink old file socket
#endif
                result = server::listeners::local::create(app, opts, params[0]);
//                result->set_precall( &precall_closure );
//                result->set_postcall( &postcall_closure );

            } else if( params.size( ) == 2 ) {  /// TCP

                result = server::listeners::tcp::create( app, opts,
                                params[0],
                                boost::lexical_cast<unsigned short>(params[1]),
                                tcp_nodelay);

            }
            return result;
        }

        class app_impl: public server::application {

        public:

            stress::application *parent_app_;

            app_impl( common::pool_pair &pp )
                :server::application(pp)
            { }

            vtrc::shared_ptr<common::rpc_service_wrapper>
                     get_service_by_name( common::connection_iface* conn,
                                          const std::string &service_name )
            {
                if( service_name == stress::service_name( ) ) {

                    vtrc::shared_ptr<gpb::Service> stress_serv
                            ( stress::create_service( conn, *parent_app_ ) );

                    return vtrc::shared_ptr<common::rpc_service_wrapper>(
                                vtrc::make_shared<common::rpc_service_wrapper>(
                                                                stress_serv ) );
                }
                return vtrc::shared_ptr<common::rpc_service_wrapper>( );
            }

        };

    }

    struct application::impl {

        common::pool_pair                   pp_;
        app_impl                            app_;
        std::vector<server::listener_sptr>  listeners_;

        vtrc::atomic<size_t>                counter_;
        common::delayed_call                retry_timer_;

        unsigned                            accept_errors_;
        unsigned                            max_clients_;
        vtrc::mutex                         counter_lock_;

        vtrc::atomic<unsigned>              next_id_;
        common::thread_pool                 timer_pool_;
        delayed_map                         map_;
        vtrc::shared_mutex                  map_lock_;

        impl( unsigned io_threads )
            :pp_(io_threads)
            ,app_(pp_)
            ,counter_(0)
            ,retry_timer_(pp_.get_io_service( ))
            ,accept_errors_(0)
            ,max_clients_(1000)
            ,next_id_(0)
            ,timer_pool_(1)
        { }

        impl( unsigned io_threads, unsigned rpc_threads )
            :pp_(io_threads, rpc_threads)
            ,app_(pp_)
            ,counter_(0)
            ,retry_timer_(pp_.get_io_service( ))
            ,max_clients_(1000)
            ,next_id_(0)
            ,timer_pool_(1)
        { }

        unsigned next_id( )
        {
            return ++next_id_;
        }

        void timer_handler( const bsys::error_code &err,
                            unsigned id,  unsigned microsec,
                            call_cb_pair tc )
        {
            unsigned err_code = err.value( );
            if( tc.second( id, err_code )) {
                start_timer( tc, id, microsec );
            }
        }

        void start_timer( call_cb_pair &tc, unsigned id,  unsigned microsec )
        {
            tc.first->call_from_now(
                        vtrc::bind( &impl::timer_handler, this,
                                    vtrc::placeholders::error,
                                    id, microsec, tc ),
                        common::timer::microseconds( microsec ) );
        }

        void on_new_connection( server::listener *l,
                                const common::connection_iface *c )
        {
            std::cout << "New connection: "
                      << "\n\tep:     " << l->name( )
                      << "\n\tclient: " << c->name( )
                      << "\n\ttotal:  " << ++counter_
                      << "\n"
                        ;
        }

        void on_stop_connection( server::listener *l,
                                 const common::connection_iface *c )
        {
            std::cout << "Close connection: "
                      << c->name( )
                      << "; count: " << --counter_
                      << "\n";
        }

        void start_retry_accept( server::listener_sptr l, unsigned rto )
        {
            retry_timer_.call_from_now(
                vtrc::bind( &impl::retry_timer_handler, this,
                            l, rto, vtrc::placeholders::_1 ),
                            common::timer::milliseconds( rto ));
        }

        void retry_timer_handler( server::listener_sptr l, unsigned retry_to,
                                  const boost::system::error_code &code)
        {
            if( !code ) {
                std::cout << "Restarting " << l->name( ) << "...";
                try {
                    l->start( );
                    std::cout << "Ok;\n";
                } catch( const std::exception &ex ) {
                    std::cout << "failed; " << ex.what( )
                              << "; Retrying...\n";
                    start_retry_accept( l, retry_to );
                };
            }
        }

        void on_accept_failed( server::listener *l,
                               unsigned retry_to,
                               const boost::system::error_code &code )
        {
            std::cout << "Accept failed at " << l->name( )
                      << " due to '" << code.message( ) << "'\n";
            start_retry_accept( l->shared_from_this( ), retry_to );
        }

        rpc::session_options options( const po::variables_map &params )
        {
            using server::listeners::default_options;
            rpc::session_options opts( default_options( ) );

            if( params.count( "message-size" ) ) {
                opts.set_max_message_length(
                            params["message-size"].as<unsigned>( ));
            }

            if( params.count( "stack-size" ) ) {
                opts.set_max_stack_size(
                            params["stack-size"].as<unsigned>( ));
            }

            if( params.count( "read-size" ) ) {
                opts.set_read_buffer_size(
                            params["read-size"].as<unsigned>( ));
            }

            if( params.count( "max-calls" ) ) {
                opts.set_max_active_calls(
                            params["max-calls"].as<unsigned>( ));
            }

            return opts;
        }

        void run( const po::variables_map &params )
        {

            typedef std::vector<std::string> string_vector;
            typedef string_vector::const_iterator vec_citer;

            string_vector ser = params["server"].as<string_vector>( );

            bool use_only_pool = !!params.count( "only-pool" );
            bool tcp_nodelay   = !!params.count( "tcp-nodelay" );

            using server::listeners::default_options;

            rpc::session_options opts( options( params ) );

            unsigned retry_to = (params.count( "accept-retry" ) != 0)
                    ? params["accept-retry"].as<unsigned>( )
                    : 1000;

            if( params.count( "max-clients" ) ) {
                max_clients_ = params["max-clients"].as<unsigned>( );
            }

            for( vec_citer b(ser.begin( )), e(ser.end( )); b != e; ++b ) {

                std::cout << "Starting listener at '" <<  *b << "'...";
                attach_start_listener( retry_to,
                    create_from_string( *b, app_, opts, tcp_nodelay) );
                std::cout << "Ok\n";

            }

            if( use_only_pool ) {
                pp_.get_io_pool( ).attach( );
            } else {
                pp_.get_rpc_pool( ).attach( );
            }
            pp_.join_all( );
        }

        void attach_start_listener( unsigned retry_to,
                                    vtrc::server::listener_sptr listen )
        {
            listen->on_new_connection_connect(
                   vtrc::bind( &impl::on_new_connection, this,
                               listen.get( ), vtrc::placeholders::_1 ));

            listen->on_stop_connection_connect(
                   vtrc::bind( &impl::on_stop_connection, this,
                               listen.get( ), vtrc::placeholders::_1 ));

            listen->on_accept_failed_connect(
                   vtrc::bind( &impl::on_accept_failed, this,
                               listen.get( ), retry_to,
                               vtrc::placeholders::_1 ) );

            listeners_.push_back( listen );

            listen->start( );

        }

        void stop( )
        {
            std::cout << "Stopping server ...";

            timer_pool_.stop( );
            timer_pool_.join_all( );

            typedef std::vector<server::listener_sptr>::const_iterator citer;
            for( citer b(listeners_.begin( )), e(listeners_.end( )); b!=e; ++b){
                (*b)->stop( );
            }
            retry_timer_.cancel( );
            pp_.stop_all( );
            std::cout << "Ok\n";
        }

    };

    application::application( unsigned io_threads )
        :impl_(new impl(io_threads))
    {
        impl_->app_.parent_app_ = this;
    }

    application::application( unsigned io_threads, unsigned rpc_threads )
        :impl_(new impl(io_threads, rpc_threads))
    {
        impl_->app_.parent_app_ = this;
    }

    application::~application( )
    {
        delete impl_;
    }

    server::application &application::get_application( )
    {
        return impl_->app_;
    }

    const server::application &application::get_application( ) const
    {
        return impl_->app_;
    }

    void application::run( const po::variables_map &params )
    {
        impl_->run( params );
    }

    void application::stop( )
    {
        impl_->stop( );
    }

    unsigned application::add_timer_event( unsigned microseconds,
                                           timer_callback cb )
    {
        unsigned id = impl_->next_id( );
        delayed_call_sptr md(
                    new delayed_call( impl_->timer_pool_.get_io_service( )) );
        call_cb_pair p( std::make_pair( md, cb ) );
        impl_->start_timer( p, id, microseconds );

        vtrc::unique_shared_lock usl(impl_->map_lock_);
        impl_->map_.insert( std::make_pair( id, p ) );

        std::cout << "Add timer " << id << " success\n";

        return id;
    }

    void application::del_timer_event( unsigned id )
    {
        vtrc::shared_lock usl(impl_->map_lock_);
        delayed_map::iterator f( impl_->map_.find( id ));
        if( f != impl_->map_.end( ) ) {
            f->second.first->cancel( );
            impl_->map_.erase( f );
        }
    }

}

<commit_msg>precall postcall server<commit_after>#include <iostream>

#include "stress-application.h"

#include "vtrc-rpc-lowlevel.pb.h"

#include "vtrc-common/vtrc-pool-pair.h"
#include "vtrc-common/vtrc-rpc-service-wrapper.h"
#include "vtrc-common/vtrc-connection-iface.h"
#include "vtrc-common/vtrc-delayed-call.h"

#include "vtrc-common/vtrc-connection-list.h"

#include "vtrc-server/vtrc-application.h"
#include "vtrc-server/vtrc-listener-tcp.h"
#include "vtrc-server/vtrc-listener-local.h"
#include "vtrc-server/vtrc-listener.h"

#include "stress-service-impl.h"

#include "boost/program_options.hpp"
#include "boost/algorithm/string.hpp"

#include "vtrc-bind.h"
#include "vtrc-ref.h"
#include "vtrc-mutex.h"
#include "vtrc-atomic.h"
#include "vtrc-thread.h"

#include "boost/asio.hpp"
#include "boost/asio/error.hpp"

#include "google/protobuf/service.h"

#include "vtrc-common/vtrc-delayed-call.h"
#include "vtrc-common/vtrc-thread-pool.h"
#include "vtrc-common/vtrc-mutex-typedefs.h"
#include "vtrc-common/vtrc-closure.h"

namespace stress {

namespace {

    void precall_closure( vtrc::common::connection_iface &,
                          const google::protobuf::MethodDescriptor *m,
                          vtrc::rpc::lowlevel_unit & )
    {
        std::cout << "Precall " << m->full_name( ) << "\n";
    }

    void postcall_closure( vtrc::common::connection_iface &,
                           vtrc::rpc::lowlevel_unit & )
    {
        std::cout << "Postcall " << "\n";
    }

}

    using namespace vtrc;
    namespace po = boost::program_options;
    namespace gpb = google::protobuf;
    namespace basio = boost::asio;
    namespace bsys  = boost::system;

    typedef vtrc::common::delayed_call            delayed_call;
    typedef vtrc::shared_ptr<delayed_call>        delayed_call_sptr;

    typedef std::pair<
            delayed_call_sptr,
            application::timer_callback > call_cb_pair;

    typedef std::map<unsigned, call_cb_pair> delayed_map;

    namespace {

        server::listener_sptr create_from_string( const std::string &name,
                                  server::application &app,
                                  const rpc::session_options &opts,
                                  bool tcp_nodelay)
        {
            /// result endpoint
            server::listener_sptr result;

            std::vector<std::string> params;

            size_t delim_pos = name.find_last_of( ':' );
            if( delim_pos == std::string::npos ) {

                /// local: <localname>
                params.push_back( name );

            } else {

                /// tcp: <addres>:<port>
                params.push_back( std::string( name.begin( ),
                                               name.begin( ) + delim_pos ) );

                params.push_back( std::string( name.begin( ) + delim_pos + 1,
                                               name.end( ) ) );
            }

            if( params.size( ) == 1 ) {         /// local endpoint
#ifndef _WIN32
                ::unlink( params[0].c_str( ) ); /// unlink old file socket
#endif
                result = server::listeners::local::create(app, opts, params[0]);
//                result->set_precall( &precall_closure );
//                result->set_postcall( &postcall_closure );

            } else if( params.size( ) == 2 ) {  /// TCP

                result = server::listeners::tcp::create( app, opts,
                                params[0],
                                boost::lexical_cast<unsigned short>(params[1]),
                                tcp_nodelay);

            }
            return result;
        }

        class app_impl: public server::application {

        public:

            stress::application *parent_app_;

            app_impl( common::pool_pair &pp )
                :server::application(pp)
            { }

            vtrc::shared_ptr<common::rpc_service_wrapper>
                     get_service_by_name( common::connection_iface* conn,
                                          const std::string &service_name )
            {
                if( service_name == stress::service_name( ) ) {

                    vtrc::shared_ptr<gpb::Service> stress_serv
                            ( stress::create_service( conn, *parent_app_ ) );

                    return vtrc::shared_ptr<common::rpc_service_wrapper>(
                                vtrc::make_shared<common::rpc_service_wrapper>(
                                                                stress_serv ) );
                }
                return vtrc::shared_ptr<common::rpc_service_wrapper>( );
            }

        };

    }

    struct application::impl {

        common::pool_pair                   pp_;
        app_impl                            app_;
        std::vector<server::listener_sptr>  listeners_;

        vtrc::atomic<size_t>                counter_;
        common::delayed_call                retry_timer_;

        unsigned                            accept_errors_;
        unsigned                            max_clients_;
        vtrc::mutex                         counter_lock_;

        vtrc::atomic<unsigned>              next_id_;
        common::thread_pool                 timer_pool_;
        delayed_map                         map_;
        vtrc::shared_mutex                  map_lock_;

        impl( unsigned io_threads )
            :pp_(io_threads)
            ,app_(pp_)
            ,counter_(0)
            ,retry_timer_(pp_.get_io_service( ))
            ,accept_errors_(0)
            ,max_clients_(1000)
            ,next_id_(0)
            ,timer_pool_(1)
        { }

        impl( unsigned io_threads, unsigned rpc_threads )
            :pp_(io_threads, rpc_threads)
            ,app_(pp_)
            ,counter_(0)
            ,retry_timer_(pp_.get_io_service( ))
            ,max_clients_(1000)
            ,next_id_(0)
            ,timer_pool_(1)
        { }

        unsigned next_id( )
        {
            return ++next_id_;
        }

        void timer_handler( const bsys::error_code &err,
                            unsigned id,  unsigned microsec,
                            call_cb_pair tc )
        {
            unsigned err_code = err.value( );
            if( tc.second( id, err_code )) {
                start_timer( tc, id, microsec );
            }
        }

        void start_timer( call_cb_pair &tc, unsigned id,  unsigned microsec )
        {
            tc.first->call_from_now(
                        vtrc::bind( &impl::timer_handler, this,
                                    vtrc::placeholders::error,
                                    id, microsec, tc ),
                        common::timer::microseconds( microsec ) );
        }

        void on_new_connection( server::listener *l,
                                const common::connection_iface *c )
        {
            std::cout << "New connection: "
                      << "\n\tep:     " << l->name( )
                      << "\n\tclient: " << c->name( )
                      << "\n\ttotal:  " << ++counter_
                      << "\n"
                        ;
        }

        void on_stop_connection( server::listener *l,
                                 const common::connection_iface *c )
        {
            std::cout << "Close connection: "
                      << c->name( )
                      << "; count: " << --counter_
                      << "\n";
        }

        void start_retry_accept( server::listener_sptr l, unsigned rto )
        {
            retry_timer_.call_from_now(
                vtrc::bind( &impl::retry_timer_handler, this,
                            l, rto, vtrc::placeholders::_1 ),
                            common::timer::milliseconds( rto ));
        }

        void retry_timer_handler( server::listener_sptr l, unsigned retry_to,
                                  const boost::system::error_code &code)
        {
            if( !code ) {
                std::cout << "Restarting " << l->name( ) << "...";
                try {
                    l->start( );
                    std::cout << "Ok;\n";
                } catch( const std::exception &ex ) {
                    std::cout << "failed; " << ex.what( )
                              << "; Retrying...\n";
                    start_retry_accept( l, retry_to );
                };
            }
        }

        void on_accept_failed( server::listener *l,
                               unsigned retry_to,
                               const boost::system::error_code &code )
        {
            std::cout << "Accept failed at " << l->name( )
                      << " due to '" << code.message( ) << "'\n";
            start_retry_accept( l->shared_from_this( ), retry_to );
        }

        rpc::session_options options( const po::variables_map &params )
        {
            using server::listeners::default_options;
            rpc::session_options opts( default_options( ) );

            if( params.count( "message-size" ) ) {
                opts.set_max_message_length(
                            params["message-size"].as<unsigned>( ));
            }

            if( params.count( "stack-size" ) ) {
                opts.set_max_stack_size(
                            params["stack-size"].as<unsigned>( ));
            }

            if( params.count( "read-size" ) ) {
                opts.set_read_buffer_size(
                            params["read-size"].as<unsigned>( ));
            }

            if( params.count( "max-calls" ) ) {
                opts.set_max_active_calls(
                            params["max-calls"].as<unsigned>( ));
            }

            return opts;
        }

        void run( const po::variables_map &params )
        {

            typedef std::vector<std::string> string_vector;
            typedef string_vector::const_iterator vec_citer;

            string_vector ser = params["server"].as<string_vector>( );

            bool use_only_pool = !!params.count( "only-pool" );
            bool tcp_nodelay   = !!params.count( "tcp-nodelay" );

            using server::listeners::default_options;

            rpc::session_options opts( options( params ) );

            unsigned retry_to = (params.count( "accept-retry" ) != 0)
                    ? params["accept-retry"].as<unsigned>( )
                    : 1000;

            if( params.count( "max-clients" ) ) {
                max_clients_ = params["max-clients"].as<unsigned>( );
            }

            for( vec_citer b(ser.begin( )), e(ser.end( )); b != e; ++b ) {

                std::cout << "Starting listener at '" <<  *b << "'...";
                attach_start_listener( retry_to,
                    create_from_string( *b, app_, opts, tcp_nodelay) );
                std::cout << "Ok\n";

            }

            if( use_only_pool ) {
                pp_.get_io_pool( ).attach( );
            } else {
                pp_.get_rpc_pool( ).attach( );
            }
            pp_.join_all( );
        }

        void attach_start_listener( unsigned retry_to,
                                    vtrc::server::listener_sptr listen )
        {
            listen->on_new_connection_connect(
                   vtrc::bind( &impl::on_new_connection, this,
                               listen.get( ), vtrc::placeholders::_1 ));

            listen->on_stop_connection_connect(
                   vtrc::bind( &impl::on_stop_connection, this,
                               listen.get( ), vtrc::placeholders::_1 ));

            listen->on_accept_failed_connect(
                   vtrc::bind( &impl::on_accept_failed, this,
                               listen.get( ), retry_to,
                               vtrc::placeholders::_1 ) );

            listeners_.push_back( listen );

            listen->start( );

        }

        void stop( )
        {
            std::cout << "Stopping server ...";

            timer_pool_.stop( );
            timer_pool_.join_all( );

            typedef std::vector<server::listener_sptr>::const_iterator citer;
            for( citer b(listeners_.begin( )), e(listeners_.end( )); b!=e; ++b){
                (*b)->stop( );
            }
            retry_timer_.cancel( );
            pp_.stop_all( );
            std::cout << "Ok\n";
        }

    };

    application::application( unsigned io_threads )
        :impl_(new impl(io_threads))
    {
        impl_->app_.parent_app_ = this;
    }

    application::application( unsigned io_threads, unsigned rpc_threads )
        :impl_(new impl(io_threads, rpc_threads))
    {
        impl_->app_.parent_app_ = this;
    }

    application::~application( )
    {
        delete impl_;
    }

    server::application &application::get_application( )
    {
        return impl_->app_;
    }

    const server::application &application::get_application( ) const
    {
        return impl_->app_;
    }

    void application::run( const po::variables_map &params )
    {
        impl_->run( params );
    }

    void application::stop( )
    {
        impl_->stop( );
    }

    unsigned application::add_timer_event( unsigned microseconds,
                                           timer_callback cb )
    {
        unsigned id = impl_->next_id( );
        delayed_call_sptr md(
                    new delayed_call( impl_->timer_pool_.get_io_service( )) );
        call_cb_pair p( std::make_pair( md, cb ) );
        impl_->start_timer( p, id, microseconds );

        vtrc::unique_shared_lock usl(impl_->map_lock_);
        impl_->map_.insert( std::make_pair( id, p ) );

        std::cout << "Add timer " << id << " success\n";

        return id;
    }

    void application::del_timer_event( unsigned id )
    {
        vtrc::shared_lock usl(impl_->map_lock_);
        delayed_map::iterator f( impl_->map_.find( id ));
        if( f != impl_->map_.end( ) ) {
            f->second.first->cancel( );
            impl_->map_.erase( f );
        }
    }

}

<|endoftext|>


<commit_before>//===--- extra/modularize/ModularizeUtilities.cpp -------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements a class for loading and validating a module map or
// header list by checking that all headers in the corresponding directories
// are accounted for.
//
//===----------------------------------------------------------------------===//

#include "clang/Basic/SourceManager.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendActions.h"
#include "CoverageChecker.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "ModularizeUtilities.h"

using namespace clang;
using namespace llvm;
using namespace Modularize;

// Subclass TargetOptions so we can construct it inline with
// the minimal option, the triple.
class ModuleMapTargetOptions : public clang::TargetOptions {
public:
  ModuleMapTargetOptions() { Triple = llvm::sys::getDefaultTargetTriple(); }
};

// ModularizeUtilities class implementation.

// Constructor.
ModularizeUtilities::ModularizeUtilities(std::vector<std::string> &InputPaths,
                                         llvm::StringRef Prefix)
  : InputFilePaths(InputPaths),
    HeaderPrefix(Prefix),
    HasModuleMap(false),
    // Init clang stuff needed for loading the module map and preprocessing.
    LangOpts(new LangOptions()), DiagIDs(new DiagnosticIDs()),
    DiagnosticOpts(new DiagnosticOptions()),
    DC(llvm::errs(), DiagnosticOpts.get()),
    Diagnostics(
    new DiagnosticsEngine(DiagIDs, DiagnosticOpts.get(), &DC, false)),
    TargetOpts(new ModuleMapTargetOptions()),
    Target(TargetInfo::CreateTargetInfo(*Diagnostics, TargetOpts)),
    FileMgr(new FileManager(FileSystemOpts)),
    SourceMgr(new SourceManager(*Diagnostics, *FileMgr, false)),
    HeaderSearchOpts(new HeaderSearchOptions()),
    HeaderInfo(new HeaderSearch(HeaderSearchOpts, *SourceMgr, *Diagnostics,
    *LangOpts, Target.get())) {
}

// Create instance of ModularizeUtilities, to simplify setting up
// subordinate objects.
ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
    std::vector<std::string> &InputPaths, llvm::StringRef Prefix) {

  return new ModularizeUtilities(InputPaths, Prefix);
}

// Load all header lists and dependencies.
std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
  typedef std::vector<std::string>::iterator Iter;
  // For each input file.
  for (Iter I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
    llvm::StringRef InputPath = *I;
    // If it's a module map.
    if (InputPath.endswith(".modulemap")) {
      // Load the module map.
      if (std::error_code EC = loadModuleMap(InputPath))
        return EC;
    }
    else {
      // Else we assume it's a header list and load it.
      if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
        errs() << "modularize: error: Unable to get header list '" << InputPath
          << "': " << EC.message() << '\n';
        return EC;
      }
    }
  }
  return std::error_code();
}

// Do coverage checks.
// For each loaded module map, do header coverage check.
// Starting from the directory of the module.map file,
// Find all header files, optionally looking only at files
// covered by the include path options, and compare against
// the headers referenced by the module.map file.
// Display warnings for unaccounted-for header files.
// Returns 0 if there were no errors or warnings, 1 if there
// were warnings, 2 if any other problem, such as a bad
// module map path argument was specified.
std::error_code ModularizeUtilities::doCoverageCheck(
    std::vector<std::string> &IncludePaths,
    llvm::ArrayRef<std::string> CommandLine) {
  int ModuleMapCount = ModuleMaps.size();
  int ModuleMapIndex;
  std::error_code EC;
  for (ModuleMapIndex = 0; ModuleMapIndex < ModuleMapCount; ++ModuleMapIndex) {
    std::unique_ptr<clang::ModuleMap> &ModMap = ModuleMaps[ModuleMapIndex];
    CoverageChecker *Checker = CoverageChecker::createCoverageChecker(
      InputFilePaths[ModuleMapIndex], IncludePaths, CommandLine, ModMap.get());
    std::error_code LocalEC = Checker->doChecks();
    if (LocalEC.value() > 0)
      EC = LocalEC;
  }
  return EC;
}

// Load single header list and dependencies.
std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies(
    llvm::StringRef InputPath) {

  // By default, use the path component of the list file name.
  SmallString<256> HeaderDirectory(InputPath);
  llvm::sys::path::remove_filename(HeaderDirectory);
  SmallString<256> CurrentDirectory;
  llvm::sys::fs::current_path(CurrentDirectory);

  // Get the prefix if we have one.
  if (HeaderPrefix.size() != 0)
    HeaderDirectory = HeaderPrefix;

  // Read the header list file into a buffer.
  ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
    MemoryBuffer::getFile(InputPath);
  if (std::error_code EC = listBuffer.getError())
    return EC;

  // Parse the header list into strings.
  SmallVector<StringRef, 32> Strings;
  listBuffer.get()->getBuffer().split(Strings, "\n", -1, false);

  // Collect the header file names from the string list.
  for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
    E = Strings.end();
    I != E; ++I) {
    StringRef Line = I->trim();
    // Ignore comments and empty lines.
    if (Line.empty() || (Line[0] == '#'))
      continue;
    std::pair<StringRef, StringRef> TargetAndDependents = Line.split(':');
    SmallString<256> HeaderFileName;
    // Prepend header file name prefix if it's not absolute.
    if (llvm::sys::path::is_absolute(TargetAndDependents.first))
      llvm::sys::path::native(TargetAndDependents.first, HeaderFileName);
    else {
      if (HeaderDirectory.size() != 0)
        HeaderFileName = HeaderDirectory;
      else
        HeaderFileName = CurrentDirectory;
      llvm::sys::path::append(HeaderFileName, TargetAndDependents.first);
      llvm::sys::path::native(HeaderFileName);
    }
    // Handle optional dependencies.
    DependentsVector Dependents;
    SmallVector<StringRef, 4> DependentsList;
    TargetAndDependents.second.split(DependentsList, " ", -1, false);
    int Count = DependentsList.size();
    for (int Index = 0; Index < Count; ++Index) {
      SmallString<256> Dependent;
      if (llvm::sys::path::is_absolute(DependentsList[Index]))
        Dependent = DependentsList[Index];
      else {
        if (HeaderDirectory.size() != 0)
          Dependent = HeaderDirectory;
        else
          Dependent = CurrentDirectory;
        llvm::sys::path::append(Dependent, DependentsList[Index]);
      }
      llvm::sys::path::native(Dependent);
      Dependents.push_back(getCanonicalPath(Dependent.str()));
    }
    // Get canonical form.
    HeaderFileName = getCanonicalPath(HeaderFileName);
    // Save the resulting header file path and dependencies.
    HeaderFileNames.push_back(HeaderFileName.str());
    Dependencies[HeaderFileName.str()] = Dependents;
  }
  return std::error_code();
}

// Load single module map and extract header file list.
std::error_code ModularizeUtilities::loadModuleMap(
    llvm::StringRef InputPath) {
  // Get file entry for module.modulemap file.
  const FileEntry *ModuleMapEntry =
    SourceMgr->getFileManager().getFile(InputPath);

  // return error if not found.
  if (!ModuleMapEntry) {
    llvm::errs() << "error: File \"" << InputPath << "\" not found.\n";
    return std::error_code(1, std::generic_category());
  }

  // Because the module map parser uses a ForwardingDiagnosticConsumer,
  // which doesn't forward the BeginSourceFile call, we do it explicitly here.
  DC.BeginSourceFile(*LangOpts, nullptr);

  // Figure out the home directory for the module map file.
  const DirectoryEntry *Dir = ModuleMapEntry->getDir();
  StringRef DirName(Dir->getName());
  if (llvm::sys::path::filename(DirName) == "Modules") {
    DirName = llvm::sys::path::parent_path(DirName);
    if (DirName.endswith(".framework"))
      Dir = FileMgr->getDirectory(DirName);
    // FIXME: This assert can fail if there's a race between the above check
    // and the removal of the directory.
    assert(Dir && "parent must exist");
  }

  std::unique_ptr<ModuleMap> ModMap;
  ModMap.reset(new ModuleMap(*SourceMgr, *Diagnostics, *LangOpts,
    Target.get(), *HeaderInfo));

  // Parse module.modulemap file into module map.
  if (ModMap->parseModuleMapFile(ModuleMapEntry, false, Dir)) {
    return std::error_code(1, std::generic_category());
  }

  // Do matching end call.
  DC.EndSourceFile();

  if (!collectModuleMapHeaders(ModMap.get()))
    return std::error_code(1, std::generic_category());

  // Save module map.
  ModuleMaps.push_back(std::move(ModMap));

  // Indicate we are using module maps.
  HasModuleMap = true;

  return std::error_code();
}

// Collect module map headers.
// Walks the modules and collects referenced headers into
// HeaderFileNames.
bool ModularizeUtilities::collectModuleMapHeaders(clang::ModuleMap *ModMap) {
  for (ModuleMap::module_iterator I = ModMap->module_begin(),
    E = ModMap->module_end();
    I != E; ++I) {
    if (!collectModuleHeaders(*I->second))
      return false;
  }
  return true;
}

// Collect referenced headers from one module.
// Collects the headers referenced in the given module into
// HeaderFileNames.
bool ModularizeUtilities::collectModuleHeaders(const Module &Mod) {

  // Ignore explicit modules because they often have dependencies
  // we can't know.
  if (Mod.IsExplicit)
    return true;

  // Treat headers in umbrella directory as dependencies.
  DependentsVector UmbrellaDependents;

  // Recursively do submodules.
  for (Module::submodule_const_iterator MI = Mod.submodule_begin(),
      MIEnd = Mod.submodule_end();
      MI != MIEnd; ++MI)
    collectModuleHeaders(**MI);

  if (const FileEntry *UmbrellaHeader = Mod.getUmbrellaHeader()) {
    std::string HeaderPath = getCanonicalPath(UmbrellaHeader->getName());
    // Collect umbrella header.
    HeaderFileNames.push_back(HeaderPath);

    // FUTURE: When needed, umbrella header header collection goes here.
  }
  else if (const DirectoryEntry *UmbrellaDir = Mod.getUmbrellaDir()) {
    // If there normal headers, assume these are umbrellas and skip collection.
    if (Mod.Headers->size() == 0) {
      // Collect headers in umbrella directory.
      if (!collectUmbrellaHeaders(UmbrellaDir->getName(), UmbrellaDependents))
        return false;
    }
  }

  // We ignore HK_Private, HK_Textual, HK_PrivateTextual, and HK_Excluded,
  // assuming they are marked as such either because of unsuitability for
  // modules or because they are meant to be included by another header,
  // and thus should be ignored by modularize.

  int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();

  for (int Index = 0; Index < NormalHeaderCount; ++Index) {
    DependentsVector NormalDependents;
    // Collect normal header.
    const clang::Module::Header &Header(
      Mod.Headers[clang::Module::HK_Normal][Index]);
    std::string HeaderPath = getCanonicalPath(Header.Entry->getName());
    HeaderFileNames.push_back(HeaderPath);
  }

  return true;
}

// Collect headers from an umbrella directory.
bool ModularizeUtilities::collectUmbrellaHeaders(StringRef UmbrellaDirName,
  DependentsVector &Dependents) {
  // Initialize directory name.
  SmallString<256> Directory(UmbrellaDirName);
  // Walk the directory.
  std::error_code EC;
  llvm::sys::fs::file_status Status;
  for (llvm::sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
    I.increment(EC)) {
    if (EC)
      return false;
    std::string File(I->path());
    I->status(Status);
    llvm::sys::fs::file_type Type = Status.type();
    // If the file is a directory, ignore the name and recurse.
    if (Type == llvm::sys::fs::file_type::directory_file) {
      if (!collectUmbrellaHeaders(File, Dependents))
        return false;
      continue;
    }
    // If the file does not have a common header extension, ignore it.
    if (!isHeader(File))
      continue;
    // Save header name.
    std::string HeaderPath = getCanonicalPath(File);
    Dependents.push_back(HeaderPath);
  }
  return true;
}

// Convert header path to canonical form.
// The canonical form is basically just use forward slashes, and remove "./".
// \param FilePath The file path, relative to the module map directory.
// \returns The file path in canonical form.
std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
  std::string Tmp(FilePath);
  std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
  StringRef Tmp2(Tmp);
  if (Tmp2.startswith("./"))
    Tmp = Tmp2.substr(2);
  return Tmp;
}

// Check for header file extension.
// If the file extension is .h, .inc, or missing, it's
// assumed to be a header.
// \param FileName The file name.  Must not be a directory.
// \returns true if it has a header extension or no extension.
bool ModularizeUtilities::isHeader(StringRef FileName) {
  StringRef Extension = llvm::sys::path::extension(FileName);
  if (Extension.size() == 0)
    return false;
  if (Extension.equals_lower(".h"))
    return true;
  if (Extension.equals_lower(".inc"))
    return true;
  return false;
}

// Get directory path component from file path.
// \returns the component of the given path, which will be
// relative if the given path is relative, absolute if the
// given path is absolute, or "." if the path has no leading
// path component.
std::string ModularizeUtilities::getDirectoryFromPath(StringRef Path) {
  SmallString<256> Directory(Path);
  sys::path::remove_filename(Directory);
  if (Directory.size() == 0)
    return ".";
  return Directory.str();
}
<commit_msg>Remove carriage returns.<commit_after>//===--- extra/modularize/ModularizeUtilities.cpp -------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements a class for loading and validating a module map or
// header list by checking that all headers in the corresponding directories
// are accounted for.
//
//===----------------------------------------------------------------------===//

#include "clang/Basic/SourceManager.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendActions.h"
#include "CoverageChecker.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "ModularizeUtilities.h"

using namespace clang;
using namespace llvm;
using namespace Modularize;

// Subclass TargetOptions so we can construct it inline with
// the minimal option, the triple.
class ModuleMapTargetOptions : public clang::TargetOptions {
public:
  ModuleMapTargetOptions() { Triple = llvm::sys::getDefaultTargetTriple(); }
};

// ModularizeUtilities class implementation.

// Constructor.
ModularizeUtilities::ModularizeUtilities(std::vector<std::string> &InputPaths,
                                         llvm::StringRef Prefix)
  : InputFilePaths(InputPaths),
    HeaderPrefix(Prefix),
    HasModuleMap(false),
    // Init clang stuff needed for loading the module map and preprocessing.
    LangOpts(new LangOptions()), DiagIDs(new DiagnosticIDs()),
    DiagnosticOpts(new DiagnosticOptions()),
    DC(llvm::errs(), DiagnosticOpts.get()),
    Diagnostics(
    new DiagnosticsEngine(DiagIDs, DiagnosticOpts.get(), &DC, false)),
    TargetOpts(new ModuleMapTargetOptions()),
    Target(TargetInfo::CreateTargetInfo(*Diagnostics, TargetOpts)),
    FileMgr(new FileManager(FileSystemOpts)),
    SourceMgr(new SourceManager(*Diagnostics, *FileMgr, false)),
    HeaderSearchOpts(new HeaderSearchOptions()),
    HeaderInfo(new HeaderSearch(HeaderSearchOpts, *SourceMgr, *Diagnostics,
    *LangOpts, Target.get())) {
}

// Create instance of ModularizeUtilities, to simplify setting up
// subordinate objects.
ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
    std::vector<std::string> &InputPaths, llvm::StringRef Prefix) {

  return new ModularizeUtilities(InputPaths, Prefix);
}

// Load all header lists and dependencies.
std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
  typedef std::vector<std::string>::iterator Iter;
  // For each input file.
  for (Iter I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
    llvm::StringRef InputPath = *I;
    // If it's a module map.
    if (InputPath.endswith(".modulemap")) {
      // Load the module map.
      if (std::error_code EC = loadModuleMap(InputPath))
        return EC;
    }
    else {
      // Else we assume it's a header list and load it.
      if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
        errs() << "modularize: error: Unable to get header list '" << InputPath
          << "': " << EC.message() << '\n';
        return EC;
      }
    }
  }
  return std::error_code();
}

// Do coverage checks.
// For each loaded module map, do header coverage check.
// Starting from the directory of the module.map file,
// Find all header files, optionally looking only at files
// covered by the include path options, and compare against
// the headers referenced by the module.map file.
// Display warnings for unaccounted-for header files.
// Returns 0 if there were no errors or warnings, 1 if there
// were warnings, 2 if any other problem, such as a bad
// module map path argument was specified.
std::error_code ModularizeUtilities::doCoverageCheck(
    std::vector<std::string> &IncludePaths,
    llvm::ArrayRef<std::string> CommandLine) {
  int ModuleMapCount = ModuleMaps.size();
  int ModuleMapIndex;
  std::error_code EC;
  for (ModuleMapIndex = 0; ModuleMapIndex < ModuleMapCount; ++ModuleMapIndex) {
    std::unique_ptr<clang::ModuleMap> &ModMap = ModuleMaps[ModuleMapIndex];
    CoverageChecker *Checker = CoverageChecker::createCoverageChecker(
      InputFilePaths[ModuleMapIndex], IncludePaths, CommandLine, ModMap.get());
    std::error_code LocalEC = Checker->doChecks();
    if (LocalEC.value() > 0)
      EC = LocalEC;
  }
  return EC;
}

// Load single header list and dependencies.
std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies(
    llvm::StringRef InputPath) {

  // By default, use the path component of the list file name.
  SmallString<256> HeaderDirectory(InputPath);
  llvm::sys::path::remove_filename(HeaderDirectory);
  SmallString<256> CurrentDirectory;
  llvm::sys::fs::current_path(CurrentDirectory);

  // Get the prefix if we have one.
  if (HeaderPrefix.size() != 0)
    HeaderDirectory = HeaderPrefix;

  // Read the header list file into a buffer.
  ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
    MemoryBuffer::getFile(InputPath);
  if (std::error_code EC = listBuffer.getError())
    return EC;

  // Parse the header list into strings.
  SmallVector<StringRef, 32> Strings;
  listBuffer.get()->getBuffer().split(Strings, "\n", -1, false);

  // Collect the header file names from the string list.
  for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
    E = Strings.end();
    I != E; ++I) {
    StringRef Line = I->trim();
    // Ignore comments and empty lines.
    if (Line.empty() || (Line[0] == '#'))
      continue;
    std::pair<StringRef, StringRef> TargetAndDependents = Line.split(':');
    SmallString<256> HeaderFileName;
    // Prepend header file name prefix if it's not absolute.
    if (llvm::sys::path::is_absolute(TargetAndDependents.first))
      llvm::sys::path::native(TargetAndDependents.first, HeaderFileName);
    else {
      if (HeaderDirectory.size() != 0)
        HeaderFileName = HeaderDirectory;
      else
        HeaderFileName = CurrentDirectory;
      llvm::sys::path::append(HeaderFileName, TargetAndDependents.first);
      llvm::sys::path::native(HeaderFileName);
    }
    // Handle optional dependencies.
    DependentsVector Dependents;
    SmallVector<StringRef, 4> DependentsList;
    TargetAndDependents.second.split(DependentsList, " ", -1, false);
    int Count = DependentsList.size();
    for (int Index = 0; Index < Count; ++Index) {
      SmallString<256> Dependent;
      if (llvm::sys::path::is_absolute(DependentsList[Index]))
        Dependent = DependentsList[Index];
      else {
        if (HeaderDirectory.size() != 0)
          Dependent = HeaderDirectory;
        else
          Dependent = CurrentDirectory;
        llvm::sys::path::append(Dependent, DependentsList[Index]);
      }
      llvm::sys::path::native(Dependent);
      Dependents.push_back(getCanonicalPath(Dependent.str()));
    }
    // Get canonical form.
    HeaderFileName = getCanonicalPath(HeaderFileName);
    // Save the resulting header file path and dependencies.
    HeaderFileNames.push_back(HeaderFileName.str());
    Dependencies[HeaderFileName.str()] = Dependents;
  }
  return std::error_code();
}

// Load single module map and extract header file list.
std::error_code ModularizeUtilities::loadModuleMap(
    llvm::StringRef InputPath) {
  // Get file entry for module.modulemap file.
  const FileEntry *ModuleMapEntry =
    SourceMgr->getFileManager().getFile(InputPath);

  // return error if not found.
  if (!ModuleMapEntry) {
    llvm::errs() << "error: File \"" << InputPath << "\" not found.\n";
    return std::error_code(1, std::generic_category());
  }

  // Because the module map parser uses a ForwardingDiagnosticConsumer,
  // which doesn't forward the BeginSourceFile call, we do it explicitly here.
  DC.BeginSourceFile(*LangOpts, nullptr);

  // Figure out the home directory for the module map file.
  const DirectoryEntry *Dir = ModuleMapEntry->getDir();
  StringRef DirName(Dir->getName());
  if (llvm::sys::path::filename(DirName) == "Modules") {
    DirName = llvm::sys::path::parent_path(DirName);
    if (DirName.endswith(".framework"))
      Dir = FileMgr->getDirectory(DirName);
    // FIXME: This assert can fail if there's a race between the above check
    // and the removal of the directory.
    assert(Dir && "parent must exist");
  }

  std::unique_ptr<ModuleMap> ModMap;
  ModMap.reset(new ModuleMap(*SourceMgr, *Diagnostics, *LangOpts,
    Target.get(), *HeaderInfo));

  // Parse module.modulemap file into module map.
  if (ModMap->parseModuleMapFile(ModuleMapEntry, false, Dir)) {
    return std::error_code(1, std::generic_category());
  }

  // Do matching end call.
  DC.EndSourceFile();

  if (!collectModuleMapHeaders(ModMap.get()))
    return std::error_code(1, std::generic_category());

  // Save module map.
  ModuleMaps.push_back(std::move(ModMap));

  // Indicate we are using module maps.
  HasModuleMap = true;

  return std::error_code();
}

// Collect module map headers.
// Walks the modules and collects referenced headers into
// HeaderFileNames.
bool ModularizeUtilities::collectModuleMapHeaders(clang::ModuleMap *ModMap) {
  for (ModuleMap::module_iterator I = ModMap->module_begin(),
    E = ModMap->module_end();
    I != E; ++I) {
    if (!collectModuleHeaders(*I->second))
      return false;
  }
  return true;
}

// Collect referenced headers from one module.
// Collects the headers referenced in the given module into
// HeaderFileNames.
bool ModularizeUtilities::collectModuleHeaders(const Module &Mod) {

  // Ignore explicit modules because they often have dependencies
  // we can't know.
  if (Mod.IsExplicit)
    return true;

  // Treat headers in umbrella directory as dependencies.
  DependentsVector UmbrellaDependents;

  // Recursively do submodules.
  for (Module::submodule_const_iterator MI = Mod.submodule_begin(),
      MIEnd = Mod.submodule_end();
      MI != MIEnd; ++MI)
    collectModuleHeaders(**MI);

  if (const FileEntry *UmbrellaHeader = Mod.getUmbrellaHeader()) {
    std::string HeaderPath = getCanonicalPath(UmbrellaHeader->getName());
    // Collect umbrella header.
    HeaderFileNames.push_back(HeaderPath);

    // FUTURE: When needed, umbrella header header collection goes here.
  }
  else if (const DirectoryEntry *UmbrellaDir = Mod.getUmbrellaDir()) {
    // If there normal headers, assume these are umbrellas and skip collection.
    if (Mod.Headers->size() == 0) {
      // Collect headers in umbrella directory.
      if (!collectUmbrellaHeaders(UmbrellaDir->getName(), UmbrellaDependents))
        return false;
    }
  }

  // We ignore HK_Private, HK_Textual, HK_PrivateTextual, and HK_Excluded,
  // assuming they are marked as such either because of unsuitability for
  // modules or because they are meant to be included by another header,
  // and thus should be ignored by modularize.

  int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();

  for (int Index = 0; Index < NormalHeaderCount; ++Index) {
    DependentsVector NormalDependents;
    // Collect normal header.
    const clang::Module::Header &Header(
      Mod.Headers[clang::Module::HK_Normal][Index]);
    std::string HeaderPath = getCanonicalPath(Header.Entry->getName());
    HeaderFileNames.push_back(HeaderPath);
  }

  return true;
}

// Collect headers from an umbrella directory.
bool ModularizeUtilities::collectUmbrellaHeaders(StringRef UmbrellaDirName,
  DependentsVector &Dependents) {
  // Initialize directory name.
  SmallString<256> Directory(UmbrellaDirName);
  // Walk the directory.
  std::error_code EC;
  llvm::sys::fs::file_status Status;
  for (llvm::sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
    I.increment(EC)) {
    if (EC)
      return false;
    std::string File(I->path());
    I->status(Status);
    llvm::sys::fs::file_type Type = Status.type();
    // If the file is a directory, ignore the name and recurse.
    if (Type == llvm::sys::fs::file_type::directory_file) {
      if (!collectUmbrellaHeaders(File, Dependents))
        return false;
      continue;
    }
    // If the file does not have a common header extension, ignore it.
    if (!isHeader(File))
      continue;
    // Save header name.
    std::string HeaderPath = getCanonicalPath(File);
    Dependents.push_back(HeaderPath);
  }
  return true;
}

// Convert header path to canonical form.
// The canonical form is basically just use forward slashes, and remove "./".
// \param FilePath The file path, relative to the module map directory.
// \returns The file path in canonical form.
std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
  std::string Tmp(FilePath);
  std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
  StringRef Tmp2(Tmp);
  if (Tmp2.startswith("./"))
    Tmp = Tmp2.substr(2);
  return Tmp;
}

// Check for header file extension.
// If the file extension is .h, .inc, or missing, it's
// assumed to be a header.
// \param FileName The file name.  Must not be a directory.
// \returns true if it has a header extension or no extension.
bool ModularizeUtilities::isHeader(StringRef FileName) {
  StringRef Extension = llvm::sys::path::extension(FileName);
  if (Extension.size() == 0)
    return false;
  if (Extension.equals_lower(".h"))
    return true;
  if (Extension.equals_lower(".inc"))
    return true;
  return false;
}

// Get directory path component from file path.
// \returns the component of the given path, which will be
// relative if the given path is relative, absolute if the
// given path is absolute, or "." if the path has no leading
// path component.
std::string ModularizeUtilities::getDirectoryFromPath(StringRef Path) {
  SmallString<256> Directory(Path);
  sys::path::remove_filename(Directory);
  if (Directory.size() == 0)
    return ".";
  return Directory.str();
}
<|endoftext|>


<commit_before>/** \file
 *
 *  Copyright (c) 2015 by Travis Gockel. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author Travis Gockel (travis@gockelhut.com)
**/
#include "test.hpp"
#include "filesystem_util.hpp"

#include <jsonv/parse.hpp>
#include <jsonv/util.hpp>
#include <jsonv/value.hpp>

#include <fstream>

namespace jsonv_test
{

using namespace jsonv;

class json_diff_test :
        public unit_test
{
public:
    json_diff_test(std::string path) :
            unit_test(std::string("diff_test/") + filename(path)),
            path(std::move(path))
    { }

    virtual void run_impl() override
    {
        value whole = [this] { std::ifstream in(path.c_str()); return parse(in); }();

        diff_result result = diff(whole.at_path(".input.left"), whole.at_path(".input.right"));
        diff_result expected;
        expected.same  = whole.at_path(".result.same");
        expected.right = whole.at_path(".result.right");
        expected.left  = whole.at_path(".result.left");

        ensure_eq(expected.same,  result.same);
        ensure_eq(expected.left,  result.left);
        ensure_eq(expected.right, result.right);
    }

private:
    std::string path;
};

class json_diff_test_initializer
{
public:
    explicit json_diff_test_initializer(const std::string& rootpath)
    {
        recursive_directory_for_each(rootpath, ".json", [this] (const std::string& p)
        {
            _tests.emplace_back(new json_diff_test(p));
        });
    }

private:
    std::deque<std::unique_ptr<unit_test>> _tests;
} json_diff_test_initializer_instance("src/jsonv-tests/data/diffs");

}
<commit_msg>MSVC: Workaround parsing error in lambda expressions.<commit_after>/** \file
 *
 *  Copyright (c) 2015 by Travis Gockel. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author Travis Gockel (travis@gockelhut.com)
**/
#include "test.hpp"
#include "filesystem_util.hpp"

#include <jsonv/parse.hpp>
#include <jsonv/util.hpp>
#include <jsonv/value.hpp>

#include <fstream>

namespace jsonv_test
{

using namespace jsonv;

class json_diff_test :
        public unit_test
{
public:
    json_diff_test(std::string path) :
            unit_test(std::string("diff_test/") + filename(path)),
            path(std::move(path))
    { }

    virtual void run_impl() override
    {
        value whole;
        {
            std::ifstream in(path.c_str());
            whole = parse(in);
        }

        diff_result result = diff(whole.at_path(".input.left"), whole.at_path(".input.right"));
        diff_result expected;
        expected.same  = whole.at_path(".result.same");
        expected.right = whole.at_path(".result.right");
        expected.left  = whole.at_path(".result.left");

        ensure_eq(expected.same,  result.same);
        ensure_eq(expected.left,  result.left);
        ensure_eq(expected.right, result.right);
    }

private:
    std::string path;
};

class json_diff_test_initializer
{
public:
    explicit json_diff_test_initializer(const std::string& rootpath)
    {
        recursive_directory_for_each(rootpath, ".json", [this] (const std::string& p)
        {
            _tests.emplace_back(new json_diff_test(p));
        });
    }

private:
    std::deque<std::unique_ptr<unit_test>> _tests;
} json_diff_test_initializer_instance("src/jsonv-tests/data/diffs");

}
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#ifndef ITEM_HH
#define ITEM_HH
#include "config.h"

#include <string>
#include <string.h>
#include <stdio.h>
#include <memcached/engine.h>

#include "mutex.hh"
#include "locks.hh"
#include "atomic.hh"

class Blob {
public:

    // Constructors.

    static Blob* New(const char *start, const size_t len) {
        size_t total_len = len + sizeof(Blob);
        Blob *t = new (::operator new(total_len)) Blob(start, len);
        assert(t->length() == len);
        return t;
    }

    static Blob* New(const std::string& s) {
        return New(s.data(), s.length());
    }

    static Blob* New(const size_t len, const char c) {
        size_t total_len = len + sizeof(Blob);
        Blob *t = new (::operator new(total_len)) Blob(c, len);
        assert(t->length() == len);
        return t;
    }

    // Actual accessorish things.

    const char* getData() const {
        return data;
    }

    size_t length() const {
        return size;
    }

    const std::string to_s() const {
        return std::string(data, size);
    }

    // This is necessary for making C++ happy when I'm doing a
    // placement new on fairly "normal" c++ heap allocations, just
    // with variable-sized objects.
    void operator delete(void* p) { ::operator delete(p); }

private:

    explicit Blob(const char *start, const size_t len) : size(static_cast<uint32_t>(len)) {
        std::memcpy(data, start, len);
    }

    explicit Blob(const char c, const size_t len) : size(static_cast<uint32_t>(len)) {
        std::memset(data, c, len);
    }

    const uint32_t size;
    char data[1];

    DISALLOW_COPY_AND_ASSIGN(Blob);
};

typedef shared_ptr<const Blob> value_t;

/**
 * The Item structure we use to pass information between the memcached
 * core and the backend. Please note that the kvstore don't store these
 * objects, so we do have an extra layer of memory copying :(
 */
class Item {
public:
    Item(const void* k, const size_t nk, const size_t nb,
         const int fl, const rel_time_t exp, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        key.assign(static_cast<const char*>(k), nk);
        assert(id != 0);
        setData(NULL, nb);
    }

    Item(const std::string &k, const int fl, const rel_time_t exp,
         const void *dta, const size_t nb, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        key.assign(k);
        assert(id != 0);
        setData(static_cast<const char*>(dta), nb);
    }

    Item(const std::string &k, const int fl, const rel_time_t exp,
         value_t val, uint64_t theCas = 0,  int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), value(val), cas(theCas), id(i), vbucketId(vbid)
    {
        assert(id != 0);
        key.assign(k);
    }

    Item(const void *k, uint16_t nk, const int fl, const rel_time_t exp,
         const void *dta, const size_t nb, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        assert(id != 0);
        key.assign(static_cast<const char*>(k), nk);
        setData(static_cast<const char*>(dta), nb);
    }

    ~Item() {}

    const char *getData() const {
        return value->getData();
    }

    value_t getValue() const {
        return value;
    }

    const std::string &getKey() const {
        return key;
    }

    int64_t getId() const {
        return id;
    }

    int getNKey() const {
        return static_cast<int>(key.length());
    }

    uint32_t getNBytes() const {
        return static_cast<uint32_t>(value->length());
    }

    rel_time_t getExptime() const {
        return exptime;
    }

    int getFlags() const {
        return flags;
    }

    uint64_t getCas() const {
        return cas;
    }

    void setCas() {
        cas = nextCas();
    }

    void setCas(uint64_t ncas) {
        cas = ncas;
    }

    /**
     * Append another item to this item
     *
     * @param item the item to append to this one
     * @return true if success
     */
    bool append(const Item &item) {
        std::string newValue(value->getData(), 0, value->length() - 2);
        newValue.append(item.getValue()->to_s());
        value.reset(Blob::New(newValue));
        return true;
    }

    /**
     * Prepend another item to this item
     *
     * @param item the item to prepend to this one
     * @return true if success
     */
    bool prepend(const Item &item) {
        std::string newValue(item.getValue()->to_s(), 0, item.getNBytes() - 2);
        newValue.append(value->to_s());
        value.reset(Blob::New(newValue));
        return true;
    }

    uint16_t getVBucketId(void) const {
        return vbucketId;
    }

    void setVBucketId(uint16_t to) {
        vbucketId = to;
    }

private:
    /**
     * Set the item's data. This is only used by constructors, so we
     * make it private.
     */
    void setData(const char *dta, const size_t nb) {
        Blob *data;
        if (dta == NULL) {
            data = Blob::New(nb, '\0');
        } else {
            data = Blob::New(dta, nb);
        }

        assert(data);
        value.reset(data);
    }

    int flags;
    rel_time_t exptime;
    std::string key;
    value_t value;
    uint64_t cas;
    int64_t id;
    uint16_t vbucketId;

    static uint64_t nextCas(void) {
        uint64_t ret;
        ret = casCounter++;
        if ((ret % casNotificationFrequency) == 0) {
            casNotifier(ret);
        }

        return ret;
    }

    static void initializeCas(uint64_t initial, void (*notifier)(uint64_t current),
                              uint64_t frequency) {
        casCounter = initial;
        casNotifier = notifier;
        casNotificationFrequency = frequency;
    }

    static uint64_t casNotificationFrequency;
    static void (*casNotifier)(uint64_t);
    static Atomic<uint64_t> casCounter;
    DISALLOW_COPY_AND_ASSIGN(Item);
};

#endif
<commit_msg>Document Blob.<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#ifndef ITEM_HH
#define ITEM_HH
#include "config.h"

#include <string>
#include <string.h>
#include <stdio.h>
#include <memcached/engine.h>

#include "mutex.hh"
#include "locks.hh"
#include "atomic.hh"

/**
 * A blob is a minimal sized storage for data up to 2^32 bytes long.
 */
class Blob {
public:

    // Constructors.

    /**
     * Create a new Blob holding the given data.
     *
     * @param start the beginning of the data to copy into this blob
     * @param len the amount of data to copy in
     *
     * @return the new Blob instance
     */
    static Blob* New(const char *start, const size_t len) {
        size_t total_len = len + sizeof(Blob);
        Blob *t = new (::operator new(total_len)) Blob(start, len);
        assert(t->length() == len);
        return t;
    }

    /**
     * Create a new Blob holding the contents of the given string.
     *
     * @param s the string whose contents go into the blob
     *
     * @return the new Blob instance
     */
    static Blob* New(const std::string& s) {
        return New(s.data(), s.length());
    }

    /**
     * Create a new Blob pre-filled with the given character.
     *
     * @param len the size of the blob
     * @param c the character to fill the blob with
     *
     * @return the new Blob instance
     */
    static Blob* New(const size_t len, const char c) {
        size_t total_len = len + sizeof(Blob);
        Blob *t = new (::operator new(total_len)) Blob(c, len);
        assert(t->length() == len);
        return t;
    }

    // Actual accessorish things.

    /**
     * Get the pointer to the contents of this Blob.
     */
    const char* getData() const {
        return data;
    }

    /**
     * Get the length of this Blob.
     */
    size_t length() const {
        return size;
    }

    /**
     * Get a std::string representation of this blob.
     */
    const std::string to_s() const {
        return std::string(data, size);
    }

    // This is necessary for making C++ happy when I'm doing a
    // placement new on fairly "normal" c++ heap allocations, just
    // with variable-sized objects.
    void operator delete(void* p) { ::operator delete(p); }

private:

    explicit Blob(const char *start, const size_t len) : size(static_cast<uint32_t>(len)) {
        std::memcpy(data, start, len);
    }

    explicit Blob(const char c, const size_t len) : size(static_cast<uint32_t>(len)) {
        std::memset(data, c, len);
    }

    const uint32_t size;
    char data[1];

    DISALLOW_COPY_AND_ASSIGN(Blob);
};

typedef shared_ptr<const Blob> value_t;

/**
 * The Item structure we use to pass information between the memcached
 * core and the backend. Please note that the kvstore don't store these
 * objects, so we do have an extra layer of memory copying :(
 */
class Item {
public:
    Item(const void* k, const size_t nk, const size_t nb,
         const int fl, const rel_time_t exp, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        key.assign(static_cast<const char*>(k), nk);
        assert(id != 0);
        setData(NULL, nb);
    }

    Item(const std::string &k, const int fl, const rel_time_t exp,
         const void *dta, const size_t nb, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        key.assign(k);
        assert(id != 0);
        setData(static_cast<const char*>(dta), nb);
    }

    Item(const std::string &k, const int fl, const rel_time_t exp,
         value_t val, uint64_t theCas = 0,  int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), value(val), cas(theCas), id(i), vbucketId(vbid)
    {
        assert(id != 0);
        key.assign(k);
    }

    Item(const void *k, uint16_t nk, const int fl, const rel_time_t exp,
         const void *dta, const size_t nb, uint64_t theCas = 0,
         int64_t i = -1, uint16_t vbid = 0) :
        flags(fl), exptime(exp), cas(theCas), id(i), vbucketId(vbid)
    {
        assert(id != 0);
        key.assign(static_cast<const char*>(k), nk);
        setData(static_cast<const char*>(dta), nb);
    }

    ~Item() {}

    const char *getData() const {
        return value->getData();
    }

    value_t getValue() const {
        return value;
    }

    const std::string &getKey() const {
        return key;
    }

    int64_t getId() const {
        return id;
    }

    int getNKey() const {
        return static_cast<int>(key.length());
    }

    uint32_t getNBytes() const {
        return static_cast<uint32_t>(value->length());
    }

    rel_time_t getExptime() const {
        return exptime;
    }

    int getFlags() const {
        return flags;
    }

    uint64_t getCas() const {
        return cas;
    }

    void setCas() {
        cas = nextCas();
    }

    void setCas(uint64_t ncas) {
        cas = ncas;
    }

    /**
     * Append another item to this item
     *
     * @param item the item to append to this one
     * @return true if success
     */
    bool append(const Item &item) {
        std::string newValue(value->getData(), 0, value->length() - 2);
        newValue.append(item.getValue()->to_s());
        value.reset(Blob::New(newValue));
        return true;
    }

    /**
     * Prepend another item to this item
     *
     * @param item the item to prepend to this one
     * @return true if success
     */
    bool prepend(const Item &item) {
        std::string newValue(item.getValue()->to_s(), 0, item.getNBytes() - 2);
        newValue.append(value->to_s());
        value.reset(Blob::New(newValue));
        return true;
    }

    uint16_t getVBucketId(void) const {
        return vbucketId;
    }

    void setVBucketId(uint16_t to) {
        vbucketId = to;
    }

private:
    /**
     * Set the item's data. This is only used by constructors, so we
     * make it private.
     */
    void setData(const char *dta, const size_t nb) {
        Blob *data;
        if (dta == NULL) {
            data = Blob::New(nb, '\0');
        } else {
            data = Blob::New(dta, nb);
        }

        assert(data);
        value.reset(data);
    }

    int flags;
    rel_time_t exptime;
    std::string key;
    value_t value;
    uint64_t cas;
    int64_t id;
    uint16_t vbucketId;

    static uint64_t nextCas(void) {
        uint64_t ret;
        ret = casCounter++;
        if ((ret % casNotificationFrequency) == 0) {
            casNotifier(ret);
        }

        return ret;
    }

    static void initializeCas(uint64_t initial, void (*notifier)(uint64_t current),
                              uint64_t frequency) {
        casCounter = initial;
        casNotifier = notifier;
        casNotificationFrequency = frequency;
    }

    static uint64_t casNotificationFrequency;
    static void (*casNotifier)(uint64_t);
    static Atomic<uint64_t> casCounter;
    DISALLOW_COPY_AND_ASSIGN(Item);
};

#endif
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <set>

#include <boost/test/unit_test.hpp>

#include <seastar/core/thread.hh>

#include "sstables/sstables.hh"
#include "compress.hh"
#include "schema_builder.hh"
#include "tests/test-utils.hh"

using namespace sstables;

class sstable_assertions final {
    shared_sstable _sst;
public:
    sstable_assertions(schema_ptr schema, const sstring& path, int generation = 1)
        : _sst(make_sstable(std::move(schema),
                            path,
                            generation,
                            sstable_version_types::mc,
                            sstable_format_types::big))
    { }
    void read_toc() {
        _sst->read_toc().get();
    }
    void read_summary() {
        _sst->read_summary(default_priority_class()).get();
    }
    void read_filter() {
        _sst->read_filter(default_priority_class()).get();
    }
    void read_statistics() {
        _sst->read_statistics(default_priority_class()).get();
    }
    void assert_toc(const std::set<component_type>& expected_components) {
        for (auto& expected : expected_components) {
            if(_sst->_recognized_components.count(expected) == 0) {
                BOOST_FAIL(sprint("Expected component of TOC missing: %s\n ... in: %s",
                                  expected,
                                  std::set<component_type>(
                                      cbegin(_sst->_recognized_components),
                                      cend(_sst->_recognized_components))));
            }
        }
        for (auto& present : _sst->_recognized_components) {
            if (expected_components.count(present) == 0) {
                BOOST_FAIL(sprint("Unexpected component of TOC: %s\n ... when expecting: %s",
                                  present,
                                  expected_components));
            }
        }
    }
};

// Following tests run on files in tests/sstables/3.x/uncompressed/simple
// They were created using following CQL statements:
//
// CREATE KEYSPACE test_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
//
// CREATE TABLE test_ks.test_table ( pk INT, ck INT, val INT, PRIMARY KEY(pk, ck))
//      WITH compression = { 'enabled' : false };
//
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(1, 101, 1001);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(2, 102, 1002);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(3, 103, 1003);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(4, 104, 1004);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(5, 105, 1005);

static thread_local const sstring UNCOMPRESSED_SIMPLE_PATH = "tests/sstables/3.x/uncompressed/simple";
static thread_local const schema_ptr UNCOMPRESSED_SIMPLE_SCHEMA =
    schema_builder("test_ks", "test_table")
        .with_column("pk", int32_type, column_kind::partition_key)
        .with_column("ck", int32_type, column_kind::clustering_key)
        .with_column("val", int32_type)
        .build();

SEASTAR_TEST_CASE(test_uncompressed_simple_read_toc) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        using ct = component_type;
        sst.assert_toc({ct::Index,
                        ct::Data,
                        ct::TOC,
                        ct::Summary,
                        ct::Digest,
                        ct::CRC,
                        ct::Filter,
                        ct::Statistics});
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_summary) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_summary();
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_filter) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_filter();
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_statistics) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_statistics();
    });
}
<commit_msg>Add test for loading the whole sstable<commit_after>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <set>

#include <boost/test/unit_test.hpp>

#include <seastar/core/thread.hh>

#include "sstables/sstables.hh"
#include "compress.hh"
#include "schema_builder.hh"
#include "tests/test-utils.hh"

using namespace sstables;

class sstable_assertions final {
    shared_sstable _sst;
public:
    sstable_assertions(schema_ptr schema, const sstring& path, int generation = 1)
        : _sst(make_sstable(std::move(schema),
                            path,
                            generation,
                            sstable_version_types::mc,
                            sstable_format_types::big))
    { }
    void read_toc() {
        _sst->read_toc().get();
    }
    void read_summary() {
        _sst->read_summary(default_priority_class()).get();
    }
    void read_filter() {
        _sst->read_filter(default_priority_class()).get();
    }
    void read_statistics() {
        _sst->read_statistics(default_priority_class()).get();
    }
    void load() {
        _sst->load().get();
    }
    void assert_toc(const std::set<component_type>& expected_components) {
        for (auto& expected : expected_components) {
            if(_sst->_recognized_components.count(expected) == 0) {
                BOOST_FAIL(sprint("Expected component of TOC missing: %s\n ... in: %s",
                                  expected,
                                  std::set<component_type>(
                                      cbegin(_sst->_recognized_components),
                                      cend(_sst->_recognized_components))));
            }
        }
        for (auto& present : _sst->_recognized_components) {
            if (expected_components.count(present) == 0) {
                BOOST_FAIL(sprint("Unexpected component of TOC: %s\n ... when expecting: %s",
                                  present,
                                  expected_components));
            }
        }
    }
};

// Following tests run on files in tests/sstables/3.x/uncompressed/simple
// They were created using following CQL statements:
//
// CREATE KEYSPACE test_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
//
// CREATE TABLE test_ks.test_table ( pk INT, ck INT, val INT, PRIMARY KEY(pk, ck))
//      WITH compression = { 'enabled' : false };
//
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(1, 101, 1001);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(2, 102, 1002);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(3, 103, 1003);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(4, 104, 1004);
// INSERT INTO test_ks.test_table(pk, ck, val) VALUES(5, 105, 1005);

static thread_local const sstring UNCOMPRESSED_SIMPLE_PATH = "tests/sstables/3.x/uncompressed/simple";
static thread_local const schema_ptr UNCOMPRESSED_SIMPLE_SCHEMA =
    schema_builder("test_ks", "test_table")
        .with_column("pk", int32_type, column_kind::partition_key)
        .with_column("ck", int32_type, column_kind::clustering_key)
        .with_column("val", int32_type)
        .build();

SEASTAR_TEST_CASE(test_uncompressed_simple_read_toc) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        using ct = component_type;
        sst.assert_toc({ct::Index,
                        ct::Data,
                        ct::TOC,
                        ct::Summary,
                        ct::Digest,
                        ct::CRC,
                        ct::Filter,
                        ct::Statistics});
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_summary) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_summary();
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_filter) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_filter();
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_read_statistics) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.read_toc();
        sst.read_statistics();
    });
}

SEASTAR_TEST_CASE(test_uncompressed_simple_load) {
    return seastar::async([] {
        sstable_assertions sst(UNCOMPRESSED_SIMPLE_SCHEMA, UNCOMPRESSED_SIMPLE_PATH);
        sst.load();
    });
}
<|endoftext|>