<commit_before>/*
 * Copyright (C) 2010, Gostai S.A.S.
 *
 * This software is provided "as is" without warranty of any kind,
 * either expressed or implied, including but not limited to the
 * implied warranties of fitness for a particular purpose.
 *
 * See the LICENSE file for more information.
 */

#include <libport/sys/utsname.h>
#include <libport/cassert>
#include <sstream>

#if ! defined LIBPORT_HAVE_SYS_UTSNAME_H
// Replacement for Windows.
int
uname(struct utsname* u)
{
  u->sysname = "unknown";
  u->nodename = "unknown";
  u->release = "unknown";
  u->version = "unknown";
  u->machine = "unknown";
  return 0;
};

#endif

namespace libport
{

  utsname::utsname()
    : major_(0)
    , minor_(0)
    , patchlevel_(0)
  {
    ERRNO_RUN(uname, &utsname_);
    std::stringstream r(release());
    r >> major_;
    assert_eq(r.get(), '.');
    r >> minor_;
    assert_eq(r.get(), '.');
    r >> patchlevel_;
  }

#define DEFINE(Function, Member)                \
  std::string                                   \
  utsname::Function() const                     \
  {                                             \
    return utsname_.Member;                     \
  }

  DEFINE(machine, machine);
  DEFINE(node,    nodename);
  DEFINE(release, release);
  DEFINE(system,  sysname);
  DEFINE(version, version);
#undef DEFINE

#define DEFINE(Function)                        \
  unsigned                                      \
  utsname::release_ ## Function() const         \
  {                                             \
    return Function ## _;                       \
  }

  DEFINE(major);
  DEFINE(minor);
  DEFINE(patchlevel);
#undef DEFINE

}
<commit_msg>utsname: fixes.<commit_after>/*
 * Copyright (C) 2010, Gostai S.A.S.
 *
 * This software is provided "as is" without warranty of any kind,
 * either expressed or implied, including but not limited to the
 * implied warranties of fitness for a particular purpose.
 *
 * See the LICENSE file for more information.
 */

#include <libport/sys/utsname.h>
#include <libport/cassert>
#include <sstream>

#if ! defined LIBPORT_HAVE_SYS_UTSNAME_H
// Replacement for Windows.
int
uname(struct utsname* u)
{
  u->sysname = "unknown";
  u->nodename = "unknown";
  u->release = "unknown";
  u->version = "unknown";
  u->machine = "unknown";
  return 0;
};

#endif

namespace libport
{

  utsname::utsname()
    : major_(0)
    , minor_(0)
    , patchlevel_(0)
  {
    ERRNO_RUN(uname, &utsname_);
    std::stringstream r(release());
    // FIXME: Improve checking.  Use Spirit?
    r >> major_;
    // Skip `.'.
    r.get();
    r >> minor_;
    // Skip `.'.
    r.get();
    r >> patchlevel_;
  }

#define DEFINE(Function, Member)                \
  std::string                                   \
  utsname::Function() const                     \
  {                                             \
    return utsname_.Member;                     \
  }

  DEFINE(machine, machine);
  DEFINE(node,    nodename);
  DEFINE(release, release);
  DEFINE(system,  sysname);
  DEFINE(version, version);
#undef DEFINE

#define DEFINE(Function)                        \
  unsigned                                      \
  utsname::release_ ## Function() const         \
  {                                             \
    return Function ## _;                       \
  }

  DEFINE(major);
  DEFINE(minor);
  DEFINE(patchlevel);
#undef DEFINE

}
<|endoftext|>


<commit_before>#include "NeuroNode.hpp"
#include "SystemCommand.hpp"
using namespace MadoopInternal;

#include "MadoopDebug.hpp"

#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <exception>
#include <list>

#include <boost/foreach.hpp>
#include <boost/optional.hpp>
#include <boost/log/trivial.hpp>

#include <boost/regex.hpp>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp>         // streaming operators etc.
#include <boost/uuid/sha1.hpp>
#include <boost/array.hpp>
typedef boost::array<boost::uint8_t,32> hashData_t;

/** @brief get a hash code.
*   @param None.
*   @return ipv6 address.
*/
static hashData_t getShaHash( const void *data, const std::size_t byte_count )
{
	boost::uuids::detail::sha1 sha1;
	sha1.process_bytes( data, byte_count );
	unsigned int digest[5];
	sha1.get_digest( digest );
	const boost::uint8_t *p_digest = reinterpret_cast<const boost::uint8_t *>( digest );
	hashData_t hash_data;
	for( int i = 0; i < 5; ++i )
	{
		hash_data[ i * 4 ]     = p_digest[ i * 4 + 3 ];
		hash_data[ i * 4 + 1 ] = p_digest[ i * 4 + 2 ];
		hash_data[ i * 4 + 2 ] = p_digest[ i * 4 + 1 ];
		hash_data[ i * 4 + 3 ] = p_digest[ i * 4 ];
	}
	return hash_data;
}

/** @brief generate a new address for self node.
*   @param None.
*   @return ipv6 address.
*/
static const string genUAddress(void)
{
	/*
	string uuidStr = boost::uuids::to_string(boost::uuids::random_generator()());
	hashData_t hash = getShaHash(uuidStr.c_str(),uuidStr.size());
	string ret("");
	for(auto &x : hash) {
		char buf[3]={0};
		std::sprintf(buf, "%02x", x);
		ret += buf;
	}
	*/
	SystemCommand cmd("openssl genrsa  4096 | openssl rsa -pubout | sha512sum");
	string privateKey = cmd.result();
	{
		boost::regex  reg( " " );
		privateKey = boost::regex_replace(privateKey,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "\n" );
		privateKey = boost::regex_replace(privateKey,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "-" );
		privateKey = boost::regex_replace(privateKey,reg,"", boost::format_all);
	}
	return privateKey;
}

/** @brief read ipv6 of address
*   @param None.
*   @return ipv6 address.
*/
static string readIPv6(void)
{
	string address = "";

//	SystemCommand cmd("ifconfig | grep inet6 | grep Scope:Global");
	SystemCommand cmd("ifconfig | grep inet6 | grep Scope:Link");
	address = cmd.result();
#if 0
	FILE* pipe = popen("ifconfig | grep inet6 | grep Scope:Global", "r");
	if (NULL == pipe)
	{
		return "ERROR";
	}
	char buffer[256];
	if(NULL != fgets(buffer, sizeof(buffer), pipe))
	{
		address = buffer;
	}
	pclose(pipe);
#endif
	{
		boost::regex  reg( "/64 Scope:Global" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "inet6 addr:" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( " " );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "\n" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	return address;
}

/** @brief constructor of self host
*   @param uniqued address.
*   @return None.
*/
NeuroNode::NeuroNode(void)
:_uAdd(genUAddress())
,_host(readIPv6())
,_port()
{
}

/** @brief constructor of remote host
*   @param uniqued address.
*   @return None.
*/
NeuroNode::NeuroNode(const string &uAdd,const string &jconf)
:_uAdd(uAdd)
,_host()
,_port()
{
	TRACE_VAR(_uAdd);
	TRACE_VAR(_host);
	TRACE_VAR(_port);
}

/** @brief build
*   @param port number.
*   @return None.
*/
void NeuroNode::build(int port)
{
	_port = port;
	TRACE_VAR(_uAdd);
	TRACE_VAR(_host);
	TRACE_VAR(_port);
}

<commit_msg>add pipe shell run,& change address to rsa public key<commit_after>#include "NeuroNode.hpp"
#include "SystemCommand.hpp"
using namespace MadoopInternal;

#include "MadoopDebug.hpp"

#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <exception>
#include <list>

#include <boost/foreach.hpp>
#include <boost/optional.hpp>
#include <boost/log/trivial.hpp>

#include <boost/regex.hpp>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp>         // streaming operators etc.
#include <boost/uuid/sha1.hpp>
#include <boost/array.hpp>
typedef boost::array<boost::uint8_t,32> hashData_t;

/** @brief get a hash code.
*   @param None.
*   @return ipv6 address.
*/
static hashData_t getShaHash( const void *data, const std::size_t byte_count )
{
	boost::uuids::detail::sha1 sha1;
	sha1.process_bytes( data, byte_count );
	unsigned int digest[5];
	sha1.get_digest( digest );
	const boost::uint8_t *p_digest = reinterpret_cast<const boost::uint8_t *>( digest );
	hashData_t hash_data;
	for( int i = 0; i < 5; ++i )
	{
		hash_data[ i * 4 ]     = p_digest[ i * 4 + 3 ];
		hash_data[ i * 4 + 1 ] = p_digest[ i * 4 + 2 ];
		hash_data[ i * 4 + 2 ] = p_digest[ i * 4 + 1 ];
		hash_data[ i * 4 + 3 ] = p_digest[ i * 4 ];
	}
	return hash_data;
}

/** @brief generate a new address for self node.
*   @param None.
*   @return ipv6 address.
*/
static const string genUAddress(void)
{
	/*
	string uuidStr = boost::uuids::to_string(boost::uuids::random_generator()());
	hashData_t hash = getShaHash(uuidStr.c_str(),uuidStr.size());
	string ret("");
	for(auto &x : hash) {
		char buf[3]={0};
		std::sprintf(buf, "%02x", x);
		ret += buf;
	}
	*/
	SystemCommand cmd("openssl genrsa  4096 | openssl rsa -pubout | sha512sum");
	string shaPublicKey = cmd.result();
	{
		boost::regex  reg( " " );
		shaPublicKey = boost::regex_replace(shaPublicKey,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "\n" );
		shaPublicKey = boost::regex_replace(shaPublicKey,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "-" );
		shaPublicKey = boost::regex_replace(shaPublicKey,reg,"", boost::format_all);
	}
	return shaPublicKey;
}

/** @brief read ipv6 of address
*   @param None.
*   @return ipv6 address.
*/
static string readIPv6(void)
{
	string address = "";

	SystemCommand cmd("ifconfig | grep inet6 | grep Scope:Global");
//	SystemCommand cmd("ifconfig | grep inet6 | grep Scope:Link");
	address = cmd.result();
	{
		boost::regex  reg( "/64 Scope:Global" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "inet6 addr:" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( " " );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	{
		boost::regex  reg( "\n" );
		address = boost::regex_replace(address,reg,"", boost::format_all);
	}
	return address;
}

/** @brief constructor of self host
*   @param uniqued address.
*   @return None.
*/
NeuroNode::NeuroNode(void)
:_uAdd(genUAddress())
,_host(readIPv6())
,_port()
{
}

/** @brief constructor of remote host
*   @param uniqued address.
*   @return None.
*/
NeuroNode::NeuroNode(const string &uAdd,const string &jconf)
:_uAdd(uAdd)
,_host()
,_port()
{
	TRACE_VAR(_uAdd);
	TRACE_VAR(_host);
	TRACE_VAR(_port);
}

/** @brief build
*   @param port number.
*   @return None.
*/
void NeuroNode::build(int port)
{
	_port = port;
	TRACE_VAR(_uAdd);
	TRACE_VAR(_host);
	TRACE_VAR(_port);
}

<|endoftext|>


<commit_before>//===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// Interceptors for malloc related functions.
///
//===----------------------------------------------------------------------===//

#include "scudo_allocator.h"

#include "interception/interception.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"

using namespace __scudo;

extern "C" {
INTERCEPTOR_ATTRIBUTE void free(void *ptr) {
  scudoDeallocate(ptr, 0, 0, FromMalloc);
}

INTERCEPTOR_ATTRIBUTE void *malloc(SIZE_T size) {
  return scudoAllocate(size, 0, FromMalloc);
}

INTERCEPTOR_ATTRIBUTE void *realloc(void *ptr, SIZE_T size) {
  return scudoRealloc(ptr, size);
}

INTERCEPTOR_ATTRIBUTE void *calloc(SIZE_T nmemb, SIZE_T size) {
  return scudoCalloc(nmemb, size);
}

INTERCEPTOR_ATTRIBUTE void *valloc(SIZE_T size) {
  return scudoValloc(size);
}

INTERCEPTOR_ATTRIBUTE
int posix_memalign(void **memptr, SIZE_T alignment, SIZE_T size) {
  return scudoPosixMemalign(memptr, alignment, size);
}

#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR_ATTRIBUTE void cfree(void *ptr) ALIAS("free");
#endif

#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR_ATTRIBUTE void *memalign(SIZE_T alignment, SIZE_T size) {
  return scudoAllocate(size, alignment, FromMemalign);
}

INTERCEPTOR_ATTRIBUTE
void *__libc_memalign(SIZE_T alignment, SIZE_T size) ALIAS("memalign");
#endif

#if SANITIZER_INTERCEPT_PVALLOC
INTERCEPTOR_ATTRIBUTE void *pvalloc(SIZE_T size) {
  return scudoPvalloc(size);
}
#endif

#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR_ATTRIBUTE void *aligned_alloc(SIZE_T alignment, SIZE_T size) {
  return scudoAlignedAlloc(alignment, size);
}
#endif

#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE
INTERCEPTOR_ATTRIBUTE SIZE_T malloc_usable_size(void *ptr) {
  return scudoMallocUsableSize(ptr);
}
#endif

#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) {
  return -1;
}
#endif
}  // extern "C"
<commit_msg>[scudo] Get rid of builtin-declaration-mismatch warnings<commit_after>//===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// Interceptors for malloc related functions.
///
//===----------------------------------------------------------------------===//

#include "scudo_allocator.h"

#include "interception/interception.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"

#include <stddef.h>

using namespace __scudo;

extern "C" {
INTERCEPTOR_ATTRIBUTE void free(void *ptr) {
  scudoDeallocate(ptr, 0, 0, FromMalloc);
}

INTERCEPTOR_ATTRIBUTE void *malloc(size_t size) {
  return scudoAllocate(size, 0, FromMalloc);
}

INTERCEPTOR_ATTRIBUTE void *realloc(void *ptr, size_t size) {
  return scudoRealloc(ptr, size);
}

INTERCEPTOR_ATTRIBUTE void *calloc(size_t nmemb, size_t size) {
  return scudoCalloc(nmemb, size);
}

INTERCEPTOR_ATTRIBUTE void *valloc(size_t size) {
  return scudoValloc(size);
}

INTERCEPTOR_ATTRIBUTE
int posix_memalign(void **memptr, size_t alignment, size_t size) {
  return scudoPosixMemalign(memptr, alignment, size);
}

#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR_ATTRIBUTE void cfree(void *ptr) ALIAS("free");
#endif

#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR_ATTRIBUTE void *memalign(size_t alignment, size_t size) {
  return scudoAllocate(size, alignment, FromMemalign);
}

INTERCEPTOR_ATTRIBUTE
void *__libc_memalign(size_t alignment, size_t size) ALIAS("memalign");
#endif

#if SANITIZER_INTERCEPT_PVALLOC
INTERCEPTOR_ATTRIBUTE void *pvalloc(size_t size) {
  return scudoPvalloc(size);
}
#endif

#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR_ATTRIBUTE void *aligned_alloc(size_t alignment, size_t size) {
  return scudoAlignedAlloc(alignment, size);
}
#endif

#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE
INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) {
  return scudoMallocUsableSize(ptr);
}
#endif

#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) {
  return -1;
}
#endif
}  // extern "C"
<|endoftext|>


<commit_before>/******************************************************************************
 *                       ____    _    _____                                   *
 *                      / ___|  / \  |  ___|    C++                           *
 *                     | |     / _ \ | |_       Actor                         *
 *                     | |___ / ___ \|  _|      Framework                     *
 *                      \____/_/   \_|_|                                      *
 *                                                                            *
 * Copyright 2011-2018 Dominik Charousset                                     *
 *                                                                            *
 * Distributed under the terms and conditions of the BSD 3-Clause License or  *
 * (at your option) under the terms and conditions of the Boost Software      *
 * License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE.       *
 *                                                                            *
 * If you did not receive a copy of the license files, see                    *
 * http://opensource.org/licenses/BSD-3-Clause and                            *
 * http://www.boost.org/LICENSE_1_0.txt.                                      *
 ******************************************************************************/

#pragma once

// this header must be generated by the build system (may be empty)
#include "caf/detail/build_config.hpp"

// Platform-specific adjustments.
#define CAF_CACHE_LINE_SIZE 64

// Config pararameters defined by the build system (usually CMake):
//
// CAF_ENABLE_RUNTIME_CHECKS:
//   - check requirements at runtime
//
// CAF_LOG_LEVEL:
//   - denotes the amount of logging, ranging from error messages only (0)
//     to complete traces (4)

/// Denotes version of CAF in the format {MAJOR}{MINOR}{PATCH},
/// whereas each number is a two-digit decimal number without
/// leading zeros (e.g. 900 is version 0.9.0).
#define CAF_VERSION 1701

/// Defined to the major version number of CAF.
#define CAF_MAJOR_VERSION (CAF_VERSION / 10000)

/// Defined to the minor version number of CAF.
#define CAF_MINOR_VERSION ((CAF_VERSION / 100) % 100)

/// Defined to the patch version number of CAF.
#define CAF_PATCH_VERSION (CAF_VERSION % 100)

// This compiler-specific block defines:
// - CAF_DEPRECATED to annotate deprecated functions
// - CAF_PUSH_WARNINGS/CAF_POP_WARNINGS to surround "noisy" header includes
// - CAF_ANNOTATE_FALLTHROUGH to suppress warnings in switch/case statements
// - CAF_COMPILER_VERSION to retrieve the compiler version in CAF_VERSION format
// - One of the following:
//   + CAF_CLANG
//   + CAF_GCC
//   + CAF_MSVC

// sets CAF_DEPRECATED, CAF_ANNOTATE_FALLTHROUGH,
// CAF_PUSH_WARNINGS and CAF_POP_WARNINGS
#if defined(__clang__)
#  define CAF_CLANG
#  define CAF_LIKELY(x) __builtin_expect((x), 1)
#  define CAF_UNLIKELY(x) __builtin_expect((x), 0)
#  define CAF_DEPRECATED __attribute__((deprecated))
#  define CAF_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
#  define CAF_PUSH_WARNINGS                                                    \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wall\"")                              \
    _Pragma("clang diagnostic ignored \"-Wextra\"")                            \
    _Pragma("clang diagnostic ignored \"-Wundef\"")                            \
    _Pragma("clang diagnostic ignored \"-Wshadow\"")                           \
    _Pragma("clang diagnostic ignored \"-Wdeprecated\"")                       \
    _Pragma("clang diagnostic ignored \"-Wextra-semi\"")                       \
    _Pragma("clang diagnostic ignored \"-Wconversion\"")                       \
    _Pragma("clang diagnostic ignored \"-Wcast-align\"")                       \
    _Pragma("clang diagnostic ignored \"-Wfloat-equal\"")                      \
    _Pragma("clang diagnostic ignored \"-Wswitch-enum\"")                      \
    _Pragma("clang diagnostic ignored \"-Wweak-vtables\"")                     \
    _Pragma("clang diagnostic ignored \"-Wdocumentation\"")                    \
    _Pragma("clang diagnostic ignored \"-Wold-style-cast\"")                   \
    _Pragma("clang diagnostic ignored \"-Wsign-conversion\"")                  \
    _Pragma("clang diagnostic ignored \"-Wunused-template\"")                  \
    _Pragma("clang diagnostic ignored \"-Wshorten-64-to-32\"")                 \
    _Pragma("clang diagnostic ignored \"-Wunreachable-code\"")                 \
    _Pragma("clang diagnostic ignored \"-Wdouble-promotion\"")                 \
    _Pragma("clang diagnostic ignored \"-Wc++14-extensions\"")                 \
    _Pragma("clang diagnostic ignored \"-Wunused-parameter\"")                 \
    _Pragma("clang diagnostic ignored \"-Wnested-anon-types\"")                \
    _Pragma("clang diagnostic ignored \"-Wreserved-id-macro\"")                \
    _Pragma("clang diagnostic ignored \"-Wconstant-conversion\"")              \
    _Pragma("clang diagnostic ignored \"-Wimplicit-fallthrough\"")             \
    _Pragma("clang diagnostic ignored \"-Wused-but-marked-unused\"")           \
    _Pragma("clang diagnostic ignored \"-Wdisabled-macro-expansion\"")
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wunused-label\"")
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wnon-virtual-dtor\"")
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"")
#  define CAF_POP_WARNINGS                                                     \
    _Pragma("clang diagnostic pop")
#  define CAF_ANNOTATE_FALLTHROUGH [[clang::fallthrough]]
#  define CAF_COMPILER_VERSION                                                 \
    (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
#  if !__has_feature(cxx_thread_local)
#    define CAF_NO_THREAD_LOCAL
#  endif
#elif defined(__GNUC__)
#  define CAF_GCC
#  define CAF_LIKELY(x) __builtin_expect((x), 1)
#  define CAF_UNLIKELY(x) __builtin_expect((x), 0)
#  define CAF_DEPRECATED __attribute__((deprecated))
#  define CAF_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
#  define CAF_PUSH_WARNINGS                                                    \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wshadow\"")                             \
    _Pragma("GCC diagnostic ignored \"-Wpragmas\"")                            \
    _Pragma("GCC diagnostic ignored \"-Wpedantic\"")                           \
    _Pragma("GCC diagnostic ignored \"-Wcast-qual\"")                          \
    _Pragma("GCC diagnostic ignored \"-Wconversion\"")                         \
    _Pragma("GCC diagnostic ignored \"-Wfloat-equal\"")                        \
    _Pragma("GCC diagnostic ignored \"-Wc++14-extensions\"")
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wunused-label\"")
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wnon-virtual-dtor\"")
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
#  define CAF_POP_WARNINGS                                                     \
    _Pragma("GCC diagnostic pop")
#  if __GNUC__ >= 7
#    define CAF_ANNOTATE_FALLTHROUGH __attribute__((fallthrough))
#  else
#    define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#  endif
#  define CAF_COMPILER_VERSION                                                 \
     (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
    // disable thread_local on GCC/macOS due to heap-use-after-free bug:
    // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67135
#  ifdef __APPLE__
#    define CAF_NO_THREAD_LOCAL
#  endif
#elif defined(_MSC_VER)
#  define CAF_MSVC
#  define CAF_LIKELY(x) x
#  define CAF_UNLIKELY(x) x
#  define CAF_DEPRECATED
#  define CAF_DEPRECATED_MSG(msg)
#  define CAF_PUSH_WARNINGS                                                    \
    __pragma(warning(push))
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    __pragma(warning(push))                                                    \
    __pragma(warning(disable: 4102))
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    __pragma(warning(push))
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    __pragma(warning(push))
#  define CAF_POP_WARNINGS __pragma(warning(pop))
#  define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#  define CAF_COMPILER_VERSION _MSC_FULL_VER
#  pragma warning( disable : 4624 )
#  pragma warning( disable : 4800 )
#  pragma warning( disable : 4503 )
#  ifndef NOMINMAX
#    define NOMINMAX
#  endif // NOMINMAX
#else
#  define CAF_LIKELY(x) x
#  define CAF_UNLIKELY(x) x
#  define CAF_DEPRECATED
#  define CAF_PUSH_WARNINGS
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING
#  define CAF_PUSH_DEPRECATED_WARNING
#  define CAF_POP_WARNINGS
#  define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#endif

// This OS-specific block defines one of the following:
// - CAF_MACOS
// - CAF_LINUX
// - CAF_BSD
// - CAF_WINDOWS
// It also defines CAF_POSIX for POSIX-compatible systems
#if defined(__APPLE__)
#  include "TargetConditionals.h"
#  if TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR
#    define CAF_IOS
#  else
#    define CAF_MACOS
#    if defined(CAF_GCC) && !defined(_GLIBCXX_HAS_GTHREADS)
#      define _GLIBCXX_HAS_GTHREADS
#    endif
#  endif
#elif defined(__ANDROID__)
#  define CAF_ANDROID
#elif defined(__linux__)
#  define CAF_LINUX
#  include <linux/version.h>
#  if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,16)
#    define CAF_POLL_IMPL
#  endif
#elif defined(__FreeBSD__)
#  define CAF_BSD
#elif defined(__CYGWIN__)
#  define CAF_CYGWIN
#elif defined(WIN32) || defined(_WIN32)
#  define CAF_WINDOWS
#else
#  error Platform and/or compiler not supported
#endif
#if defined(CAF_MACOS) || defined(CAF_LINUX) || defined(CAF_BSD) || defined(CAF_CYGWIN)
#  define CAF_POSIX
#endif

#if defined(CAF_WINDOWS) && defined(CAF_CLANG)
// Fix for issue with static_cast<> in objbase.h.
// See: https://github.com/philsquared/Catch/issues/690.
struct IUnknown;
#endif

#include <cstdio>
#include <cstdlib>

// Optionally enable CAF_ASSERT
#ifndef CAF_ENABLE_RUNTIME_CHECKS
# define CAF_ASSERT(unused) static_cast<void>(0)
#elif defined(CAF_WINDOWS) || defined(CAF_BSD)
# define CAF_ASSERT(stmt)                                                      \
  if (static_cast<bool>(stmt) == false) {                                      \
    printf("%s:%u: requirement failed '%s'\n", __FILE__, __LINE__, #stmt);     \
    ::abort();                                                                 \
  } static_cast<void>(0)
#else // defined(CAF_LINUX) || defined(CAF_MACOS)
# include <execinfo.h>
# define CAF_ASSERT(stmt)                                                      \
  if (static_cast<bool>(stmt) == false) {                                      \
    printf("%s:%u: requirement failed '%s'\n", __FILE__, __LINE__, #stmt);     \
    void* array[20];                                                           \
    auto caf_bt_size = ::backtrace(array, 20);                                 \
    ::backtrace_symbols_fd(array, caf_bt_size, 2);                             \
    ::abort();                                                                 \
  } static_cast<void>(0)
#endif

// Convenience macros.
#define CAF_IGNORE_UNUSED(x) static_cast<void>(x)

#define CAF_CRITICAL(error)                                                    \
  do {                                                                         \
    fprintf(stderr, "[FATAL] %s:%u: critical error: '%s'\n",                   \
            __FILE__, __LINE__, error);                                        \
    ::abort();                                                                 \
  } while (false)
<commit_msg>Add macros for explicit symbol visibility<commit_after>/******************************************************************************
 *                       ____    _    _____                                   *
 *                      / ___|  / \  |  ___|    C++                           *
 *                     | |     / _ \ | |_       Actor                         *
 *                     | |___ / ___ \|  _|      Framework                     *
 *                      \____/_/   \_|_|                                      *
 *                                                                            *
 * Copyright 2011-2018 Dominik Charousset                                     *
 *                                                                            *
 * Distributed under the terms and conditions of the BSD 3-Clause License or  *
 * (at your option) under the terms and conditions of the Boost Software      *
 * License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE.       *
 *                                                                            *
 * If you did not receive a copy of the license files, see                    *
 * http://opensource.org/licenses/BSD-3-Clause and                            *
 * http://www.boost.org/LICENSE_1_0.txt.                                      *
 ******************************************************************************/

#pragma once

// this header must be generated by the build system (may be empty)
#include "caf/detail/build_config.hpp"

// Platform-specific adjustments.
#define CAF_CACHE_LINE_SIZE 64

// Config pararameters defined by the build system (usually CMake):
//
// CAF_ENABLE_RUNTIME_CHECKS:
//   - check requirements at runtime
//
// CAF_LOG_LEVEL:
//   - denotes the amount of logging, ranging from error messages only (0)
//     to complete traces (4)

/// Denotes version of CAF in the format {MAJOR}{MINOR}{PATCH},
/// whereas each number is a two-digit decimal number without
/// leading zeros (e.g. 900 is version 0.9.0).
#define CAF_VERSION 1701

/// Defined to the major version number of CAF.
#define CAF_MAJOR_VERSION (CAF_VERSION / 10000)

/// Defined to the minor version number of CAF.
#define CAF_MINOR_VERSION ((CAF_VERSION / 100) % 100)

/// Defined to the patch version number of CAF.
#define CAF_PATCH_VERSION (CAF_VERSION % 100)

// This compiler-specific block defines:
// - CAF_DEPRECATED to annotate deprecated functions
// - CAF_PUSH_WARNINGS/CAF_POP_WARNINGS to surround "noisy" header includes
// - CAF_ANNOTATE_FALLTHROUGH to suppress warnings in switch/case statements
// - CAF_COMPILER_VERSION to retrieve the compiler version in CAF_VERSION format
// - One of the following:
//   + CAF_CLANG
//   + CAF_GCC
//   + CAF_MSVC

// sets CAF_DEPRECATED, CAF_ANNOTATE_FALLTHROUGH,
// CAF_PUSH_WARNINGS and CAF_POP_WARNINGS
#if defined(__clang__)
#  define CAF_CLANG
#  define CAF_LIKELY(x) __builtin_expect((x), 1)
#  define CAF_UNLIKELY(x) __builtin_expect((x), 0)
#  define CAF_DEPRECATED __attribute__((deprecated))
#  define CAF_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
#  define CAF_PUSH_WARNINGS                                                    \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wall\"")                              \
    _Pragma("clang diagnostic ignored \"-Wextra\"")                            \
    _Pragma("clang diagnostic ignored \"-Wundef\"")                            \
    _Pragma("clang diagnostic ignored \"-Wshadow\"")                           \
    _Pragma("clang diagnostic ignored \"-Wdeprecated\"")                       \
    _Pragma("clang diagnostic ignored \"-Wextra-semi\"")                       \
    _Pragma("clang diagnostic ignored \"-Wconversion\"")                       \
    _Pragma("clang diagnostic ignored \"-Wcast-align\"")                       \
    _Pragma("clang diagnostic ignored \"-Wfloat-equal\"")                      \
    _Pragma("clang diagnostic ignored \"-Wswitch-enum\"")                      \
    _Pragma("clang diagnostic ignored \"-Wweak-vtables\"")                     \
    _Pragma("clang diagnostic ignored \"-Wdocumentation\"")                    \
    _Pragma("clang diagnostic ignored \"-Wold-style-cast\"")                   \
    _Pragma("clang diagnostic ignored \"-Wsign-conversion\"")                  \
    _Pragma("clang diagnostic ignored \"-Wunused-template\"")                  \
    _Pragma("clang diagnostic ignored \"-Wshorten-64-to-32\"")                 \
    _Pragma("clang diagnostic ignored \"-Wunreachable-code\"")                 \
    _Pragma("clang diagnostic ignored \"-Wdouble-promotion\"")                 \
    _Pragma("clang diagnostic ignored \"-Wc++14-extensions\"")                 \
    _Pragma("clang diagnostic ignored \"-Wunused-parameter\"")                 \
    _Pragma("clang diagnostic ignored \"-Wnested-anon-types\"")                \
    _Pragma("clang diagnostic ignored \"-Wreserved-id-macro\"")                \
    _Pragma("clang diagnostic ignored \"-Wconstant-conversion\"")              \
    _Pragma("clang diagnostic ignored \"-Wimplicit-fallthrough\"")             \
    _Pragma("clang diagnostic ignored \"-Wused-but-marked-unused\"")           \
    _Pragma("clang diagnostic ignored \"-Wdisabled-macro-expansion\"")
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wunused-label\"")
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wnon-virtual-dtor\"")
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    _Pragma("clang diagnostic push")                                           \
    _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"")
#  define CAF_POP_WARNINGS                                                     \
    _Pragma("clang diagnostic pop")
#  define CAF_ANNOTATE_FALLTHROUGH [[clang::fallthrough]]
#  define CAF_COMPILER_VERSION                                                 \
    (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
#  if !__has_feature(cxx_thread_local)
#    define CAF_NO_THREAD_LOCAL
#  endif
#elif defined(__GNUC__)
#  define CAF_GCC
#  define CAF_LIKELY(x) __builtin_expect((x), 1)
#  define CAF_UNLIKELY(x) __builtin_expect((x), 0)
#  define CAF_DEPRECATED __attribute__((deprecated))
#  define CAF_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
#  define CAF_PUSH_WARNINGS                                                    \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wshadow\"")                             \
    _Pragma("GCC diagnostic ignored \"-Wpragmas\"")                            \
    _Pragma("GCC diagnostic ignored \"-Wpedantic\"")                           \
    _Pragma("GCC diagnostic ignored \"-Wcast-qual\"")                          \
    _Pragma("GCC diagnostic ignored \"-Wconversion\"")                         \
    _Pragma("GCC diagnostic ignored \"-Wfloat-equal\"")                        \
    _Pragma("GCC diagnostic ignored \"-Wc++14-extensions\"")
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wunused-label\"")
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wnon-virtual-dtor\"")
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    _Pragma("GCC diagnostic push")                                             \
    _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
#  define CAF_POP_WARNINGS                                                     \
    _Pragma("GCC diagnostic pop")
#  if __GNUC__ >= 7
#    define CAF_ANNOTATE_FALLTHROUGH __attribute__((fallthrough))
#  else
#    define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#  endif
#  define CAF_COMPILER_VERSION                                                 \
     (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
    // disable thread_local on GCC/macOS due to heap-use-after-free bug:
    // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67135
#  ifdef __APPLE__
#    define CAF_NO_THREAD_LOCAL
#  endif
#elif defined(_MSC_VER)
#  define CAF_MSVC
#  define CAF_LIKELY(x) x
#  define CAF_UNLIKELY(x) x
#  define CAF_DEPRECATED
#  define CAF_DEPRECATED_MSG(msg)
#  define CAF_PUSH_WARNINGS                                                    \
    __pragma(warning(push))
#  define CAF_PUSH_UNUSED_LABEL_WARNING                                        \
    __pragma(warning(push))                                                    \
    __pragma(warning(disable: 4102))
#  define CAF_PUSH_DEPRECATED_WARNING                                          \
    __pragma(warning(push))
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING                                    \
    __pragma(warning(push))
#  define CAF_POP_WARNINGS __pragma(warning(pop))
#  define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#  define CAF_COMPILER_VERSION _MSC_FULL_VER
#  pragma warning( disable : 4624 )
#  pragma warning( disable : 4800 )
#  pragma warning( disable : 4503 )
#  ifndef NOMINMAX
#    define NOMINMAX
#  endif // NOMINMAX
#else
#  define CAF_LIKELY(x) x
#  define CAF_UNLIKELY(x) x
#  define CAF_DEPRECATED
#  define CAF_PUSH_WARNINGS
#  define CAF_PUSH_NON_VIRTUAL_DTOR_WARNING
#  define CAF_PUSH_DEPRECATED_WARNING
#  define CAF_POP_WARNINGS
#  define CAF_ANNOTATE_FALLTHROUGH static_cast<void>(0)
#endif

// This OS-specific block defines one of the following:
// - CAF_MACOS
// - CAF_LINUX
// - CAF_BSD
// - CAF_WINDOWS
// It also defines CAF_POSIX for POSIX-compatible systems
#if defined(__APPLE__)
#  include "TargetConditionals.h"
#  if TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR
#    define CAF_IOS
#  else
#    define CAF_MACOS
#    if defined(CAF_GCC) && !defined(_GLIBCXX_HAS_GTHREADS)
#      define _GLIBCXX_HAS_GTHREADS
#    endif
#  endif
#elif defined(__ANDROID__)
#  define CAF_ANDROID
#elif defined(__linux__)
#  define CAF_LINUX
#  include <linux/version.h>
#  if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,16)
#    define CAF_POLL_IMPL
#  endif
#elif defined(__FreeBSD__)
#  define CAF_BSD
#elif defined(__CYGWIN__)
#  define CAF_CYGWIN
#elif defined(WIN32) || defined(_WIN32)
#  define CAF_WINDOWS
#else
#  error Platform and/or compiler not supported
#endif
#if defined(CAF_MACOS) || defined(CAF_LINUX) || defined(CAF_BSD) || defined(CAF_CYGWIN)
#  define CAF_POSIX
#endif

#if defined(CAF_WINDOWS) && defined(CAF_CLANG)
// Fix for issue with static_cast<> in objbase.h.
// See: https://github.com/philsquared/Catch/issues/690.
struct IUnknown;
#endif

#include <cstdio>
#include <cstdlib>

// Optionally enable CAF_ASSERT
#ifndef CAF_ENABLE_RUNTIME_CHECKS
# define CAF_ASSERT(unused) static_cast<void>(0)
#elif defined(CAF_WINDOWS) || defined(CAF_BSD)
# define CAF_ASSERT(stmt)                                                      \
  if (static_cast<bool>(stmt) == false) {                                      \
    printf("%s:%u: requirement failed '%s'\n", __FILE__, __LINE__, #stmt);     \
    ::abort();                                                                 \
  } static_cast<void>(0)
#else // defined(CAF_LINUX) || defined(CAF_MACOS)
# include <execinfo.h>
# define CAF_ASSERT(stmt)                                                      \
  if (static_cast<bool>(stmt) == false) {                                      \
    printf("%s:%u: requirement failed '%s'\n", __FILE__, __LINE__, #stmt);     \
    void* array[20];                                                           \
    auto caf_bt_size = ::backtrace(array, 20);                                 \
    ::backtrace_symbols_fd(array, caf_bt_size, 2);                             \
    ::abort();                                                                 \
  } static_cast<void>(0)
#endif

// Convenience macros.
#define CAF_IGNORE_UNUSED(x) static_cast<void>(x)

#define CAF_CRITICAL(error)                                                    \
  do {                                                                         \
    fprintf(stderr, "[FATAL] %s:%u: critical error: '%s'\n",                   \
            __FILE__, __LINE__, error);                                        \
    ::abort();                                                                 \
  } while (false)

// Explicit symbol visibility macros.
#ifdef CAF_MSVC
#  define CAF_API_EXPORT __declspec(dllexport)
#  define CAF_API_IMPORT __declspec(dllimport)
#elif defined(CAF_CLANG) || defined(CAF_GCC)
#  define CAF_API_EXPORT __attribute__ ((visibility("default")))
#  define CAF_API_IMPORT
#else
#  define CAF_API_EXPORT
#  define CAF_API_IMPORT
#endif

#ifdef libcaf_core_shared_EXPORTS
#  define CAF_API CAF_API_EXPORT
#else
#  define CAF_API CAF_API_IMPORT
#endif

<|endoftext|>


<commit_before>/******************************************************************************
 *                       ____    _    _____                                   *
 *                      / ___|  / \  |  ___|    C++                           *
 *                     | |     / _ \ | |_       Actor                         *
 *                     | |___ / ___ \|  _|      Framework                     *
 *                      \____/_/   \_|_|                                      *
 *                                                                            *
 * Copyright (C) 2011 - 2015                                                  *
 * Dominik Charousset <dominik.charousset (at) haw-hamburg.de>                *
 *                                                                            *
 * Distributed under the terms and conditions of the BSD 3-Clause License or  *
 * (at your option) under the terms and conditions of the Boost Software      *
 * License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE.       *
 *                                                                            *
 * If you did not receive a copy of the license files, see                    *
 * http://opensource.org/licenses/BSD-3-Clause and                            *
 * http://www.boost.org/LICENSE_1_0.txt.                                      *
 ******************************************************************************/

#include "caf/config.hpp"

#define CAF_SUITE maybe
#include "caf/test/unit_test.hpp"

#include <string>

#include "caf/maybe.hpp"

using namespace std;
using namespace caf;

namespace {

struct qwertz {
  qwertz(int i, int j) : i_(i), j_(j) {
    // nop
  }
  int i_;
  int j_;
};

bool operator==(const qwertz& lhs, const qwertz& rhs) {
  return lhs.i_ == rhs.i_ && lhs.j_ == rhs.j_;
}

enum class test_errc : uint8_t {
  first_error = 1,
  second_error
};

error make_error(test_errc x) {
  return {static_cast<uint8_t>(x), atom("test_errc")};
}

} // namespace <anonymous>

CAF_TEST(empties) {
  maybe<int> i;
  maybe<int> j;
  CAF_CHECK(i == j);
  CAF_CHECK(!(i != j));
}

CAF_TEST(unequal) {
  maybe<int> i = 5;
  maybe<int> j = 6;
  CAF_CHECK(!(i == j));
  CAF_CHECK(i != j);
}

CAF_TEST(distinct_types) {
  maybe<int> i;
  maybe<double> j;
  CAF_CHECK(i == j);
  CAF_CHECK(!(i != j));
}

CAF_TEST(custom_type_none) {
  maybe<qwertz> i;
  CAF_CHECK(i == none);
}

CAF_TEST(custom_type_engaged) {
  qwertz obj{1, 2};
  maybe<qwertz> j = obj;
  CAF_CHECK(j != none);
  CAF_CHECK(obj == j);
  CAF_CHECK(j == obj );
  CAF_CHECK(obj == *j);
  CAF_CHECK(*j == obj);
}

CAF_TEST(error_construct_and_assign) {
  auto f = []() -> maybe<int> {
    return test_errc::second_error;
  };
  auto val = f();
  CAF_CHECK(! val && val.error() == test_errc::second_error);
  val = 42;
  CAF_CHECK(val && *val == 42);
  val = test_errc::first_error;
  CAF_CHECK(! val && val.error() == test_errc::first_error);
}

CAF_TEST(maybe_void) {
  maybe<void> m;
  CAF_CHECK(! m);
  CAF_CHECK(m.empty());
  CAF_CHECK(! m.error());
  // Assign erroneous state.
  m = test_errc::second_error;
  CAF_CHECK(! m);
  CAF_CHECK(! m.empty());
  CAF_CHECK(m.error());
  CAF_CHECK(m.error() == test_errc::second_error);
  // Implicit construction.
  auto f = []() -> maybe<void> {
    return test_errc::second_error;
  };
  auto val = f();
  CAF_CHECK(! val && val.error() == test_errc::second_error);
}
<commit_msg>Remove now superfluous unit tests for maybe<T><commit_after><|endoftext|>


<commit_before>/*
    Copyright (c) 2008 Volker Krause <vkrause@kde.org>

    This library is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public License as published by
    the Free Software Foundation; either version 2 of the License, or (at your
    option) any later version.

    This library is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
    License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to the
    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
*/

#include "uistatesaver.h"

#include <KConfigGroup>
#include <KDebug>

#include <QComboBox>
#include <QHeaderView>
#include <QSplitter>
#include <QTabWidget>
#include <QTreeView>

using namespace KPIM;

struct Saver {
  static void process( QSplitter *splitter, KConfigGroup &config )
  {
    if ( splitter->sizes().count( 0 ) == splitter->sizes().count() )
      return;
    config.writeEntry( splitter->objectName(), splitter->sizes() );
  }

  static void process( QTabWidget *tab, KConfigGroup &config )
  {
    config.writeEntry( tab->objectName(), tab->currentIndex() );
  }

  static void process( QTreeView *tv, KConfigGroup &config )
  {
    config.writeEntry( tv->objectName(), tv->header()->saveState() );
  }

  static void process( QComboBox *cb, KConfigGroup &config )
  {
    config.writeEntry( cb->objectName(), cb->currentIndex() );
  }
};

struct Restorer {
  static void process( QSplitter *splitter, const KConfigGroup &config )
  {
    const QList<int> sizes = config.readEntry( splitter->objectName(), QList<int>() );
    if ( !sizes.isEmpty() && splitter->count() == sizes.count() && splitter->count() != splitter->sizes().count( 0 ) )
      splitter->setSizes( sizes );
  }

  static void process( QTabWidget *tab, const KConfigGroup &config )
  {
    const int index = config.readEntry( tab->objectName(), -1 );
    if ( index >= 0 && index < tab->count() )
      tab->setCurrentIndex( index );
  }

  static void process( QTreeView *tv, const KConfigGroup &config )
  {
    const QByteArray state = config.readEntry( tv->objectName(), QByteArray() );
    if ( !state.isEmpty() )
      tv->header()->restoreState( state );
  }

  static void process( QComboBox *cb, const KConfigGroup &config )
  {
    const int index = config.readEntry( cb->objectName(), -1 );
    if ( index >= 0 && index < cb->count() )
      cb->setCurrentIndex( index );
  }
};

#define PROCESS_TYPE( T ) \
{ \
  T *obj = qobject_cast<T*>( w ); \
  if ( obj ) { \
    Op::process( obj, config ); \
    continue; \
  } \
}

template <typename Op, typename Config> static void processWidgets( QWidget *widget, Config config )
{
  QList<QWidget*> widgets = widget->findChildren<QWidget*>();
  widgets << widget;
  foreach ( QWidget* w, widgets ) {
    if ( w->objectName().isEmpty() )
      continue;
    PROCESS_TYPE( QSplitter );
    PROCESS_TYPE( QTabWidget );
    PROCESS_TYPE( QTreeView );
    PROCESS_TYPE( QComboBox );
  }
}

#undef PROCESS_TYPE

void UiStateSaver::saveState(QWidget * widget, KConfigGroup & config)
{
  processWidgets<Saver, KConfigGroup&>( widget, config );
}

void UiStateSaver::restoreState(QWidget * widget, const KConfigGroup & config)
{
  processWidgets<Restorer, const KConfigGroup&>( widget, config );
}
<commit_msg>Don't blindly copy and paste...<commit_after>/*
    Copyright (c) 2008 Volker Krause <vkrause@kde.org>

    This library is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public License as published by
    the Free Software Foundation; either version 2 of the License, or (at your
    option) any later version.

    This library is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
    License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to the
    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
*/

#include "uistatesaver.h"

#include <KConfigGroup>
#include <KDebug>

#include <QComboBox>
#include <QHeaderView>
#include <QSplitter>
#include <QTabWidget>
#include <QTreeView>

using namespace KPIM;

struct Saver {
  static void process( QSplitter *splitter, KConfigGroup &config )
  {
    if ( splitter->sizes().count( 0 ) == splitter->sizes().count() )
      return;
    config.writeEntry( splitter->objectName(), splitter->sizes() );
  }

  static void process( QTabWidget *tab, KConfigGroup &config )
  {
    config.writeEntry( tab->objectName(), tab->currentIndex() );
  }

  static void process( QTreeView *tv, KConfigGroup &config )
  {
    config.writeEntry( tv->objectName(), tv->header()->saveState() );
  }

  static void process( QComboBox *cb, KConfigGroup &config )
  {
    config.writeEntry( cb->objectName(), cb->currentIndex() );
  }
};

struct Restorer {
  static void process( QSplitter *splitter, const KConfigGroup &config )
  {
    const QList<int> sizes = config.readEntry( splitter->objectName(), QList<int>() );
    if ( !sizes.isEmpty() && splitter->count() == sizes.count() && sizes.count() != sizes.count( 0 ) )
      splitter->setSizes( sizes );
  }

  static void process( QTabWidget *tab, const KConfigGroup &config )
  {
    const int index = config.readEntry( tab->objectName(), -1 );
    if ( index >= 0 && index < tab->count() )
      tab->setCurrentIndex( index );
  }

  static void process( QTreeView *tv, const KConfigGroup &config )
  {
    const QByteArray state = config.readEntry( tv->objectName(), QByteArray() );
    if ( !state.isEmpty() )
      tv->header()->restoreState( state );
  }

  static void process( QComboBox *cb, const KConfigGroup &config )
  {
    const int index = config.readEntry( cb->objectName(), -1 );
    if ( index >= 0 && index < cb->count() )
      cb->setCurrentIndex( index );
  }
};

#define PROCESS_TYPE( T ) \
{ \
  T *obj = qobject_cast<T*>( w ); \
  if ( obj ) { \
    Op::process( obj, config ); \
    continue; \
  } \
}

template <typename Op, typename Config> static void processWidgets( QWidget *widget, Config config )
{
  QList<QWidget*> widgets = widget->findChildren<QWidget*>();
  widgets << widget;
  foreach ( QWidget* w, widgets ) {
    if ( w->objectName().isEmpty() )
      continue;
    PROCESS_TYPE( QSplitter );
    PROCESS_TYPE( QTabWidget );
    PROCESS_TYPE( QTreeView );
    PROCESS_TYPE( QComboBox );
  }
}

#undef PROCESS_TYPE

void UiStateSaver::saveState(QWidget * widget, KConfigGroup & config)
{
  processWidgets<Saver, KConfigGroup&>( widget, config );
}

void UiStateSaver::restoreState(QWidget * widget, const KConfigGroup & config)
{
  processWidgets<Restorer, const KConfigGroup&>( widget, config );
}
<|endoftext|>


<commit_before>/*
 *  Udp.cpp: Library to send/receive UDP packets with the Arduino ethernet shield.
 *  This version only offers minimal wrapping of socket.c/socket.h
 *  Drop Udp.h/.cpp into the Ethernet library directory at hardware/libraries/Ethernet/
 *
 * MIT License:
 * Copyright (c) 2008 Bjoern Hartmann
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * bjoern@cs.stanford.edu 12/30/2008
 */

#include "w5100.h"
#include "socket.h"
#include "ethernet.h"
#include "udp.h"

/* Start UDP socket, listening at local port PORT */
void UdpClass::begin(uint16_t port) {
  _port = port;
  _sock = 0; //TODO: should not be hardcoded
  socket(_sock, SnMR::UDP, _port, 0);
}

/* Send packet contained in buf of length len to peer at specified ip, and port */
/* Use this function to transmit binary data that might contain 0x00 bytes*/
/* This function returns sent data size for success else -1. */
uint16_t UdpClass::sendPacket(uint8_t * buf, uint16_t len,  uint8_t * ip, uint16_t port){
  return sendto(_sock,(const uint8_t *)buf,len,ip,port);
}

/* Send  zero-terminated string str as packet to peer at specified ip, and port */
/* This function returns sent data size for success else -1. */
uint16_t UdpClass::sendPacket(const char str[], uint8_t * ip, uint16_t port){
  // compute strlen
  const char *s;
  for(s = str; *s; ++s);
  uint16_t len = (s-str);
  // send packet
  return sendto(_sock,(const uint8_t *)str,len,ip,port);
}
/* Is data available in rx buffer? Returns 0 if no, number of available bytes if yes.
 * returned value includes 8 byte UDP header!*/
int UdpClass::available() {
  return W5100.getRXReceivedSize(_sock);
}

/* Read a received packet into buffer buf (which is of maximum length len); */
/* store calling ip and port as well. Call available() to make sure data is ready first. */
/* NOTE: I don't believe len is ever checked in implementation of recvfrom(),*/
/*       so it's easy to overflow buffer. so we check and truncate. */
/* returns number of bytes read, or negative number of bytes we would have needed if we truncated */
int UdpClass::readPacket(uint8_t * buf, uint16_t bufLen, uint8_t *ip, uint16_t *port) {
  int packetLen = available()-8; //skip UDP header;
  if(packetLen < 0 ) return 0; // no real data here
  if(packetLen > (int)bufLen) {
    //packet is too large - truncate
    //HACK - hand-parse the UDP packet using TCP recv method
    uint8_t tmpBuf[8];
    int i;
    //read 8 header bytes and get IP and port from it
    recv(_sock,tmpBuf,8);
    ip[0] = tmpBuf[0];
    ip[1] = tmpBuf[1];
    ip[2] = tmpBuf[2];
    ip[3] = tmpBuf[3];
    *port = tmpBuf[4];
    *port = (*port << 8) + tmpBuf[5];

    //now copy first (bufLen) bytes into buf
    for(i=0;i<(int)bufLen;i++) {
      recv(_sock,tmpBuf,1);
      buf[i]=tmpBuf[0];
    }

    //and just read the rest byte by byte and throw it away
    while(available()) {
      recv(_sock,tmpBuf,1);
    }

    return (-1*packetLen);

    //ALTERNATIVE: requires stdlib - takes a bunch of space
    /*//create new buffer and read everything into it
     		uint8_t * tmpBuf = (uint8_t *)malloc(packetLen);
     		recvfrom(_sock,tmpBuf,packetLen,ip,port);
     		if(!tmpBuf) return 0; //couldn't allocate
     		// copy first bufLen bytes
     		for(unsigned int i=0; i<bufLen; i++) {
     			buf[i]=tmpBuf[i];
     		}
     		//free temp buffer
     		free(tmpBuf);
     		*/

  }
  return recvfrom(_sock,buf,bufLen,ip,port);
}

/* Read a received packet, throw away peer's ip and port.  See note above. */
int UdpClass::readPacket(uint8_t * buf, uint16_t len) {
  uint8_t ip[4];
  uint16_t port[1];
  return recvfrom(_sock,buf,len,ip,port);
}

/* Create one global object */
UdpClass Udp;
<commit_msg>Correcting case of #include's in Ethernet library.<commit_after>/*
 *  Udp.cpp: Library to send/receive UDP packets with the Arduino ethernet shield.
 *  This version only offers minimal wrapping of socket.c/socket.h
 *  Drop Udp.h/.cpp into the Ethernet library directory at hardware/libraries/Ethernet/
 *
 * MIT License:
 * Copyright (c) 2008 Bjoern Hartmann
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * bjoern@cs.stanford.edu 12/30/2008
 */

#include "w5100.h"
#include "socket.h"
#include "Ethernet.h"
#include "Udp.h"

/* Start UDP socket, listening at local port PORT */
void UdpClass::begin(uint16_t port) {
  _port = port;
  _sock = 0; //TODO: should not be hardcoded
  socket(_sock, SnMR::UDP, _port, 0);
}

/* Send packet contained in buf of length len to peer at specified ip, and port */
/* Use this function to transmit binary data that might contain 0x00 bytes*/
/* This function returns sent data size for success else -1. */
uint16_t UdpClass::sendPacket(uint8_t * buf, uint16_t len,  uint8_t * ip, uint16_t port){
  return sendto(_sock,(const uint8_t *)buf,len,ip,port);
}

/* Send  zero-terminated string str as packet to peer at specified ip, and port */
/* This function returns sent data size for success else -1. */
uint16_t UdpClass::sendPacket(const char str[], uint8_t * ip, uint16_t port){
  // compute strlen
  const char *s;
  for(s = str; *s; ++s);
  uint16_t len = (s-str);
  // send packet
  return sendto(_sock,(const uint8_t *)str,len,ip,port);
}
/* Is data available in rx buffer? Returns 0 if no, number of available bytes if yes.
 * returned value includes 8 byte UDP header!*/
int UdpClass::available() {
  return W5100.getRXReceivedSize(_sock);
}

/* Read a received packet into buffer buf (which is of maximum length len); */
/* store calling ip and port as well. Call available() to make sure data is ready first. */
/* NOTE: I don't believe len is ever checked in implementation of recvfrom(),*/
/*       so it's easy to overflow buffer. so we check and truncate. */
/* returns number of bytes read, or negative number of bytes we would have needed if we truncated */
int UdpClass::readPacket(uint8_t * buf, uint16_t bufLen, uint8_t *ip, uint16_t *port) {
  int packetLen = available()-8; //skip UDP header;
  if(packetLen < 0 ) return 0; // no real data here
  if(packetLen > (int)bufLen) {
    //packet is too large - truncate
    //HACK - hand-parse the UDP packet using TCP recv method
    uint8_t tmpBuf[8];
    int i;
    //read 8 header bytes and get IP and port from it
    recv(_sock,tmpBuf,8);
    ip[0] = tmpBuf[0];
    ip[1] = tmpBuf[1];
    ip[2] = tmpBuf[2];
    ip[3] = tmpBuf[3];
    *port = tmpBuf[4];
    *port = (*port << 8) + tmpBuf[5];

    //now copy first (bufLen) bytes into buf
    for(i=0;i<(int)bufLen;i++) {
      recv(_sock,tmpBuf,1);
      buf[i]=tmpBuf[0];
    }

    //and just read the rest byte by byte and throw it away
    while(available()) {
      recv(_sock,tmpBuf,1);
    }

    return (-1*packetLen);

    //ALTERNATIVE: requires stdlib - takes a bunch of space
    /*//create new buffer and read everything into it
     		uint8_t * tmpBuf = (uint8_t *)malloc(packetLen);
     		recvfrom(_sock,tmpBuf,packetLen,ip,port);
     		if(!tmpBuf) return 0; //couldn't allocate
     		// copy first bufLen bytes
     		for(unsigned int i=0; i<bufLen; i++) {
     			buf[i]=tmpBuf[i];
     		}
     		//free temp buffer
     		free(tmpBuf);
     		*/

  }
  return recvfrom(_sock,buf,bufLen,ip,port);
}

/* Read a received packet, throw away peer's ip and port.  See note above. */
int UdpClass::readPacket(uint8_t * buf, uint16_t len) {
  uint8_t ip[4];
  uint16_t port[1];
  return recvfrom(_sock,buf,len,ip,port);
}

/* Create one global object */
UdpClass Udp;
<|endoftext|>


<commit_before>/*
   Copyright 2014 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

#include "libsyndicate/ms/volume.h"
#include "libsyndicate/ms/url.h"

// free a volume
void ms_volume_free( struct ms_volume* vol ) {
   if( vol == NULL ) {
      return;
   }

   dbprintf("Destroy Volume '%s'\n", vol->name );

   if( vol->volume_public_key ) {
      EVP_PKEY_free( vol->volume_public_key );
      vol->volume_public_key = NULL;
   }

   ms_cert_bundle* all_certs[MS_NUM_CERT_BUNDLES+1];
   ms_client_cert_bundles( vol, all_certs );

   for( int i = 1; all_certs[i] != NULL; i++ ) {
      ms_cert_bundle* certs = all_certs[i];

      for( ms_cert_bundle::iterator itr = certs->begin(); itr != certs->end(); itr++ ) {
         ms_client_gateway_cert_free( itr->second );
         free( itr->second );
      }
   }

   delete vol->UG_certs;
   delete vol->RG_certs;
   delete vol->AG_certs;

   vol->UG_certs = NULL;
   vol->RG_certs = NULL;
   vol->AG_certs = NULL;

   if( vol->name ) {
      free( vol->name );
      vol->name = NULL;
   }

   if( vol->root ) {
      md_entry_free( vol->root );
      free( vol->root );
   }

   if( vol->cache_closure ) {
      md_closure_shutdown( vol->cache_closure );
      free( vol->cache_closure );
   }

   memset( vol, 0, sizeof(struct ms_volume) );
}

// synchronously download a volume's metadata by name
// on success, populate the given ms_volume structure with the metadata
int ms_client_download_volume_by_name( struct ms_client* client, char const* volume_name, struct ms_volume* vol, char const* volume_pubkey_pem ) {
   ms::ms_volume_metadata volume_md;
   char* buf = NULL;
   size_t len = 0;
   int rc = 0;
   int http_status = 0;

   char* volume_url = ms_client_volume_url_by_name( client->url, volume_name );

   http_status = ms_client_download( client, volume_url, &buf, &len );

   free( volume_url );

   if( http_status != 200 ) {
      errorf("ms_client_download(%s) rc = %d\n", volume_url, http_status );

      return -abs(http_status);
   }

   // extract the message
   bool valid = volume_md.ParseFromString( string(buf, len) );
   free( buf );

   if( !valid ) {
      errorf("Invalid data for Volume %s\n", volume_name );
      return -EINVAL;
   }

   rc = ms_client_volume_init( vol, &volume_md, volume_pubkey_pem, client->conf, client->my_pubkey, client->my_key );
   if( rc != 0 ) {
      errorf("ms_client_volume_init rc = %d\n", rc );
      return rc;
   }

   return 0;
}

// reload volume metadata
// * download new volume metadata, and load it in
// * if the cert version has changed, then reload the certs as well
// client must NOT be locked.
int ms_client_reload_volume( struct ms_client* client ) {

   int rc = 0;
   ms::ms_volume_metadata volume_md;
   char* buf = NULL;
   size_t len = 0;
   int http_status = 0;

   ms_client_view_rlock( client );

   struct ms_volume* vol = client->volume;

   if( vol == NULL ) {
      errorf("%s", "ERR: unbound from Volume!\n" );
      ms_client_view_unlock( client );
      return -ENOENT;
   }

   // get the Volume ID for later
   uint64_t volume_id = vol->volume_id;

   char* volume_url = ms_client_volume_url( client->url, vol->volume_id );

   ms_client_view_unlock( client );

   http_status = ms_client_download( client, volume_url, &buf, &len );

   free( volume_url );

   if( http_status != 200 ) {
      errorf("ms_client_download(%s) rc = %d\n", volume_url,http_status );

      return -abs(http_status);
   }

   // extract the message
   bool valid = volume_md.ParseFromString( string(buf, len) );
   free( buf );

   if( !valid ) {
      errorf("Invalid data for Volume %" PRIu64 "\n", volume_id );
      return -EINVAL;
   }

   ms_client_view_wlock( client );

   // re-find the Volume
   vol = client->volume;
   if( vol == NULL ) {
      errorf("%s", "ERR: unbound from Volume!" );
      ms_client_view_unlock( client );
      return -ENOENT;
   }

   uint64_t old_version = vol->volume_version;
   uint64_t old_cert_version = vol->volume_cert_version;

   // get the new versions, and make sure they've advanced.
   uint64_t new_version = volume_md.volume_version();
   uint64_t new_cert_version = volume_md.cert_version();

   if( new_version < old_version ) {
      errorf("Invalid volume version (expected greater than %" PRIu64 ", got %" PRIu64 ")\n", old_version, new_version );
      ms_client_view_unlock( client );
      return -ENOTCONN;
   }

   if( new_cert_version < old_cert_version ) {
      errorf("Invalid certificate version (expected greater than %" PRIu64 ", got %" PRIu64 ")\n", old_cert_version, new_cert_version );
      ms_client_view_unlock( client );
      return -ENOTCONN;
   }

   if( new_version > old_version ) {
      // have new data--load it in
      rc = ms_client_volume_init( vol, &volume_md, NULL, client->conf, client->my_pubkey, client->my_key );
   }
   else {
      rc = 0;
   }

   ms_client_view_unlock( client );

   if( rc != 0 ) {
      errorf("ms_client_volume_init(%" PRIu64 ") rc = %d\n", volume_id, rc );
      return rc;
   }

   // do we need to download the UGs and/or RGs as well?
   dbprintf("Volume  version %" PRIu64 " --> %" PRIu64 "\n", old_version, new_version );
   dbprintf("Cert    version %" PRIu64 " --> %" PRIu64 "\n", old_cert_version, new_cert_version );

   // load new certificate information, if we have any
   if( new_cert_version > old_cert_version ) {
      rc = ms_client_reload_certs( client, new_cert_version );
      if( rc != 0 ) {
         errorf("ms_client_reload_certs rc = %d\n", rc );

         return rc;
      }
   }
   return 0;
}

// populate a Volume structure with the volume metadata
int ms_client_volume_init( struct ms_volume* vol, ms::ms_volume_metadata* volume_md, char const* volume_pubkey_pem, struct md_syndicate_conf* conf, EVP_PKEY* gateway_pubkey, EVP_PKEY* gateway_privkey ) {

   int rc = 0;

   // get the new public key
   if( vol->reload_volume_key || vol->volume_public_key == NULL || volume_pubkey_pem != NULL ) {
      vol->reload_volume_key = false;

      // trust it this time, but not in the future
      if( volume_pubkey_pem != NULL ) {
         rc = md_load_pubkey( &vol->volume_public_key, volume_pubkey_pem );
      }
      else {
         rc = md_load_pubkey( &vol->volume_public_key, volume_md->volume_public_key().c_str() );
      }

      if( rc != 0 ) {
         errorf("md_load_pubkey rc = %d\n", rc );
         return -ENOTCONN;
      }
   }

   if( vol->volume_public_key == NULL ) {
      errorf("%s", "unable to verify integrity of metadata for Volume!  No public key given!\n");
      return -ENOTCONN;
   }

   // verify metadata
   rc = md_verify<ms::ms_volume_metadata>( vol->volume_public_key, volume_md );
   if( rc != 0 ) {
      errorf("Signature verification failed on Volume %s (%" PRIu64 "), rc = %d\n", volume_md->name().c_str(), volume_md->volume_id(), rc );
      return rc;
   }

   // sanity check
   if( vol->name ) {
      char* new_name = strdup( volume_md->name().c_str() );
      if( strcmp( new_name, vol->name ) != 0 ) {
         errorf("Invalid Volume metadata: tried to change name from '%s' to '%s'\n", vol->name, new_name );
         free( new_name );
         return -EINVAL;
      }
      free( new_name );
   }

   struct md_entry* root = NULL;

   if( volume_md->has_root() ) {
      root = CALLOC_LIST( struct md_entry, 1 );
      ms_entry_to_md_entry( volume_md->root(), root );
   }

   if( vol->root ) {
      md_entry_free( vol->root );
      free( vol->root );
   }

   vol->root = root;

   if( vol->UG_certs == NULL ) {
      vol->UG_certs = new ms_cert_bundle();
   }

   if( vol->RG_certs == NULL ) {
      vol->RG_certs = new ms_cert_bundle();
   }

   if( vol->AG_certs == NULL ) {
      vol->AG_certs = new ms_cert_bundle();
   }

   vol->volume_cert_version = volume_md->cert_version();
   vol->volume_id = volume_md->volume_id();
   vol->volume_owner_id = volume_md->owner_id();
   vol->blocksize = volume_md->blocksize();
   vol->volume_version = volume_md->volume_version();
   vol->num_files = volume_md->num_files();

   if( vol->name == NULL ) {
      vol->name = strdup( volume_md->name().c_str() );
   }

   vol->cache_closure = NULL;

   if( volume_md->has_cache_closure_text() ) {
      char const* method = NULL;

      if( vol->cache_closure == NULL ) {
         method = "md_closure_init";

         vol->cache_closure = CALLOC_LIST( struct md_closure, 1 );
         rc = md_closure_init( vol->cache_closure, conf, gateway_pubkey, gateway_privkey,
                               MS_CLIENT_CACHE_CLOSURE_PROTOTYPE,
                               volume_md->cache_closure_text().data(), volume_md->cache_closure_text().size(),
                               false, false );
      }
      else {
         method = "md_closure_reload";

         rc = md_closure_reload( vol->cache_closure, conf, gateway_pubkey, gateway_privkey, volume_md->cache_closure_text().data(), volume_md->cache_closure_text().size() );
      }

      if( rc != 0 ) {
         errorf("%s rc = %d\n", method, rc );
         return rc;
      }
      else {
         dbprintf("(Re)initialized CDN closure %p for Volume %s\n", vol->cache_closure, vol->name );
      }
   }
   else {
      errorf("WARN: no CDN closure for Volume %s\n", vol->name );
   }
   return 0;
}

<commit_msg>Don't track the number of files in a volume<commit_after>/*
   Copyright 2014 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

#include "libsyndicate/ms/volume.h"
#include "libsyndicate/ms/url.h"

// free a volume
void ms_volume_free( struct ms_volume* vol ) {
   if( vol == NULL ) {
      return;
   }

   dbprintf("Destroy Volume '%s'\n", vol->name );

   if( vol->volume_public_key ) {
      EVP_PKEY_free( vol->volume_public_key );
      vol->volume_public_key = NULL;
   }

   ms_cert_bundle* all_certs[MS_NUM_CERT_BUNDLES+1];
   ms_client_cert_bundles( vol, all_certs );

   for( int i = 1; all_certs[i] != NULL; i++ ) {
      ms_cert_bundle* certs = all_certs[i];

      for( ms_cert_bundle::iterator itr = certs->begin(); itr != certs->end(); itr++ ) {
         ms_client_gateway_cert_free( itr->second );
         free( itr->second );
      }
   }

   delete vol->UG_certs;
   delete vol->RG_certs;
   delete vol->AG_certs;

   vol->UG_certs = NULL;
   vol->RG_certs = NULL;
   vol->AG_certs = NULL;

   if( vol->name ) {
      free( vol->name );
      vol->name = NULL;
   }

   if( vol->root ) {
      md_entry_free( vol->root );
      free( vol->root );
   }

   if( vol->cache_closure ) {
      md_closure_shutdown( vol->cache_closure );
      free( vol->cache_closure );
   }

   memset( vol, 0, sizeof(struct ms_volume) );
}

// synchronously download a volume's metadata by name
// on success, populate the given ms_volume structure with the metadata
int ms_client_download_volume_by_name( struct ms_client* client, char const* volume_name, struct ms_volume* vol, char const* volume_pubkey_pem ) {
   ms::ms_volume_metadata volume_md;
   char* buf = NULL;
   size_t len = 0;
   int rc = 0;
   int http_status = 0;

   char* volume_url = ms_client_volume_url_by_name( client->url, volume_name );

   http_status = ms_client_download( client, volume_url, &buf, &len );

   free( volume_url );

   if( http_status != 200 ) {
      errorf("ms_client_download(%s) rc = %d\n", volume_url, http_status );

      return -abs(http_status);
   }

   // extract the message
   bool valid = volume_md.ParseFromString( string(buf, len) );
   free( buf );

   if( !valid ) {
      errorf("Invalid data for Volume %s\n", volume_name );
      return -EINVAL;
   }

   rc = ms_client_volume_init( vol, &volume_md, volume_pubkey_pem, client->conf, client->my_pubkey, client->my_key );
   if( rc != 0 ) {
      errorf("ms_client_volume_init rc = %d\n", rc );
      return rc;
   }

   return 0;
}

// reload volume metadata
// * download new volume metadata, and load it in
// * if the cert version has changed, then reload the certs as well
// client must NOT be locked.
int ms_client_reload_volume( struct ms_client* client ) {

   int rc = 0;
   ms::ms_volume_metadata volume_md;
   char* buf = NULL;
   size_t len = 0;
   int http_status = 0;

   ms_client_view_rlock( client );

   struct ms_volume* vol = client->volume;

   if( vol == NULL ) {
      errorf("%s", "ERR: unbound from Volume!\n" );
      ms_client_view_unlock( client );
      return -ENOENT;
   }

   // get the Volume ID for later
   uint64_t volume_id = vol->volume_id;

   char* volume_url = ms_client_volume_url( client->url, vol->volume_id );

   ms_client_view_unlock( client );

   http_status = ms_client_download( client, volume_url, &buf, &len );

   free( volume_url );

   if( http_status != 200 ) {
      errorf("ms_client_download(%s) rc = %d\n", volume_url,http_status );

      return -abs(http_status);
   }

   // extract the message
   bool valid = volume_md.ParseFromString( string(buf, len) );
   free( buf );

   if( !valid ) {
      errorf("Invalid data for Volume %" PRIu64 "\n", volume_id );
      return -EINVAL;
   }

   ms_client_view_wlock( client );

   // re-find the Volume
   vol = client->volume;
   if( vol == NULL ) {
      errorf("%s", "ERR: unbound from Volume!" );
      ms_client_view_unlock( client );
      return -ENOENT;
   }

   uint64_t old_version = vol->volume_version;
   uint64_t old_cert_version = vol->volume_cert_version;

   // get the new versions, and make sure they've advanced.
   uint64_t new_version = volume_md.volume_version();
   uint64_t new_cert_version = volume_md.cert_version();

   if( new_version < old_version ) {
      errorf("Invalid volume version (expected greater than %" PRIu64 ", got %" PRIu64 ")\n", old_version, new_version );
      ms_client_view_unlock( client );
      return -ENOTCONN;
   }

   if( new_cert_version < old_cert_version ) {
      errorf("Invalid certificate version (expected greater than %" PRIu64 ", got %" PRIu64 ")\n", old_cert_version, new_cert_version );
      ms_client_view_unlock( client );
      return -ENOTCONN;
   }

   if( new_version > old_version ) {
      // have new data--load it in
      rc = ms_client_volume_init( vol, &volume_md, NULL, client->conf, client->my_pubkey, client->my_key );
   }
   else {
      rc = 0;
   }

   ms_client_view_unlock( client );

   if( rc != 0 ) {
      errorf("ms_client_volume_init(%" PRIu64 ") rc = %d\n", volume_id, rc );
      return rc;
   }

   // do we need to download the UGs and/or RGs as well?
   dbprintf("Volume  version %" PRIu64 " --> %" PRIu64 "\n", old_version, new_version );
   dbprintf("Cert    version %" PRIu64 " --> %" PRIu64 "\n", old_cert_version, new_cert_version );

   // load new certificate information, if we have any
   if( new_cert_version > old_cert_version ) {
      rc = ms_client_reload_certs( client, new_cert_version );
      if( rc != 0 ) {
         errorf("ms_client_reload_certs rc = %d\n", rc );

         return rc;
      }
   }
   return 0;
}

// populate a Volume structure with the volume metadata
int ms_client_volume_init( struct ms_volume* vol, ms::ms_volume_metadata* volume_md, char const* volume_pubkey_pem, struct md_syndicate_conf* conf, EVP_PKEY* gateway_pubkey, EVP_PKEY* gateway_privkey ) {

   int rc = 0;

   // get the new public key
   if( vol->reload_volume_key || vol->volume_public_key == NULL || volume_pubkey_pem != NULL ) {
      vol->reload_volume_key = false;

      // trust it this time, but not in the future
      if( volume_pubkey_pem != NULL ) {
         rc = md_load_pubkey( &vol->volume_public_key, volume_pubkey_pem );
      }
      else {
         rc = md_load_pubkey( &vol->volume_public_key, volume_md->volume_public_key().c_str() );
      }

      if( rc != 0 ) {
         errorf("md_load_pubkey rc = %d\n", rc );
         return -ENOTCONN;
      }
   }

   if( vol->volume_public_key == NULL ) {
      errorf("%s", "unable to verify integrity of metadata for Volume!  No public key given!\n");
      return -ENOTCONN;
   }

   // verify metadata
   rc = md_verify<ms::ms_volume_metadata>( vol->volume_public_key, volume_md );
   if( rc != 0 ) {
      errorf("Signature verification failed on Volume %s (%" PRIu64 "), rc = %d\n", volume_md->name().c_str(), volume_md->volume_id(), rc );
      return rc;
   }

   // sanity check
   if( vol->name ) {
      char* new_name = strdup( volume_md->name().c_str() );
      if( strcmp( new_name, vol->name ) != 0 ) {
         errorf("Invalid Volume metadata: tried to change name from '%s' to '%s'\n", vol->name, new_name );
         free( new_name );
         return -EINVAL;
      }
      free( new_name );
   }

   struct md_entry* root = NULL;

   if( volume_md->has_root() ) {
      root = CALLOC_LIST( struct md_entry, 1 );
      ms_entry_to_md_entry( volume_md->root(), root );
   }

   if( vol->root ) {
      md_entry_free( vol->root );
      free( vol->root );
   }

   vol->root = root;

   if( vol->UG_certs == NULL ) {
      vol->UG_certs = new ms_cert_bundle();
   }

   if( vol->RG_certs == NULL ) {
      vol->RG_certs = new ms_cert_bundle();
   }

   if( vol->AG_certs == NULL ) {
      vol->AG_certs = new ms_cert_bundle();
   }

   vol->volume_cert_version = volume_md->cert_version();
   vol->volume_id = volume_md->volume_id();
   vol->volume_owner_id = volume_md->owner_id();
   vol->blocksize = volume_md->blocksize();
   vol->volume_version = volume_md->volume_version();

   if( vol->name == NULL ) {
      vol->name = strdup( volume_md->name().c_str() );
   }

   vol->cache_closure = NULL;

   if( volume_md->has_cache_closure_text() ) {
      char const* method = NULL;

      if( vol->cache_closure == NULL ) {
         method = "md_closure_init";

         vol->cache_closure = CALLOC_LIST( struct md_closure, 1 );
         rc = md_closure_init( vol->cache_closure, conf, gateway_pubkey, gateway_privkey,
                               MS_CLIENT_CACHE_CLOSURE_PROTOTYPE,
                               volume_md->cache_closure_text().data(), volume_md->cache_closure_text().size(),
                               false, false );
      }
      else {
         method = "md_closure_reload";

         rc = md_closure_reload( vol->cache_closure, conf, gateway_pubkey, gateway_privkey, volume_md->cache_closure_text().data(), volume_md->cache_closure_text().size() );
      }

      if( rc != 0 ) {
         errorf("%s rc = %d\n", method, rc );
         return rc;
      }
      else {
         dbprintf("(Re)initialized CDN closure %p for Volume %s\n", vol->cache_closure, vol->name );
      }
   }
   else {
      errorf("WARN: no CDN closure for Volume %s\n", vol->name );
   }
   return 0;
}

<|endoftext|>


<commit_before>#include "lm/builder/corpus_count.hh"

#include "lm/builder/ngram.hh"
#include "lm/lm_exception.hh"
#include "lm/word_index.hh"
#include "util/file.hh"
#include "util/file_piece.hh"
#include "util/murmur_hash.hh"
#include "util/probing_hash_table.hh"
#include "util/scoped.hh"
#include "util/stream/chain.hh"
#include "util/stream/timer.hh"
#include "util/tokenize_piece.hh"

#include <boost/unordered_set.hpp>
#include <boost/unordered_map.hpp>

#include <functional>

#include <stdint.h>

namespace lm {
namespace builder {
namespace {

class VocabHandout {
  public:
    explicit VocabHandout(int fd) {
      util::scoped_fd duped(util::DupOrThrow(fd));
      word_list_.reset(util::FDOpenOrThrow(duped));

      Lookup("<unk>"); // Force 0
      Lookup("<s>"); // Force 1
      Lookup("</s>"); // Force 2
    }

    WordIndex Lookup(const StringPiece &word) {
      uint64_t hashed = util::MurmurHashNative(word.data(), word.size());
      std::pair<Seen::iterator, bool> ret(seen_.insert(std::pair<uint64_t, lm::WordIndex>(hashed, seen_.size())));
      if (ret.second) {
        char null_delimit = 0;
        util::WriteOrThrow(word_list_.get(), word.data(), word.size());
        util::WriteOrThrow(word_list_.get(), &null_delimit, 1);
        UTIL_THROW_IF(seen_.size() >= std::numeric_limits<lm::WordIndex>::max(), VocabLoadException, "Too many vocabulary words.  Change WordIndex to uint64_t in lm/word_index.hh.");
      }
      return ret.first->second;
    }

    WordIndex Size() const {
      return seen_.size();
    }

  private:
    typedef boost::unordered_map<uint64_t, lm::WordIndex> Seen;

    Seen seen_;

    util::scoped_FILE word_list_;
};

class DedupeHash : public std::unary_function<const WordIndex *, bool> {
  public:
    explicit DedupeHash(std::size_t order) : size_(order * sizeof(WordIndex)) {}

    std::size_t operator()(const WordIndex *start) const {
      return util::MurmurHashNative(start, size_);
    }

  private:
    const std::size_t size_;
};

class DedupeEquals : public std::binary_function<const WordIndex *, const WordIndex *, bool> {
  public:
    explicit DedupeEquals(std::size_t order) : size_(order * sizeof(WordIndex)) {}

    bool operator()(const WordIndex *first, const WordIndex *second) const {
      return !memcmp(first, second, size_);
    }

  private:
    const std::size_t size_;
};

struct DedupeEntry {
  typedef WordIndex *Key;
  Key GetKey() const { return key; }
  Key key;
  static DedupeEntry Construct(WordIndex *at) {
    DedupeEntry ret;
    ret.key = at;
    return ret;
  }
};

typedef util::ProbingHashTable<DedupeEntry, DedupeHash, DedupeEquals> Dedupe;

const float kProbingMultiplier = 1.5;

class Writer {
  public:
    Writer(std::size_t order, const util::stream::ChainPosition &position, void *dedupe_mem, std::size_t dedupe_mem_size)
      : block_(position), gram_(block_->Get(), order),
        dedupe_invalid_(order, std::numeric_limits<WordIndex>::max()),
        dedupe_(dedupe_mem, dedupe_mem_size, &dedupe_invalid_[0], DedupeHash(order), DedupeEquals(order)),
        buffer_(new WordIndex[order - 1]),
        block_size_(position.GetChain().BlockSize()) {
      dedupe_.Clear(DedupeEntry::Construct(&dedupe_invalid_[0]));
      assert(Dedupe::Size(position.GetChain().BlockSize() / position.GetChain().EntrySize(), kProbingMultiplier) == dedupe_mem_size);
      if (order == 1) {
        // Add special words.  AdjustCounts is responsible if order != 1.
        AddUnigramWord(kUNK);
        AddUnigramWord(kBOS);
      }
    }

    ~Writer() {
      block_->SetValidSize(reinterpret_cast<const uint8_t*>(gram_.begin()) - static_cast<const uint8_t*>(block_->Get()));
      (++block_).Poison();
    }

    // Write context with a bunch of <s>
    void StartSentence() {
      for (WordIndex *i = gram_.begin(); i != gram_.end() - 1; ++i) {
        *i = kBOS;
      }
    }

    void Append(WordIndex word) {
      *(gram_.end() - 1) = word;
      Dedupe::MutableIterator at;
      bool found = dedupe_.FindOrInsert(DedupeEntry::Construct(gram_.begin()), at);
      if (found) {
        // Already present.
        NGram already(at->key, gram_.Order());
        ++(already.Count());
        // Shift left by one.
        memmove(gram_.begin(), gram_.begin() + 1, sizeof(WordIndex) * (gram_.Order() - 1));
        return;
      }
      // Complete the write.
      gram_.Count() = 1;
      // Prepare the next n-gram.
      if (reinterpret_cast<uint8_t*>(gram_.begin()) + gram_.TotalSize() != static_cast<uint8_t*>(block_->Get()) + block_size_) {
        NGram last(gram_);
        gram_.NextInMemory();
        std::copy(last.begin() + 1, last.end(), gram_.begin());
        return;
      }
      // Block end.  Need to store the context in a temporary buffer.
      std::copy(gram_.begin() + 1, gram_.end(), buffer_.get());
      dedupe_.Clear(DedupeEntry::Construct(&dedupe_invalid_[0]));
      block_->SetValidSize(block_size_);
      gram_.ReBase((++block_)->Get());
      std::copy(buffer_.get(), buffer_.get() + gram_.Order() - 1, gram_.begin());
    }

  private:
    void AddUnigramWord(WordIndex index) {
      *gram_.begin() = index;
      gram_.Count() = 0;
      gram_.NextInMemory();
      if (gram_.Base() == static_cast<uint8_t*>(block_->Get()) + block_size_) {
        block_->SetValidSize(block_size_);
        gram_.ReBase((++block_)->Get());
      }
    }

    util::stream::Link block_;

    NGram gram_;

    // This is the memory behind the invalid value in dedupe_.
    std::vector<WordIndex> dedupe_invalid_;
    // Hash table combiner implementation.
    Dedupe dedupe_;

    // Small buffer to hold existing ngrams when shifting across a block boundary.
    boost::scoped_array<WordIndex> buffer_;

    const std::size_t block_size_;
};

} // namespace

float CorpusCount::DedupeMultiplier(std::size_t order) {
  return kProbingMultiplier * static_cast<float>(sizeof(DedupeEntry)) / static_cast<float>(NGram::TotalSize(order));
}

CorpusCount::CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block)
  : from_(from), vocab_write_(vocab_write), token_count_(token_count), type_count_(type_count),
    dedupe_mem_size_(Dedupe::Size(entries_per_block, kProbingMultiplier)),
    dedupe_mem_(util::MallocOrThrow(dedupe_mem_size_)) {
  token_count_ = 0;
  type_count_ = 0;
}

void CorpusCount::Run(const util::stream::ChainPosition &position) {
  UTIL_TIMER("(%w s) Counted n-grams\n");

  VocabHandout vocab(vocab_write_);
  const WordIndex end_sentence = vocab.Lookup("</s>");
  Writer writer(NGram::OrderFromSize(position.GetChain().EntrySize()), position, dedupe_mem_.get(), dedupe_mem_size_);
  uint64_t count = 0;
  try {
    while(true) {
      StringPiece line(from_.ReadLine());
      writer.StartSentence();
      for (util::TokenIter<util::AnyCharacter, true> w(line, " \t"); w; ++w) {
        WordIndex word = vocab.Lookup(*w);
        UTIL_THROW_IF(word <= 2, FormatLoadException, "Special word " << *w << " is not allowed in the corpus.  I plan to support models containing <unk> in the future.");
        writer.Append(word);
        ++count;
      }
      writer.Append(end_sentence);
    }
  } catch (const util::EndOfFileException &e) {}
  token_count_ = count;
  type_count_ = vocab.Size();
}

} // namespace builder
} // namespace lm
<commit_msg>Tolerate odd characters in corpora.<commit_after>#include "lm/builder/corpus_count.hh"

#include "lm/builder/ngram.hh"
#include "lm/lm_exception.hh"
#include "lm/word_index.hh"
#include "util/file.hh"
#include "util/file_piece.hh"
#include "util/murmur_hash.hh"
#include "util/probing_hash_table.hh"
#include "util/scoped.hh"
#include "util/stream/chain.hh"
#include "util/stream/timer.hh"
#include "util/tokenize_piece.hh"

#include <boost/unordered_set.hpp>
#include <boost/unordered_map.hpp>

#include <functional>

#include <stdint.h>

namespace lm {
namespace builder {
namespace {

class VocabHandout {
  public:
    explicit VocabHandout(int fd) {
      util::scoped_fd duped(util::DupOrThrow(fd));
      word_list_.reset(util::FDOpenOrThrow(duped));

      Lookup("<unk>"); // Force 0
      Lookup("<s>"); // Force 1
      Lookup("</s>"); // Force 2
    }

    WordIndex Lookup(const StringPiece &word) {
      uint64_t hashed = util::MurmurHashNative(word.data(), word.size());
      std::pair<Seen::iterator, bool> ret(seen_.insert(std::pair<uint64_t, lm::WordIndex>(hashed, seen_.size())));
      if (ret.second) {
        char null_delimit = 0;
        util::WriteOrThrow(word_list_.get(), word.data(), word.size());
        util::WriteOrThrow(word_list_.get(), &null_delimit, 1);
        UTIL_THROW_IF(seen_.size() >= std::numeric_limits<lm::WordIndex>::max(), VocabLoadException, "Too many vocabulary words.  Change WordIndex to uint64_t in lm/word_index.hh.");
      }
      return ret.first->second;
    }

    WordIndex Size() const {
      return seen_.size();
    }

  private:
    typedef boost::unordered_map<uint64_t, lm::WordIndex> Seen;

    Seen seen_;

    util::scoped_FILE word_list_;
};

class DedupeHash : public std::unary_function<const WordIndex *, bool> {
  public:
    explicit DedupeHash(std::size_t order) : size_(order * sizeof(WordIndex)) {}

    std::size_t operator()(const WordIndex *start) const {
      return util::MurmurHashNative(start, size_);
    }

  private:
    const std::size_t size_;
};

class DedupeEquals : public std::binary_function<const WordIndex *, const WordIndex *, bool> {
  public:
    explicit DedupeEquals(std::size_t order) : size_(order * sizeof(WordIndex)) {}

    bool operator()(const WordIndex *first, const WordIndex *second) const {
      return !memcmp(first, second, size_);
    }

  private:
    const std::size_t size_;
};

struct DedupeEntry {
  typedef WordIndex *Key;
  Key GetKey() const { return key; }
  Key key;
  static DedupeEntry Construct(WordIndex *at) {
    DedupeEntry ret;
    ret.key = at;
    return ret;
  }
};

typedef util::ProbingHashTable<DedupeEntry, DedupeHash, DedupeEquals> Dedupe;

const float kProbingMultiplier = 1.5;

class Writer {
  public:
    Writer(std::size_t order, const util::stream::ChainPosition &position, void *dedupe_mem, std::size_t dedupe_mem_size)
      : block_(position), gram_(block_->Get(), order),
        dedupe_invalid_(order, std::numeric_limits<WordIndex>::max()),
        dedupe_(dedupe_mem, dedupe_mem_size, &dedupe_invalid_[0], DedupeHash(order), DedupeEquals(order)),
        buffer_(new WordIndex[order - 1]),
        block_size_(position.GetChain().BlockSize()) {
      dedupe_.Clear(DedupeEntry::Construct(&dedupe_invalid_[0]));
      assert(Dedupe::Size(position.GetChain().BlockSize() / position.GetChain().EntrySize(), kProbingMultiplier) == dedupe_mem_size);
      if (order == 1) {
        // Add special words.  AdjustCounts is responsible if order != 1.
        AddUnigramWord(kUNK);
        AddUnigramWord(kBOS);
      }
    }

    ~Writer() {
      block_->SetValidSize(reinterpret_cast<const uint8_t*>(gram_.begin()) - static_cast<const uint8_t*>(block_->Get()));
      (++block_).Poison();
    }

    // Write context with a bunch of <s>
    void StartSentence() {
      for (WordIndex *i = gram_.begin(); i != gram_.end() - 1; ++i) {
        *i = kBOS;
      }
    }

    void Append(WordIndex word) {
      *(gram_.end() - 1) = word;
      Dedupe::MutableIterator at;
      bool found = dedupe_.FindOrInsert(DedupeEntry::Construct(gram_.begin()), at);
      if (found) {
        // Already present.
        NGram already(at->key, gram_.Order());
        ++(already.Count());
        // Shift left by one.
        memmove(gram_.begin(), gram_.begin() + 1, sizeof(WordIndex) * (gram_.Order() - 1));
        return;
      }
      // Complete the write.
      gram_.Count() = 1;
      // Prepare the next n-gram.
      if (reinterpret_cast<uint8_t*>(gram_.begin()) + gram_.TotalSize() != static_cast<uint8_t*>(block_->Get()) + block_size_) {
        NGram last(gram_);
        gram_.NextInMemory();
        std::copy(last.begin() + 1, last.end(), gram_.begin());
        return;
      }
      // Block end.  Need to store the context in a temporary buffer.
      std::copy(gram_.begin() + 1, gram_.end(), buffer_.get());
      dedupe_.Clear(DedupeEntry::Construct(&dedupe_invalid_[0]));
      block_->SetValidSize(block_size_);
      gram_.ReBase((++block_)->Get());
      std::copy(buffer_.get(), buffer_.get() + gram_.Order() - 1, gram_.begin());
    }

  private:
    void AddUnigramWord(WordIndex index) {
      *gram_.begin() = index;
      gram_.Count() = 0;
      gram_.NextInMemory();
      if (gram_.Base() == static_cast<uint8_t*>(block_->Get()) + block_size_) {
        block_->SetValidSize(block_size_);
        gram_.ReBase((++block_)->Get());
      }
    }

    util::stream::Link block_;

    NGram gram_;

    // This is the memory behind the invalid value in dedupe_.
    std::vector<WordIndex> dedupe_invalid_;
    // Hash table combiner implementation.
    Dedupe dedupe_;

    // Small buffer to hold existing ngrams when shifting across a block boundary.
    boost::scoped_array<WordIndex> buffer_;

    const std::size_t block_size_;
};

} // namespace

float CorpusCount::DedupeMultiplier(std::size_t order) {
  return kProbingMultiplier * static_cast<float>(sizeof(DedupeEntry)) / static_cast<float>(NGram::TotalSize(order));
}

CorpusCount::CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block)
  : from_(from), vocab_write_(vocab_write), token_count_(token_count), type_count_(type_count),
    dedupe_mem_size_(Dedupe::Size(entries_per_block, kProbingMultiplier)),
    dedupe_mem_(util::MallocOrThrow(dedupe_mem_size_)) {
  token_count_ = 0;
  type_count_ = 0;
}

void CorpusCount::Run(const util::stream::ChainPosition &position) {
  UTIL_TIMER("(%w s) Counted n-grams\n");

  VocabHandout vocab(vocab_write_);
  const WordIndex end_sentence = vocab.Lookup("</s>");
  Writer writer(NGram::OrderFromSize(position.GetChain().EntrySize()), position, dedupe_mem_.get(), dedupe_mem_size_);
  uint64_t count = 0;
  StringPiece delimiters("\0\t\r ", 4);
  try {
    while(true) {
      StringPiece line(from_.ReadLine());
      writer.StartSentence();
      for (util::TokenIter<util::AnyCharacter, true> w(line, delimiters); w; ++w) {
        WordIndex word = vocab.Lookup(*w);
        UTIL_THROW_IF(word <= 2, FormatLoadException, "Special word " << *w << " is not allowed in the corpus.  I plan to support models containing <unk> in the future.");
        writer.Append(word);
        ++count;
      }
      writer.Append(end_sentence);
    }
  } catch (const util::EndOfFileException &e) {}
  token_count_ = count;
  type_count_ = vocab.Size();
}

} // namespace builder
} // namespace lm
<|endoftext|>