<commit_before>/**
 * This file is part of the CernVM File System.
 *
 * cvmfs_talk runs query-response cycles against a running cvmfs instance.
 */

#include <errno.h>
#include <unistd.h>

#include <cassert>
#include <cstring>
#include <string>
#include <vector>

#include "logging.h"
#include "options.h"
#include "util/pointer.h"
#include "util/posix.h"
#include "util/string.h"

struct InstanceInfo {
  bool IsDefined() {
    return !socket_path.empty() || !instance_name.empty();
  }

  // Called at most once by DeterminePath
  static std::string GetDefaultDomain() {
    std::string result;
    BashOptionsManager options_mgr;
    options_mgr.ParseDefault("");
    bool retval = options_mgr.GetValue("CVMFS_DEFAULT_DOMAIN", &result);
    if (!retval) {
      LogCvmfs(kLogCvmfs, kLogStderr,
               "Error: could not determin CVMFS_DEFAULT_DOMAIN");
    }
    return result;
  }

  bool DeterminePaths() {
    std::string fqrn = instance_name;
    if (fqrn.find('.') == std::string::npos) {
      static std::string default_domain = GetDefaultDomain();
      fqrn = fqrn + "." + default_domain;
    }

    BashOptionsManager options_mgr;
    options_mgr.ParseDefault(fqrn);
    if (!options_mgr.GetValue("CVMFS_WORKSPACE", &workspace)) {
      if (!options_mgr.GetValue("CVMFS_CACHE_DIR", &workspace)) {
        bool retval = options_mgr.GetValue("CVMFS_CACHE_BASE", &workspace);
        if (!retval) {
          LogCvmfs(kLogCvmfs, kLogStderr,
                   "CVMFS_WORKSPACE, CVMFS_CACHE_DIR, and CVMFS_CACHE_BASE "
                   "missing");
          return false;
        }

        std::string optarg;
        if (options_mgr.GetValue("CVMFS_SHARED_CACHE", &optarg) &&
            options_mgr.IsOn(optarg))
        {
          workspace += "/shared";
        } else {
          workspace += "/" + fqrn;
        }
      }
    }

    socket_path = workspace + "/cvmfs_io." + fqrn;
    return true;
  }

  bool CompleteInfo() {
    assert(IsDefined());

    if (socket_path.empty()) {
      bool retval = DeterminePaths();
      if (!retval)
        return false;
      identifier = "instance '" + instance_name + "' active in " + workspace;
    } else {
      workspace = GetParentPath(socket_path);
      identifier = "instance listening at " + socket_path;
    }
    return true;
  }

  std::string socket_path;
  std::string instance_name;
  std::string workspace;
  std::string identifier;
};

static bool ReadResponse(int fd) {
  std::string line;
  char buf;
  int retval;
  while ((retval = read(fd, &buf, 1)) == 1) {
    if (buf == '\n') {
      LogCvmfs(kLogCvmfs, kLogStdout, "%s", line.c_str());
      line.clear();
      continue;
    }
    line.push_back(buf);
  }
  return retval == 0;
}

bool SendCommand(const std::string &command, InstanceInfo instance_info) {
  bool retval = instance_info.CompleteInfo();
  if (!retval) return false;

  int fd = ConnectSocket(instance_info.socket_path);
  if (fd < 0) {
    if (errno == ENOENT) {
      LogCvmfs(kLogCvmfs, kLogStderr,
               "Seems like CernVM-FS is not running in %s (not found: %s)",
               instance_info.workspace.c_str(),
               instance_info.socket_path.c_str());
    } else {
      LogCvmfs(kLogCvmfs, kLogStderr, "Could not access %s (%d - %s)",
               instance_info.identifier.c_str(), errno, strerror(errno));
    }
    return false;
  }

  WritePipe(fd, command.data(), command.size());
  retval = ReadResponse(fd);
  close(fd);

  if (!retval) {
    LogCvmfs(kLogCvmfs, kLogStderr, "Broken connection to %s (%d - %s)",
             instance_info.identifier.c_str(), errno, strerror(errno));
    return false;
  }
  return true;
}

static void Usage(const std::string &exe) {
  LogCvmfs(kLogCvmfs, kLogStdout,
    "Usage: %s [-i instance | -p socket] <command>                     \n"
    "   By default, iteratate through all instances.                   \n"
    "\n"
    "Example:                                                          \n"
    "  %s -i atlas.cern.ch pid                                         \n"
    "\n"
    "Commands:                                                         \n"
    "  tracebuffer flush      flushes the trace buffer to disk         \n"
    "  cache instance         describes the active cache manager       \n"
    "  cache size             gets current size of file cache          \n"
    "  cache list             gets files in cache                      \n"
    "  cache list pinned      gets pinned file catalogs in cache       \n"
    "  cache list catalogs    gets all file catalogs in cache          \n"
    "  cleanup <MB>           cleans file cache until size <= <MB>     \n"
    "  cleanup rate <period>  n.o. cleanups in the last <period> min   \n"
    "  evict <path>           removes <path> from the cache            \n"
    "  pin <path>             pins <path> in the cache                 \n"
    "  mountpoint             returns the mount point                  \n"
    "  remount [sync]         look for new catalogs                    \n"
    "  revision               gets the repository revision             \n"
    "  max ttl info           gets the maximum ttl                     \n"
    "  max ttl set <minutes>  sets the maximum ttl                     \n"
    "  nameserver get         get the DNS server                       \n"
    "  nameserver set <host>  sets a DNS server                        \n"
    "  host info              get host chain and their rtt,            \n"
    "                         if already probed                        \n"
    "  host probe             orders the host chain according to rtt   \n"
    "  host probe geo         let Stratum 1s order the host chain and  \n"
    "                         fallback proxies using the Geo-API       \n"
    "  host switch            switches to the next host in the chain   \n"
    "  host set <host list>   sets a new host chain                    \n"
    "  proxy info             gets load-balance proxy groups           \n"
    "  proxy rebalance        randomly selects a new proxy server      \n"
    "                         from the current load-balance group      \n"
    "  proxy group switch     switches to the next load-balance        \n"
    "                         proxy group in the chain                 \n"
    "  proxy set <proxy list> sets a new chain of load-balance proxy   \n"
    "                         groups (not including fallback proxies)  \n"
    "  proxy fallback <list>  sets a new list of fallback proxies      \n"
    "  external host info     gets info about external host chain      \n"
    "  external host switch   switches to the next external host       \n"
    "  external host set                                               \n"
    "       <host list>       sets external host chain                 \n"
    "  external proxy info    gets info about external proxy groups    \n"
    "  external proxy set                                              \n"
    "       <proxy list>      sets chain of external proxy groups      \n"
    "  timeout info           gets the network timeouts                \n"
    "  timeout set                                                     \n"
    "       <proxy> <direct>  sets the network timeouts in seconds     \n"
    "  pid                    gets the pid                             \n"
    "  pid cachemgr           gets the pid of the shared cache manager \n"
    "  pid watchdog           gets the pid of the crash handler process\n"
    "  parameters             dumps the effective parameters           \n"
    "  reset error counters   resets the counter for I/O errors        \n"
    "  hotpatch history       shows timestamps and version info of     \n"
    "                         loaded (hotpatched) Fuse modules         \n"
    "  version                gets cvmfs version                       \n"
    "  version patchlevel     gets cvmfs patchlevel                    \n"
    "  open catalogs          shows information about currently        \n"
    "                         loaded catalogs (_not_ all cached ones)  \n"
    "  latency                show the latencies of different fuse     \n"
    "                         calls (requires CVMFS_INSTRUMENT_FUSE)   \n"
    "\n",
    exe.c_str(), exe.c_str());
}

int main(int argc, char *argv[]) {
  InstanceInfo instance_info;
  std::string command;

  int c;
  // 's' for socket would have been a better option letter but we keep 'p'
  // for backwards compatibility.  The '+' at the beginning of the option stirng
  // prevents permutation of the option and non-option arguments.
  while ((c = getopt(argc, argv, "+hi:p:")) != -1) {
    switch (c) {
      case 'h':
        Usage(argv[0]);
        return 0;
      case 'p':
        instance_info.socket_path = optarg;
        break;
      case 'i':
        instance_info.instance_name = optarg;
        break;
      case '?':
      default:
        Usage(argv[0]);
        return 1;
    }
  }

  for (; optind < argc; ++optind) {
    command += argv[optind];
    if (optind < (argc - 1))
      command.push_back(' ');
  }
  if (command.empty()) {
    LogCvmfs(kLogCvmfs, kLogStderr, "Command missing");
    return 1;
  }

  int retcode = 0;
  if (!instance_info.IsDefined()) {
    BashOptionsManager options_mgr;
    options_mgr.ParseDefault("");
    std::string opt_repos;
    options_mgr.GetValue("CVMFS_REPOSITORIES", &opt_repos);
    std::vector<std::string> repos = SplitString(opt_repos, ',');
    for (unsigned i = 0; i < repos.size(); ++i) {
      if (repos[i].empty())
        continue;
      instance_info.instance_name = repos[i];
      LogCvmfs(kLogCvmfs, kLogStdout, "%s:", repos[i].c_str());
      bool retval = SendCommand(command, instance_info);
      if (!retval) retcode = 1;
    }
  } else {
    bool retval = SendCommand(command, instance_info);
    if (!retval) retcode = 1;
  }
  return retcode;
}
<commit_msg>add 'device id' to cvmfs_talk help text<commit_after>/**
 * This file is part of the CernVM File System.
 *
 * cvmfs_talk runs query-response cycles against a running cvmfs instance.
 */

#include <errno.h>
#include <unistd.h>

#include <cassert>
#include <cstring>
#include <string>
#include <vector>

#include "logging.h"
#include "options.h"
#include "util/pointer.h"
#include "util/posix.h"
#include "util/string.h"

struct InstanceInfo {
  bool IsDefined() {
    return !socket_path.empty() || !instance_name.empty();
  }

  // Called at most once by DeterminePath
  static std::string GetDefaultDomain() {
    std::string result;
    BashOptionsManager options_mgr;
    options_mgr.ParseDefault("");
    bool retval = options_mgr.GetValue("CVMFS_DEFAULT_DOMAIN", &result);
    if (!retval) {
      LogCvmfs(kLogCvmfs, kLogStderr,
               "Error: could not determin CVMFS_DEFAULT_DOMAIN");
    }
    return result;
  }

  bool DeterminePaths() {
    std::string fqrn = instance_name;
    if (fqrn.find('.') == std::string::npos) {
      static std::string default_domain = GetDefaultDomain();
      fqrn = fqrn + "." + default_domain;
    }

    BashOptionsManager options_mgr;
    options_mgr.ParseDefault(fqrn);
    if (!options_mgr.GetValue("CVMFS_WORKSPACE", &workspace)) {
      if (!options_mgr.GetValue("CVMFS_CACHE_DIR", &workspace)) {
        bool retval = options_mgr.GetValue("CVMFS_CACHE_BASE", &workspace);
        if (!retval) {
          LogCvmfs(kLogCvmfs, kLogStderr,
                   "CVMFS_WORKSPACE, CVMFS_CACHE_DIR, and CVMFS_CACHE_BASE "
                   "missing");
          return false;
        }

        std::string optarg;
        if (options_mgr.GetValue("CVMFS_SHARED_CACHE", &optarg) &&
            options_mgr.IsOn(optarg))
        {
          workspace += "/shared";
        } else {
          workspace += "/" + fqrn;
        }
      }
    }

    socket_path = workspace + "/cvmfs_io." + fqrn;
    return true;
  }

  bool CompleteInfo() {
    assert(IsDefined());

    if (socket_path.empty()) {
      bool retval = DeterminePaths();
      if (!retval)
        return false;
      identifier = "instance '" + instance_name + "' active in " + workspace;
    } else {
      workspace = GetParentPath(socket_path);
      identifier = "instance listening at " + socket_path;
    }
    return true;
  }

  std::string socket_path;
  std::string instance_name;
  std::string workspace;
  std::string identifier;
};

static bool ReadResponse(int fd) {
  std::string line;
  char buf;
  int retval;
  while ((retval = read(fd, &buf, 1)) == 1) {
    if (buf == '\n') {
      LogCvmfs(kLogCvmfs, kLogStdout, "%s", line.c_str());
      line.clear();
      continue;
    }
    line.push_back(buf);
  }
  return retval == 0;
}

bool SendCommand(const std::string &command, InstanceInfo instance_info) {
  bool retval = instance_info.CompleteInfo();
  if (!retval) return false;

  int fd = ConnectSocket(instance_info.socket_path);
  if (fd < 0) {
    if (errno == ENOENT) {
      LogCvmfs(kLogCvmfs, kLogStderr,
               "Seems like CernVM-FS is not running in %s (not found: %s)",
               instance_info.workspace.c_str(),
               instance_info.socket_path.c_str());
    } else {
      LogCvmfs(kLogCvmfs, kLogStderr, "Could not access %s (%d - %s)",
               instance_info.identifier.c_str(), errno, strerror(errno));
    }
    return false;
  }

  WritePipe(fd, command.data(), command.size());
  retval = ReadResponse(fd);
  close(fd);

  if (!retval) {
    LogCvmfs(kLogCvmfs, kLogStderr, "Broken connection to %s (%d - %s)",
             instance_info.identifier.c_str(), errno, strerror(errno));
    return false;
  }
  return true;
}

static void Usage(const std::string &exe) {
  LogCvmfs(kLogCvmfs, kLogStdout,
    "Usage: %s [-i instance | -p socket] <command>                     \n"
    "   By default, iteratate through all instances.                   \n"
    "\n"
    "Example:                                                          \n"
    "  %s -i atlas.cern.ch pid                                         \n"
    "\n"
    "Commands:                                                         \n"
    "  tracebuffer flush      flushes the trace buffer to disk         \n"
    "  cache instance         describes the active cache manager       \n"
    "  cache size             gets current size of file cache          \n"
    "  cache list             gets files in cache                      \n"
    "  cache list pinned      gets pinned file catalogs in cache       \n"
    "  cache list catalogs    gets all file catalogs in cache          \n"
    "  cleanup <MB>           cleans file cache until size <= <MB>     \n"
    "  cleanup rate <period>  n.o. cleanups in the last <period> min   \n"
    "  evict <path>           removes <path> from the cache            \n"
    "  pin <path>             pins <path> in the cache                 \n"
    "  mountpoint             returns the mount point                  \n"
    "  device id              returns major:minor virtual device id    \n"
    "                         on Linux and 0:0 on macOS                \n"
    "  remount [sync]         look for new catalogs                    \n"
    "  revision               gets the repository revision             \n"
    "  max ttl info           gets the maximum ttl                     \n"
    "  max ttl set <minutes>  sets the maximum ttl                     \n"
    "  nameserver get         get the DNS server                       \n"
    "  nameserver set <host>  sets a DNS server                        \n"
    "  host info              get host chain and their rtt,            \n"
    "                         if already probed                        \n"
    "  host probe             orders the host chain according to rtt   \n"
    "  host probe geo         let Stratum 1s order the host chain and  \n"
    "                         fallback proxies using the Geo-API       \n"
    "  host switch            switches to the next host in the chain   \n"
    "  host set <host list>   sets a new host chain                    \n"
    "  proxy info             gets load-balance proxy groups           \n"
    "  proxy rebalance        randomly selects a new proxy server      \n"
    "                         from the current load-balance group      \n"
    "  proxy group switch     switches to the next load-balance        \n"
    "                         proxy group in the chain                 \n"
    "  proxy set <proxy list> sets a new chain of load-balance proxy   \n"
    "                         groups (not including fallback proxies)  \n"
    "  proxy fallback <list>  sets a new list of fallback proxies      \n"
    "  external host info     gets info about external host chain      \n"
    "  external host switch   switches to the next external host       \n"
    "  external host set                                               \n"
    "       <host list>       sets external host chain                 \n"
    "  external proxy info    gets info about external proxy groups    \n"
    "  external proxy set                                              \n"
    "       <proxy list>      sets chain of external proxy groups      \n"
    "  timeout info           gets the network timeouts                \n"
    "  timeout set                                                     \n"
    "       <proxy> <direct>  sets the network timeouts in seconds     \n"
    "  pid                    gets the pid                             \n"
    "  pid cachemgr           gets the pid of the shared cache manager \n"
    "  pid watchdog           gets the pid of the crash handler process\n"
    "  parameters             dumps the effective parameters           \n"
    "  reset error counters   resets the counter for I/O errors        \n"
    "  hotpatch history       shows timestamps and version info of     \n"
    "                         loaded (hotpatched) Fuse modules         \n"
    "  version                gets cvmfs version                       \n"
    "  version patchlevel     gets cvmfs patchlevel                    \n"
    "  open catalogs          shows information about currently        \n"
    "                         loaded catalogs (_not_ all cached ones)  \n"
    "  latency                show the latencies of different fuse     \n"
    "                         calls (requires CVMFS_INSTRUMENT_FUSE)   \n"
    "\n",
    exe.c_str(), exe.c_str());
}

int main(int argc, char *argv[]) {
  InstanceInfo instance_info;
  std::string command;

  int c;
  // 's' for socket would have been a better option letter but we keep 'p'
  // for backwards compatibility.  The '+' at the beginning of the option stirng
  // prevents permutation of the option and non-option arguments.
  while ((c = getopt(argc, argv, "+hi:p:")) != -1) {
    switch (c) {
      case 'h':
        Usage(argv[0]);
        return 0;
      case 'p':
        instance_info.socket_path = optarg;
        break;
      case 'i':
        instance_info.instance_name = optarg;
        break;
      case '?':
      default:
        Usage(argv[0]);
        return 1;
    }
  }

  for (; optind < argc; ++optind) {
    command += argv[optind];
    if (optind < (argc - 1))
      command.push_back(' ');
  }
  if (command.empty()) {
    LogCvmfs(kLogCvmfs, kLogStderr, "Command missing");
    return 1;
  }

  int retcode = 0;
  if (!instance_info.IsDefined()) {
    BashOptionsManager options_mgr;
    options_mgr.ParseDefault("");
    std::string opt_repos;
    options_mgr.GetValue("CVMFS_REPOSITORIES", &opt_repos);
    std::vector<std::string> repos = SplitString(opt_repos, ',');
    for (unsigned i = 0; i < repos.size(); ++i) {
      if (repos[i].empty())
        continue;
      instance_info.instance_name = repos[i];
      LogCvmfs(kLogCvmfs, kLogStdout, "%s:", repos[i].c_str());
      bool retval = SendCommand(command, instance_info);
      if (!retval) retcode = 1;
    }
  } else {
    bool retval = SendCommand(command, instance_info);
    if (!retval) retcode = 1;
  }
  return retcode;
}
<|endoftext|>


<commit_before>#include <common/buffer.h>
#include <common/endian.h>

#include <event/event_callback.h>

#include <io/pipe/pipe.h>
#include <io/pipe/pipe_pair.h>

#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_decoder.h>
#include <xcodec/xcodec_encoder.h>
#include <xcodec/xcodec_hash.h>
#include <xcodec/xcodec_pipe_pair.h>

/*
 * And now for something completely different, a note on how end-of-stream
 * indication works with the XCodec.
 *
 * When things are going along smoothly, the XCodec is a nice one-way stream
 * compressor.  All you need is state that you already have or state from
 * earlier in the stream.  However, it doesn't take much for things to go less
 * smoothly.  When you have two connections, a symbol may be defined in the
 * first and referenced in the second, and the reference in the second stream
 * may be decoded before the definition in the first one.  In this case, we
 * have <ASK> and <LEARN> in the <OOB> stream to communicate bidirectionally
 * to get the reference.  If we're actually going to get the definition soon,
 * that's a bit wasteful, and there are a lot of optimizations we can make,
 * but the basic principle needs to be robust in case, say, the first
 * connection goes down.
 *
 * Because of this, we can't just pass through end-of-stream indicators
 * freely.  When the encoder receives EOS from a StreamChannel, we could then
 * send EOS out to the StreamChannel that connects us to the decoder on the
 * other side of the network.  But what if that decoder needs to <ASK> us
 * about a symbol we sent a reference to just before EOS?
 *
 * So we send <EOS> rather than EOS, a message saying that the encoded stream
 * has ended.
 *
 * When the decoder receives <EOS> it can send EOS on to the StreamChannel it
 * is writing to, assuming it has processed all outstanding frame data.  And
 * when it has finished processing all outstanding frame data, it will send
 * <EOS_ACK> on the encoder's output StreamChannel, to the remote decoder.
 * When both sides have sent <EOS_ACK>, the encoder's StreamChannels may be
 * shut down and no more communication will occur.
 */

/*
 * Usage:
 * 	<OP_HELLO> length[uint8_t] data[uint8_t x length]
 *
 * Effects:
 * 	Must appear at the start of and only at the start of an encoded	stream.
 *
 * Sife-effects:
 * 	Possibly many.
 */
#define	XCODEC_PIPE_OP_HELLO	((uint8_t)0xff)

/*
 * Usage:
 * 	<OP_LEARN> data[uint8_t x XCODEC_PIPE_SEGMENT_LENGTH]
 *
 * Effects:
 * 	The `data' is hashed, the hash is associated with the data if possible.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_LEARN	((uint8_t)0xfe)

/*
 * Usage:
 * 	<OP_ASK> hash[uint64_t]
 *
 * Effects:
 * 	An OP_LEARN will be sent in response with the data corresponding to the
 * 	hash.
 *
 * 	If the hash is unknown, error will be indicated.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_ASK	((uint8_t)0xfd)

/*
 * Usage:
 * 	<OP_EOS>
 *
 * Effects:
 * 	Alert the other party that we have no intention of sending more data.
 *
 * Side-effects:
 * 	The other party will send <OP_EOS_ACK> when it has processed all of
 * 	the data we have sent.
 */
#define	XCODEC_PIPE_OP_EOS	((uint8_t)0xfc)

/*
 * Usage:
 * 	<OP_EOS_ACK>
 *
 * Effects:
 * 	Alert the other party that we have no intention of reading more data.
 *
 * Side-effects:
 * 	The connection will be torn down.
 */
#define	XCODEC_PIPE_OP_EOS_ACK	((uint8_t)0xfb)

/*
 * Usage:
 * 	<FRAME> length[uint16_t] data[uint8_t x length]
 *
 * Effects:
 * 	Frames an encoded chunk.  As distinct to OP_OOB below.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_FRAME	((uint8_t)0x00)

#define	XCODEC_PIPE_MAX_FRAME	(32768)

static void encode_frame(Buffer *, Buffer *);

void
XCodecPipePair::decoder_consume(Buffer *buf)
{
	if (buf->empty()) {
		if (!decoder_buffer_.empty())
			ERROR(log_) << "Remote encoder closed connection with data outstanding.";
		if (!decoder_frame_buffer_.empty())
			ERROR(log_) << "Remote encoder closed connection with frame data outstanding.";
		if (!decoder_sent_eos_) {
			decoder_sent_eos_ = true;
			decoder_produce(buf);
		}
		return;
	}

	decoder_buffer_.append(buf);
	buf->clear();

	while (!decoder_buffer_.empty()) {
		uint8_t op = decoder_buffer_.peek();
		switch (op) {
		case XCODEC_PIPE_OP_HELLO:
			if (decoder_cache_ != NULL) {
				ERROR(log_) << "Got <HELLO> twice.";
				decoder_error();
				return;
			} else {
				uint8_t len;
				if (decoder_buffer_.length() < sizeof op + sizeof len)
					return;
				decoder_buffer_.copyout(&len, sizeof op, sizeof len);

				if (decoder_buffer_.length() < sizeof op + sizeof len + len)
					return;

				if (len != UUID_SIZE) {
					ERROR(log_) << "Unsupported <HELLO> length: " << (unsigned)len;
					decoder_error();
					return;
				}

				Buffer uubuf;
				decoder_buffer_.moveout(&uubuf, sizeof op + sizeof len, UUID_SIZE);

				UUID uuid;
				if (!uuid.decode(&uubuf)) {
					ERROR(log_) << "Invalid UUID in <HELLO>.";
					decoder_error();
					return;
				}

				decoder_cache_ = XCodecCache::lookup(uuid);
				ASSERT(decoder_cache_ != NULL);

				ASSERT(decoder_ == NULL);
				decoder_ = new XCodecDecoder(decoder_cache_);

				DEBUG(log_) << "Peer connected with UUID: " << uuid.string_;
			}
			break;
		case XCODEC_PIPE_OP_ASK:
			if (encoder_ == NULL) {
				ERROR(log_) << "Got <ASK> before sending <HELLO>.";
				decoder_error();
				return;
			} else {
				uint64_t hash;
				if (decoder_buffer_.length() < sizeof op + sizeof hash)
					return;

				decoder_buffer_.skip(sizeof op);

				decoder_buffer_.extract(&hash);
				decoder_buffer_.skip(sizeof hash);
				hash = BigEndian::decode(hash);

				BufferSegment *oseg = codec_->cache()->lookup(hash);
				if (oseg == NULL) {
					ERROR(log_) << "Unknown hash in <ASK>: " << hash;
					decoder_error();
					return;
				}

				DEBUG(log_) << "Responding to <ASK> with <LEARN>.";

				Buffer learn;
				learn.append(XCODEC_PIPE_OP_LEARN);
				learn.append(oseg);
				oseg->unref();

				encoder_produce(&learn);
			}
			break;
		case XCODEC_PIPE_OP_LEARN:
			if (decoder_cache_ == NULL) {
				ERROR(log_) << "Got <LEARN> before <HELLO>.";
				decoder_error();
				return;
			} else {
				if (decoder_buffer_.length() < sizeof op + XCODEC_SEGMENT_LENGTH)
					return;

				decoder_buffer_.skip(sizeof op);

				BufferSegment *seg;
				decoder_buffer_.copyout(&seg, XCODEC_SEGMENT_LENGTH);
				decoder_buffer_.skip(XCODEC_SEGMENT_LENGTH);

				uint64_t hash = XCodecHash<XCODEC_SEGMENT_LENGTH>::hash(seg->data());
				if (decoder_unknown_hashes_.find(hash) == decoder_unknown_hashes_.end()) {
					INFO(log_) << "Gratuitous <LEARN> without <ASK>.";
				} else {
					decoder_unknown_hashes_.erase(hash);
				}

				BufferSegment *oseg = decoder_cache_->lookup(hash);
				if (oseg != NULL) {
					if (!oseg->equal(seg)) {
						oseg->unref();
						ERROR(log_) << "Collision in <LEARN>.";
						seg->unref();
						decoder_error();
						return;
					}
					oseg->unref();
					DEBUG(log_) << "Redundant <LEARN>.";
				} else {
					DEBUG(log_) << "Successful <LEARN>.";
					decoder_cache_->enter(hash, seg);
				}
				seg->unref();
			}
			break;
		case XCODEC_PIPE_OP_EOS:
			if (decoder_received_eos_) {
				ERROR(log_) << "Duplicate <EOS>.";
				decoder_error();
				return;
			}
			decoder_received_eos_ = true;
			break;
		case XCODEC_PIPE_OP_EOS_ACK:
			if (!encoder_sent_eos_) {
				ERROR(log_) << "Got <EOS_ACK> before sending <EOS>.";
				decoder_error();
				return;
			}
			if (decoder_received_eos_ack_) {
				ERROR(log_) << "Duplicate <EOS_ACK>.";
				decoder_error();
				return;
			}
			decoder_received_eos_ack_ = true;
			break;
		case XCODEC_PIPE_OP_FRAME:
			if (decoder_ == NULL) {
				ERROR(log_) << "Got frame data before decoder initialized.";
				decoder_error();
				return;
			} else {
				uint16_t len;
				decoder_buffer_.extract(&len, sizeof op);
				len = BigEndian::decode(len);
				if (len == 0 || len > XCODEC_PIPE_MAX_FRAME) {
					ERROR(log_) << "Invalid framed data length.";
					decoder_error();
					return;
				}

				if (decoder_buffer_.length() < sizeof op + sizeof len + len)
					return;

				decoder_buffer_.moveout(&decoder_frame_buffer_, sizeof op + sizeof len, len);
			}
			break;
		default:
			ERROR(log_) << "Unsupported operation in pipe stream.";
			decoder_error();
			return;
		}

		if (decoder_frame_buffer_.empty()) {
			if (decoder_received_eos_ && !encoder_sent_eos_ack_) {
				DEBUG(log_) << "Decoder finished, got <EOS>, sending <EOS_ACK>.";

				Buffer eos_ack;
				eos_ack.append(XCODEC_PIPE_OP_EOS_ACK);

				encoder_produce(&eos_ack);
				encoder_sent_eos_ack_ = true;
			}
			continue;
		}

		if (!decoder_unknown_hashes_.empty()) {
			DEBUG(log_) << "Waiting for unknown hashes to continue processing data.";
			continue;
		}

		Buffer output;
		if (!decoder_->decode(&output, &decoder_frame_buffer_, decoder_unknown_hashes_)) {
			ERROR(log_) << "Decoder exiting with error.";
			decoder_error();
			return;
		}

		if (!output.empty()) {
			decoder_produce(&output);
		} else {
			/*
			 * We should only get no output from the decoder if
			 * we're waiting on the next frame or we need an
			 * unknown hash.  It would be nice to make the
			 * encoder framing aware so that it would not end
			 * up with encoded data that straddles a frame
			 * boundary.  (Fixing that would also allow us to
			 * simplify length checking within the decoder
			 * considerably.)
			 */
			ASSERT(!decoder_frame_buffer_.empty() || !decoder_unknown_hashes_.empty());
		}

		Buffer ask;
		std::set<uint64_t>::const_iterator it;
		for (it = decoder_unknown_hashes_.begin(); it != decoder_unknown_hashes_.end(); ++it) {
			uint64_t hash = *it;
			hash = BigEndian::encode(hash);

			ask.append(XCODEC_PIPE_OP_ASK);
			ask.append(&hash);
		}
		if (!ask.empty()) {
			DEBUG(log_) << "Sending <ASK>s.";
			encoder_produce(&ask);
		}
	}

	if (decoder_received_eos_ && !decoder_sent_eos_) {
		DEBUG(log_) << "Decoder finished, got <EOS>, shutting down decoder output channel.";
		Buffer eos;
		decoder_produce(&eos);
		decoder_sent_eos_ = true;
	}

	if (encoder_sent_eos_ack_ && decoder_received_eos_ack_) {
		ASSERT(decoder_buffer_.empty());
		ASSERT(decoder_frame_buffer_.empty());

		DEBUG(log_) << "Decoder finished, got <EOS_ACK>, shutting down encoder output channel.";

		Buffer eos;
		encoder_produce(&eos);
	}
}

void
XCodecPipePair::encoder_consume(Buffer *buf)
{
	ASSERT(!encoder_sent_eos_);

	Buffer output;

	if (encoder_ == NULL) {
		Buffer extra;
		if (!codec_->cache()->uuid_encode(&extra)) {
			ERROR(log_) << "Could not encode UUID for <HELLO>.";
			encoder_error();
			return;
		}

		uint8_t len = extra.length();
		ASSERT(len == UUID_SIZE);

		output.append(XCODEC_PIPE_OP_HELLO);
		output.append(len);
		output.append(extra);

		ASSERT(output.length() == 2 + UUID_SIZE);

		encoder_ = new XCodecEncoder(codec_->cache());
	}

	if (!buf->empty()) {
		Buffer encoded;
		encoder_->encode(&encoded, buf);
		ASSERT(!encoded.empty());

		encode_frame(&output, &encoded);
		ASSERT(!output.empty());
	} else {
		output.append(XCODEC_PIPE_OP_EOS);
		encoder_sent_eos_ = true;
	}
	encoder_produce(&output);
}

static void
encode_frame(Buffer *out, Buffer *in)
{
	while (!in->empty()) {
		uint16_t framelen;
		if (in->length() <= XCODEC_PIPE_MAX_FRAME)
			framelen = in->length();
		else
			framelen = XCODEC_PIPE_MAX_FRAME;

		Buffer frame;
		in->moveout(&frame, framelen);

		framelen = BigEndian::encode(framelen);

		out->append(XCODEC_PIPE_OP_FRAME);
		out->append(&framelen);
		out->append(frame);
	}
}
<commit_msg>Skip opcodes for <EOS|EOS_ACK>.<commit_after>#include <common/buffer.h>
#include <common/endian.h>

#include <event/event_callback.h>

#include <io/pipe/pipe.h>
#include <io/pipe/pipe_pair.h>

#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_decoder.h>
#include <xcodec/xcodec_encoder.h>
#include <xcodec/xcodec_hash.h>
#include <xcodec/xcodec_pipe_pair.h>

/*
 * And now for something completely different, a note on how end-of-stream
 * indication works with the XCodec.
 *
 * When things are going along smoothly, the XCodec is a nice one-way stream
 * compressor.  All you need is state that you already have or state from
 * earlier in the stream.  However, it doesn't take much for things to go less
 * smoothly.  When you have two connections, a symbol may be defined in the
 * first and referenced in the second, and the reference in the second stream
 * may be decoded before the definition in the first one.  In this case, we
 * have <ASK> and <LEARN> in the <OOB> stream to communicate bidirectionally
 * to get the reference.  If we're actually going to get the definition soon,
 * that's a bit wasteful, and there are a lot of optimizations we can make,
 * but the basic principle needs to be robust in case, say, the first
 * connection goes down.
 *
 * Because of this, we can't just pass through end-of-stream indicators
 * freely.  When the encoder receives EOS from a StreamChannel, we could then
 * send EOS out to the StreamChannel that connects us to the decoder on the
 * other side of the network.  But what if that decoder needs to <ASK> us
 * about a symbol we sent a reference to just before EOS?
 *
 * So we send <EOS> rather than EOS, a message saying that the encoded stream
 * has ended.
 *
 * When the decoder receives <EOS> it can send EOS on to the StreamChannel it
 * is writing to, assuming it has processed all outstanding frame data.  And
 * when it has finished processing all outstanding frame data, it will send
 * <EOS_ACK> on the encoder's output StreamChannel, to the remote decoder.
 * When both sides have sent <EOS_ACK>, the encoder's StreamChannels may be
 * shut down and no more communication will occur.
 */

/*
 * Usage:
 * 	<OP_HELLO> length[uint8_t] data[uint8_t x length]
 *
 * Effects:
 * 	Must appear at the start of and only at the start of an encoded	stream.
 *
 * Sife-effects:
 * 	Possibly many.
 */
#define	XCODEC_PIPE_OP_HELLO	((uint8_t)0xff)

/*
 * Usage:
 * 	<OP_LEARN> data[uint8_t x XCODEC_PIPE_SEGMENT_LENGTH]
 *
 * Effects:
 * 	The `data' is hashed, the hash is associated with the data if possible.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_LEARN	((uint8_t)0xfe)

/*
 * Usage:
 * 	<OP_ASK> hash[uint64_t]
 *
 * Effects:
 * 	An OP_LEARN will be sent in response with the data corresponding to the
 * 	hash.
 *
 * 	If the hash is unknown, error will be indicated.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_ASK	((uint8_t)0xfd)

/*
 * Usage:
 * 	<OP_EOS>
 *
 * Effects:
 * 	Alert the other party that we have no intention of sending more data.
 *
 * Side-effects:
 * 	The other party will send <OP_EOS_ACK> when it has processed all of
 * 	the data we have sent.
 */
#define	XCODEC_PIPE_OP_EOS	((uint8_t)0xfc)

/*
 * Usage:
 * 	<OP_EOS_ACK>
 *
 * Effects:
 * 	Alert the other party that we have no intention of reading more data.
 *
 * Side-effects:
 * 	The connection will be torn down.
 */
#define	XCODEC_PIPE_OP_EOS_ACK	((uint8_t)0xfb)

/*
 * Usage:
 * 	<FRAME> length[uint16_t] data[uint8_t x length]
 *
 * Effects:
 * 	Frames an encoded chunk.  As distinct to OP_OOB below.
 *
 * Side-effects:
 * 	None.
 */
#define	XCODEC_PIPE_OP_FRAME	((uint8_t)0x00)

#define	XCODEC_PIPE_MAX_FRAME	(32768)

static void encode_frame(Buffer *, Buffer *);

void
XCodecPipePair::decoder_consume(Buffer *buf)
{
	if (buf->empty()) {
		if (!decoder_buffer_.empty())
			ERROR(log_) << "Remote encoder closed connection with data outstanding.";
		if (!decoder_frame_buffer_.empty())
			ERROR(log_) << "Remote encoder closed connection with frame data outstanding.";
		if (!decoder_sent_eos_) {
			decoder_sent_eos_ = true;
			decoder_produce(buf);
		}
		return;
	}

	decoder_buffer_.append(buf);
	buf->clear();

	while (!decoder_buffer_.empty()) {
		uint8_t op = decoder_buffer_.peek();
		switch (op) {
		case XCODEC_PIPE_OP_HELLO:
			if (decoder_cache_ != NULL) {
				ERROR(log_) << "Got <HELLO> twice.";
				decoder_error();
				return;
			} else {
				uint8_t len;
				if (decoder_buffer_.length() < sizeof op + sizeof len)
					return;
				decoder_buffer_.copyout(&len, sizeof op, sizeof len);

				if (decoder_buffer_.length() < sizeof op + sizeof len + len)
					return;

				if (len != UUID_SIZE) {
					ERROR(log_) << "Unsupported <HELLO> length: " << (unsigned)len;
					decoder_error();
					return;
				}

				Buffer uubuf;
				decoder_buffer_.moveout(&uubuf, sizeof op + sizeof len, UUID_SIZE);

				UUID uuid;
				if (!uuid.decode(&uubuf)) {
					ERROR(log_) << "Invalid UUID in <HELLO>.";
					decoder_error();
					return;
				}

				decoder_cache_ = XCodecCache::lookup(uuid);
				ASSERT(decoder_cache_ != NULL);

				ASSERT(decoder_ == NULL);
				decoder_ = new XCodecDecoder(decoder_cache_);

				DEBUG(log_) << "Peer connected with UUID: " << uuid.string_;
			}
			break;
		case XCODEC_PIPE_OP_ASK:
			if (encoder_ == NULL) {
				ERROR(log_) << "Got <ASK> before sending <HELLO>.";
				decoder_error();
				return;
			} else {
				uint64_t hash;
				if (decoder_buffer_.length() < sizeof op + sizeof hash)
					return;

				decoder_buffer_.skip(sizeof op);

				decoder_buffer_.extract(&hash);
				decoder_buffer_.skip(sizeof hash);
				hash = BigEndian::decode(hash);

				BufferSegment *oseg = codec_->cache()->lookup(hash);
				if (oseg == NULL) {
					ERROR(log_) << "Unknown hash in <ASK>: " << hash;
					decoder_error();
					return;
				}

				DEBUG(log_) << "Responding to <ASK> with <LEARN>.";

				Buffer learn;
				learn.append(XCODEC_PIPE_OP_LEARN);
				learn.append(oseg);
				oseg->unref();

				encoder_produce(&learn);
			}
			break;
		case XCODEC_PIPE_OP_LEARN:
			if (decoder_cache_ == NULL) {
				ERROR(log_) << "Got <LEARN> before <HELLO>.";
				decoder_error();
				return;
			} else {
				if (decoder_buffer_.length() < sizeof op + XCODEC_SEGMENT_LENGTH)
					return;

				decoder_buffer_.skip(sizeof op);

				BufferSegment *seg;
				decoder_buffer_.copyout(&seg, XCODEC_SEGMENT_LENGTH);
				decoder_buffer_.skip(XCODEC_SEGMENT_LENGTH);

				uint64_t hash = XCodecHash<XCODEC_SEGMENT_LENGTH>::hash(seg->data());
				if (decoder_unknown_hashes_.find(hash) == decoder_unknown_hashes_.end()) {
					INFO(log_) << "Gratuitous <LEARN> without <ASK>.";
				} else {
					decoder_unknown_hashes_.erase(hash);
				}

				BufferSegment *oseg = decoder_cache_->lookup(hash);
				if (oseg != NULL) {
					if (!oseg->equal(seg)) {
						oseg->unref();
						ERROR(log_) << "Collision in <LEARN>.";
						seg->unref();
						decoder_error();
						return;
					}
					oseg->unref();
					DEBUG(log_) << "Redundant <LEARN>.";
				} else {
					DEBUG(log_) << "Successful <LEARN>.";
					decoder_cache_->enter(hash, seg);
				}
				seg->unref();
			}
			break;
		case XCODEC_PIPE_OP_EOS:
			if (decoder_received_eos_) {
				ERROR(log_) << "Duplicate <EOS>.";
				decoder_error();
				return;
			}
			decoder_buffer_.skip(1);
			decoder_received_eos_ = true;
			break;
		case XCODEC_PIPE_OP_EOS_ACK:
			if (!encoder_sent_eos_) {
				ERROR(log_) << "Got <EOS_ACK> before sending <EOS>.";
				decoder_error();
				return;
			}
			if (decoder_received_eos_ack_) {
				ERROR(log_) << "Duplicate <EOS_ACK>.";
				decoder_error();
				return;
			}
			decoder_buffer_.skip(1);
			decoder_received_eos_ack_ = true;
			break;
		case XCODEC_PIPE_OP_FRAME:
			if (decoder_ == NULL) {
				ERROR(log_) << "Got frame data before decoder initialized.";
				decoder_error();
				return;
			} else {
				uint16_t len;
				decoder_buffer_.extract(&len, sizeof op);
				len = BigEndian::decode(len);
				if (len == 0 || len > XCODEC_PIPE_MAX_FRAME) {
					ERROR(log_) << "Invalid framed data length.";
					decoder_error();
					return;
				}

				if (decoder_buffer_.length() < sizeof op + sizeof len + len)
					return;

				decoder_buffer_.moveout(&decoder_frame_buffer_, sizeof op + sizeof len, len);
			}
			break;
		default:
			ERROR(log_) << "Unsupported operation in pipe stream.";
			decoder_error();
			return;
		}

		if (decoder_frame_buffer_.empty()) {
			if (decoder_received_eos_ && !encoder_sent_eos_ack_) {
				DEBUG(log_) << "Decoder finished, got <EOS>, sending <EOS_ACK>.";

				Buffer eos_ack;
				eos_ack.append(XCODEC_PIPE_OP_EOS_ACK);

				encoder_produce(&eos_ack);
				encoder_sent_eos_ack_ = true;
			}
			continue;
		}

		if (!decoder_unknown_hashes_.empty()) {
			DEBUG(log_) << "Waiting for unknown hashes to continue processing data.";
			continue;
		}

		Buffer output;
		if (!decoder_->decode(&output, &decoder_frame_buffer_, decoder_unknown_hashes_)) {
			ERROR(log_) << "Decoder exiting with error.";
			decoder_error();
			return;
		}

		if (!output.empty()) {
			decoder_produce(&output);
		} else {
			/*
			 * We should only get no output from the decoder if
			 * we're waiting on the next frame or we need an
			 * unknown hash.  It would be nice to make the
			 * encoder framing aware so that it would not end
			 * up with encoded data that straddles a frame
			 * boundary.  (Fixing that would also allow us to
			 * simplify length checking within the decoder
			 * considerably.)
			 */
			ASSERT(!decoder_frame_buffer_.empty() || !decoder_unknown_hashes_.empty());
		}

		Buffer ask;
		std::set<uint64_t>::const_iterator it;
		for (it = decoder_unknown_hashes_.begin(); it != decoder_unknown_hashes_.end(); ++it) {
			uint64_t hash = *it;
			hash = BigEndian::encode(hash);

			ask.append(XCODEC_PIPE_OP_ASK);
			ask.append(&hash);
		}
		if (!ask.empty()) {
			DEBUG(log_) << "Sending <ASK>s.";
			encoder_produce(&ask);
		}
	}

	if (decoder_received_eos_ && !decoder_sent_eos_) {
		DEBUG(log_) << "Decoder finished, got <EOS>, shutting down decoder output channel.";
		Buffer eos;
		decoder_produce(&eos);
		decoder_sent_eos_ = true;
	}

	if (encoder_sent_eos_ack_ && decoder_received_eos_ack_) {
		ASSERT(decoder_buffer_.empty());
		ASSERT(decoder_frame_buffer_.empty());

		DEBUG(log_) << "Decoder finished, got <EOS_ACK>, shutting down encoder output channel.";

		Buffer eos;
		encoder_produce(&eos);
	}
}

void
XCodecPipePair::encoder_consume(Buffer *buf)
{
	ASSERT(!encoder_sent_eos_);

	Buffer output;

	if (encoder_ == NULL) {
		Buffer extra;
		if (!codec_->cache()->uuid_encode(&extra)) {
			ERROR(log_) << "Could not encode UUID for <HELLO>.";
			encoder_error();
			return;
		}

		uint8_t len = extra.length();
		ASSERT(len == UUID_SIZE);

		output.append(XCODEC_PIPE_OP_HELLO);
		output.append(len);
		output.append(extra);

		ASSERT(output.length() == 2 + UUID_SIZE);

		encoder_ = new XCodecEncoder(codec_->cache());
	}

	if (!buf->empty()) {
		Buffer encoded;
		encoder_->encode(&encoded, buf);
		ASSERT(!encoded.empty());

		encode_frame(&output, &encoded);
		ASSERT(!output.empty());
	} else {
		output.append(XCODEC_PIPE_OP_EOS);
		encoder_sent_eos_ = true;
	}
	encoder_produce(&output);
}

static void
encode_frame(Buffer *out, Buffer *in)
{
	while (!in->empty()) {
		uint16_t framelen;
		if (in->length() <= XCODEC_PIPE_MAX_FRAME)
			framelen = in->length();
		else
			framelen = XCODEC_PIPE_MAX_FRAME;

		Buffer frame;
		in->moveout(&frame, framelen);

		framelen = BigEndian::encode(framelen);

		out->append(XCODEC_PIPE_OP_FRAME);
		out->append(&framelen);
		out->append(frame);
	}
}
<|endoftext|>


<commit_before>#include "dsb/test.h"
#include <stdio.h>

unsigned int check_count = 0;
unsigned int check_failed = 0;
unsigned int test_count = 0;
unsigned int test_failed = 0;
int test_pass = 1;
int test_pretty = 1;

void fdsb_test_done(const char *function)
{
	if (test_pass == 1)
	{
		printf("%s passed\n",function);
	}
	else
	{
		printf("%s failed\n",function);
	}
}

void fdsb_test_checkfailed(int line, const char *function, const char *file)
{
	printf("%s:%d failed!\n",function,line);

	test_pass = 0;
	check_count++;
	check_failed++;
}

void fdsb_test_checkpassed()
{
	check_count++;
}

void fdsb_test(void (*test)(void))
{
	test_pass = 1;
	test_count++;
	test();
}

<commit_msg>Fixed incorrect include name<commit_after>#include "fdsb/test.hpp"
#include <stdio.h>

unsigned int check_count = 0;
unsigned int check_failed = 0;
unsigned int test_count = 0;
unsigned int test_failed = 0;
int test_pass = 1;
int test_pretty = 1;

void fdsb_test_done(const char *function)
{
	if (test_pass == 1)
	{
		printf("%s passed\n",function);
	}
	else
	{
		printf("%s failed\n",function);
	}
}

void fdsb_test_checkfailed(int line, const char *function, const char *file)
{
	printf("%s:%d failed!\n",function,line);

	test_pass = 0;
	check_count++;
	check_failed++;
}

void fdsb_test_checkpassed()
{
	check_count++;
}

void fdsb_test(void (*test)(void))
{
	test_pass = 1;
	test_count++;
	test();
}

<|endoftext|>


<commit_before>#include "memcached/memcached_client.hxx"
#include "http_response.hxx"
#include "async.hxx"
#include "fd-util.h"
#include "growing_buffer.hxx"
#include "header_writer.hxx"
#include "lease.hxx"
#include "direct.hxx"
#include "istream/istream_internal.hxx"
#include "fd_util.h"
#include "fb_pool.hxx"
#include "util/Cast.hxx"

#include <glib.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <event.h>

static inline GQuark
test_quark(void)
{
    return g_quark_from_static_string("test");
}

static int
connect_fake_server(void)
{
    int ret, sv[2];
    pid_t pid;

    ret = socketpair_cloexec(AF_UNIX, SOCK_STREAM, 0, sv);
    if (ret < 0) {
        perror("socketpair() failed");
        exit(EXIT_FAILURE);
    }

    pid = fork();
    if (pid < 0) {
        perror("fork() failed");
        exit(EXIT_FAILURE);
    }

    if (pid == 0) {
        dup2(sv[1], 0);
        dup2(sv[1], 1);
        close(sv[0]);
        close(sv[1]);
        execl("./test/fake_memcached_server", "fake_memcached_server",
              nullptr);
        perror("exec() failed");
        exit(EXIT_FAILURE);
    }

    close(sv[1]);

    fd_set_nonblock(sv[0], true);

    return sv[0];
}

struct context {
    struct pool *pool;

    unsigned data_blocking;
    bool close_value_early, close_value_late, close_value_data;
    struct async_operation_ref async_ref;
    int fd;
    bool released, reuse, got_response;
    enum memcached_response_status status;

    struct istream *delayed;

    struct istream *value;
    off_t value_data, consumed_value_data;
    bool value_eof, value_abort, value_closed;

    struct istream request_value;
};

/*
 * lease
 *
 */

static void
my_release(bool reuse, void *ctx)
{
    struct context *c = (struct context *)ctx;

    close(c->fd);
    c->fd = -1;
    c->released = true;
    c->reuse = reuse;
}

static const struct lease my_lease = {
    .release = my_release,
};

/*
 * request value istream
 *
 */

static char request_value[8192];

struct request_value {
    struct istream base;

    struct async_operation_ref async_ref;

    bool read_close, read_abort;

    size_t sent;
};

static inline struct request_value *
istream_to_value(struct istream *istream)
{
    return &ContainerCast2(*istream, &request_value::base);
}

static off_t
istream_request_value_available(struct istream *istream, gcc_unused bool partial)
{
    const struct request_value *v = istream_to_value(istream);

    return sizeof(request_value) - v->sent;
}

static void
istream_request_value_read(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    if (v->read_close) {
        GError *error = g_error_new_literal(test_quark(), 0, "read_close");
        istream_deinit_abort(&v->base, error);
    } else if (v->read_abort)
        v->async_ref.Abort();
    else if (v->sent >= sizeof(request_value))
        istream_deinit_eof(&v->base);
    else {
        size_t nbytes =
            istream_invoke_data(&v->base, request_value + v->sent,
                                sizeof(request_value) - v->sent);
        if (nbytes == 0)
            return;

        v->sent += nbytes;

        if (v->sent >= sizeof(request_value))
            istream_deinit_eof(&v->base);
    }
}

static void
istream_request_value_close(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    istream_deinit(&v->base);
}

static const struct istream_class istream_request_value = {
    .available = istream_request_value_available,
    .read = istream_request_value_read,
    .close = istream_request_value_close,
};

static struct istream *
request_value_new(struct pool *pool, bool read_close, bool read_abort)
{
    struct request_value *v = (struct request_value *)
        istream_new(pool, &istream_request_value, sizeof(*v));

    v->read_close = read_close;
    v->read_abort = read_abort;
    v->sent = 0;

    return &v->base;
}

static struct async_operation_ref *
request_value_async_ref(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    return &v->async_ref;
}

/*
 * istream handler
 *
 */

static size_t
my_istream_data(gcc_unused const void *data, size_t length, void *ctx)
{
    struct context *c = (struct context *)ctx;

    c->value_data += length;

    if (c->close_value_data) {
        c->value_closed = true;
        istream_free_handler(&c->value);
        return 0;
    }

    if (c->data_blocking) {
        --c->data_blocking;
        return 0;
    }

    c->consumed_value_data += length;
    return length;
}

static void
my_istream_eof(void *ctx)
{
    struct context *c = (struct context *)ctx;

    c->value = NULL;
    c->value_eof = true;
}

static void
my_istream_abort(GError *error, void *ctx)
{
    struct context *c = (struct context *)ctx;

    g_error_free(error);

    c->value = NULL;
    c->value_abort = true;
}

static const struct istream_handler my_istream_handler = {
    .data = my_istream_data,
    .eof = my_istream_eof,
    .abort = my_istream_abort,
};

/*
 * memcached_response_handler_t
 *
 */

static void
my_mcd_response(enum memcached_response_status status,
                gcc_unused const void *extras,
                gcc_unused size_t extras_length,
                gcc_unused const void *key,
                gcc_unused size_t key_length,
                struct istream *value, void *ctx)
{
    struct context *c = (struct context *)ctx;

    assert(!c->got_response);

    c->got_response = true;
    c->status = status;

    if (c->close_value_early)
        istream_close_unused(value);
    else if (value != NULL)
        istream_assign_handler(&c->value, value, &my_istream_handler, c, 0);

    if (c->close_value_late) {
        c->value_closed = true;
        istream_free_handler(&c->value);
    }
}

static void
my_mcd_error(GError *error, gcc_unused void *ctx)
{
    struct context *c = (struct context *)ctx;

    assert(!c->got_response);

    g_error_free(error);

    c->got_response = true;
    c->status = (memcached_response_status)-1;
}

static const struct memcached_client_handler my_mcd_handler = {
    .response = my_mcd_response,
    .error = my_mcd_error,
};

/*
 * tests
 *
 */

static void
test_basic(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(c->value_eof);
    assert(!c->value_abort);
}

static void
test_close_early(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_early = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_data == 0);
}

static void
test_close_late(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_late = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_closed);
    assert(c->value_data == 0);
}

static void
test_close_data(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_data = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_closed);
    assert(c->value_data > 0);
}

static void
test_abort(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_data = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    c->async_ref.Abort();

    assert(!c->got_response);
    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
}

static void
test_request_value(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, false, false);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(c->value_eof);
    assert(!c->value_abort);
}

static void
test_request_value_close(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, true, false);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
}

static void
test_request_value_abort(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, false, true);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
}

/*
 * main
 *
 */

static void
run_test(struct pool *pool, void (*test)(struct pool *pool, struct context *c)) {
    struct context c;

    memset(&c, 0, sizeof(c));
    c.pool = pool_new_linear(pool, "test", 16384);
    test(c.pool, &c);
    pool_commit();
}

int main(int argc, char **argv) {
    struct event_base *event_base;
    struct pool *pool;

    (void)argc;
    (void)argv;

    signal(SIGPIPE, SIG_IGN);

    direct_global_init();
    event_base = event_init();
    fb_pool_init(false);

    pool = pool_new_libc(NULL, "root");

    run_test(pool, test_basic);
    run_test(pool, test_close_early);
    run_test(pool, test_close_late);
    run_test(pool, test_close_data);
    run_test(pool, test_abort);
    run_test(pool, test_request_value);
    run_test(pool, test_request_value_close);
    run_test(pool, test_request_value_abort);

    pool_unref(pool);
    pool_commit();
    pool_recycler_clear();

    fb_pool_deinit();
    event_base_free(event_base);
    direct_global_deinit();
}
<commit_msg>test/t_memcached_client: use C++11 initializer instead of memset()<commit_after>#include "memcached/memcached_client.hxx"
#include "http_response.hxx"
#include "async.hxx"
#include "fd-util.h"
#include "growing_buffer.hxx"
#include "header_writer.hxx"
#include "lease.hxx"
#include "direct.hxx"
#include "istream/istream_internal.hxx"
#include "fd_util.h"
#include "fb_pool.hxx"
#include "util/Cast.hxx"

#include <glib.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <event.h>

static inline GQuark
test_quark(void)
{
    return g_quark_from_static_string("test");
}

static int
connect_fake_server(void)
{
    int ret, sv[2];
    pid_t pid;

    ret = socketpair_cloexec(AF_UNIX, SOCK_STREAM, 0, sv);
    if (ret < 0) {
        perror("socketpair() failed");
        exit(EXIT_FAILURE);
    }

    pid = fork();
    if (pid < 0) {
        perror("fork() failed");
        exit(EXIT_FAILURE);
    }

    if (pid == 0) {
        dup2(sv[1], 0);
        dup2(sv[1], 1);
        close(sv[0]);
        close(sv[1]);
        execl("./test/fake_memcached_server", "fake_memcached_server",
              nullptr);
        perror("exec() failed");
        exit(EXIT_FAILURE);
    }

    close(sv[1]);

    fd_set_nonblock(sv[0], true);

    return sv[0];
}

struct context {
    struct pool *pool;

    unsigned data_blocking = 0;
    bool close_value_early = false;
    bool close_value_late = false;
    bool close_value_data = false;
    struct async_operation_ref async_ref;
    int fd = -1;
    bool released = false, reuse = false, got_response = false;
    enum memcached_response_status status;

    struct istream *delayed = nullptr;

    struct istream *value = nullptr;
    off_t value_data = 0, consumed_value_data = 0;
    bool value_eof = false, value_abort = false, value_closed = false;

    struct istream request_value;
};

/*
 * lease
 *
 */

static void
my_release(bool reuse, void *ctx)
{
    struct context *c = (struct context *)ctx;

    close(c->fd);
    c->fd = -1;
    c->released = true;
    c->reuse = reuse;
}

static const struct lease my_lease = {
    .release = my_release,
};

/*
 * request value istream
 *
 */

static char request_value[8192];

struct request_value {
    struct istream base;

    struct async_operation_ref async_ref;

    bool read_close, read_abort;

    size_t sent;
};

static inline struct request_value *
istream_to_value(struct istream *istream)
{
    return &ContainerCast2(*istream, &request_value::base);
}

static off_t
istream_request_value_available(struct istream *istream, gcc_unused bool partial)
{
    const struct request_value *v = istream_to_value(istream);

    return sizeof(request_value) - v->sent;
}

static void
istream_request_value_read(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    if (v->read_close) {
        GError *error = g_error_new_literal(test_quark(), 0, "read_close");
        istream_deinit_abort(&v->base, error);
    } else if (v->read_abort)
        v->async_ref.Abort();
    else if (v->sent >= sizeof(request_value))
        istream_deinit_eof(&v->base);
    else {
        size_t nbytes =
            istream_invoke_data(&v->base, request_value + v->sent,
                                sizeof(request_value) - v->sent);
        if (nbytes == 0)
            return;

        v->sent += nbytes;

        if (v->sent >= sizeof(request_value))
            istream_deinit_eof(&v->base);
    }
}

static void
istream_request_value_close(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    istream_deinit(&v->base);
}

static const struct istream_class istream_request_value = {
    .available = istream_request_value_available,
    .read = istream_request_value_read,
    .close = istream_request_value_close,
};

static struct istream *
request_value_new(struct pool *pool, bool read_close, bool read_abort)
{
    struct request_value *v = (struct request_value *)
        istream_new(pool, &istream_request_value, sizeof(*v));

    v->read_close = read_close;
    v->read_abort = read_abort;
    v->sent = 0;

    return &v->base;
}

static struct async_operation_ref *
request_value_async_ref(struct istream *istream)
{
    struct request_value *v = istream_to_value(istream);

    return &v->async_ref;
}

/*
 * istream handler
 *
 */

static size_t
my_istream_data(gcc_unused const void *data, size_t length, void *ctx)
{
    struct context *c = (struct context *)ctx;

    c->value_data += length;

    if (c->close_value_data) {
        c->value_closed = true;
        istream_free_handler(&c->value);
        return 0;
    }

    if (c->data_blocking) {
        --c->data_blocking;
        return 0;
    }

    c->consumed_value_data += length;
    return length;
}

static void
my_istream_eof(void *ctx)
{
    struct context *c = (struct context *)ctx;

    c->value = NULL;
    c->value_eof = true;
}

static void
my_istream_abort(GError *error, void *ctx)
{
    struct context *c = (struct context *)ctx;

    g_error_free(error);

    c->value = NULL;
    c->value_abort = true;
}

static const struct istream_handler my_istream_handler = {
    .data = my_istream_data,
    .eof = my_istream_eof,
    .abort = my_istream_abort,
};

/*
 * memcached_response_handler_t
 *
 */

static void
my_mcd_response(enum memcached_response_status status,
                gcc_unused const void *extras,
                gcc_unused size_t extras_length,
                gcc_unused const void *key,
                gcc_unused size_t key_length,
                struct istream *value, void *ctx)
{
    struct context *c = (struct context *)ctx;

    assert(!c->got_response);

    c->got_response = true;
    c->status = status;

    if (c->close_value_early)
        istream_close_unused(value);
    else if (value != NULL)
        istream_assign_handler(&c->value, value, &my_istream_handler, c, 0);

    if (c->close_value_late) {
        c->value_closed = true;
        istream_free_handler(&c->value);
    }
}

static void
my_mcd_error(GError *error, gcc_unused void *ctx)
{
    struct context *c = (struct context *)ctx;

    assert(!c->got_response);

    g_error_free(error);

    c->got_response = true;
    c->status = (memcached_response_status)-1;
}

static const struct memcached_client_handler my_mcd_handler = {
    .response = my_mcd_response,
    .error = my_mcd_error,
};

/*
 * tests
 *
 */

static void
test_basic(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(c->value_eof);
    assert(!c->value_abort);
}

static void
test_close_early(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_early = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_data == 0);
}

static void
test_close_late(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_late = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_closed);
    assert(c->value_data == 0);
}

static void
test_close_data(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_data = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
    assert(c->value_closed);
    assert(c->value_data > 0);
}

static void
test_abort(struct pool *pool, struct context *c)
{
    c->fd = connect_fake_server();
    c->close_value_data = true;

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            NULL,
                            &my_mcd_handler, c,
                            &c->async_ref);
    pool_unref(pool);
    pool_commit();

    c->async_ref.Abort();

    assert(!c->got_response);
    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
    assert(c->value == NULL);
    assert(!c->value_eof);
    assert(!c->value_abort);
}

static void
test_request_value(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, false, false);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(c->reuse);
    assert(c->fd < 0);
    assert(c->status == MEMCACHED_STATUS_NO_ERROR);
    assert(c->value == NULL);
    assert(c->value_eof);
    assert(!c->value_abort);
}

static void
test_request_value_close(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, true, false);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
}

static void
test_request_value_abort(struct pool *pool, struct context *c)
{
    struct istream *value;

    c->fd = connect_fake_server();

    value = request_value_new(c->pool, false, true);

    memcached_client_invoke(pool, c->fd, FdType::FD_SOCKET, &my_lease, c,
                            MEMCACHED_OPCODE_SET,
                            NULL, 0,
                            "foo", 3,
                            value,
                            &my_mcd_handler, c,
                            request_value_async_ref(value));
    pool_unref(pool);
    pool_commit();

    event_dispatch();

    assert(c->released);
    assert(!c->reuse);
    assert(c->fd < 0);
}

/*
 * main
 *
 */

static void
run_test(struct pool *pool, void (*test)(struct pool *pool, struct context *c)) {
    struct context c;

    c.pool = pool_new_linear(pool, "test", 16384);
    test(c.pool, &c);
    pool_commit();
}

int main(int argc, char **argv) {
    struct event_base *event_base;
    struct pool *pool;

    (void)argc;
    (void)argv;

    signal(SIGPIPE, SIG_IGN);

    direct_global_init();
    event_base = event_init();
    fb_pool_init(false);

    pool = pool_new_libc(NULL, "root");

    run_test(pool, test_basic);
    run_test(pool, test_close_early);
    run_test(pool, test_close_late);
    run_test(pool, test_close_data);
    run_test(pool, test_abort);
    run_test(pool, test_request_value);
    run_test(pool, test_request_value_close);
    run_test(pool, test_request_value_abort);

    pool_unref(pool);
    pool_commit();
    pool_recycler_clear();

    fb_pool_deinit();
    event_base_free(event_base);
    direct_global_deinit();
}
<|endoftext|>


<commit_before>/*
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Willow Garage, Inc.
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of Willow Garage, Inc. nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *
 * $Id: convex_hull.hpp 36101 2011-02-21 19:39:30Z aaldoma $
 *
 */

#ifndef PCL_SURFACE_IMPL_CONCAVE_HULL_H_
#define PCL_SURFACE_IMPL_CONCAVE_HULL_H_

#include <pcl/surface/concave_hull.h>
#include <pcl/common/common.h>
#include <pcl/registration/transforms.h>

//////////////////////////////////////////////////////////////////////////
template<typename PointInT> void
pcl::ConcaveHull<PointInT>::reconstruct (PointCloud &output)
{
  output.header = input_->header;
  if (!initCompute ())
  {
    output.points.clear ();
    return;
  }

  // Perform the actual surface reconstruction
  std::vector<pcl::Vertices> polygons;
  performReconstruction (output, polygons);

  output.width = output.points.size ();
  output.height = 1;
  output.is_dense = true;

  deinitCompute ();
}

//////////////////////////////////////////////////////////////////////////
template<typename PointInT> void
pcl::ConcaveHull<PointInT>::reconstruct (PointCloud &points,
                                         std::vector<pcl::Vertices> &polygons)
{
  points.header = input_->header;
  if (!initCompute ())
  {
    points.points.clear ();
    return;
  }

  // Perform the actual surface reconstruction
  performReconstruction (points, polygons);

  points.width = points.points.size ();
  points.height = 1;
  points.is_dense = true;

  deinitCompute ();
}

//////////////////////////////////////////////////////////////////////////
template<typename PointInT>
void
pcl::ConcaveHull<PointInT>::performReconstruction (PointCloud &alpha_shape,
                                                   std::vector<pcl::Vertices> &polygons)
{
  EIGEN_ALIGN16 Eigen::Matrix3f covariance_matrix;
  Eigen::Vector4f xyz_centroid;
  compute3DCentroid (*input_, *indices_, xyz_centroid);
  computeCovarianceMatrix (*input_, *indices_, xyz_centroid, covariance_matrix);
  EIGEN_ALIGN16 Eigen::Vector3f eigen_values;
  EIGEN_ALIGN16 Eigen::Matrix3f eigen_vectors;
  pcl::eigen33 (covariance_matrix, eigen_vectors, eigen_values);

  Eigen::Affine3f transform1;
  transform1.setIdentity();
  int dim = 3;
  if (eigen_values[0] / eigen_values[2] < 1.0e-5)
  {
   //we have points laying on a plane, using 2d convex hull
   //compute transformation bring eigen_vectors.col(i) to z-axis

   eigen_vectors.col (2) = eigen_vectors.col(0).cross(eigen_vectors.col(1));
   eigen_vectors.col (1) = eigen_vectors.col(2).cross(eigen_vectors.col(0));

   transform1 (0, 2) = eigen_vectors (0, 0);
   transform1 (1, 2) = eigen_vectors (1, 0);
   transform1 (2, 2) = eigen_vectors (2, 0);

   transform1 (0, 1) = eigen_vectors (0, 1);
   transform1 (1, 1) = eigen_vectors (1, 1);
   transform1 (2, 1) = eigen_vectors (2, 1);
   transform1 (0, 0) = eigen_vectors (0, 2);
   transform1 (1, 0) = eigen_vectors (1, 2);
   transform1 (2, 0) = eigen_vectors (2, 2);

   transform1 = transform1.inverse ();
   dim = 2;
  }
  else
  {
   transform1.setIdentity ();
  }

  // For 3D point clouds, alpha shapes is not yet implemented!
  if (dim == 3)
  {
    ROS_WARN ("Alpha shapes for 3-dimensional point clouds not yet implemented..., returning empty PolygonMesh and alpha_shape_points");
    alpha_shape.points.resize (0);
    alpha_shape.width = alpha_shape.height = 0;
    polygons.resize (0);
    return;
  }

  PointCloud cloud_transformed;
  pcl::demeanPointCloud(*input_, *indices_,xyz_centroid,cloud_transformed);
  pcl::transformPointCloud (cloud_transformed, cloud_transformed, transform1);

  // True if qhull should free points in qh_freeqhull() or reallocation
  boolT ismalloc = True;
  // option flags for qhull, see qh_opt.htm
  char flags[] = "qhull d QJ";
  // output from qh_produce_output(), use NULL to skip qh_produce_output()
  FILE *outfile = NULL;
  // error messages from qhull code
  FILE *errfile = stderr;
  // 0 if no error from qhull
  int exitcode;

  // Array of coordinates for each point
  coordT *points = (coordT *)calloc (cloud_transformed.points.size () * dim, sizeof (coordT));

  for (size_t i = 0; i < cloud_transformed.points.size (); ++i)
  {
    points[i * dim + 0] = (coordT)cloud_transformed.points[i].x;
    points[i * dim + 1] = (coordT)cloud_transformed.points[i].y;

    if (dim > 2)
      points[i * dim + 2] = (coordT)cloud_transformed.points[i].z;
  }

  // Compute concave hull
  exitcode = qh_new_qhull (dim, cloud_transformed.points.size (), points, ismalloc, flags,
                           outfile, errfile);
  qh_setvoronoi_all ();

  int num_vertices = qh num_vertices;
  alpha_shape.points.resize (num_vertices);

  vertexT *vertex;
  // Max vertex id
  int max_vertex_id=-1;
  FORALLvertices
  {
    if (vertex->id > max_vertex_id)
      max_vertex_id = vertex->id;
  }

  facetT *facet; // set by FORALLfacets
  int non_upperdelaunay=0;
  FORALLfacets
  {
    if (!facet->upperdelaunay)
      non_upperdelaunay++;
  }

  ++max_vertex_id;
  std::vector<int> qhid_to_pcidx (max_vertex_id);

  int num_facets = qh num_facets;
  int dd = 0;
  // Compute the alpha complex for the set of points
  // Filters the delaunay triangles
  setT* edges_set = qh_settemp (3 * num_facets);
  voronoi_centers_.points.resize (num_facets);
  FORALLfacets
  {
    // Facets are the delaunay triangles (2d) or tetrahedrons (3d)
    if (!facet->upperdelaunay)
    {
      // Check if the distance from any vertex to the facet->center
      // (center of the voronoi cell) is smaller than alpha
      vertexT* anyVertex = (vertexT *)(facet->vertices->e[0].p);
      double r = computeDistVertexCenter2D (anyVertex, facet->center);
      if (r < alpha_)
      {
        pcl::Vertices facet_vertices;
        qh_makeridges (facet);
        facet->good = true;

        ridgeT *ridge, **ridgep;
        FOREACHridge_(facet->ridges)
          qh_setappend (&edges_set, ridge);

        voronoi_centers_.points[dd].x = facet->center[0];
        voronoi_centers_.points[dd].y = facet->center[1];
        voronoi_centers_.points[dd].z = 0;
        ++dd;
      }
      else
        facet->good = false;
    }
  }

  int vertices = 0;
  std::vector<bool> added_vertices (max_vertex_id, false);
  ridgeT *ridge, **ridgep;
  FOREACHridge_(edges_set)
  {
    if (ridge->bottom->upperdelaunay || ridge->top->upperdelaunay || !ridge->top->good || !ridge->bottom->good)
    {
      int vertex_n, vertex_i;
      FOREACHvertex_i_((*ridge).vertices)
      {
        if (!added_vertices[vertex->id])
        {
          alpha_shape.points[vertices].x = vertex->point[0];
          alpha_shape.points[vertices].y = vertex->point[1];

          if (dim > 2)
            alpha_shape.points[vertices].z = vertex->point[2];
          else
            alpha_shape.points[vertices].z = 0;

          qhid_to_pcidx[vertex->id] = vertices; //map the vertex id of qhull to the point cloud index
          added_vertices[vertex->id] = true;
          vertices++;
        }
      }
    }
  }

  alpha_shape.points.resize (vertices);
  polygons.resize (dd);
  voronoi_centers_.points.resize (dd);

  if (dim == 2)
  {
    Eigen::Affine3f transInverse = transform1.inverse ();
    pcl::transformPointCloud (alpha_shape, alpha_shape, transInverse);
    xyz_centroid[0] = -xyz_centroid[0];
    xyz_centroid[1] = -xyz_centroid[1];
    xyz_centroid[2] = -xyz_centroid[2];
    pcl::demeanPointCloud(alpha_shape,xyz_centroid,alpha_shape);
  }
}

#define PCL_INSTANTIATE_ConcaveHull(T) template class pcl::ConcaveHull<T>;

#endif    // PCL_SURFACE_IMPL_CONCAVE_HULL_H_
<commit_msg>Fixes issue 90. The points in the concave hull were not being sorted. Now one or more unconnected sorted polygons are returned.<commit_after>/*
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Willow Garage, Inc.
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of Willow Garage, Inc. nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *
 * $Id: convex_hull.hpp 36101 2011-02-21 19:39:30Z aaldoma $
 *
 */

#ifndef PCL_SURFACE_IMPL_CONCAVE_HULL_H_
#define PCL_SURFACE_IMPL_CONCAVE_HULL_H_

#include <pcl/surface/concave_hull.h>
#include <pcl/common/common.h>
#include <pcl/registration/transforms.h>

//////////////////////////////////////////////////////////////////////////
template<typename PointInT> void
pcl::ConcaveHull<PointInT>::reconstruct (PointCloud &output)
{
  output.header = input_->header;
  if (!initCompute ())
  {
    output.points.clear ();
    return;
  }

  // Perform the actual surface reconstruction
  std::vector<pcl::Vertices> polygons;
  performReconstruction (output, polygons);

  output.width = output.points.size ();
  output.height = 1;
  output.is_dense = true;

  deinitCompute ();
}

//////////////////////////////////////////////////////////////////////////
template<typename PointInT> void
pcl::ConcaveHull<PointInT>::reconstruct (PointCloud &points,
                                         std::vector<pcl::Vertices> &polygons)
{
  points.header = input_->header;
  if (!initCompute ())
  {
    points.points.clear ();
    return;
  }

  // Perform the actual surface reconstruction
  performReconstruction (points, polygons);

  points.width = points.points.size ();
  points.height = 1;
  points.is_dense = true;

  deinitCompute ();
}

//////////////////////////////////////////////////////////////////////////
template<typename PointInT>
void
pcl::ConcaveHull<PointInT>::performReconstruction (PointCloud &alpha_shape,
                                                   std::vector<pcl::Vertices> &polygons)
{
  EIGEN_ALIGN16 Eigen::Matrix3f covariance_matrix;
  Eigen::Vector4f xyz_centroid;
  compute3DCentroid (*input_, *indices_, xyz_centroid);
  computeCovarianceMatrix (*input_, *indices_, xyz_centroid, covariance_matrix);
  EIGEN_ALIGN16 Eigen::Vector3f eigen_values;
  EIGEN_ALIGN16 Eigen::Matrix3f eigen_vectors;
  pcl::eigen33 (covariance_matrix, eigen_vectors, eigen_values);

  Eigen::Affine3f transform1;
  transform1.setIdentity();
  int dim = 3;
  if (eigen_values[0] / eigen_values[2] < 1.0e-5)
  {
   //we have points laying on a plane, using 2d convex hull
   //compute transformation bring eigen_vectors.col(i) to z-axis

   eigen_vectors.col (2) = eigen_vectors.col(0).cross(eigen_vectors.col(1));
   eigen_vectors.col (1) = eigen_vectors.col(2).cross(eigen_vectors.col(0));

   transform1 (0, 2) = eigen_vectors (0, 0);
   transform1 (1, 2) = eigen_vectors (1, 0);
   transform1 (2, 2) = eigen_vectors (2, 0);

   transform1 (0, 1) = eigen_vectors (0, 1);
   transform1 (1, 1) = eigen_vectors (1, 1);
   transform1 (2, 1) = eigen_vectors (2, 1);
   transform1 (0, 0) = eigen_vectors (0, 2);
   transform1 (1, 0) = eigen_vectors (1, 2);
   transform1 (2, 0) = eigen_vectors (2, 2);

   transform1 = transform1.inverse ();
   dim = 2;
  }
  else
  {
   transform1.setIdentity ();
  }

  // For 3D point clouds, alpha shapes is not yet implemented!
  if (dim == 3)
  {
    ROS_WARN ("Alpha shapes for 3-dimensional point clouds not yet implemented..., returning empty PolygonMesh and alpha_shape_points");
    alpha_shape.points.resize (0);
    alpha_shape.width = alpha_shape.height = 0;
    polygons.resize (0);
    return;
  }

  PointCloud cloud_transformed;
  pcl::demeanPointCloud(*input_, *indices_,xyz_centroid,cloud_transformed);
  pcl::transformPointCloud (cloud_transformed, cloud_transformed, transform1);

  // True if qhull should free points in qh_freeqhull() or reallocation
  boolT ismalloc = True;
  // option flags for qhull, see qh_opt.htm
  char flags[] = "qhull d QJ";
  // output from qh_produce_output(), use NULL to skip qh_produce_output()
  FILE *outfile = NULL;
  // error messages from qhull code
  FILE *errfile = stderr;
  // 0 if no error from qhull
  int exitcode;

  // Array of coordinates for each point
  coordT *points = (coordT *)calloc (cloud_transformed.points.size () * dim, sizeof (coordT));

  for (size_t i = 0; i < cloud_transformed.points.size (); ++i)
  {
    points[i * dim + 0] = (coordT)cloud_transformed.points[i].x;
    points[i * dim + 1] = (coordT)cloud_transformed.points[i].y;

    if (dim > 2)
      points[i * dim + 2] = (coordT)cloud_transformed.points[i].z;
  }

  // Compute concave hull
  exitcode = qh_new_qhull (dim, cloud_transformed.points.size (), points, ismalloc, flags,
                           outfile, errfile);
  qh_setvoronoi_all ();

  int num_vertices = qh num_vertices;
  alpha_shape.points.resize (num_vertices);

  vertexT *vertex;
  // Max vertex id
  int max_vertex_id=-1;
  FORALLvertices
  {
    if (vertex->id > max_vertex_id)
      max_vertex_id = vertex->id;
  }

  facetT *facet; // set by FORALLfacets
  int non_upperdelaunay=0;
  FORALLfacets
  {
    if (!facet->upperdelaunay)
      non_upperdelaunay++;
  }

  ++max_vertex_id;
  std::vector<int> qhid_to_pcidx (max_vertex_id);

  int num_facets = qh num_facets;
  int dd = 0;
  // Compute the alpha complex for the set of points
  // Filters the delaunay triangles
  setT* edges_set = qh_settemp (3 * num_facets);
  voronoi_centers_.points.resize (num_facets);
  FORALLfacets
  {
    // Facets are the delaunay triangles (2d) or tetrahedrons (3d)
    if (!facet->upperdelaunay)
    {
      // Check if the distance from any vertex to the facet->center
      // (center of the voronoi cell) is smaller than alpha
      vertexT* anyVertex = (vertexT *)(facet->vertices->e[0].p);
      double r = computeDistVertexCenter2D (anyVertex, facet->center);
      if (r < alpha_)
      {
        pcl::Vertices facet_vertices;
        qh_makeridges (facet);
        facet->good = true;

        ridgeT *ridge, **ridgep;
        FOREACHridge_(facet->ridges)
          qh_setappend (&edges_set, ridge);

        voronoi_centers_.points[dd].x = facet->center[0];
        voronoi_centers_.points[dd].y = facet->center[1];
        voronoi_centers_.points[dd].z = 0;
        ++dd;
      }
      else
        facet->good = false;
    }
  }

  int vertices = 0;
  std::vector<bool> added_vertices (max_vertex_id, false);
  std::map<int, std::vector<int> > edges;

  ridgeT * ridge, **ridgep;
  FOREACHridge_(edges_set)
  {
    if (ridge->bottom->upperdelaunay || ridge->top->upperdelaunay || !ridge->top->good || !ridge->bottom->good)
    {
      int vertex_n, vertex_i;
      int vertices_in_ridge=0;
      std::vector<int> pcd_indices;
      pcd_indices.resize(2);

      FOREACHvertex_i_((*ridge).vertices) //in 2-dim, 2 vertices per ridge!

      {
        if (!added_vertices[vertex->id])
        {
          alpha_shape.points[vertices].x = vertex->point[0];
          alpha_shape.points[vertices].y = vertex->point[1];

          if (dim > 2)
          alpha_shape.points[vertices].z = vertex->point[2];
          else
          alpha_shape.points[vertices].z = 0;

          qhid_to_pcidx[vertex->id] = vertices; //map the vertex id of qhull to the point cloud index
          added_vertices[vertex->id] = true;
          pcd_indices[vertices_in_ridge] = vertices;
          vertices++;
        }
        else
        {
          pcd_indices[vertices_in_ridge] = qhid_to_pcidx[vertex->id];
        }

        vertices_in_ridge++;
      }

      //make edges bidirectional and pointing to alpha_shape pointcloud...
      edges[pcd_indices[0]].push_back(pcd_indices[1]);
      edges[pcd_indices[1]].push_back(pcd_indices[0]);
    }
  }

  alpha_shape.points.resize (vertices);

  std::vector < std::vector<int> > connected;
  PointCloud alpha_shape_sorted;
  alpha_shape_sorted.points.resize (vertices);

  //iterate over edges until they are empty!
  std::map<int, std::vector<int> >::iterator curr = edges.begin ();
  int next;
  std::vector<bool> used (vertices, false); //used to decide which direction should we take!
  std::vector<int> pcd_idx_start_polygons;
  pcd_idx_start_polygons.push_back (0);

  //start following edges and removing elements
  int sorted_idx = 0;
  while (!edges.empty ())
  {
    alpha_shape_sorted.points[sorted_idx] = alpha_shape.points[(*curr).first];
    //check where we can go from (*curr).first
    for (size_t i = 0; i < (*curr).second.size (); i++)
    {
      if (!used[((*curr).second)[i]])
      {
        //we can go there
        next = ((*curr).second)[i];
        break;
      }
    }

    used[(*curr).first] = true;
    edges.erase (curr); //remove edges starting from curr

    sorted_idx++;

    if (edges.empty ())
      break;

    //reassign current
    curr = edges.find (next); //if next is not found, then we have unconnected polygons.
    if (curr == edges.end ())
    {
      //set current to any of the remaining in edge!
      curr = edges.begin ();
      pcd_idx_start_polygons.push_back (sorted_idx);
    }
  }

  pcd_idx_start_polygons.push_back (sorted_idx);

  alpha_shape.points = alpha_shape_sorted.points;

  polygons.resize (pcd_idx_start_polygons.size () - 1);

  for (size_t poly_id = 0; poly_id < polygons.size (); poly_id++)
  {
    //std::cout << "indices => start: " << pcd_idx_start_polygons[poly_id] << "end:" << pcd_idx_start_polygons[poly_id+1]  << std::endl;
    polygons[poly_id].vertices.resize (pcd_idx_start_polygons[poly_id + 1] - pcd_idx_start_polygons[poly_id] + 1);
    //populate points in the corresponding polygon
    for (size_t j = (size_t)pcd_idx_start_polygons[poly_id]; j < (size_t)pcd_idx_start_polygons[poly_id + 1]; ++j)
      polygons[poly_id].vertices[j - pcd_idx_start_polygons[poly_id]] = j;

    polygons[poly_id].vertices[polygons[poly_id].vertices.size () - 1] = pcd_idx_start_polygons[poly_id];
  }

  voronoi_centers_.points.resize (dd);

  Eigen::Affine3f transInverse = transform1.inverse ();
  pcl::transformPointCloud (alpha_shape, alpha_shape, transInverse);
  xyz_centroid[0] = -xyz_centroid[0];
  xyz_centroid[1] = -xyz_centroid[1];
  xyz_centroid[2] = -xyz_centroid[2];
  pcl::demeanPointCloud (alpha_shape, xyz_centroid, alpha_shape);

}

#define PCL_INSTANTIATE_ConcaveHull(T) template class pcl::ConcaveHull<T>;

#endif    // PCL_SURFACE_IMPL_CONCAVE_HULL_H_
<|endoftext|>