<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include "session_context.h"

namespace upload {

SessionContextBase::SessionContextBase()
    : upload_results_(1000, 1000),
      api_url_(),
      session_token_(),
      drop_lease_(true),
      max_pack_size_(ObjectPack::kDefaultLimit),
      current_pack_(NULL),
      mtx_(),
      stats_() {}

SessionContextBase::~SessionContextBase() {}

bool SessionContextBase::Initialize(const std::string& api_url,
                                    const std::string& session_token,
                                    bool drop_lease, uint64_t max_pack_size) {
  bool ret = true;

  // Initialize session context lock
  pthread_mutexattr_t attr;
  if (pthread_mutexattr_init(&attr) ||
      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
      pthread_mutex_init(&mtx_, &attr) || pthread_mutexattr_destroy(&attr)) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Could not initialize SessionContext lock.");
    return false;
  }

  {
    MutexLockGuard lock(mtx_);

    // Set upstream URL and session token
    api_url_ = api_url;
    session_token_ = session_token;
    drop_lease_ = drop_lease;
    max_pack_size_ = max_pack_size;

    // Reset internal counters
    stats_ = Stats();

    // Ensure that the upload job and result queues are empty
    if (!upload_results_.IsEmpty()) {
      LogCvmfs(
          kLogUploadGateway, kLogStderr,
          "Could not initialize SessionContext - Upload queues are not empty.");
      ret = false;
    }

    // Ensure that there are not open object packs
    if (current_pack_) {
      LogCvmfs(
          kLogUploadGateway, kLogStderr,
          "Could not initialize SessionContext - Existing open object packs.");
      ret = false;
    }

    ret = InitializeDerived() && ret;
  }

  return ret;
}

bool SessionContextBase::Finalize() {
  bool results = true;
  {
    MutexLockGuard lock(mtx_);

    if (drop_lease_ && !DropLease()) {
      LogCvmfs(kLogUploadGateway, kLogStderr,
               "SessionContext finalization - Could not drop active lease");
    }

    while (!upload_results_.IsEmpty() ||
           (stats_.jobs_finished < NumJobsSubmitted())) {
      Future<bool>* future = upload_results_.Dequeue();
      results = future->Get() && results;
      delete future;
      stats_.jobs_finished++;
    }

    results = FinalizeDerived() &&
              (stats_.bytes_committed == stats_.bytes_dispatched) && results;
  }

  pthread_mutex_destroy(&mtx_);
  return results;
}

ObjectPack::BucketHandle SessionContextBase::NewBucket() {
  MutexLockGuard lock(mtx_);
  ObjectPack::BucketHandle hd = CurrentPack()->NewBucket();
  stats_.buckets_created++;
  return hd;
}

bool SessionContextBase::CommitBucket(const ObjectPack::BucketContentType type,
                                      const shash::Any& id,
                                      const ObjectPack::BucketHandle handle,
                                      const std::string& name,
                                      const bool force_dispatch) {
  MutexLockGuard lock(mtx_);

  uint64_t size0 = CurrentPack()->size();
  bool committed = CurrentPack()->CommitBucket(type, id, handle, name);

  if (committed && !force_dispatch) {  // Current pack is still not full
    uint64_t size1 = CurrentPack()->size();
    stats_.buckets_committed++;
    stats_.bytes_committed += size1 - size0;
  } else {  // Current pack is full and can be dispatched
    ObjectPack* new_pack = new ObjectPack(max_pack_size_);
    CurrentPack()->TransferBucket(handle, new_pack);

    upload_results_.Enqueue(DispatchObjectPack(CurrentPack()));

    atomic_inc64(&stats_.objects_dispatched);
    stats_.bytes_dispatched += CurrentPack()->size();
    current_pack_ = new_pack;

    CommitBucket(type, id, handle, name, false);
  }
  return true;
}

ObjectPack* SessionContextBase::CurrentPack() {
  MutexLockGuard lock(mtx_);
  if (!current_pack_) {
    current_pack_ = new ObjectPack(max_pack_size_);
  }
  return current_pack_;
}

SessionContext::SessionContext()
    : SessionContextBase(), upload_jobs_(1000, 900) {}

bool SessionContext::InitializeDerived() {
  // Start worker thread
  atomic_init32(&worker_terminate_);
  atomic_write32(&worker_terminate_, 0);
  int retval =
      pthread_create(&worker_, NULL, UploadLoop, reinterpret_cast<void*>(this));

  return !retval;
}

bool SessionContext::FinalizeDerived() {
  atomic_write32(&worker_terminate_, 1);

  pthread_join(worker_, NULL);

  return true;
}

bool SessionContext::DropLease() { return true; }

Future<bool>* SessionContext::DispatchObjectPack(const ObjectPack* pack) {
  UploadJob* job = new UploadJob;
  job->pack = pack;
  job->result = new Future<bool>();
  upload_jobs_.Enqueue(job);
  return job->result;
}

bool SessionContext::DoUpload(const SessionContext::UploadJob* /*job*/) {
  return true;
}

void* SessionContext::UploadLoop(void* data) {
  SessionContext* ctx = reinterpret_cast<SessionContext*>(data);
  LogCvmfs(kLogUploadGateway, kLogStderr,
           "Upload session context - worker started with context @%ld", data);

  atomic_int64 jobs_processed;
  atomic_init64(&jobs_processed);
  while (!ctx->ShouldTerminate()) {
    while (jobs_processed < ctx->NumJobsSubmitted()) {
      UploadJob* job = ctx->upload_jobs_.Dequeue();
      ctx->DoUpload(job);
      jobs_processed++;
    }
  }

  return NULL;
}

bool SessionContext::ShouldTerminate() {
  return atomic_read32(&worker_terminate_);
}

}  // namespace upload
<commit_msg>SessionContext - only drop lease after all upload jobs are finished<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include "session_context.h"

namespace upload {

SessionContextBase::SessionContextBase()
    : upload_results_(1000, 1000),
      api_url_(),
      session_token_(),
      drop_lease_(true),
      max_pack_size_(ObjectPack::kDefaultLimit),
      current_pack_(NULL),
      mtx_(),
      stats_() {}

SessionContextBase::~SessionContextBase() {}

bool SessionContextBase::Initialize(const std::string& api_url,
                                    const std::string& session_token,
                                    bool drop_lease, uint64_t max_pack_size) {
  bool ret = true;

  // Initialize session context lock
  pthread_mutexattr_t attr;
  if (pthread_mutexattr_init(&attr) ||
      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
      pthread_mutex_init(&mtx_, &attr) || pthread_mutexattr_destroy(&attr)) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Could not initialize SessionContext lock.");
    return false;
  }

  {
    MutexLockGuard lock(mtx_);

    // Set upstream URL and session token
    api_url_ = api_url;
    session_token_ = session_token;
    drop_lease_ = drop_lease;
    max_pack_size_ = max_pack_size;

    // Reset internal counters
    stats_ = Stats();

    // Ensure that the upload job and result queues are empty
    if (!upload_results_.IsEmpty()) {
      LogCvmfs(
          kLogUploadGateway, kLogStderr,
          "Could not initialize SessionContext - Upload queues are not empty.");
      ret = false;
    }

    // Ensure that there are not open object packs
    if (current_pack_) {
      LogCvmfs(
          kLogUploadGateway, kLogStderr,
          "Could not initialize SessionContext - Existing open object packs.");
      ret = false;
    }

    ret = InitializeDerived() && ret;
  }

  return ret;
}

bool SessionContextBase::Finalize() {
  bool results = true;
  {
    MutexLockGuard lock(mtx_);

    while (!upload_results_.IsEmpty() ||
           (stats_.jobs_finished < NumJobsSubmitted())) {
      Future<bool>* future = upload_results_.Dequeue();
      results = future->Get() && results;
      delete future;
      stats_.jobs_finished++;
    }

    if (drop_lease_ && !DropLease()) {
      LogCvmfs(kLogUploadGateway, kLogStderr,
               "SessionContext finalization - Could not drop active lease");
    }

    results = FinalizeDerived() &&
              (stats_.bytes_committed == stats_.bytes_dispatched) && results;
  }

  pthread_mutex_destroy(&mtx_);
  return results;
}

ObjectPack::BucketHandle SessionContextBase::NewBucket() {
  MutexLockGuard lock(mtx_);
  ObjectPack::BucketHandle hd = CurrentPack()->NewBucket();
  stats_.buckets_created++;
  return hd;
}

bool SessionContextBase::CommitBucket(const ObjectPack::BucketContentType type,
                                      const shash::Any& id,
                                      const ObjectPack::BucketHandle handle,
                                      const std::string& name,
                                      const bool force_dispatch) {
  MutexLockGuard lock(mtx_);

  uint64_t size0 = CurrentPack()->size();
  bool committed = CurrentPack()->CommitBucket(type, id, handle, name);

  if (committed && !force_dispatch) {  // Current pack is still not full
    uint64_t size1 = CurrentPack()->size();
    stats_.buckets_committed++;
    stats_.bytes_committed += size1 - size0;
  } else {  // Current pack is full and can be dispatched
    ObjectPack* new_pack = new ObjectPack(max_pack_size_);
    CurrentPack()->TransferBucket(handle, new_pack);

    upload_results_.Enqueue(DispatchObjectPack(CurrentPack()));

    atomic_inc64(&stats_.objects_dispatched);
    stats_.bytes_dispatched += CurrentPack()->size();
    current_pack_ = new_pack;

    CommitBucket(type, id, handle, name, false);
  }
  return true;
}

ObjectPack* SessionContextBase::CurrentPack() {
  MutexLockGuard lock(mtx_);
  if (!current_pack_) {
    current_pack_ = new ObjectPack(max_pack_size_);
  }
  return current_pack_;
}

SessionContext::SessionContext()
    : SessionContextBase(), upload_jobs_(1000, 900) {}

bool SessionContext::InitializeDerived() {
  // Start worker thread
  atomic_init32(&worker_terminate_);
  atomic_write32(&worker_terminate_, 0);
  int retval =
      pthread_create(&worker_, NULL, UploadLoop, reinterpret_cast<void*>(this));

  return !retval;
}

bool SessionContext::FinalizeDerived() {
  atomic_write32(&worker_terminate_, 1);

  pthread_join(worker_, NULL);

  return true;
}

bool SessionContext::DropLease() { return true; }

Future<bool>* SessionContext::DispatchObjectPack(const ObjectPack* pack) {
  UploadJob* job = new UploadJob;
  job->pack = pack;
  job->result = new Future<bool>();
  upload_jobs_.Enqueue(job);
  return job->result;
}

bool SessionContext::DoUpload(const SessionContext::UploadJob* /*job*/) {
  return true;
}

void* SessionContext::UploadLoop(void* data) {
  SessionContext* ctx = reinterpret_cast<SessionContext*>(data);
  LogCvmfs(kLogUploadGateway, kLogStderr,
           "Upload session context - worker started with context @%ld", data);

  atomic_int64 jobs_processed;
  atomic_init64(&jobs_processed);
  while (!ctx->ShouldTerminate()) {
    while (jobs_processed < ctx->NumJobsSubmitted()) {
      UploadJob* job = ctx->upload_jobs_.Dequeue();
      ctx->DoUpload(job);
      jobs_processed++;
    }
  }

  return NULL;
}

bool SessionContext::ShouldTerminate() {
  return atomic_read32(&worker_terminate_);
}

}  // namespace upload
<|endoftext|>


<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include "session_context.h"

#include <algorithm>
#include <limits>

#include "curl/curl.h"
#include "cvmfs_config.h"
#include "gateway_util.h"
#include "json_document.h"
#include "json_document_write.h"
#include "swissknife_lease_curl.h"
#include "util/exception.h"
#include "util/pointer.h"
#include "util/string.h"

namespace {
// Maximum number of jobs during a session. No limit, for practical
// purposes.
const uint64_t kMaxNumJobs = std::numeric_limits<uint64_t>::max();
}

namespace upload {

size_t SendCB(void* ptr, size_t size, size_t nmemb, void* userp) {
  CurlSendPayload* payload = static_cast<CurlSendPayload*>(userp);

  size_t max_chunk_size = size * nmemb;
  if (max_chunk_size < 1) {
    return 0;
  }

  size_t current_chunk_size = 0;
  while (current_chunk_size < max_chunk_size) {
    if (payload->index < payload->json_message->size()) {
      // Can add a chunk from the JSON message
      const size_t read_size =
          std::min(max_chunk_size - current_chunk_size,
                   payload->json_message->size() - payload->index);
      current_chunk_size += read_size;
      std::memcpy(ptr, payload->json_message->data() + payload->index,
                  read_size);
      payload->index += read_size;
    } else {
      // Can add a chunk from the payload
      const size_t max_read_size = max_chunk_size - current_chunk_size;
      const unsigned nbytes = payload->pack_serializer->ProduceNext(
          max_read_size, static_cast<unsigned char*>(ptr) + current_chunk_size);
      current_chunk_size += nbytes;

      if (!nbytes) {
        break;
      }
    }
  }

  return current_chunk_size;
}

size_t RecvCB(void* buffer, size_t size, size_t nmemb, void* userp) {
  std::string* my_buffer = static_cast<std::string*>(userp);

  if (size * nmemb < 1) {
    return 0;
  }

  *my_buffer = static_cast<char*>(buffer);

  return my_buffer->size();
}

SessionContextBase::SessionContextBase()
    : upload_results_(kMaxNumJobs, kMaxNumJobs),
      api_url_(),
      session_token_(),
      key_id_(),
      secret_(),
      queue_was_flushed_(1, 1),
      max_pack_size_(ObjectPack::kDefaultLimit),
      active_handles_(),
      current_pack_(NULL),
      current_pack_mtx_(),
      objects_dispatched_(0),
      bytes_committed_(0),
      bytes_dispatched_(0),
      initialized_(false) {}

SessionContextBase::~SessionContextBase() {}

bool SessionContextBase::Initialize(const std::string& api_url,
                                    const std::string& session_token,
                                    const std::string& key_id,
                                    const std::string& secret,
                                    uint64_t max_pack_size,
                                    uint64_t max_queue_size) {
  bool ret = true;

  // Initialize session context lock
  pthread_mutexattr_t attr;
  if (pthread_mutexattr_init(&attr) ||
      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
      pthread_mutex_init(&current_pack_mtx_, &attr) ||
      pthread_mutexattr_destroy(&attr)) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Could not initialize SessionContext lock.");
    return false;
  }

  // Set upstream URL and session token
  api_url_ = api_url;
  session_token_ = session_token;
  key_id_ = key_id;
  secret_ = secret;
  max_pack_size_ = max_pack_size;

  atomic_init64(&objects_dispatched_);
  bytes_committed_ = 0u;
  bytes_dispatched_ = 0u;

  // Ensure that the upload job and result queues are empty
  upload_results_.Drop();

  queue_was_flushed_.Drop();
  queue_was_flushed_.Enqueue(true);

  // Ensure that there are not open object packs
  if (current_pack_) {
    LogCvmfs(
        kLogUploadGateway, kLogStderr,
        "Could not initialize SessionContext - Existing open object packs.");
    ret = false;
  }

  ret = InitializeDerived(max_queue_size) && ret;

  initialized_ = true;

  return ret;
}

bool SessionContextBase::Finalize(bool commit, const std::string& old_root_hash,
                                  const std::string& new_root_hash,
                                  const RepositoryTag& tag) {
  assert(active_handles_.empty());
  if (!initialized_) {
    assert(!commit);
    return true;
  }

  {
    MutexLockGuard lock(current_pack_mtx_);

    if (current_pack_ && current_pack_->GetNoObjects() > 0) {
      Dispatch();
      current_pack_ = NULL;
    }
  }

  bool results = true;
  int64_t jobs_finished = 0;
  while (!upload_results_.IsEmpty() || (jobs_finished < NumJobsSubmitted())) {
    Future<bool>* future = upload_results_.Dequeue();
    results = future->Get() && results;
    delete future;
    jobs_finished++;
  }

  if (commit) {
    if (old_root_hash.empty() || new_root_hash.empty()) {
      return false;
    }
    bool commit_result = Commit(old_root_hash, new_root_hash, tag);
    if (!commit_result) {
      LogCvmfs(kLogUploadGateway, kLogStderr,
               "SessionContext: could not commit session. Aborting.");
      FinalizeDerived();
      pthread_mutex_destroy(&current_pack_mtx_);
      initialized_ = false;
      return false;
    }
  }

  results &= FinalizeDerived() && (bytes_committed_ == bytes_dispatched_);

  pthread_mutex_destroy(&current_pack_mtx_);

  initialized_ = false;

  return results;
}

void SessionContextBase::WaitForUpload() {
  if (!upload_results_.IsEmpty()) {
    queue_was_flushed_.Dequeue();
  }
}

ObjectPack::BucketHandle SessionContextBase::NewBucket() {
  MutexLockGuard lock(current_pack_mtx_);
  if (!current_pack_) {
    current_pack_ = new ObjectPack(max_pack_size_);
  }
  ObjectPack::BucketHandle hd = current_pack_->NewBucket();
  active_handles_.push_back(hd);
  return hd;
}

bool SessionContextBase::CommitBucket(const ObjectPack::BucketContentType type,
                                      const shash::Any& id,
                                      const ObjectPack::BucketHandle handle,
                                      const std::string& name,
                                      const bool force_dispatch) {
  MutexLockGuard lock(current_pack_mtx_);

  if (!current_pack_) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Error: Called SessionBaseContext::CommitBucket without an open "
             "ObjectPack.");
    return false;
  }

  uint64_t size0 = current_pack_->size();
  bool committed = current_pack_->CommitBucket(type, id, handle, name);

  if (committed) {  // Current pack is still not full
    active_handles_.erase(
        std::remove(active_handles_.begin(), active_handles_.end(), handle),
        active_handles_.end());
    uint64_t size1 = current_pack_->size();
    bytes_committed_ += size1 - size0;
    if (force_dispatch) {
      Dispatch();
      current_pack_ = NULL;
    }
  } else {  // Current pack is full and can be dispatched
    uint64_t new_size = 0;
    if (handle->capacity > max_pack_size_) {
      new_size = handle->capacity + 1;
    } else {
      new_size = max_pack_size_;
    }
    ObjectPack* new_pack = new ObjectPack(new_size);
    for (size_t i = 0u; i < active_handles_.size(); ++i) {
      current_pack_->TransferBucket(active_handles_[i], new_pack);
    }

    if (current_pack_->GetNoObjects() > 0) {
      Dispatch();
    }
    current_pack_ = new_pack;

    CommitBucket(type, id, handle, name, false);
  }

  return true;
}

int64_t SessionContextBase::NumJobsSubmitted() const {
  return atomic_read64(&objects_dispatched_);
}

void SessionContextBase::Dispatch() {
  MutexLockGuard lock(current_pack_mtx_);

  if (!current_pack_) {
    return;
  }

  atomic_inc64(&objects_dispatched_);
  bytes_dispatched_ += current_pack_->size();
  upload_results_.Enqueue(DispatchObjectPack(current_pack_));
}

SessionContext::SessionContext()
    : SessionContextBase(),
      upload_jobs_(),
      worker_terminate_(),
      worker_()
{
  // Set to 0 in InitializeDerived()
  atomic_write32(&worker_terminate_, 1);
}

bool SessionContext::InitializeDerived(uint64_t max_queue_size) {
  // Start worker thread
  atomic_init32(&worker_terminate_);

  upload_jobs_ = new FifoChannel<UploadJob*>(max_queue_size, max_queue_size);
  upload_jobs_->Drop();

  int retval =
      pthread_create(&worker_, NULL, UploadLoop, reinterpret_cast<void*>(this));

  return !retval;
}

bool SessionContext::FinalizeDerived() {
  if (atomic_cas32(&worker_terminate_, 0, 1))
    pthread_join(worker_, NULL);

  return true;
}

bool SessionContext::Commit(const std::string& old_root_hash,
                            const std::string& new_root_hash,
                            const RepositoryTag& tag) {
  JsonStringGenerator request_input;
  request_input.Add("old_root_hash", old_root_hash);
  request_input.Add("new_root_hash", new_root_hash);
  request_input.Add("tag_name", tag.name_);
  request_input.Add("tag_channel", tag.channel_);
  request_input.Add("tag_description", tag.description_);
  std::string request = request_input.GenerateString();
  CurlBuffer buffer;
  return MakeEndRequest("POST", key_id_, secret_, session_token_, api_url_,
                        request, &buffer);
}

Future<bool>* SessionContext::DispatchObjectPack(ObjectPack* pack) {
  UploadJob* job = new UploadJob;
  job->pack = pack;
  job->result = new Future<bool>();
  upload_jobs_->Enqueue(job);
  return job->result;
}

bool SessionContext::DoUpload(const SessionContext::UploadJob* job) {
  // Set up the object pack serializer
  ObjectPackProducer serializer(job->pack);

  shash::Any payload_digest(shash::kSha1);
  serializer.GetDigest(&payload_digest);
  const std::string json_msg =
      "{\"session_token\" : \"" + session_token_ +
      "\", \"payload_digest\" : \"" + payload_digest.ToString(false) +
      "\", \"header_size\" : \"" + StringifyInt(serializer.GetHeaderSize()) +
      "\", \"api_version\" : \"" + StringifyInt(gateway::APIVersion()) + "\"}";

  // Compute HMAC
  shash::Any hmac(shash::kSha1);
  shash::HmacString(secret_, json_msg, &hmac);

  CurlSendPayload payload;
  payload.json_message = &json_msg;
  payload.pack_serializer = &serializer;
  payload.index = 0;

  const size_t payload_size =
      json_msg.size() + serializer.GetHeaderSize() + job->pack->size();

  // Prepare the Curl POST request
  CURL* h_curl = curl_easy_init();

  if (!h_curl) {
    return false;
  }

  // Set HTTP headers (Authorization and Message-Size)
  std::string header_str = std::string("Authorization: ") + key_id_ + " " +
                           Base64(hmac.ToString(false));
  struct curl_slist* auth_header = NULL;
  auth_header = curl_slist_append(auth_header, header_str.c_str());
  header_str = std::string("Message-Size: ") + StringifyInt(json_msg.size());
  auth_header = curl_slist_append(auth_header, header_str.c_str());
  curl_easy_setopt(h_curl, CURLOPT_HTTPHEADER, auth_header);

  std::string reply;
  curl_easy_setopt(h_curl, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt(h_curl, CURLOPT_USERAGENT, "cvmfs/" VERSION);
  curl_easy_setopt(h_curl, CURLOPT_MAXREDIRS, 50L);
  curl_easy_setopt(h_curl, CURLOPT_CUSTOMREQUEST, "POST");
  curl_easy_setopt(h_curl, CURLOPT_URL, (api_url_ + "/payloads").c_str());
  curl_easy_setopt(h_curl, CURLOPT_POSTFIELDS, NULL);
  curl_easy_setopt(h_curl, CURLOPT_POSTFIELDSIZE_LARGE,
                   static_cast<curl_off_t>(payload_size));
  curl_easy_setopt(h_curl, CURLOPT_READDATA, &payload);
  curl_easy_setopt(h_curl, CURLOPT_READFUNCTION, SendCB);
  curl_easy_setopt(h_curl, CURLOPT_WRITEFUNCTION, RecvCB);
  curl_easy_setopt(h_curl, CURLOPT_WRITEDATA, &reply);

  // Perform the Curl POST request
  CURLcode ret = curl_easy_perform(h_curl);
  if (ret) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "SessionContext::DoUpload - curl_easy_perform failed: %d",
             ret);
  }

  UniquePtr<JsonDocument> reply_json(JsonDocument::Create(reply));
  const JSON *reply_status =
    JsonDocument::SearchInObject(reply_json->root(), "status", JSON_STRING);
  const bool ok = (reply_status != NULL &&
                   std::string(reply_status->string_value) == "ok");
  if (!ok) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "SessionContext::DoUpload - error reply: %s",
             reply.c_str());
  }

  curl_easy_cleanup(h_curl);
  h_curl = NULL;

  return ok && !ret;
}

void* SessionContext::UploadLoop(void* data) {
  SessionContext* ctx = reinterpret_cast<SessionContext*>(data);

  int64_t jobs_processed = 0;
  while (!ctx->ShouldTerminate()) {
    while (jobs_processed < ctx->NumJobsSubmitted()) {
      UploadJob* job = ctx->upload_jobs_->Dequeue();
      if (!ctx->DoUpload(job)) {
        PANIC(kLogStderr,
              "SessionContext: could not submit payload. Aborting.");
      }
      job->result->Set(true);
      delete job->pack;
      delete job;
      jobs_processed++;
    }
    if (ctx->queue_was_flushed_.IsEmpty()) {
      ctx->queue_was_flushed_.Enqueue(true);
    }
  }

  return NULL;
}

bool SessionContext::ShouldTerminate() {
  return atomic_read32(&worker_terminate_);
}

}  // namespace upload
<commit_msg>Fix SessionContext on 32bit platforms<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include "session_context.h"

#include <algorithm>
#include <limits>

#include "curl/curl.h"
#include "cvmfs_config.h"
#include "gateway_util.h"
#include "json_document.h"
#include "json_document_write.h"
#include "swissknife_lease_curl.h"
#include "util/exception.h"
#include "util/pointer.h"
#include "util/string.h"

namespace {
// Maximum number of jobs during a session. No limit, for practical
// purposes. Note that we use uint32_t so that the FifoChannel code works
// correctly with this limit on 32bit systems.
const uint32_t kMaxNumJobs = std::numeric_limits<uint32_t>::max();
}

namespace upload {

size_t SendCB(void* ptr, size_t size, size_t nmemb, void* userp) {
  CurlSendPayload* payload = static_cast<CurlSendPayload*>(userp);

  size_t max_chunk_size = size * nmemb;
  if (max_chunk_size < 1) {
    return 0;
  }

  size_t current_chunk_size = 0;
  while (current_chunk_size < max_chunk_size) {
    if (payload->index < payload->json_message->size()) {
      // Can add a chunk from the JSON message
      const size_t read_size =
          std::min(max_chunk_size - current_chunk_size,
                   payload->json_message->size() - payload->index);
      current_chunk_size += read_size;
      std::memcpy(ptr, payload->json_message->data() + payload->index,
                  read_size);
      payload->index += read_size;
    } else {
      // Can add a chunk from the payload
      const size_t max_read_size = max_chunk_size - current_chunk_size;
      const unsigned nbytes = payload->pack_serializer->ProduceNext(
          max_read_size, static_cast<unsigned char*>(ptr) + current_chunk_size);
      current_chunk_size += nbytes;

      if (!nbytes) {
        break;
      }
    }
  }

  return current_chunk_size;
}

size_t RecvCB(void* buffer, size_t size, size_t nmemb, void* userp) {
  std::string* my_buffer = static_cast<std::string*>(userp);

  if (size * nmemb < 1) {
    return 0;
  }

  *my_buffer = static_cast<char*>(buffer);

  return my_buffer->size();
}

SessionContextBase::SessionContextBase()
    : upload_results_(kMaxNumJobs, kMaxNumJobs),
      api_url_(),
      session_token_(),
      key_id_(),
      secret_(),
      queue_was_flushed_(1, 1),
      max_pack_size_(ObjectPack::kDefaultLimit),
      active_handles_(),
      current_pack_(NULL),
      current_pack_mtx_(),
      objects_dispatched_(0),
      bytes_committed_(0),
      bytes_dispatched_(0),
      initialized_(false) {}

SessionContextBase::~SessionContextBase() {}

bool SessionContextBase::Initialize(const std::string& api_url,
                                    const std::string& session_token,
                                    const std::string& key_id,
                                    const std::string& secret,
                                    uint64_t max_pack_size,
                                    uint64_t max_queue_size) {
  bool ret = true;

  // Initialize session context lock
  pthread_mutexattr_t attr;
  if (pthread_mutexattr_init(&attr) ||
      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
      pthread_mutex_init(&current_pack_mtx_, &attr) ||
      pthread_mutexattr_destroy(&attr)) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Could not initialize SessionContext lock.");
    return false;
  }

  // Set upstream URL and session token
  api_url_ = api_url;
  session_token_ = session_token;
  key_id_ = key_id;
  secret_ = secret;
  max_pack_size_ = max_pack_size;

  atomic_init64(&objects_dispatched_);
  bytes_committed_ = 0u;
  bytes_dispatched_ = 0u;

  // Ensure that the upload job and result queues are empty
  upload_results_.Drop();

  queue_was_flushed_.Drop();
  queue_was_flushed_.Enqueue(true);

  // Ensure that there are not open object packs
  if (current_pack_) {
    LogCvmfs(
        kLogUploadGateway, kLogStderr,
        "Could not initialize SessionContext - Existing open object packs.");
    ret = false;
  }

  ret = InitializeDerived(max_queue_size) && ret;

  initialized_ = true;

  return ret;
}

bool SessionContextBase::Finalize(bool commit, const std::string& old_root_hash,
                                  const std::string& new_root_hash,
                                  const RepositoryTag& tag) {
  assert(active_handles_.empty());
  if (!initialized_) {
    assert(!commit);
    return true;
  }

  {
    MutexLockGuard lock(current_pack_mtx_);

    if (current_pack_ && current_pack_->GetNoObjects() > 0) {
      Dispatch();
      current_pack_ = NULL;
    }
  }

  bool results = true;
  int64_t jobs_finished = 0;
  while (!upload_results_.IsEmpty() || (jobs_finished < NumJobsSubmitted())) {
    Future<bool>* future = upload_results_.Dequeue();
    results = future->Get() && results;
    delete future;
    jobs_finished++;
  }

  if (commit) {
    if (old_root_hash.empty() || new_root_hash.empty()) {
      return false;
    }
    bool commit_result = Commit(old_root_hash, new_root_hash, tag);
    if (!commit_result) {
      LogCvmfs(kLogUploadGateway, kLogStderr,
               "SessionContext: could not commit session. Aborting.");
      FinalizeDerived();
      pthread_mutex_destroy(&current_pack_mtx_);
      initialized_ = false;
      return false;
    }
  }

  results &= FinalizeDerived() && (bytes_committed_ == bytes_dispatched_);

  pthread_mutex_destroy(&current_pack_mtx_);

  initialized_ = false;

  return results;
}

void SessionContextBase::WaitForUpload() {
  if (!upload_results_.IsEmpty()) {
    queue_was_flushed_.Dequeue();
  }
}

ObjectPack::BucketHandle SessionContextBase::NewBucket() {
  MutexLockGuard lock(current_pack_mtx_);
  if (!current_pack_) {
    current_pack_ = new ObjectPack(max_pack_size_);
  }
  ObjectPack::BucketHandle hd = current_pack_->NewBucket();
  active_handles_.push_back(hd);
  return hd;
}

bool SessionContextBase::CommitBucket(const ObjectPack::BucketContentType type,
                                      const shash::Any& id,
                                      const ObjectPack::BucketHandle handle,
                                      const std::string& name,
                                      const bool force_dispatch) {
  MutexLockGuard lock(current_pack_mtx_);

  if (!current_pack_) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "Error: Called SessionBaseContext::CommitBucket without an open "
             "ObjectPack.");
    return false;
  }

  uint64_t size0 = current_pack_->size();
  bool committed = current_pack_->CommitBucket(type, id, handle, name);

  if (committed) {  // Current pack is still not full
    active_handles_.erase(
        std::remove(active_handles_.begin(), active_handles_.end(), handle),
        active_handles_.end());
    uint64_t size1 = current_pack_->size();
    bytes_committed_ += size1 - size0;
    if (force_dispatch) {
      Dispatch();
      current_pack_ = NULL;
    }
  } else {  // Current pack is full and can be dispatched
    uint64_t new_size = 0;
    if (handle->capacity > max_pack_size_) {
      new_size = handle->capacity + 1;
    } else {
      new_size = max_pack_size_;
    }
    ObjectPack* new_pack = new ObjectPack(new_size);
    for (size_t i = 0u; i < active_handles_.size(); ++i) {
      current_pack_->TransferBucket(active_handles_[i], new_pack);
    }

    if (current_pack_->GetNoObjects() > 0) {
      Dispatch();
    }
    current_pack_ = new_pack;

    CommitBucket(type, id, handle, name, false);
  }

  return true;
}

int64_t SessionContextBase::NumJobsSubmitted() const {
  return atomic_read64(&objects_dispatched_);
}

void SessionContextBase::Dispatch() {
  MutexLockGuard lock(current_pack_mtx_);

  if (!current_pack_) {
    return;
  }

  atomic_inc64(&objects_dispatched_);
  bytes_dispatched_ += current_pack_->size();
  upload_results_.Enqueue(DispatchObjectPack(current_pack_));
}

SessionContext::SessionContext()
    : SessionContextBase(),
      upload_jobs_(),
      worker_terminate_(),
      worker_()
{
  // Set to 0 in InitializeDerived()
  atomic_write32(&worker_terminate_, 1);
}

bool SessionContext::InitializeDerived(uint64_t max_queue_size) {
  // Start worker thread
  atomic_init32(&worker_terminate_);

  upload_jobs_ = new FifoChannel<UploadJob*>(max_queue_size, max_queue_size);
  upload_jobs_->Drop();

  int retval =
      pthread_create(&worker_, NULL, UploadLoop, reinterpret_cast<void*>(this));

  return !retval;
}

bool SessionContext::FinalizeDerived() {
  if (atomic_cas32(&worker_terminate_, 0, 1))
    pthread_join(worker_, NULL);

  return true;
}

bool SessionContext::Commit(const std::string& old_root_hash,
                            const std::string& new_root_hash,
                            const RepositoryTag& tag) {
  JsonStringGenerator request_input;
  request_input.Add("old_root_hash", old_root_hash);
  request_input.Add("new_root_hash", new_root_hash);
  request_input.Add("tag_name", tag.name_);
  request_input.Add("tag_channel", tag.channel_);
  request_input.Add("tag_description", tag.description_);
  std::string request = request_input.GenerateString();
  CurlBuffer buffer;
  return MakeEndRequest("POST", key_id_, secret_, session_token_, api_url_,
                        request, &buffer);
}

Future<bool>* SessionContext::DispatchObjectPack(ObjectPack* pack) {
  UploadJob* job = new UploadJob;
  job->pack = pack;
  job->result = new Future<bool>();
  upload_jobs_->Enqueue(job);
  return job->result;
}

bool SessionContext::DoUpload(const SessionContext::UploadJob* job) {
  // Set up the object pack serializer
  ObjectPackProducer serializer(job->pack);

  shash::Any payload_digest(shash::kSha1);
  serializer.GetDigest(&payload_digest);
  const std::string json_msg =
      "{\"session_token\" : \"" + session_token_ +
      "\", \"payload_digest\" : \"" + payload_digest.ToString(false) +
      "\", \"header_size\" : \"" + StringifyInt(serializer.GetHeaderSize()) +
      "\", \"api_version\" : \"" + StringifyInt(gateway::APIVersion()) + "\"}";

  // Compute HMAC
  shash::Any hmac(shash::kSha1);
  shash::HmacString(secret_, json_msg, &hmac);

  CurlSendPayload payload;
  payload.json_message = &json_msg;
  payload.pack_serializer = &serializer;
  payload.index = 0;

  const size_t payload_size =
      json_msg.size() + serializer.GetHeaderSize() + job->pack->size();

  // Prepare the Curl POST request
  CURL* h_curl = curl_easy_init();

  if (!h_curl) {
    return false;
  }

  // Set HTTP headers (Authorization and Message-Size)
  std::string header_str = std::string("Authorization: ") + key_id_ + " " +
                           Base64(hmac.ToString(false));
  struct curl_slist* auth_header = NULL;
  auth_header = curl_slist_append(auth_header, header_str.c_str());
  header_str = std::string("Message-Size: ") + StringifyInt(json_msg.size());
  auth_header = curl_slist_append(auth_header, header_str.c_str());
  curl_easy_setopt(h_curl, CURLOPT_HTTPHEADER, auth_header);

  std::string reply;
  curl_easy_setopt(h_curl, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt(h_curl, CURLOPT_USERAGENT, "cvmfs/" VERSION);
  curl_easy_setopt(h_curl, CURLOPT_MAXREDIRS, 50L);
  curl_easy_setopt(h_curl, CURLOPT_CUSTOMREQUEST, "POST");
  curl_easy_setopt(h_curl, CURLOPT_URL, (api_url_ + "/payloads").c_str());
  curl_easy_setopt(h_curl, CURLOPT_POSTFIELDS, NULL);
  curl_easy_setopt(h_curl, CURLOPT_POSTFIELDSIZE_LARGE,
                   static_cast<curl_off_t>(payload_size));
  curl_easy_setopt(h_curl, CURLOPT_READDATA, &payload);
  curl_easy_setopt(h_curl, CURLOPT_READFUNCTION, SendCB);
  curl_easy_setopt(h_curl, CURLOPT_WRITEFUNCTION, RecvCB);
  curl_easy_setopt(h_curl, CURLOPT_WRITEDATA, &reply);

  // Perform the Curl POST request
  CURLcode ret = curl_easy_perform(h_curl);
  if (ret) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "SessionContext::DoUpload - curl_easy_perform failed: %d",
             ret);
  }

  UniquePtr<JsonDocument> reply_json(JsonDocument::Create(reply));
  const JSON *reply_status =
    JsonDocument::SearchInObject(reply_json->root(), "status", JSON_STRING);
  const bool ok = (reply_status != NULL &&
                   std::string(reply_status->string_value) == "ok");
  if (!ok) {
    LogCvmfs(kLogUploadGateway, kLogStderr,
             "SessionContext::DoUpload - error reply: %s",
             reply.c_str());
  }

  curl_easy_cleanup(h_curl);
  h_curl = NULL;

  return ok && !ret;
}

void* SessionContext::UploadLoop(void* data) {
  SessionContext* ctx = reinterpret_cast<SessionContext*>(data);

  int64_t jobs_processed = 0;
  while (!ctx->ShouldTerminate()) {
    while (jobs_processed < ctx->NumJobsSubmitted()) {
      UploadJob* job = ctx->upload_jobs_->Dequeue();
      if (!ctx->DoUpload(job)) {
        PANIC(kLogStderr,
              "SessionContext: could not submit payload. Aborting.");
      }
      job->result->Set(true);
      delete job->pack;
      delete job;
      jobs_processed++;
    }
    if (ctx->queue_was_flushed_.IsEmpty()) {
      ctx->queue_was_flushed_.Enqueue(true);
    }
  }

  return NULL;
}

bool SessionContext::ShouldTerminate() {
  return atomic_read32(&worker_terminate_);
}

}  // namespace upload
<|endoftext|>


<commit_before>///////////////////////////////////////////////////////////////////////
// File:        svutil.cpp
// Description: ScrollView Utilities
// Author:      Joern Wanke
// Created:     Thu Nov 29 2007
//
// (C) Copyright 2007, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////
//
// SVUtil contains the SVSync and SVNetwork classes, which are used for
// thread/process creation & synchronization and network connection.

#include <cstdio>
#ifdef _WIN32
#include <windows.h>
struct addrinfo {
  struct sockaddr* ai_addr;
  int ai_addrlen;
  int ai_family;
  int ai_socktype;
  int ai_protocol;
};
#else
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
#include <semaphore.h>
#include <csignal>
#include <sys/select.h>
#include <sys/socket.h>
#ifdef __linux__
#include <sys/prctl.h>
#endif
#include <unistd.h>
#endif

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <memory>
#include <string>

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "svutil.h"

SVMutex::SVMutex() {
#ifdef _WIN32
  mutex_ = CreateMutex(0, FALSE, 0);
#else
  pthread_mutex_init(&mutex_, nullptr);
#endif
}

void SVMutex::Lock() {
#ifdef _WIN32
  WaitForSingleObject(mutex_, INFINITE);
#else
  pthread_mutex_lock(&mutex_);
#endif
}

void SVMutex::Unlock() {
#ifdef _WIN32
  ReleaseMutex(mutex_);
#else
  pthread_mutex_unlock(&mutex_);
#endif
}

// Create new thread.
void SVSync::StartThread(void* (*func)(void*), void* arg) {
#ifdef _WIN32
  LPTHREAD_START_ROUTINE f = (LPTHREAD_START_ROUTINE)func;
  DWORD threadid;
  HANDLE newthread = CreateThread(nullptr,        // default security attributes
                                  0,           // use default stack size
                                  f,           // thread function
                                  arg,         // argument to thread function
                                  0,           // use default creation flags
                                  &threadid);  // returns the thread identifier
#else
  pthread_t helper;
  pthread_attr_t attr;
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  pthread_create(&helper, &attr, func, arg);
#endif
}

#ifndef GRAPHICS_DISABLED

const int kMaxMsgSize = 4096;

// Signals a thread to exit.
void SVSync::ExitThread() {
#ifdef _WIN32
  // ExitThread(0);
#else
  pthread_exit(nullptr);
#endif
}

// Starts a new process.
void SVSync::StartProcess(const char* executable, const char* args) {
  std::string proc;
  proc.append(executable);
  proc.append(" ");
  proc.append(args);
  std::cout << "Starting " << proc << std::endl;
#ifdef _WIN32
  STARTUPINFO start_info;
  PROCESS_INFORMATION proc_info;
  GetStartupInfo(&start_info);
  if (!CreateProcess(nullptr, const_cast<char*>(proc.c_str()), nullptr, nullptr, FALSE,
                CREATE_NO_WINDOW | DETACHED_PROCESS, nullptr, nullptr,
                &start_info, &proc_info))
    return;
#else
  int pid = fork();
  if (pid != 0) {   // The father process returns
  } else {
#ifdef __linux__
    // Make sure the java process terminates on exit, since its
    // broken socket detection seems to be useless.
    prctl(PR_SET_PDEATHSIG, 2, 0, 0, 0);
#endif
    char* mutable_args = strdup(args);
    int argc = 1;
    for (int i = 0; mutable_args[i]; ++i) {
      if (mutable_args[i] == ' ') {
        ++argc;
      }
    }
    std::unique_ptr<char*[]> argv(new char*[argc + 2]);
    argv[0] = strdup(executable);
    argv[1] = mutable_args;
    argc = 2;
    bool inquote = false;
    for (int i = 0; mutable_args[i]; ++i) {
      if (!inquote && mutable_args[i] == ' ') {
        mutable_args[i] = '\0';
        argv[argc++] = mutable_args + i + 1;
      } else if (mutable_args[i] == '"') {
        inquote = !inquote;
        mutable_args[i] = ' ';
      }
    }
    argv[argc] = nullptr;
    execvp(executable, argv.get());
    free(argv[0]);
    free(argv[1]);
  }
#endif
}

SVSemaphore::SVSemaphore() {
#ifdef _WIN32
  semaphore_ = CreateSemaphore(0, 0, 10, 0);
#elif defined(__APPLE__)
  char name[50];
  snprintf(name, sizeof(name), "%ld", random());
  sem_unlink(name);
  semaphore_ = sem_open(name, O_CREAT , S_IWUSR, 0);
  if (semaphore_ == SEM_FAILED) {
    perror("sem_open");
  }
#else
  sem_init(&semaphore_, 0, 0);
#endif
}

void SVSemaphore::Signal() {
#ifdef _WIN32
  ReleaseSemaphore(semaphore_, 1, nullptr);
#elif defined(__APPLE__)
  sem_post(semaphore_);
#else
  sem_post(&semaphore_);
#endif
}

void SVSemaphore::Wait() {
#ifdef _WIN32
  WaitForSingleObject(semaphore_, INFINITE);
#elif defined(__APPLE__)
  sem_wait(semaphore_);
#else
  sem_wait(&semaphore_);
#endif
}

// Place a message in the message buffer (and flush it).
void SVNetwork::Send(const char* msg) {
  mutex_send_.Lock();
  msg_buffer_out_.append(msg);
  mutex_send_.Unlock();
}

// Send the whole buffer.
void SVNetwork::Flush() {
  mutex_send_.Lock();
  while (!msg_buffer_out_.empty()) {
    int i = send(stream_, msg_buffer_out_.c_str(), msg_buffer_out_.length(), 0);
    msg_buffer_out_.erase(0, i);
  }
  mutex_send_.Unlock();
}

// Receive a message from the server.
// This will always return one line of char* (denoted by \n).
char* SVNetwork::Receive() {
  char* result = nullptr;
#if defined(_WIN32) || defined(__CYGWIN__)
  if (has_content) { result = strtok (nullptr, "\n"); }
#else
  if (buffer_ptr_ != nullptr) { result = strtok_r(nullptr, "\n", &buffer_ptr_); }
#endif

  // This means there is something left in the buffer and we return it.
  if (result != nullptr) { return result;
  // Otherwise, we read from the stream_.
  } else {
    buffer_ptr_ = nullptr;
    has_content = false;

    // The timeout length is not really important since we are looping anyway
    // until a new message is delivered.
    struct timeval tv;
    tv.tv_sec = 10;
    tv.tv_usec = 0;

    // Set the flags to return when the stream_ is ready to be read.
    fd_set readfds;
    FD_ZERO(&readfds);
    FD_SET(stream_, &readfds);

    int i = select(stream_+1, &readfds, nullptr, nullptr, &tv);

    // The stream_ died.
    if (i == 0) { return nullptr; }

    // Read the message buffer.
    i = recv(stream_, msg_buffer_in_, kMaxMsgSize, 0);

    // Server quit (0) or error (-1).
    if (i <= 0) { return nullptr; }
    msg_buffer_in_[i] = '\0';
    has_content = true;
#ifdef _WIN32
    return strtok(msg_buffer_in_, "\n");
#else
    // Setup a new string tokenizer.
    return strtok_r(msg_buffer_in_, "\n", &buffer_ptr_);
#endif
  }
}

// Close the connection to the server.
void SVNetwork::Close() {
#ifdef _WIN32
  closesocket(stream_);
#else
  close(stream_);
#endif
  // Mark stream_ as invalid.
  stream_ = -1;
}

// The program to invoke to start ScrollView
static const char* ScrollViewProg() {
#ifdef _WIN32
  const char* prog = "java -Xms512m -Xmx1024m";
#else
  const char* prog = "sh";
#endif
  return prog;
}

// The arguments to the program to invoke to start ScrollView
static std::string ScrollViewCommand(std::string scrollview_path) {
  // The following ugly ifdef is to enable the output of the java runtime
  // to be sent down a black hole on non-windows to ignore all the
  // exceptions in piccolo. Ideally piccolo would be debugged to make
  // this unnecessary.
  // Also the path has to be separated by ; on windows and : otherwise.
#ifdef _WIN32
  const char* cmd_template = "-Djava.library.path=%s -jar %s/ScrollView.jar";

#else
  const char* cmd_template =
      "-c \"trap 'kill %%1' 0 1 2 ; java "
      "-Xms1024m -Xmx2048m -jar %s/ScrollView.jar"
      " & wait\"";
#endif
  size_t cmdlen = strlen(cmd_template) + 4 * strlen(scrollview_path.c_str()) + 1;
  std::unique_ptr<char[]> cmd(new char[cmdlen]);
  const char* sv_path = scrollview_path.c_str();
  snprintf(cmd.get(), cmdlen, cmd_template, sv_path, sv_path, sv_path, sv_path);
  std::string command(cmd.get());
  return command;
}

// Platform-independent freeaddrinfo()
static void FreeAddrInfo(struct addrinfo* addr_info) {
  #if defined(__linux__)
  freeaddrinfo(addr_info);
  #else
  delete addr_info->ai_addr;
  delete addr_info;
  #endif
}

// Non-linux version of getaddrinfo()
#if !defined(__linux__)
static int GetAddrInfoNonLinux(const char* hostname, int port,
                               struct addrinfo** addr_info) {
// Get the host data depending on the OS.
  struct sockaddr_in* address;
  *addr_info = new struct addrinfo;
  memset(*addr_info, 0, sizeof(struct addrinfo));
  address = new struct sockaddr_in;
  memset(address, 0, sizeof(struct sockaddr_in));

  (*addr_info)->ai_addr = (struct sockaddr*) address;
  (*addr_info)->ai_addrlen = sizeof(struct sockaddr);
  (*addr_info)->ai_family = AF_INET;
  (*addr_info)->ai_socktype = SOCK_STREAM;

  struct hostent *name;
#ifdef _WIN32
  WSADATA wsaData;
  WSAStartup(MAKEWORD(1, 1), &wsaData);
  name = gethostbyname(hostname);
#else
  name = gethostbyname(hostname);
#endif

  if (name == nullptr) {
    FreeAddrInfo(*addr_info);
    *addr_info = nullptr;
    return -1;
  }

  // Fill in the appropriate variables to be able to connect to the server.
  address->sin_family = name->h_addrtype;
  memcpy(&address->sin_addr.s_addr, name->h_addr_list[0], name->h_length);
  address->sin_port = htons(port);
  return 0;
}
#endif

// Platform independent version of getaddrinfo()
//   Given a hostname:port, produce an addrinfo struct
static int GetAddrInfo(const char* hostname, int port,
                       struct addrinfo** address) {
#if defined(__linux__)
  char port_str[40];
  snprintf(port_str, 40, "%d", port);
  return getaddrinfo(hostname, port_str, nullptr, address);
#else
  return GetAddrInfoNonLinux(hostname, port, address);
#endif
}

// Set up a connection to a ScrollView on hostname:port.
SVNetwork::SVNetwork(const char* hostname, int port) {
  msg_buffer_in_ = new char[kMaxMsgSize + 1];
  msg_buffer_in_[0] = '\0';

  has_content = false;
  buffer_ptr_ = nullptr;

  struct addrinfo *addr_info = nullptr;

  if (GetAddrInfo(hostname, port, &addr_info) != 0) {
    std::cerr << "Error resolving name for ScrollView host "
              << std::string(hostname) << ":" << port << std::endl;
  }

  stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                   addr_info->ai_protocol);

  // If server is not there, we will start a new server as local child process.
  if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) < 0) {
    const char* scrollview_path = getenv("SCROLLVIEW_PATH");
    if (scrollview_path == nullptr) {
#ifdef SCROLLVIEW_PATH
#define _STR(a) #a
#define _XSTR(a) _STR(a)
      scrollview_path = _XSTR(SCROLLVIEW_PATH);
#undef _XSTR
#undef _STR
#else
      scrollview_path = ".";
#endif
    }
    const char *prog = ScrollViewProg();
    std::string command = ScrollViewCommand(scrollview_path);
    SVSync::StartProcess(prog, command.c_str());

    // Wait for server to show up.
    // Note: There is no exception handling in case the server never turns up.

    Close();
    for (;;) {
      stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                       addr_info->ai_protocol);
      if (stream_ >= 0) {
        if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) == 0) {
          break;
        }

        Close();

        std::cout << "ScrollView: Waiting for server...\n";
#ifdef _WIN32
        Sleep(1000);
#else
        sleep(1);
#endif
      }
    }
  }
  FreeAddrInfo(addr_info);
}

SVNetwork::~SVNetwork() {
  Close();
  delete[] msg_buffer_in_;
}

#endif  // GRAPHICS_DISABLED
<commit_msg>SVNetwork: Handle failed socket call (CID 1164597)<commit_after>///////////////////////////////////////////////////////////////////////
// File:        svutil.cpp
// Description: ScrollView Utilities
// Author:      Joern Wanke
// Created:     Thu Nov 29 2007
//
// (C) Copyright 2007, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////
//
// SVUtil contains the SVSync and SVNetwork classes, which are used for
// thread/process creation & synchronization and network connection.

#include <cstdio>
#ifdef _WIN32
#include <windows.h>
struct addrinfo {
  struct sockaddr* ai_addr;
  int ai_addrlen;
  int ai_family;
  int ai_socktype;
  int ai_protocol;
};
#else
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
#include <semaphore.h>
#include <csignal>
#include <sys/select.h>
#include <sys/socket.h>
#ifdef __linux__
#include <sys/prctl.h>
#endif
#include <unistd.h>
#endif

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <memory>
#include <string>

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "svutil.h"

SVMutex::SVMutex() {
#ifdef _WIN32
  mutex_ = CreateMutex(0, FALSE, 0);
#else
  pthread_mutex_init(&mutex_, nullptr);
#endif
}

void SVMutex::Lock() {
#ifdef _WIN32
  WaitForSingleObject(mutex_, INFINITE);
#else
  pthread_mutex_lock(&mutex_);
#endif
}

void SVMutex::Unlock() {
#ifdef _WIN32
  ReleaseMutex(mutex_);
#else
  pthread_mutex_unlock(&mutex_);
#endif
}

// Create new thread.
void SVSync::StartThread(void* (*func)(void*), void* arg) {
#ifdef _WIN32
  LPTHREAD_START_ROUTINE f = (LPTHREAD_START_ROUTINE)func;
  DWORD threadid;
  HANDLE newthread = CreateThread(nullptr,        // default security attributes
                                  0,           // use default stack size
                                  f,           // thread function
                                  arg,         // argument to thread function
                                  0,           // use default creation flags
                                  &threadid);  // returns the thread identifier
#else
  pthread_t helper;
  pthread_attr_t attr;
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  pthread_create(&helper, &attr, func, arg);
#endif
}

#ifndef GRAPHICS_DISABLED

const int kMaxMsgSize = 4096;

// Signals a thread to exit.
void SVSync::ExitThread() {
#ifdef _WIN32
  // ExitThread(0);
#else
  pthread_exit(nullptr);
#endif
}

// Starts a new process.
void SVSync::StartProcess(const char* executable, const char* args) {
  std::string proc;
  proc.append(executable);
  proc.append(" ");
  proc.append(args);
  std::cout << "Starting " << proc << std::endl;
#ifdef _WIN32
  STARTUPINFO start_info;
  PROCESS_INFORMATION proc_info;
  GetStartupInfo(&start_info);
  if (!CreateProcess(nullptr, const_cast<char*>(proc.c_str()), nullptr, nullptr, FALSE,
                CREATE_NO_WINDOW | DETACHED_PROCESS, nullptr, nullptr,
                &start_info, &proc_info))
    return;
#else
  int pid = fork();
  if (pid != 0) {   // The father process returns
  } else {
#ifdef __linux__
    // Make sure the java process terminates on exit, since its
    // broken socket detection seems to be useless.
    prctl(PR_SET_PDEATHSIG, 2, 0, 0, 0);
#endif
    char* mutable_args = strdup(args);
    int argc = 1;
    for (int i = 0; mutable_args[i]; ++i) {
      if (mutable_args[i] == ' ') {
        ++argc;
      }
    }
    std::unique_ptr<char*[]> argv(new char*[argc + 2]);
    argv[0] = strdup(executable);
    argv[1] = mutable_args;
    argc = 2;
    bool inquote = false;
    for (int i = 0; mutable_args[i]; ++i) {
      if (!inquote && mutable_args[i] == ' ') {
        mutable_args[i] = '\0';
        argv[argc++] = mutable_args + i + 1;
      } else if (mutable_args[i] == '"') {
        inquote = !inquote;
        mutable_args[i] = ' ';
      }
    }
    argv[argc] = nullptr;
    execvp(executable, argv.get());
    free(argv[0]);
    free(argv[1]);
  }
#endif
}

SVSemaphore::SVSemaphore() {
#ifdef _WIN32
  semaphore_ = CreateSemaphore(0, 0, 10, 0);
#elif defined(__APPLE__)
  char name[50];
  snprintf(name, sizeof(name), "%ld", random());
  sem_unlink(name);
  semaphore_ = sem_open(name, O_CREAT , S_IWUSR, 0);
  if (semaphore_ == SEM_FAILED) {
    perror("sem_open");
  }
#else
  sem_init(&semaphore_, 0, 0);
#endif
}

void SVSemaphore::Signal() {
#ifdef _WIN32
  ReleaseSemaphore(semaphore_, 1, nullptr);
#elif defined(__APPLE__)
  sem_post(semaphore_);
#else
  sem_post(&semaphore_);
#endif
}

void SVSemaphore::Wait() {
#ifdef _WIN32
  WaitForSingleObject(semaphore_, INFINITE);
#elif defined(__APPLE__)
  sem_wait(semaphore_);
#else
  sem_wait(&semaphore_);
#endif
}

// Place a message in the message buffer (and flush it).
void SVNetwork::Send(const char* msg) {
  mutex_send_.Lock();
  msg_buffer_out_.append(msg);
  mutex_send_.Unlock();
}

// Send the whole buffer.
void SVNetwork::Flush() {
  mutex_send_.Lock();
  while (!msg_buffer_out_.empty()) {
    int i = send(stream_, msg_buffer_out_.c_str(), msg_buffer_out_.length(), 0);
    msg_buffer_out_.erase(0, i);
  }
  mutex_send_.Unlock();
}

// Receive a message from the server.
// This will always return one line of char* (denoted by \n).
char* SVNetwork::Receive() {
  char* result = nullptr;
#if defined(_WIN32) || defined(__CYGWIN__)
  if (has_content) { result = strtok (nullptr, "\n"); }
#else
  if (buffer_ptr_ != nullptr) { result = strtok_r(nullptr, "\n", &buffer_ptr_); }
#endif

  // This means there is something left in the buffer and we return it.
  if (result != nullptr) { return result;
  // Otherwise, we read from the stream_.
  } else {
    buffer_ptr_ = nullptr;
    has_content = false;

    // The timeout length is not really important since we are looping anyway
    // until a new message is delivered.
    struct timeval tv;
    tv.tv_sec = 10;
    tv.tv_usec = 0;

    // Set the flags to return when the stream_ is ready to be read.
    fd_set readfds;
    FD_ZERO(&readfds);
    FD_SET(stream_, &readfds);

    int i = select(stream_+1, &readfds, nullptr, nullptr, &tv);

    // The stream_ died.
    if (i == 0) { return nullptr; }

    // Read the message buffer.
    i = recv(stream_, msg_buffer_in_, kMaxMsgSize, 0);

    // Server quit (0) or error (-1).
    if (i <= 0) { return nullptr; }
    msg_buffer_in_[i] = '\0';
    has_content = true;
#ifdef _WIN32
    return strtok(msg_buffer_in_, "\n");
#else
    // Setup a new string tokenizer.
    return strtok_r(msg_buffer_in_, "\n", &buffer_ptr_);
#endif
  }
}

// Close the connection to the server.
void SVNetwork::Close() {
#ifdef _WIN32
  closesocket(stream_);
#else
  close(stream_);
#endif
  // Mark stream_ as invalid.
  stream_ = -1;
}

// The program to invoke to start ScrollView
static const char* ScrollViewProg() {
#ifdef _WIN32
  const char* prog = "java -Xms512m -Xmx1024m";
#else
  const char* prog = "sh";
#endif
  return prog;
}

// The arguments to the program to invoke to start ScrollView
static std::string ScrollViewCommand(std::string scrollview_path) {
  // The following ugly ifdef is to enable the output of the java runtime
  // to be sent down a black hole on non-windows to ignore all the
  // exceptions in piccolo. Ideally piccolo would be debugged to make
  // this unnecessary.
  // Also the path has to be separated by ; on windows and : otherwise.
#ifdef _WIN32
  const char* cmd_template = "-Djava.library.path=%s -jar %s/ScrollView.jar";

#else
  const char* cmd_template =
      "-c \"trap 'kill %%1' 0 1 2 ; java "
      "-Xms1024m -Xmx2048m -jar %s/ScrollView.jar"
      " & wait\"";
#endif
  size_t cmdlen = strlen(cmd_template) + 4 * strlen(scrollview_path.c_str()) + 1;
  std::unique_ptr<char[]> cmd(new char[cmdlen]);
  const char* sv_path = scrollview_path.c_str();
  snprintf(cmd.get(), cmdlen, cmd_template, sv_path, sv_path, sv_path, sv_path);
  std::string command(cmd.get());
  return command;
}

// Platform-independent freeaddrinfo()
static void FreeAddrInfo(struct addrinfo* addr_info) {
  #if defined(__linux__)
  freeaddrinfo(addr_info);
  #else
  delete addr_info->ai_addr;
  delete addr_info;
  #endif
}

// Non-linux version of getaddrinfo()
#if !defined(__linux__)
static int GetAddrInfoNonLinux(const char* hostname, int port,
                               struct addrinfo** addr_info) {
// Get the host data depending on the OS.
  struct sockaddr_in* address;
  *addr_info = new struct addrinfo;
  memset(*addr_info, 0, sizeof(struct addrinfo));
  address = new struct sockaddr_in;
  memset(address, 0, sizeof(struct sockaddr_in));

  (*addr_info)->ai_addr = (struct sockaddr*) address;
  (*addr_info)->ai_addrlen = sizeof(struct sockaddr);
  (*addr_info)->ai_family = AF_INET;
  (*addr_info)->ai_socktype = SOCK_STREAM;

  struct hostent *name;
#ifdef _WIN32
  WSADATA wsaData;
  WSAStartup(MAKEWORD(1, 1), &wsaData);
  name = gethostbyname(hostname);
#else
  name = gethostbyname(hostname);
#endif

  if (name == nullptr) {
    FreeAddrInfo(*addr_info);
    *addr_info = nullptr;
    return -1;
  }

  // Fill in the appropriate variables to be able to connect to the server.
  address->sin_family = name->h_addrtype;
  memcpy(&address->sin_addr.s_addr, name->h_addr_list[0], name->h_length);
  address->sin_port = htons(port);
  return 0;
}
#endif

// Platform independent version of getaddrinfo()
//   Given a hostname:port, produce an addrinfo struct
static int GetAddrInfo(const char* hostname, int port,
                       struct addrinfo** address) {
#if defined(__linux__)
  char port_str[40];
  snprintf(port_str, 40, "%d", port);
  return getaddrinfo(hostname, port_str, nullptr, address);
#else
  return GetAddrInfoNonLinux(hostname, port, address);
#endif
}

// Set up a connection to a ScrollView on hostname:port.
SVNetwork::SVNetwork(const char* hostname, int port) {
  msg_buffer_in_ = new char[kMaxMsgSize + 1];
  msg_buffer_in_[0] = '\0';

  has_content = false;
  buffer_ptr_ = nullptr;

  struct addrinfo *addr_info = nullptr;

  if (GetAddrInfo(hostname, port, &addr_info) != 0) {
    std::cerr << "Error resolving name for ScrollView host "
              << std::string(hostname) << ":" << port << std::endl;
  }

  stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                   addr_info->ai_protocol);

  if (stream_ < 0) {
    std::cerr << "Failed to open socket" << std::endl;
  } else if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) < 0) {
    // If server is not there, we will start a new server as local child process.
    const char* scrollview_path = getenv("SCROLLVIEW_PATH");
    if (scrollview_path == nullptr) {
#ifdef SCROLLVIEW_PATH
#define _STR(a) #a
#define _XSTR(a) _STR(a)
      scrollview_path = _XSTR(SCROLLVIEW_PATH);
#undef _XSTR
#undef _STR
#else
      scrollview_path = ".";
#endif
    }
    const char *prog = ScrollViewProg();
    std::string command = ScrollViewCommand(scrollview_path);
    SVSync::StartProcess(prog, command.c_str());

    // Wait for server to show up.
    // Note: There is no exception handling in case the server never turns up.

    Close();
    for (;;) {
      stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                       addr_info->ai_protocol);
      if (stream_ >= 0) {
        if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) == 0) {
          break;
        }

        Close();

        std::cout << "ScrollView: Waiting for server...\n";
#ifdef _WIN32
        Sleep(1000);
#else
        sleep(1);
#endif
      }
    }
  }
  FreeAddrInfo(addr_info);
}

SVNetwork::~SVNetwork() {
  Close();
  delete[] msg_buffer_in_;
}

#endif  // GRAPHICS_DISABLED
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "AutoCompleteServlet.h"
#include "CTRCounter.h"
#include "fnord-base/Language.h"
#include "fnord-base/wallclock.h"
#include "fnord-base/io/fileutil.h"

using namespace fnord;

namespace cm {

AutoCompleteServlet::AutoCompleteServlet(
    RefPtr<fts::Analyzer> analyzer) :
    analyzer_(analyzer) {
  cat_names_.emplace("de~e1-1",  "Mode");
  cat_names_.emplace("de~e1-2",  "Accessoires");
  cat_names_.emplace("de~e1-3",  "Taschen");
  cat_names_.emplace("de~e1-4",  "Schmuck");
  cat_names_.emplace("de~e1-5",  "Kinder");
  cat_names_.emplace("de~e1-6",  "Wohnen");
  cat_names_.emplace("de~e1-7",  "Kunst");
  cat_names_.emplace("de~e1-8",  "Handarbeitsbedarf");
  cat_names_.emplace("de~e1-9",  "Vintage");
  cat_names_.emplace("de~e1-10", "Baby");
  cat_names_.emplace("de~e1-11", "Anlass");
  cat_names_.emplace("de~e1-12", "Sale");
  cat_names_.emplace("de~e1-13", "Maenner");
  cat_names_.emplace("de~e1-14", "Schreibwaren");
}

void AutoCompleteServlet::addTermInfo(const String& term, const TermInfo& ti) {
  if (ti.score < 1000) return;
  term_info_.emplace(term, ti);
}

void AutoCompleteServlet::handleHTTPRequest(
    http::HTTPRequest* req,
    http::HTTPResponse* res) {
  URI uri(req->uri());
  ResultListType results;
  const auto& params = uri.queryParams();

  /* arguments */
  String lang_str;
  if (!URI::getParam(params, "lang", &lang_str)) {
    res->addBody("error: missing ?lang=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  String qstr;
  if (!URI::getParam(params, "q", &qstr)) {
    res->addBody("error: missing ?q=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  Vector<String> terms;
  Vector<String> valid_terms;
  auto lang = languageFromString(lang_str);
  analyzer_->tokenize(lang, qstr, &terms);
  if (terms.size() == 0) {
    res->addBody("error: invalid ?q=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  for (const auto& t : terms) {
    if (term_info_.count(lang_str + "~" + t) > 0) {
      valid_terms.emplace_back(t);
    }
  }

  if (terms.size() == 1 || valid_terms.size() == 0) {
    suggestSingleTerm(lang, terms, &results);
  } else {
    suggestMultiTerm(lang, terms, valid_terms, &results);
  }

  if (results.size() == 0 && qstr.length() > 2) {
    suggestFuzzy(lang, terms, &results);
  }

#ifndef FNORD_NODEBUG
  fnord::logDebug(
      "cm.autocompleteserver",
      "suggest lang=$0 qstr=$1 terms=$2 valid_terms=$3 results=$4",
      lang_str,
      qstr,
      inspect(terms),
      inspect(valid_terms),
      results.size());
#endif

  /* write response */
  res->setStatus(http::kStatusOK);
  res->addHeader("Content-Type", "application/json; charset=utf-8");
  json::JSONOutputStream json(res->getBodyOutputStream());

  json.beginObject();
  json.addObjectEntry("query");
  json.addString(qstr);
  json.addComma();
  json.addObjectEntry("suggestions");
  json.beginArray();

  for (int i = 0; i < results.size() && i < 12; ++i) {
    if (i > 0) {
      json.addComma();
    }
    json.beginObject();
    json.addObjectEntry("text");
    json.addString(std::get<0>(results[i]));
    json.addComma();
    json.addObjectEntry("score");
    json.addFloat(std::get<1>(results[i]));
    json.addComma();
    json.addObjectEntry("url");
    json.addString(std::get<2>(results[i]));
    json.endObject();
  }

  json.endArray();
  json.endObject();
}

void AutoCompleteServlet::suggestSingleTerm(
    Language lang,
    Vector<String> terms,
    ResultListType* results) {
  auto lang_str = fnord::languageToString(lang);
  auto prefix = lang_str + "~" + terms.back();
  terms.pop_back();
  String qstr_prefix;

  if (terms.size() > 0 ) {
    qstr_prefix += StringUtil::join(terms, " ") + " ";
  }

  Vector<Pair<String, double>> matches;
  double best_match = 0;

  for (auto iter = term_info_.lower_bound(prefix);
      iter != term_info_.end() && StringUtil::beginsWith(iter->first, prefix);
      ++iter) {
    matches.emplace_back(iter->first, iter->second.score);
    if (iter->second.score > best_match) {
      best_match = iter->second.score;
    }
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  if (matches.size() > 0) {
    const auto& best_match_ti = term_info_.find(matches[0].first);
    for (const auto& c : best_match_ti->second.top_categories) {
      if ((c.second / best_match_ti->second.top_categories[0].second) < 0.2) {
        break;
      }

      auto label = StringUtil::format(
          "$0$1 in $2",
          qstr_prefix,
          matches[0].first.substr(3),
          cat_names_[lang_str + "~" + c.first]);

      results->emplace_back(label, c.second, "");
    }
  }

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first.substr(3), score, "");
  }

  if (matches.size() > 0) {
    const auto& best_match_ti = term_info_.find(matches[0].first);

    for (const auto& r : best_match_ti->second.related_terms) {
      auto label = StringUtil::format(
          "$0$1 $2",
          qstr_prefix,
          matches[0].first.substr(3),
          r.first);

      results->emplace_back(label, r.second, "");
    }
  }
}

void AutoCompleteServlet::suggestMultiTerm(
    Language lang,
    Vector<String> terms,
    const Vector<String>& valid_terms,
    ResultListType* results) {
  Set<String> search_terms;
  auto lang_str = fnord::languageToString(lang);
  auto last_term = terms.back();
  terms.pop_back();
  String qstr_prefix;

  if (terms.size() > 0 ) {
    qstr_prefix += StringUtil::join(terms, " ") + " ";
  }

  HashMap<String, double> topcats_h;
  double best_topcat = 0;
  HashMap<String, double> matches_h;
  double best_match = 0;
  for (const auto& vt : valid_terms) {
    const auto& vtinfo = term_info_.find(lang_str + "~" + vt)->second;
    search_terms.emplace(vt);

    for (const auto& topcat : vtinfo.top_categories) {
      topcats_h[topcat.first] += topcat.second;
      topcats_h[topcat.first] *= 2;

      if (topcats_h[topcat.first] > best_topcat) {
        best_topcat = topcats_h[topcat.first];
      }
    }

    for (const auto& related : vtinfo.related_terms) {
      if (!StringUtil::beginsWith(related.first, last_term)) {
        continue;
      }

      matches_h[related.first] += related.second;

      if (matches_h[related.first] > best_match) {
        best_match = matches_h[related.first];
      }
    }
  }

  Vector<Pair<String, double>> topcats;
  for (const auto& m : topcats_h) {
    topcats.emplace_back(m);
  }

  std::sort(topcats.begin(), topcats.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  for (int m = 0; m < topcats.size() && m < 3; ++m) {
    auto score = topcats[m].second;

    if ((score / best_topcat) < 0.2) {
      break;
    }

    auto label = StringUtil::format(
        "$0 in $1",
        StringUtil::join(valid_terms, " "),
        cat_names_[lang_str + "~" + topcats[m].first]);

    results->emplace_back(label, score, "");
  }

  Vector<Pair<String, double>> matches;
  for (const auto& m : matches_h) {
    matches.emplace_back(m);
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  if (matches.size() > 0) {
    search_terms.emplace(matches[0].first);
  }

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first, score, "");
  }

  matches_h.clear();
  matches.clear();
  best_match = 0;

  for (const auto& vt : search_terms) {
    auto vtinfo_iter = term_info_.find(lang_str + "~" + vt);
    if (vtinfo_iter == term_info_.end()) {
      continue;
    }

    const auto& vtinfo = vtinfo_iter->second;
    for (const auto& related : vtinfo.related_terms) {
      matches_h[related.first] += related.second;

      if (matches_h[related.first] > best_match) {
        best_match = matches_h[related.first];
      }
    }
  }

  for (const auto& m : matches_h) {
    matches.emplace_back(m);
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first, score, "");
  }
}

void AutoCompleteServlet::suggestFuzzy(
    Language lang,
    Vector<String> terms,
    ResultListType* results) {
  results->emplace_back("here be dragons: fuzzy suggestion", 1.0, "");
}

}
<commit_msg>add cross-origin header, remove fuzzy sug placeholder<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "AutoCompleteServlet.h"
#include "CTRCounter.h"
#include "fnord-base/Language.h"
#include "fnord-base/wallclock.h"
#include "fnord-base/io/fileutil.h"

using namespace fnord;

namespace cm {

AutoCompleteServlet::AutoCompleteServlet(
    RefPtr<fts::Analyzer> analyzer) :
    analyzer_(analyzer) {
  cat_names_.emplace("de~e1-1",  "Mode");
  cat_names_.emplace("de~e1-2",  "Accessoires");
  cat_names_.emplace("de~e1-3",  "Taschen");
  cat_names_.emplace("de~e1-4",  "Schmuck");
  cat_names_.emplace("de~e1-5",  "Kinder");
  cat_names_.emplace("de~e1-6",  "Wohnen");
  cat_names_.emplace("de~e1-7",  "Kunst");
  cat_names_.emplace("de~e1-8",  "Handarbeitsbedarf");
  cat_names_.emplace("de~e1-9",  "Vintage");
  cat_names_.emplace("de~e1-10", "Baby");
  cat_names_.emplace("de~e1-11", "Anlass");
  cat_names_.emplace("de~e1-12", "Sale");
  cat_names_.emplace("de~e1-13", "Maenner");
  cat_names_.emplace("de~e1-14", "Schreibwaren");
}

void AutoCompleteServlet::addTermInfo(const String& term, const TermInfo& ti) {
  if (ti.score < 1000) return;
  term_info_.emplace(term, ti);
}

void AutoCompleteServlet::handleHTTPRequest(
    http::HTTPRequest* req,
    http::HTTPResponse* res) {
  res->addHeader("Access-Control-Allow-Origin", "*");

  URI uri(req->uri());
  ResultListType results;
  const auto& params = uri.queryParams();

  /* arguments */
  String lang_str;
  if (!URI::getParam(params, "lang", &lang_str)) {
    res->addBody("error: missing ?lang=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  String qstr;
  if (!URI::getParam(params, "q", &qstr)) {
    res->addBody("error: missing ?q=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  Vector<String> terms;
  Vector<String> valid_terms;
  auto lang = languageFromString(lang_str);
  analyzer_->tokenize(lang, qstr, &terms);
  if (terms.size() == 0) {
    res->addBody("error: invalid ?q=... parameter");
    res->setStatus(http::kStatusBadRequest);
    return;
  }

  for (const auto& t : terms) {
    if (term_info_.count(lang_str + "~" + t) > 0) {
      valid_terms.emplace_back(t);
    }
  }

  if (terms.size() == 1 || valid_terms.size() == 0) {
    suggestSingleTerm(lang, terms, &results);
  } else {
    suggestMultiTerm(lang, terms, valid_terms, &results);
  }

  if (results.size() == 0 && qstr.length() > 2) {
    suggestFuzzy(lang, terms, &results);
  }

#ifndef FNORD_NODEBUG
  fnord::logDebug(
      "cm.autocompleteserver",
      "suggest lang=$0 qstr=$1 terms=$2 valid_terms=$3 results=$4",
      lang_str,
      qstr,
      inspect(terms),
      inspect(valid_terms),
      results.size());
#endif

  /* write response */
  res->setStatus(http::kStatusOK);
  res->addHeader("Content-Type", "application/json; charset=utf-8");
  json::JSONOutputStream json(res->getBodyOutputStream());

  json.beginObject();
  json.addObjectEntry("query");
  json.addString(qstr);
  json.addComma();
  json.addObjectEntry("suggestions");
  json.beginArray();

  for (int i = 0; i < results.size() && i < 12; ++i) {
    if (i > 0) {
      json.addComma();
    }
    json.beginObject();
    json.addObjectEntry("text");
    json.addString(std::get<0>(results[i]));
    json.addComma();
    json.addObjectEntry("score");
    json.addFloat(std::get<1>(results[i]));
    json.addComma();
    json.addObjectEntry("url");
    json.addString(std::get<2>(results[i]));
    json.endObject();
  }

  json.endArray();
  json.endObject();
}

void AutoCompleteServlet::suggestSingleTerm(
    Language lang,
    Vector<String> terms,
    ResultListType* results) {
  auto lang_str = fnord::languageToString(lang);
  auto prefix = lang_str + "~" + terms.back();
  terms.pop_back();
  String qstr_prefix;

  if (terms.size() > 0 ) {
    qstr_prefix += StringUtil::join(terms, " ") + " ";
  }

  Vector<Pair<String, double>> matches;
  double best_match = 0;

  for (auto iter = term_info_.lower_bound(prefix);
      iter != term_info_.end() && StringUtil::beginsWith(iter->first, prefix);
      ++iter) {
    matches.emplace_back(iter->first, iter->second.score);
    if (iter->second.score > best_match) {
      best_match = iter->second.score;
    }
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  if (matches.size() > 0) {
    const auto& best_match_ti = term_info_.find(matches[0].first);
    for (const auto& c : best_match_ti->second.top_categories) {
      if ((c.second / best_match_ti->second.top_categories[0].second) < 0.2) {
        break;
      }

      auto label = StringUtil::format(
          "$0$1 in $2",
          qstr_prefix,
          matches[0].first.substr(3),
          cat_names_[lang_str + "~" + c.first]);

      results->emplace_back(label, c.second, "");
    }
  }

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first.substr(3), score, "");
  }

  if (matches.size() > 0) {
    const auto& best_match_ti = term_info_.find(matches[0].first);

    for (const auto& r : best_match_ti->second.related_terms) {
      auto label = StringUtil::format(
          "$0$1 $2",
          qstr_prefix,
          matches[0].first.substr(3),
          r.first);

      results->emplace_back(label, r.second, "");
    }
  }
}

void AutoCompleteServlet::suggestMultiTerm(
    Language lang,
    Vector<String> terms,
    const Vector<String>& valid_terms,
    ResultListType* results) {
  Set<String> search_terms;
  auto lang_str = fnord::languageToString(lang);
  auto last_term = terms.back();
  terms.pop_back();
  String qstr_prefix;

  if (terms.size() > 0 ) {
    qstr_prefix += StringUtil::join(terms, " ") + " ";
  }

  HashMap<String, double> topcats_h;
  double best_topcat = 0;
  HashMap<String, double> matches_h;
  double best_match = 0;
  for (const auto& vt : valid_terms) {
    const auto& vtinfo = term_info_.find(lang_str + "~" + vt)->second;
    search_terms.emplace(vt);

    for (const auto& topcat : vtinfo.top_categories) {
      topcats_h[topcat.first] += topcat.second;
      topcats_h[topcat.first] *= 2;

      if (topcats_h[topcat.first] > best_topcat) {
        best_topcat = topcats_h[topcat.first];
      }
    }

    for (const auto& related : vtinfo.related_terms) {
      if (!StringUtil::beginsWith(related.first, last_term)) {
        continue;
      }

      matches_h[related.first] += related.second;

      if (matches_h[related.first] > best_match) {
        best_match = matches_h[related.first];
      }
    }
  }

  Vector<Pair<String, double>> topcats;
  for (const auto& m : topcats_h) {
    topcats.emplace_back(m);
  }

  std::sort(topcats.begin(), topcats.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  for (int m = 0; m < topcats.size() && m < 3; ++m) {
    auto score = topcats[m].second;

    if ((score / best_topcat) < 0.2) {
      break;
    }

    auto label = StringUtil::format(
        "$0 in $1",
        StringUtil::join(valid_terms, " "),
        cat_names_[lang_str + "~" + topcats[m].first]);

    results->emplace_back(label, score, "");
  }

  Vector<Pair<String, double>> matches;
  for (const auto& m : matches_h) {
    matches.emplace_back(m);
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  if (matches.size() > 0) {
    search_terms.emplace(matches[0].first);
  }

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first, score, "");
  }

  matches_h.clear();
  matches.clear();
  best_match = 0;

  for (const auto& vt : search_terms) {
    auto vtinfo_iter = term_info_.find(lang_str + "~" + vt);
    if (vtinfo_iter == term_info_.end()) {
      continue;
    }

    const auto& vtinfo = vtinfo_iter->second;
    for (const auto& related : vtinfo.related_terms) {
      matches_h[related.first] += related.second;

      if (matches_h[related.first] > best_match) {
        best_match = matches_h[related.first];
      }
    }
  }

  for (const auto& m : matches_h) {
    matches.emplace_back(m);
  }

  std::sort(matches.begin(), matches.end(), [] (
      const Pair<String, double>& a,
      const Pair<String, double>& b) {
    return b.second < a.second;
  });

  for (int m = 0; m < matches.size(); ++m) {
    auto score = matches[m].second;

    if ((score / best_match) < 0.1) {
      break;
    }

    results->emplace_back(qstr_prefix + matches[m].first, score, "");
  }
}

void AutoCompleteServlet::suggestFuzzy(
    Language lang,
    Vector<String> terms,
    ResultListType* results) {
  //results->emplace_back("here be dragons: fuzzy suggestion", 1.0, "");
}

}
<|endoftext|>