<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <limits.h>
#include <ctype.h>
#include <algorithm>

#include "base/file_util.h"
#include "base/histogram.h"
#include "base/logging.h"
#include "net/base/sdch_filter.h"
#include "net/base/sdch_manager.h"

#include "sdch/open-vcdiff/src/google/vcdecoder.h"

SdchFilter::SdchFilter()
    : decoding_status_(DECODING_UNINITIALIZED),
      vcdiff_streaming_decoder_(NULL),
      dictionary_hash_(),
      dictionary_hash_is_plausible_(false),
      dictionary_(NULL),
      dest_buffer_excess_(),
      dest_buffer_excess_index_(0),
      source_bytes_(0),
      output_bytes_(0),
      possible_pass_through_(false) {
}

SdchFilter::~SdchFilter() {
  static int filter_use_count = 0;
  ++filter_use_count;
  if (META_REFRESH_RECOVERY == decoding_status_) {
    UMA_HISTOGRAM_COUNTS(L"Sdch.FilterUseBeforeDisabling", filter_use_count);
  }

  if (vcdiff_streaming_decoder_.get()) {
    if (!vcdiff_streaming_decoder_->FinishDecoding()) {
      decoding_status_ = DECODING_ERROR;
      SdchManager::SdchErrorRecovery(SdchManager::INCOMPLETE_SDCH_CONTENT);
      // Make it possible for the user to hit reload, and get non-sdch content.
      // Note this will "wear off" quickly enough, and is just meant to assure
      // in some rare case that the user is not stuck.
      SdchManager::BlacklistDomain(url());
    }
  }

  if (!was_cached()
      && base::Time() != connect_time()
      && read_times_.size() > 0
      && base::Time() != read_times_.back()) {
    base::TimeDelta duration = read_times_.back() - connect_time();
    // We clip our logging at 10 minutes to prevent anamolous data from being
    // considered (per suggestion from Jake Brutlag).
    if (10 >= duration.InMinutes()) {
      if (DECODING_IN_PROGRESS == decoding_status_) {
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_Latency_F", duration,
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_1st_To_Last",
                                    read_times_.back() - read_times_[0],
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        if (read_times_.size() > 3)
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_3rd_To_4th",
                                      read_times_[3] - read_times_[2],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
        UMA_HISTOGRAM_COUNTS_100(L"Sdch.Network_Decode_Reads",
                                 read_times_.size());
        UMA_HISTOGRAM_COUNTS(L"Sdch.Network_Decode_Bytes_Read", output_bytes_);
      } else if (PASS_THROUGH == decoding_status_) {
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_Latency_F",
                                    duration,
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_1st_To_Last",
                                    read_times_.back() - read_times_[0],
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        if (read_times_.size() > 3)
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_3rd_To_4th",
                                      read_times_[3] - read_times_[2],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
        UMA_HISTOGRAM_COUNTS_100(L"Sdch.Network_Pass-through_Reads",
                                 read_times_.size());
      }
    }
  }

  if (dictionary_)
    dictionary_->Release();
}

bool SdchFilter::InitDecoding(Filter::FilterType filter_type) {
  if (decoding_status_ != DECODING_UNINITIALIZED)
    return false;

  // Handle case  where sdch filter is guessed, but not required.
  if (FILTER_TYPE_SDCH_POSSIBLE == filter_type)
    possible_pass_through_ = true;

  // Initialize decoder only after we have a dictionary in hand.
  decoding_status_ = WAITING_FOR_DICTIONARY_SELECTION;
  return true;
}

static const char* kDecompressionErrorHtml =
  "<META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\">"
  "
"
  "An error occurred. This page will be reloaded shortly. "
  "Or press the \"reload\" button now to reload it immediately."
  "
";

Filter::FilterStatus SdchFilter::ReadFilteredData(char* dest_buffer,
                                                  int* dest_len) {
  int available_space = *dest_len;
  *dest_len = 0;  // Nothing output yet.

  if (!dest_buffer || available_space <= 0)
    return FILTER_ERROR;

  // Don't update when we're called to just flush out our internal buffers.
  if (next_stream_data_ && stream_data_len_ > 0)
    read_times_.push_back(base::Time::Now());

  if (WAITING_FOR_DICTIONARY_SELECTION == decoding_status_) {
    FilterStatus status = InitializeDictionary();
    if (FILTER_NEED_MORE_DATA == status)
      return FILTER_NEED_MORE_DATA;
    if (FILTER_ERROR == status) {
      DCHECK(DECODING_ERROR == decoding_status_);
      DCHECK(0 == dest_buffer_excess_index_);
      DCHECK(dest_buffer_excess_.empty());
      if (possible_pass_through_) {
        // We added the sdch coding tag, and it should not have been added.
        // This can happen in server experiments, where the server decides
        // not to use sdch, even though there is a dictionary.  To be
        // conservative, we locally added the tentative sdch (fearing that a
        // proxy stripped it!) and we must now recant (pass through).
        SdchManager::SdchErrorRecovery(SdchManager::DISCARD_TENTATIVE_SDCH);
        decoding_status_ = PASS_THROUGH;
        dest_buffer_excess_ = dictionary_hash_;  // Send what we scanned.
      } else if (!dictionary_hash_is_plausible_) {
        // One of the first 9 bytes precluded consideration as a hash.
        // This can't be an SDCH payload, even though the server said it was.
        // This is a major error, as the server or proxy tagged this SDCH even
        // though it is not!
        // The good news is that error recovery is clear...
        SdchManager::SdchErrorRecovery(SdchManager::PASSING_THROUGH_NON_SDCH);
        decoding_status_ = PASS_THROUGH;
        dest_buffer_excess_ = dictionary_hash_;  // Send what we scanned.
      } else {
        // We don't have the dictionary that was demanded.
        // With very low probability, random garbage data looked like a
        // dictionary specifier (8 ASCII characters followed by a null), but
        // that is sufficiently unlikely that we ignore it.
        if (std::string::npos == mime_type().find_first_of("text/html")) {
          SdchManager::BlacklistDomainForever(url());
          if (was_cached_)
            SdchManager::SdchErrorRecovery(
                SdchManager::CACHED_META_REFRESH_UNSUPPORTED);
          else
            SdchManager::SdchErrorRecovery(
                SdchManager::META_REFRESH_UNSUPPORTED);
          return FILTER_ERROR;
        }
        // HTML content means we can issue a meta-refresh, and get the content
        // again, perhaps without SDCH (to be safe).
        if (was_cached_) {
          // Cached content is probably a startup tab, so we'll just get fresh
          // content and try again, without disabling sdch.
          SdchManager::SdchErrorRecovery(
              SdchManager::META_REFRESH_CACHED_RECOVERY);
        } else {
          // Since it wasn't in the cache, we definately need at least some
          // period of blacklisting to get the correct content.
          SdchManager::BlacklistDomain(url());
          SdchManager::SdchErrorRecovery(SdchManager::META_REFRESH_RECOVERY);
        }
        decoding_status_ = META_REFRESH_RECOVERY;
        // Issue a meta redirect with SDCH disabled.
        dest_buffer_excess_ = kDecompressionErrorHtml;
      }
    } else {
      DCHECK(DECODING_IN_PROGRESS == decoding_status_);
    }
  }

  int amount = OutputBufferExcess(dest_buffer, available_space);
  *dest_len += amount;
  dest_buffer += amount;
  available_space -= amount;
  DCHECK(available_space >= 0);

  if (available_space <= 0)
    return FILTER_OK;
  DCHECK(dest_buffer_excess_.empty());
  DCHECK(0 == dest_buffer_excess_index_);

  if (decoding_status_ != DECODING_IN_PROGRESS) {
    if (META_REFRESH_RECOVERY == decoding_status_) {
      // Absorb all input data.  We've already output page reload HTML.
      next_stream_data_ = NULL;
      stream_data_len_ = 0;
      return FILTER_NEED_MORE_DATA;
    }
    if (PASS_THROUGH == decoding_status_) {
      // We must pass in available_space, but it will be changed to bytes_used.
      FilterStatus result = CopyOut(dest_buffer, &available_space);
      // Accumulate the returned count of bytes_used (a.k.a., available_space).
      *dest_len += available_space;
      return result;
    }
    DCHECK(false);
    decoding_status_ = DECODING_ERROR;
    return FILTER_ERROR;
  }

  if (!next_stream_data_ || stream_data_len_ <= 0)
    return FILTER_NEED_MORE_DATA;

  bool ret = vcdiff_streaming_decoder_->DecodeChunk(
    next_stream_data_, stream_data_len_, &dest_buffer_excess_);
  // Assume all data was used in decoding.
  next_stream_data_ = NULL;
  source_bytes_ += stream_data_len_;
  stream_data_len_ = 0;
  output_bytes_ += dest_buffer_excess_.size();
  if (!ret) {
    vcdiff_streaming_decoder_.reset(NULL);  // Don't call it again.
    decoding_status_ = DECODING_ERROR;
    SdchManager::SdchErrorRecovery(SdchManager::DECODE_BODY_ERROR);
    return FILTER_ERROR;
  }

  amount = OutputBufferExcess(dest_buffer, available_space);
  *dest_len += amount;
  dest_buffer += amount;
  if (0 == available_space && !dest_buffer_excess_.empty())
      return FILTER_OK;
  return FILTER_NEED_MORE_DATA;
}

Filter::FilterStatus SdchFilter::InitializeDictionary() {
  const size_t kServerIdLength = 9;  // Dictionary hash plus null from server.
  size_t bytes_needed = kServerIdLength - dictionary_hash_.size();
  DCHECK(bytes_needed > 0);
  if (!next_stream_data_)
    return FILTER_NEED_MORE_DATA;
  if (static_cast<size_t>(stream_data_len_) < bytes_needed) {
    dictionary_hash_.append(next_stream_data_, stream_data_len_);
    next_stream_data_ = NULL;
    stream_data_len_ = 0;
    return FILTER_NEED_MORE_DATA;
  }
  dictionary_hash_.append(next_stream_data_, bytes_needed);
  DCHECK(kServerIdLength == dictionary_hash_.size());
  stream_data_len_ -= bytes_needed;
  DCHECK(0 <= stream_data_len_);
  if (stream_data_len_ > 0)
    next_stream_data_ += bytes_needed;
  else
    next_stream_data_ = NULL;

  DCHECK(!dictionary_);
  dictionary_hash_is_plausible_ = true;  // Assume plausible, but check.
  if ('\0' == dictionary_hash_[kServerIdLength - 1])
    SdchManager::Global()->GetVcdiffDictionary(std::string(dictionary_hash_, 0,
                                                           kServerIdLength - 1),
                                               url(), &dictionary_);
  else
    dictionary_hash_is_plausible_ = false;

  if (!dictionary_) {
    DCHECK(dictionary_hash_.size() == kServerIdLength);
    // Since dictionary was not found, check to see if hash was even plausible.
    for (size_t i = 0; i < kServerIdLength - 1; ++i) {
      char base64_char = dictionary_hash_[i];
      if (!isalnum(base64_char) && '-' != base64_char && '_' != base64_char) {
        dictionary_hash_is_plausible_ = false;
        break;
      }
    }
    if (dictionary_hash_is_plausible_)
      SdchManager::SdchErrorRecovery(SdchManager::DICTIONARY_HASH_NOT_FOUND);
    else
      SdchManager::SdchErrorRecovery(SdchManager::DICTIONARY_HASH_MALFORMED);
    decoding_status_ = DECODING_ERROR;
    return FILTER_ERROR;
  }
  dictionary_->AddRef();
  vcdiff_streaming_decoder_.reset(new open_vcdiff::VCDiffStreamingDecoder);
  vcdiff_streaming_decoder_->StartDecoding(dictionary_->text().data(),
                                           dictionary_->text().size());
  decoding_status_ = DECODING_IN_PROGRESS;
  return FILTER_OK;
}

int SdchFilter::OutputBufferExcess(char* const dest_buffer,
                                   size_t available_space) {
  if (dest_buffer_excess_.empty())
    return 0;
  DCHECK(dest_buffer_excess_.size() > dest_buffer_excess_index_);
  size_t amount = std::min(available_space,
      dest_buffer_excess_.size() - dest_buffer_excess_index_);
  memcpy(dest_buffer, dest_buffer_excess_.data() + dest_buffer_excess_index_,
         amount);
  dest_buffer_excess_index_ += amount;
  if (dest_buffer_excess_.size() <= dest_buffer_excess_index_) {
    DCHECK(dest_buffer_excess_.size() == dest_buffer_excess_index_);
    dest_buffer_excess_.clear();
    dest_buffer_excess_index_ = 0;
  }
  return amount;
}
<commit_msg>Add one more histogram to SDCH decoder<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <limits.h>
#include <ctype.h>
#include <algorithm>

#include "base/file_util.h"
#include "base/histogram.h"
#include "base/logging.h"
#include "net/base/sdch_filter.h"
#include "net/base/sdch_manager.h"

#include "sdch/open-vcdiff/src/google/vcdecoder.h"

SdchFilter::SdchFilter()
    : decoding_status_(DECODING_UNINITIALIZED),
      vcdiff_streaming_decoder_(NULL),
      dictionary_hash_(),
      dictionary_hash_is_plausible_(false),
      dictionary_(NULL),
      dest_buffer_excess_(),
      dest_buffer_excess_index_(0),
      source_bytes_(0),
      output_bytes_(0),
      possible_pass_through_(false) {
}

SdchFilter::~SdchFilter() {
  static int filter_use_count = 0;
  ++filter_use_count;
  if (META_REFRESH_RECOVERY == decoding_status_) {
    UMA_HISTOGRAM_COUNTS(L"Sdch.FilterUseBeforeDisabling", filter_use_count);
  }

  if (vcdiff_streaming_decoder_.get()) {
    if (!vcdiff_streaming_decoder_->FinishDecoding()) {
      decoding_status_ = DECODING_ERROR;
      SdchManager::SdchErrorRecovery(SdchManager::INCOMPLETE_SDCH_CONTENT);
      // Make it possible for the user to hit reload, and get non-sdch content.
      // Note this will "wear off" quickly enough, and is just meant to assure
      // in some rare case that the user is not stuck.
      SdchManager::BlacklistDomain(url());
    }
  }

  if (!was_cached()
      && base::Time() != connect_time()
      && read_times_.size() > 0
      && base::Time() != read_times_.back()) {
    base::TimeDelta duration = read_times_.back() - connect_time();
    // We clip our logging at 10 minutes to prevent anamolous data from being
    // considered (per suggestion from Jake Brutlag).
    if (10 >= duration.InMinutes()) {
      if (DECODING_IN_PROGRESS == decoding_status_) {
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_Latency_F", duration,
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_1st_To_Last",
                                    read_times_.back() - read_times_[0],
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        if (read_times_.size() > 3) {
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_3rd_To_4th",
                                      read_times_[3] - read_times_[2],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Decode_2nd_To_3rd",
                                      read_times_[2] - read_times_[1],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
        }
        UMA_HISTOGRAM_COUNTS_100(L"Sdch.Network_Decode_Reads",
                                 read_times_.size());
        UMA_HISTOGRAM_COUNTS(L"Sdch.Network_Decode_Bytes_Read", output_bytes_);
      } else if (PASS_THROUGH == decoding_status_) {
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_Latency_F",
                                    duration,
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_1st_To_Last",
                                    read_times_.back() - read_times_[0],
                                    base::TimeDelta::FromMilliseconds(20),
                                    base::TimeDelta::FromMinutes(10), 100);
        if (read_times_.size() > 3) {
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_3rd_To_4th",
                                      read_times_[3] - read_times_[2],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
          UMA_HISTOGRAM_CLIPPED_TIMES(L"Sdch.Network_Pass-through_2nd_To_3rd",
                                      read_times_[2] - read_times_[1],
                                      base::TimeDelta::FromMilliseconds(10),
                                      base::TimeDelta::FromSeconds(3), 100);
        }
        UMA_HISTOGRAM_COUNTS_100(L"Sdch.Network_Pass-through_Reads",
                                 read_times_.size());
      }
    }
  }

  if (dictionary_)
    dictionary_->Release();
}

bool SdchFilter::InitDecoding(Filter::FilterType filter_type) {
  if (decoding_status_ != DECODING_UNINITIALIZED)
    return false;

  // Handle case  where sdch filter is guessed, but not required.
  if (FILTER_TYPE_SDCH_POSSIBLE == filter_type)
    possible_pass_through_ = true;

  // Initialize decoder only after we have a dictionary in hand.
  decoding_status_ = WAITING_FOR_DICTIONARY_SELECTION;
  return true;
}

static const char* kDecompressionErrorHtml =
  "<META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\">"
  "
"
  "An error occurred. This page will be reloaded shortly. "
  "Or press the \"reload\" button now to reload it immediately."
  "
";

Filter::FilterStatus SdchFilter::ReadFilteredData(char* dest_buffer,
                                                  int* dest_len) {
  int available_space = *dest_len;
  *dest_len = 0;  // Nothing output yet.

  if (!dest_buffer || available_space <= 0)
    return FILTER_ERROR;

  // Don't update when we're called to just flush out our internal buffers.
  if (next_stream_data_ && stream_data_len_ > 0)
    read_times_.push_back(base::Time::Now());

  if (WAITING_FOR_DICTIONARY_SELECTION == decoding_status_) {
    FilterStatus status = InitializeDictionary();
    if (FILTER_NEED_MORE_DATA == status)
      return FILTER_NEED_MORE_DATA;
    if (FILTER_ERROR == status) {
      DCHECK(DECODING_ERROR == decoding_status_);
      DCHECK(0 == dest_buffer_excess_index_);
      DCHECK(dest_buffer_excess_.empty());
      if (possible_pass_through_) {
        // We added the sdch coding tag, and it should not have been added.
        // This can happen in server experiments, where the server decides
        // not to use sdch, even though there is a dictionary.  To be
        // conservative, we locally added the tentative sdch (fearing that a
        // proxy stripped it!) and we must now recant (pass through).
        SdchManager::SdchErrorRecovery(SdchManager::DISCARD_TENTATIVE_SDCH);
        decoding_status_ = PASS_THROUGH;
        dest_buffer_excess_ = dictionary_hash_;  // Send what we scanned.
      } else if (!dictionary_hash_is_plausible_) {
        // One of the first 9 bytes precluded consideration as a hash.
        // This can't be an SDCH payload, even though the server said it was.
        // This is a major error, as the server or proxy tagged this SDCH even
        // though it is not!
        // The good news is that error recovery is clear...
        SdchManager::SdchErrorRecovery(SdchManager::PASSING_THROUGH_NON_SDCH);
        decoding_status_ = PASS_THROUGH;
        dest_buffer_excess_ = dictionary_hash_;  // Send what we scanned.
      } else {
        // We don't have the dictionary that was demanded.
        // With very low probability, random garbage data looked like a
        // dictionary specifier (8 ASCII characters followed by a null), but
        // that is sufficiently unlikely that we ignore it.
        if (std::string::npos == mime_type().find_first_of("text/html")) {
          SdchManager::BlacklistDomainForever(url());
          if (was_cached_)
            SdchManager::SdchErrorRecovery(
                SdchManager::CACHED_META_REFRESH_UNSUPPORTED);
          else
            SdchManager::SdchErrorRecovery(
                SdchManager::META_REFRESH_UNSUPPORTED);
          return FILTER_ERROR;
        }
        // HTML content means we can issue a meta-refresh, and get the content
        // again, perhaps without SDCH (to be safe).
        if (was_cached_) {
          // Cached content is probably a startup tab, so we'll just get fresh
          // content and try again, without disabling sdch.
          SdchManager::SdchErrorRecovery(
              SdchManager::META_REFRESH_CACHED_RECOVERY);
        } else {
          // Since it wasn't in the cache, we definately need at least some
          // period of blacklisting to get the correct content.
          SdchManager::BlacklistDomain(url());
          SdchManager::SdchErrorRecovery(SdchManager::META_REFRESH_RECOVERY);
        }
        decoding_status_ = META_REFRESH_RECOVERY;
        // Issue a meta redirect with SDCH disabled.
        dest_buffer_excess_ = kDecompressionErrorHtml;
      }
    } else {
      DCHECK(DECODING_IN_PROGRESS == decoding_status_);
    }
  }

  int amount = OutputBufferExcess(dest_buffer, available_space);
  *dest_len += amount;
  dest_buffer += amount;
  available_space -= amount;
  DCHECK(available_space >= 0);

  if (available_space <= 0)
    return FILTER_OK;
  DCHECK(dest_buffer_excess_.empty());
  DCHECK(0 == dest_buffer_excess_index_);

  if (decoding_status_ != DECODING_IN_PROGRESS) {
    if (META_REFRESH_RECOVERY == decoding_status_) {
      // Absorb all input data.  We've already output page reload HTML.
      next_stream_data_ = NULL;
      stream_data_len_ = 0;
      return FILTER_NEED_MORE_DATA;
    }
    if (PASS_THROUGH == decoding_status_) {
      // We must pass in available_space, but it will be changed to bytes_used.
      FilterStatus result = CopyOut(dest_buffer, &available_space);
      // Accumulate the returned count of bytes_used (a.k.a., available_space).
      *dest_len += available_space;
      return result;
    }
    DCHECK(false);
    decoding_status_ = DECODING_ERROR;
    return FILTER_ERROR;
  }

  if (!next_stream_data_ || stream_data_len_ <= 0)
    return FILTER_NEED_MORE_DATA;

  bool ret = vcdiff_streaming_decoder_->DecodeChunk(
    next_stream_data_, stream_data_len_, &dest_buffer_excess_);
  // Assume all data was used in decoding.
  next_stream_data_ = NULL;
  source_bytes_ += stream_data_len_;
  stream_data_len_ = 0;
  output_bytes_ += dest_buffer_excess_.size();
  if (!ret) {
    vcdiff_streaming_decoder_.reset(NULL);  // Don't call it again.
    decoding_status_ = DECODING_ERROR;
    SdchManager::SdchErrorRecovery(SdchManager::DECODE_BODY_ERROR);
    return FILTER_ERROR;
  }

  amount = OutputBufferExcess(dest_buffer, available_space);
  *dest_len += amount;
  dest_buffer += amount;
  if (0 == available_space && !dest_buffer_excess_.empty())
      return FILTER_OK;
  return FILTER_NEED_MORE_DATA;
}

Filter::FilterStatus SdchFilter::InitializeDictionary() {
  const size_t kServerIdLength = 9;  // Dictionary hash plus null from server.
  size_t bytes_needed = kServerIdLength - dictionary_hash_.size();
  DCHECK(bytes_needed > 0);
  if (!next_stream_data_)
    return FILTER_NEED_MORE_DATA;
  if (static_cast<size_t>(stream_data_len_) < bytes_needed) {
    dictionary_hash_.append(next_stream_data_, stream_data_len_);
    next_stream_data_ = NULL;
    stream_data_len_ = 0;
    return FILTER_NEED_MORE_DATA;
  }
  dictionary_hash_.append(next_stream_data_, bytes_needed);
  DCHECK(kServerIdLength == dictionary_hash_.size());
  stream_data_len_ -= bytes_needed;
  DCHECK(0 <= stream_data_len_);
  if (stream_data_len_ > 0)
    next_stream_data_ += bytes_needed;
  else
    next_stream_data_ = NULL;

  DCHECK(!dictionary_);
  dictionary_hash_is_plausible_ = true;  // Assume plausible, but check.
  if ('\0' == dictionary_hash_[kServerIdLength - 1])
    SdchManager::Global()->GetVcdiffDictionary(std::string(dictionary_hash_, 0,
                                                           kServerIdLength - 1),
                                               url(), &dictionary_);
  else
    dictionary_hash_is_plausible_ = false;

  if (!dictionary_) {
    DCHECK(dictionary_hash_.size() == kServerIdLength);
    // Since dictionary was not found, check to see if hash was even plausible.
    for (size_t i = 0; i < kServerIdLength - 1; ++i) {
      char base64_char = dictionary_hash_[i];
      if (!isalnum(base64_char) && '-' != base64_char && '_' != base64_char) {
        dictionary_hash_is_plausible_ = false;
        break;
      }
    }
    if (dictionary_hash_is_plausible_)
      SdchManager::SdchErrorRecovery(SdchManager::DICTIONARY_HASH_NOT_FOUND);
    else
      SdchManager::SdchErrorRecovery(SdchManager::DICTIONARY_HASH_MALFORMED);
    decoding_status_ = DECODING_ERROR;
    return FILTER_ERROR;
  }
  dictionary_->AddRef();
  vcdiff_streaming_decoder_.reset(new open_vcdiff::VCDiffStreamingDecoder);
  vcdiff_streaming_decoder_->StartDecoding(dictionary_->text().data(),
                                           dictionary_->text().size());
  decoding_status_ = DECODING_IN_PROGRESS;
  return FILTER_OK;
}

int SdchFilter::OutputBufferExcess(char* const dest_buffer,
                                   size_t available_space) {
  if (dest_buffer_excess_.empty())
    return 0;
  DCHECK(dest_buffer_excess_.size() > dest_buffer_excess_index_);
  size_t amount = std::min(available_space,
      dest_buffer_excess_.size() - dest_buffer_excess_index_);
  memcpy(dest_buffer, dest_buffer_excess_.data() + dest_buffer_excess_index_,
         amount);
  dest_buffer_excess_index_ += amount;
  if (dest_buffer_excess_.size() <= dest_buffer_excess_index_) {
    DCHECK(dest_buffer_excess_.size() == dest_buffer_excess_index_);
    dest_buffer_excess_.clear();
    dest_buffer_excess_index_ = 0;
  }
  return amount;
}
<|endoftext|>


<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "net/test/test_server.h"

#include <algorithm>
#include <string>
#include <vector>

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include "net/base/x509_certificate.h"
#endif

#include "base/base64.h"
#include "base/command_line.h"
#include "base/debug/leak_annotations.h"
#include "base/json/json_reader.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/scoped_ptr.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "base/values.h"
#include "googleurl/src/gurl.h"
#include "net/base/cert_test_util.h"
#include "net/base/host_port_pair.h"
#include "net/base/host_resolver.h"
#include "net/base/test_completion_callback.h"
#include "net/socket/tcp_client_socket.h"
#include "net/test/python_utils.h"
#include "testing/platform_test.h"

namespace net {

namespace {

// Number of connection attempts for tests.
const int kServerConnectionAttempts = 10;

// Connection timeout in milliseconds for tests.
const int kServerConnectionTimeoutMs = 1000;

std::string GetHostname(TestServer::Type type,
                        const TestServer::HTTPSOptions& options) {
  if (type == TestServer::TYPE_HTTPS &&
      options.server_certificate ==
          TestServer::HTTPSOptions::CERT_MISMATCHED_NAME) {
    // Return a different hostname string that resolves to the same hostname.
    return "localhost";
  }

  return "127.0.0.1";
}

}  // namespace

#if defined(OS_MACOSX)
void SetMacTestCertificate(X509Certificate* cert);
#endif

TestServer::HTTPSOptions::HTTPSOptions()
    : server_certificate(CERT_OK),
      request_client_certificate(false),
      bulk_ciphers(HTTPSOptions::BULK_CIPHER_ANY) {}

TestServer::HTTPSOptions::HTTPSOptions(
    TestServer::HTTPSOptions::ServerCertificate cert)
    : server_certificate(cert),
      request_client_certificate(false),
      bulk_ciphers(HTTPSOptions::BULK_CIPHER_ANY) {}

TestServer::HTTPSOptions::~HTTPSOptions() {}

FilePath TestServer::HTTPSOptions::GetCertificateFile() const {
  switch (server_certificate) {
    case CERT_OK:
    case CERT_MISMATCHED_NAME:
      return FilePath(FILE_PATH_LITERAL("ok_cert.pem"));
    case CERT_EXPIRED:
      return FilePath(FILE_PATH_LITERAL("expired_cert.pem"));
    default:
      NOTREACHED();
  }
  return FilePath();
}

TestServer::TestServer(Type type, const FilePath& document_root)
    : type_(type),
      started_(false) {
  Init(document_root);
}

TestServer::TestServer(const HTTPSOptions& https_options,
                       const FilePath& document_root)
    : https_options_(https_options),
      type_(TYPE_HTTPS),
      started_(false) {
  Init(document_root);
}

TestServer::~TestServer() {
#if defined(OS_MACOSX)
  SetMacTestCertificate(NULL);
#endif
  Stop();
}

void TestServer::Init(const FilePath& document_root) {
  // At this point, the port that the testserver will listen on is unknown.
  // The testserver will listen on an ephemeral port, and write the port
  // number out over a pipe that this TestServer object will read from. Once
  // that is complete, the host_port_pair_ will contain the actual port.
  host_port_pair_ = HostPortPair(GetHostname(type_, https_options_), 0);
  process_handle_ = base::kNullProcessHandle;

  FilePath src_dir;
  PathService::Get(base::DIR_SOURCE_ROOT, &src_dir);

  document_root_ = src_dir.Append(document_root);

  certificates_dir_ = src_dir.Append(FILE_PATH_LITERAL("net"))
                       .Append(FILE_PATH_LITERAL("data"))
                       .Append(FILE_PATH_LITERAL("ssl"))
                       .Append(FILE_PATH_LITERAL("certificates"));
}

bool TestServer::Start() {
  if (type_ == TYPE_HTTPS) {
    if (!LoadTestRootCert())
      return false;
    if (!CheckCATrusted())
      return false;
  }

  // Get path to python server script
  FilePath testserver_path;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &testserver_path)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  testserver_path = testserver_path
      .Append(FILE_PATH_LITERAL("net"))
      .Append(FILE_PATH_LITERAL("tools"))
      .Append(FILE_PATH_LITERAL("testserver"))
      .Append(FILE_PATH_LITERAL("testserver.py"));

  if (!SetPythonPath())
    return false;

  if (!LaunchPython(testserver_path))
    return false;

  if (!WaitToStart()) {
    Stop();
    return false;
  }

  started_ = true;
  return true;
}

bool TestServer::Stop() {
  if (!process_handle_)
    return true;

  started_ = false;

  // First check if the process has already terminated.
  bool ret = base::WaitForSingleProcess(process_handle_, 0);
  if (!ret)
    ret = base::KillProcess(process_handle_, 1, true);

  if (ret) {
    base::CloseProcessHandle(process_handle_);
    process_handle_ = base::kNullProcessHandle;
  } else {
    VLOG(1) << "Kill failed?";
  }

  return ret;
}

const HostPortPair& TestServer::host_port_pair() const {
  DCHECK(started_);
  return host_port_pair_;
}

std::string TestServer::GetScheme() const {
  switch (type_) {
    case TYPE_FTP:
      return "ftp";
    case TYPE_HTTP:
    case TYPE_SYNC:
      return "http";
    case TYPE_HTTPS:
      return "https";
    default:
      NOTREACHED();
  }
  return std::string();
}

bool TestServer::GetAddressList(AddressList* address_list) const {
  DCHECK(address_list);

  scoped_ptr<HostResolver> resolver(
      CreateSystemHostResolver(HostResolver::kDefaultParallelism, NULL, NULL));
  HostResolver::RequestInfo info(host_port_pair_);
  int rv = resolver->Resolve(info, address_list, NULL, NULL, BoundNetLog());
  if (rv != net::OK) {
    LOG(ERROR) << "Failed to resolve hostname: " << host_port_pair_.host();
    return false;
  }
  return true;
}

GURL TestServer::GetURL(const std::string& path) const {
  return GURL(GetScheme() + "://" + host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUser(const std::string& path,
                                const std::string& user) const {
  return GURL(GetScheme() + "://" + user + "@" +
              host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUserAndPassword(const std::string& path,
                                           const std::string& user,
                                           const std::string& password) const {
  return GURL(GetScheme() + "://" + user + ":" + password +
              "@" + host_port_pair_.ToString() +
              "/" + path);
}

// static
bool TestServer::GetFilePathWithReplacements(
    const std::string& original_file_path,
    const std::vector<StringPair>& text_to_replace,
    std::string* replacement_path) {
  std::string new_file_path = original_file_path;
  bool first_query_parameter = true;
  const std::vector<StringPair>::const_iterator end = text_to_replace.end();
  for (std::vector<StringPair>::const_iterator it = text_to_replace.begin();
       it != end;
       ++it) {
    const std::string& old_text = it->first;
    const std::string& new_text = it->second;
    std::string base64_old;
    std::string base64_new;
    if (!base::Base64Encode(old_text, &base64_old))
      return false;
    if (!base::Base64Encode(new_text, &base64_new))
      return false;
    if (first_query_parameter) {
      new_file_path += "?";
      first_query_parameter = false;
    } else {
      new_file_path += "&";
    }
    new_file_path += "replace_text=";
    new_file_path += base64_old;
    new_file_path += ":";
    new_file_path += base64_new;
  }

  *replacement_path = new_file_path;
  return true;
}

bool TestServer::SetPythonPath() {
  FilePath third_party_dir;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &third_party_dir)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  third_party_dir = third_party_dir.Append(FILE_PATH_LITERAL("third_party"));

  // For simplejson. (simplejson, unlike all the other python modules
  // we include, doesn't have an extra 'simplejson' directory, so we
  // need to include its parent directory, i.e. third_party_dir).
  AppendToPythonPath(third_party_dir);

  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("tlslite")));
  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("pyftpdlib")));

  // Locate the Python code generated by the protocol buffers compiler.
  FilePath pyproto_code_dir;
  if (!GetPyProtoPath(&pyproto_code_dir)) {
    LOG(ERROR) << "Failed to get python dir for generated code.";
    return false;
  }

  AppendToPythonPath(pyproto_code_dir);
  AppendToPythonPath(pyproto_code_dir.Append(FILE_PATH_LITERAL("sync_pb")));
  AppendToPythonPath(pyproto_code_dir.Append(
      FILE_PATH_LITERAL("device_management_pb")));

  return true;
}

FilePath TestServer::GetRootCertificatePath() {
  return certificates_dir_.AppendASCII("root_ca_cert.crt");
}

bool TestServer::LoadTestRootCert() {
#if defined(USE_NSS)
  if (cert_)
    return true;

  // TODO(dkegel): figure out how to get this to only happen once?

  // This currently leaks a little memory.
  // TODO(dkegel): fix the leak and remove the entry in
  // tools/valgrind/memcheck/suppressions.txt
  ANNOTATE_SCOPED_MEMORY_LEAK;  // Tell heap checker about the leak.
  cert_ = LoadTemporaryRootCert(GetRootCertificatePath());
  return (cert_ != NULL);
#elif defined(OS_MACOSX)
  X509Certificate* cert = LoadTemporaryRootCert(GetRootCertificatePath());
  if (!cert)
    return false;
  SetMacTestCertificate(cert);
  return true;
#else
  return true;
#endif
}

bool TestServer::AddCommandLineArguments(CommandLine* command_line) const {
  command_line->AppendSwitchASCII("port",
                                  base::IntToString(host_port_pair_.port()));
  command_line->AppendSwitchPath("data-dir", document_root_);

  if (type_ == TYPE_FTP) {
    command_line->AppendArg("-f");
  } else if (type_ == TYPE_SYNC) {
    command_line->AppendArg("--sync");
  } else if (type_ == TYPE_HTTPS) {
    FilePath certificate_path(certificates_dir_);
    certificate_path = certificate_path.Append(
        https_options_.GetCertificateFile());
    if (!file_util::PathExists(certificate_path)) {
      LOG(ERROR) << "Certificate path " << certificate_path.value()
                 << " doesn't exist. Can't launch https server.";
      return false;
    }
    command_line->AppendSwitchPath("https", certificate_path);

    if (https_options_.request_client_certificate)
      command_line->AppendSwitch("ssl-client-auth");

    for (std::vector<FilePath>::const_iterator it =
             https_options_.client_authorities.begin();
         it != https_options_.client_authorities.end(); ++it) {
      if (!file_util::PathExists(*it)) {
        LOG(ERROR) << "Client authority path " << it->value()
                   << " doesn't exist. Can't launch https server.";
        return false;
      }

      command_line->AppendSwitchPath("ssl-client-ca", *it);
    }

    const char kBulkCipherSwitch[] = "ssl-bulk-cipher";
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_RC4)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "rc4");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_AES128)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "aes128");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_AES256)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "aes256");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_3DES)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "3des");
  }

  return true;
}

bool TestServer::ParseServerData(const std::string& server_data) {
  VLOG(1) << "Server data: " << server_data;
  base::JSONReader json_reader;
  scoped_ptr<Value> value(json_reader.JsonToValue(server_data, true, false));
  if (!value.get() ||
      !value->IsType(Value::TYPE_DICTIONARY)) {
    LOG(ERROR) << "Could not parse server data: "
               << json_reader.GetErrorMessage();
    return false;
  }
  DictionaryValue* dict = static_cast<DictionaryValue*>(value.get());
  int port = 0;
  if (!dict->GetInteger("port", &port)) {
    LOG(ERROR) << "Could not find port value";
    return false;
  }
  if ((port <= 0) || (port >= kuint16max)) {
    LOG(ERROR) << "Invalid port value: " << port;
    return false;
  }
  host_port_pair_.set_port(port);
  return true;
}

}  // namespace net
<commit_msg>Fixed port boundary check in test_server.cc<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "net/test/test_server.h"

#include <algorithm>
#include <string>
#include <vector>

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include "net/base/x509_certificate.h"
#endif

#include "base/base64.h"
#include "base/command_line.h"
#include "base/debug/leak_annotations.h"
#include "base/json/json_reader.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/scoped_ptr.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "base/values.h"
#include "googleurl/src/gurl.h"
#include "net/base/cert_test_util.h"
#include "net/base/host_port_pair.h"
#include "net/base/host_resolver.h"
#include "net/base/test_completion_callback.h"
#include "net/socket/tcp_client_socket.h"
#include "net/test/python_utils.h"
#include "testing/platform_test.h"

namespace net {

namespace {

// Number of connection attempts for tests.
const int kServerConnectionAttempts = 10;

// Connection timeout in milliseconds for tests.
const int kServerConnectionTimeoutMs = 1000;

std::string GetHostname(TestServer::Type type,
                        const TestServer::HTTPSOptions& options) {
  if (type == TestServer::TYPE_HTTPS &&
      options.server_certificate ==
          TestServer::HTTPSOptions::CERT_MISMATCHED_NAME) {
    // Return a different hostname string that resolves to the same hostname.
    return "localhost";
  }

  return "127.0.0.1";
}

}  // namespace

#if defined(OS_MACOSX)
void SetMacTestCertificate(X509Certificate* cert);
#endif

TestServer::HTTPSOptions::HTTPSOptions()
    : server_certificate(CERT_OK),
      request_client_certificate(false),
      bulk_ciphers(HTTPSOptions::BULK_CIPHER_ANY) {}

TestServer::HTTPSOptions::HTTPSOptions(
    TestServer::HTTPSOptions::ServerCertificate cert)
    : server_certificate(cert),
      request_client_certificate(false),
      bulk_ciphers(HTTPSOptions::BULK_CIPHER_ANY) {}

TestServer::HTTPSOptions::~HTTPSOptions() {}

FilePath TestServer::HTTPSOptions::GetCertificateFile() const {
  switch (server_certificate) {
    case CERT_OK:
    case CERT_MISMATCHED_NAME:
      return FilePath(FILE_PATH_LITERAL("ok_cert.pem"));
    case CERT_EXPIRED:
      return FilePath(FILE_PATH_LITERAL("expired_cert.pem"));
    default:
      NOTREACHED();
  }
  return FilePath();
}

TestServer::TestServer(Type type, const FilePath& document_root)
    : type_(type),
      started_(false) {
  Init(document_root);
}

TestServer::TestServer(const HTTPSOptions& https_options,
                       const FilePath& document_root)
    : https_options_(https_options),
      type_(TYPE_HTTPS),
      started_(false) {
  Init(document_root);
}

TestServer::~TestServer() {
#if defined(OS_MACOSX)
  SetMacTestCertificate(NULL);
#endif
  Stop();
}

void TestServer::Init(const FilePath& document_root) {
  // At this point, the port that the testserver will listen on is unknown.
  // The testserver will listen on an ephemeral port, and write the port
  // number out over a pipe that this TestServer object will read from. Once
  // that is complete, the host_port_pair_ will contain the actual port.
  host_port_pair_ = HostPortPair(GetHostname(type_, https_options_), 0);
  process_handle_ = base::kNullProcessHandle;

  FilePath src_dir;
  PathService::Get(base::DIR_SOURCE_ROOT, &src_dir);

  document_root_ = src_dir.Append(document_root);

  certificates_dir_ = src_dir.Append(FILE_PATH_LITERAL("net"))
                       .Append(FILE_PATH_LITERAL("data"))
                       .Append(FILE_PATH_LITERAL("ssl"))
                       .Append(FILE_PATH_LITERAL("certificates"));
}

bool TestServer::Start() {
  if (type_ == TYPE_HTTPS) {
    if (!LoadTestRootCert())
      return false;
    if (!CheckCATrusted())
      return false;
  }

  // Get path to python server script
  FilePath testserver_path;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &testserver_path)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  testserver_path = testserver_path
      .Append(FILE_PATH_LITERAL("net"))
      .Append(FILE_PATH_LITERAL("tools"))
      .Append(FILE_PATH_LITERAL("testserver"))
      .Append(FILE_PATH_LITERAL("testserver.py"));

  if (!SetPythonPath())
    return false;

  if (!LaunchPython(testserver_path))
    return false;

  if (!WaitToStart()) {
    Stop();
    return false;
  }

  started_ = true;
  return true;
}

bool TestServer::Stop() {
  if (!process_handle_)
    return true;

  started_ = false;

  // First check if the process has already terminated.
  bool ret = base::WaitForSingleProcess(process_handle_, 0);
  if (!ret)
    ret = base::KillProcess(process_handle_, 1, true);

  if (ret) {
    base::CloseProcessHandle(process_handle_);
    process_handle_ = base::kNullProcessHandle;
  } else {
    VLOG(1) << "Kill failed?";
  }

  return ret;
}

const HostPortPair& TestServer::host_port_pair() const {
  DCHECK(started_);
  return host_port_pair_;
}

std::string TestServer::GetScheme() const {
  switch (type_) {
    case TYPE_FTP:
      return "ftp";
    case TYPE_HTTP:
    case TYPE_SYNC:
      return "http";
    case TYPE_HTTPS:
      return "https";
    default:
      NOTREACHED();
  }
  return std::string();
}

bool TestServer::GetAddressList(AddressList* address_list) const {
  DCHECK(address_list);

  scoped_ptr<HostResolver> resolver(
      CreateSystemHostResolver(HostResolver::kDefaultParallelism, NULL, NULL));
  HostResolver::RequestInfo info(host_port_pair_);
  int rv = resolver->Resolve(info, address_list, NULL, NULL, BoundNetLog());
  if (rv != net::OK) {
    LOG(ERROR) << "Failed to resolve hostname: " << host_port_pair_.host();
    return false;
  }
  return true;
}

GURL TestServer::GetURL(const std::string& path) const {
  return GURL(GetScheme() + "://" + host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUser(const std::string& path,
                                const std::string& user) const {
  return GURL(GetScheme() + "://" + user + "@" +
              host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUserAndPassword(const std::string& path,
                                           const std::string& user,
                                           const std::string& password) const {
  return GURL(GetScheme() + "://" + user + ":" + password +
              "@" + host_port_pair_.ToString() +
              "/" + path);
}

// static
bool TestServer::GetFilePathWithReplacements(
    const std::string& original_file_path,
    const std::vector<StringPair>& text_to_replace,
    std::string* replacement_path) {
  std::string new_file_path = original_file_path;
  bool first_query_parameter = true;
  const std::vector<StringPair>::const_iterator end = text_to_replace.end();
  for (std::vector<StringPair>::const_iterator it = text_to_replace.begin();
       it != end;
       ++it) {
    const std::string& old_text = it->first;
    const std::string& new_text = it->second;
    std::string base64_old;
    std::string base64_new;
    if (!base::Base64Encode(old_text, &base64_old))
      return false;
    if (!base::Base64Encode(new_text, &base64_new))
      return false;
    if (first_query_parameter) {
      new_file_path += "?";
      first_query_parameter = false;
    } else {
      new_file_path += "&";
    }
    new_file_path += "replace_text=";
    new_file_path += base64_old;
    new_file_path += ":";
    new_file_path += base64_new;
  }

  *replacement_path = new_file_path;
  return true;
}

bool TestServer::SetPythonPath() {
  FilePath third_party_dir;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &third_party_dir)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  third_party_dir = third_party_dir.Append(FILE_PATH_LITERAL("third_party"));

  // For simplejson. (simplejson, unlike all the other python modules
  // we include, doesn't have an extra 'simplejson' directory, so we
  // need to include its parent directory, i.e. third_party_dir).
  AppendToPythonPath(third_party_dir);

  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("tlslite")));
  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("pyftpdlib")));

  // Locate the Python code generated by the protocol buffers compiler.
  FilePath pyproto_code_dir;
  if (!GetPyProtoPath(&pyproto_code_dir)) {
    LOG(ERROR) << "Failed to get python dir for generated code.";
    return false;
  }

  AppendToPythonPath(pyproto_code_dir);
  AppendToPythonPath(pyproto_code_dir.Append(FILE_PATH_LITERAL("sync_pb")));
  AppendToPythonPath(pyproto_code_dir.Append(
      FILE_PATH_LITERAL("device_management_pb")));

  return true;
}

FilePath TestServer::GetRootCertificatePath() {
  return certificates_dir_.AppendASCII("root_ca_cert.crt");
}

bool TestServer::LoadTestRootCert() {
#if defined(USE_NSS)
  if (cert_)
    return true;

  // TODO(dkegel): figure out how to get this to only happen once?

  // This currently leaks a little memory.
  // TODO(dkegel): fix the leak and remove the entry in
  // tools/valgrind/memcheck/suppressions.txt
  ANNOTATE_SCOPED_MEMORY_LEAK;  // Tell heap checker about the leak.
  cert_ = LoadTemporaryRootCert(GetRootCertificatePath());
  return (cert_ != NULL);
#elif defined(OS_MACOSX)
  X509Certificate* cert = LoadTemporaryRootCert(GetRootCertificatePath());
  if (!cert)
    return false;
  SetMacTestCertificate(cert);
  return true;
#else
  return true;
#endif
}

bool TestServer::AddCommandLineArguments(CommandLine* command_line) const {
  command_line->AppendSwitchASCII("port",
                                  base::IntToString(host_port_pair_.port()));
  command_line->AppendSwitchPath("data-dir", document_root_);

  if (type_ == TYPE_FTP) {
    command_line->AppendArg("-f");
  } else if (type_ == TYPE_SYNC) {
    command_line->AppendArg("--sync");
  } else if (type_ == TYPE_HTTPS) {
    FilePath certificate_path(certificates_dir_);
    certificate_path = certificate_path.Append(
        https_options_.GetCertificateFile());
    if (!file_util::PathExists(certificate_path)) {
      LOG(ERROR) << "Certificate path " << certificate_path.value()
                 << " doesn't exist. Can't launch https server.";
      return false;
    }
    command_line->AppendSwitchPath("https", certificate_path);

    if (https_options_.request_client_certificate)
      command_line->AppendSwitch("ssl-client-auth");

    for (std::vector<FilePath>::const_iterator it =
             https_options_.client_authorities.begin();
         it != https_options_.client_authorities.end(); ++it) {
      if (!file_util::PathExists(*it)) {
        LOG(ERROR) << "Client authority path " << it->value()
                   << " doesn't exist. Can't launch https server.";
        return false;
      }

      command_line->AppendSwitchPath("ssl-client-ca", *it);
    }

    const char kBulkCipherSwitch[] = "ssl-bulk-cipher";
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_RC4)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "rc4");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_AES128)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "aes128");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_AES256)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "aes256");
    if (https_options_.bulk_ciphers & HTTPSOptions::BULK_CIPHER_3DES)
      command_line->AppendSwitchASCII(kBulkCipherSwitch, "3des");
  }

  return true;
}

bool TestServer::ParseServerData(const std::string& server_data) {
  VLOG(1) << "Server data: " << server_data;
  base::JSONReader json_reader;
  scoped_ptr<Value> value(json_reader.JsonToValue(server_data, true, false));
  if (!value.get() ||
      !value->IsType(Value::TYPE_DICTIONARY)) {
    LOG(ERROR) << "Could not parse server data: "
               << json_reader.GetErrorMessage();
    return false;
  }
  DictionaryValue* dict = static_cast<DictionaryValue*>(value.get());
  int port = 0;
  if (!dict->GetInteger("port", &port)) {
    LOG(ERROR) << "Could not find port value";
    return false;
  }
  if ((port <= 0) || (port > kuint16max)) {
    LOG(ERROR) << "Invalid port value: " << port;
    return false;
  }
  host_port_pair_.set_port(port);
  return true;
}

}  // namespace net
<|endoftext|>


<commit_before>#include <ecto/ecto.hpp>
namespace bp = boost::python;
using namespace ecto::except;
namespace ecto
{

  namespace py
  {
    template<typename ExceptionType>
    struct Translate_
    {
      static void translate(const ExceptionType & x)
      {
        PyErr_SetObject(Exc_Type_, bp::object(x).ptr());
        PyErr_SetString(Exc_Type_, x.what());
        bp::throw_error_already_set();
      }
      static PyObject* Exc_Type_;
    };

    template<typename ExceptionType>
    PyObject* Translate_<ExceptionType>::Exc_Type_;

    template<typename ExceptionType>
    void register_exception(const std::string& name)
    {
      bp::class_<ExceptionType> exc_cl(name.c_str());
      Translate_<ExceptionType>::Exc_Type_ = exc_cl.ptr();
      bp::register_exception_translator<ExceptionType>(&Translate_<ExceptionType>::translate);
    }

    void wrap_except()
    {
      register_exception<EctoException>("EctoException");
      register_exception<ValueNone>("ValueNone");
      register_exception<TypeMismatch>("TypeMismatch");
      register_exception<ValueRequired>("ValueRequired");
    }
  }
}
<commit_msg>Exceptions.<commit_after>#include <ecto/ecto.hpp>
#include <boost/python/errors.hpp>
namespace bp = boost::python;
using namespace ecto::except;
namespace ecto
{

  namespace py
  {
    template<typename ExceptionType>
    struct Translate_
    {
      static void translate(const ExceptionType & x)
      {
        PyErr_SetObject(Exc_Type_, bp::object(x).ptr());
        PyErr_SetString(Exc_Type_, x.what());
        bp::throw_error_already_set();
      }
      static PyObject* Exc_Type_;
    };

    template<typename ExceptionType>
    PyObject* Translate_<ExceptionType>::Exc_Type_;

    template<typename ExceptionType>
    void register_exception(const std::string& name)
    {
      bp::class_<ExceptionType> exc_cl(name.c_str());
      Translate_<ExceptionType>::Exc_Type_ = exc_cl.ptr();
      bp::register_exception_translator<ExceptionType>(&Translate_<ExceptionType>::translate);
    }

    void wrap_except()
    {
      register_exception<EctoException>("EctoException");
      register_exception<ValueNone>("ValueNone");
      register_exception<TypeMismatch>("TypeMismatch");
      register_exception<ValueRequired>("ValueRequired");
    }
  }
}
<|endoftext|>


<commit_before>#ifndef DEAMORTIZED_HASH_SET
#define DEAMORTIZED_HASH_SET

#include <cstdint>
#include <functional>
#include <cstdlib>
#include <cassert>
#include <cstddef>
#include <memory>
#include <vector>

using std::vector;
using std::pair;
using std::make_pair;

#include "tlsf_allocator.hpp"

template<typename Key,
         typename Hash = std::hash<Key>,
         typename Equal = std::equal_to<Key>,
         typename Allocate = TlsfAllocator<Key> >
class LimitedHashSet {
  typedef std::size_t size_t;
public:

  LimitedHashSet& operator=(const LimitedHashSet&) = delete;
  LimitedHashSet(const LimitedHashSet&) = delete;
  LimitedHashSet(LimitedHashSet&&) = delete;
  LimitedHashSet& operator=(LimitedHashSet&&) = delete;

public:

  struct Slot {
    bool occupied;
    Key key;
  };
  Hash hasher;
  Equal equaler;
  typedef typename Allocate::template rebind<Slot>::other A;
  A allocator;
  size_t capacity, size;
  Slot * data;
  size_t progress;
  enum ACT : char {
    CLEAR, INIT, RESET, FILL
  } state;

public:

  // TODO: overload for std::swap
  // TODO: copy and move and assignment constructors
  void swap(LimitedHashSet & that) {
    std::swap(allocator, that.allocator);
    std::swap(capacity, that.capacity);
    std::swap(size, that.size);
    std::swap(data, that.data);
    std::swap(progress, that.progress);
    std::swap(state, that.state);
  }

  explicit LimitedHashSet(const Allocate& a, const size_t capacity)
    : hasher(), equaler(), allocator(a), capacity(capacity), size(0),
      data(allocator.allocate(capacity)), progress(0), state(INIT)
  {}

  // TODO: this need to take state into account
  ~LimitedHashSet() {
    switch (state) {
    case CLEAR:
      while (progress < capacity) {
        allocator.destroy(&data[progress++]);
      }
      break;
    case INIT: // no data - still initializing
      break;
    case RESET: // no data - not even initialized yet
      break;
    case FILL:
      for (size_t i = 0; i < capacity; ++i) {
        if (data[i].occupied) {
          allocator.destroy(&data[i]);
        }
      }
    }
    allocator.deallocate(data, capacity);
  }

  void clear(const size_t left) {
    assert ((0 == capacity) || (state == FILL) || (state == CLEAR));
    if (state == FILL) {
      progress = 0;
    }
    state = CLEAR;
    const size_t many = left ? ((capacity - progress + left - 1)/left) : (capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      if (data[progress].occupied) allocator.destroy(&data[progress]);
      ++progress;
    }
  }

  void init(const size_t left) {
    assert ((0 == capacity) || (state == RESET) || (state == INIT));
    if (RESET == state) {
      progress = 0;
    }
    state = INIT;
    const size_t many = left ? ((capacity - progress + left - 1)/left) : (capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      data[progress].occupied = false;
      ++progress;
    }
  }

  void reset(const size_t new_capacity) {
    assert ((0 == capacity) || (state == CLEAR) || (INIT == state) || (FILL == state));
    state = RESET;
    size = 0;
    if (new_capacity != capacity) {
      allocator.deallocate(data, capacity);
      capacity = new_capacity;
      data = allocator.allocate(capacity);
    }
    progress = 0;
  }

  void fill(const size_t left, const LimitedHashSet& that) {
    assert ((0 == capacity) || (state == FILL) || (state == INIT));
    if (INIT == state) {
      progress = 0;
    }
    state = FILL;
    // don't care about *my* progress
    const size_t many = left ? ((that.capacity - progress + left - 1)/left) : (that.capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      if (that.data[progress].occupied) insert(that.data[progress].key);
      ++progress;
    }
  }

  void catch_up(const vector<pair<Key *, pair<size_t, size_t> > > & moved_back) {
    for (const auto & kft : moved_back) {
      if ((kft.second.first >= progress)
          and (kft.second.second < progress)) {
        insert(*kft.first);
      }
    }
  }

private:

  // find a key. capacity must be > 0. if key is not present, returns
  // location where it would be if it were newly placed.
  size_t locate(const Key& k) const {
    const size_t cap_mask = capacity-1;
    auto h = hasher(k) & cap_mask;
    while (true) {
      if (not data[h].occupied)    return h;
      if (equaler(data[h].key, k)) return h;
      h = (h+1) & cap_mask;
    }
  }

  // i < j <= k, mod capacity
  static bool cyclic_between(const size_t& i, const size_t& j, const size_t& k) {
    return (((i < j) and (j <= k))
            or ((i < j) and (k < i))
            or ((j <= k) and (k < i)));
  }

public:
  Key* find(const Key& k) {
    if (NULL == data) return nullptr;
    const size_t i = locate(k);
    if (data[i].occupied && equaler(data[i].key, k)) {
      return &data[i].key;
    }
    return nullptr;
  }

public:

  vector<pair<Key *, pair<size_t, size_t> > > erase(const Key& k) {
    vector<pair<Key *, pair<size_t, size_t> > > ans;
    const size_t cap_mask = capacity-1;
    size_t start = locate(k); // the empty location
    if (not data[start].occupied) return ans;
    data[start].occupied = false;
    data[start].key.~Key();
    --size;
    size_t i = (start + 1) & cap_mask;
    while (data[i].occupied) {
      const size_t h = hasher(data[i].key) & cap_mask;
      if (not cyclic_between(start,h,i)) {
        data[start].occupied = true;
        new (&data[start].key) Key(std::move(data[i].key));
        ans.push_back(make_pair(&data[start].key, make_pair(i, start)));
        start = i;
        data[i].occupied = false;
        //data[i].key.~Key(); // no : already moved it?
      }
      i = (i+1) & cap_mask;
    }
    return ans;
  }

  bool insert(const Key& k) {
    const size_t i = locate(k);
    if (data[i].occupied) return false;
    new (&data[i].key) Key(k);
    ++size;
    data[i].occupied = true;
    return true;
  }

};

// IDEA: store hash values, ones means "empty"

template<typename Key,
         typename Hash = std::hash<Key>,
         typename Equal = std::equal_to<Key>,
         typename Allocate = TlsfAllocator<Key> >
struct DeamortizedHashSet {

  DeamortizedHashSet() :
    allocator(), far(allocator,0), near(allocator,32) {
    near.init(1);
    near.state = LimitedHashSet<Key, Hash, Equal,Allocate>::ACT::FILL;
  }

  Allocate allocator;
  LimitedHashSet<Key, Hash, Equal, Allocate> far, near;
  // 1/4: clear old
  // 1/4: init new
  // 1/2: fill new

  enum ACT : char {
    CLEAR, MAKE_BIGGER, INIT_BIGGER, MAKE_SMALLER, INIT_SMALLER, FILL, SWAP, RELAX
  };

  // how much time you have to do this
  pair<ACT, size_t> get_act_help() const {
    if (near.size < near.capacity/8) return make_pair(SWAP, 1);
    if (near.size < 3*(near.capacity/16)) return make_pair(FILL, near.size - near.capacity/8);
    if (near.size < 7*(near.capacity/32) - 1) return make_pair(INIT_SMALLER, near.size - 3*(near.capacity/16));
    if (near.size < 7*(near.capacity/32)) return make_pair(MAKE_SMALLER, near.size - 3*(near.capacity/16));
    if (near.size < near.capacity/4) return make_pair(CLEAR, near.size - 7*(near.capacity/32));
    if (near.size < 5*(near.capacity/16)) return make_pair(CLEAR, 5*(near.capacity/16) - near.size);
    if (near.size < 5*(near.capacity/16) + 1) return make_pair(MAKE_BIGGER, 5*(near.capacity/16) - near.size);
    if (near.size < 3*(near.capacity/8)) return make_pair(INIT_BIGGER, 3*(near.capacity/8) - near.size);
    if (near.size < near.capacity/2) return make_pair(FILL, near.capacity/2 - near.size);
    return make_pair(SWAP, 1);
  }

  pair<ACT, size_t> get_act() const {
    if ((near.capacity <= 32) and (near.size < 7)) return make_pair(RELAX, 0);
    return get_act_help();
  }

  void erase(const Key& k) {
    vector<pair<Key *, pair<size_t, size_t> > >  moved_back = near.erase(k);
    const auto act = get_act();
    switch (act.first) {
    case RELAX:
      break;
    case CLEAR:
      far.clear(act.second);
      break;
    case MAKE_BIGGER:
      far.reset(near.capacity*2);
    case INIT_BIGGER:
      far.init(act.second);
      break;
    case MAKE_SMALLER:
      far.reset(near.capacity/2);
    case INIT_SMALLER:
      far.init(act.second);
      break;
    case FILL:
      far.fill(act.second, near);
      far.erase(k);
      far.catch_up(moved_back);
      break;
    default: // SWAP
      far.swap(near);
      far.erase(k);
    }
  }

  bool insert(const Key& k) {
    const bool ans = near.insert(k);
    const auto act = get_act();
    switch (act.first) {
    case RELAX:
      break;
    case CLEAR:
      far.clear(act.second);
      break;
    case MAKE_BIGGER:
      far.reset(near.capacity*2);
    case INIT_BIGGER:
      far.init(act.second);
      break;
    case MAKE_SMALLER:
      far.reset(near.capacity/2);
    case INIT_SMALLER:
      far.init(act.second);
      break;
    case FILL:
      far.fill(act.second, near);
      far.insert(k);
      break;
    default: // SWAP
      far.swap(near);
      near.insert(k);
    }
    return ans;
  }
};

#endif // LINEAR_HASH_SET
<commit_msg>wrong doubleside<commit_after>#ifndef DEAMORTIZED_HASH_SET
#define DEAMORTIZED_HASH_SET

#include <cstdint>
#include <functional>
#include <cstdlib>
#include <cassert>
#include <cstddef>
#include <memory>
#include <vector>

using std::vector;
using std::pair;
using std::make_pair;

#include "tlsf_allocator.hpp"

template<typename Key,
         typename Hash = std::hash<Key>,
         typename Equal = std::equal_to<Key>,
         typename Allocate = TlsfAllocator<Key> >
class LimitedHashSet {
  typedef std::size_t size_t;
public:

  LimitedHashSet& operator=(const LimitedHashSet&) = delete;
  LimitedHashSet(const LimitedHashSet&) = delete;
  LimitedHashSet(LimitedHashSet&&) = delete;
  LimitedHashSet& operator=(LimitedHashSet&&) = delete;

public:

  struct Slot {
    bool occupied;
    Key key;
  };
  Hash hasher;
  Equal equaler;
  typedef typename Allocate::template rebind<Slot>::other A;
  A allocator;
  size_t capacity, size;
  Slot * data;
  size_t progress;
  enum ACT : char {
    CLEAR, INIT, RESET, FILL
  } state;

public:

  // TODO: overload for std::swap
  // TODO: copy and move and assignment constructors
  void swap(LimitedHashSet & that) {
    std::swap(allocator, that.allocator);
    std::swap(capacity, that.capacity);
    std::swap(size, that.size);
    std::swap(data, that.data);
    std::swap(progress, that.progress);
    std::swap(state, that.state);
  }

  explicit LimitedHashSet(const Allocate& a, const size_t capacity)
    : hasher(), equaler(), allocator(a), capacity(capacity), size(0),
      data(allocator.allocate(capacity)), progress(0), state(INIT)
  {}

  // TODO: this need to take state into account
  ~LimitedHashSet() {
    switch (state) {
    case CLEAR:
      while (progress < capacity) {
        allocator.destroy(&data[progress++]);
      }
      break;
    case INIT: // no data - still initializing
      break;
    case RESET: // no data - not even initialized yet
      break;
    case FILL:
      for (size_t i = 0; i < capacity; ++i) {
        if (data[i].occupied) {
          allocator.destroy(&data[i]);
        }
      }
    }
    allocator.deallocate(data, capacity);
  }

  void clear(const size_t left) {
    assert ((0 == capacity) || (state == FILL) || (state == CLEAR));
    if (state == FILL) {
      progress = 0;
    }
    state = CLEAR;
    const size_t many = left ? ((capacity - progress + left - 1)/left) : (capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      if (data[progress].occupied) allocator.destroy(&data[progress]);
      ++progress;
    }
  }

  void init(const size_t left) {
    assert ((0 == capacity) || (state == RESET) || (state == INIT) || (FILL == state));
    if (RESET == state) {
      progress = 0;
    }
    state = INIT;
    const size_t many = left ? ((capacity - progress + left - 1)/left) : (capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      data[progress].occupied = false;
      ++progress;
    }
  }

  void reset(const size_t new_capacity) {
    assert ((0 == capacity) || (state == CLEAR) || (INIT == state));
    state = RESET;
    size = 0;
    if (new_capacity != capacity) {
      allocator.deallocate(data, capacity);
      capacity = new_capacity;
      data = allocator.allocate(capacity);
    }
    progress = 0;
  }

  void fill(const size_t left, const LimitedHashSet& that) {
    assert ((0 == capacity) || (state == FILL) || (state == INIT));
    if (INIT == state) {
      progress = 0;
    }
    state = FILL;
    // don't care about *my* progress
    const size_t many = left ? ((that.capacity - progress + left - 1)/left) : (that.capacity - progress);
    for (size_t i = 0; i < many; ++i) {
      if (that.data[progress].occupied) insert(that.data[progress].key);
      ++progress;
    }
  }

  void catch_up(const vector<pair<Key *, pair<size_t, size_t> > > & moved_back) {
    for (const auto & kft : moved_back) {
      if ((kft.second.first >= progress)
          and (kft.second.second < progress)) {
        insert(*kft.first);
      }
    }
  }

private:

  // find a key. capacity must be > 0. if key is not present, returns
  // location where it would be if it were newly placed.
  size_t locate(const Key& k) const {
    const size_t cap_mask = capacity-1;
    auto h = hasher(k) & cap_mask;
    while (true) {
      if (not data[h].occupied)    return h;
      if (equaler(data[h].key, k)) return h;
      h = (h+1) & cap_mask;
    }
  }

  // i < j <= k, mod capacity
  static bool cyclic_between(const size_t& i, const size_t& j, const size_t& k) {
    return (((i < j) and (j <= k))
            or ((i < j) and (k < i))
            or ((j <= k) and (k < i)));
  }

public:
  Key* find(const Key& k) {
    if (NULL == data) return nullptr;
    const size_t i = locate(k);
    if (data[i].occupied && equaler(data[i].key, k)) {
      return &data[i].key;
    }
    return nullptr;
  }

public:

  vector<pair<Key *, pair<size_t, size_t> > > erase(const Key& k) {
    vector<pair<Key *, pair<size_t, size_t> > > ans;
    const size_t cap_mask = capacity-1;
    size_t start = locate(k); // the empty location
    if (not data[start].occupied) return ans;
    data[start].occupied = false;
    data[start].key.~Key();
    --size;
    size_t i = (start + 1) & cap_mask;
    while (data[i].occupied) {
      const size_t h = hasher(data[i].key) & cap_mask;
      if (not cyclic_between(start,h,i)) {
        data[start].occupied = true;
        new (&data[start].key) Key(std::move(data[i].key));
        ans.push_back(make_pair(&data[start].key, make_pair(i, start)));
        start = i;
        data[i].occupied = false;
        //data[i].key.~Key(); // no : already moved it?
      }
      i = (i+1) & cap_mask;
    }
    return ans;
  }

  bool insert(const Key& k) {
    const size_t i = locate(k);
    if (data[i].occupied) return false;
    new (&data[i].key) Key(k);
    ++size;
    data[i].occupied = true;
    return true;
  }

};

// IDEA: store hash values, ones means "empty"

template<typename Key,
         typename Hash = std::hash<Key>,
         typename Equal = std::equal_to<Key>,
         typename Allocate = TlsfAllocator<Key> >
struct DeamortizedHashSet {

  DeamortizedHashSet() :
    allocator(), far(allocator,0), near(allocator,32) {
    near.init(1);
    near.state = LimitedHashSet<Key, Hash, Equal,Allocate>::ACT::FILL;
  }

  Allocate allocator;
  LimitedHashSet<Key, Hash, Equal, Allocate> far, near;
  // 1/4: clear old
  // 1/4: init new
  // 1/2: fill new

  enum ACT : char {
    CLEAR, MAKE_BIGGER, INIT_BIGGER, MAKE_SMALLER, INIT_SMALLER, FILL, SWAP, RELAX
  };

  // how much time you have to do this
  pair<ACT, size_t> get_act_help() const {
    if (near.size < near.capacity/8) return make_pair(SWAP, 1);
    if (near.size < 3*(near.capacity/16)) return make_pair(FILL, near.size - near.capacity/8);
    if (near.size < 7*(near.capacity/32) - 1) return make_pair(INIT_SMALLER, near.size - 3*(near.capacity/16));
    if (near.size < 7*(near.capacity/32)) return make_pair(MAKE_SMALLER, near.size - 3*(near.capacity/16));
    if (near.size < near.capacity/4) return make_pair(CLEAR, near.size - 7*(near.capacity/32));
    if (near.size < 5*(near.capacity/16)) return make_pair(CLEAR, 5*(near.capacity/16) - near.size);
    if (near.size < 5*(near.capacity/16) + 1) return make_pair(MAKE_BIGGER, 5*(near.capacity/16) - near.size);
    if (near.size < 3*(near.capacity/8)) return make_pair(INIT_BIGGER, 3*(near.capacity/8) - near.size);
    if (near.size < near.capacity/2) return make_pair(FILL, near.capacity/2 - near.size);
    return make_pair(SWAP, 1);
  }

  pair<ACT, size_t> get_act() const {
    if ((near.capacity <= 32) and (near.size < 7)) return make_pair(RELAX, 0);
    return get_act_help();
  }

  void erase(const Key& k) {
    vector<pair<Key *, pair<size_t, size_t> > >  moved_back = near.erase(k);
    const auto act = get_act();
    switch (act.first) {
    case RELAX:
      break;
    case CLEAR:
      far.clear(act.second);
      break;
    case MAKE_BIGGER:
      far.reset(near.capacity*2);
    case INIT_BIGGER:
      far.init(act.second);
      break;
    case MAKE_SMALLER:
      far.reset(near.capacity/2);
    case INIT_SMALLER:
      far.init(act.second);
      break;
    case FILL:
      far.fill(act.second, near);
      far.erase(k);
      far.catch_up(moved_back);
      break;
    default: // SWAP
      far.swap(near);
      far.erase(k);
    }
  }

  bool insert(const Key& k) {
    const bool ans = near.insert(k);
    const auto act = get_act();
    switch (act.first) {
    case RELAX:
      break;
    case CLEAR:
      far.clear(act.second);
      break;
    case MAKE_BIGGER:
      far.reset(near.capacity*2);
    case INIT_BIGGER:
      far.init(act.second);
      break;
    case MAKE_SMALLER:
      far.reset(near.capacity/2);
    case INIT_SMALLER:
      far.init(act.second);
      break;
    case FILL:
      far.fill(act.second, near);
      far.insert(k);
      break;
    default: // SWAP
      far.swap(near);
      near.insert(k);
    }
    return ans;
  }
};

#endif // LINEAR_HASH_SET
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/detail/MemoryIdler.h>
#include <folly/Logging.h>
#include <folly/Malloc.h>
#include <folly/ScopeGuard.h>
#include <folly/detail/CacheLocality.h>
#include <limits.h>
#include <pthread.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <utility>

// weak linking means the symbol will be null if not available, instead
// of a link failure
extern "C" int mallctl(const char *name, void *oldp, size_t *oldlenp,
                       void *newp, size_t newlen)
    __attribute__((__weak__));

namespace folly { namespace detail {

AtomicStruct<std::chrono::steady_clock::duration>
MemoryIdler::defaultIdleTimeout(std::chrono::seconds(5));

/// Calls mallctl, optionally reading and/or writing an unsigned value
/// if in and/or out is non-null.  Logs on error
static unsigned mallctlWrapper(const char* cmd, const unsigned* in,
                               unsigned* out) {
  size_t outLen = sizeof(unsigned);
  int err = mallctl(cmd,
                    out, out ? &outLen : nullptr,
                    const_cast<unsigned*>(in), in ? sizeof(unsigned) : 0);
  if (err != 0) {
    FB_LOG_EVERY_MS(WARNING, 10000)
      << "mallctl " << cmd << ": " << strerror(err) << " (" << err << ")";
  }
  return err;
}

void MemoryIdler::flushLocalMallocCaches() {
  if (usingJEMalloc()) {
    if (!mallctl) {
      FB_LOG_EVERY_MS(ERROR, 10000) << "mallctl weak link failed";
      return;
    }

    // "tcache.flush" was renamed to "thread.tcache.flush" in jemalloc 3
    (void)mallctlWrapper("thread.tcache.flush", nullptr, nullptr);

    // By default jemalloc has 4 arenas per cpu, and then assigns each
    // thread to one of those arenas.  This means that in any service
    // that doesn't perform a lot of context switching, the chances that
    // another thread will be using the current thread's arena (and hence
    // doing the appropriate dirty-page purging) are low.  Some good
    // tuned configurations (such as that used by hhvm) use fewer arenas
    // and then pin threads to avoid contended access.  In that case,
    // purging the arenas is counter-productive.  We use the heuristic
    // that if narenas <= 2 * num_cpus then we shouldn't do anything here,
    // which detects when the narenas has been reduced from the default
    unsigned narenas;
    unsigned arenaForCurrent;
    if (mallctlWrapper("arenas.narenas", nullptr, &narenas) == 0 &&
        narenas > 2 * CacheLocality::system().numCpus &&
        mallctlWrapper("thread.arena", nullptr, &arenaForCurrent) == 0) {
      (void)mallctlWrapper("arenas.purge", &arenaForCurrent, nullptr);
    }
  }
}

#if FOLLY_X64

static const size_t s_pageSize = sysconf(_SC_PAGESIZE);
static FOLLY_TLS uintptr_t tls_stackLimit;
static FOLLY_TLS size_t tls_stackSize;

static void fetchStackLimits() {
  pthread_attr_t attr;
#if defined(_GNU_SOURCE) && defined(__linux__) // Linux+GNU extension
  pthread_getattr_np(pthread_self(), &attr);
#else
  pthread_attr_init(&attr);
#endif
  SCOPE_EXIT { pthread_attr_destroy(&attr); };

  void* addr;
  size_t rawSize;
  int err;
  if ((err = pthread_attr_getstack(&attr, &addr, &rawSize))) {
    // unexpected, but it is better to continue in prod than do nothing
    FB_LOG_EVERY_MS(ERROR, 10000) << "pthread_attr_getstack error " << err;
    assert(false);
    tls_stackSize = 1;
    return;
  }
  assert(addr != nullptr);
  assert(rawSize >= PTHREAD_STACK_MIN);

  // glibc subtracts guard page from stack size, even though pthread docs
  // seem to imply the opposite
  size_t guardSize;
  if (pthread_attr_getguardsize(&attr, &guardSize) != 0) {
    guardSize = 0;
  }
  assert(rawSize > guardSize);

  // stack goes down, so guard page adds to the base addr
  tls_stackLimit = uintptr_t(addr) + guardSize;
  tls_stackSize = rawSize - guardSize;

  assert((tls_stackLimit & (s_pageSize - 1)) == 0);
}

FOLLY_NOINLINE static uintptr_t getStackPtr() {
  char marker;
  auto rv = uintptr_t(&marker);
  return rv;
}

void MemoryIdler::unmapUnusedStack(size_t retain) {
  if (tls_stackSize == 0) {
    fetchStackLimits();
  }
  if (tls_stackSize <= std::max(size_t(1), retain)) {
    // covers both missing stack info, and impossibly large retain
    return;
  }

  auto sp = getStackPtr();
  assert(sp >= tls_stackLimit);
  assert(sp - tls_stackLimit < tls_stackSize);

  auto end = (sp - retain) & ~(s_pageSize - 1);
  if (end <= tls_stackLimit) {
    // no pages are eligible for unmapping
    return;
  }

  size_t len = end - tls_stackLimit;
  assert((len & (s_pageSize - 1)) == 0);
  if (madvise((void*)tls_stackLimit, len, MADV_DONTNEED) != 0) {
    // It is likely that the stack vma hasn't been fully grown.  In this
    // case madvise will apply dontneed to the present vmas, then return
    // errno of ENOMEM.  We can also get an EAGAIN, theoretically.
    // EINVAL means either an invalid alignment or length, or that some
    // of the pages are locked or shared.  Neither should occur.
    assert(errno == EAGAIN || errno == ENOMEM);
  }
}

#else

void MemoryIdler::unmapUnusedStack(size_t retain) {
}

#endif

}}
<commit_msg>detail::MemoryIdler should use Malloc.h's mallctl decl<commit_after>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/detail/MemoryIdler.h>
#include <folly/Logging.h>
#include <folly/Malloc.h>
#include <folly/ScopeGuard.h>
#include <folly/detail/CacheLocality.h>
#include <limits.h>
#include <pthread.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <utility>

namespace folly { namespace detail {

AtomicStruct<std::chrono::steady_clock::duration>
MemoryIdler::defaultIdleTimeout(std::chrono::seconds(5));

/// Calls mallctl, optionally reading and/or writing an unsigned value
/// if in and/or out is non-null.  Logs on error
static unsigned mallctlWrapper(const char* cmd, const unsigned* in,
                               unsigned* out) {
  size_t outLen = sizeof(unsigned);
  int err = mallctl(cmd,
                    out, out ? &outLen : nullptr,
                    const_cast<unsigned*>(in), in ? sizeof(unsigned) : 0);
  if (err != 0) {
    FB_LOG_EVERY_MS(WARNING, 10000)
      << "mallctl " << cmd << ": " << strerror(err) << " (" << err << ")";
  }
  return err;
}

void MemoryIdler::flushLocalMallocCaches() {
  if (usingJEMalloc()) {
    if (!mallctl) {
      FB_LOG_EVERY_MS(ERROR, 10000) << "mallctl weak link failed";
      return;
    }

    // "tcache.flush" was renamed to "thread.tcache.flush" in jemalloc 3
    (void)mallctlWrapper("thread.tcache.flush", nullptr, nullptr);

    // By default jemalloc has 4 arenas per cpu, and then assigns each
    // thread to one of those arenas.  This means that in any service
    // that doesn't perform a lot of context switching, the chances that
    // another thread will be using the current thread's arena (and hence
    // doing the appropriate dirty-page purging) are low.  Some good
    // tuned configurations (such as that used by hhvm) use fewer arenas
    // and then pin threads to avoid contended access.  In that case,
    // purging the arenas is counter-productive.  We use the heuristic
    // that if narenas <= 2 * num_cpus then we shouldn't do anything here,
    // which detects when the narenas has been reduced from the default
    unsigned narenas;
    unsigned arenaForCurrent;
    if (mallctlWrapper("arenas.narenas", nullptr, &narenas) == 0 &&
        narenas > 2 * CacheLocality::system().numCpus &&
        mallctlWrapper("thread.arena", nullptr, &arenaForCurrent) == 0) {
      (void)mallctlWrapper("arenas.purge", &arenaForCurrent, nullptr);
    }
  }
}

#if FOLLY_X64

static const size_t s_pageSize = sysconf(_SC_PAGESIZE);
static FOLLY_TLS uintptr_t tls_stackLimit;
static FOLLY_TLS size_t tls_stackSize;

static void fetchStackLimits() {
  pthread_attr_t attr;
#if defined(_GNU_SOURCE) && defined(__linux__) // Linux+GNU extension
  pthread_getattr_np(pthread_self(), &attr);
#else
  pthread_attr_init(&attr);
#endif
  SCOPE_EXIT { pthread_attr_destroy(&attr); };

  void* addr;
  size_t rawSize;
  int err;
  if ((err = pthread_attr_getstack(&attr, &addr, &rawSize))) {
    // unexpected, but it is better to continue in prod than do nothing
    FB_LOG_EVERY_MS(ERROR, 10000) << "pthread_attr_getstack error " << err;
    assert(false);
    tls_stackSize = 1;
    return;
  }
  assert(addr != nullptr);
  assert(rawSize >= PTHREAD_STACK_MIN);

  // glibc subtracts guard page from stack size, even though pthread docs
  // seem to imply the opposite
  size_t guardSize;
  if (pthread_attr_getguardsize(&attr, &guardSize) != 0) {
    guardSize = 0;
  }
  assert(rawSize > guardSize);

  // stack goes down, so guard page adds to the base addr
  tls_stackLimit = uintptr_t(addr) + guardSize;
  tls_stackSize = rawSize - guardSize;

  assert((tls_stackLimit & (s_pageSize - 1)) == 0);
}

FOLLY_NOINLINE static uintptr_t getStackPtr() {
  char marker;
  auto rv = uintptr_t(&marker);
  return rv;
}

void MemoryIdler::unmapUnusedStack(size_t retain) {
  if (tls_stackSize == 0) {
    fetchStackLimits();
  }
  if (tls_stackSize <= std::max(size_t(1), retain)) {
    // covers both missing stack info, and impossibly large retain
    return;
  }

  auto sp = getStackPtr();
  assert(sp >= tls_stackLimit);
  assert(sp - tls_stackLimit < tls_stackSize);

  auto end = (sp - retain) & ~(s_pageSize - 1);
  if (end <= tls_stackLimit) {
    // no pages are eligible for unmapping
    return;
  }

  size_t len = end - tls_stackLimit;
  assert((len & (s_pageSize - 1)) == 0);
  if (madvise((void*)tls_stackLimit, len, MADV_DONTNEED) != 0) {
    // It is likely that the stack vma hasn't been fully grown.  In this
    // case madvise will apply dontneed to the present vmas, then return
    // errno of ENOMEM.  We can also get an EAGAIN, theoretically.
    // EINVAL means either an invalid alignment or length, or that some
    // of the pages are locked or shared.  Neither should occur.
    assert(errno == EAGAIN || errno == ENOMEM);
  }
}

#else

void MemoryIdler::unmapUnusedStack(size_t retain) {
}

#endif

}}
<|endoftext|>


<commit_before>//
// MessagePack for C++ static resolution routine
//
// Copyright (C) 2008 FURUHASHI Sadayuki
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
#ifndef MSGPACK_OBJECT_HPP__
#define MSGPACK_OBJECT_HPP__

#include "msgpack/pack.hpp"
#include <stdint.h>
#include <stdexcept>
#include <typeinfo>
#include <limits>
#include <ostream>

namespace msgpack {

class type_error : public std::bad_cast { };

namespace type {
	static const unsigned char NIL					= 0x01;
	static const unsigned char BOOLEAN				= 0x02;
	static const unsigned char POSITIVE_INTEGER		= 0x03;
	static const unsigned char NEGATIVE_INTEGER		= 0x04;
	static const unsigned char DOUBLE				= 0x05;
	static const unsigned char RAW					= 0x06;
	static const unsigned char ARRAY				= 0x07;
	static const unsigned char MAP					= 0x08;
}

struct object {
	unsigned char type;
	union {
		bool boolean;
		uint64_t u64;
		int64_t  i64;
		double   dec;
		struct {
			object* ptr;
			uint32_t size;
		} container;
		struct {
			const char* ptr;
			uint32_t size;
		} ref;
	} via;

	template <typename T>
	operator T() { T v; convert(v, *this); return v; };

	template <typename T>
	T as() { T v; convert(v, *this); return v; }

	bool is_nil() { return type == type::NIL; }
};

std::ostream& operator<< (std::ostream& s, const object o);

bool operator==(const object x, const object y);
inline bool operator!=(const object x, const object y) { return !(x == y); }

inline object& operator>> (object o, object& v)
	{ v = o; return v; }

template <typename Stream>
packer<Stream>& operator<< (packer<Stream>& o, const object& v);

template <typename T>
inline void convert(T& v, object o)
{
	o >> v;
}

template <typename Stream, typename T>
inline void pack(packer<Stream>& o, const T& v)
{
	o << v;
}

template <typename Stream, typename T>
inline void pack(Stream& s, const T& v)
{
	packer<Stream> pk(s);
	pack(pk, v);
}

template <typename Stream, typename T>
inline void pack_copy(packer<Stream>& o, T v)
{
	pack(o, v);
}

template <typename T>
inline T& operator>> (object o, T& v)
{
	v.msgpack_unpack(o);
	return v;
}

template <typename Stream, typename T>
inline packer<Stream>& operator<< (packer<Stream>& o, const T& v)
{
	o << v.msgpack_pack();
	return o;
}

template <typename Type>
class define : public Type {
public:
	typedef Type msgpack_type;
	typedef define<Type> define_type;

	define() {}
	define(msgpack_type v) : msgpack_type(v) {}

	msgpack_type msgpack_pack() const
	{
		return *this;
	}

	void msgpack_unpack(object o)
	{
		convert(static_cast<msgpack_type&>(*this), o);
	}
};

template <typename Stream>
packer<Stream>& operator<< (packer<Stream>& o, const object& v)
{
	switch(v.type) {
	case type::NIL:
		o.pack_nil();
		return o;

	case type::BOOLEAN:
		if(v.via.boolean) {
			o.pack_true();
		} else {
			o.pack_false();
		}
		return o;

	case type::POSITIVE_INTEGER:
		if(v.via.u64 <= (uint64_t)std::numeric_limits<uint16_t>::max()) {
			if(v.via.u64 <= (uint16_t)std::numeric_limits<uint8_t>::max()) {
				o.pack_uint8(v.via.u64);
			} else {
				o.pack_uint16(v.via.u64);
			}
		} else {
			if(v.via.u64 <= (uint64_t)std::numeric_limits<uint32_t>::max()) {
				o.pack_uint32(v.via.u64);
			} else {
				o.pack_uint64(v.via.u64);
			}
		}
		return o;

	case type::NEGATIVE_INTEGER:
		if(v.via.i64 >= (int64_t)std::numeric_limits<int16_t>::min()) {
			if(v.via.i64 >= (int64_t)std::numeric_limits<int8_t>::min()) {
				o.pack_int8(v.via.i64);
			} else {
				o.pack_int16(v.via.i64);
			}
		} else {
			if(v.via.i64 >= (int64_t)std::numeric_limits<int32_t>::min()) {
				o.pack_int64(v.via.i64);
			} else {
				o.pack_int64(v.via.i64);
			}
		}
		return o;

	case type::RAW:
		o.pack_raw(v.via.ref.size);
		o.pack_raw_body(v.via.ref.ptr, v.via.ref.size);
		return o;

	case type::ARRAY:
		o.pack_array(v.via.container.size);
		for(object* p(v.via.container.ptr),
				* const pend(v.via.container.ptr + v.via.container.size);
				p < pend; ++p) {
			*p >> o;
		}
		return o;
		// FIXME loop optimiziation

	case type::MAP:
		o.pack_map(v.via.container.size);
		for(object* p(v.via.container.ptr),
				* const pend(v.via.container.ptr + v.via.container.size*2);
				p < pend; ) {
			*p >> o; ++p;
			*p >> o; ++p;
		}
		return o;
		// FIXME loop optimiziation

	default:
		throw type_error();
	}
}

}  // namespace msgpack

#include "msgpack/type.hpp"

#endif /* msgpack/object.hpp */

<commit_msg>lang/c/msgpack: C++ binding: abolished implicit convertion and added object::convert()<commit_after>//
// MessagePack for C++ static resolution routine
//
// Copyright (C) 2008 FURUHASHI Sadayuki
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
#ifndef MSGPACK_OBJECT_HPP__
#define MSGPACK_OBJECT_HPP__

#include "msgpack/pack.hpp"
#include <stdint.h>
#include <stdexcept>
#include <typeinfo>
#include <limits>
#include <ostream>

namespace msgpack {

class type_error : public std::bad_cast { };

namespace type {
	static const unsigned char NIL					= 0x01;
	static const unsigned char BOOLEAN				= 0x02;
	static const unsigned char POSITIVE_INTEGER		= 0x03;
	static const unsigned char NEGATIVE_INTEGER		= 0x04;
	static const unsigned char DOUBLE				= 0x05;
	static const unsigned char RAW					= 0x06;
	static const unsigned char ARRAY				= 0x07;
	static const unsigned char MAP					= 0x08;
}

struct object {
	union union_type {
		bool boolean;
		uint64_t u64;
		int64_t  i64;
		double   dec;
		struct {
			object* ptr;
			uint32_t size;
		} container;
		struct {
			const char* ptr;
			uint32_t size;
		} ref;
	};

	unsigned char type;
	union_type via;

	bool is_nil() { return type == type::NIL; }

	template <typename T>
	T as();

	template <typename T>
	void convert(T& v);

private:
	struct implicit_type;

public:
	implicit_type convert();
};

struct object::implicit_type {
	implicit_type(object o) : obj(o) { }
	~implicit_type() { }

	template <typename T>
	operator T() { return obj.as<T>(); }

private:
	object obj;
};

std::ostream& operator<< (std::ostream& s, const object o);

bool operator==(const object x, const object y);
inline bool operator!=(const object x, const object y) { return !(x == y); }

inline object& operator>> (object o, object& v)
{
	v = o;
	return v;
}

template <typename Stream>
packer<Stream>& operator<< (packer<Stream>& o, const object& v);

template <typename T>
inline void convert(T& v, object o)
{
	o >> v;
}

template <typename Stream, typename T>
inline void pack(packer<Stream>& o, const T& v)
{
	o << v;
}

template <typename Stream, typename T>
inline void pack(Stream& s, const T& v)
{
	packer<Stream> pk(s);
	pack(pk, v);
}

template <typename Stream, typename T>
inline void pack_copy(packer<Stream>& o, T v)
{
	pack(o, v);
}

template <typename T>
inline T& operator>> (object o, T& v)
{
	v.msgpack_unpack(o);
	return v;
}

template <typename Stream, typename T>
inline packer<Stream>& operator<< (packer<Stream>& o, const T& v)
{
	o << v.msgpack_pack();
	return o;
}

template <typename Type>
class define : public Type {
public:
	typedef Type msgpack_type;
	typedef define<Type> define_type;

	define() {}
	define(msgpack_type v) : msgpack_type(v) {}

	msgpack_type msgpack_pack() const
	{
		return *this;
	}

	void msgpack_unpack(object o)
	{
		convert(static_cast<msgpack_type&>(*this), o);
	}
};

inline object::implicit_type object::convert()
{
	return implicit_type(*this);
}

template <typename T>
inline T object::as()
{
	T v;
	msgpack::convert(v, *this);
	return v;
}

template <typename T>
void object::convert(T& v)
{
	msgpack::convert(v, *this);
}

template <typename Stream>
packer<Stream>& operator<< (packer<Stream>& o, const object& v)
{
	switch(v.type) {
	case type::NIL:
		o.pack_nil();
		return o;

	case type::BOOLEAN:
		if(v.via.boolean) {
			o.pack_true();
		} else {
			o.pack_false();
		}
		return o;

	case type::POSITIVE_INTEGER:
		if(v.via.u64 <= (uint64_t)std::numeric_limits<uint16_t>::max()) {
			if(v.via.u64 <= (uint16_t)std::numeric_limits<uint8_t>::max()) {
				o.pack_uint8(v.via.u64);
			} else {
				o.pack_uint16(v.via.u64);
			}
		} else {
			if(v.via.u64 <= (uint64_t)std::numeric_limits<uint32_t>::max()) {
				o.pack_uint32(v.via.u64);
			} else {
				o.pack_uint64(v.via.u64);
			}
		}
		return o;

	case type::NEGATIVE_INTEGER:
		if(v.via.i64 >= (int64_t)std::numeric_limits<int16_t>::min()) {
			if(v.via.i64 >= (int64_t)std::numeric_limits<int8_t>::min()) {
				o.pack_int8(v.via.i64);
			} else {
				o.pack_int16(v.via.i64);
			}
		} else {
			if(v.via.i64 >= (int64_t)std::numeric_limits<int32_t>::min()) {
				o.pack_int64(v.via.i64);
			} else {
				o.pack_int64(v.via.i64);
			}
		}
		return o;

	case type::RAW:
		o.pack_raw(v.via.ref.size);
		o.pack_raw_body(v.via.ref.ptr, v.via.ref.size);
		return o;

	case type::ARRAY:
		o.pack_array(v.via.container.size);
		for(object* p(v.via.container.ptr),
				* const pend(v.via.container.ptr + v.via.container.size);
				p < pend; ++p) {
			*p >> o;
		}
		return o;
		// FIXME loop optimiziation

	case type::MAP:
		o.pack_map(v.via.container.size);
		for(object* p(v.via.container.ptr),
				* const pend(v.via.container.ptr + v.via.container.size*2);
				p < pend; ) {
			*p >> o; ++p;
			*p >> o; ++p;
		}
		return o;
		// FIXME loop optimiziation

	default:
		throw type_error();
	}
}

}  // namespace msgpack

#include "msgpack/type.hpp"

#endif /* msgpack/object.hpp */

<|endoftext|>