<commit_before>/*
 * console_printer.cpp
 *
 *  Created on : Mar 26, 2016
 *      Author : Omar Makke (O jMakke)
 *      Email  : ojmakke@yahoo.com

This file is part of GNU Nets also known as GNUNets

GNU Nets is free software: you can redistribute it and/or modify
it under the terms of the Affero GNU General Public License as published by
the Free Software Foundation, version 3 of the License.

GNU Nets is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Affero
GNU General Public License for more details.

You should have received a copy of the Affero GNU General Public License
along with GNU Nets.  If not, see <http://www.gnu.org/licenses/>.
 */
#include <ncurses.h>
#include <string.h>
#include <stdlib.h>

#include "../common.h"
#include "console_printer.h"

#define CONTROL 0
#define INPUT 1

const char *feedback1[] = {
                            "F1 to quit anytime",
                            "F2 to rescale window",
                            "i to insert commands (INSERT mode)",
                            "ESC to quit INSERT mode",
                            "s to save network parameters",
                            "v for basic network parameters",
                            "h for more help",
                          };

ConsolePrinter::ConsolePrinter()
{
  ii_feedback = 0;
  ii_network = 0;
  ii_input = 0;
  init_screen();
}

ConsolePrinter& ConsolePrinter::instance()
{
  static ConsolePrinter instance;
  return instance;
}

void ConsolePrinter::print_feedback1()
{
  const int ROWOFFSET = 1;
  const int COL       = 2;
  for(size_t i = 0; i < ARRAY_SIZE(feedback1); i++)
    {
      mvwprintw(w_feedback, i+ROWOFFSET, COL, feedback1[i]);
    }
  wrefresh(w_feedback);
}

void ConsolePrinter::draw_input()
{
  const int COLOFFSET = 3;
  const int IOFFSET = 1;
  wattron(w_input, COLOR_PAIR(2));
  werase(w_input);
  box(w_input, 0, 0);
  mvwprintw(w_input, 0, COLOFFSET, "User Input");
  mvwprintw(w_input, YINPUT, IOFFSET, ">");
  keypad(w_input, true);
  noecho();
  wrefresh(w_input);
  clear_command();
  ii_input = 0;
}

void ConsolePrinter::draw_feedback()
{
  const int COLOFFSET = 3;
  wattron(w_feedback, COLOR_PAIR(1));
  werase(w_feedback);
  box(w_feedback, 0, 0);
  mvwprintw(w_feedback, 0, COLOFFSET, "Feedback");
  print_feedback1();
  wrefresh(w_feedback);
}

void ConsolePrinter::draw_network()
{
  const int COLOFFSET = 3;
  wattron(w_network, COLOR_PAIR(3));
  werase(w_network);
  box(w_network, 0, 0);
  mvwprintw(w_network, 0, COLOFFSET, "Network Info");
  wrefresh(w_network);
}

void ConsolePrinter::init_screen()
{
  int row, col;
  initscr();
  clear();
  cbreak();
  noecho();
  refresh();
  getmaxyx(stdscr,row,col);
  start_color();
  init_pair(1, COLOR_RED, COLOR_BLACK);
  init_pair(2, COLOR_GREEN, COLOR_BLACK);
  init_pair(3, COLOR_CYAN, COLOR_BLACK);

  w_input = create_newwin(4, col, row-4, 0);
  w_feedback = create_newwin(row-4, col/2-1,0,0);
  w_network = create_newwin(row-4, col/2+1, 0,col/2-1);
  draw_feedback();
  draw_network();
  draw_input();

}

void ConsolePrinter::destroy_screen()
{
  destroy_win(w_input);
  destroy_win(w_network);
  destroy_win(w_feedback);
  endwin();
}

WINDOW* ConsolePrinter::create_newwin(int height, int width, int starty, int startx)
{
  WINDOW *local_win;
  local_win = newwin(height, width, starty, startx);
  box(local_win, 0 , 0);
  wrefresh(local_win);
  return local_win;
}

void ConsolePrinter::destroy_win(WINDOW *local_win)
{
  wborder(local_win, ' ', ' ', ' ',' ',' ',' ',' ',' ');
  wrefresh(local_win);
  delwin(local_win);
}

void ConsolePrinter::rescale()
{
  destroy_screen();
  clear_command();
  init_screen();
}

void ConsolePrinter::reset_input_cursor()
{
  // move the cursor to input location
  mvwprintw(w_input, YINPUT,ii_input+XOFFSET, "");
  wrefresh(w_input);
}

// TODO consider what happens when screen is less than 80
// What happens if we attempt to write beyond visible limit?
void ConsolePrinter::input_write(char c)
{
  int row, col;
  getmaxyx(stdscr,row,col);
  // backspace
  if(c == KEY_BACKSPACE  || c == KEY_DC || c == 07 || c == 8)
    {
      if(ii_input == 0)
        {
          return; //Do nothing
        }
      commands[ii_input--] = 0; // Remove buffer char
      // To erase on screen, write space to clear pixels and then "" to clear space.
      // TODO: Find better method
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET, " ");
      wrefresh(w_input);
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET, "");
      wrefresh(w_input);
      return;
    }
   if(c == 10 || c == 13)
    {
      commands[ii_input] = 0; // Terminating null.
      ii_input = 0;
      // parse_command
      draw_input();
      reset_input_cursor();
    }
  // Escape
  else if(c == 27)
    {
     return;  // do nothing
    }
  // Attempts to protect against writing beyond visible screen.
  else if(ii_input >= (size_t) col - XOFFSET-2)
    {
      return;
    }
  // Ignore invalid chars
  else if( c < 33 || c > 126)
    {
      return;
    }
  // Cap input and replace last char in input
  else if(ii_input >= BUFFERSIZE - 2)
    {
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET - 1, "%c", c);
      reset_input_cursor();
      commands[BUFFERSIZE - 2] = c;
      wrefresh(w_input);
      return;
    }
  mvwprintw(w_input, YINPUT, ii_input + XOFFSET, "%c", c);
  commands[ii_input++] = c;
  return;
}

void ConsolePrinter::clear_command()
{
  memset(commands, 0, BUFFERSIZE);
}

void ConsolePrinter::interact()
{
  int ch;
  unsigned int state = CONTROL;
  bool completed = false;
  while(!completed)
    {
      ch = wgetch(w_input);
      switch(ch)
        {
        // Never allowed in input. This quits
        case KEY_F(1):
          {
            return;
            break;
          }
        // Reset everything
        case KEY_F(2):
          {
            rescale();
            state = CONTROL;
          }
          break;
         // In control mode, do nothing. Otherwise, insert \0
        case 10:
        case 13:
          {
            if(state == CONTROL)
              {
                continue;
              }
            // Parse input first
            input_write(ch);
          }
          break;
        // Escape resets everything
        case 27:
          {
            if(state == CONTROL)
              {
                continue;
              }
            switch_console_state(&state);
          }
          break;
        case 'i':
          {
            if(state == CONTROL)
              {
                switch_console_state(&state);
                break;
              }
            input_write(ch);
          }
          break;
        default:
          if(state == CONTROL)
            {
              continue;
            }
          input_write(ch);
          break;
        }
    }
  return;
}

void ConsolePrinter::switch_console_state(unsigned int *state)
{
  switch(*state)
    {
    case CONTROL:
      {
        *state = INPUT;
      }
      break;
    case INPUT:
      {
        *state = CONTROL;
      }
    break;
    }
}

ConsolePrinter::~ConsolePrinter()
{
  destroy_screen();
}

<commit_msg>Fixed return key going to new line issue<commit_after>/*
 * console_printer.cpp
 *
 *  Created on : Mar 26, 2016
 *      Author : Omar Makke (O jMakke)
 *      Email  : ojmakke@yahoo.com

This file is part of GNU Nets also known as GNUNets

GNU Nets is free software: you can redistribute it and/or modify
it under the terms of the Affero GNU General Public License as published by
the Free Software Foundation, version 3 of the License.

GNU Nets is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Affero
GNU General Public License for more details.

You should have received a copy of the Affero GNU General Public License
along with GNU Nets.  If not, see <http://www.gnu.org/licenses/>.
 */
#include <ncurses.h>
#include <string.h>
#include <stdlib.h>

#include "../common.h"
#include "console_printer.h"

#define CONTROL 0
#define INPUT 1

const char *feedback1[] = {
                            "F1 to quit anytime",
                            "F2 to rescale window",
                            "i to insert commands (INSERT mode)",
                            "ESC to quit INSERT mode",
                            "s to save network parameters",
                            "v for basic network parameters",
                            "h for more help",
                          };

ConsolePrinter::ConsolePrinter()
{
  ii_feedback = 0;
  ii_network = 0;
  ii_input = 0;
  init_screen();
}

ConsolePrinter& ConsolePrinter::instance()
{
  static ConsolePrinter instance;
  return instance;
}

void ConsolePrinter::print_feedback1()
{
  const int ROWOFFSET = 1;
  const int COL       = 2;
  for(size_t i = 0; i < ARRAY_SIZE(feedback1); i++)
    {
      mvwprintw(w_feedback, i+ROWOFFSET, COL, feedback1[i]);
    }
  wrefresh(w_feedback);
}

void ConsolePrinter::draw_input()
{
  const int COLOFFSET = 3;
  const int IOFFSET = 1;
  wattron(w_input, COLOR_PAIR(2));
  werase(w_input);
  box(w_input, 0, 0);
  mvwprintw(w_input, 0, COLOFFSET, "User Input");
  mvwprintw(w_input, YINPUT, IOFFSET, ">");
  keypad(w_input, true);
  noecho();
  wrefresh(w_input);
  clear_command();
  ii_input = 0;
}

void ConsolePrinter::draw_feedback()
{
  const int COLOFFSET = 3;
  wattron(w_feedback, COLOR_PAIR(1));
  werase(w_feedback);
  box(w_feedback, 0, 0);
  mvwprintw(w_feedback, 0, COLOFFSET, "Feedback");
  print_feedback1();
  wrefresh(w_feedback);
}

void ConsolePrinter::draw_network()
{
  const int COLOFFSET = 3;
  wattron(w_network, COLOR_PAIR(3));
  werase(w_network);
  box(w_network, 0, 0);
  mvwprintw(w_network, 0, COLOFFSET, "Network Info");
  wrefresh(w_network);
}

void ConsolePrinter::init_screen()
{
  int row, col;
  initscr();
  clear();
  cbreak();
  noecho();
  refresh();
  getmaxyx(stdscr,row,col);
  start_color();
  init_pair(1, COLOR_RED, COLOR_BLACK);
  init_pair(2, COLOR_GREEN, COLOR_BLACK);
  init_pair(3, COLOR_CYAN, COLOR_BLACK);

  w_input = create_newwin(4, col, row-4, 0);
  w_feedback = create_newwin(row-4, col/2-1,0,0);
  w_network = create_newwin(row-4, col/2+1, 0,col/2-1);
  draw_feedback();
  draw_network();
  draw_input();

}

void ConsolePrinter::destroy_screen()
{
  destroy_win(w_input);
  destroy_win(w_network);
  destroy_win(w_feedback);
  endwin();
}

WINDOW* ConsolePrinter::create_newwin(int height, int width, int starty, int startx)
{
  WINDOW *local_win;
  local_win = newwin(height, width, starty, startx);
  box(local_win, 0 , 0);
  wrefresh(local_win);
  return local_win;
}

void ConsolePrinter::destroy_win(WINDOW *local_win)
{
  wborder(local_win, ' ', ' ', ' ',' ',' ',' ',' ',' ');
  wrefresh(local_win);
  delwin(local_win);
}

void ConsolePrinter::rescale()
{
  destroy_screen();
  clear_command();
  init_screen();
}

void ConsolePrinter::reset_input_cursor()
{
  // move the cursor to input location
  mvwprintw(w_input, YINPUT,ii_input+XOFFSET, "");
  wrefresh(w_input);
}

// TODO consider what happens when screen is less than 80
// What happens if we attempt to write beyond visible limit?
void ConsolePrinter::input_write(char c)
{
  int row, col;
  getmaxyx(stdscr,row,col);
  // backspace
  if(c == KEY_BACKSPACE  || c == KEY_DC || c == 07 || c == 8)
    {
      if(ii_input == 0)
        {
          return; //Do nothing
        }
      commands[ii_input--] = 0; // Remove buffer char
      // To erase on screen, write space to clear pixels and then "" to clear space.
      // TODO: Find better method
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET, " ");
      wrefresh(w_input);
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET, "");
      wrefresh(w_input);
      return;
    }
   if(c == 10 || c == 13)
    {
      commands[ii_input] = 0; // Terminating null.
      ii_input = 0;
      // parse_command
      draw_input();
      reset_input_cursor();
      return;
    }
  // Escape
  else if(c == 27)
    {
     return;  // do nothing
    }
  // Attempts to protect against writing beyond visible screen.
  else if(ii_input >= (size_t) col - XOFFSET-2)
    {
      return;
    }
  // Ignore invalid chars
  else if( c < 33 || c > 126)
    {
      return;
    }
  // Cap input and replace last char in input
  else if(ii_input >= BUFFERSIZE - 2)
    {
      mvwprintw(w_input, YINPUT, ii_input + XOFFSET - 1, "%c", c);
      reset_input_cursor();
      commands[BUFFERSIZE - 2] = c;
      wrefresh(w_input);
      return;
    }
  mvwprintw(w_input, YINPUT, ii_input + XOFFSET, "%c", c);
  commands[ii_input++] = c;
  return;
}

void ConsolePrinter::clear_command()
{
  memset(commands, 0, BUFFERSIZE);
}

void ConsolePrinter::interact()
{
  int ch;
  unsigned int state = CONTROL;
  bool completed = false;
  while(!completed)
    {
      ch = wgetch(w_input);
      switch(ch)
        {
        // Never allowed in input. This quits
        case KEY_F(1):
          {
            return;
            break;
          }
        // Reset everything
        case KEY_F(2):
          {
            rescale();
            state = CONTROL;
          }
          break;
         // In control mode, do nothing. Otherwise, insert \0
        case 10:
        case 13:
          {
            if(state == CONTROL)
              {
                continue;
              }
            // Parse input first
            input_write(ch);
          }
          break;
        // Escape resets everything
        case 27:
          {
            if(state == CONTROL)
              {
                continue;
              }
            switch_console_state(&state);
          }
          break;
        case 'i':
          {
            if(state == CONTROL)
              {
                switch_console_state(&state);
                break;
              }
            input_write(ch);
          }
          break;
        default:
          if(state == CONTROL)
            {
              continue;
            }
          input_write(ch);
          break;
        }
    }
  return;
}

void ConsolePrinter::switch_console_state(unsigned int *state)
{
  switch(*state)
    {
    case CONTROL:
      {
        *state = INPUT;
      }
      break;
    case INPUT:
      {
        *state = CONTROL;
      }
    break;
    }
}

ConsolePrinter::~ConsolePrinter()
{
  destroy_screen();
}

<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This test performs a series false positive checks using a list of URLs
// against a known set of SafeBrowsing data.
//
// It uses a normal SafeBrowsing database to create a bloom filter where it
// looks up all the URLs in the url file. A URL that has a prefix found in the
// bloom filter and found in the database is considered a hit: a valid lookup
// that will result in a gethash request. A URL that has a prefix found in the
// bloom filter but not in the database is a miss: a false positive lookup that
// will result in an unnecessary gethash request.
//
// By varying the size of the bloom filter and using a constant set of
// SafeBrowsing data, we can check a known set of URLs against the filter and
// determine the false positive rate.
//
// False positive calculation usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.FalsePositives
//                      --filter-start=<integer>
//                      --filter-steps=<integer>
//                      --filter-verbose
//
//  --filter-start:   The filter multiplier to begin with. This represents the
//                    number of bits per prefix of memory to use in the filter.
//                    The default value is identical to the current SafeBrowsing
//                    database value.
//  --filter-steps:   The number of iterations to run, with each iteration
//                    increasing the filter multiplier by 1. The default value
//                    is 1.
//  --filter-verbose: Used to print out the hit / miss results per URL.
//  --filter-csv:     The URL file contains information about the number of
//                    unique views (the popularity) of each URL. See the format
//                    description below.
//
//
// Hash compute time usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.HashTime
//                      --filter-num-checks=<integer>
//
//  --filter-num-checks: The number of hash look ups to perform on the bloom
//                       filter. The default is 10 million.
//
// Data files:
//    chrome/test/data/safe_browsing/filter/database
//    chrome/test/data/safe_browsing/filter/urls
//
// database: A normal SafeBrowsing database.
// urls:     A text file containing a list of URLs, one per line. If the option
//           --filter-csv is specified, the format of each line in the file is
//           <url>,<weight> where weight is an integer indicating the number of
//           unique views for the URL.

#include <algorithm>
#include <fstream>
#include <vector>

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/path_service.h"
#include "base/rand_util.h"
#include "base/sha2.h"
#include "base/string_number_conversions.h"
#include "base/string_util.h"
#include "base/time.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_util.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/sqlite_compiled_statement.h"
#include "chrome/common/sqlite_utils.h"
#include "googleurl/src/gurl.h"
#include "testing/gtest/include/gtest/gtest.h"

using base::Time;
using base::TimeDelta;

namespace {

// Ensures the SafeBrowsing database is closed properly.
class ScopedPerfDatabase {
 public:
  explicit ScopedPerfDatabase(sqlite3* db) : db_(db) {}
  ~ScopedPerfDatabase() {
    sqlite3_close(db_);
  }

 private:
  sqlite3* db_;

  DISALLOW_COPY_AND_ASSIGN(ScopedPerfDatabase);
};

// Command line flags.
const char kFilterVerbose[] = "filter-verbose";
const char kFilterStart[] = "filter-start";
const char kFilterSteps[] = "filter-steps";
const char kFilterCsv[] = "filter-csv";
const char kFilterNumChecks[] = "filter-num-checks";

// Number of hash checks to make during performance testing.
static const int kNumHashChecks = 10000000;

// Returns the path to the data used in this test, relative to the top of the
// source directory.
FilePath GetFullDataPath() {
  FilePath full_path;
  CHECK(PathService::Get(chrome::DIR_TEST_DATA, &full_path));
  full_path = full_path.Append(FILE_PATH_LITERAL("safe_browsing"));
  full_path = full_path.Append(FILE_PATH_LITERAL("filter"));
  CHECK(file_util::PathExists(full_path));
  return full_path;
}

// Constructs a bloom filter of the appropriate size from the provided prefixes.
void BuildBloomFilter(int size_multiplier,
                      const std::vector<SBPrefix>& prefixes,
                      BloomFilter** bloom_filter) {
  // Create a BloomFilter with the specified size.
  const int key_count = std::max(static_cast<int>(prefixes.size()),
                                 BloomFilter::kBloomFilterMinSize);
  const int filter_size = key_count * size_multiplier;
  *bloom_filter = new BloomFilter(filter_size);

  // Add the prefixes to it.
  for (size_t i = 0; i < prefixes.size(); ++i)
    (*bloom_filter)->Insert(prefixes[i]);

  std::cout << "Bloom filter with prefixes: " << prefixes.size()
            << ", multiplier: " << size_multiplier
            << ", size (bytes): " << (*bloom_filter)->size()
            << std::endl;
}

// Reads the set of add prefixes contained in a SafeBrowsing database into a
// sorted array suitable for fast searching. This takes significantly less time
// to look up a given prefix than performing SQL queries.
bool ReadDatabase(const FilePath& path, std::vector<SBPrefix>* prefixes) {
  FilePath database_file = path.Append(FILE_PATH_LITERAL("database"));
  sqlite3* db = NULL;
  if (sqlite_utils::OpenSqliteDb(database_file, &db) != SQLITE_OK) {
    sqlite3_close(db);
    return false;
  }

  ScopedPerfDatabase database(db);
  scoped_ptr<SqliteStatementCache> sql_cache(new SqliteStatementCache(db));

  // Get the number of items in the add_prefix table.
  std::string sql = "SELECT COUNT(*) FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(count_statement, *sql_cache, sql.c_str());
  if (!count_statement.is_valid())
    return false;

  if (count_statement->step() != SQLITE_ROW)
    return false;

  const int count = count_statement->column_int(0);

  // Load them into a prefix vector and sort
  prefixes->reserve(count);
  sql = "SELECT prefix FROM add_prefix";
  SQLITE_UNIQUE_STATEMENT(prefix_statement, *sql_cache, sql.c_str());
  if (!prefix_statement.is_valid())
    return false;

  while (prefix_statement->step() == SQLITE_ROW)
    prefixes->push_back(prefix_statement->column_int(0));

  DCHECK(static_cast<int>(prefixes->size()) == count);
  sort(prefixes->begin(), prefixes->end());

  return true;
}

// Generates all legal SafeBrowsing prefixes for the specified URL, and returns
// the set of Prefixes that exist in the bloom filter. The function returns the
// number of host + path combinations checked.
int GeneratePrefixHits(const std::string url,
                       BloomFilter* bloom_filter,
                       std::vector<SBPrefix>* prefixes) {
  GURL url_check(url);
  std::vector<std::string> hosts;
  if (url_check.HostIsIPAddress()) {
    hosts.push_back(url_check.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url_check, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url_check, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBPrefix prefix;
      base::SHA256HashString(hosts[i] + paths[j], &prefix, sizeof(prefix));
      if (bloom_filter->Exists(prefix))
        prefixes->push_back(prefix);
    }
  }

  return hosts.size() * paths.size();
}

// Binary search of sorted prefixes.
bool IsPrefixInDatabase(SBPrefix prefix,
                        const std::vector<SBPrefix>& prefixes) {
  if (prefixes.empty())
    return false;

  int low = 0;
  int high = prefixes.size() - 1;
  while (low <= high) {
    int mid = ((unsigned int)low + (unsigned int)high) >> 1;
    SBPrefix prefix_mid = prefixes[mid];
    if (prefix_mid == prefix)
      return true;
    if (prefix_mid < prefix)
      low = mid + 1;
    else
      high = mid - 1;
  }

  return false;
}

// Construct a bloom filter with the given prefixes and multiplier, and test the
// false positive rate (misses) against a URL list.
void CalculateBloomFilterFalsePositives(
    int size_multiplier,
    const FilePath& data_dir,
    const std::vector<SBPrefix>& prefix_list) {
  BloomFilter* bloom_filter = NULL;
  BuildBloomFilter(size_multiplier, prefix_list, &bloom_filter);
  scoped_refptr<BloomFilter> scoped_filter(bloom_filter);

  // Read in data file line at a time.
  FilePath url_file = data_dir.Append(FILE_PATH_LITERAL("urls"));
  std::ifstream url_stream(url_file.value().c_str());

  // Keep track of stats
  int hits = 0;
  int misses = 0;
  int weighted_hits = 0;
  int weighted_misses = 0;
  int url_count = 0;
  int prefix_count = 0;

  // Print out volumes of data (per URL hit and miss information).
  bool verbose = CommandLine::ForCurrentProcess()->HasSwitch(kFilterVerbose);
  bool use_weights = CommandLine::ForCurrentProcess()->HasSwitch(kFilterCsv);

  std::string url;
  while (std::getline(url_stream, url)) {
    ++url_count;

    // Handle a format that contains URLs weighted by unique views.
    int weight = 1;
    if (use_weights) {
      std::string::size_type pos = url.find_last_of(",");
      if (pos != std::string::npos) {
        base::StringToInt(url.begin() + pos + 1, url.end(), &weight);
        url = url.substr(0, pos);
      }
    }

    // See if the URL is in the bloom filter.
    std::vector<SBPrefix> prefixes;
    prefix_count += GeneratePrefixHits(url, bloom_filter, &prefixes);

    // See if the prefix is actually in the database (in-memory prefix list).
    for (size_t i = 0; i < prefixes.size(); ++i) {
      if (IsPrefixInDatabase(prefixes[i], prefix_list)) {
        ++hits;
        weighted_hits += weight;
        if (verbose) {
          std::cout << "Hit for URL: " << url
                    << " (prefix = "   << prefixes[i] << ")"
                    << std::endl;
        }
      } else {
        ++misses;
        weighted_misses += weight;
        if (verbose) {
          std::cout << "Miss for URL: " << url
                    << " (prefix = "    << prefixes[i] << ")"
                    << std::endl;
        }
      }
    }
  }

  // Print out the results for this test.
  std::cout << "URLs checked: "      << url_count
            << ", prefix compares: " << prefix_count
            << ", hits: "            << hits
            << ", misses: "          << misses;
  if (use_weights) {
    std::cout << ", weighted hits: "   << weighted_hits
              << ", weighted misses: " << weighted_misses;
  }
  std::cout << std::endl;
}

}  // namespace

// This test can take several minutes to perform its calculations, so it should
// be disabled until you need to run it.
TEST(SafeBrowsingBloomFilter, FalsePositives) {
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();

  int start = BloomFilter::kBloomFilterSizeRatio;
  if (cmd_line.HasSwitch(kFilterStart)) {
    ASSERT_TRUE(base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterStart),
                                  &start));
  }

  int steps = 1;
  if (cmd_line.HasSwitch(kFilterSteps)) {
    ASSERT_TRUE(base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterSteps),
                                  &steps));
  }

  int stop = start + steps;

  for (int multiplier = start; multiplier < stop; ++multiplier)
    CalculateBloomFilterFalsePositives(multiplier, data_dir, prefix_list);
}

// Computes the time required for performing a number of look ups in a bloom
// filter. This is useful for measuring the performance of new hash functions.
TEST(SafeBrowsingBloomFilter, HashTime) {
  // Read the data from the database.
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();

  int num_checks = kNumHashChecks;
  if (cmd_line.HasSwitch(kFilterNumChecks)) {
    ASSERT_TRUE(
        base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterNumChecks),
                          &num_checks));
  }

  // Populate the bloom filter and measure the time.
  BloomFilter* bloom_filter = NULL;
  Time populate_before = Time::Now();
  BuildBloomFilter(BloomFilter::kBloomFilterSizeRatio,
                   prefix_list, &bloom_filter);
  TimeDelta populate = Time::Now() - populate_before;

  // Check a large number of random prefixes against the filter.
  int hits = 0;
  Time check_before = Time::Now();
  for (int i = 0; i < num_checks; ++i) {
    uint32 prefix = static_cast<uint32>(base::RandUint64());
    if (bloom_filter->Exists(prefix))
      ++hits;
  }
  TimeDelta check = Time::Now() - check_before;

  int64 time_per_insert = populate.InMicroseconds() /
                          static_cast<int>(prefix_list.size());
  int64 time_per_check = check.InMicroseconds() / num_checks;

  std::cout << "Time results for checks: " << num_checks
            << ", prefixes: "              << prefix_list.size()
            << ", populate time (ms): "    << populate.InMilliseconds()
            << ", check time (ms): "       << check.InMilliseconds()
            << ", hits: "                  << hits
            << ", per-populate (us): "     << time_per_insert
            << ", per-check (us): "        << time_per_check
            << std::endl;
}
<commit_msg>Cleanup: Remove sqlite_utils.h from filter_false_positive_perftest.cc.<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This test performs a series false positive checks using a list of URLs
// against a known set of SafeBrowsing data.
//
// It uses a normal SafeBrowsing database to create a bloom filter where it
// looks up all the URLs in the url file. A URL that has a prefix found in the
// bloom filter and found in the database is considered a hit: a valid lookup
// that will result in a gethash request. A URL that has a prefix found in the
// bloom filter but not in the database is a miss: a false positive lookup that
// will result in an unnecessary gethash request.
//
// By varying the size of the bloom filter and using a constant set of
// SafeBrowsing data, we can check a known set of URLs against the filter and
// determine the false positive rate.
//
// False positive calculation usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.FalsePositives
//                      --filter-start=<integer>
//                      --filter-steps=<integer>
//                      --filter-verbose
//
//  --filter-start:   The filter multiplier to begin with. This represents the
//                    number of bits per prefix of memory to use in the filter.
//                    The default value is identical to the current SafeBrowsing
//                    database value.
//  --filter-steps:   The number of iterations to run, with each iteration
//                    increasing the filter multiplier by 1. The default value
//                    is 1.
//  --filter-verbose: Used to print out the hit / miss results per URL.
//  --filter-csv:     The URL file contains information about the number of
//                    unique views (the popularity) of each URL. See the format
//                    description below.
//
//
// Hash compute time usage:
//   $ ./perf_tests.exe --gtest_filter=SafeBrowsingBloomFilter.HashTime
//                      --filter-num-checks=<integer>
//
//  --filter-num-checks: The number of hash look ups to perform on the bloom
//                       filter. The default is 10 million.
//
// Data files:
//    chrome/test/data/safe_browsing/filter/database
//    chrome/test/data/safe_browsing/filter/urls
//
// database: A normal SafeBrowsing database.
// urls:     A text file containing a list of URLs, one per line. If the option
//           --filter-csv is specified, the format of each line in the file is
//           <url>,<weight> where weight is an integer indicating the number of
//           unique views for the URL.

#include <algorithm>
#include <fstream>
#include <vector>

#include "app/sql/statement.h"
#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/path_service.h"
#include "base/rand_util.h"
#include "base/sha2.h"
#include "base/string_number_conversions.h"
#include "base/string_util.h"
#include "base/time.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_util.h"
#include "chrome/common/chrome_paths.h"
#include "googleurl/src/gurl.h"
#include "testing/gtest/include/gtest/gtest.h"

using base::Time;
using base::TimeDelta;

namespace {

// Command line flags.
const char kFilterVerbose[] = "filter-verbose";
const char kFilterStart[] = "filter-start";
const char kFilterSteps[] = "filter-steps";
const char kFilterCsv[] = "filter-csv";
const char kFilterNumChecks[] = "filter-num-checks";

// Number of hash checks to make during performance testing.
const int kNumHashChecks = 10000000;

// Returns the path to the data used in this test, relative to the top of the
// source directory.
FilePath GetFullDataPath() {
  FilePath full_path;
  CHECK(PathService::Get(chrome::DIR_TEST_DATA, &full_path));
  full_path = full_path.Append(FILE_PATH_LITERAL("safe_browsing"));
  full_path = full_path.Append(FILE_PATH_LITERAL("filter"));
  CHECK(file_util::PathExists(full_path));
  return full_path;
}

// Constructs a bloom filter of the appropriate size from the provided prefixes.
void BuildBloomFilter(int size_multiplier,
                      const std::vector<SBPrefix>& prefixes,
                      BloomFilter** bloom_filter) {
  // Create a BloomFilter with the specified size.
  const int key_count = std::max(static_cast<int>(prefixes.size()),
                                 BloomFilter::kBloomFilterMinSize);
  const int filter_size = key_count * size_multiplier;
  *bloom_filter = new BloomFilter(filter_size);

  // Add the prefixes to it.
  for (size_t i = 0; i < prefixes.size(); ++i)
    (*bloom_filter)->Insert(prefixes[i]);

  std::cout << "Bloom filter with prefixes: " << prefixes.size()
            << ", multiplier: " << size_multiplier
            << ", size (bytes): " << (*bloom_filter)->size()
            << std::endl;
}

// Reads the set of add prefixes contained in a SafeBrowsing database into a
// sorted array suitable for fast searching. This takes significantly less time
// to look up a given prefix than performing SQL queries.
bool ReadDatabase(const FilePath& path, std::vector<SBPrefix>* prefixes) {
  FilePath database_file = path.Append(FILE_PATH_LITERAL("database"));
  sql::Connection db;
  if (!db.Open(database_file))
    return false;

  // Get the number of items in the add_prefix table.
  const char* query = "SELECT COUNT(*) FROM add_prefix";
  sql::Statement count_statement(db.GetCachedStatement(SQL_FROM_HERE, query));
  if (!count_statement)
    return false;

  if (!count_statement.Step())
    return false;

  const int count = count_statement.ColumnInt(0);

  // Load them into a prefix vector and sort.
  prefixes->reserve(count);
  query = "SELECT prefix FROM add_prefix";
  sql::Statement prefix_statement(db.GetCachedStatement(SQL_FROM_HERE, query));
  if (!prefix_statement)
    return false;

  while (prefix_statement.Step())
    prefixes->push_back(prefix_statement.ColumnInt(0));

  DCHECK(static_cast<int>(prefixes->size()) == count);
  sort(prefixes->begin(), prefixes->end());

  return true;
}

// Generates all legal SafeBrowsing prefixes for the specified URL, and returns
// the set of Prefixes that exist in the bloom filter. The function returns the
// number of host + path combinations checked.
int GeneratePrefixHits(const std::string url,
                       BloomFilter* bloom_filter,
                       std::vector<SBPrefix>* prefixes) {
  GURL url_check(url);
  std::vector<std::string> hosts;
  if (url_check.HostIsIPAddress()) {
    hosts.push_back(url_check.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url_check, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url_check, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBPrefix prefix;
      base::SHA256HashString(hosts[i] + paths[j], &prefix, sizeof(prefix));
      if (bloom_filter->Exists(prefix))
        prefixes->push_back(prefix);
    }
  }

  return hosts.size() * paths.size();
}

// Binary search of sorted prefixes.
bool IsPrefixInDatabase(SBPrefix prefix,
                        const std::vector<SBPrefix>& prefixes) {
  if (prefixes.empty())
    return false;

  int low = 0;
  int high = prefixes.size() - 1;
  while (low <= high) {
    int mid = ((unsigned int)low + (unsigned int)high) >> 1;
    SBPrefix prefix_mid = prefixes[mid];
    if (prefix_mid == prefix)
      return true;
    if (prefix_mid < prefix)
      low = mid + 1;
    else
      high = mid - 1;
  }

  return false;
}

// Construct a bloom filter with the given prefixes and multiplier, and test the
// false positive rate (misses) against a URL list.
void CalculateBloomFilterFalsePositives(
    int size_multiplier,
    const FilePath& data_dir,
    const std::vector<SBPrefix>& prefix_list) {
  BloomFilter* bloom_filter = NULL;
  BuildBloomFilter(size_multiplier, prefix_list, &bloom_filter);
  scoped_refptr<BloomFilter> scoped_filter(bloom_filter);

  // Read in data file line at a time.
  FilePath url_file = data_dir.Append(FILE_PATH_LITERAL("urls"));
  std::ifstream url_stream(url_file.value().c_str());

  // Keep track of stats
  int hits = 0;
  int misses = 0;
  int weighted_hits = 0;
  int weighted_misses = 0;
  int url_count = 0;
  int prefix_count = 0;

  // Print out volumes of data (per URL hit and miss information).
  bool verbose = CommandLine::ForCurrentProcess()->HasSwitch(kFilterVerbose);
  bool use_weights = CommandLine::ForCurrentProcess()->HasSwitch(kFilterCsv);

  std::string url;
  while (std::getline(url_stream, url)) {
    ++url_count;

    // Handle a format that contains URLs weighted by unique views.
    int weight = 1;
    if (use_weights) {
      std::string::size_type pos = url.find_last_of(",");
      if (pos != std::string::npos) {
        base::StringToInt(url.begin() + pos + 1, url.end(), &weight);
        url = url.substr(0, pos);
      }
    }

    // See if the URL is in the bloom filter.
    std::vector<SBPrefix> prefixes;
    prefix_count += GeneratePrefixHits(url, bloom_filter, &prefixes);

    // See if the prefix is actually in the database (in-memory prefix list).
    for (size_t i = 0; i < prefixes.size(); ++i) {
      if (IsPrefixInDatabase(prefixes[i], prefix_list)) {
        ++hits;
        weighted_hits += weight;
        if (verbose) {
          std::cout << "Hit for URL: " << url
                    << " (prefix = "   << prefixes[i] << ")"
                    << std::endl;
        }
      } else {
        ++misses;
        weighted_misses += weight;
        if (verbose) {
          std::cout << "Miss for URL: " << url
                    << " (prefix = "    << prefixes[i] << ")"
                    << std::endl;
        }
      }
    }
  }

  // Print out the results for this test.
  std::cout << "URLs checked: "      << url_count
            << ", prefix compares: " << prefix_count
            << ", hits: "            << hits
            << ", misses: "          << misses;
  if (use_weights) {
    std::cout << ", weighted hits: "   << weighted_hits
              << ", weighted misses: " << weighted_misses;
  }
  std::cout << std::endl;
}

}  // namespace

// This test can take several minutes to perform its calculations, so it should
// be disabled until you need to run it.
TEST(SafeBrowsingBloomFilter, FalsePositives) {
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();

  int start = BloomFilter::kBloomFilterSizeRatio;
  if (cmd_line.HasSwitch(kFilterStart)) {
    ASSERT_TRUE(base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterStart),
                                  &start));
  }

  int steps = 1;
  if (cmd_line.HasSwitch(kFilterSteps)) {
    ASSERT_TRUE(base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterSteps),
                                  &steps));
  }

  int stop = start + steps;

  for (int multiplier = start; multiplier < stop; ++multiplier)
    CalculateBloomFilterFalsePositives(multiplier, data_dir, prefix_list);
}

// Computes the time required for performing a number of look ups in a bloom
// filter. This is useful for measuring the performance of new hash functions.
TEST(SafeBrowsingBloomFilter, HashTime) {
  // Read the data from the database.
  std::vector<SBPrefix> prefix_list;
  FilePath data_dir = GetFullDataPath();
  ASSERT_TRUE(ReadDatabase(data_dir, &prefix_list));

  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();

  int num_checks = kNumHashChecks;
  if (cmd_line.HasSwitch(kFilterNumChecks)) {
    ASSERT_TRUE(
        base::StringToInt(cmd_line.GetSwitchValueASCII(kFilterNumChecks),
                          &num_checks));
  }

  // Populate the bloom filter and measure the time.
  BloomFilter* bloom_filter = NULL;
  Time populate_before = Time::Now();
  BuildBloomFilter(BloomFilter::kBloomFilterSizeRatio,
                   prefix_list, &bloom_filter);
  TimeDelta populate = Time::Now() - populate_before;

  // Check a large number of random prefixes against the filter.
  int hits = 0;
  Time check_before = Time::Now();
  for (int i = 0; i < num_checks; ++i) {
    uint32 prefix = static_cast<uint32>(base::RandUint64());
    if (bloom_filter->Exists(prefix))
      ++hits;
  }
  TimeDelta check = Time::Now() - check_before;

  int64 time_per_insert = populate.InMicroseconds() /
                          static_cast<int>(prefix_list.size());
  int64 time_per_check = check.InMicroseconds() / num_checks;

  std::cout << "Time results for checks: " << num_checks
            << ", prefixes: "              << prefix_list.size()
            << ", populate time (ms): "    << populate.InMilliseconds()
            << ", check time (ms): "       << check.InMilliseconds()
            << ", hits: "                  << hits
            << ", per-populate (us): "     << time_per_insert
            << ", per-check (us): "        << time_per_check
            << std::endl;
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Modified by ScyllaDB
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "log.hh"
#include "streaming/stream_detail.hh"
#include "streaming/stream_transfer_task.hh"
#include "streaming/stream_session.hh"
#include "streaming/stream_manager.hh"
#include "streaming/stream_reason.hh"
#include "streaming/stream_mutation_fragments_cmd.hh"
#include "mutation_reader.hh"
#include "frozen_mutation.hh"
#include "mutation.hh"
#include "message/messaging_service.hh"
#include "range.hh"
#include "dht/i_partitioner.hh"
#include "service/priority_manager.hh"
#include <boost/range/irange.hpp>
#include "service/storage_service.hh"
#include <boost/icl/interval.hpp>
#include <boost/icl/interval_set.hpp>
#include "sstables/sstables.hh"
#include "database.hh"

namespace streaming {

extern logging::logger sslog;

stream_transfer_task::stream_transfer_task(shared_ptr<stream_session> session, UUID cf_id, dht::token_range_vector ranges, long total_size)
    : stream_task(session, cf_id)
    , _ranges(std::move(ranges))
    , _total_size(total_size) {
}

stream_transfer_task::~stream_transfer_task() = default;

dht::partition_range_vector to_partition_ranges(const dht::token_range_vector& ranges) {
    dht::partition_range_vector prs;
    prs.reserve(ranges.size());
    for (auto& range : ranges) {
        prs.push_back(dht::to_partition_range(range));
    }
    return prs;
}

struct send_info {
    database& db;
    utils::UUID plan_id;
    utils::UUID cf_id;
    netw::messaging_service::msg_addr id;
    uint32_t dst_cpu_id;
    stream_reason reason;
    size_t mutations_nr{0};
    semaphore mutations_done{0};
    bool error_logged = false;
    column_family& cf;
    dht::token_range_vector ranges;
    dht::partition_range_vector prs;
    flat_mutation_reader reader;
    send_info(database& db_, utils::UUID plan_id_, utils::UUID cf_id_,
              dht::token_range_vector ranges_, netw::messaging_service::msg_addr id_,
              uint32_t dst_cpu_id_, stream_reason reason_)
        : db(db_)
        , plan_id(plan_id_)
        , cf_id(cf_id_)
        , id(id_)
        , dst_cpu_id(dst_cpu_id_)
        , reason(reason_)
        , cf(db.find_column_family(cf_id))
        , ranges(std::move(ranges_))
        , prs(to_partition_ranges(ranges))
        , reader(cf.make_streaming_reader(cf.schema(), prs)) {
    }
    future<bool> has_relevant_range_on_this_shard() {
        return do_with(false, [this] (bool& found_relevant_range) {
            return do_for_each(ranges, [this, &found_relevant_range] (dht::token_range range) {
                if (!found_relevant_range) {
                    auto sharder = dht::selective_token_range_sharder(range, engine().cpu_id());
                    auto range_shard = sharder.next();
                    if (range_shard) {
                        found_relevant_range = true;
                    }
                }
            }).then([&found_relevant_range] {
                return found_relevant_range;
            });
        });
    }
    future<size_t> estimate_partitions() {
        return do_with(cf.get_sstables(), size_t(0), [this] (auto& sstables, size_t& partition_count) {
            return do_for_each(*sstables, [this, &partition_count] (auto& sst) {
                return do_for_each(ranges, [this, &sst, &partition_count] (auto& range) {
                    partition_count += sst->estimated_keys_for_range(range);
                });
            }).then([&partition_count] {
                return partition_count;
            });
        });
    }
};

future<stop_iteration> do_send_mutations(lw_shared_ptr<send_info> si, frozen_mutation fm, bool fragmented) {
    return get_local_stream_manager().mutation_send_limiter().wait().then([si, fragmented, fm = std::move(fm)] () mutable {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION to {}, cf_id={}", si->plan_id, si->id, si->cf_id);
        auto fm_size = fm.representation().size();
        // Do it in the background.
        (void)netw::get_local_messaging_service().send_stream_mutation(si->id, si->plan_id, std::move(fm), si->dst_cpu_id, fragmented, si->reason).then([si, fm_size] {
            sslog.debug("[Stream #{}] GOT STREAM_MUTATION Reply from {}", si->plan_id, si->id.addr);
            get_local_stream_manager().update_progress(si->plan_id, si->id.addr, progress_info::direction::OUT, fm_size);
            si->mutations_done.signal();
        }).handle_exception([si] (auto ep) {
            // There might be larger number of STREAM_MUTATION inflight.
            // Log one error per column_family per range
            if (!si->error_logged) {
                si->error_logged = true;
                sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION to {}: {}", si->plan_id, si->id, ep);
            }
            si->mutations_done.broken();
        }).finally([] {
            get_local_stream_manager().mutation_send_limiter().signal();
        });
        return stop_iteration::no;
    });
}

future<> send_mutations(lw_shared_ptr<send_info> si) {
    size_t fragment_size = default_frozen_fragment_size;
    // Mutations cannot be sent fragmented if the receiving side doesn't support that.
    if (!service::get_local_storage_service().cluster_supports_large_partitions()) {
        fragment_size = std::numeric_limits<size_t>::max();
    }
    return fragment_and_freeze(std::move(si->reader), [si] (auto fm, bool fragmented) {
        if (!si->db.column_family_exists(si->cf_id)) {
            return make_ready_future<stop_iteration>(stop_iteration::yes);
        }
        si->mutations_nr++;
        return do_send_mutations(si, std::move(fm), fragmented);
    }, fragment_size).then([si] {
        return si->mutations_done.wait(si->mutations_nr);
    });
}

future<> send_mutation_fragments(lw_shared_ptr<send_info> si) {
  return si->estimate_partitions().then([si] (size_t estimated_partitions) {
    sslog.info("[Stream #{}] Start sending ks={}, cf={}, estimated_partitions={}, with new rpc streaming", si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name(), estimated_partitions);
    return netw::get_local_messaging_service().make_sink_and_source_for_stream_mutation_fragments(si->reader.schema()->version(), si->plan_id, si->cf_id, estimated_partitions, si->reason, si->id).then([si] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd> sink, rpc::source<int32_t> source) mutable {
        auto got_error_from_peer = make_lw_shared<bool>(false);

        auto source_op = [source, got_error_from_peer, si] () mutable -> future<> {
            return repeat([source, got_error_from_peer, si] () mutable {
                return source().then([source, got_error_from_peer, si] (std::optional<std::tuple<int32_t>> status_opt) mutable {
                    if (status_opt) {
                        auto status = std::get<0>(*status_opt);
                        *got_error_from_peer = status == -1;
                        sslog.debug("Got status code from peer={}, plan_id={}, cf_id={}, status={}", si->id.addr, si->plan_id, si->cf_id, status);
                        // we've got an error from the other side, but we cannot just abandon rpc::source we
                        // need to continue reading until EOS since this will signal that no more work
                        // is left and rpc::source can be destroyed. The sender closes connection immediately
                        // after sending the status, so EOS should arrive shortly.
                        return stop_iteration::no;
                    } else {
                        return stop_iteration::yes;
                    }
                });
            });
        }();

        auto sink_op = [sink, si, got_error_from_peer] () mutable -> future<> {
            return do_with(std::move(sink), [si, got_error_from_peer] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd>& sink) {
                return repeat([&sink, si, got_error_from_peer] () mutable {
                    return si->reader(db::no_timeout).then([&sink, si, s = si->reader.schema(), got_error_from_peer] (mutation_fragment_opt mf) mutable {
                        if (mf && !(*got_error_from_peer)) {
                            frozen_mutation_fragment fmf = freeze(*s, *mf);
                            auto size = fmf.representation().size();
                            streaming::get_local_stream_manager().update_progress(si->plan_id, si->id.addr, streaming::progress_info::direction::OUT, size);
                            return sink(fmf, stream_mutation_fragments_cmd::mutation_fragment_data).then([] { return stop_iteration::no; });
                        } else {
                            return make_ready_future<stop_iteration>(stop_iteration::yes);
                        }
                    });
                }).then([&sink] () mutable {
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::end_of_stream);
                }).handle_exception([&sink] (std::exception_ptr ep) mutable {
                    // Notify the receiver the sender has failed
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::error).then([ep = std::move(ep)] () mutable {
                        return make_exception_future<>(std::move(ep));
                    });
                }).finally([&sink] () mutable {
                    return sink.close();
                });
            });
        }();

        return when_all_succeed(std::move(source_op), std::move(sink_op)).then([got_error_from_peer, si] {
            if (*got_error_from_peer) {
                throw std::runtime_error(format("Peer failed to process mutation_fragment peer={}, plan_id={}, cf_id={}", si->id.addr, si->plan_id, si->cf_id));
            }
        });
    });
  });
}

future<> stream_transfer_task::execute() {
    auto plan_id = session->plan_id();
    auto cf_id = this->cf_id;
    auto dst_cpu_id = session->dst_cpu_id;
    auto id = netw::messaging_service::msg_addr{session->peer, session->dst_cpu_id};
    sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}", plan_id, cf_id);
    sort_and_merge_ranges();
    bool streaming_with_rpc_stream = service::get_local_storage_service().cluster_supports_stream_with_rpc_stream();
    auto reason = session->get_reason();
    return session->get_db().invoke_on_all([plan_id, cf_id, id, dst_cpu_id, ranges=this->_ranges, streaming_with_rpc_stream, reason] (database& db) {
        auto si = make_lw_shared<send_info>(db, plan_id, cf_id, std::move(ranges), id, dst_cpu_id, reason);
        return si->has_relevant_range_on_this_shard().then([si, plan_id, cf_id, streaming_with_rpc_stream] (bool has_relevant_range_on_this_shard) {
            if (!has_relevant_range_on_this_shard) {
                sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}: ignore ranges on shard={}",
                        plan_id, cf_id, engine().cpu_id());
                return make_ready_future<>();
            }
            if (streaming_with_rpc_stream) {
                return send_mutation_fragments(std::move(si));
            } else {
                return send_mutations(std::move(si));
            }
        });
    }).then([this, plan_id, cf_id, id, streaming_with_rpc_stream] {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION_DONE to {}, cf_id={}", plan_id, id, cf_id);
        return session->ms().send_stream_mutation_done(id, plan_id, _ranges,
                cf_id, session->dst_cpu_id).handle_exception([plan_id, id, cf_id] (auto ep) {
            sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION_DONE to {}: {}", plan_id, id, ep);
            std::rethrow_exception(ep);
        });
    }).then([this, id, plan_id, cf_id] {
        sslog.debug("[Stream #{}] GOT STREAM_MUTATION_DONE Reply from {}", plan_id, id.addr);
    }).handle_exception([this, plan_id, id] (auto ep){
        sslog.warn("[Stream #{}] stream_transfer_task: Fail to send to {}: {}", plan_id, id, ep);
        std::rethrow_exception(ep);
    });
}

void stream_transfer_task::append_ranges(const dht::token_range_vector& ranges) {
    _ranges.insert(_ranges.end(), ranges.begin(), ranges.end());
}

void stream_transfer_task::sort_and_merge_ranges() {
    boost::icl::interval_set<dht::token> myset;
    dht::token_range_vector ranges;
    sslog.debug("cf_id = {}, before ranges = {}, size={}", cf_id, _ranges, _ranges.size());
    _ranges.swap(ranges);
    for (auto& range : ranges) {
        // TODO: We should convert range_to_interval and interval_to_range to
        // take nonwrapping_range ranges.
        myset += locator::token_metadata::range_to_interval(range);
    }
    ranges.clear();
    ranges.shrink_to_fit();
    for (auto& i : myset) {
        auto r = locator::token_metadata::interval_to_range(i);
        _ranges.push_back(dht::token_range(r));
    }
    sslog.debug("cf_id = {}, after  ranges = {}, size={}", cf_id, _ranges, _ranges.size());
}

} // namespace streaming
<commit_msg>streaming:  Do not open rpc stream connection if reader has no data <commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Modified by ScyllaDB
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "log.hh"
#include "streaming/stream_detail.hh"
#include "streaming/stream_transfer_task.hh"
#include "streaming/stream_session.hh"
#include "streaming/stream_manager.hh"
#include "streaming/stream_reason.hh"
#include "streaming/stream_mutation_fragments_cmd.hh"
#include "mutation_reader.hh"
#include "frozen_mutation.hh"
#include "mutation.hh"
#include "message/messaging_service.hh"
#include "range.hh"
#include "dht/i_partitioner.hh"
#include "service/priority_manager.hh"
#include <boost/range/irange.hpp>
#include "service/storage_service.hh"
#include <boost/icl/interval.hpp>
#include <boost/icl/interval_set.hpp>
#include "sstables/sstables.hh"
#include "database.hh"

namespace streaming {

extern logging::logger sslog;

stream_transfer_task::stream_transfer_task(shared_ptr<stream_session> session, UUID cf_id, dht::token_range_vector ranges, long total_size)
    : stream_task(session, cf_id)
    , _ranges(std::move(ranges))
    , _total_size(total_size) {
}

stream_transfer_task::~stream_transfer_task() = default;

dht::partition_range_vector to_partition_ranges(const dht::token_range_vector& ranges) {
    dht::partition_range_vector prs;
    prs.reserve(ranges.size());
    for (auto& range : ranges) {
        prs.push_back(dht::to_partition_range(range));
    }
    return prs;
}

struct send_info {
    database& db;
    utils::UUID plan_id;
    utils::UUID cf_id;
    netw::messaging_service::msg_addr id;
    uint32_t dst_cpu_id;
    stream_reason reason;
    size_t mutations_nr{0};
    semaphore mutations_done{0};
    bool error_logged = false;
    column_family& cf;
    dht::token_range_vector ranges;
    dht::partition_range_vector prs;
    flat_mutation_reader reader;
    send_info(database& db_, utils::UUID plan_id_, utils::UUID cf_id_,
              dht::token_range_vector ranges_, netw::messaging_service::msg_addr id_,
              uint32_t dst_cpu_id_, stream_reason reason_)
        : db(db_)
        , plan_id(plan_id_)
        , cf_id(cf_id_)
        , id(id_)
        , dst_cpu_id(dst_cpu_id_)
        , reason(reason_)
        , cf(db.find_column_family(cf_id))
        , ranges(std::move(ranges_))
        , prs(to_partition_ranges(ranges))
        , reader(cf.make_streaming_reader(cf.schema(), prs)) {
    }
    future<bool> has_relevant_range_on_this_shard() {
        return do_with(false, [this] (bool& found_relevant_range) {
            return do_for_each(ranges, [this, &found_relevant_range] (dht::token_range range) {
                if (!found_relevant_range) {
                    auto sharder = dht::selective_token_range_sharder(range, engine().cpu_id());
                    auto range_shard = sharder.next();
                    if (range_shard) {
                        found_relevant_range = true;
                    }
                }
            }).then([&found_relevant_range] {
                return found_relevant_range;
            });
        });
    }
    future<size_t> estimate_partitions() {
        return do_with(cf.get_sstables(), size_t(0), [this] (auto& sstables, size_t& partition_count) {
            return do_for_each(*sstables, [this, &partition_count] (auto& sst) {
                return do_for_each(ranges, [this, &sst, &partition_count] (auto& range) {
                    partition_count += sst->estimated_keys_for_range(range);
                });
            }).then([&partition_count] {
                return partition_count;
            });
        });
    }
};

future<stop_iteration> do_send_mutations(lw_shared_ptr<send_info> si, frozen_mutation fm, bool fragmented) {
    return get_local_stream_manager().mutation_send_limiter().wait().then([si, fragmented, fm = std::move(fm)] () mutable {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION to {}, cf_id={}", si->plan_id, si->id, si->cf_id);
        auto fm_size = fm.representation().size();
        // Do it in the background.
        (void)netw::get_local_messaging_service().send_stream_mutation(si->id, si->plan_id, std::move(fm), si->dst_cpu_id, fragmented, si->reason).then([si, fm_size] {
            sslog.debug("[Stream #{}] GOT STREAM_MUTATION Reply from {}", si->plan_id, si->id.addr);
            get_local_stream_manager().update_progress(si->plan_id, si->id.addr, progress_info::direction::OUT, fm_size);
            si->mutations_done.signal();
        }).handle_exception([si] (auto ep) {
            // There might be larger number of STREAM_MUTATION inflight.
            // Log one error per column_family per range
            if (!si->error_logged) {
                si->error_logged = true;
                sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION to {}: {}", si->plan_id, si->id, ep);
            }
            si->mutations_done.broken();
        }).finally([] {
            get_local_stream_manager().mutation_send_limiter().signal();
        });
        return stop_iteration::no;
    });
}

future<> send_mutations(lw_shared_ptr<send_info> si) {
    size_t fragment_size = default_frozen_fragment_size;
    // Mutations cannot be sent fragmented if the receiving side doesn't support that.
    if (!service::get_local_storage_service().cluster_supports_large_partitions()) {
        fragment_size = std::numeric_limits<size_t>::max();
    }
    return fragment_and_freeze(std::move(si->reader), [si] (auto fm, bool fragmented) {
        if (!si->db.column_family_exists(si->cf_id)) {
            return make_ready_future<stop_iteration>(stop_iteration::yes);
        }
        si->mutations_nr++;
        return do_send_mutations(si, std::move(fm), fragmented);
    }, fragment_size).then([si] {
        return si->mutations_done.wait(si->mutations_nr);
    });
}

future<> send_mutation_fragments(lw_shared_ptr<send_info> si) {
 return si->reader.peek(db::no_timeout).then([si] (mutation_fragment* mfp) {
  if (!mfp) {
    // The reader contains no data
    sslog.info("[Stream #{}] Skip sending ks={}, cf={}, reader contains no data, with new rpc streaming",
        si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name());
    return make_ready_future<>();
  }
  return si->estimate_partitions().then([si] (size_t estimated_partitions) {
    sslog.info("[Stream #{}] Start sending ks={}, cf={}, estimated_partitions={}, with new rpc streaming", si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name(), estimated_partitions);
    return netw::get_local_messaging_service().make_sink_and_source_for_stream_mutation_fragments(si->reader.schema()->version(), si->plan_id, si->cf_id, estimated_partitions, si->reason, si->id).then([si] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd> sink, rpc::source<int32_t> source) mutable {
        auto got_error_from_peer = make_lw_shared<bool>(false);

        auto source_op = [source, got_error_from_peer, si] () mutable -> future<> {
            return repeat([source, got_error_from_peer, si] () mutable {
                return source().then([source, got_error_from_peer, si] (std::optional<std::tuple<int32_t>> status_opt) mutable {
                    if (status_opt) {
                        auto status = std::get<0>(*status_opt);
                        *got_error_from_peer = status == -1;
                        sslog.debug("Got status code from peer={}, plan_id={}, cf_id={}, status={}", si->id.addr, si->plan_id, si->cf_id, status);
                        // we've got an error from the other side, but we cannot just abandon rpc::source we
                        // need to continue reading until EOS since this will signal that no more work
                        // is left and rpc::source can be destroyed. The sender closes connection immediately
                        // after sending the status, so EOS should arrive shortly.
                        return stop_iteration::no;
                    } else {
                        return stop_iteration::yes;
                    }
                });
            });
        }();

        auto sink_op = [sink, si, got_error_from_peer] () mutable -> future<> {
            return do_with(std::move(sink), [si, got_error_from_peer] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd>& sink) {
                return repeat([&sink, si, got_error_from_peer] () mutable {
                    return si->reader(db::no_timeout).then([&sink, si, s = si->reader.schema(), got_error_from_peer] (mutation_fragment_opt mf) mutable {
                        if (mf && !(*got_error_from_peer)) {
                            frozen_mutation_fragment fmf = freeze(*s, *mf);
                            auto size = fmf.representation().size();
                            streaming::get_local_stream_manager().update_progress(si->plan_id, si->id.addr, streaming::progress_info::direction::OUT, size);
                            return sink(fmf, stream_mutation_fragments_cmd::mutation_fragment_data).then([] { return stop_iteration::no; });
                        } else {
                            return make_ready_future<stop_iteration>(stop_iteration::yes);
                        }
                    });
                }).then([&sink] () mutable {
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::end_of_stream);
                }).handle_exception([&sink] (std::exception_ptr ep) mutable {
                    // Notify the receiver the sender has failed
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::error).then([ep = std::move(ep)] () mutable {
                        return make_exception_future<>(std::move(ep));
                    });
                }).finally([&sink] () mutable {
                    return sink.close();
                });
            });
        }();

        return when_all_succeed(std::move(source_op), std::move(sink_op)).then([got_error_from_peer, si] {
            if (*got_error_from_peer) {
                throw std::runtime_error(format("Peer failed to process mutation_fragment peer={}, plan_id={}, cf_id={}", si->id.addr, si->plan_id, si->cf_id));
            }
        });
    });
  });
 });
}

future<> stream_transfer_task::execute() {
    auto plan_id = session->plan_id();
    auto cf_id = this->cf_id;
    auto dst_cpu_id = session->dst_cpu_id;
    auto id = netw::messaging_service::msg_addr{session->peer, session->dst_cpu_id};
    sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}", plan_id, cf_id);
    sort_and_merge_ranges();
    bool streaming_with_rpc_stream = service::get_local_storage_service().cluster_supports_stream_with_rpc_stream();
    auto reason = session->get_reason();
    return session->get_db().invoke_on_all([plan_id, cf_id, id, dst_cpu_id, ranges=this->_ranges, streaming_with_rpc_stream, reason] (database& db) {
        auto si = make_lw_shared<send_info>(db, plan_id, cf_id, std::move(ranges), id, dst_cpu_id, reason);
        return si->has_relevant_range_on_this_shard().then([si, plan_id, cf_id, streaming_with_rpc_stream] (bool has_relevant_range_on_this_shard) {
            if (!has_relevant_range_on_this_shard) {
                sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}: ignore ranges on shard={}",
                        plan_id, cf_id, engine().cpu_id());
                return make_ready_future<>();
            }
            if (streaming_with_rpc_stream) {
                return send_mutation_fragments(std::move(si));
            } else {
                return send_mutations(std::move(si));
            }
        });
    }).then([this, plan_id, cf_id, id, streaming_with_rpc_stream] {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION_DONE to {}, cf_id={}", plan_id, id, cf_id);
        return session->ms().send_stream_mutation_done(id, plan_id, _ranges,
                cf_id, session->dst_cpu_id).handle_exception([plan_id, id, cf_id] (auto ep) {
            sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION_DONE to {}: {}", plan_id, id, ep);
            std::rethrow_exception(ep);
        });
    }).then([this, id, plan_id, cf_id] {
        sslog.debug("[Stream #{}] GOT STREAM_MUTATION_DONE Reply from {}", plan_id, id.addr);
    }).handle_exception([this, plan_id, id] (auto ep){
        sslog.warn("[Stream #{}] stream_transfer_task: Fail to send to {}: {}", plan_id, id, ep);
        std::rethrow_exception(ep);
    });
}

void stream_transfer_task::append_ranges(const dht::token_range_vector& ranges) {
    _ranges.insert(_ranges.end(), ranges.begin(), ranges.end());
}

void stream_transfer_task::sort_and_merge_ranges() {
    boost::icl::interval_set<dht::token> myset;
    dht::token_range_vector ranges;
    sslog.debug("cf_id = {}, before ranges = {}, size={}", cf_id, _ranges, _ranges.size());
    _ranges.swap(ranges);
    for (auto& range : ranges) {
        // TODO: We should convert range_to_interval and interval_to_range to
        // take nonwrapping_range ranges.
        myset += locator::token_metadata::range_to_interval(range);
    }
    ranges.clear();
    ranges.shrink_to_fit();
    for (auto& i : myset) {
        auto r = locator::token_metadata::interval_to_range(i);
        _ranges.push_back(dht::token_range(r));
    }
    sslog.debug("cf_id = {}, after  ranges = {}, size={}", cf_id, _ranges, _ranges.size());
}

} // namespace streaming
<|endoftext|>


<commit_before>#include <cassert>
#include <cstring>
#include <iostream>

#include "simdjson/portability.h"
#include "simdjson/common_defs.h"
#include "jsoncharutils.h"

using namespace simdjson;

WARN_UNUSED
really_inline bool is_valid_true_atom(const uint8_t *loc) {
  uint64_t tv = *reinterpret_cast<const uint64_t *>("true    ");
  uint64_t mask4 = 0x00000000ffffffff;
  uint32_t error = 0;
  uint64_t
      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
  // this can read up to 7 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc, sizeof(uint64_t));
  error = (locval & mask4) ^ tv;
  error |= is_not_structural_or_whitespace(loc[4]);
  return error == 0;
}

WARN_UNUSED
really_inline bool is_valid_false_atom(const uint8_t *loc) {
  // We have to use an integer constant because the space in the cast
  // below would lead to values illegally being qualified
  // uint64_t fv = *reinterpret_cast<const uint64_t *>("false   ");
  // using this constant (that is the same false) but nulls out the
  // unused bits solves that
  uint64_t fv = 0x00000065736c6166; // takes into account endianness
  uint64_t mask5 = 0x000000ffffffffff;
  // we can't use the 32 bit value for checking for errors otherwise
  // the last character of false (it being 5 byte long!) would be
  // ignored
  uint64_t error = 0;
  uint64_t
      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
  // this can read up to 7 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc, sizeof(uint64_t));
  error = (locval & mask5) ^ fv;
  error |= is_not_structural_or_whitespace(loc[5]);
  return error == 0;
}

WARN_UNUSED
really_inline bool is_valid_null_atom(const uint8_t *loc) {
  uint64_t nv = *reinterpret_cast<const uint64_t *>("null    ");
  uint64_t mask4 = 0x00000000ffffffff;
  uint32_t error = 0;
  uint64_t
      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
  // this can read up to 7 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc, sizeof(uint64_t));
  error = (locval & mask4) ^ nv;
  error |= is_not_structural_or_whitespace(loc[4]);
  return error == 0;
}

#ifdef JSON_TEST_STRINGS
void found_string(const uint8_t *buf, const uint8_t *parsed_begin,
                  const uint8_t *parsed_end);
void found_bad_string(const uint8_t *buf);
#endif

#include "arm64/stage2_build_tape.h"
#include "haswell/stage2_build_tape.h"
#include "westmere/stage2_build_tape.h"
<commit_msg>This is implementing @jkeiser optimization idea. (#431)<commit_after>#include <cassert>
#include <cstring>
#include <iostream>

#include "simdjson/portability.h"
#include "simdjson/common_defs.h"
#include "jsoncharutils.h"

using namespace simdjson;

WARN_UNUSED
really_inline bool is_valid_true_atom(const uint8_t *loc) {
  uint32_t tv = *reinterpret_cast<const uint32_t *>("true");
  uint32_t error = 0;
  uint32_t
      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
  // this can read up to 3 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc, sizeof(uint32_t));
  error = locval ^ tv;
  error |= is_not_structural_or_whitespace(loc[4]);
  return error == 0;
}

WARN_UNUSED
really_inline bool is_valid_false_atom(const uint8_t *loc) {
  // assume that loc starts with "f"
  uint32_t fv = *reinterpret_cast<const uint32_t *>("alse");
  uint32_t error = 0;
  uint32_t
      locval; // we want to avoid unaligned 32-bit loads (undefined in C/C++)
  // this can read up to 4 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc + 1, sizeof(uint32_t));
  error = locval ^ fv;
  error |= is_not_structural_or_whitespace(loc[5]);
  return error == 0;
}

WARN_UNUSED
really_inline bool is_valid_null_atom(const uint8_t *loc) {
  uint32_t nv = *reinterpret_cast<const uint32_t *>("null");
  uint32_t error = 0;
  uint32_t
      locval; // we want to avoid unaligned 32-bit loads (undefined in C/C++)
  // this can read up to 2 bytes beyond the buffer size, but we require
  // SIMDJSON_PADDING of padding
  static_assert(sizeof(uint32_t) - 1 <= SIMDJSON_PADDING);
  std::memcpy(&locval, loc, sizeof(uint32_t));
  error = locval ^ nv;
  error |= is_not_structural_or_whitespace(loc[4]);
  return error == 0;
}

#ifdef JSON_TEST_STRINGS
void found_string(const uint8_t *buf, const uint8_t *parsed_begin,
                  const uint8_t *parsed_end);
void found_bad_string(const uint8_t *buf);
#endif

#include "arm64/stage2_build_tape.h"
#include "haswell/stage2_build_tape.h"
#include "westmere/stage2_build_tape.h"
<|endoftext|>


<commit_before>// @(#)root/net:$Name:  $:$Id: TMessage.cxx,v 1.4 2002/03/19 10:57:25 brun Exp $
// Author: Fons Rademakers   19/12/96

/*************************************************************************
 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TMessage                                                             //
//                                                                      //
// Message buffer class used for serializing objects and sending them   //
// over a network. This class inherits from TBuffer the basic I/O       //
// serializer.                                                          //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TMessage.h"
#include "Bytes.h"
#include "TFile.h"

extern "C" void R__zip (Int_t cxlevel, Int_t *nin, char *bufin, Int_t *lout, char *bufout, Int_t *nout);
extern "C" void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout);
const Int_t kMAXBUF = 0xffffff;

ClassImp(TMessage)

//______________________________________________________________________________
TMessage::TMessage(UInt_t what) : TBuffer(kWrite)
{
   // Create a TMessage object for storing objects. The "what" integer
   // describes the type of message. Predifined ROOT system message types
   // can be found in MessageTypes.h. Make sure your own message types are
   // unique from the ROOT defined message types (i.e. 0 - 10000 are
   // reserved by ROOT). In case you OR "what" with kMESS_ACK, the message
   // will wait for an acknowledgement from the remote side. This makes
   // the sending process synchronous. In case you OR "what" with kMESS_ZIP,
   // the message will be compressed in TSocket using the zip algorithm
   // (only if message is > 256 bytes).

   // space at the beginning of the message reserved for the message length
   UInt_t   reserved = 0;
   *this << reserved;

   fWhat  = what;
   *this << what;

   fClass      = 0;
   fCompress   = 0;
   fBufComp    = 0;
   fBufCompCur = 0;
   fCompPos    = 0;
}

//______________________________________________________________________________
TMessage::TMessage(void *buf, Int_t bufsize) : TBuffer(kRead, bufsize, buf)
{
   // Create a TMessage object for reading objects. The objects will be
   // read from buf. Use the What() method to get the message type.

   // skip space at the beginning of the message reserved for the message length
   fBufCur += sizeof(UInt_t);

   *this >> fWhat;

   fCompress   = 0;
   fBufComp    = 0;
   fBufCompCur = 0;
   fCompPos    = 0;

   if (fWhat & kMESS_ZIP) {
      // if buffer has kMESS_ZIP set, move it to fBufComp and uncompress
      fBufComp    = fBuffer;
      fBufCompCur = fBuffer + bufsize;
      fBuffer     = 0;
      Uncompress();
   }

   if (fWhat == kMESS_OBJECT) {
      InitMap();
      fClass = ReadClass();     // get first the class stored in message
      SetBufferOffset(sizeof(UInt_t) + sizeof(fWhat));
      ResetMap();
   } else {
      fClass = 0;
   }
}

//______________________________________________________________________________
TMessage::~TMessage()
{
   // Clean up compression buffer.

   delete [] fBufComp;
}

//______________________________________________________________________________
void TMessage::Forward()
{
   // Change a buffer that was received into one that can be send, i.e.
   // forward a just received message.

   if (IsReading()) {
      SetWriteMode();
      SetBufferOffset(fBufSize);

      if (fBufComp) {
         fCompPos = fBufCur;
      }
   }
}

//______________________________________________________________________________
void TMessage::Reset()
{
   // Reset the message buffer so we can use (i.e. fill) it again.

   SetBufferOffset(sizeof(UInt_t) + sizeof(fWhat));
   ResetMap();

   if (fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }
}

//______________________________________________________________________________
void TMessage::SetLength() const
{
   // Set the message length at the beginning of the message buffer.
   // This method is only called by TSocket::Send().

   if (IsWriting()) {
      char *buf = Buffer();
      tobuf(buf, (UInt_t)(Length() - sizeof(UInt_t)));

      if (fBufComp) {
         buf = fBufComp;
         tobuf(buf, (UInt_t)(CompLength() - sizeof(UInt_t)));
      }
   }
}

//______________________________________________________________________________
void TMessage::SetWhat(UInt_t what)
{
   // Using this method one can change the message type a-posteriory.
   // In case you OR "what" with kMESS_ACK, the message will wait for
   // an acknowledgement from the remote side. This makes the sending
   // process synchronous.

   fWhat = what;

   char *buf = Buffer();
   buf += sizeof(UInt_t);   // skip reserved length space
   tobuf(buf, what);

   if (fBufComp) {
      buf = fBufComp;
      buf += sizeof(UInt_t);   // skip reserved length space
      tobuf(buf, what | kMESS_ZIP);
   }
}

//______________________________________________________________________________
void TMessage::SetCompressionLevel(Int_t level)
{
   // Set the message compression level. Can be between 0 and 9 with 0
   // being no compression and 9 maximum compression. In general the default
   // level of 1 is the best compromise between achieved compression and
   // cpu time. Compression will only happen when the message is > 256 bytes.

   if (level < 0) level = 0;
   if (level > 9) level = 9;

   if (level != fCompress && fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }
   fCompress = level;
}

//______________________________________________________________________________
Int_t TMessage::Compress()
{
   // Compress the message. The message will only be compressed if the
   // compression level > 0 and the if the message is > 256 bytes.
   // Returns -1 in case of error (when compression fails or
   // when the message increases in size in some pathological cases),
   // otherwise returns 0.

   if (fCompress == 0) {
      // no compression specified
      if (fBufComp) {
         delete [] fBufComp;
         fBufComp    = 0;
         fBufCompCur = 0;
         fCompPos    = 0;
      }
      return 0;
   }

   if (fBufComp && fCompPos == fBufCur) {
      // the message was already compressed
      return 0;
   }

   // remove any existing compressed buffer before compressing modified message
   if (fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }

   if (Length() <= 256 + 2*sizeof(UInt_t)) {
      // this message is too small to be compressed
      return 0;
   }

   Int_t hdrlen   = 2*sizeof(UInt_t);
   Int_t messlen  = Length() - hdrlen;
   Int_t nbuffers = messlen / kMAXBUF;
   Int_t chdrlen  = 3*sizeof(UInt_t);   // compressed buffer header length
   Int_t buflen   = TMath::Max(512, chdrlen + messlen + 9*nbuffers);
   fBufComp       = new char[buflen];
   char *messbuf  = Buffer() + hdrlen;
   char *bufcur   = fBufComp + chdrlen;
   Int_t noutot   = 0;
   Int_t nzip     = 0;
   Int_t nout, bufmax;
   for (Int_t i = 0; i <= nbuffers; i++) {
      if (i == nbuffers)
         bufmax = messlen - nzip;
      else
         bufmax = kMAXBUF;
      R__zip(fCompress, &bufmax, messbuf, &bufmax, bufcur, &nout);
      if (nout == 0 || nout >= messlen) {
         //this happens when the buffer cannot be compressed
         delete [] fBufComp;
         fBufComp    = 0;
         fBufCompCur = 0;
         fCompPos    = 0;
         return -1;
      }
      bufcur  += nout;
      noutot  += nout;
      messbuf += kMAXBUF;
      nzip    += kMAXBUF;
   }
   fBufCompCur = bufcur;
   fCompPos    = fBufCur;

   bufcur = fBufComp;
   tobuf(bufcur, (UInt_t)(CompLength() - sizeof(UInt_t)));
   Int_t what = fWhat | kMESS_ZIP;
   tobuf(bufcur, what);
   tobuf(bufcur, Length());    // original uncompressed buffer length

   return 0;
}

//______________________________________________________________________________
Int_t TMessage::Uncompress()
{
   // Uncompress the message. The message will only be uncompressed when
   // kMESS_ZIP is set. Returns -1 in case of error, 0 otherwise.

   if (!fBufComp || !(fWhat & kMESS_ZIP))
      return -1;

   Int_t buflen;
   Int_t hdrlen = 2*sizeof(UInt_t);
   UChar_t *bufcur = (UChar_t*)fBufComp + hdrlen;
   frombuf((char *&)bufcur, &buflen);
   fBuffer  = new char[buflen];
   fBufSize = buflen;
   fBufCur  = fBuffer + sizeof(UInt_t) + sizeof(fWhat);
   fBufMax  = fBuffer + fBufSize;
   char *messbuf = fBuffer + hdrlen;

   Int_t nin, nout, nbuf;
   Int_t noutot = 0;
   while (1) {
      nin  = 9 + ((Int_t)bufcur[3] | ((Int_t)bufcur[4] << 8) | ((Int_t)bufcur[5] << 16));
      nbuf = (Int_t)bufcur[6] | ((Int_t)bufcur[7] << 8) | ((Int_t)bufcur[8] << 16);
      R__unzip(&nin, bufcur, &nbuf, messbuf, &nout);
      if (!nout) break;
      noutot += nout;
      if (noutot >= buflen - hdrlen) break;
      bufcur  += nin;
      messbuf += nout;
   }

   fWhat &= ~kMESS_ZIP;
   fCompress = 1;

   return 0;
}

<commit_msg>Remove a compliation warning (comparing integer with unsigned integer)<commit_after>// @(#)root/net:$Name:  $:$Id: TMessage.cxx,v 1.5 2004/05/05 14:43:34 rdm Exp $
// Author: Fons Rademakers   19/12/96

/*************************************************************************
 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TMessage                                                             //
//                                                                      //
// Message buffer class used for serializing objects and sending them   //
// over a network. This class inherits from TBuffer the basic I/O       //
// serializer.                                                          //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TMessage.h"
#include "Bytes.h"
#include "TFile.h"

extern "C" void R__zip (Int_t cxlevel, Int_t *nin, char *bufin, Int_t *lout, char *bufout, Int_t *nout);
extern "C" void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout);
const Int_t kMAXBUF = 0xffffff;

ClassImp(TMessage)

//______________________________________________________________________________
TMessage::TMessage(UInt_t what) : TBuffer(kWrite)
{
   // Create a TMessage object for storing objects. The "what" integer
   // describes the type of message. Predifined ROOT system message types
   // can be found in MessageTypes.h. Make sure your own message types are
   // unique from the ROOT defined message types (i.e. 0 - 10000 are
   // reserved by ROOT). In case you OR "what" with kMESS_ACK, the message
   // will wait for an acknowledgement from the remote side. This makes
   // the sending process synchronous. In case you OR "what" with kMESS_ZIP,
   // the message will be compressed in TSocket using the zip algorithm
   // (only if message is > 256 bytes).

   // space at the beginning of the message reserved for the message length
   UInt_t   reserved = 0;
   *this << reserved;

   fWhat  = what;
   *this << what;

   fClass      = 0;
   fCompress   = 0;
   fBufComp    = 0;
   fBufCompCur = 0;
   fCompPos    = 0;
}

//______________________________________________________________________________
TMessage::TMessage(void *buf, Int_t bufsize) : TBuffer(kRead, bufsize, buf)
{
   // Create a TMessage object for reading objects. The objects will be
   // read from buf. Use the What() method to get the message type.

   // skip space at the beginning of the message reserved for the message length
   fBufCur += sizeof(UInt_t);

   *this >> fWhat;

   fCompress   = 0;
   fBufComp    = 0;
   fBufCompCur = 0;
   fCompPos    = 0;

   if (fWhat & kMESS_ZIP) {
      // if buffer has kMESS_ZIP set, move it to fBufComp and uncompress
      fBufComp    = fBuffer;
      fBufCompCur = fBuffer + bufsize;
      fBuffer     = 0;
      Uncompress();
   }

   if (fWhat == kMESS_OBJECT) {
      InitMap();
      fClass = ReadClass();     // get first the class stored in message
      SetBufferOffset(sizeof(UInt_t) + sizeof(fWhat));
      ResetMap();
   } else {
      fClass = 0;
   }
}

//______________________________________________________________________________
TMessage::~TMessage()
{
   // Clean up compression buffer.

   delete [] fBufComp;
}

//______________________________________________________________________________
void TMessage::Forward()
{
   // Change a buffer that was received into one that can be send, i.e.
   // forward a just received message.

   if (IsReading()) {
      SetWriteMode();
      SetBufferOffset(fBufSize);

      if (fBufComp) {
         fCompPos = fBufCur;
      }
   }
}

//______________________________________________________________________________
void TMessage::Reset()
{
   // Reset the message buffer so we can use (i.e. fill) it again.

   SetBufferOffset(sizeof(UInt_t) + sizeof(fWhat));
   ResetMap();

   if (fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }
}

//______________________________________________________________________________
void TMessage::SetLength() const
{
   // Set the message length at the beginning of the message buffer.
   // This method is only called by TSocket::Send().

   if (IsWriting()) {
      char *buf = Buffer();
      tobuf(buf, (UInt_t)(Length() - sizeof(UInt_t)));

      if (fBufComp) {
         buf = fBufComp;
         tobuf(buf, (UInt_t)(CompLength() - sizeof(UInt_t)));
      }
   }
}

//______________________________________________________________________________
void TMessage::SetWhat(UInt_t what)
{
   // Using this method one can change the message type a-posteriory.
   // In case you OR "what" with kMESS_ACK, the message will wait for
   // an acknowledgement from the remote side. This makes the sending
   // process synchronous.

   fWhat = what;

   char *buf = Buffer();
   buf += sizeof(UInt_t);   // skip reserved length space
   tobuf(buf, what);

   if (fBufComp) {
      buf = fBufComp;
      buf += sizeof(UInt_t);   // skip reserved length space
      tobuf(buf, what | kMESS_ZIP);
   }
}

//______________________________________________________________________________
void TMessage::SetCompressionLevel(Int_t level)
{
   // Set the message compression level. Can be between 0 and 9 with 0
   // being no compression and 9 maximum compression. In general the default
   // level of 1 is the best compromise between achieved compression and
   // cpu time. Compression will only happen when the message is > 256 bytes.

   if (level < 0) level = 0;
   if (level > 9) level = 9;

   if (level != fCompress && fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }
   fCompress = level;
}

//______________________________________________________________________________
Int_t TMessage::Compress()
{
   // Compress the message. The message will only be compressed if the
   // compression level > 0 and the if the message is > 256 bytes.
   // Returns -1 in case of error (when compression fails or
   // when the message increases in size in some pathological cases),
   // otherwise returns 0.

   if (fCompress == 0) {
      // no compression specified
      if (fBufComp) {
         delete [] fBufComp;
         fBufComp    = 0;
         fBufCompCur = 0;
         fCompPos    = 0;
      }
      return 0;
   }

   if (fBufComp && fCompPos == fBufCur) {
      // the message was already compressed
      return 0;
   }

   // remove any existing compressed buffer before compressing modified message
   if (fBufComp) {
      delete [] fBufComp;
      fBufComp    = 0;
      fBufCompCur = 0;
      fCompPos    = 0;
   }

   if (Length() <= (Int_t)(256 + 2*sizeof(UInt_t))) {
      // this message is too small to be compressed
      return 0;
   }

   Int_t hdrlen   = 2*sizeof(UInt_t);
   Int_t messlen  = Length() - hdrlen;
   Int_t nbuffers = messlen / kMAXBUF;
   Int_t chdrlen  = 3*sizeof(UInt_t);   // compressed buffer header length
   Int_t buflen   = TMath::Max(512, chdrlen + messlen + 9*nbuffers);
   fBufComp       = new char[buflen];
   char *messbuf  = Buffer() + hdrlen;
   char *bufcur   = fBufComp + chdrlen;
   Int_t noutot   = 0;
   Int_t nzip     = 0;
   Int_t nout, bufmax;
   for (Int_t i = 0; i <= nbuffers; i++) {
      if (i == nbuffers)
         bufmax = messlen - nzip;
      else
         bufmax = kMAXBUF;
      R__zip(fCompress, &bufmax, messbuf, &bufmax, bufcur, &nout);
      if (nout == 0 || nout >= messlen) {
         //this happens when the buffer cannot be compressed
         delete [] fBufComp;
         fBufComp    = 0;
         fBufCompCur = 0;
         fCompPos    = 0;
         return -1;
      }
      bufcur  += nout;
      noutot  += nout;
      messbuf += kMAXBUF;
      nzip    += kMAXBUF;
   }
   fBufCompCur = bufcur;
   fCompPos    = fBufCur;

   bufcur = fBufComp;
   tobuf(bufcur, (UInt_t)(CompLength() - sizeof(UInt_t)));
   Int_t what = fWhat | kMESS_ZIP;
   tobuf(bufcur, what);
   tobuf(bufcur, Length());    // original uncompressed buffer length

   return 0;
}

//______________________________________________________________________________
Int_t TMessage::Uncompress()
{
   // Uncompress the message. The message will only be uncompressed when
   // kMESS_ZIP is set. Returns -1 in case of error, 0 otherwise.

   if (!fBufComp || !(fWhat & kMESS_ZIP))
      return -1;

   Int_t buflen;
   Int_t hdrlen = 2*sizeof(UInt_t);
   UChar_t *bufcur = (UChar_t*)fBufComp + hdrlen;
   frombuf((char *&)bufcur, &buflen);
   fBuffer  = new char[buflen];
   fBufSize = buflen;
   fBufCur  = fBuffer + sizeof(UInt_t) + sizeof(fWhat);
   fBufMax  = fBuffer + fBufSize;
   char *messbuf = fBuffer + hdrlen;

   Int_t nin, nout, nbuf;
   Int_t noutot = 0;
   while (1) {
      nin  = 9 + ((Int_t)bufcur[3] | ((Int_t)bufcur[4] << 8) | ((Int_t)bufcur[5] << 16));
      nbuf = (Int_t)bufcur[6] | ((Int_t)bufcur[7] << 8) | ((Int_t)bufcur[8] << 16);
      R__unzip(&nin, bufcur, &nbuf, messbuf, &nout);
      if (!nout) break;
      noutot += nout;
      if (noutot >= buflen - hdrlen) break;
      bufcur  += nin;
      messbuf += nout;
   }

   fWhat &= ~kMESS_ZIP;
   fCompress = 1;

   return 0;
}

<|endoftext|>