<commit_before>/**
 * \file tests.cpp
 *
 * \author Matt Hammerly
 */

#include <postgresql/libpq-fe.h>
#include <string>
#include <fstream>
#include <stdio.h>
#include <iostream>
#include <curl/curl.h>
#include <plist/plist++.h>
#include <dirent.h>
#include <sys/types.h>
#include <cctype>
#include <cstring>
#include <cassert>
#include "ma.h"
#include "db.h"
#include "tests.h"

/**
 * \brief Entry point into test program
 */
int main() {

    PGconn* conn = connectToDatabase();

    dropTables(conn);

    createTables(conn);

    PList::Dictionary* library = openLibrary();

    // test caseSensitiveFilePath()
    test_caseSensitiveFilePath();

    // test adding track to database
    test_addTrackToDatabase(conn);

    // test adding playlist to database
    test_addPlaylistToDatabase(conn);

    // test fractional inserts into playlists
    test_track_inserts(conn);

    // test track removal from playlists
    test_removal(conn);

    puts( PQerrorMessage(conn) );

    return 0;
}

/**
 * \brief Test caseSensitiveFilePath() function
 */
void test_caseSensitiveFilePath() {
    std::string new_library_path = "/home/matt/hdd/dev/yamm/test_data/";
    std::string wrong = new_library_path + "beirut/THe Rip tiDe/01 A candle's Fire.mp3";
    std::string right = new_library_path + "Beirut/The Rip Tide/01 A Candle's Fire.mp3";

    assert( caseSensitiveFilePath(wrong) == right );
    puts( "caseSensitiveFilePath() works correctly!" );
}

/**
 * \brief Test addTrackToDatabase(...)
 *
 * \param conn Pointer to database connection struct
 */
void test_addTrackToDatabase(PGconn* conn) {
    std::string new_library_path = "/home/matt/hdd/dev/yamm/test_data/";

    PGresult* res = addTrackToDatabase( conn, "", new_library_path + "Beirut/The Rip Tide/01 A Candle's Fire.mp3", 0 );
    char* track_id = PQgetvalue( res, 0, 0 );
    char const *expected_id = "1";

    assert ( strcmp(track_id, expected_id) == 0 );
    puts ("addTrackToDatabase(...) works correctly!" );

    PQclear(res);
}

/**
 * \brief Test addPlaylistToDatabase(...)
 *
 * \param conn Pointer to database connection struct
 */
void test_addPlaylistToDatabase(PGconn* conn) {
    PGresult* res = addPlaylistToDatabase( conn, "", "Test Playlist", 0 );
    char *playlist_id = PQgetvalue( res, 0, 0 );
    char const *expected_id = "1";

    assert( strcmp( playlist_id, expected_id) == 0 );
    puts ("addPlaylistToDatabase(...) works correctly!" );

    PQclear(res);
}

/**
 * \brief Test appendTrackToPlaylist(...) and addTrackToPlaylist(...) functions
 *
 * \param conn Pointer to database connection struct
 *
 * Todo: check to make sure the fractional insert is ordered correctly
 */
void test_track_inserts(PGconn* conn) {
    // SELECT * FROM tracks_playlists Link LEFT JOIN playlists Playlist ON Link.playlist_id = Playlist.id WHERE Playlist.title = 'testing fractional inserts' ORDER BY position;
    PGresult* playlist_insert_test_res = addPlaylistToDatabase( conn, "", "testing fractional inserts", 0);
    char* playlist_id = PQgetvalue( playlist_insert_test_res, 0, 0 );
    PGresult* track_insert_res = appendTrackToPlaylist( conn, "1", playlist_id );
    char* association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp( association_id, "1") == 0 );
    track_insert_res = appendTrackToPlaylist( conn, "2", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "4", playlist_id );
    association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp(association_id, "3") == 0 );
    puts ("appendTrackToPlaylist(...) works correctly!");
    track_insert_res = addTrackToPlaylist( conn, "3", playlist_id, 2.5 );
    association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp(association_id, "4") == 0 );
    puts ("addTrackToPlaylist(...) works correctly!");
    PQclear( playlist_insert_test_res );
    PQclear( track_insert_res );
}

/**
 * \brief Test removeTrackFromPlaylist(...) assuming appendTrackToPlaylist works fine
 *
 * \param conn Pointer to database connection struct
 */
void test_removal(PGconn* conn) {
    // SELECT * FROM tracks_playlists Link LEFT JOIN playlists Playlist ON Link.playlist_id = Playlist.id WHERE Playlist.title = 'testing track removal' ORDER BY position;
    PGresult* playlist_insert_test_res = addPlaylistToDatabase( conn, "", "testing track removal", 0);
    char* playlist_id = PQgetvalue( playlist_insert_test_res, 0, 0 );
    PGresult* track_insert_res = appendTrackToPlaylist( conn, "1", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "2", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "4", playlist_id );
    char* track_playlist_id = PQgetvalue( track_insert_res, 0, 0 );
    track_insert_res = removeTrackFromPlaylist( conn, track_playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "3", playlist_id );
    PQclear( playlist_insert_test_res );
    PQclear( track_insert_res );
}
<commit_msg>finished up another test<commit_after>/**
 * \file tests.cpp
 *
 * \author Matt Hammerly
 */

#include <postgresql/libpq-fe.h>
#include <string>
#include <fstream>
#include <stdio.h>
#include <iostream>
#include <curl/curl.h>
#include <plist/plist++.h>
#include <dirent.h>
#include <sys/types.h>
#include <cctype>
#include <cstring>
#include <cassert>
#include "ma.h"
#include "db.h"
#include "tests.h"

/**
 * \brief Entry point into test program
 */
int main() {

    PGconn* conn = connectToDatabase();

    dropTables(conn);

    createTables(conn);

    PList::Dictionary* library = openLibrary();

    // test caseSensitiveFilePath()
    test_caseSensitiveFilePath();

    // test adding track to database
    test_addTrackToDatabase(conn);

    // test adding playlist to database
    test_addPlaylistToDatabase(conn);

    // test fractional inserts into playlists
    test_track_inserts(conn);

    // test track removal from playlists
    test_removal(conn);

    puts( PQerrorMessage(conn) );

    return 0;
}

/**
 * \brief Test caseSensitiveFilePath() function
 */
void test_caseSensitiveFilePath() {
    std::string new_library_path = "/home/matt/hdd/dev/yamm/test_data/";
    std::string wrong = new_library_path + "beirut/THe Rip tiDe/01 A candle's Fire.mp3";
    std::string right = new_library_path + "Beirut/The Rip Tide/01 A Candle's Fire.mp3";

    assert( caseSensitiveFilePath(wrong) == right );
    puts( "caseSensitiveFilePath() works correctly!" );
}

/**
 * \brief Test addTrackToDatabase(...)
 *
 * \param conn Pointer to database connection struct
 */
void test_addTrackToDatabase(PGconn* conn) {
    std::string new_library_path = "/home/matt/hdd/dev/yamm/test_data/";

    PGresult* res = addTrackToDatabase( conn, "", new_library_path + "Beirut/The Rip Tide/01 A Candle's Fire.mp3", 0 );
    char* track_id = PQgetvalue( res, 0, 0 );
    char const *expected_id = "1";

    assert ( strcmp(track_id, expected_id) == 0 );
    puts ("addTrackToDatabase(...) works correctly!" );

    PQclear(res);
}

/**
 * \brief Test addPlaylistToDatabase(...)
 *
 * \param conn Pointer to database connection struct
 */
void test_addPlaylistToDatabase(PGconn* conn) {
    PGresult* res = addPlaylistToDatabase( conn, "", "Test Playlist", 0 );
    char *playlist_id = PQgetvalue( res, 0, 0 );
    char const *expected_id = "1";

    assert( strcmp( playlist_id, expected_id) == 0 );
    puts ("addPlaylistToDatabase(...) works correctly!" );

    PQclear(res);
}

/**
 * \brief Test appendTrackToPlaylist(...) and addTrackToPlaylist(...) functions
 *
 * \param conn Pointer to database connection struct
 *
 * Todo: check to make sure the fractional insert is ordered correctly
 */
void test_track_inserts(PGconn* conn) {
    // SELECT * FROM tracks_playlists Link LEFT JOIN playlists Playlist ON Link.playlist_id = Playlist.id WHERE Playlist.title = 'testing fractional inserts' ORDER BY position;
    PGresult* playlist_insert_test_res = addPlaylistToDatabase( conn, "", "testing fractional inserts", 0);
    char* playlist_id = PQgetvalue( playlist_insert_test_res, 0, 0 );
    PGresult* track_insert_res = appendTrackToPlaylist( conn, "1", playlist_id );
    char* association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp( association_id, "1") == 0 );
    track_insert_res = appendTrackToPlaylist( conn, "2", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "4", playlist_id );
    association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp(association_id, "3") == 0 );
    puts ("appendTrackToPlaylist(...) works correctly!");
    track_insert_res = addTrackToPlaylist( conn, "3", playlist_id, 2.5 );
    association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp(association_id, "4") == 0 );
    puts ("addTrackToPlaylist(...) works correctly!");
    PQclear( playlist_insert_test_res );
    PQclear( track_insert_res );
}

/**
 * \brief Test removeTrackFromPlaylist(...) assuming appendTrackToPlaylist works fine
 *
 * \param conn Pointer to database connection struct
 */
void test_removal(PGconn* conn) {
    // SELECT * FROM tracks_playlists Link LEFT JOIN playlists Playlist ON Link.playlist_id = Playlist.id WHERE Playlist.title = 'testing track removal' ORDER BY position;
    PGresult* playlist_insert_test_res = addPlaylistToDatabase( conn, "", "testing track removal", 0);
    char* playlist_id = PQgetvalue( playlist_insert_test_res, 0, 0 );
    PGresult* track_insert_res = appendTrackToPlaylist( conn, "1", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "2", playlist_id );
    track_insert_res = appendTrackToPlaylist( conn, "4", playlist_id );
    char* association_id = PQgetvalue( track_insert_res, 0, 0 );
    track_insert_res = removeTrackFromPlaylist( conn, association_id );
    association_id = PQgetvalue( track_insert_res, 0, 0 );
    assert( strcmp(association_id, "7") == 0 );
    puts("removeTrackFromPlaylist(...) works correctly!");
    track_insert_res = appendTrackToPlaylist( conn, "3", playlist_id );
    PQclear( playlist_insert_test_res );
    PQclear( track_insert_res );
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Nicholas Corgan (n.corgan@gmail.com)
 *
 * Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt
 * or copy at http://opensource.org/licenses/MIT)
 */

#include <cstdlib>
#include <fstream>
#include <stdexcept>
#include <cstdlib>
#include <string>

#include <boost/filesystem.hpp>

#include <pkmn/paths.hpp>

#include "SQLiteCpp/src/SQLiteC++.h"

namespace fs = boost::filesystem;

namespace pkmn
{
    //Get location of APPDATA
    std::string get_appdata_path()
    {
        //Windows
        #ifdef _MSC_VER
        char* appdata_dir;
        size_t len;
        _dupenv_s(&appdata_dir, &len, "APPDATA");
        #else
        const char* appdata_dir = getenv("APPDATA");
        #endif
        if(appdata_dir and fs::exists(fs::path(appdata_dir))) return std::string(appdata_dir);

        //Linux
        #ifdef _MSC_VER
        _dupenv_s(&appdata_dir, &len, "APPDATA");
        #else
        appdata_dir = getenv("HOME");
        #endif
        if(appdata_dir and fs::exists(fs::path(appdata_dir))) return std::string(appdata_dir);

        //If it's gotten to this point, fall back to the temp directory
        return get_tmp_dir();
    }

    //Get location of database
    std::string get_database_path()
    {
        #ifdef _MSC_VER
        char* database_dir;
        size_t len;
        _dupenv_s(&database_dir, &len, "LIBPKMN_DATABASE_DIR");
        #else
        const char* database_dir = getenv("LIBPKMN_DATABASE_DIR"); //Environment variable
        #endif
        if(database_dir == NULL) database_dir = "@LIBPKMN_DATABASE_DIR@"; //CMake variable

        fs::path database_path = fs::path(database_dir) / "libpkmn.db";
        if(not fs::exists(database_path)) throw std::runtime_error("Could not find database!");
        else
        {
            /*
             * Use SQLiteCpp to check to see if database is valid.
             * The SQLite::Database constructor will throw an exception if
             * the given filename is not a valid SQLite database.
             */
            try {SQLite::Database db(database_path.string().c_str());}
            catch(std::exception &e) {return ":memory:";}
        }

        return database_path.string();
    }

    //Get images directory
    std::string get_images_dir()
    {
        #ifdef _MSC_VER
        char* images_dir;
        size_t len;
        _dupenv_s(&images_dir, &len, "LIBPKMN_IMAGES_DIR");
        #else
        const char* images_dir = getenv("LIBPKMN_IMAGES_DIR"); //Environment variable
        #endif
        if(images_dir == NULL) images_dir = "@LIBPKMN_IMAGES_DIR@"; //CMake variable

        if(not fs::exists(fs::path(images_dir))) throw std::runtime_error("Could not find images directory!");

        return std::string(images_dir);
    }

    //Get path for system's temporary directory
    std::string get_tmp_dir()
    {
        #ifdef _MSC_VER
        char* tmp_dir;
        size_t len;
        _dupenv_s(&tmp_dir, &len, "TMP");
        #else
        const char* tmp_dir = getenv("TMP");
        #endif

        if(tmp_dir and fs::exists(fs::path(tmp_dir))) return std::string(tmp_dir);
        else return "/tmp";
    }
} /* namespace pkmn */
<commit_msg>paths: removed duplicate code and removed Windows memory leaks<commit_after>/*
 * Copyright (c) 2013-2014 Nicholas Corgan (n.corgan@gmail.com)
 *
 * Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt
 * or copy at http://opensource.org/licenses/MIT)
 */

#include <cstdlib>
#include <fstream>
#include <stdexcept>
#include <cstdlib>
#include <string>

#include <boost/filesystem.hpp>

#include <pkmn/paths.hpp>

#include "SQLiteCpp/src/SQLiteC++.h"

namespace fs = boost::filesystem;

namespace pkmn
{
    static std::string _getenv(const char* var)
    {
        std::string value;

        #ifdef PKMN_PLATFORM_WIN32
        char* arr;
        size_t len;
        _dupenv_s(&arr, &len, var);
        if(!arr) return std::string("");
        else
        {
            value = std::string(arr);
            free(arr);
            return value;
        }
        #else
        const char* arr = getenv(var);
        return (arr) ? std::string(arr) : std::string("");
        #endif
    }

    //Get location of APPDATA
    std::string get_appdata_path()
    {
        #ifdef PKMN_PLATFORM_WIN32
        std::string appdata = _getenv("APPDATA");
        if(appdata != "" and fs::exists(fs::path(appdata))) return appdata;
        #else
        std::string home = _getenv("HOME");
        if(home != "" and fs::exists(fs::path(home))) return home;
        #endif

        //It should never get to this point, this is a last resort
        return get_tmp_dir();
    }

    //Get location of database
    std::string get_database_path()
    {
        std::string database_dir = _getenv("LIBPKMN_DATABASE_DIR");
        if(database_dir == "") database_dir = "@LIBPKMN_DATABASE_DIR@"; //Generated by CMake

        //At this point, we should have a database directory, so throw an error otherwise
        if(database_dir == "") throw std::runtime_error("Could not find database!");

        fs::path database_path = fs::path(database_dir) / "libpkmn.db";
        if(fs::exists(database_path)) return database_path.string();
        else throw std::runtime_error("Could not find database!");
    }

    //Get images directory
    std::string get_images_dir()
    {
        std::string images_dir = _getenv("LIBPKMN_IMAGES_DIR");
        if(images_dir == "") images_dir = "@LIBPKMN_IMAGES_DIR@";

        //At this point, we should have an images directory, so throw an error otherwise
        if(images_dir == "" or not fs::exists(fs::path(images_dir)))
            throw std::runtime_error("Could not find images directory!");
        else return images_dir;
    }

    //Get path for system's temporary directory
    std::string get_tmp_dir()
    {
        std::string tmp = _getenv("TMP");
        return (tmp != "" and fs::exists(fs::path(tmp))) ? tmp : "/tmp";
    }
} /* namespace pkmn */
<|endoftext|>


<commit_before>//===----------------------------------------------------------------------===//
//
//                         PelotonDB
//
// materialization_executor.cpp
//
// Identification: src/backend/executor/materialization_executor.cpp
//
// Copyright (c) 2015, Carnegie Mellon University Database Group
//
//===----------------------------------------------------------------------===//

#include "backend/executor/materialization_executor.h"

#include <cassert>
#include <memory>
#include <utility>

#include "backend/planner/materialization_plan.h"
#include "backend/executor/logical_tile.h"
#include "backend/executor/logical_tile_factory.h"
#include "backend/storage/tile.h"
#include "backend/storage/data_table.h"

namespace peloton {
namespace executor {

// Row-oriented materialization
void MaterializeRowAtAtATime(LogicalTile *source_tile,
                             const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                             const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                             storage::Tile *dest_tile);

// Column-oriented materialization
void MaterializeColumnAtATime(LogicalTile *source_tile,
                              const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                              const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                              storage::Tile *dest_tile);

/**
 * @brief Constructor for the materialization executor.
 * @param node Materialization node corresponding to this executor.
 */
MaterializationExecutor::MaterializationExecutor(
    const planner::AbstractPlan *node, ExecutorContext *executor_context)
: AbstractExecutor(node, executor_context) {}

/**
 * @brief Nothing to init at the moment.
 * @return true on success, false otherwise.
 */
bool MaterializationExecutor::DInit() {
  assert(children_.size() == 1);

  return true;
}

/**
 * @brief Generates map from each base tile to columns originally from that
 *        base tile to be materialized.
 * @param column_ids Ids of columns to be materialized.
 * @param source_tile Logical tile that contains mapping from columns to
 *        base tiles.
 * @param tile_to_cols Map to be populated with mappings from tile to columns.
 *
 * We generate this mapping so that we can materialize columns tile by tile for
 * efficiency reasons.
 */
void MaterializationExecutor::GenerateTileToColMap(
    const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
    LogicalTile *source_tile,
    std::unordered_map<storage::Tile *, std::vector<oid_t>> &
    cols_in_physical_tile) {
  for (const auto &kv : old_to_new_cols) {
    oid_t col = kv.first;

    // figure out base physical tile for column in logical tile
    storage::Tile *base_tile = source_tile->GetBaseTile(col);

    std::vector<oid_t> &cols_from_tile = cols_in_physical_tile[base_tile];
    cols_from_tile.push_back(col);
  }
}

/**
 * @brief Does the actual copying of data into the new physical tile.
 * @param source_tile Source tile to copy data from.
 * @param tile_to_cols Map from base tile to columns in that tile
 *        to be materialized.
 * @param dest_tile New tile to copy data into.
 */
void MaterializationExecutor::MaterializeByTiles(
    LogicalTile *source_tile,
    const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
    const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
    storage::Tile *dest_tile) {

  auto dest_tile_column_count = dest_tile->GetColumnCount();
  // TODO: Make this a parameter
  oid_t column_count_threshold = 20;
  bool row_wise_materialization = true;

  if(peloton_layout == LAYOUT_HYBRID &&
      dest_tile_column_count < column_count_threshold)
    row_wise_materialization = false;

  // Materialize as needed
  if(row_wise_materialization == true) {
    MaterializeRowAtAtATime(source_tile, old_to_new_cols, tile_to_cols, dest_tile);
  }
  else {
    MaterializeColumnAtATime(source_tile, old_to_new_cols, tile_to_cols, dest_tile);
  }

}

void MaterializeRowAtAtATime(LogicalTile *source_tile,
                             const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                             const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                             storage::Tile *dest_tile) {

  ///////////////////////////
  // EACH PHYSICAL TILE
  ///////////////////////////
  // Copy over all data from each base tile.
  for (const auto &kv : tile_to_cols) {
    const std::vector<oid_t> &old_column_ids = kv.second;

    auto &schema = source_tile->GetSchema();
    oid_t new_tuple_id = 0;

    auto& column_position_lists = source_tile->GetPositionLists();

    // Get old column information
    std::vector<oid_t> old_column_position_idxs;
    std::vector<size_t> old_column_offsets;
    std::vector<ValueType> old_column_types;
    std::vector<bool> old_is_inlineds;
    std::vector<storage::Tile*> old_tiles;

    // Get new column information
    std::vector<size_t> new_column_offsets;
    std::vector<bool> new_is_inlineds;
    std::vector<size_t> new_column_lengths;

    // Amortize schema lookups once per column
    for (oid_t old_col_id : old_column_ids) {
      auto& column_info = schema[old_col_id];

      // Get the position list
      old_column_position_idxs.push_back(column_info.position_list_idx);

      // Get old column information
      storage::Tile *old_tile = column_info.base_tile;
      old_tiles.push_back(old_tile);
      auto old_schema = old_tile->GetSchema();
      oid_t old_column_id = column_info.origin_column_id;
      const size_t old_column_offset = old_schema->GetOffset(old_column_id);
      old_column_offsets.push_back(old_column_offset);
      const ValueType old_column_type = old_schema->GetType(old_column_id);
      old_column_types.push_back(old_column_type);
      const bool old_is_inlined = old_schema->IsInlined(old_column_id);
      old_is_inlineds.push_back(old_is_inlined);

      // Old to new column mapping
      auto it = old_to_new_cols.find(old_col_id);
      assert(it != old_to_new_cols.end());

      // Get new column information
      oid_t new_column_id = it->second;
      auto new_schema = dest_tile->GetSchema();
      const size_t new_column_offset = new_schema->GetOffset(new_column_id);
      new_column_offsets.push_back(new_column_offset);
      const bool new_is_inlined = new_schema->IsInlined(new_column_id);
      new_is_inlineds.push_back(new_is_inlined);
      const size_t new_column_length = new_schema->GetAppropriateLength(new_column_id);
      new_column_lengths.push_back(new_column_length);
    }

    assert(new_column_offsets.size() == old_column_ids.size());

    ///////////////////////////
    // EACH TUPLE
    ///////////////////////////
    // Copy all values in the tuple to the physical tile
    // This uses fast getter and setter functions
    for (oid_t old_tuple_id : *source_tile) {

      ///////////////////////////
      // EACH COLUMN
      ///////////////////////////
      // Go over each column in given base physical tile
      oid_t col_itr = 0;

      for (oid_t old_col_id : old_column_position_idxs) {

        auto& column_position_list = column_position_lists[old_col_id];

        oid_t base_tuple_id = column_position_list[old_tuple_id];

        auto value = old_tiles[col_itr]->GetValueFast(base_tuple_id,
                                                      old_column_offsets[col_itr],
                                                      old_column_types[col_itr],
                                                      old_is_inlineds[col_itr]);

        LOG_TRACE("Old Tuple : %u Column : %u \n", old_tuple_id, old_col_id);
        LOG_TRACE("New Tuple : %u Column : %u \n", new_tuple_id, new_column_id);

        dest_tile->SetValueFast(value, new_tuple_id,
                                new_column_offsets[col_itr],
                                new_is_inlineds[col_itr],
                                new_column_lengths[col_itr]);

        // Go to next column
        col_itr++;
      }

      // Go to next tuple
      new_tuple_id++;
    }

  }

}

void MaterializeColumnAtATime(LogicalTile *source_tile,
                              const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                              const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                              storage::Tile *dest_tile) {

  ///////////////////////////
  // EACH PHYSICAL TILE
  ///////////////////////////
  // Copy over all data from each base tile.
  for (const auto &kv : tile_to_cols) {
    const std::vector<oid_t> &old_column_ids = kv.second;

    ///////////////////////////
    // EACH COLUMN
    ///////////////////////////
    // Go over each column in given base physical tile
    for (oid_t old_col_id : old_column_ids) {
      auto &column_info = source_tile->GetColumnInfo(old_col_id);

      // Amortize schema lookups once per column
      storage::Tile *old_tile = column_info.base_tile;
      auto old_schema = old_tile->GetSchema();

      // Get old column information
      oid_t old_column_id = column_info.origin_column_id;
      const size_t old_column_offset = old_schema->GetOffset(old_column_id);
      const ValueType old_column_type = old_schema->GetType(old_column_id);
      const bool old_is_inlined = old_schema->IsInlined(old_column_id);

      // Old to new column mapping
      auto it = old_to_new_cols.find(old_col_id);
      assert(it != old_to_new_cols.end());

      // Get new column information
      oid_t new_column_id = it->second;
      auto new_schema = dest_tile->GetSchema();
      const size_t new_column_offset = new_schema->GetOffset(new_column_id);
      const bool new_is_inlined = new_schema->IsInlined(new_column_id);
      const size_t new_column_length = new_schema->GetAppropriateLength(new_column_id);

      // Get the position list
      auto& column_position_list = source_tile->GetPositionList(column_info.position_list_idx);
      oid_t new_tuple_id = 0;

      // Copy all values in the column to the physical tile
      // This uses fast getter and setter functions
      ///////////////////////////
      // EACH TUPLE
      ///////////////////////////
      for (oid_t old_tuple_id : *source_tile) {
        oid_t base_tuple_id = column_position_list[old_tuple_id];
        auto value = old_tile->GetValueFast(base_tuple_id,
                                            old_column_offset,
                                            old_column_type,
                                            old_is_inlined);

        LOG_TRACE("Old Tuple : %u Column : %u \n", old_tuple_id, old_col_id);
        LOG_TRACE("New Tuple : %u Column : %u \n", new_tuple_id, new_column_id);

        dest_tile->SetValueFast(value, new_tuple_id,
                                new_column_offset,
                                new_is_inlined,
                                new_column_length);

        // Go to next tuple
        new_tuple_id++;
      }

    }

  }

}

std::unordered_map<oid_t, oid_t> MaterializationExecutor::BuildIdentityMapping(
    const catalog::Schema *schema) {
  std::unordered_map<oid_t, oid_t> old_to_new_cols;
  oid_t column_count = schema->GetColumnCount();
  for (oid_t col = 0; col < column_count; col++) {
    old_to_new_cols[col] = col;
  }

  return old_to_new_cols;
}

/**
 * @brief Create a physical tile for the given logical tile
 * @param source_tile Source tile from which the physical tile is created
 * @return a logical tile wrapper for the created physical tile
 */
LogicalTile *MaterializationExecutor::Physify(LogicalTile *source_tile) {
  std::unique_ptr<catalog::Schema> source_tile_schema(
      source_tile->GetPhysicalSchema());
  const int num_tuples = source_tile->GetTupleCount();
  const catalog::Schema *output_schema;
  std::unordered_map<oid_t, oid_t> old_to_new_cols;
  auto node = GetRawNode();

  // Create a default identity mapping node if we did not get one
  if (node == nullptr) {
    assert(source_tile_schema.get());
    output_schema = source_tile_schema.get();

    old_to_new_cols = BuildIdentityMapping(output_schema);
  }
  // Else use the mapping in the given plan node
  else {
    const planner::MaterializationPlan &node =
        GetPlanNode<planner::MaterializationPlan>();
    if (node.GetSchema()) {
      output_schema = node.GetSchema();
      old_to_new_cols = node.old_to_new_cols();
    } else {
      output_schema = source_tile_schema.get();
      old_to_new_cols = BuildIdentityMapping(output_schema);
    }
  }

  // Generate mappings.
  std::unordered_map<storage::Tile *, std::vector<oid_t>> tile_to_cols;
  GenerateTileToColMap(old_to_new_cols, source_tile, tile_to_cols);

  // Create new physical tile.
  std::unique_ptr<storage::Tile> dest_tile(
      storage::TileFactory::GetTempTile(*output_schema, num_tuples));

  // Proceed to materialize logical tile by physical tile at a time.
  MaterializeByTiles(source_tile, old_to_new_cols, tile_to_cols,
                     dest_tile.get());

  // Wrap physical tile in logical tile.
  return LogicalTileFactory::WrapTiles({dest_tile.release()});
}

/**
 * @brief Creates materialized physical tile from logical tile and wraps it
 *        in a new logical tile.
 *
 * @return true on success, false otherwise.
 */
bool MaterializationExecutor::DExecute() {
  // Retrieve child tile.
  const bool success = children_[0]->Execute();
  if (!success) {
    return false;
  }

  std::unique_ptr<LogicalTile> source_tile(children_[0]->GetOutput());
  LogicalTile *output_tile = nullptr;

  // Check the number of tuples in input logical tile
  // If none, then just return false
  const int num_tuples = source_tile->GetTupleCount();
  if (num_tuples == 0) {
    return false;
  }

  auto node = GetRawNode();
  bool physify_flag = true;  // by default, we create a physical tile

  if (node != nullptr) {
    const planner::MaterializationPlan &node =
        GetPlanNode<planner::MaterializationPlan>();
    physify_flag = node.GetPhysifyFlag();
  }

  if (physify_flag) {
    /* create a physical tile and a logical tile wrapper to be the output */
    output_tile = Physify(source_tile.get());
  } else {
    /* just pass thru the underlying logical tile */
    output_tile = source_tile.release();
  }

  SetOutput(output_tile);

  return true;
}

}  // namespace executor
}  // namespace peloton
<commit_msg>Restore mat<commit_after>//===----------------------------------------------------------------------===//
//
//                         PelotonDB
//
// materialization_executor.cpp
//
// Identification: src/backend/executor/materialization_executor.cpp
//
// Copyright (c) 2015, Carnegie Mellon University Database Group
//
//===----------------------------------------------------------------------===//

#include "backend/executor/materialization_executor.h"

#include <cassert>
#include <memory>
#include <utility>

#include "backend/planner/materialization_plan.h"
#include "backend/executor/logical_tile.h"
#include "backend/executor/logical_tile_factory.h"
#include "backend/storage/tile.h"
#include "backend/storage/data_table.h"

namespace peloton {
namespace executor {

// Row-oriented materialization
void MaterializeRowAtAtATime(LogicalTile *source_tile,
                             const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                             const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                             storage::Tile *dest_tile);

// Column-oriented materialization
void MaterializeColumnAtATime(LogicalTile *source_tile,
                              const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                              const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                              storage::Tile *dest_tile);

/**
 * @brief Constructor for the materialization executor.
 * @param node Materialization node corresponding to this executor.
 */
MaterializationExecutor::MaterializationExecutor(
    const planner::AbstractPlan *node, ExecutorContext *executor_context)
: AbstractExecutor(node, executor_context) {}

/**
 * @brief Nothing to init at the moment.
 * @return true on success, false otherwise.
 */
bool MaterializationExecutor::DInit() {
  assert(children_.size() == 1);

  return true;
}

/**
 * @brief Generates map from each base tile to columns originally from that
 *        base tile to be materialized.
 * @param column_ids Ids of columns to be materialized.
 * @param source_tile Logical tile that contains mapping from columns to
 *        base tiles.
 * @param tile_to_cols Map to be populated with mappings from tile to columns.
 *
 * We generate this mapping so that we can materialize columns tile by tile for
 * efficiency reasons.
 */
void MaterializationExecutor::GenerateTileToColMap(
    const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
    LogicalTile *source_tile,
    std::unordered_map<storage::Tile *, std::vector<oid_t>> &
    cols_in_physical_tile) {
  for (const auto &kv : old_to_new_cols) {
    oid_t col = kv.first;

    // figure out base physical tile for column in logical tile
    storage::Tile *base_tile = source_tile->GetBaseTile(col);

    std::vector<oid_t> &cols_from_tile = cols_in_physical_tile[base_tile];
    cols_from_tile.push_back(col);
  }
}

/**
 * @brief Does the actual copying of data into the new physical tile.
 * @param source_tile Source tile to copy data from.
 * @param tile_to_cols Map from base tile to columns in that tile
 *        to be materialized.
 * @param dest_tile New tile to copy data into.
 */
void MaterializationExecutor::MaterializeByTiles(
    LogicalTile *source_tile,
    const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
    const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
    storage::Tile *dest_tile) {

  auto dest_tile_column_count = dest_tile->GetColumnCount();
  // TODO: Make this a parameter
  oid_t column_count_threshold = 20;
  bool row_wise_materialization = true;

  if(peloton_layout == LAYOUT_COLUMN)
      row_wise_materialization = false;

  if(peloton_layout == LAYOUT_HYBRID &&
      dest_tile_column_count < column_count_threshold)
    row_wise_materialization = false;

  // Materialize as needed
  if(row_wise_materialization == true) {
    MaterializeRowAtAtATime(source_tile, old_to_new_cols, tile_to_cols, dest_tile);
  }
  else {
    MaterializeColumnAtATime(source_tile, old_to_new_cols, tile_to_cols, dest_tile);
  }

}

void MaterializeRowAtAtATime(LogicalTile *source_tile,
                             const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                             const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                             storage::Tile *dest_tile) {

  ///////////////////////////
  // EACH PHYSICAL TILE
  ///////////////////////////
  // Copy over all data from each base tile.
  for (const auto &kv : tile_to_cols) {
    const std::vector<oid_t> &old_column_ids = kv.second;

    auto &schema = source_tile->GetSchema();
    oid_t new_tuple_id = 0;

    auto& column_position_lists = source_tile->GetPositionLists();

    // Get old column information
    std::vector<oid_t> old_column_position_idxs;
    std::vector<size_t> old_column_offsets;
    std::vector<ValueType> old_column_types;
    std::vector<bool> old_is_inlineds;
    std::vector<storage::Tile*> old_tiles;

    // Get new column information
    std::vector<size_t> new_column_offsets;
    std::vector<bool> new_is_inlineds;
    std::vector<size_t> new_column_lengths;

    // Amortize schema lookups once per column
    for (oid_t old_col_id : old_column_ids) {
      auto& column_info = schema[old_col_id];

      // Get the position list
      old_column_position_idxs.push_back(column_info.position_list_idx);

      // Get old column information
      storage::Tile *old_tile = column_info.base_tile;
      old_tiles.push_back(old_tile);
      auto old_schema = old_tile->GetSchema();
      oid_t old_column_id = column_info.origin_column_id;
      const size_t old_column_offset = old_schema->GetOffset(old_column_id);
      old_column_offsets.push_back(old_column_offset);
      const ValueType old_column_type = old_schema->GetType(old_column_id);
      old_column_types.push_back(old_column_type);
      const bool old_is_inlined = old_schema->IsInlined(old_column_id);
      old_is_inlineds.push_back(old_is_inlined);

      // Old to new column mapping
      auto it = old_to_new_cols.find(old_col_id);
      assert(it != old_to_new_cols.end());

      // Get new column information
      oid_t new_column_id = it->second;
      auto new_schema = dest_tile->GetSchema();
      const size_t new_column_offset = new_schema->GetOffset(new_column_id);
      new_column_offsets.push_back(new_column_offset);
      const bool new_is_inlined = new_schema->IsInlined(new_column_id);
      new_is_inlineds.push_back(new_is_inlined);
      const size_t new_column_length = new_schema->GetAppropriateLength(new_column_id);
      new_column_lengths.push_back(new_column_length);
    }

    assert(new_column_offsets.size() == old_column_ids.size());

    ///////////////////////////
    // EACH TUPLE
    ///////////////////////////
    // Copy all values in the tuple to the physical tile
    // This uses fast getter and setter functions
    for (oid_t old_tuple_id : *source_tile) {

      ///////////////////////////
      // EACH COLUMN
      ///////////////////////////
      // Go over each column in given base physical tile
      oid_t col_itr = 0;

      for (oid_t old_col_id : old_column_position_idxs) {

        auto& column_position_list = column_position_lists[old_col_id];

        oid_t base_tuple_id = column_position_list[old_tuple_id];

        auto value = old_tiles[col_itr]->GetValueFast(base_tuple_id,
                                                      old_column_offsets[col_itr],
                                                      old_column_types[col_itr],
                                                      old_is_inlineds[col_itr]);

        LOG_TRACE("Old Tuple : %u Column : %u \n", old_tuple_id, old_col_id);
        LOG_TRACE("New Tuple : %u Column : %u \n", new_tuple_id, new_column_id);

        dest_tile->SetValueFast(value, new_tuple_id,
                                new_column_offsets[col_itr],
                                new_is_inlineds[col_itr],
                                new_column_lengths[col_itr]);

        // Go to next column
        col_itr++;
      }

      // Go to next tuple
      new_tuple_id++;
    }

  }

}

void MaterializeColumnAtATime(LogicalTile *source_tile,
                              const std::unordered_map<oid_t, oid_t> &old_to_new_cols,
                              const std::unordered_map<storage::Tile *, std::vector<oid_t>> &tile_to_cols,
                              storage::Tile *dest_tile) {

  ///////////////////////////
  // EACH PHYSICAL TILE
  ///////////////////////////
  // Copy over all data from each base tile.
  for (const auto &kv : tile_to_cols) {
    const std::vector<oid_t> &old_column_ids = kv.second;

    ///////////////////////////
    // EACH COLUMN
    ///////////////////////////
    // Go over each column in given base physical tile
    for (oid_t old_col_id : old_column_ids) {
      auto &column_info = source_tile->GetColumnInfo(old_col_id);

      // Amortize schema lookups once per column
      storage::Tile *old_tile = column_info.base_tile;
      auto old_schema = old_tile->GetSchema();

      // Get old column information
      oid_t old_column_id = column_info.origin_column_id;
      const size_t old_column_offset = old_schema->GetOffset(old_column_id);
      const ValueType old_column_type = old_schema->GetType(old_column_id);
      const bool old_is_inlined = old_schema->IsInlined(old_column_id);

      // Old to new column mapping
      auto it = old_to_new_cols.find(old_col_id);
      assert(it != old_to_new_cols.end());

      // Get new column information
      oid_t new_column_id = it->second;
      auto new_schema = dest_tile->GetSchema();
      const size_t new_column_offset = new_schema->GetOffset(new_column_id);
      const bool new_is_inlined = new_schema->IsInlined(new_column_id);
      const size_t new_column_length = new_schema->GetAppropriateLength(new_column_id);

      // Get the position list
      auto& column_position_list = source_tile->GetPositionList(column_info.position_list_idx);
      oid_t new_tuple_id = 0;

      // Copy all values in the column to the physical tile
      // This uses fast getter and setter functions
      ///////////////////////////
      // EACH TUPLE
      ///////////////////////////
      for (oid_t old_tuple_id : *source_tile) {
        oid_t base_tuple_id = column_position_list[old_tuple_id];
        auto value = old_tile->GetValueFast(base_tuple_id,
                                            old_column_offset,
                                            old_column_type,
                                            old_is_inlined);

        LOG_TRACE("Old Tuple : %u Column : %u \n", old_tuple_id, old_col_id);
        LOG_TRACE("New Tuple : %u Column : %u \n", new_tuple_id, new_column_id);

        dest_tile->SetValueFast(value, new_tuple_id,
                                new_column_offset,
                                new_is_inlined,
                                new_column_length);

        // Go to next tuple
        new_tuple_id++;
      }

    }

  }

}

std::unordered_map<oid_t, oid_t> MaterializationExecutor::BuildIdentityMapping(
    const catalog::Schema *schema) {
  std::unordered_map<oid_t, oid_t> old_to_new_cols;
  oid_t column_count = schema->GetColumnCount();
  for (oid_t col = 0; col < column_count; col++) {
    old_to_new_cols[col] = col;
  }

  return old_to_new_cols;
}

/**
 * @brief Create a physical tile for the given logical tile
 * @param source_tile Source tile from which the physical tile is created
 * @return a logical tile wrapper for the created physical tile
 */
LogicalTile *MaterializationExecutor::Physify(LogicalTile *source_tile) {
  std::unique_ptr<catalog::Schema> source_tile_schema(
      source_tile->GetPhysicalSchema());
  const int num_tuples = source_tile->GetTupleCount();
  const catalog::Schema *output_schema;
  std::unordered_map<oid_t, oid_t> old_to_new_cols;
  auto node = GetRawNode();

  // Create a default identity mapping node if we did not get one
  if (node == nullptr) {
    assert(source_tile_schema.get());
    output_schema = source_tile_schema.get();

    old_to_new_cols = BuildIdentityMapping(output_schema);
  }
  // Else use the mapping in the given plan node
  else {
    const planner::MaterializationPlan &node =
        GetPlanNode<planner::MaterializationPlan>();
    if (node.GetSchema()) {
      output_schema = node.GetSchema();
      old_to_new_cols = node.old_to_new_cols();
    } else {
      output_schema = source_tile_schema.get();
      old_to_new_cols = BuildIdentityMapping(output_schema);
    }
  }

  // Generate mappings.
  std::unordered_map<storage::Tile *, std::vector<oid_t>> tile_to_cols;
  GenerateTileToColMap(old_to_new_cols, source_tile, tile_to_cols);

  // Create new physical tile.
  std::unique_ptr<storage::Tile> dest_tile(
      storage::TileFactory::GetTempTile(*output_schema, num_tuples));

  // Proceed to materialize logical tile by physical tile at a time.
  MaterializeByTiles(source_tile, old_to_new_cols, tile_to_cols,
                     dest_tile.get());

  // Wrap physical tile in logical tile.
  return LogicalTileFactory::WrapTiles({dest_tile.release()});
}

/**
 * @brief Creates materialized physical tile from logical tile and wraps it
 *        in a new logical tile.
 *
 * @return true on success, false otherwise.
 */
bool MaterializationExecutor::DExecute() {
  // Retrieve child tile.
  const bool success = children_[0]->Execute();
  if (!success) {
    return false;
  }

  std::unique_ptr<LogicalTile> source_tile(children_[0]->GetOutput());
  LogicalTile *output_tile = nullptr;

  // Check the number of tuples in input logical tile
  // If none, then just return false
  const int num_tuples = source_tile->GetTupleCount();
  if (num_tuples == 0) {
    return false;
  }

  auto node = GetRawNode();
  bool physify_flag = true;  // by default, we create a physical tile

  if (node != nullptr) {
    const planner::MaterializationPlan &node =
        GetPlanNode<planner::MaterializationPlan>();
    physify_flag = node.GetPhysifyFlag();
  }

  if (physify_flag) {
    /* create a physical tile and a logical tile wrapper to be the output */
    output_tile = Physify(source_tile.get());
  } else {
    /* just pass thru the underlying logical tile */
    output_tile = source_tile.release();
  }

  SetOutput(output_tile);

  return true;
}

}  // namespace executor
}  // namespace peloton
<|endoftext|>


<commit_before>
#ifndef __BUFFER_CACHE_WRITEBACK_HPP__
#define __BUFFER_CACHE_WRITEBACK_HPP__

#include "concurrency/rwi_lock.hpp"
#include "utils.hpp"

template<class mc_config_t>
struct writeback_tmpl_t :
    public lock_available_callback_t,
    public mc_config_t::serializer_t::write_txn_callback_t
{
    typedef mc_cache_t<mc_config_t> cache_t;
    typedef mc_buf_t<mc_config_t> buf_t;
    typedef mc_transaction_t<mc_config_t> transaction_t;
    typedef mc_transaction_begin_callback_t<mc_config_t> transaction_begin_callback_t;
    typedef mc_transaction_commit_callback_t<mc_config_t> transaction_commit_callback_t;

public:
    writeback_tmpl_t(
        cache_t *cache,
        bool wait_for_flush,
        unsigned int flush_timer_ms,
        unsigned int flush_threshold);
    virtual ~writeback_tmpl_t();

    struct sync_callback_t : public intrusive_list_node_t<sync_callback_t> {
        virtual ~sync_callback_t() {}
        virtual void on_sync() = 0;
    };

    void start();

    /* Forces a writeback to happen soon. If there is nothing to write, return 'true'; otherwise,
    returns 'false' and calls 'callback' as soon as the next writeback cycle is over. */
    bool sync(sync_callback_t *callback);

    /* Same as sync(), but doesn't hurry up the writeback in any way. */
    bool sync_patiently(sync_callback_t *callback);

    bool begin_transaction(transaction_t *txn, transaction_begin_callback_t *cb);
    void on_transaction_commit(transaction_t *txn);

    // This is called by buf_t when the OS informs the buf that a write operation completed
    void buf_was_written(buf_t *buf);

    unsigned int num_dirty_blocks();

    class local_buf_t : public intrusive_list_node_t<local_buf_t> {

        friend class writeback_tmpl_t;

    public:
        explicit local_buf_t(buf_t *gbuf)
            : gbuf(gbuf), dirty(false) {}

        void set_dirty(bool _dirty = true);

        bool safe_to_unload() const { return !dirty; }

    private:
        buf_t *gbuf;
#ifndef NDEBUG
    public:
#endif
        bool dirty;
    };

    /* User-controlled settings. */

    bool wait_for_flush;
    int flush_timer_ms;
    unsigned int flush_threshold;   // Number of blocks, not percentage

private:
    // The writeback system has a mechanism to keep data safe if the server crashes. If modified
    // data sits in memory for longer than flush_timer_ms milliseconds, a writeback will be
    // automatically started to store it on disk. flush_timer is the timer to keep track of how
    // much longer the data can sit in memory.
    event_queue_t::timer_t *flush_timer;
    static void flush_timer_callback(void *ctx);

    virtual void on_lock_available();
    void on_serializer_write_txn();

    // Start or continue a writeback. Should only be called by the writeback itself. Returns 'true'
    // if the writeback completes, or 'false' if it is waiting for something and will complete
    // later.
    bool next_writeback_step();

    /* Internal variables used at all times. */

    enum state_t {
        state_unstarted,
        state_ready,
        state_locking,
        state_locked,
        state_write_bufs,
        state_cleanup
    } state;

    cache_t *cache;

    /* The flush lock is necessary because if we acquire dirty blocks
     * in random order during the flush, there might be a deadlock
     * with set_fsm. When the flush is initiated, the flush_lock is
     * grabbed, all write transactions are drained, and no new write transactions
     * are allowed in the meantime. Once all transactions are drained,
     * the writeback code locks all dirty blocks for reading and
     * releases the flush lock. */

    /* Note that the write transactions lock the flush lock for reading, because there can be more
    than one write transaction proceeding at the same time (not on the same bufs, but the bufs'
    locks will take care of that) while the writeback locks the flush lock for writing, because it
    must exclude all of the transactions.
    */

    rwi_lock_t *flush_lock;

    // List of things waiting for their data to be written to disk. They will be called back after
    // the next complete writeback cycle completes.
    intrusive_list_t<sync_callback_t> sync_callbacks;

    // If something requests a sync but a sync is already in progress, then
    // start_next_sync_immediately is set so that a new sync operation is started as soon as the
    // old one finishes. Note that start_next_sync_immediately being true is not equivalent to
    // sync_callbacks being nonempty, because when wait_for_flush is set, transactions will sit
    // patiently in sync_callbacks without setting start_next_sync_immediately.
    bool start_next_sync_immediately;

    // List of bufs that are currenty dirty
    intrusive_list_t<local_buf_t> dirty_bufs;

    /* Internal variables used only during a flush operation. */

    // Transaction that the writeback is using to grab buffers
    transaction_t *transaction;

    // List of things to call back as soon as the writeback currently in progress is over.
    intrusive_list_t<sync_callback_t> current_sync_callbacks;
};

#include "writeback.tcc"

#endif // __BUFFER_CACHE_WRITEBACK_HPP__

<commit_msg>Fix compiling in release mode *sheepish look.<commit_after>
#ifndef __BUFFER_CACHE_WRITEBACK_HPP__
#define __BUFFER_CACHE_WRITEBACK_HPP__

#include "concurrency/rwi_lock.hpp"
#include "utils.hpp"

template<class mc_config_t>
struct writeback_tmpl_t :
    public lock_available_callback_t,
    public mc_config_t::serializer_t::write_txn_callback_t
{
    typedef mc_cache_t<mc_config_t> cache_t;
    typedef mc_buf_t<mc_config_t> buf_t;
    typedef mc_transaction_t<mc_config_t> transaction_t;
    typedef mc_transaction_begin_callback_t<mc_config_t> transaction_begin_callback_t;
    typedef mc_transaction_commit_callback_t<mc_config_t> transaction_commit_callback_t;

public:
    writeback_tmpl_t(
        cache_t *cache,
        bool wait_for_flush,
        unsigned int flush_timer_ms,
        unsigned int flush_threshold);
    virtual ~writeback_tmpl_t();

    struct sync_callback_t : public intrusive_list_node_t<sync_callback_t> {
        virtual ~sync_callback_t() {}
        virtual void on_sync() = 0;
    };

    void start();

    /* Forces a writeback to happen soon. If there is nothing to write, return 'true'; otherwise,
    returns 'false' and calls 'callback' as soon as the next writeback cycle is over. */
    bool sync(sync_callback_t *callback);

    /* Same as sync(), but doesn't hurry up the writeback in any way. */
    bool sync_patiently(sync_callback_t *callback);

    bool begin_transaction(transaction_t *txn, transaction_begin_callback_t *cb);
    void on_transaction_commit(transaction_t *txn);

    // This is called by buf_t when the OS informs the buf that a write operation completed
    void buf_was_written(buf_t *buf);

    unsigned int num_dirty_blocks();

    class local_buf_t : public intrusive_list_node_t<local_buf_t> {

        friend class writeback_tmpl_t;

    public:
        explicit local_buf_t(buf_t *gbuf)
            : gbuf(gbuf), dirty(false) {}

        void set_dirty(bool _dirty = true);

        bool safe_to_unload() const { return !dirty; }

    private:
        buf_t *gbuf;
    public: //TODO make this private again @jdoliner
        bool dirty;
    };

    /* User-controlled settings. */

    bool wait_for_flush;
    int flush_timer_ms;
    unsigned int flush_threshold;   // Number of blocks, not percentage

private:
    // The writeback system has a mechanism to keep data safe if the server crashes. If modified
    // data sits in memory for longer than flush_timer_ms milliseconds, a writeback will be
    // automatically started to store it on disk. flush_timer is the timer to keep track of how
    // much longer the data can sit in memory.
    event_queue_t::timer_t *flush_timer;
    static void flush_timer_callback(void *ctx);

    virtual void on_lock_available();
    void on_serializer_write_txn();

    // Start or continue a writeback. Should only be called by the writeback itself. Returns 'true'
    // if the writeback completes, or 'false' if it is waiting for something and will complete
    // later.
    bool next_writeback_step();

    /* Internal variables used at all times. */

    enum state_t {
        state_unstarted,
        state_ready,
        state_locking,
        state_locked,
        state_write_bufs,
        state_cleanup
    } state;

    cache_t *cache;

    /* The flush lock is necessary because if we acquire dirty blocks
     * in random order during the flush, there might be a deadlock
     * with set_fsm. When the flush is initiated, the flush_lock is
     * grabbed, all write transactions are drained, and no new write transactions
     * are allowed in the meantime. Once all transactions are drained,
     * the writeback code locks all dirty blocks for reading and
     * releases the flush lock. */

    /* Note that the write transactions lock the flush lock for reading, because there can be more
    than one write transaction proceeding at the same time (not on the same bufs, but the bufs'
    locks will take care of that) while the writeback locks the flush lock for writing, because it
    must exclude all of the transactions.
    */

    rwi_lock_t *flush_lock;

    // List of things waiting for their data to be written to disk. They will be called back after
    // the next complete writeback cycle completes.
    intrusive_list_t<sync_callback_t> sync_callbacks;

    // If something requests a sync but a sync is already in progress, then
    // start_next_sync_immediately is set so that a new sync operation is started as soon as the
    // old one finishes. Note that start_next_sync_immediately being true is not equivalent to
    // sync_callbacks being nonempty, because when wait_for_flush is set, transactions will sit
    // patiently in sync_callbacks without setting start_next_sync_immediately.
    bool start_next_sync_immediately;

    // List of bufs that are currenty dirty
    intrusive_list_t<local_buf_t> dirty_bufs;

    /* Internal variables used only during a flush operation. */

    // Transaction that the writeback is using to grab buffers
    transaction_t *transaction;

    // List of things to call back as soon as the writeback currently in progress is over.
    intrusive_list_t<sync_callback_t> current_sync_callbacks;
};

#include "writeback.tcc"

#endif // __BUFFER_CACHE_WRITEBACK_HPP__

<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/autofill/autofill_download.h"

#include <algorithm>
#include <ostream>
#include <vector>

#include "base/logging.h"
#include "base/rand_util.h"
#include "base/stl_util.h"
#include "base/string_util.h"
#include "chrome/browser/autofill/autofill_metrics.h"
#include "chrome/browser/autofill/autofill_xml_parser.h"
#include "chrome/browser/autofill/form_structure.h"
#include "chrome/browser/prefs/pref_service.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/common/pref_names.h"
#include "content/public/common/url_fetcher.h"
#include "googleurl/src/gurl.h"
#include "net/http/http_response_headers.h"
#include "third_party/libjingle/source/talk/xmllite/xmlparser.h"

namespace {
const char kAutofillQueryServerRequestUrl[] =
    "https://toolbarqueries.google.com/tbproxy/af/query";
const char kAutofillUploadServerRequestUrl[] =
    "https://toolbarqueries.google.com/tbproxy/af/upload";
const char kAutofillQueryServerNameStartInHeader[] = "GFE/";

const size_t kMaxFormCacheSize = 16;
};

struct AutofillDownloadManager::FormRequestData {
  std::vector<std::string> form_signatures;
  AutofillRequestType request_type;
};

AutofillDownloadManager::AutofillDownloadManager(Profile* profile,
                                                 Observer* observer)
    : profile_(profile),
      observer_(observer),
      max_form_cache_size_(kMaxFormCacheSize),
      next_query_request_(base::Time::Now()),
      next_upload_request_(base::Time::Now()),
      positive_upload_rate_(0),
      negative_upload_rate_(0),
      fetcher_id_for_unittest_(0) {
  DCHECK(observer_);
  PrefService* preferences = profile_->GetPrefs();
  positive_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillPositiveUploadRate);
  negative_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillNegativeUploadRate);
}

AutofillDownloadManager::~AutofillDownloadManager() {
  STLDeleteContainerPairFirstPointers(url_fetchers_.begin(),
                                      url_fetchers_.end());
}

bool AutofillDownloadManager::StartQueryRequest(
    const std::vector<FormStructure*>& forms,
    const AutofillMetrics& metric_logger) {
  if (next_query_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    return false;
  }
  std::string form_xml;
  FormRequestData request_data;
  if (!FormStructure::EncodeQueryRequest(forms, &request_data.form_signatures,
                                         &form_xml)) {
    return false;
  }

  request_data.request_type = AutofillDownloadManager::REQUEST_QUERY;
  metric_logger.LogServerQueryMetric(AutofillMetrics::QUERY_SENT);

  std::string query_data;
  if (CheckCacheForQueryRequest(request_data.form_signatures, &query_data)) {
    DVLOG(1) << "AutofillDownloadManager: query request has been retrieved from"
             << "the cache";
    observer_->OnLoadedServerPredictions(query_data);
    return true;
  }

  return StartRequest(form_xml, request_data);
}

bool AutofillDownloadManager::StartUploadRequest(
    const FormStructure& form,
    bool form_was_autofilled,
    const FieldTypeSet& available_field_types) {
  if (next_upload_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    DVLOG(1) << "AutofillDownloadManager: Upload request is throttled.";
    return false;
  }

  // Flip a coin to see if we should upload this form.
  double upload_rate = form_was_autofilled ? GetPositiveUploadRate() :
                                             GetNegativeUploadRate();
  if (form.upload_required() == UPLOAD_NOT_REQUIRED ||
      (form.upload_required() == USE_UPLOAD_RATES &&
       base::RandDouble() > upload_rate)) {
    DVLOG(1) << "AutofillDownloadManager: Upload request is ignored.";
    // If we ever need notification that upload was skipped, add it here.
    return false;
  }

  std::string form_xml;
  if (!form.EncodeUploadRequest(available_field_types, form_was_autofilled,
                                &form_xml))
    return false;

  FormRequestData request_data;
  request_data.form_signatures.push_back(form.FormSignature());
  request_data.request_type = AutofillDownloadManager::REQUEST_UPLOAD;

  return StartRequest(form_xml, request_data);
}

double AutofillDownloadManager::GetPositiveUploadRate() const {
  return positive_upload_rate_;
}

double AutofillDownloadManager::GetNegativeUploadRate() const {
  return negative_upload_rate_;
}

void AutofillDownloadManager::SetPositiveUploadRate(double rate) {
  if (rate == positive_upload_rate_)
    return;
  positive_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefService* preferences = profile_->GetPrefs();
  preferences->SetDouble(prefs::kAutofillPositiveUploadRate, rate);
}

void AutofillDownloadManager::SetNegativeUploadRate(double rate) {
  if (rate == negative_upload_rate_)
    return;
  negative_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefService* preferences = profile_->GetPrefs();
  preferences->SetDouble(prefs::kAutofillNegativeUploadRate, rate);
}

bool AutofillDownloadManager::StartRequest(
    const std::string& form_xml,
    const FormRequestData& request_data) {
  net::URLRequestContextGetter* request_context = profile_->GetRequestContext();
  DCHECK(request_context);
  std::string request_url;
  if (request_data.request_type == AutofillDownloadManager::REQUEST_QUERY)
    request_url = kAutofillQueryServerRequestUrl;
  else
    request_url = kAutofillUploadServerRequestUrl;

  // Id is ignored for regular chrome, in unit test id's for fake fetcher
  // factory will be 0, 1, 2, ...
  content::URLFetcher* fetcher = content::URLFetcher::Create(
      fetcher_id_for_unittest_++, GURL(request_url), content::URLFetcher::POST,
      this);
  url_fetchers_[fetcher] = request_data;
  fetcher->SetAutomaticallyRetryOn5xx(false);
  fetcher->SetRequestContext(request_context);
  fetcher->SetUploadData("text/plain", form_xml);
  fetcher->Start();
  return true;
}

void AutofillDownloadManager::CacheQueryRequest(
    const std::vector<std::string>& forms_in_query,
    const std::string& query_data) {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, move to the first position and return.
      std::pair<std::string, std::string> data = *it;
      cached_forms_.erase(it);
      cached_forms_.push_front(data);
      return;
    }
  }
  std::pair<std::string, std::string> data;
  data.first = signature;
  data.second = query_data;
  cached_forms_.push_front(data);
  while (cached_forms_.size() > max_form_cache_size_)
    cached_forms_.pop_back();
}

bool AutofillDownloadManager::CheckCacheForQueryRequest(
    const std::vector<std::string>& forms_in_query,
    std::string* query_data) const {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::const_iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, fill the data and return.
      *query_data = it->second;
      return true;
    }
  }
  return false;
}

std::string AutofillDownloadManager::GetCombinedSignature(
    const std::vector<std::string>& forms_in_query) const {
  size_t total_size = forms_in_query.size();
  for (size_t i = 0; i < forms_in_query.size(); ++i)
    total_size += forms_in_query[i].length();
  std::string signature;

  signature.reserve(total_size);

  for (size_t i = 0; i < forms_in_query.size(); ++i) {
    if (i)
      signature.append(",");
    signature.append(forms_in_query[i]);
  }
  return signature;
}

void AutofillDownloadManager::OnURLFetchComplete(
    const content::URLFetcher* source) {
  std::map<content::URLFetcher *, FormRequestData>::iterator it =
      url_fetchers_.find(const_cast<content::URLFetcher*>(source));
  if (it == url_fetchers_.end()) {
    // Looks like crash on Mac is possibly caused with callback entering here
    // with unknown fetcher when network is refreshed.
    return;
  }
  std::string type_of_request(
      it->second.request_type == AutofillDownloadManager::REQUEST_QUERY ?
          "query" : "upload");
  const int kHttpResponseOk = 200;
  const int kHttpInternalServerError = 500;
  const int kHttpBadGateway = 502;
  const int kHttpServiceUnavailable = 503;

  CHECK(it->second.form_signatures.size());
  if (source->GetResponseCode() != kHttpResponseOk) {
    bool back_off = false;
    std::string server_header;
    switch (source->GetResponseCode()) {
      case kHttpBadGateway:
        if (!source->GetResponseHeaders()->EnumerateHeader(NULL, "server",
                                                           &server_header) ||
            StartsWithASCII(server_header.c_str(),
                            kAutofillQueryServerNameStartInHeader,
                            false) != 0)
          break;
        // Bad gateway was received from Autofill servers. Fall through to back
        // off.
      case kHttpInternalServerError:
      case kHttpServiceUnavailable:
        back_off = true;
        break;
    }

    if (back_off) {
      base::Time back_off_time(base::Time::Now() + source->GetBackoffDelay());
      if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
        next_query_request_ = back_off_time;
      } else {
        next_upload_request_ = back_off_time;
      }
    }

    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has failed with response "
             << source->GetResponseCode();
    observer_->OnServerRequestError(it->second.form_signatures[0],
                                    it->second.request_type,
                                    source->GetResponseCode());
  } else {
    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has succeeded";
    std::string response_body;
    source->GetResponseAsString(&response_body);
    if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
      CacheQueryRequest(it->second.form_signatures, response_body);
      observer_->OnLoadedServerPredictions(response_body);
    } else {
      double new_positive_upload_rate = 0;
      double new_negative_upload_rate = 0;
      AutofillUploadXmlParser parse_handler(&new_positive_upload_rate,
                                            &new_negative_upload_rate);
      buzz::XmlParser parser(&parse_handler);
      parser.Parse(response_body.data(), response_body.length(), true);
      if (parse_handler.succeeded()) {
        SetPositiveUploadRate(new_positive_upload_rate);
        SetNegativeUploadRate(new_negative_upload_rate);
      }

      observer_->OnUploadedPossibleFieldTypes();
    }
  }
  delete it->first;
  url_fetchers_.erase(it);
}
<commit_msg>Update Autofill server URL to https://clients1.google.com<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/autofill/autofill_download.h"

#include <algorithm>
#include <ostream>
#include <vector>

#include "base/logging.h"
#include "base/rand_util.h"
#include "base/stl_util.h"
#include "base/string_util.h"
#include "chrome/browser/autofill/autofill_metrics.h"
#include "chrome/browser/autofill/autofill_xml_parser.h"
#include "chrome/browser/autofill/form_structure.h"
#include "chrome/browser/prefs/pref_service.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/common/pref_names.h"
#include "content/public/common/url_fetcher.h"
#include "googleurl/src/gurl.h"
#include "net/http/http_response_headers.h"
#include "third_party/libjingle/source/talk/xmllite/xmlparser.h"

namespace {
const char kAutofillQueryServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/query";
const char kAutofillUploadServerRequestUrl[] =
    "https://clients1.google.com/tbproxy/af/upload";
const char kAutofillQueryServerNameStartInHeader[] = "GFE/";

const size_t kMaxFormCacheSize = 16;
};

struct AutofillDownloadManager::FormRequestData {
  std::vector<std::string> form_signatures;
  AutofillRequestType request_type;
};

AutofillDownloadManager::AutofillDownloadManager(Profile* profile,
                                                 Observer* observer)
    : profile_(profile),
      observer_(observer),
      max_form_cache_size_(kMaxFormCacheSize),
      next_query_request_(base::Time::Now()),
      next_upload_request_(base::Time::Now()),
      positive_upload_rate_(0),
      negative_upload_rate_(0),
      fetcher_id_for_unittest_(0) {
  DCHECK(observer_);
  PrefService* preferences = profile_->GetPrefs();
  positive_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillPositiveUploadRate);
  negative_upload_rate_ =
      preferences->GetDouble(prefs::kAutofillNegativeUploadRate);
}

AutofillDownloadManager::~AutofillDownloadManager() {
  STLDeleteContainerPairFirstPointers(url_fetchers_.begin(),
                                      url_fetchers_.end());
}

bool AutofillDownloadManager::StartQueryRequest(
    const std::vector<FormStructure*>& forms,
    const AutofillMetrics& metric_logger) {
  if (next_query_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    return false;
  }
  std::string form_xml;
  FormRequestData request_data;
  if (!FormStructure::EncodeQueryRequest(forms, &request_data.form_signatures,
                                         &form_xml)) {
    return false;
  }

  request_data.request_type = AutofillDownloadManager::REQUEST_QUERY;
  metric_logger.LogServerQueryMetric(AutofillMetrics::QUERY_SENT);

  std::string query_data;
  if (CheckCacheForQueryRequest(request_data.form_signatures, &query_data)) {
    DVLOG(1) << "AutofillDownloadManager: query request has been retrieved from"
             << "the cache";
    observer_->OnLoadedServerPredictions(query_data);
    return true;
  }

  return StartRequest(form_xml, request_data);
}

bool AutofillDownloadManager::StartUploadRequest(
    const FormStructure& form,
    bool form_was_autofilled,
    const FieldTypeSet& available_field_types) {
  if (next_upload_request_ > base::Time::Now()) {
    // We are in back-off mode: do not do the request.
    DVLOG(1) << "AutofillDownloadManager: Upload request is throttled.";
    return false;
  }

  // Flip a coin to see if we should upload this form.
  double upload_rate = form_was_autofilled ? GetPositiveUploadRate() :
                                             GetNegativeUploadRate();
  if (form.upload_required() == UPLOAD_NOT_REQUIRED ||
      (form.upload_required() == USE_UPLOAD_RATES &&
       base::RandDouble() > upload_rate)) {
    DVLOG(1) << "AutofillDownloadManager: Upload request is ignored.";
    // If we ever need notification that upload was skipped, add it here.
    return false;
  }

  std::string form_xml;
  if (!form.EncodeUploadRequest(available_field_types, form_was_autofilled,
                                &form_xml))
    return false;

  FormRequestData request_data;
  request_data.form_signatures.push_back(form.FormSignature());
  request_data.request_type = AutofillDownloadManager::REQUEST_UPLOAD;

  return StartRequest(form_xml, request_data);
}

double AutofillDownloadManager::GetPositiveUploadRate() const {
  return positive_upload_rate_;
}

double AutofillDownloadManager::GetNegativeUploadRate() const {
  return negative_upload_rate_;
}

void AutofillDownloadManager::SetPositiveUploadRate(double rate) {
  if (rate == positive_upload_rate_)
    return;
  positive_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefService* preferences = profile_->GetPrefs();
  preferences->SetDouble(prefs::kAutofillPositiveUploadRate, rate);
}

void AutofillDownloadManager::SetNegativeUploadRate(double rate) {
  if (rate == negative_upload_rate_)
    return;
  negative_upload_rate_ = rate;
  DCHECK_GE(rate, 0.0);
  DCHECK_LE(rate, 1.0);
  PrefService* preferences = profile_->GetPrefs();
  preferences->SetDouble(prefs::kAutofillNegativeUploadRate, rate);
}

bool AutofillDownloadManager::StartRequest(
    const std::string& form_xml,
    const FormRequestData& request_data) {
  net::URLRequestContextGetter* request_context = profile_->GetRequestContext();
  DCHECK(request_context);
  std::string request_url;
  if (request_data.request_type == AutofillDownloadManager::REQUEST_QUERY)
    request_url = kAutofillQueryServerRequestUrl;
  else
    request_url = kAutofillUploadServerRequestUrl;

  // Id is ignored for regular chrome, in unit test id's for fake fetcher
  // factory will be 0, 1, 2, ...
  content::URLFetcher* fetcher = content::URLFetcher::Create(
      fetcher_id_for_unittest_++, GURL(request_url), content::URLFetcher::POST,
      this);
  url_fetchers_[fetcher] = request_data;
  fetcher->SetAutomaticallyRetryOn5xx(false);
  fetcher->SetRequestContext(request_context);
  fetcher->SetUploadData("text/plain", form_xml);
  fetcher->Start();
  return true;
}

void AutofillDownloadManager::CacheQueryRequest(
    const std::vector<std::string>& forms_in_query,
    const std::string& query_data) {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, move to the first position and return.
      std::pair<std::string, std::string> data = *it;
      cached_forms_.erase(it);
      cached_forms_.push_front(data);
      return;
    }
  }
  std::pair<std::string, std::string> data;
  data.first = signature;
  data.second = query_data;
  cached_forms_.push_front(data);
  while (cached_forms_.size() > max_form_cache_size_)
    cached_forms_.pop_back();
}

bool AutofillDownloadManager::CheckCacheForQueryRequest(
    const std::vector<std::string>& forms_in_query,
    std::string* query_data) const {
  std::string signature = GetCombinedSignature(forms_in_query);
  for (QueryRequestCache::const_iterator it = cached_forms_.begin();
       it != cached_forms_.end(); ++it) {
    if (it->first == signature) {
      // We hit the cache, fill the data and return.
      *query_data = it->second;
      return true;
    }
  }
  return false;
}

std::string AutofillDownloadManager::GetCombinedSignature(
    const std::vector<std::string>& forms_in_query) const {
  size_t total_size = forms_in_query.size();
  for (size_t i = 0; i < forms_in_query.size(); ++i)
    total_size += forms_in_query[i].length();
  std::string signature;

  signature.reserve(total_size);

  for (size_t i = 0; i < forms_in_query.size(); ++i) {
    if (i)
      signature.append(",");
    signature.append(forms_in_query[i]);
  }
  return signature;
}

void AutofillDownloadManager::OnURLFetchComplete(
    const content::URLFetcher* source) {
  std::map<content::URLFetcher *, FormRequestData>::iterator it =
      url_fetchers_.find(const_cast<content::URLFetcher*>(source));
  if (it == url_fetchers_.end()) {
    // Looks like crash on Mac is possibly caused with callback entering here
    // with unknown fetcher when network is refreshed.
    return;
  }
  std::string type_of_request(
      it->second.request_type == AutofillDownloadManager::REQUEST_QUERY ?
          "query" : "upload");
  const int kHttpResponseOk = 200;
  const int kHttpInternalServerError = 500;
  const int kHttpBadGateway = 502;
  const int kHttpServiceUnavailable = 503;

  CHECK(it->second.form_signatures.size());
  if (source->GetResponseCode() != kHttpResponseOk) {
    bool back_off = false;
    std::string server_header;
    switch (source->GetResponseCode()) {
      case kHttpBadGateway:
        if (!source->GetResponseHeaders()->EnumerateHeader(NULL, "server",
                                                           &server_header) ||
            StartsWithASCII(server_header.c_str(),
                            kAutofillQueryServerNameStartInHeader,
                            false) != 0)
          break;
        // Bad gateway was received from Autofill servers. Fall through to back
        // off.
      case kHttpInternalServerError:
      case kHttpServiceUnavailable:
        back_off = true;
        break;
    }

    if (back_off) {
      base::Time back_off_time(base::Time::Now() + source->GetBackoffDelay());
      if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
        next_query_request_ = back_off_time;
      } else {
        next_upload_request_ = back_off_time;
      }
    }

    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has failed with response "
             << source->GetResponseCode();
    observer_->OnServerRequestError(it->second.form_signatures[0],
                                    it->second.request_type,
                                    source->GetResponseCode());
  } else {
    DVLOG(1) << "AutofillDownloadManager: " << type_of_request
             << " request has succeeded";
    std::string response_body;
    source->GetResponseAsString(&response_body);
    if (it->second.request_type == AutofillDownloadManager::REQUEST_QUERY) {
      CacheQueryRequest(it->second.form_signatures, response_body);
      observer_->OnLoadedServerPredictions(response_body);
    } else {
      double new_positive_upload_rate = 0;
      double new_negative_upload_rate = 0;
      AutofillUploadXmlParser parse_handler(&new_positive_upload_rate,
                                            &new_negative_upload_rate);
      buzz::XmlParser parser(&parse_handler);
      parser.Parse(response_body.data(), response_body.length(), true);
      if (parse_handler.succeeded()) {
        SetPositiveUploadRate(new_positive_upload_rate);
        SetNegativeUploadRate(new_negative_upload_rate);
      }

      observer_->OnUploadedPossibleFieldTypes();
    }
  }
  delete it->first;
  url_fetchers_.erase(it);
}
<|endoftext|>