<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

  This software is distributed WITHOUT ANY WARRANTY; without even
  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbImage.h"
#include "otbDrawLineSpatialObjectListFilter.h"
#include "otbLineSpatialObjectList.h"

#include "otbImageFileReader.h"
#include "otbImageFileWriter.h"

//  Software Guide : BeginCommandLineArgs
//    INPUTS: {QB_Suburb.png}
//    OUTPUTS: {QB_SuburbLSD.png}
//  Software Guide : EndCommandLineArgs

// Software Guide : BeginLatex
//
// This example illustrates the use of the
// \doxygen{otb}{LineSegmentDetector}\cite{LSD}, also known as {\em Lucy in the
// Sky with Diamonds}.
//
// The first step required to use this filter is to include its header file.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet
#include "otbLineSegmentDetector.h"
// Software Guide : EndCodeSnippet

int main( int argc, char * argv[] )
{
  const   char * infname  = argv[1];
  const   char * outfname  = argv[2];

  typedef unsigned char   InputPixelType;
  typedef double   PrecisionType;
  const   unsigned int    Dimension = 2;

// Software Guide : BeginLatex
//
// As usual, we start by defining the types for the input image and
// the image file reader.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet
  typedef otb::Image< InputPixelType,  Dimension >    ImageType;
  typedef otb::ImageFileReader<ImageType>        ReaderType;
// Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We instantiate the reader and set the file name for the input image.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  ReaderType::Pointer reader = ReaderType::New();
  reader->SetFileName(infname);
// Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We define now the type for the segment detector filter. It is
// templated over the input image type and the precision with which
// the coordinates of the detected segments will be given. It is
// recommended to set this precision to a real type. The output of the
// filter will be a list of \doxygen{itk}{LineSpatialObject}s.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::LineSegmentDetector<ImageType,
                                                 PrecisionType> LsdFilterType;

  LsdFilterType::Pointer  lsdFilter = LsdFilterType::New();

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// In order to be able to display the results, we will draw the
// detected segments on top of the input image. For this matter, we
// will use the \doxygen{otb}{DrawLineSpatialObjectListFilter} which
// is templated over the input and output image types.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::DrawLineSpatialObjectListFilter< ImageType,
                                       ImageType > DrawLineListType;
  DrawLineListType::Pointer drawLineFilter =   DrawLineListType::New();

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We can now define the type for the writer, instantiate it and set
// the file name for the output image.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::ImageFileWriter<ImageType>        WriterType;
  WriterType::Pointer writer = WriterType::New();
  writer->SetFileName(outfname);

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We plug the pipeline.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  lsdFilter->SetInput(reader->GetOutput());
  writer->SetInput(drawLineFilter->GetOutput());

  drawLineFilter->SetInput(reader->GetOutput());
  drawLineFilter->SetInputLineSpatialObjectList(lsdFilter->GetOutput());

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// Before calling the \code{Update()} method of the writer in order to
// trigger the pipeline execution, we call the
// \doxygen{GenerateOutputInformation()} of the reader, so the LSD
// filter gets the information about image size and spacing.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  reader->GenerateOutputInformation();
  writer->Update();

  // Software Guide : EndCodeSnippet

  //  Software Guide : BeginLatex
  // Figure~\ref{fig:LSD} shows the result of applying the line segment
  // detection to a small patch extracted from a VHR image.
  // \begin{figure}
  // \center
  // \includegraphics[width=0.35\textwidth]{QB_Suburb.eps}
  // \includegraphics[width=0.35\textwidth]{QB_SuburbLSD.eps}
  // \itkcaption[LSD Application]{Result of applying the
  // \doxygen{otb}{LineSegmentDetector} to a VHR image of a suburb.}
  // \label{fig:LSD}
  // \end{figure}
  //
  //  Software Guide : EndLatex

  return EXIT_SUCCESS;
}

<commit_msg>ENH LSD Example Test : Specify  value to draw the lines over the image<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

  This software is distributed WITHOUT ANY WARRANTY; without even
  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbImage.h"
#include "otbDrawLineSpatialObjectListFilter.h"
#include "otbLineSpatialObjectList.h"

#include "otbImageFileReader.h"
#include "otbImageFileWriter.h"

//  Software Guide : BeginCommandLineArgs
//    INPUTS: {QB_Suburb.png}
//    OUTPUTS: {QB_SuburbLSD.png}
//  Software Guide : EndCommandLineArgs

// Software Guide : BeginLatex
//
// This example illustrates the use of the
// \doxygen{otb}{LineSegmentDetector}\cite{LSD}, also known as {\em Lucy in the
// Sky with Diamonds}.
//
// The first step required to use this filter is to include its header file.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet
#include "otbLineSegmentDetector.h"
// Software Guide : EndCodeSnippet

int main( int argc, char * argv[] )
{
  const   char * infname  = argv[1];
  const   char * outfname  = argv[2];

  typedef unsigned char   InputPixelType;
  typedef double   PrecisionType;
  const   unsigned int    Dimension = 2;

// Software Guide : BeginLatex
//
// As usual, we start by defining the types for the input image and
// the image file reader.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet
  typedef otb::Image< InputPixelType,  Dimension >    ImageType;
  typedef otb::ImageFileReader<ImageType>        ReaderType;
// Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We instantiate the reader and set the file name for the input image.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  ReaderType::Pointer reader = ReaderType::New();
  reader->SetFileName(infname);
// Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We define now the type for the segment detector filter. It is
// templated over the input image type and the precision with which
// the coordinates of the detected segments will be given. It is
// recommended to set this precision to a real type. The output of the
// filter will be a list of \doxygen{itk}{LineSpatialObject}s.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::LineSegmentDetector<ImageType,
                                                 PrecisionType> LsdFilterType;

  LsdFilterType::Pointer  lsdFilter = LsdFilterType::New();

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// In order to be able to display the results, we will draw the
// detected segments on top of the input image. For this matter, we
// will use the \doxygen{otb}{DrawLineSpatialObjectListFilter} which
// is templated over the input and output image types.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::DrawLineSpatialObjectListFilter< ImageType,
                                       ImageType > DrawLineListType;
  DrawLineListType::Pointer drawLineFilter =   DrawLineListType::New();

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We can now define the type for the writer, instantiate it and set
// the file name for the output image.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  typedef otb::ImageFileWriter<ImageType>        WriterType;
  WriterType::Pointer writer = WriterType::New();
  writer->SetFileName(outfname);

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// We plug the pipeline.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  lsdFilter->SetInput(reader->GetOutput());
  writer->SetInput(drawLineFilter->GetOutput());

  drawLineFilter->SetInput(reader->GetOutput());
  drawLineFilter->SetInputLineSpatialObjectList(lsdFilter->GetOutput());
  drawLineFilter->SetValue(1000);

  // Software Guide : EndCodeSnippet
// Software Guide : BeginLatex
//
// Before calling the \code{Update()} method of the writer in order to
// trigger the pipeline execution, we call the
// \doxygen{GenerateOutputInformation()} of the reader, so the LSD
// filter gets the information about image size and spacing.
//
// Software Guide : EndLatex

// Software Guide : BeginCodeSnippet

  reader->GenerateOutputInformation();
  writer->Update();

  // Software Guide : EndCodeSnippet

  //  Software Guide : BeginLatex
  // Figure~\ref{fig:LSD} shows the result of applying the line segment
  // detection to a small patch extracted from a VHR image.
  // \begin{figure}
  // \center
  // \includegraphics[width=0.35\textwidth]{QB_Suburb.eps}
  // \includegraphics[width=0.35\textwidth]{QB_SuburbLSD.eps}
  // \itkcaption[LSD Application]{Result of applying the
  // \doxygen{otb}{LineSegmentDetector} to a VHR image of a suburb.}
  // \label{fig:LSD}
  // \end{figure}
  //
  //  Software Guide : EndLatex

  return EXIT_SUCCESS;
}

<|endoftext|>


<commit_before>#ifndef _DEFINES_H_
#define _DEFINES_H_

#define MAX_ROUNDS 1
#define BOARD_W 10
#define BOARD_H 10

#define ATTACKERS_ALLOC 40
#define DEFENDERS_ALLOC 100

#define ATTACK_HEALTH 200
#define ATTACK_POWER 50
#define ATTACK_SPEED 1

#define DEFENSE_HEALTH  100
#define DEFENSE_POWER 10
#define DEFENSE_SPEED 1

#define PLAYER_HEALTH 1000
#define PLAYER_POWER 1000
#define PLAYER_SPEED 0

// Where the attackers will respawn. This is relative to the board coordinates.
#define RESPAWN_X 0
#define RESPAWN_Y 0

// The player's tower coordinate. The tower NEVER moves.
#define PLAYER_X (BOARD_W - 1)
#define PLAYER_Y (BOARD_H - 1)

#define WINDOW_W 800
#define WINDOW_H 600
#define WINDOW_BPP 32

#define GRID_W ((float)WINDOW_W / BOARD_W)
#define GRID_H ((float)WINDOW_H / BOARD_H)

// Score points.

#define ADD_DESTROY_POINTS 150 	// Give points for each enemy destroyed.
#define ADD_LIFE_POINTS 3 		// Give points for each life point kept.

#define SUB_NUMBER_OF_TOWERS 20	// Take points for each tower put.
#define SUB_RESPAWN_BLOCK 2000	// Penalize for blocking the respawn point with a tower.
#define SUB_BLOCK_PATH 7000	// Penalize for blocking all the paths from respawn to player's tower.
#define SUB_TOWER_DESTROYED 4500 // Penalize for losing the game.

// This value depends on the previous values.
#define MAX_VALUE 8760
#define MIN_VALUE -11500

#define POPULATION_SIZE 100
#define CHROM_LENGTH 100

#endif //_DEFINES_H_
<commit_msg>Defense tower attack power decreased from 10 to 8.<commit_after>#ifndef _DEFINES_H_
#define _DEFINES_H_

#define MAX_ROUNDS 1
#define BOARD_W 10
#define BOARD_H 10

#define ATTACKERS_ALLOC 40
#define DEFENDERS_ALLOC 100

#define ATTACK_HEALTH 200
#define ATTACK_POWER 50
#define ATTACK_SPEED 1

#define DEFENSE_HEALTH  100
#define DEFENSE_POWER 8
#define DEFENSE_SPEED 1

#define PLAYER_HEALTH 1000
#define PLAYER_POWER 1000
#define PLAYER_SPEED 0

// Where the attackers will respawn. This is relative to the board coordinates.
#define RESPAWN_X 0
#define RESPAWN_Y 0

// The player's tower coordinate. The tower NEVER moves.
#define PLAYER_X (BOARD_W - 1)
#define PLAYER_Y (BOARD_H - 1)

#define WINDOW_W 800
#define WINDOW_H 600
#define WINDOW_BPP 32

#define GRID_W ((float)WINDOW_W / BOARD_W)
#define GRID_H ((float)WINDOW_H / BOARD_H)

// Score points.

#define ADD_DESTROY_POINTS 150 	// Give points for each enemy destroyed.
#define ADD_LIFE_POINTS 3 		// Give points for each life point kept.

#define SUB_NUMBER_OF_TOWERS 20	// Take points for each tower put.
#define SUB_RESPAWN_BLOCK 2000	// Penalize for blocking the respawn point with a tower.
#define SUB_BLOCK_PATH 7000	// Penalize for blocking all the paths from respawn to player's tower.
#define SUB_TOWER_DESTROYED 4500 // Penalize for losing the game.

// This value depends on the previous values.
#define MAX_VALUE 8760
#define MIN_VALUE -11500

#define POPULATION_SIZE 100
#define CHROM_LENGTH 100

#endif //_DEFINES_H_
<|endoftext|>


<commit_before>/***********************************************
 * Author: Jun Jiang - jiangjun4@sina.com
 * Create: 2017-08-19 18:50
 * Last modified : 2017-08-19 18:50
 * Filename      : DNN.cpp
 * Description   :
 **********************************************/
#include "dnn/DNN.h"
#include "dnn/Layer.h"
#include "utils/Log.h"
#include "utils/Shuffler.h"
#include <vector>

namespace abcdl{
namespace dnn{

void DNN::train(const abcdl::algebra::Mat& train_data,
                const abcdl::algebra::Mat& train_label,
                const abcdl::algebra::Mat& test_data,
                const abcdl::algebra::Mat& test_label){
    LOG(INFO) << "dnn start training...";

    size_t layer_size = _layers.size();
    size_t num_train_data = train_data.rows();

    CHECK(num_train_data == train_label.rows());
    CHECK(train_data.cols() == _layers[0]->get_input_dim());
    CHECK(train_label.cols() == _layers[_layers.size() -1]->get_output_dim());

    abcdl::utils::Shuffler shuffler(num_train_data);
    abcdl::algebra::Mat data;
    abcdl::algebra::Mat label;

    for(size_t i = 0; i != _epoch; i++){
        shuffler.shuffle();
        for(size_t j = 0; j != num_train_data; j++){
            train_data.get_row(&data, shuffler.get_row(j));
            train_label.get_row(&label, shuffler.get_row(j));

            for(size_t k = 0; k != layer_size; k++){
                if(k == 0){
                    ((InputLayer*)_layers[k])->set_x(data);
                }
                _layers[k]->forward(_layers[k-1]);
            }

            for(size_t k = layer_size - 1; k > 0; k--){
                if(k == layer_size - 1){
                    ((OutputLayer*)_layers[k])->set_y(label);
                    _layers[k]->backward(_layers[k-1], nullptr);
                }else{
                    _layers[k]->backward(_layers[k-1], _layers[k+1]);
                }

                //mini_batch_update
                if(j % _batch_size == _batch_size - 1 || j == num_train_data - 1){
                    _layers[k]->update_gradient(j % _batch_size + 1, _alpha, _lamda);
                }
            }

            if(j % 100 == 0){
                printf("Epoch[%ld/%ld] Train[%ld/%ld]\r", i + 1, _epoch, j, num_train_data);
            }
        }

        if(test_data.rows() > 0){
            size_t num = evaluate(test_data, test_label);
            printf("Epoch[%ld][%ld/%ld] rate[%f]\n", i + 1, num, test_data.rows(), num/(real)test_data.rows());
        }
    }
}

size_t DNN::evaluate(const abcdl::algebra::Mat& test_data, const abcdl::algebra::Mat& test_label){
    CHECK(test_data.cols() == _layers[0]->get_input_dim());
    CHECK(test_data.rows() == test_label.rows());
    CHECK(test_label.cols() == _layers[_layers.size() - 1]->get_output_dim());

    size_t rows = test_data.rows();
    size_t predict_num = 0;
    abcdl::algebra::Mat mat;

    for(size_t i = 0; i != rows; i++){
        predict(mat, test_data.get_row(i));
        if(mat.argmax() == test_label.get_row(i).argmax()){
            ++predict_num;
        }
    }
    return predict_num;
}

void DNN::predict(abcdl::algebra::Mat& result, const abcdl::algebra::Mat& predict_data){
    CHECK(predict_data.cols() == _layers[0]->get_input_dim());
    size_t layer_size = _layers.size();
    for(size_t k = 0; k != layer_size; k++){
        if(k == 0){
            ((InputLayer*)_layers[k])->set_x(predict_data);
        }
        _layers[k]->forward(_layers[k-1]);
    }
    result = _layers[layer_size - 1]->get_activate_data();
}

bool DNN::load_model(const std::string& path){
    std::vector<abcdl::algebra::Mat*> models;
    if(!_model_loader.read<real>(path, &models, "DNNMODEL") || models.size() % 2 != 0){
        for(auto& model : models){
            delete model;
        }
        models.clear();
        return false;
    }

    for(auto& layer : _layers){
        delete layer;
    }
    _layers.clear();

    for(size_t i = 0; i != models.size() / 2; i++){
        if(i == 0){
            _layers.push_back(new InputLayer(models[0]->rows()));
        }
        if(i == models.size() / 2 - 1){
            _layers.push_back(new OutputLayer(models[i * 2]->rows(), models[i * 2]->cols(), new SigmoidActivateFunc(), new CrossEntropyCost(), *models[i * 2], *models[i * 2 + 1]));
        }else{
            _layers.push_back(new FullConnLayer(models[i * 2]->rows(), models[i * 2]->cols(), new SigmoidActivateFunc(), *models[i * 2], *models[i * 2 + 1]));
        }
    }

    for(auto& model : models){
        delete model;
    }
    models.clear();
    return true;
}
bool DNN::write_model(const std::string& path){
    std::vector<abcdl::algebra::Mat*> models;
    for(size_t i = 1; i != _layers.size(); i++){
        models.push_back(&_layers[i]->get_weight());
        models.push_back(&_layers[i]->get_bias());
    }
    return _model_loader.write<real>(models, path, "DNNMODEL", false);
}

}//namespace dnn
}//namespace abcdl
<commit_msg>run time stat<commit_after>/***********************************************
 * Author: Jun Jiang - jiangjun4@sina.com
 * Create: 2017-08-19 18:50
 * Last modified : 2017-08-19 18:50
 * Filename      : DNN.cpp
 * Description   :
 **********************************************/
#include "dnn/DNN.h"
#include "dnn/Layer.h"
#include "utils/Log.h"
#include "utils/Shuffler.h"
#include <vector>

namespace abcdl{
namespace dnn{

void DNN::train(const abcdl::algebra::Mat& train_data,
                const abcdl::algebra::Mat& train_label,
                const abcdl::algebra::Mat& test_data,
                const abcdl::algebra::Mat& test_label){
    LOG(INFO) << "dnn start training...";

    size_t layer_size = _layers.size();
    size_t num_train_data = train_data.rows();

    CHECK(num_train_data == train_label.rows());
    CHECK(train_data.cols() == _layers[0]->get_input_dim());
    CHECK(train_label.cols() == _layers[_layers.size() -1]->get_output_dim());

    abcdl::algebra::Mat data;
    abcdl::algebra::Mat label;
    abcdl::utils::Shuffler shuffler(num_train_data);
    auto now = []{return std::chrono::system_clock::now();};

    for(size_t i = 0; i != _epoch; i++){

        shuffler.shuffle();
        auto start_time = now();

        for(size_t j = 0; j != num_train_data; j++){
            train_data.get_row(&data, shuffler.get_row(j));
            train_label.get_row(&label, shuffler.get_row(j));

            for(size_t k = 0; k != layer_size; k++){
                if(k == 0){
                    ((InputLayer*)_layers[k])->set_x(data);
                }
                _layers[k]->forward(_layers[k-1]);
            }

            for(size_t k = layer_size - 1; k > 0; k--){
                if(k == layer_size - 1){
                    ((OutputLayer*)_layers[k])->set_y(label);
                    _layers[k]->backward(_layers[k-1], nullptr);
                }else{
                    _layers[k]->backward(_layers[k-1], _layers[k+1]);
                }

                //mini_batch_update
                if(j % _batch_size == _batch_size - 1 || j == num_train_data - 1){
                    _layers[k]->update_gradient(j % _batch_size + 1, _alpha, _lamda);
                }
            }

            if(j % 100 == 0){
                printf("Epoch[%ld/%ld] Train[%ld/%ld]\r", i + 1, _epoch, j, num_train_data);
            }
        }

        auto train_time = now();
        printf("Epoch [%ld] training run time:[%ld]ms\n", i, std::chrono::duration_cast<std::chrono::milliseconds>(train_time - start_time).count());

        if(test_data.rows() > 0){
            size_t num = evaluate(test_data, test_label);
            printf("Epoch[%ld][%ld/%ld] rate[%f]\n", i + 1, num, test_data.rows(), num/(real)test_data.rows());
    	    printf("Epoch[%ld] predict run time:[%ld]ms\n", i, std::chrono::duration_cast<std::chrono::milliseconds>(now() - train_time).count());
        }
    }
}

size_t DNN::evaluate(const abcdl::algebra::Mat& test_data, const abcdl::algebra::Mat& test_label){
    CHECK(test_data.cols() == _layers[0]->get_input_dim());
    CHECK(test_data.rows() == test_label.rows());
    CHECK(test_label.cols() == _layers[_layers.size() - 1]->get_output_dim());

    size_t rows = test_data.rows();
    size_t predict_num = 0;
    abcdl::algebra::Mat mat;

    for(size_t i = 0; i != rows; i++){
        predict(mat, test_data.get_row(i));
        if(mat.argmax() == test_label.get_row(i).argmax()){
            ++predict_num;
        }
    }
    return predict_num;
}

void DNN::predict(abcdl::algebra::Mat& result, const abcdl::algebra::Mat& predict_data){
    CHECK(predict_data.cols() == _layers[0]->get_input_dim());
    size_t layer_size = _layers.size();
    for(size_t k = 0; k != layer_size; k++){
        if(k == 0){
            ((InputLayer*)_layers[k])->set_x(predict_data);
        }
        _layers[k]->forward(_layers[k-1]);
    }
    result = _layers[layer_size - 1]->get_activate_data();
}

bool DNN::load_model(const std::string& path){
    std::vector<abcdl::algebra::Mat*> models;
    if(!_model_loader.read<real>(path, &models, "DNNMODEL") || models.size() % 2 != 0){
        for(auto& model : models){
            delete model;
        }
        models.clear();
        return false;
    }

    for(auto& layer : _layers){
        delete layer;
    }
    _layers.clear();

    for(size_t i = 0; i != models.size() / 2; i++){
        if(i == 0){
            _layers.push_back(new InputLayer(models[0]->rows()));
        }
        if(i == models.size() / 2 - 1){
            _layers.push_back(new OutputLayer(models[i * 2]->rows(), models[i * 2]->cols(), new SigmoidActivateFunc(), new CrossEntropyCost(), *models[i * 2], *models[i * 2 + 1]));
        }else{
            _layers.push_back(new FullConnLayer(models[i * 2]->rows(), models[i * 2]->cols(), new SigmoidActivateFunc(), *models[i * 2], *models[i * 2 + 1]));
        }
    }

    for(auto& model : models){
        delete model;
    }
    models.clear();
    return true;
}
bool DNN::write_model(const std::string& path){
    std::vector<abcdl::algebra::Mat*> models;
    for(size_t i = 1; i != _layers.size(); i++){
        models.push_back(&_layers[i]->get_weight());
        models.push_back(&_layers[i]->get_bias());
    }
    return _model_loader.write<real>(models, path, "DNNMODEL", false);
}

}//namespace dnn
}//namespace abcdl
<|endoftext|>


<commit_before>#include <catch.hpp>
#include <map>
#include <string>

#include <boost/variant/get.hpp>

#include "../../src/HTMLParser/HTMLTokenizer.hpp"
#include "../../src/HTMLParser/tokens/StartToken.hpp"
#include "../../src/HTMLParser/tokens/DoctypeToken.hpp"
#include "../../src/HTMLParser/tokens/EndToken.hpp"
#include "../../src/HTMLParser/tokens/CommentToken.hpp"
#include "../../src/HTMLParser/tokens/CharacterToken.hpp"

TEST_CASE("HTML tokenization")
{
    std::map<std::wstring, std::wstring> empty_map;
    SECTION("Manually creating tokens")
    {
        SECTION("Manually creating doctype token")
        {
            DoctypeToken doctype_token = DoctypeToken();
            CHECK_FALSE(doctype_token.quirks_required());
            CHECK_FALSE(doctype_token.is_name_set());
            CHECK_FALSE(doctype_token.is_public_identifier_set());
            CHECK_FALSE(doctype_token.is_system_identifier_set());
        }

        SECTION("Manually creating HTML root token")
        {
            StartToken html_token = StartToken();

            CHECK_FALSE(html_token.is_self_closing());
            CHECK(html_token.get_attributes() == empty_map);
            CHECK(html_token.get_tag_name() == L"");
        }
    }

    SECTION("Creating tokens with tokenizer")
    {
        HTMLTokenizer tokenizer;

        SECTION("Creating doctype tokens with tokenizer")
        {
            SECTION("Normal doctype token")
            {
                auto doctype_token =
                    tokenizer.create_token_from_string(L"<!DOCTYPE html>");

                REQUIRE(doctype_token->is_doctype_token());
                CHECK_FALSE(doctype_token->quirks_required());
                CHECK(doctype_token->is_name_set());
                CHECK(doctype_token->get_tag_name() == L"html");
            }

            SECTION("Doctype name not set")
            {
                auto doctype_token =
                    tokenizer.create_token_from_string(L"<!DOCTYPE>");

                REQUIRE(doctype_token->is_doctype_token());
                CHECK(doctype_token->quirks_required());
                CHECK_FALSE(doctype_token->is_name_set());
            }

            SECTION("Correctly handles extra identifiers")
            {
                std::wstring long_doctype = L"<!DOCTYPE HTML PUBLIC "
                    L"\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
                    L"\"http://www.w3.org/TR/html4/loose.dtd\">";
                std::shared_ptr<HTMLToken> doctype_token =
                    tokenizer.create_token_from_string(long_doctype);

                REQUIRE(doctype_token->is_doctype_token());
                CHECK(doctype_token->quirks_required());
                CHECK(doctype_token->get_tag_name() == L"html");
                CHECK(doctype_token->is_name_set());
            }
        }

        SECTION("Creating start tokens with tokenizer")
        {
            std::map<std::wstring, std::wstring> lang_map = {{L"lang", L"en"}};

            SECTION("Creating html root token with tokenizer")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");
                std::map<std::wstring, std::wstring> attributes_map =
                    token->get_attributes();

                REQUIRE(token->is_start_token());
                CHECK_FALSE(token->is_self_closing());
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK(token->get_tag_name() == L"html");
            }

            SECTION("Creating self-closing tag with tokenizer")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"
");

                REQUIRE(token->is_start_token());
                CHECK(token->is_self_closing());
                CHECK(token->get_tag_name() == L"br");
            }

            SECTION("Tag with multiple attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                        L"");
                std::map<std::wstring, std::wstring> attributes_map =
                    token->get_attributes();

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"img");
                CHECK_FALSE(token->is_self_closing());
                CHECK(token->contains_attribute(L"src"));
                CHECK(token->contains_attribute(L"width"));
                CHECK(token->contains_attribute(L"height"));
                CHECK(token->get_attribute_value(L"src") == L"example.png");
                CHECK(token->get_attribute_value(L"width") == L"10");
                CHECK(token->get_attribute_value(L"height") == L"20");
            }

            SECTION("Tag with repeated attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                                L"");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"html");
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK_FALSE(token->get_attribute_value(L"lang") == L"br");
                CHECK_FALSE(token->is_self_closing());
            }

            SECTION("Capitalization in attribute name/value")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"html");
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK_FALSE(token->contains_attribute(L"lAnG"));
                CHECK_FALSE(token->is_self_closing());
            }

            SECTION("Self-closing tag with attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                                L"<area shape=\"circle\"/>");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"area");
                CHECK(token->is_self_closing());
                CHECK(token->contains_attribute(L"shape"));
                CHECK(token->get_attribute_value(L"shape") == L"circle");
            }
        }

        SECTION("Creating end tags with tokenizer")
        {
            std::shared_ptr<HTMLToken> token =
                tokenizer.create_token_from_string(L"
");

            REQUIRE(token->is_end_token());
            CHECK(token->get_tag_name() == L"p");
            CHECK_FALSE(token->is_self_closing());
        }

        SECTION("Creating character tokens with tokenizer")
        {
            std::shared_ptr<HTMLToken> token =
                tokenizer.create_token_from_string(L"a");

            REQUIRE(token->is_char_token());
            CHECK(token->get_char() == L'a');
            CHECK_FALSE(token->is_self_closing());
        }

        SECTION("Creating comment tags with tokenizer")
        {
            SECTION("Normal comment")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                            L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"This is a comment");
            }

            SECTION("Comment with no data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"");
            }

            SECTION("Comment with hyphen in data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test-string");
            }

            SECTION("Comment with hyphen before data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"-Test string");
            }

            SECTION("Comment with two hyphens in the middle")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test--string");
            }

            SECTION("Comment with exclamation mark and hyphens in the middle")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                            L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test--!string");
            }
        }
    }

    SECTION("Tokenizing multi-tag strings")
    {
        HTMLTokenizer tokenizer;

        SECTION("Well-formatted minimal string")
        {
            std::vector<std::shared_ptr<HTMLToken>> tokens =
                tokenizer.tokenize_string(
                    L"<!DOCTYPE html>");

            std::vector<std::shared_ptr<HTMLToken>>::iterator it =
                tokens.begin();
            CHECK((*it)->is_doctype_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"head");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"head");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"body");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"body");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"html");
            CHECK((*it)->is_end_token());
        }

        SECTION("Well-formatted string with character tokens")
        {
            std::vector<std::shared_ptr<HTMLToken>> tokens =
                tokenizer.tokenize_string(
                L"<!DOCTYPE html>Test");

            std::vector<std::shared_ptr<HTMLToken>>::iterator it =
                tokens.begin();
            CHECK((*it)->is_doctype_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"head");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"head");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"body");
            std::advance(it, 1);
            CHECK((*it)->get_char() == L'T');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L'e');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L's');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L't');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"body");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"html");
            CHECK((*it)->is_end_token());
        }
    }
}
<commit_msg>Test tokenizing with newlines inside of tags<commit_after>#include <catch.hpp>
#include <map>
#include <string>

#include <boost/variant/get.hpp>

#include "../../src/HTMLParser/HTMLTokenizer.hpp"
#include "../../src/HTMLParser/tokens/StartToken.hpp"
#include "../../src/HTMLParser/tokens/DoctypeToken.hpp"
#include "../../src/HTMLParser/tokens/EndToken.hpp"
#include "../../src/HTMLParser/tokens/CommentToken.hpp"
#include "../../src/HTMLParser/tokens/CharacterToken.hpp"

TEST_CASE("HTML tokenization")
{
    std::map<std::wstring, std::wstring> empty_map;
    SECTION("Manually creating tokens")
    {
        SECTION("Manually creating doctype token")
        {
            DoctypeToken doctype_token = DoctypeToken();
            CHECK_FALSE(doctype_token.quirks_required());
            CHECK_FALSE(doctype_token.is_name_set());
            CHECK_FALSE(doctype_token.is_public_identifier_set());
            CHECK_FALSE(doctype_token.is_system_identifier_set());
        }

        SECTION("Manually creating HTML root token")
        {
            StartToken html_token = StartToken();

            CHECK_FALSE(html_token.is_self_closing());
            CHECK(html_token.get_attributes() == empty_map);
            CHECK(html_token.get_tag_name() == L"");
        }
    }

    SECTION("Creating tokens with tokenizer")
    {
        HTMLTokenizer tokenizer;

        SECTION("Creating doctype tokens with tokenizer")
        {
            SECTION("Normal doctype token")
            {
                auto doctype_token =
                    tokenizer.create_token_from_string(L"<!DOCTYPE html>");

                REQUIRE(doctype_token->is_doctype_token());
                CHECK_FALSE(doctype_token->quirks_required());
                CHECK(doctype_token->is_name_set());
                CHECK(doctype_token->get_tag_name() == L"html");
            }

            SECTION("Doctype name not set")
            {
                auto doctype_token =
                    tokenizer.create_token_from_string(L"<!DOCTYPE>");

                REQUIRE(doctype_token->is_doctype_token());
                CHECK(doctype_token->quirks_required());
                CHECK_FALSE(doctype_token->is_name_set());
            }

            SECTION("Correctly handles extra identifiers")
            {
                std::wstring long_doctype = L"<!DOCTYPE HTML PUBLIC "
                    L"\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
                    L"\"http://www.w3.org/TR/html4/loose.dtd\">";
                std::shared_ptr<HTMLToken> doctype_token =
                    tokenizer.create_token_from_string(long_doctype);

                REQUIRE(doctype_token->is_doctype_token());
                CHECK(doctype_token->quirks_required());
                CHECK(doctype_token->get_tag_name() == L"html");
                CHECK(doctype_token->is_name_set());
            }
        }

        SECTION("Creating start tokens with tokenizer")
        {
            std::map<std::wstring, std::wstring> lang_map = {{L"lang", L"en"}};

            SECTION("Creating html root token with tokenizer")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");
                std::map<std::wstring, std::wstring> attributes_map =
                    token->get_attributes();

                REQUIRE(token->is_start_token());
                CHECK_FALSE(token->is_self_closing());
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK(token->get_tag_name() == L"html");
            }

            SECTION("Creating self-closing tag with tokenizer")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"
");

                REQUIRE(token->is_start_token());
                CHECK(token->is_self_closing());
                CHECK(token->get_tag_name() == L"br");
            }

            SECTION("Tag with multiple attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                        L"");
                std::map<std::wstring, std::wstring> attributes_map =
                    token->get_attributes();

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"img");
                CHECK_FALSE(token->is_self_closing());
                CHECK(token->contains_attribute(L"src"));
                CHECK(token->contains_attribute(L"width"));
                CHECK(token->contains_attribute(L"height"));
                CHECK(token->get_attribute_value(L"src") == L"example.png");
                CHECK(token->get_attribute_value(L"width") == L"10");
                CHECK(token->get_attribute_value(L"height") == L"20");
            }

            SECTION("Tag with repeated attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                                L"");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"html");
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK_FALSE(token->get_attribute_value(L"lang") == L"br");
                CHECK_FALSE(token->is_self_closing());
            }

            SECTION("Capitalization in attribute name/value")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"html");
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK_FALSE(token->contains_attribute(L"lAnG"));
                CHECK_FALSE(token->is_self_closing());
            }

            SECTION("Self-closing tag with attributes")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                                L"<area shape=\"circle\"/>");

                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"area");
                CHECK(token->is_self_closing());
                CHECK(token->contains_attribute(L"shape"));
                CHECK(token->get_attribute_value(L"shape") == L"circle");
            }

            SECTION("Newline inside of tag")
            {
                auto token =
                    tokenizer.create_token_from_string(L"");
                REQUIRE(token->is_start_token());
                CHECK(token->get_tag_name() == L"html");
                CHECK(token->contains_attribute(L"lang"));
                CHECK(token->get_attribute_value(L"lang") == L"en");
                CHECK_FALSE(token->is_self_closing());
            }
        }

        SECTION("Creating end tags with tokenizer")
        {
            std::shared_ptr<HTMLToken> token =
                tokenizer.create_token_from_string(L"
");

            REQUIRE(token->is_end_token());
            CHECK(token->get_tag_name() == L"p");
            CHECK_FALSE(token->is_self_closing());

            SECTION("Newline inside of tag")
            {
                auto token =
                    tokenizer.create_token_from_string(L"
");
                REQUIRE(token->is_end_token());
                CHECK(token->get_tag_name() == L"p");
                CHECK_FALSE(token->is_self_closing());
            }
        }

        SECTION("Creating character tokens with tokenizer")
        {
            std::shared_ptr<HTMLToken> token =
                tokenizer.create_token_from_string(L"a");

            REQUIRE(token->is_char_token());
            CHECK(token->get_char() == L'a');
            CHECK_FALSE(token->is_self_closing());
        }

        SECTION("Creating comment tags with tokenizer")
        {
            SECTION("Normal comment")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                            L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"This is a comment");
            }

            SECTION("Normal comment with newline")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                            L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"This is\na comment");
            }

            SECTION("Comment with no data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"");
            }

            SECTION("Comment with hyphen in data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test-string");
            }

            SECTION("Comment with hyphen before data")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"-Test string");
            }

            SECTION("Comment with two hyphens in the middle")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test--string");
            }

            SECTION("Comment with exclamation mark and hyphens in the middle")
            {
                std::shared_ptr<HTMLToken> token =
                    tokenizer.create_token_from_string(
                            L"");

                REQUIRE(token->is_comment_token());
                CHECK(token->get_data() == L"Test--!string");
            }
        }
    }

    SECTION("Tokenizing multi-tag strings")
    {
        HTMLTokenizer tokenizer;

        SECTION("Well-formatted minimal string")
        {
            std::vector<std::shared_ptr<HTMLToken>> tokens =
                tokenizer.tokenize_string(
                    L"<!DOCTYPE html>");

            std::vector<std::shared_ptr<HTMLToken>>::iterator it =
                tokens.begin();
            CHECK((*it)->is_doctype_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"head");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"head");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"body");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"body");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"html");
            CHECK((*it)->is_end_token());
        }

        SECTION("Well-formatted string with character tokens")
        {
            std::vector<std::shared_ptr<HTMLToken>> tokens =
                tokenizer.tokenize_string(
                L"<!DOCTYPE html>Test");

            std::vector<std::shared_ptr<HTMLToken>>::iterator it =
                tokens.begin();
            CHECK((*it)->is_doctype_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"html");
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"head");
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"head");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->is_start_token());
            CHECK((*it)->get_tag_name() == L"body");
            std::advance(it, 1);
            CHECK((*it)->get_char() == L'T');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L'e');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L's');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_char() == L't');
            CHECK((*it)->is_char_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"body");
            CHECK((*it)->is_end_token());
            std::advance(it, 1);
            CHECK((*it)->get_tag_name() == L"html");
            CHECK((*it)->is_end_token());
        }
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp/concurrency/PosixThreadFactory.h>
#include <thrift/lib/cpp/concurrency/Exception.h>
#include <thrift/lib/cpp/concurrency/Mutex.h>
#include <thrift/lib/cpp/concurrency/SpinLock.h>
#include <thrift/lib/cpp/thrift_config.h>

#if GOOGLE_PERFTOOLS_REGISTER_THREAD
#  include "base/Profiler.h"
#endif

#include <assert.h>
#include <pthread.h>
#include <sys/resource.h>

#include <iostream>

#include <boost/weak_ptr.hpp>

#include <glog/logging.h>

namespace apache { namespace thrift { namespace concurrency {

using std::shared_ptr;
using std::weak_ptr;

const PosixThreadFactory::POLICY PosixThreadFactory::kDefaultPolicy;
const PosixThreadFactory::PRIORITY PosixThreadFactory::kDefaultPriority;
const int PosixThreadFactory::kDefaultStackSizeMB;

// global set to track the ids of live threads.
std::set<pthread_t> liveThreadIds;
Mutex liveThreadIdMutex;

void addLiveThreadId(pthread_t tid) {
  Guard g(liveThreadIdMutex);
  liveThreadIds.insert(tid);
}

void removeLiveThreadId(pthread_t tid) {
  Guard g(liveThreadIdMutex);
  liveThreadIds.erase(tid);
}

void getLiveThreadIds(std::set<pthread_t>* tids) {
  Guard g(liveThreadIdMutex);
  for (std::set<pthread_t>::const_iterator it = liveThreadIds.begin();
       it != liveThreadIds.end(); ++it) {
    tids->insert(*it);
  }
}

// push our given name upstream into pthreads
bool PthreadThread::updateName() {
  if (!pthread_ || name_.empty()) {
    return false;
  }
  return setPosixThreadName(pthread_, name_);
}

PthreadThread::PthreadThread(int policy, int priority, int stackSize,
                             bool detached,
                             shared_ptr<Runnable> runnable) :
  pthread_(0),
  state_(uninitialized),
  policy_(policy),
  priority_(priority),
  stackSize_(stackSize),
  detached_(detached) {

  this->Thread::runnable(runnable);
}

PthreadThread::~PthreadThread() {
  /* Nothing references this thread, if is is not detached, do a join
     now, otherwise the thread-id and, possibly, other resources will
     be leaked. */
  if(!detached_) {
    try {
      join();
    } catch(...) {
      // We're really hosed.
    }
  }
  removeLiveThreadId(pthread_);
}

void PthreadThread::start() {
  Guard g(stateLock_);

  if (state_ != uninitialized) {
    return;
  }

  pthread_attr_t thread_attr;
  if (pthread_attr_init(&thread_attr) != 0) {
      throw SystemResourceException("pthread_attr_init failed");
  }

  if(pthread_attr_setdetachstate(&thread_attr,
                                 detached_ ?
                                 PTHREAD_CREATE_DETACHED :
                                 PTHREAD_CREATE_JOINABLE) != 0) {
      throw SystemResourceException("pthread_attr_setdetachstate failed");
  }

  // Set thread stack size
  if (pthread_attr_setstacksize(&thread_attr, MB * stackSize_) != 0) {
    throw SystemResourceException("pthread_attr_setstacksize failed");
  }

  // Create reference
  shared_ptr<PthreadThread>* selfRef = new shared_ptr<PthreadThread>();
  *selfRef = self_.lock();

  state_ = starting;

  if (pthread_create(&pthread_, &thread_attr, threadMain, (void*)selfRef) != 0) {
    throw SystemResourceException("pthread_create failed");
  }

  // Now that we have a thread, we can set the name we've been given, if any.
  updateName();
  addLiveThreadId(pthread_);
}

void PthreadThread::join() {
  Guard g(stateLock_);
  STATE join_state = state_;

  if (!detached_ && join_state != uninitialized) {
    void* ignore;
    /* XXX
       If join fails it is most likely due to the fact
       that the last reference was the thread itself and cannot
       join.  This results in leaked threads and will eventually
       cause the process to run out of thread resources.
       We're beyond the point of throwing an exception.  Not clear how
       best to handle this. */
    int res = pthread_join(pthread_, &ignore);
    detached_ = (res == 0);
    if (res != 0) {
      LOG(ERROR) << "PthreadThread::join(): fail with code " << res;
    }
  } else {
    LOG(ERROR) << "PthreadThread::join(): detached thread";
  }
}

Thread::id_t PthreadThread::getId() {
  return (Thread::id_t)pthread_;
}

shared_ptr<Runnable> PthreadThread::runnable() const {
  return Thread::runnable();
}

void PthreadThread::runnable(shared_ptr<Runnable> value) {
  Thread::runnable(value);
}

void PthreadThread::weakRef(shared_ptr<PthreadThread> self) {
  assert(self.get() == this);
  self_ = weak_ptr<PthreadThread>(self);
}

bool PthreadThread::setName(const std::string& name) {
  Guard g(stateLock_);
  name_ = name;
  return updateName();
}

void* PthreadThread::threadMain(void* arg) {
  shared_ptr<PthreadThread> thread = *(shared_ptr<PthreadThread>*)arg;
  delete reinterpret_cast<shared_ptr<PthreadThread>*>(arg);

  if (thread == nullptr) {
    return (void*)0;
  }

#if GOOGLE_PERFTOOLS_REGISTER_THREAD
  ProfilerRegisterThread();
#endif
  // Using pthread_attr_setschedparam() at thread creation doesn't actually
  // change the new thread's priority for some reason... Other people on the
  // 'net also complain about it.  The solution is to set priority inside the
  // new thread's threadMain.
  if (thread->policy_ == SCHED_FIFO || thread->policy_ == SCHED_RR) {
    struct sched_param sched_param;
    sched_param.sched_priority = thread->priority_;
    int err =
      pthread_setschedparam(pthread_self(), thread->policy_, &sched_param);
    if (err != 0) {
      VLOG(1) << "pthread_setschedparam failed (are you root?) with error " <<
        err, strerror(err);
    }
  } else if (thread->policy_ == SCHED_OTHER) {
    if (setpriority(PRIO_PROCESS, 0, thread->priority_) != 0) {
      VLOG(1) << "setpriority failed (are you root?) with error " <<
        errno, strerror(errno);
    }
  }

  thread->runnable()->run();

  return (void*)0;
}

int PosixThreadFactory::Impl::toPthreadPolicy(POLICY policy) {
  switch (policy) {
  case OTHER:
    return SCHED_OTHER;
  case FIFO:
    return SCHED_FIFO;
  case ROUND_ROBIN:
    return SCHED_RR;
  }
  return SCHED_OTHER;
}

int PosixThreadFactory::Impl::toPthreadPriority(
    POLICY policy, PRIORITY priority) {
  int pthread_policy = toPthreadPolicy(policy);
  int min_priority = 0;
  int max_priority = 0;
  if (pthread_policy == SCHED_FIFO || pthread_policy == SCHED_RR) {
#ifdef THRIFT_HAVE_SCHED_GET_PRIORITY_MIN
    min_priority = sched_get_priority_min(pthread_policy);
#endif
#ifdef THRIFT_HAVE_SCHED_GET_PRIORITY_MAX
    max_priority = sched_get_priority_max(pthread_policy);
#endif
  } else if (pthread_policy == SCHED_OTHER) {
    min_priority = 19;
    max_priority = -20;
  }
  int quanta = HIGHEST - LOWEST;
  float stepsperquanta = (float)(max_priority - min_priority) / quanta;

  if (priority <= HIGHEST) {
    return (int)(min_priority + stepsperquanta * priority);
  } else {
    // should never get here for priority increments.
    assert(false);
    return (int)(min_priority + stepsperquanta * NORMAL);
  }
}

PosixThreadFactory::Impl::Impl(
  POLICY policy, PRIORITY priority, int stackSize, DetachState detached) :
  policy_(policy),
  priority_(priority),
  stackSize_(stackSize),
  detached_(detached) {}

shared_ptr<Thread> PosixThreadFactory::Impl::newThread(
    const shared_ptr<Runnable>& runnable,
    PosixThreadFactory::DetachState detachState) const {
  shared_ptr<PthreadThread> result = shared_ptr<PthreadThread>(
      new PthreadThread(toPthreadPolicy(policy_),
                        toPthreadPriority(policy_, priority_), stackSize_,
                        detachState == DETACHED, runnable));
  result->weakRef(result);
  runnable->thread(result);
  return result;
}

int PosixThreadFactory::Impl::getStackSize() const {
  return stackSize_;
}

void PosixThreadFactory::Impl::setStackSize(int value) {
  stackSize_ = value;
}

PosixThreadFactory::PRIORITY PosixThreadFactory::Impl::getPriority()
    const {
  return priority_;
}

/**
 * Sets priority.
 *
 *  XXX
 *  Need to handle incremental priorities properly.
 */
void PosixThreadFactory::Impl::setPriority(PRIORITY value) {
  priority_ = value;
}

PosixThreadFactory::DetachState PosixThreadFactory::Impl::getDetachState()
    const {
  return detached_;
}

void PosixThreadFactory::Impl::setDetachState(DetachState value) {
  detached_ = value;
}

Thread::id_t PosixThreadFactory::Impl::getCurrentThreadId() const {
  return static_cast<Thread::id_t>(pthread_self());
}

PosixThreadFactory::PosixThreadFactory(POLICY policy, PRIORITY priority,
                                       int stackSize, bool detached)
  : impl_(new PosixThreadFactory::Impl(policy, priority, stackSize,
                                       detached ? DETACHED : ATTACHED)) {
}

PosixThreadFactory::PosixThreadFactory(DetachState detached)
  : impl_(new PosixThreadFactory::Impl(kDefaultPolicy, kDefaultPriority,
                                       kDefaultStackSizeMB, detached)) {
}

shared_ptr<Thread> PosixThreadFactory::newThread(
    const shared_ptr<Runnable>& runnable) const {
  return impl_->newThread(runnable, impl_->getDetachState());
}

shared_ptr<Thread> PosixThreadFactory::newThread(
    const shared_ptr<Runnable>& runnable,
    DetachState detachState) const {
  return impl_->newThread(runnable, detachState);
}

int PosixThreadFactory::getStackSize() const { return impl_->getStackSize(); }

void PosixThreadFactory::setStackSize(int value) { impl_->setStackSize(value); }

PosixThreadFactory::PRIORITY PosixThreadFactory::getPriority() const { return impl_->getPriority(); }

void PosixThreadFactory::setPriority(PosixThreadFactory::PRIORITY value) { impl_->setPriority(value); }

bool PosixThreadFactory::isDetached() const {
  return impl_->getDetachState() == DETACHED;
}

void PosixThreadFactory::setDetached(bool value) {
  impl_->setDetachState(value ? DETACHED : ATTACHED);
}

void PosixThreadFactory::setDetached(DetachState value) {
  impl_->setDetachState(value);
}

Thread::id_t PosixThreadFactory::getCurrentThreadId() const { return impl_->getCurrentThreadId(); }

}}} // apache::thrift::concurrency
<commit_msg>Reduce references to thrift spinlock part 1<commit_after>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp/concurrency/PosixThreadFactory.h>
#include <thrift/lib/cpp/concurrency/Exception.h>
#include <thrift/lib/cpp/concurrency/Mutex.h>
#include <thrift/lib/cpp/thrift_config.h>

#if GOOGLE_PERFTOOLS_REGISTER_THREAD
#  include "base/Profiler.h"
#endif

#include <assert.h>
#include <pthread.h>
#include <sys/resource.h>

#include <iostream>

#include <boost/weak_ptr.hpp>

#include <glog/logging.h>

namespace apache { namespace thrift { namespace concurrency {

using std::shared_ptr;
using std::weak_ptr;

const PosixThreadFactory::POLICY PosixThreadFactory::kDefaultPolicy;
const PosixThreadFactory::PRIORITY PosixThreadFactory::kDefaultPriority;
const int PosixThreadFactory::kDefaultStackSizeMB;

// global set to track the ids of live threads.
std::set<pthread_t> liveThreadIds;
Mutex liveThreadIdMutex;

void addLiveThreadId(pthread_t tid) {
  Guard g(liveThreadIdMutex);
  liveThreadIds.insert(tid);
}

void removeLiveThreadId(pthread_t tid) {
  Guard g(liveThreadIdMutex);
  liveThreadIds.erase(tid);
}

void getLiveThreadIds(std::set<pthread_t>* tids) {
  Guard g(liveThreadIdMutex);
  for (std::set<pthread_t>::const_iterator it = liveThreadIds.begin();
       it != liveThreadIds.end(); ++it) {
    tids->insert(*it);
  }
}

// push our given name upstream into pthreads
bool PthreadThread::updateName() {
  if (!pthread_ || name_.empty()) {
    return false;
  }
  return setPosixThreadName(pthread_, name_);
}

PthreadThread::PthreadThread(int policy, int priority, int stackSize,
                             bool detached,
                             shared_ptr<Runnable> runnable) :
  pthread_(0),
  state_(uninitialized),
  policy_(policy),
  priority_(priority),
  stackSize_(stackSize),
  detached_(detached) {

  this->Thread::runnable(runnable);
}

PthreadThread::~PthreadThread() {
  /* Nothing references this thread, if is is not detached, do a join
     now, otherwise the thread-id and, possibly, other resources will
     be leaked. */
  if(!detached_) {
    try {
      join();
    } catch(...) {
      // We're really hosed.
    }
  }
  removeLiveThreadId(pthread_);
}

void PthreadThread::start() {
  Guard g(stateLock_);

  if (state_ != uninitialized) {
    return;
  }

  pthread_attr_t thread_attr;
  if (pthread_attr_init(&thread_attr) != 0) {
      throw SystemResourceException("pthread_attr_init failed");
  }

  if(pthread_attr_setdetachstate(&thread_attr,
                                 detached_ ?
                                 PTHREAD_CREATE_DETACHED :
                                 PTHREAD_CREATE_JOINABLE) != 0) {
      throw SystemResourceException("pthread_attr_setdetachstate failed");
  }

  // Set thread stack size
  if (pthread_attr_setstacksize(&thread_attr, MB * stackSize_) != 0) {
    throw SystemResourceException("pthread_attr_setstacksize failed");
  }

  // Create reference
  shared_ptr<PthreadThread>* selfRef = new shared_ptr<PthreadThread>();
  *selfRef = self_.lock();

  state_ = starting;

  if (pthread_create(&pthread_, &thread_attr, threadMain, (void*)selfRef) != 0) {
    throw SystemResourceException("pthread_create failed");
  }

  // Now that we have a thread, we can set the name we've been given, if any.
  updateName();
  addLiveThreadId(pthread_);
}

void PthreadThread::join() {
  Guard g(stateLock_);
  STATE join_state = state_;

  if (!detached_ && join_state != uninitialized) {
    void* ignore;
    /* XXX
       If join fails it is most likely due to the fact
       that the last reference was the thread itself and cannot
       join.  This results in leaked threads and will eventually
       cause the process to run out of thread resources.
       We're beyond the point of throwing an exception.  Not clear how
       best to handle this. */
    int res = pthread_join(pthread_, &ignore);
    detached_ = (res == 0);
    if (res != 0) {
      LOG(ERROR) << "PthreadThread::join(): fail with code " << res;
    }
  } else {
    LOG(ERROR) << "PthreadThread::join(): detached thread";
  }
}

Thread::id_t PthreadThread::getId() {
  return (Thread::id_t)pthread_;
}

shared_ptr<Runnable> PthreadThread::runnable() const {
  return Thread::runnable();
}

void PthreadThread::runnable(shared_ptr<Runnable> value) {
  Thread::runnable(value);
}

void PthreadThread::weakRef(shared_ptr<PthreadThread> self) {
  assert(self.get() == this);
  self_ = weak_ptr<PthreadThread>(self);
}

bool PthreadThread::setName(const std::string& name) {
  Guard g(stateLock_);
  name_ = name;
  return updateName();
}

void* PthreadThread::threadMain(void* arg) {
  shared_ptr<PthreadThread> thread = *(shared_ptr<PthreadThread>*)arg;
  delete reinterpret_cast<shared_ptr<PthreadThread>*>(arg);

  if (thread == nullptr) {
    return (void*)0;
  }

#if GOOGLE_PERFTOOLS_REGISTER_THREAD
  ProfilerRegisterThread();
#endif
  // Using pthread_attr_setschedparam() at thread creation doesn't actually
  // change the new thread's priority for some reason... Other people on the
  // 'net also complain about it.  The solution is to set priority inside the
  // new thread's threadMain.
  if (thread->policy_ == SCHED_FIFO || thread->policy_ == SCHED_RR) {
    struct sched_param sched_param;
    sched_param.sched_priority = thread->priority_;
    int err =
      pthread_setschedparam(pthread_self(), thread->policy_, &sched_param);
    if (err != 0) {
      VLOG(1) << "pthread_setschedparam failed (are you root?) with error " <<
        err, strerror(err);
    }
  } else if (thread->policy_ == SCHED_OTHER) {
    if (setpriority(PRIO_PROCESS, 0, thread->priority_) != 0) {
      VLOG(1) << "setpriority failed (are you root?) with error " <<
        errno, strerror(errno);
    }
  }

  thread->runnable()->run();

  return (void*)0;
}

int PosixThreadFactory::Impl::toPthreadPolicy(POLICY policy) {
  switch (policy) {
  case OTHER:
    return SCHED_OTHER;
  case FIFO:
    return SCHED_FIFO;
  case ROUND_ROBIN:
    return SCHED_RR;
  }
  return SCHED_OTHER;
}

int PosixThreadFactory::Impl::toPthreadPriority(
    POLICY policy, PRIORITY priority) {
  int pthread_policy = toPthreadPolicy(policy);
  int min_priority = 0;
  int max_priority = 0;
  if (pthread_policy == SCHED_FIFO || pthread_policy == SCHED_RR) {
#ifdef THRIFT_HAVE_SCHED_GET_PRIORITY_MIN
    min_priority = sched_get_priority_min(pthread_policy);
#endif
#ifdef THRIFT_HAVE_SCHED_GET_PRIORITY_MAX
    max_priority = sched_get_priority_max(pthread_policy);
#endif
  } else if (pthread_policy == SCHED_OTHER) {
    min_priority = 19;
    max_priority = -20;
  }
  int quanta = HIGHEST - LOWEST;
  float stepsperquanta = (float)(max_priority - min_priority) / quanta;

  if (priority <= HIGHEST) {
    return (int)(min_priority + stepsperquanta * priority);
  } else {
    // should never get here for priority increments.
    assert(false);
    return (int)(min_priority + stepsperquanta * NORMAL);
  }
}

PosixThreadFactory::Impl::Impl(
  POLICY policy, PRIORITY priority, int stackSize, DetachState detached) :
  policy_(policy),
  priority_(priority),
  stackSize_(stackSize),
  detached_(detached) {}

shared_ptr<Thread> PosixThreadFactory::Impl::newThread(
    const shared_ptr<Runnable>& runnable,
    PosixThreadFactory::DetachState detachState) const {
  shared_ptr<PthreadThread> result = shared_ptr<PthreadThread>(
      new PthreadThread(toPthreadPolicy(policy_),
                        toPthreadPriority(policy_, priority_), stackSize_,
                        detachState == DETACHED, runnable));
  result->weakRef(result);
  runnable->thread(result);
  return result;
}

int PosixThreadFactory::Impl::getStackSize() const {
  return stackSize_;
}

void PosixThreadFactory::Impl::setStackSize(int value) {
  stackSize_ = value;
}

PosixThreadFactory::PRIORITY PosixThreadFactory::Impl::getPriority()
    const {
  return priority_;
}

/**
 * Sets priority.
 *
 *  XXX
 *  Need to handle incremental priorities properly.
 */
void PosixThreadFactory::Impl::setPriority(PRIORITY value) {
  priority_ = value;
}

PosixThreadFactory::DetachState PosixThreadFactory::Impl::getDetachState()
    const {
  return detached_;
}

void PosixThreadFactory::Impl::setDetachState(DetachState value) {
  detached_ = value;
}

Thread::id_t PosixThreadFactory::Impl::getCurrentThreadId() const {
  return static_cast<Thread::id_t>(pthread_self());
}

PosixThreadFactory::PosixThreadFactory(POLICY policy, PRIORITY priority,
                                       int stackSize, bool detached)
  : impl_(new PosixThreadFactory::Impl(policy, priority, stackSize,
                                       detached ? DETACHED : ATTACHED)) {
}

PosixThreadFactory::PosixThreadFactory(DetachState detached)
  : impl_(new PosixThreadFactory::Impl(kDefaultPolicy, kDefaultPriority,
                                       kDefaultStackSizeMB, detached)) {
}

shared_ptr<Thread> PosixThreadFactory::newThread(
    const shared_ptr<Runnable>& runnable) const {
  return impl_->newThread(runnable, impl_->getDetachState());
}

shared_ptr<Thread> PosixThreadFactory::newThread(
    const shared_ptr<Runnable>& runnable,
    DetachState detachState) const {
  return impl_->newThread(runnable, detachState);
}

int PosixThreadFactory::getStackSize() const { return impl_->getStackSize(); }

void PosixThreadFactory::setStackSize(int value) { impl_->setStackSize(value); }

PosixThreadFactory::PRIORITY PosixThreadFactory::getPriority() const { return impl_->getPriority(); }

void PosixThreadFactory::setPriority(PosixThreadFactory::PRIORITY value) { impl_->setPriority(value); }

bool PosixThreadFactory::isDetached() const {
  return impl_->getDetachState() == DETACHED;
}

void PosixThreadFactory::setDetached(bool value) {
  impl_->setDetachState(value ? DETACHED : ATTACHED);
}

void PosixThreadFactory::setDetached(DetachState value) {
  impl_->setDetachState(value);
}

Thread::id_t PosixThreadFactory::getCurrentThreadId() const { return impl_->getCurrentThreadId(); }

}}} // apache::thrift::concurrency
<|endoftext|>


<commit_before>#include "fetcher.h"

// TODO: Move this to config.h!
#ifdef __linux__
#define USE_OPENSSL
#endif

// TODO: Make a crypto interface and remove ifdefs!
#ifdef USE_OPENSSL
#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/sha.h>
#else
#include <resolv.h>
#include <CommonCrypto/CommonDigest.h>
#include <CommonCrypto/CommonKeyDerivation.h>
#endif

#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xpath.h>

#include <memory>
#include <stdexcept>

namespace lastpass
{

namespace
{

class Xml
{
public:
    explicit Xml(std::string const &text): document_(nullptr)
    {
        document_ = xmlReadMemory(text.c_str(), text.size(), "", nullptr, 0);
        if (document_ == nullptr)
            throw std::runtime_error("Failed to parse XML");
    }

    ~Xml()
    {
        xmlFreeDoc(document_);
    }

    std::string get_attribute(std::string const &xpath) const
    {
        std::unique_ptr<xmlXPathContext, decltype(&xmlXPathFreeContext)>context(
            xmlXPathNewContext(document_),
            &xmlXPathFreeContext);
        if (context.get() == nullptr)
            return "";

        std::unique_ptr<xmlXPathObject, decltype(&xmlXPathFreeObject)>result(
            xmlXPathEvalExpression(reinterpret_cast<xmlChar const *>(xpath.c_str()), context.get()),
            &xmlXPathFreeObject);
        if (result.get() == nullptr)
            return "";

        xmlNodeSet const *nodes = result->nodesetval;
        if (nodes == nullptr ||
            nodes->nodeNr <= 0 ||
            nodes->nodeTab[0]->type != XML_ATTRIBUTE_NODE)
            return "";

        return reinterpret_cast<char const *>(((xmlAttrPtr)nodes->nodeTab[0])->children->content);
    }

private:
    xmlDocPtr document_;
};

}

Session Fetcher::login(std::string const &username, std::string const &password, WebClient &web_client)
{
    return login(username, password, request_iteration_count(username, web_client), web_client);
}

Session Fetcher::login(std::string const &username, std::string const &password, int iteration_count, WebClient &web_client)
{
    auto response = web_client.post("https://lastpass.com/login.php", {
        {"method", "mobile"},
        {"web", "1"},
        {"xml", "1"},
        {"username", username},
        {"hash", make_hash(username, password, iteration_count)},
        {"iterations", std::to_string(iteration_count)}
    });
    Xml xml(response);
    auto id = xml.get_attribute("//ok/@sessionid");

    if (id.empty())
        throw std::runtime_error("Failed to login");

    // TODO: Handle errors here!

    return {id, iteration_count};
}

Blob Fetcher::fetch(Session const &session, WebClient &web_client)
{
    auto response = web_client.get("https://lastpass.com/getaccts.php",
                                   {{"mobile", "1"}, {"b64", "1"}, {"hash", "0.0"}},
                                   {{"PHPSESSID", session.id()}});

    return {to_bytes(response), session.key_iteration_count()};
}

int Fetcher::request_iteration_count(std::string const &username, WebClient &web_client)
{
    return std::stoi(web_client.post("https://lastpass.com/iterations.php", {{"email", username}}));
}

std::vector<uint8_t> Fetcher::make_key(std::string const &username, std::string const &password, int iteration_count)
{
    return iteration_count == 1
        ? sha256(username + password)
        : pbkdf2_sha256(to_bytes(password), to_bytes(username), iteration_count, 32);
}

std::string Fetcher::make_hash(std::string const &username, std::string const &password, int iteration_count)
{
    auto key = make_key(username, password, iteration_count);
    return iteration_count == 1
        ? to_hex(sha256(to_hex(key) + password))
        : to_hex(pbkdf2_sha256(key, to_bytes(password), 1, 32));
}

std::vector<uint8_t> Fetcher::pbkdf2_sha256(std::vector<uint8_t> const &password,
                                            std::vector<uint8_t> const &salt,
                                            int iteration_count,
                                            size_t size)
{
    static_assert(sizeof(uint8_t) == sizeof(char), "uint8_t should be the same size as char");

    std::vector<uint8_t> key(size);

#ifdef USE_OPENSSL
    PKCS5_PBKDF2_HMAC(reinterpret_cast<char const *>(password.data()),
                      password.size(),
                      salt.data(),
                      salt.size(),
                      iteration_count,
                      EVP_sha256(),
                      size,
                      &key[0]);
#else
    CCKeyDerivationPBKDF(kCCPBKDF2,
                         reinterpret_cast<char const *>(password.data()),
                         password.size(),
                         salt.data(),
                         salt.size(),
                         kCCPRFHmacAlgSHA256,
                         iteration_count,
                         &key[0],
                         size);
#endif

    return key;
}

std::vector<uint8_t> Fetcher::sha256(std::string const &text)
{
#ifdef USE_OPENSSL
    std::vector<uint8_t> hash(SHA256_DIGEST_LENGTH);
    SHA256_CTX sha256;
    SHA256_Init(&sha256);
    SHA256_Update(&sha256, text.c_str(), text.size());
    SHA256_Final(&hash[0], &sha256);
#else
    std::vector<uint8_t> hash(CC_SHA256_DIGEST_LENGTH);
    CC_SHA256(text.c_str(), text.size(), &hash[0]);
#endif

    return hash;
}

std::vector<uint8_t> Fetcher::to_bytes(std::string const &text)
{
    std::vector<uint8_t> bytes(text.size());
    std::copy(text.begin(), text.end(), bytes.begin());
    return bytes;
}

std::string Fetcher::to_hex(std::vector<uint8_t> const &bytes)
{
    static char const hex_chars[16] = {'0', '1', '2', '3', '4', '5', '6', '7',
                                       '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};

    std::string hex;
    hex.reserve(bytes.size() * 2);

    for (auto i: bytes)
    {
        hex += hex_chars[i / 16];
        hex += hex_chars[i % 16];
    }

    return hex;
}

std::vector<uint8_t> Fetcher::decode_base64(std::string const &base64_text)
{
    // The size is the upper bound, the actual size could be smaller.
    // After decoding we need to trim unused space.
    std::vector<uint8_t> decoded(base64_text.size() * 3 / 4);

#ifdef USE_OPENSSL
    BIO *context = BIO_push(BIO_new(BIO_f_base64(),
                            BIO_new_mem_buf((void *)base64_text.c_str(), base64_text.size()));
    BIO_set_flags(context, BIO_FLAGS_BASE64_NO_NL | BIO_FLAGS_MEM_RDONLY);
    size_t actual_size = BIO_read(context, decoded.data(), decoded.size());
    BIO_free_all(context);
#else
    size_t actual_size = b64_pton(base64_text.c_str(), decoded.data(), decoded.size());
#endif

    decoded.resize(actual_size);
    return decoded;
}

}
<commit_msg>Another fix for OpenSSL base64<commit_after>#include "fetcher.h"

// TODO: Move this to config.h!
#ifdef __linux__
#define USE_OPENSSL
#endif

// TODO: Make a crypto interface and remove ifdefs!
#ifdef USE_OPENSSL
#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/sha.h>
#else
#include <resolv.h>
#include <CommonCrypto/CommonDigest.h>
#include <CommonCrypto/CommonKeyDerivation.h>
#endif

#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xpath.h>

#include <memory>
#include <stdexcept>

namespace lastpass
{

namespace
{

class Xml
{
public:
    explicit Xml(std::string const &text): document_(nullptr)
    {
        document_ = xmlReadMemory(text.c_str(), text.size(), "", nullptr, 0);
        if (document_ == nullptr)
            throw std::runtime_error("Failed to parse XML");
    }

    ~Xml()
    {
        xmlFreeDoc(document_);
    }

    std::string get_attribute(std::string const &xpath) const
    {
        std::unique_ptr<xmlXPathContext, decltype(&xmlXPathFreeContext)>context(
            xmlXPathNewContext(document_),
            &xmlXPathFreeContext);
        if (context.get() == nullptr)
            return "";

        std::unique_ptr<xmlXPathObject, decltype(&xmlXPathFreeObject)>result(
            xmlXPathEvalExpression(reinterpret_cast<xmlChar const *>(xpath.c_str()), context.get()),
            &xmlXPathFreeObject);
        if (result.get() == nullptr)
            return "";

        xmlNodeSet const *nodes = result->nodesetval;
        if (nodes == nullptr ||
            nodes->nodeNr <= 0 ||
            nodes->nodeTab[0]->type != XML_ATTRIBUTE_NODE)
            return "";

        return reinterpret_cast<char const *>(((xmlAttrPtr)nodes->nodeTab[0])->children->content);
    }

private:
    xmlDocPtr document_;
};

}

Session Fetcher::login(std::string const &username, std::string const &password, WebClient &web_client)
{
    return login(username, password, request_iteration_count(username, web_client), web_client);
}

Session Fetcher::login(std::string const &username, std::string const &password, int iteration_count, WebClient &web_client)
{
    auto response = web_client.post("https://lastpass.com/login.php", {
        {"method", "mobile"},
        {"web", "1"},
        {"xml", "1"},
        {"username", username},
        {"hash", make_hash(username, password, iteration_count)},
        {"iterations", std::to_string(iteration_count)}
    });
    Xml xml(response);
    auto id = xml.get_attribute("//ok/@sessionid");

    if (id.empty())
        throw std::runtime_error("Failed to login");

    // TODO: Handle errors here!

    return {id, iteration_count};
}

Blob Fetcher::fetch(Session const &session, WebClient &web_client)
{
    auto response = web_client.get("https://lastpass.com/getaccts.php",
                                   {{"mobile", "1"}, {"b64", "1"}, {"hash", "0.0"}},
                                   {{"PHPSESSID", session.id()}});

    return {to_bytes(response), session.key_iteration_count()};
}

int Fetcher::request_iteration_count(std::string const &username, WebClient &web_client)
{
    return std::stoi(web_client.post("https://lastpass.com/iterations.php", {{"email", username}}));
}

std::vector<uint8_t> Fetcher::make_key(std::string const &username, std::string const &password, int iteration_count)
{
    return iteration_count == 1
        ? sha256(username + password)
        : pbkdf2_sha256(to_bytes(password), to_bytes(username), iteration_count, 32);
}

std::string Fetcher::make_hash(std::string const &username, std::string const &password, int iteration_count)
{
    auto key = make_key(username, password, iteration_count);
    return iteration_count == 1
        ? to_hex(sha256(to_hex(key) + password))
        : to_hex(pbkdf2_sha256(key, to_bytes(password), 1, 32));
}

std::vector<uint8_t> Fetcher::pbkdf2_sha256(std::vector<uint8_t> const &password,
                                            std::vector<uint8_t> const &salt,
                                            int iteration_count,
                                            size_t size)
{
    static_assert(sizeof(uint8_t) == sizeof(char), "uint8_t should be the same size as char");

    std::vector<uint8_t> key(size);

#ifdef USE_OPENSSL
    PKCS5_PBKDF2_HMAC(reinterpret_cast<char const *>(password.data()),
                      password.size(),
                      salt.data(),
                      salt.size(),
                      iteration_count,
                      EVP_sha256(),
                      size,
                      &key[0]);
#else
    CCKeyDerivationPBKDF(kCCPBKDF2,
                         reinterpret_cast<char const *>(password.data()),
                         password.size(),
                         salt.data(),
                         salt.size(),
                         kCCPRFHmacAlgSHA256,
                         iteration_count,
                         &key[0],
                         size);
#endif

    return key;
}

std::vector<uint8_t> Fetcher::sha256(std::string const &text)
{
#ifdef USE_OPENSSL
    std::vector<uint8_t> hash(SHA256_DIGEST_LENGTH);
    SHA256_CTX sha256;
    SHA256_Init(&sha256);
    SHA256_Update(&sha256, text.c_str(), text.size());
    SHA256_Final(&hash[0], &sha256);
#else
    std::vector<uint8_t> hash(CC_SHA256_DIGEST_LENGTH);
    CC_SHA256(text.c_str(), text.size(), &hash[0]);
#endif

    return hash;
}

std::vector<uint8_t> Fetcher::to_bytes(std::string const &text)
{
    std::vector<uint8_t> bytes(text.size());
    std::copy(text.begin(), text.end(), bytes.begin());
    return bytes;
}

std::string Fetcher::to_hex(std::vector<uint8_t> const &bytes)
{
    static char const hex_chars[16] = {'0', '1', '2', '3', '4', '5', '6', '7',
                                       '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};

    std::string hex;
    hex.reserve(bytes.size() * 2);

    for (auto i: bytes)
    {
        hex += hex_chars[i / 16];
        hex += hex_chars[i % 16];
    }

    return hex;
}

std::vector<uint8_t> Fetcher::decode_base64(std::string const &base64_text)
{
    // The size is the upper bound, the actual size could be smaller.
    // After decoding we need to trim unused space.
    std::vector<uint8_t> decoded(base64_text.size() * 3 / 4);

#ifdef USE_OPENSSL
    BIO *context = BIO_push(BIO_new(BIO_f_base64()),
                            BIO_new_mem_buf((void *)base64_text.c_str(), base64_text.size()));
    BIO_set_flags(context, BIO_FLAGS_BASE64_NO_NL | BIO_FLAGS_MEM_RDONLY);
    size_t actual_size = BIO_read(context, decoded.data(), decoded.size());
    BIO_free_all(context);
#else
    size_t actual_size = b64_pton(base64_text.c_str(), decoded.data(), decoded.size());
#endif

    decoded.resize(actual_size);
    return decoded;
}

}
<|endoftext|>