<commit_before>#include <iostream>
#include <fstream>

#include <cstdlib>
#include <cassert>
#include <cmath>

#include <gsl/gsl_math.h>

#include "dataset.hh"

using std::cerr;
using std::ofstream;

MSZDataSet::MSZDataSet(double **matrix, size_t nrows, size_t ncols, size_t outputs)
  : matrix(matrix), nrows(nrows), rfeatures(ncols - outputs), ncols(ncols), outputs(outputs) {}

MSZDataSet::~MSZDataSet() {

  // Please God, do not let this Seg Fault!

  // Clean up matrix by deleting each line.
  for(size_t row = 0; row < nrows; row++)
    free(matrix[row]);

  free(matrix);

}

void MSZDataSet::dumpPlotFiles(const vector<string> &labels, const string &prefix) const {

  if(labels.size() != ncols) {
    cerr << "Warning: Trying to dump plot files but labels vector is too small.\n";
    return;
  }

  // Dump the files
  for(size_t out = 0; out < outputs; out++) {
    for(size_t feature = outputs; feature < ncols; feature++) {
      ofstream file;
      file.open((prefix + "_" + labels[out] + "_" + labels[feature] + ".dat").c_str());

      file << "# This file is generated by MatSATZilla\n"
	   << "# It should be used with gnuplot to plot\n"
	   << "# the feature " << labels[feature] << " against\n"
	   << "# the output " << labels[out] << "\n"
	   << "#\n"
	// Outputting timestamp could be nice.
	//<< "# Timestamp: " <<
	   << "#\n"
	   << "# " << labels[feature] << "\t\t" << labels[out] << "\n";

      // For each instance
      for(size_t i = 0; i < nrows; i++)
	file << matrix[i][feature] << "\t\t" << matrix[i][out] << "\n";

      file.close();
    }
  }
}

void MSZDataSet::dumpPlotFiles(char **labels, size_t len, char *prefix) const {

  vector<string> vec(len);

  for(size_t i = 0; i < len; i++)
    vec[i] = string(labels[i]);

  dumpPlotFiles(vec, string(prefix));
}

double MSZDataSet::getOutputValue(size_t row, size_t col) const {
  assert(row < nrows);
  assert(col < outputs);

  return matrix[row][col];
}

double MSZDataSet::getFeatureValue(size_t row, size_t col) const {
  assert(row < nrows);
  assert(col < ncols - outputs);

  return matrix[row][col+outputs];
}

void MSZDataSet::standardize() {

  static bool stdDone = false;

  if(!stdDone) {
    for(size_t c = outputs; c < ncols; c++) {

      // Compute column mean and compute column variance.
      double mean = 0.0;
      for(size_t r = 0; r < nrows; r++)
	mean += matrix[r][c];
      mean /= nrows;

      // Compute column standard deviation.
      double sdv = 0.0;
      for(size_t r = 0; r < nrows; r++)
	sdv += gsl_pow_2(matrix[r][c] - mean);
      sdv /= nrows;

      for(size_t r = 0; r < nrows; r++)
	matrix[r][c] = (matrix[r][c] - mean) / sdv;

    }

  }

  stdDone = true;

}

void MSZDataSet::standardizeOutputs() {

  static bool stdDone = false;

  if(!stdDone) {
    for(size_t c = 0; c < outputs; c++)
      for(size_t r = 0; r < nrows; r++)
	matrix[r][c] = log(matrix[r][c]);
  }

  stdDone = true;
}

void MSZDataSet::expand(size_t n) {
  vector<size_t> pvec(ncols - outputs);
  for(size_t i = 0; i < pvec.size(); i++)
    pvec[i] = i;

  expandOnPartition(n, pvec);
}

void MSZDataSet::expand(size_t n, const vector<vector<size_t> > &pvec) {
  for(size_t i = 0; i < pvec.size(); i++)
    expandOnPartition(n, pvec[i]);
}

void MSZDataSet::expandOnPartition(size_t k, const vector<size_t> &pvec) {

  // expansion is quadratic at the minimum and
  // and be bigger than partition size.
  assert(k > 2 && k <= pvec.size());

  // Standardize features
  standardize();

  // Reallocate the data.
  // If we have N features, then, to expand to a order-K polynomial
  // we are adding
  // n + (n !) / (k! (n-k!))
  size_t num = 1;
  for(size_t i = 0; i < k; i++) num *= rfeatures - i; // for big k hell breaks loose
  size_t den = 1;
  for(size_t i = k; i > 0; i--) den *= i;
  size_t nNewF = rfeatures;
  nNewF += num / den;
  ncols += nNewF; // Update number of columns

  // Now, we need to realloc all the structure
  for(size_t r = 0; r < nrows; r++) {
    matrix[r] = (double*)realloc(matrix[r], sizeof(**matrix) * (nNewF + ncols));
    if(matrix[r] == 0) {
      cerr << "Bad news... while trying to allocate memory...\n";
      exit(EXIT_FAILURE);
    }
  }

  // Compute the cross products
  // To do this we compute all the indices combinations and use them
  // to compute the cross product on columns defined in pvec.
  size_t currColumn = rfeatures + outputs;
  size_t *ind = new size_t [k];
  for(size_t i = 0; i < k; i++) ind[i] = i;

  // Compute cross product for initial configuration
  for(size_t r = 0; r < nrows; r++) {
    matrix[r][currColumn] = computeCrossProduct(r, ind, k, pvec);
  }
  currColumn++;

  size_t n = pvec.size();
  while(1) {
    if(ind[0] == n - k) break;

    for(size_t p = 0; p < k; p++) {
      size_t i = k - p - 1;
      bool incp = false;

      // Can I increment this index?
      if(ind[i] < n - p - 1) {
	ind[i]++; // YES

	for(size_t inc = 1; inc + i < k; inc++)
	  ind[inc+i] = ind[i] + inc;
	incp = true;

	// Compute cross product in current configuration
	for(size_t r = 0; r < nrows; r++) {
	  matrix[r][currColumn] = computeCrossProduct(r, ind, k, pvec);
	}
	currColumn++;
      }
      if(incp) break;
    }
  }
  delete ind;

  // Compute Powers
  for(size_t i = 0; i < pvec.size(); i++) {
    for(size_t r = 0; r < nrows; r++) {
      // Compute powers for pvec[i] feature
      matrix[r][currColumn] = gsl_pow_int(matrix[r][pvec[i]], k);
    }
  }

}

double MSZDataSet::computeCrossProduct(size_t r, size_t *ind, size_t k, const vector<size_t> &vec) {
  double cp = 1.0; // Cross-product
  for(size_t i = 0; i < k; i++)
   cp *= matrix[r][vec[ind[i]]];
  return cp;
}

/////////////////////////////////////////
/////////////////////////////////////////
//
// API Entrace Function for Data Set creation
//
/////////////////////////////////////////
/////////////////////////////////////////

MSZDataSet *createDataSet(double** matrix, size_t nrows, size_t ncols, size_t outputs) {
  assert(nrows > 0);
  assert(ncols > 0);
  assert(matrix != 0);
  assert(outputs > 0);
  assert(nrows >= ncols - outputs);

#ifndef NDEBUG
  for(size_t r = 0; r < nrows; r++)
    assert(matrix[r] != 0);
#endif // NDEBUG

  // The call!
  MSZDataSet *ds = new MSZDataSet(matrix, nrows, ncols, outputs);
  if(!ds) {
    cerr << "Error: Allocation of Dataset\n";
    exit(EXIT_FAILURE);
  }

  return ds;
}
<commit_msg>Now combinations use GSL structure.<commit_after>#include <iostream>
#include <fstream>

#include <cstdlib>
#include <cassert>
#include <cmath>

#include <gsl/gsl_math.h>
#include <gsl/gsl_combination.h>

#include "dataset.hh"

using std::cerr;
using std::ofstream;

MSZDataSet::MSZDataSet(double **matrix, size_t nrows, size_t ncols, size_t outputs)
  : matrix(matrix), nrows(nrows), rfeatures(ncols - outputs), ncols(ncols), outputs(outputs) {}

MSZDataSet::~MSZDataSet() {

  // Please God, do not let this Seg Fault!

  // Clean up matrix by deleting each line.
  for(size_t row = 0; row < nrows; row++)
    free(matrix[row]);

  free(matrix);

}

void MSZDataSet::dumpPlotFiles(const vector<string> &labels, const string &prefix) const {

  if(labels.size() != ncols) {
    cerr << "Warning: Trying to dump plot files but labels vector is too small.\n";
    return;
  }

  // Dump the files
  for(size_t out = 0; out < outputs; out++) {
    for(size_t feature = outputs; feature < ncols; feature++) {
      ofstream file;
      file.open((prefix + "_" + labels[out] + "_" + labels[feature] + ".dat").c_str());

      file << "# This file is generated by MatSATZilla\n"
	   << "# It should be used with gnuplot to plot\n"
	   << "# the feature " << labels[feature] << " against\n"
	   << "# the output " << labels[out] << "\n"
	   << "#\n"
	// Outputting timestamp could be nice.
	//<< "# Timestamp: " <<
	   << "#\n"
	   << "# " << labels[feature] << "\t\t" << labels[out] << "\n";

      // For each instance
      for(size_t i = 0; i < nrows; i++)
	file << matrix[i][feature] << "\t\t" << matrix[i][out] << "\n";

      file.close();
    }
  }
}

void MSZDataSet::dumpPlotFiles(char **labels, size_t len, char *prefix) const {

  vector<string> vec(len);

  for(size_t i = 0; i < len; i++)
    vec[i] = string(labels[i]);

  dumpPlotFiles(vec, string(prefix));
}

double MSZDataSet::getOutputValue(size_t row, size_t col) const {
  assert(row < nrows);
  assert(col < outputs);

  return matrix[row][col];
}

double MSZDataSet::getFeatureValue(size_t row, size_t col) const {
  assert(row < nrows);
  assert(col < ncols - outputs);

  return matrix[row][col+outputs];
}

void MSZDataSet::standardize() {

  static bool stdDone = false;

  if(!stdDone) {
    for(size_t c = outputs; c < ncols; c++) {

      // Compute column mean and compute column variance.
      double mean = 0.0;
      for(size_t r = 0; r < nrows; r++)
	mean += matrix[r][c];
      mean /= nrows;

      // Compute column standard deviation.
      double sdv = 0.0;
      for(size_t r = 0; r < nrows; r++)
	sdv += gsl_pow_2(matrix[r][c] - mean);
      sdv /= nrows;

      for(size_t r = 0; r < nrows; r++)
	matrix[r][c] = (matrix[r][c] - mean) / sdv;

    }

  }

  stdDone = true;

}

void MSZDataSet::standardizeOutputs() {

  static bool stdDone = false;

  if(!stdDone) {
    for(size_t c = 0; c < outputs; c++)
      for(size_t r = 0; r < nrows; r++)
	matrix[r][c] = log(matrix[r][c]);
  }

  stdDone = true;
}

void MSZDataSet::expand(size_t n) {
  vector<size_t> pvec(ncols - outputs);
  for(size_t i = 0; i < pvec.size(); i++)
    pvec[i] = i;

  expandOnPartition(n, pvec);
}

void MSZDataSet::expand(size_t n, const vector<vector<size_t> > &pvec) {
  for(size_t i = 0; i < pvec.size(); i++)
    expandOnPartition(n, pvec[i]);
}

void MSZDataSet::expandOnPartition(size_t k, const vector<size_t> &pvec) {

  // expansion is quadratic at the minimum and
  // and be bigger than partition size.
  assert(k > 2 && k <= pvec.size());

  // Standardize features
  standardize();

  // Reallocate the data.
  // If we have N features, then, to expand to a order-K polynomial
  // we are adding
  // n + (n !) / (k! (n-k!))
  size_t num = 1;
  for(size_t i = 0; i < k; i++) num *= rfeatures - i; // for big k hell breaks loose
  size_t den = 1;
  for(size_t i = k; i > 0; i--) den *= i;
  size_t nNewF = rfeatures;
  nNewF += num / den;
  ncols += nNewF; // Update number of columns

  // Now, we need to realloc all the structure
  for(size_t r = 0; r < nrows; r++) {
    matrix[r] = (double*)realloc(matrix[r], sizeof(**matrix) * (nNewF + ncols));
    if(matrix[r] == 0) {
      cerr << "Bad news... while trying to allocate memory...\n";
      exit(EXIT_FAILURE);
    }
  }

  // Compute the cross products
  // To do this we compute all the indices combinations and use them
  // to compute the cross product on columns defined in pvec.
  size_t currColumn = rfeatures + outputs;
  const size_t n = pvec.size();
  gsl_combination * c;
  c = gsl_combination_calloc (n, k);
  size_t *ind = new size_t [k];

  // Compute cross product for initial configuration
  // Now using the very nice combination structure from GSL.
  // Up to rev. 121, we used custom combination generator.
  do {
    for(size_t i = 0; i < k; i++)
      ind[i] = gsl_combination_get(c, i);

    // Compute cross product in current configuration
    for(size_t r = 0; r < nrows; r++) {
      matrix[r][currColumn] = computeCrossProduct(r, ind, k, pvec);
    }
    currColumn++;
  } while(gsl_combination_next (c) == GSL_SUCCESS);
  gsl_combination_free (c);
  delete ind;

  // Compute Powers
  for(size_t i = 0; i < pvec.size(); i++) {
    for(size_t r = 0; r < nrows; r++) {
      // Compute powers for pvec[i] feature
      matrix[r][currColumn] = gsl_pow_int(matrix[r][pvec[i]], k);
    }
  }

}

double MSZDataSet::computeCrossProduct(size_t r, size_t *ind, size_t k, const vector<size_t> &vec) {
  double cp = 1.0; // Cross-product
  for(size_t i = 0; i < k; i++)
   cp *= matrix[r][vec[ind[i]]];
  return cp;
}

/////////////////////////////////////////
/////////////////////////////////////////
//
// API Entrace Function for Data Set creation
//
/////////////////////////////////////////
/////////////////////////////////////////

MSZDataSet *createDataSet(double** matrix, size_t nrows, size_t ncols, size_t outputs) {
  assert(nrows > 0);
  assert(ncols > 0);
  assert(matrix != 0);
  assert(outputs > 0);
  assert(nrows >= ncols - outputs);

#ifndef NDEBUG
  for(size_t r = 0; r < nrows; r++)
    assert(matrix[r] != 0);
#endif // NDEBUG

  // The call!
  MSZDataSet *ds = new MSZDataSet(matrix, nrows, ncols, outputs);
  if(!ds) {
    cerr << "Error: Allocation of Dataset\n";
    exit(EXIT_FAILURE);
  }

  return ds;
}
<|endoftext|>


<commit_before>#include <stan/agrad/rev/atan2.hpp>
#include <test/agrad/util.hpp>
#include <gtest/gtest.h>

<commit_msg>splitting atan2 tests out of test/agrad/agrad_test.cpp<commit_after>#include <stan/agrad/rev/atan2.hpp>
#include <test/agrad/util.hpp>
#include <gtest/gtest.h>

TEST(AgradRev,atan2_var_var) {
  AVAR a = 1.2;
  AVAR b = 3.9;
  AVAR f = atan2(a,b);
  EXPECT_FLOAT_EQ(atan2(1.2,3.9),f.val());

  AVEC x = createAVEC(a,b);
  VEC g;
  f.grad(x,g);
  EXPECT_FLOAT_EQ(3.9 / (1.2 * 1.2 + 3.9 * 3.9), g[0]);
  EXPECT_FLOAT_EQ(-1.2 / (1.2 * 1.2 + 3.9 * 3.9), g[1]);
}

TEST(AgradRev,atan2_dvd) {
  AVAR sigma = 1;
  AVEC x = createAVEC(sigma);
  AVAR f = atan2(1.0,sigma) / 3.14;
  VEC g;
  f.grad(x,g);

  AVAR sigma1 = 1;
  AVEC x1 = createAVEC(sigma1);
  AVAR f1 = atan2(1.0,sigma1);
  VEC g1;
  f1.grad(x1,g1);

  EXPECT_FLOAT_EQ(3.14 * g[0],g1[0]);
}
TEST(AgradRev,atan2_var_var__integration) {
  double c = 5.0;
  AVAR a = 1.2;
  AVAR b = 3.9;
  AVAR f = atan2(a,b) * c;
  EXPECT_FLOAT_EQ(atan2(1.2,3.9)*c,f.val());

  AVEC x = createAVEC(a,b);
  VEC g;
  f.grad(x,g);
  EXPECT_FLOAT_EQ(3.9 / (1.2 * 1.2 + 3.9 * 3.9) * c, g[0]);
  EXPECT_FLOAT_EQ(-1.2 / (1.2 * 1.2 + 3.9 * 3.9) * c, g[1]);
}

TEST(AgradRev,atan2_var_double) {
  AVAR a = 1.2;

  double b = 3.9;
  AVAR f = atan2(a,b);
  EXPECT_FLOAT_EQ(atan2(1.2,3.9),f.val());

  AVEC x = createAVEC(a);
  VEC g;
  f.grad(x,g);
  EXPECT_FLOAT_EQ(3.9 / (1.2 * 1.2 + 3.9 * 3.9), g[0]);
}

TEST(AgradRev,atan2_double_var) {
  double a = 1.2;
  AVAR b = 3.9;
  AVAR f = atan2(a,b);
  EXPECT_FLOAT_EQ(atan2(1.2,3.9),f.val());

  AVEC x = createAVEC(b);
  VEC g;
  f.grad(x,g);
  EXPECT_FLOAT_EQ(-1.2 / (1.2 * 1.2 + 3.9 * 3.9), g[0]);
}
<|endoftext|>


<commit_before>// Copyright (c) 2020-2021 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chainparams.h>
#include <index/coinstatsindex.h>
#include <interfaces/chain.h>
#include <kernel/coinstats.h>
#include <test/util/setup_common.h>
#include <test/util/validation.h>
#include <util/time.h>
#include <validation.h>

#include <boost/test/unit_test.hpp>

#include <chrono>

BOOST_AUTO_TEST_SUITE(coinstatsindex_tests)

static void IndexWaitSynced(BaseIndex& index)
{
    // Allow the CoinStatsIndex to catch up with the block index that is syncing
    // in a background thread.
    const auto timeout = GetTime<std::chrono::seconds>() + 120s;
    while (!index.BlockUntilSyncedToCurrentChain()) {
        BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>());
        UninterruptibleSleep(100ms);
    }
}

BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup)
{
    CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true};

    const CBlockIndex* block_index;
    {
        LOCK(cs_main);
        block_index = m_node.chainman->ActiveChain().Tip();
    }

    // CoinStatsIndex should not be found before it is started.
    BOOST_CHECK(!coin_stats_index.LookUpStats(*block_index));

    // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex
    // is started.
    BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain());

    BOOST_REQUIRE(coin_stats_index.Start());

    IndexWaitSynced(coin_stats_index);

    // Check that CoinStatsIndex works for genesis block.
    const CBlockIndex* genesis_block_index;
    {
        LOCK(cs_main);
        genesis_block_index = m_node.chainman->ActiveChain().Genesis();
    }
    BOOST_CHECK(coin_stats_index.LookUpStats(*genesis_block_index));

    // Check that CoinStatsIndex updates with new blocks.
    BOOST_CHECK(coin_stats_index.LookUpStats(*block_index));

    const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG};
    std::vector<CMutableTransaction> noTxns;
    CreateAndProcessBlock(noTxns, script_pub_key);

    // Let the CoinStatsIndex to catch up again.
    BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain());

    const CBlockIndex* new_block_index;
    {
        LOCK(cs_main);
        new_block_index = m_node.chainman->ActiveChain().Tip();
    }
    BOOST_CHECK(coin_stats_index.LookUpStats(*new_block_index));

    BOOST_CHECK(block_index != new_block_index);

    // Shutdown sequence (c.f. Shutdown() in init.cpp)
    coin_stats_index.Stop();

    // Rest of shutdown sequence and destructors happen in ~TestingSetup()
}

// Test shutdown between BlockConnected and ChainStateFlushed notifications,
// make sure index is not corrupted and is able to reload.
BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup)
{
    Chainstate& chainstate = Assert(m_node.chainman)->ActiveChainstate();
    const CChainParams& params = Params();
    {
        CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
        BOOST_REQUIRE(index.Start());
        IndexWaitSynced(index);
        std::shared_ptr<const CBlock> new_block;
        CBlockIndex* new_block_index = nullptr;
        {
            const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG};
            const CBlock block = this->CreateBlock({}, script_pub_key, chainstate);

            new_block = std::make_shared<CBlock>(block);

            LOCK(cs_main);
            BlockValidationState state;
            BOOST_CHECK(CheckBlock(block, state, params.GetConsensus()));
            BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true));
            CCoinsViewCache view(&chainstate.CoinsTip());
            BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view));
        }
        // Send block connected notification, then stop the index without
        // sending a chainstate flushed notification. Prior to #24138, this
        // would cause the index to be corrupted and fail to reload.
        ValidationInterfaceTest::BlockConnected(index, new_block, new_block_index);
        index.Stop();
    }

    {
        CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
        // Make sure the index can be loaded.
        BOOST_REQUIRE(index.Start());
        index.Stop();
    }
}

BOOST_AUTO_TEST_SUITE_END()
<commit_msg>test: silence TSAN false positive in coinstatsindex_initial_sync<commit_after>// Copyright (c) 2020-2021 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chainparams.h>
#include <index/coinstatsindex.h>
#include <interfaces/chain.h>
#include <kernel/coinstats.h>
#include <test/util/setup_common.h>
#include <test/util/validation.h>
#include <util/time.h>
#include <validation.h>

#include <boost/test/unit_test.hpp>

#include <chrono>

BOOST_AUTO_TEST_SUITE(coinstatsindex_tests)

static void IndexWaitSynced(BaseIndex& index)
{
    // Allow the CoinStatsIndex to catch up with the block index that is syncing
    // in a background thread.
    const auto timeout = GetTime<std::chrono::seconds>() + 120s;
    while (!index.BlockUntilSyncedToCurrentChain()) {
        BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>());
        UninterruptibleSleep(100ms);
    }
}

BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup)
{
    CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true};

    const CBlockIndex* block_index;
    {
        LOCK(cs_main);
        block_index = m_node.chainman->ActiveChain().Tip();
    }

    // CoinStatsIndex should not be found before it is started.
    BOOST_CHECK(!coin_stats_index.LookUpStats(*block_index));

    // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex
    // is started.
    BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain());

    BOOST_REQUIRE(coin_stats_index.Start());

    IndexWaitSynced(coin_stats_index);

    // Check that CoinStatsIndex works for genesis block.
    const CBlockIndex* genesis_block_index;
    {
        LOCK(cs_main);
        genesis_block_index = m_node.chainman->ActiveChain().Genesis();
    }
    BOOST_CHECK(coin_stats_index.LookUpStats(*genesis_block_index));

    // Check that CoinStatsIndex updates with new blocks.
    BOOST_CHECK(coin_stats_index.LookUpStats(*block_index));

    const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG};
    std::vector<CMutableTransaction> noTxns;
    CreateAndProcessBlock(noTxns, script_pub_key);

    // Let the CoinStatsIndex to catch up again.
    BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain());

    const CBlockIndex* new_block_index;
    {
        LOCK(cs_main);
        new_block_index = m_node.chainman->ActiveChain().Tip();
    }
    BOOST_CHECK(coin_stats_index.LookUpStats(*new_block_index));

    BOOST_CHECK(block_index != new_block_index);

    // It is not safe to stop and destroy the index until it finishes handling
    // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain()
    // call above is sufficient to ensure this, but the
    // SyncWithValidationInterfaceQueue() call below is also needed to ensure
    // TSAN always sees the test thread waiting for the notification thread, and
    // avoid potential false positive reports.
    SyncWithValidationInterfaceQueue();

    // Shutdown sequence (c.f. Shutdown() in init.cpp)
    coin_stats_index.Stop();
}

// Test shutdown between BlockConnected and ChainStateFlushed notifications,
// make sure index is not corrupted and is able to reload.
BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup)
{
    Chainstate& chainstate = Assert(m_node.chainman)->ActiveChainstate();
    const CChainParams& params = Params();
    {
        CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
        BOOST_REQUIRE(index.Start());
        IndexWaitSynced(index);
        std::shared_ptr<const CBlock> new_block;
        CBlockIndex* new_block_index = nullptr;
        {
            const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG};
            const CBlock block = this->CreateBlock({}, script_pub_key, chainstate);

            new_block = std::make_shared<CBlock>(block);

            LOCK(cs_main);
            BlockValidationState state;
            BOOST_CHECK(CheckBlock(block, state, params.GetConsensus()));
            BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true));
            CCoinsViewCache view(&chainstate.CoinsTip());
            BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view));
        }
        // Send block connected notification, then stop the index without
        // sending a chainstate flushed notification. Prior to #24138, this
        // would cause the index to be corrupted and fail to reload.
        ValidationInterfaceTest::BlockConnected(index, new_block, new_block_index);
        index.Stop();
    }

    {
        CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
        // Make sure the index can be loaded.
        BOOST_REQUIRE(index.Start());
        index.Stop();
    }
}

BOOST_AUTO_TEST_SUITE_END()
<|endoftext|>


<commit_before>// -----------------------------------------------------------------------
// pion-common: a collection of common libraries used by the Pion Platform
// -----------------------------------------------------------------------
// Copyright (C) 2007-2008 Atomic Labs, Inc.  (http://www.atomiclabs.com)
//
// Distributed under the Boost Software License, Version 1.0.
// See http://www.boost.org/LICENSE_1_0.txt
//

#ifndef __PION_PIONLOCKFREEQUEUE_HEADER__
#define __PION_PIONLOCKFREEQUEUE_HEADER__

#include <boost/array.hpp>
#include <boost/cstdint.hpp>
#include <boost/noncopyable.hpp>
#include <pion/PionConfig.hpp>
#include <pion/PionScheduler.hpp>
#include <apr-1/apr_atomic.h>

// NOTE: the data structures contained in this file are based upon algorithms
// published in the paper "Simple, Fast, and Practical Non-Blocking and Blocking
// Concurrent Queue Algorithms" (1996, Maged M. Michael and Michael L. Scott,
// Department of Computer Science, University of Rochester).
// See http://www.cs.rochester.edu/u/scott/papers/1996_PODC_queues.pdf

namespace pion {	// begin namespace pion

///
/// PionLockFreeQueue: a FIFO queue that is thread-safe, lock-free and uses fixed memory
///
template <typename T, boost::uint16_t MaxSize = 50000, boost::uint32_t SleepNanoSec = 100000>
class PionLockFreeQueue :
	private boost::noncopyable
{
protected:

	/// an object used to point to a QueueNode
	union QueueNodePtr {
		/// the actual data contained within the QueueNodePtr object
		struct {
			/// index for the QueueNode object that is pointed to
			boost::uint16_t		index;
			/// used to check compare and swap operations & protect against ABA problems
			boost::uint16_t		counter;
		} data;
		/// the 32-bit value of the QueueNode object, used for CAS operations
		apr_uint32_t			value;
	};

	/// an object used to wrap each item in the queue
	struct QueueNode {
		/// default constructor used for base QueueNode
		QueueNode(void) { m_next.value = 0; }
		/// the data wrapped by the node object
		T							m_data;
		/// points to the next node in the list
		volatile QueueNodePtr		m_next;
	};

	/**
	 * returns a reference to a QueueNode based on the index of a QueueNodePtr
	 *
	 * @param node_ptr pointer to the the QueueNode object
	 *
	 * @return QueueNode& reference to the QueueNode object
	 */
	inline QueueNode& getQueueNode(QueueNodePtr node_ptr) {
		return m_index_table[node_ptr.data.index];
	}

	/**
	 * changes the QueueNode pointed to using an atomic compare-and-swap operation
	 *
	 * @param cur_ptr reference to the QueueNode pointer that will be changed
	 * @param old_ptr the old (existing) pointer value, used to check cas operation
	 * @param new_index the new index position for the object pointed to
	 *
	 * @return bool if the cas operation was successful, or false if not changed
	 */
	static inline bool cas(volatile QueueNodePtr& cur_ptr, QueueNodePtr old_ptr,
		boost::uint16_t new_index)
	{
		QueueNodePtr new_value;
		new_value.data.index = new_index;
		new_value.data.counter = old_ptr.data.counter + 1;
		return (apr_atomic_cas32(&(cur_ptr.value), new_value.value, old_ptr.value) == old_ptr.value);
	}

	/// returns the index position for a QueueNode that is available for use (may block)
	inline boost::uint16_t acquireNode(void) {
		QueueNodePtr	current_idx_ptr;
		boost::uint16_t	new_index;
		boost::uint16_t	result_index;

		while (true) {
			while (true) {
				// get current avail_index value
				current_idx_ptr.value = m_avail_index.value;
				// sleep if current avail_index value == 0
				if (current_idx_ptr.data.index > 0)
					break;
				PionScheduler::sleep(0, SleepNanoSec);
			}

			// prepare what will become the new avail_index value
			new_index = current_idx_ptr.value - 1;

			// fetch the result value optimistically
			result_index = m_nodes_avail[new_index].data.index;

			// use cas operation to update avail_index value
			if (cas(m_avail_index, current_idx_ptr, new_index))
				break;	// cas successful - all done!
		}

		return result_index;
	}

	/// releases a QueueNode that is no longer in use
	inline void releaseNode(const boost::uint16_t node_index) {
		QueueNodePtr	current_idx_ptr;
		QueueNodePtr	current_node_ptr;

		while (true) {
			while (true) {
				// get current avail_index value
				current_idx_ptr.value = m_avail_index.value;
				// sleep until current avail_index value is < MaxSize
				if (current_idx_ptr.data.index < MaxSize)
					break;
				// this should never be possible
				PION_ASSERT(false);
			};

			// get value for the next available index position
			current_node_ptr.value = m_nodes_avail[current_idx_ptr.data.index].value;

			// re-check current avail_index to avoid unnecessary cas calls
			if (current_idx_ptr.value == m_avail_index.value) {
				// use cas operation to optimistically assign the new node_index value
				if (cas(m_nodes_avail[current_idx_ptr.data.index],
					current_node_ptr, node_index))
				{
					// use cas operation to update avail_index value
					if (cas(m_avail_index, current_idx_ptr, current_idx_ptr.data.index + 1))
						break;	// both cas were successful - all done!
				}
			}
		}
	}

public:

	/// constructs a new PionLockFreeQueue
	PionLockFreeQueue(void)
	{
		// point head and tail to the node at index 1 (0 is reserved for NULL)
		m_head_ptr.data.index = m_tail_ptr.data.index = 1;
		m_head_ptr.data.counter = m_tail_ptr.data.counter = 0;
		// initialize avail_index to zero
		m_avail_index.value = 0;
		// initialize nodes_avil to zero
		for (boost::uint16_t n = 0; n < MaxSize; ++n)
			m_nodes_avail[n].value = 0;
		// initialize next values to zero
		for (boost::uint16_t n = 0; n < MaxSize+2; ++n)
			m_index_table[n].m_next.value = 0;
		// push everything but the first two nodes into the available stack
		for (boost::uint16_t n = 2; n < MaxSize+2; ++n)
			releaseNode(n);
	}

	/// virtual destructor
	virtual ~PionLockFreeQueue() {}

	/// returns the number of items that are currently in the queue
	inline boost::uint16_t size(void) const { return m_avail_index.data.index; }

	/// returns true if the queue is empty; false if it is not
	inline bool is_empty(void) const { return m_avail_index.data.index == 0; }

	/**
	 * pushes a new item into the back of the queue
	 *
	 * @param t the item to add to the back of the queue
	 */
	inline void push(T& t) {
		// retrieve a new list node for the queue item
		const boost::uint16_t node_index(acquireNode());

		// prepare it to be added to the list
		QueueNode& node_ref = m_index_table[node_index];
		node_ref.m_data = t;
		node_ref.m_next.data.index = 0;

		// append node to the end of the list
		QueueNodePtr tail_ptr;
		QueueNodePtr next_ptr;
		while (true) {
			tail_ptr.value = m_tail_ptr.value;
			next_ptr.value = getQueueNode(tail_ptr).m_next.value;
			// make sure that the tail pointer has not changed since reading next
			if (tail_ptr.value == m_tail_ptr.value) {
				// check if tail was pointing to the last node
				if (next_ptr.data.index == 0) {
					// try to link the new node at the end of the list
					if (cas(getQueueNode(tail_ptr).m_next, next_ptr, node_index))
						break;
				} else {
					// try to swing tail to the next node
					cas(m_tail_ptr, tail_ptr, next_ptr.data.index);
				}
			}
		}

		// done with enqueue; try to swing tail to the inserted node
		cas(m_tail_ptr, tail_ptr, node_index);
	}

	/**
	 * pops the next item from the top of the queue
	 *
	 * @param t assigned to the item at the top of the queue, if it is not empty
	 *
	 * @return true if an item was retrieved, false if the queue is empty
	 */
	inline bool pop(T& t) {
		QueueNodePtr head_ptr;
		QueueNodePtr tail_ptr;
		QueueNodePtr next_ptr;

		while (true) {
			// read current pointer values
			head_ptr.value = m_head_ptr.value;
			tail_ptr.value = m_tail_ptr.value;
			next_ptr.value = getQueueNode(head_ptr).m_next.value;
			// check consistency
			if (head_ptr.value == m_head_ptr.value) {
				// check if queue is empty, or tail is falling behind
				if (head_ptr.data.index == tail_ptr.data.index) {
					// is queue empty?
					if (next_ptr.data.index == 0)
						return false;
					// not empty; try to advance tail to catch it up
					cas(m_tail_ptr, tail_ptr, next_ptr.data.index);
				} else {
					// tail is OK
					// read value before CAS, otherwise another dequeue might
					// free the next node
					t = getQueueNode(next_ptr).m_data;
					// try to swing head to the next node
					if (cas(m_head_ptr, head_ptr, next_ptr.data.index))
						break;	// success -> exit loop
				}
			}
		}

		// item successfully retrieved
		releaseNode(const_cast<boost::uint16_t&>(head_ptr.data.index));
		return true;
	}

private:

	/// lookup table that maps uint16 index numbers to QueueNode pointers
	boost::array<QueueNode, MaxSize+2>		m_index_table;

	/// keeps track of all the QueueNode objects that are available for use
	boost::array<QueueNodePtr, MaxSize>		m_nodes_avail;

	/// pointer to the first item in the list
	volatile QueueNodePtr					m_head_ptr;

	/// pointer to the last item in the list
	volatile QueueNodePtr					m_tail_ptr;

	/// index pointer to the next QueueNode object that is available for use
	volatile QueueNodePtr					m_avail_index;
};

}	// end namespace pion

#endif
<commit_msg>Changes needed to use PionLockFreeQueue.<commit_after>// -----------------------------------------------------------------------
// pion-common: a collection of common libraries used by the Pion Platform
// -----------------------------------------------------------------------
// Copyright (C) 2007-2008 Atomic Labs, Inc.  (http://www.atomiclabs.com)
//
// Distributed under the Boost Software License, Version 1.0.
// See http://www.boost.org/LICENSE_1_0.txt
//

#ifndef __PION_PIONLOCKFREEQUEUE_HEADER__
#define __PION_PIONLOCKFREEQUEUE_HEADER__

#include <boost/array.hpp>
#include <boost/cstdint.hpp>
#include <boost/noncopyable.hpp>
#include <pion/PionConfig.hpp>
#include <pion/PionScheduler.hpp>
#ifdef _MSC_VER
	#include <apr_atomic.h>
#else
	#include <apr-1/apr_atomic.h>
#endif

// NOTE: the data structures contained in this file are based upon algorithms
// published in the paper "Simple, Fast, and Practical Non-Blocking and Blocking
// Concurrent Queue Algorithms" (1996, Maged M. Michael and Michael L. Scott,
// Department of Computer Science, University of Rochester).
// See http://www.cs.rochester.edu/u/scott/papers/1996_PODC_queues.pdf

namespace pion {	// begin namespace pion

///
/// PionLockFreeQueue: a FIFO queue that is thread-safe, lock-free and uses fixed memory
///
template <typename T, boost::uint16_t MaxSize = 50000, boost::uint32_t SleepNanoSec = 100000>
class PionLockFreeQueue :
	private boost::noncopyable
{
protected:

	/// an object used to point to a QueueNode
	union QueueNodePtr {
		/// the actual data contained within the QueueNodePtr object
		struct {
			/// index for the QueueNode object that is pointed to
			boost::uint16_t		index;
			/// used to check compare and swap operations & protect against ABA problems
			boost::uint16_t		counter;
		} data;
		/// the 32-bit value of the QueueNode object, used for CAS operations
		apr_uint32_t			value;
	};

	/// an object used to wrap each item in the queue
	struct QueueNode {
		/// default constructor used for base QueueNode
		QueueNode(void) { m_next.value = 0; }
		/// the data wrapped by the node object
		T							m_data;
		/// points to the next node in the list
		volatile QueueNodePtr		m_next;
	};

	/**
	 * returns a reference to a QueueNode based on the index of a QueueNodePtr
	 *
	 * @param node_ptr pointer to the the QueueNode object
	 *
	 * @return QueueNode& reference to the QueueNode object
	 */
	inline QueueNode& getQueueNode(QueueNodePtr node_ptr) {
		return m_index_table[node_ptr.data.index];
	}

	/**
	 * changes the QueueNode pointed to using an atomic compare-and-swap operation
	 *
	 * @param cur_ptr reference to the QueueNode pointer that will be changed
	 * @param old_ptr the old (existing) pointer value, used to check cas operation
	 * @param new_index the new index position for the object pointed to
	 *
	 * @return bool if the cas operation was successful, or false if not changed
	 */
	static inline bool cas(volatile QueueNodePtr& cur_ptr, QueueNodePtr old_ptr,
		boost::uint16_t new_index)
	{
		QueueNodePtr new_value;
		new_value.data.index = new_index;
		new_value.data.counter = old_ptr.data.counter + 1;
		return (apr_atomic_cas32(&(cur_ptr.value), new_value.value, old_ptr.value) == old_ptr.value);
	}

	/// returns the index position for a QueueNode that is available for use (may block)
	inline boost::uint16_t acquireNode(void) {
		QueueNodePtr	current_idx_ptr;
		boost::uint16_t	new_index;
		boost::uint16_t	result_index;

		while (true) {
			while (true) {
				// get current avail_index value
				current_idx_ptr.value = m_avail_index.value;
				// sleep if current avail_index value == 0
				if (current_idx_ptr.data.index > 0)
					break;
				PionScheduler::sleep(0, SleepNanoSec);
			}

			// prepare what will become the new avail_index value
			new_index = current_idx_ptr.value - 1;

			// fetch the result value optimistically
			result_index = m_nodes_avail[new_index].data.index;

			// use cas operation to update avail_index value
			if (cas(m_avail_index, current_idx_ptr, new_index))
				break;	// cas successful - all done!
		}

		return result_index;
	}

	/// releases a QueueNode that is no longer in use
	inline void releaseNode(const boost::uint16_t node_index) {
		QueueNodePtr	current_idx_ptr;
		QueueNodePtr	current_node_ptr;

		while (true) {
			while (true) {
				// get current avail_index value
				current_idx_ptr.value = m_avail_index.value;
				// sleep until current avail_index value is < MaxSize
				if (current_idx_ptr.data.index < MaxSize)
					break;
				// this should never be possible
				PION_ASSERT(false);
			};

			// get value for the next available index position
			current_node_ptr.value = m_nodes_avail[current_idx_ptr.data.index].value;

			// re-check current avail_index to avoid unnecessary cas calls
			if (current_idx_ptr.value == m_avail_index.value) {
				// use cas operation to optimistically assign the new node_index value
				if (cas(m_nodes_avail[current_idx_ptr.data.index],
					current_node_ptr, node_index))
				{
					// use cas operation to update avail_index value
					if (cas(m_avail_index, current_idx_ptr, current_idx_ptr.data.index + 1))
						break;	// both cas were successful - all done!
				}
			}
		}
	}

public:

	/// constructs a new PionLockFreeQueue
	PionLockFreeQueue(void)
	{
		// point head and tail to the node at index 1 (0 is reserved for NULL)
		m_head_ptr.data.index = m_tail_ptr.data.index = 1;
		m_head_ptr.data.counter = m_tail_ptr.data.counter = 0;
		// initialize avail_index to zero
		m_avail_index.value = 0;
		// initialize nodes_avil to zero
		for (boost::uint16_t n = 0; n < MaxSize; ++n)
			m_nodes_avail[n].value = 0;
		// initialize next values to zero
		for (boost::uint16_t n = 0; n < MaxSize+2; ++n)
			m_index_table[n].m_next.value = 0;
		// push everything but the first two nodes into the available stack
		for (boost::uint16_t n = 2; n < MaxSize+2; ++n)
			releaseNode(n);
	}

	/// virtual destructor
	virtual ~PionLockFreeQueue() {}

	/// returns the number of items that are currently in the queue
	inline boost::uint16_t size(void) const { return m_avail_index.data.index; }

	/// returns true if the queue is empty; false if it is not
	inline bool is_empty(void) const { return m_avail_index.data.index == 0; }

	/**
	 * pushes a new item into the back of the queue
	 *
	 * @param t the item to add to the back of the queue
	 */
	inline void push(T& t) {
		// retrieve a new list node for the queue item
		const boost::uint16_t node_index(acquireNode());

		// prepare it to be added to the list
		QueueNode& node_ref = m_index_table[node_index];
		node_ref.m_data = t;
		node_ref.m_next.data.index = 0;

		// append node to the end of the list
		QueueNodePtr tail_ptr;
		QueueNodePtr next_ptr;
		while (true) {
			tail_ptr.value = m_tail_ptr.value;
			next_ptr.value = getQueueNode(tail_ptr).m_next.value;
			// make sure that the tail pointer has not changed since reading next
			if (tail_ptr.value == m_tail_ptr.value) {
				// check if tail was pointing to the last node
				if (next_ptr.data.index == 0) {
					// try to link the new node at the end of the list
					if (cas(getQueueNode(tail_ptr).m_next, next_ptr, node_index))
						break;
				} else {
					// try to swing tail to the next node
					cas(m_tail_ptr, tail_ptr, next_ptr.data.index);
				}
			}
		}

		// done with enqueue; try to swing tail to the inserted node
		cas(m_tail_ptr, tail_ptr, node_index);
	}

	/**
	 * pops the next item from the top of the queue
	 *
	 * @param t assigned to the item at the top of the queue, if it is not empty
	 *
	 * @return true if an item was retrieved, false if the queue is empty
	 */
	inline bool pop(T& t) {
		QueueNodePtr head_ptr;
		QueueNodePtr tail_ptr;
		QueueNodePtr next_ptr;

		while (true) {
			// read current pointer values
			head_ptr.value = m_head_ptr.value;
			tail_ptr.value = m_tail_ptr.value;
			next_ptr.value = getQueueNode(head_ptr).m_next.value;
			// check consistency
			if (head_ptr.value == m_head_ptr.value) {
				// check if queue is empty, or tail is falling behind
				if (head_ptr.data.index == tail_ptr.data.index) {
					// is queue empty?
					if (next_ptr.data.index == 0)
						return false;
					// not empty; try to advance tail to catch it up
					cas(m_tail_ptr, tail_ptr, next_ptr.data.index);
				} else {
					// tail is OK
					// read value before CAS, otherwise another dequeue might
					// free the next node
					t = getQueueNode(next_ptr).m_data;
					// try to swing head to the next node
					if (cas(m_head_ptr, head_ptr, next_ptr.data.index))
						break;	// success -> exit loop
				}
			}
		}

		// item successfully retrieved
		releaseNode(const_cast<boost::uint16_t&>(head_ptr.data.index));
		return true;
	}

private:

	/// lookup table that maps uint16 index numbers to QueueNode pointers
	boost::array<QueueNode, MaxSize+2>		m_index_table;

	/// keeps track of all the QueueNode objects that are available for use
	boost::array<QueueNodePtr, MaxSize>		m_nodes_avail;

	/// pointer to the first item in the list
	volatile QueueNodePtr					m_head_ptr;

	/// pointer to the last item in the list
	volatile QueueNodePtr					m_tail_ptr;

	/// index pointer to the next QueueNode object that is available for use
	volatile QueueNodePtr					m_avail_index;
};

}	// end namespace pion

#endif
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <functional>

namespace detail {

template<typename T, typename Comparator>
class extremum_tracker {
    T _default_value;
    bool _is_set = false;
    T _value;
public:
    extremum_tracker() {}

    explicit extremum_tracker(T default_value) {
        _default_value = default_value;
    }

    void update(T value) {
        if (!_is_set) {
            _value = value;
            _is_set = true;
        } else {
            if (Comparator{}(value,_value)) {
                _value = value;
            }
        }
    }

    T get() const {
        if (_is_set) {
            return _value;
        }
        return _default_value;
    }
};

} // namespace detail

template <typename T>
using min_tracker = detail::extremum_tracker<T, std::less<T>>;

template <typename T>
using max_tracker = detail::extremum_tracker<T, std::greater<T>>;

template <typename T>
class min_max_tracker {
    min_tracker<T> _min_tracker;
    max_tracker<T> _max_tracker;
public:
    min_max_tracker()
        : _min_tracker(std::numeric_limits<T>::min())
        , _max_tracker(std::numeric_limits<T>::max())
    {}

    min_max_tracker(T default_min, T default_max)
        : _min_tracker(default_min)
        , _max_tracker(default_max)
    {}

    void update(T value) {
        _min_tracker.update(value);
        _max_tracker.update(value);
    }

    void update(const min_max_tracker<T>& other) {
        _min_tracker.update(other.min());
        _max_tracker.update(other.max());
    }

    T min() const {
        return _min_tracker.get();
    }

    T max() const {
        return _max_tracker.get();
    }
};

<commit_msg>utils/extremum_tracking: preserve "not-set" status on merge<commit_after>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <functional>

namespace detail {

template<typename T, typename Comparator>
class extremum_tracker {
    T _default_value;
    bool _is_set = false;
    T _value;
public:
    extremum_tracker() {}

    explicit extremum_tracker(T default_value) {
        _default_value = default_value;
    }

    void update(T value) {
        if (!_is_set) {
            _value = value;
            _is_set = true;
        } else {
            if (Comparator{}(value,_value)) {
                _value = value;
            }
        }
    }

    void update(const extremum_tracker& other) {
        if (other._is_set) {
            update(other._value);
        }
    }

    T get() const {
        if (_is_set) {
            return _value;
        }
        return _default_value;
    }
};

} // namespace detail

template <typename T>
using min_tracker = detail::extremum_tracker<T, std::less<T>>;

template <typename T>
using max_tracker = detail::extremum_tracker<T, std::greater<T>>;

template <typename T>
class min_max_tracker {
    min_tracker<T> _min_tracker;
    max_tracker<T> _max_tracker;
public:
    min_max_tracker()
        : _min_tracker(std::numeric_limits<T>::min())
        , _max_tracker(std::numeric_limits<T>::max())
    {}

    min_max_tracker(T default_min, T default_max)
        : _min_tracker(default_min)
        , _max_tracker(default_max)
    {}

    void update(T value) {
        _min_tracker.update(value);
        _max_tracker.update(value);
    }

    void update(const min_max_tracker<T>& other) {
        _min_tracker.update(other._min_tracker);
        _max_tracker.update(other._max_tracker);
    }

    T min() const {
        return _min_tracker.get();
    }

    T max() const {
        return _max_tracker.get();
    }
};
<|endoftext|>


<commit_before>#include <u5e/utf8.hpp>
#include <u5e/codepoint.hpp>
#include "gtest/gtest.h"
#include <experimental/string_view>

using u5e::codepoint;
using u5e::utf8;
using std::experimental::string_view;
using std::string;

TEST(t_003_utf8_encoded_iterator, constructor1a) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e(str);
}

TEST(t_003_utf8_encoded_iterator, constructor1b) {
  string str("Ol\xC3\x81!");
  utf8<string> e(str);
}

TEST(t_003_utf8_encoded_iterator, constructor2a) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e1(str);
  utf8<string_view> e2(e1);
}

TEST(t_003_utf8_encoded_iterator, constructor2b) {
  string str("Ol\xC3\x81!");
  utf8<string> e1(str);
  utf8<string> e2(e1);
}

TEST(t_003_utf8_encoded_iterator, iterator) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e(str);
  utf8<string_view>::iterator it = e.begin();
  ASSERT_EQ(*it, 'O');
  it++;
  ASSERT_EQ(*it, 'l');
  it++;
  ASSERT_EQ(*it, 0xC1);
  it++;
  ASSERT_EQ(*it, '!');
  it++;
  ASSERT_EQ(it, e.end());
}

TEST(t_003_utf8_encoded_iterator_over_std_string, iterator) {
  string str("Ol\xC3\x81!");
  utf8<string> e(str);
  utf8<string>::iterator it = e.begin();
  ASSERT_EQ(*it, 'O');
  it++;
  ASSERT_EQ(*it, 'l');
  it++;
  ASSERT_EQ(*it, 0xC1);
  it++;
  ASSERT_EQ(*it, '!');
  it++;
  ASSERT_EQ(it, e.end());
}

<commit_msg>Add test for iterating over const char*<commit_after>#include <u5e/utf8.hpp>
#include <u5e/codepoint.hpp>
#include "gtest/gtest.h"
#include <experimental/string_view>

using u5e::codepoint;
using u5e::utf8;
using std::experimental::string_view;
using std::string;

TEST(t_003_utf8_encoded_iterator, constructor1a) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e(str);
}

TEST(t_003_utf8_encoded_iterator, constructor1b) {
  string str("Ol\xC3\x81!");
  utf8<string> e(str);
}

TEST(t_003_utf8_encoded_iterator, constructor2a) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e1(str);
  utf8<string_view> e2(e1);
}

TEST(t_003_utf8_encoded_iterator, constructor2b) {
  string str("Ol\xC3\x81!");
  utf8<string> e1(str);
  utf8<string> e2(e1);
}

TEST(t_003_utf8_encoded_iterator, iteratora) {
  string_view str("Ol\xC3\x81!");
  utf8<string_view> e(str);
  utf8<string_view>::iterator it = e.begin();
  ASSERT_EQ(*it, 'O');
  it++;
  ASSERT_EQ(*it, 'l');
  it++;
  ASSERT_EQ(*it, 0xC1);
  it++;
  ASSERT_EQ(*it, '!');
  it++;
  ASSERT_EQ(it, e.end());
}

TEST(t_003_utf8_encoded_iterator, iteratorb) {
  string str("Ol\xC3\x81!");
  utf8<string> e(str);
  utf8<string>::iterator it = e.begin();
  ASSERT_EQ(*it, 'O');
  it++;
  ASSERT_EQ(*it, 'l');
  it++;
  ASSERT_EQ(*it, 0xC1);
  it++;
  ASSERT_EQ(*it, '!');
  it++;
  ASSERT_EQ(it, e.end());
}

TEST(t_003_utf8_encoded_iterator, iteratorc) {
  const char* foo = "Ol\xC3\x81!";
  utf8_iterator<const char*> it = e.begin();
  ASSERT_EQ(*it, 'O');
  it++;
  ASSERT_EQ(*it, 'l');
  it++;
  ASSERT_EQ(*it, 0xC1);
  it++;
  ASSERT_EQ(*it, '!');
  it++;
  ASSERT_EQ(it, e.end());
}
<|endoftext|>


<commit_before>/*
 * Copyright 2009-2011 The VOTCA Development Team (http://www.votca.org)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <iostream>
#include <fstream>
#include <boost/format.hpp>
#include <votca/csg/csgapplication.h>

using namespace votca::csg;
using namespace std;
using boost::format;

/**
    \brief class for writing dlpoly topology files

    This class encapsulates the dlpoly topology writing functions

*/

class DLPTopolApp
    : public CsgApplication
{
public:
    string ProgramName() { return "csg_dlptopol"; }
    void HelpText(ostream &out) {
        out << "Create a dlpoly topology template based on an existing (atomistic) topology "
	    << "and a mapping xml-file.\n"
	    << "NOTE: the created template file needs to be inspected and amended by the user!\n\n"
	    << "Examples of usage:\n"
	    << "'csg_dlptopol --top .dlpf --out .dlpf --cg cg-map.xml'      by convention transforms FIELD -> FIELD_CGV\n"
	    << "'csg_dlptopol --top FA-dlpoly.dlpf --out CG-dlpoly.dlpf --cg cg-map.xml'  transforms FA-dlpoly.dlpf -> CG-dlpoly.dlpf\n"
	    << "'csg_dlptopol --top FA-gromacs.tpr --out FA-dlpoly.dlpf --no-map'         transforms FA-gromacs.tpr -> FA-dlpoly.dlpf";
    }

    bool DoMapping(void) { return true; }

    void Initialize(void);
    bool EvaluateOptions(void) {
        CsgApplication::EvaluateOptions();
        CheckRequired("out", "no output topology specified");
        return true;
    }
    bool EvaluateTopology(Topology *top, Topology *top_ref);

protected:
    void WriteMoleculeAtoms(ostream &out, Molecule &cg);
    void WriteMoleculeInteractions(ostream &out, Molecule &cg);
    void WriteVDWInteractions(ostream &out, Molecule &cg);
    void WriteMolecularType(ostream &out, Molecule &cg, int nummol);
};

void DLPTopolApp::Initialize(void)
{
    CsgApplication::Initialize();
    //AddProgramOptions()
    //("top", boost::program_options::value<string>(),
    //"  input topology in any known format:\n  <name>.dlpf for dlpoly, <name>.tpr for gromacs\n  (convention: '.dlpf'='use FIELD')");
    AddProgramOptions()
      ("out", boost::program_options::value<string>(),
       "  output topology in dlpoly format:\n  <name>.dlpf (convention: '.dlpf'='use FIELD_CGV')");
}

bool DLPTopolApp::EvaluateTopology(Topology *top, Topology *top_ref)
{
  // check the file names from options

  string fname=OptionsMap()["top"].as<string>();

  if( fname == ".dlpf" ) {
    fname = "FIELD";
  }

  cout << "input  file-name: " << fname << endl;

  fname=OptionsMap()["out"].as<string>();

#ifdef DEBUG
  cout << "output file-name given: " << fname << endl;
#endif

  if( fname == ".dlpf" ) {
    fname = "FIELD_CGV";
  }

#ifdef DEBUG
  cout << "output file-name actual: " << fname << endl;
#else
  cout << "output file-name: " << fname << endl;
#endif

  // do CG mapping

  MoleculeContainer &mols = top->Molecules();
  MoleculeContainer MolecularTypes;
  MoleculeContainer::iterator iter;

  int prv_mol_number = 1;
  string prv_mol_name;
  vector<int> nummols;

  vector<string> vdw_pairs;

  // find all unique molecular types

  for(iter=mols.begin(); iter!=mols.end(); ++iter) {
    Molecule *mol = *iter;

    // molecules are ignored during the mapping stage
    // i.e. the ignored ones do not enter the CG topology (*top) - ?
    //if( IsIgnored(mol->getName()) ) continue;

    if( mol->getName()==prv_mol_name ) {
      prv_mol_number++;
      continue;
    }

    nummols.push_back(prv_mol_number);
    prv_mol_number = 1;
    prv_mol_name   = mol->getName();

    //#ifdef DEBUG
    cout << "'" << mol->getName() << "' added to CG molecular types" << endl;
    //#endif

    MolecularTypes.push_back(mol);

    // collect unique bead pairs over all molecular types found

    for(int ib1=0; ib1<mol->BeadCount(); ib1++) {
      string bead_name1 = mol->getBead(ib1)->getType()->getName();

      for(int imt=0; imt<MolecularTypes.size(); imt++) {

	for(int ib2=0; ib2<MolecularTypes[imt]->BeadCount(); ib2++) {

	  string bead_name2 = MolecularTypes[imt]->getBead(ib2)->getType()->getName();
	  stringstream ss_bp1,ss_bp2;

	  ss_bp1 << format("%8s%8s" ) % bead_name1 % bead_name2;
	  ss_bp2 << format("%8s%8s" ) % bead_name2 % bead_name1;

          bool is_new_pair=true;

	  for(int ibp=0; ibp<vdw_pairs.size(); ibp++) {
	    if( ss_bp1.str()==vdw_pairs[ibp] || ss_bp2.str()==vdw_pairs[ibp] ) {
	      is_new_pair=false;
	      break;
	    }
	  }
	  if( is_new_pair ) {
	    vdw_pairs.push_back(ss_bp1.str());
#ifdef DEBUG
	    cout << "'" << ss_bp1.str() << "' added to CG vdw pairs" << endl;
#endif
	  }
	}
      }
    }
  }
  nummols.push_back(prv_mol_number);

  if(MolecularTypes.size() > 1)
    cout << "WARNING: creation of topology for multiple molecular types "
      "is experimental at this stage\n";

  ofstream fl;
  fl.open(fname.c_str());

  fl << "From VOTCA with love" << " # check/amend this file if needed!\n";
  fl << "units kJ\n";
  fl << "molecular types " << MolecularTypes.size() << endl;

  for(int i=0; i<MolecularTypes.size(); i++) {
    WriteMolecularType(fl, *(MolecularTypes[i]), nummols[i+1]);
  }

  // vdw seciton (pairwise vdw/CG interactions)

  if( vdw_pairs.size()>0 ) {

    fl << "vdw "<< vdw_pairs.size() << endl;

    for(int ibp=0; ibp<vdw_pairs.size(); ibp++) {
      fl << vdw_pairs[ibp] << " tab   1.00000  0.00000\n";
    }
  }

  fl << "close" << endl;

  cout << "Created template for dlpoly topology - please, check & amend if needed!" << endl;

  fl.close();
  return true;
}

void DLPTopolApp::WriteMoleculeAtoms(ostream &out, Molecule &cg)
{
  out << "atoms " << cg.BeadCount() << endl;
    out << "# name  mass  charge  nrept  ifrozen (optional: ngroup, index, name/type, type/residue, index/res-ID) \n";
    for(int i=0; i<cg.BeadCount(); ++i) {
        Bead *b=cg.getBead(i);

        out << format("%8s  %10f  %10f     1     0     1 %10d  %8s  %8s %10d \n")
            % b->getType()->getName() % b->getM() % b->getQ() % (i+1) % b->getType()->getName() % b->getName() % (i+1);
    }
}

void DLPTopolApp::WriteMoleculeInteractions(ostream &out, Molecule &cg)
{
    InteractionContainer ics=cg.Interactions();
    vector<Interaction *>::iterator iter;

    stringstream sout;

    int n_entries = 0;
    int nb=-1;

    for(iter=ics.begin(); iter!=ics.end(); ++iter) {
        Interaction *ic = *iter;
        if(nb != ic->BeadCount()) {

	    if(sout.str()!="")
	      out << n_entries << endl << sout.str();

	    sout.str("");
	    n_entries = 0;

            nb=ic->BeadCount();
            switch(nb) {
                case 2:
		  out << "bonds ";
                    break;
                case 3:
                    out << "angles ";
                    break;
                case 4:
                    out << "dihedrals ";
                    break;
                default:
                    throw runtime_error(string("cannot handle number of beads in interaction:") +
                            ic->getName());
            }
        }
	n_entries++;
	// to do: is it possible to use bond/angle/dihedral function types for 1:1 mapping? (CG map overwrites ic->Group anyway)
        //sout << ic->getInteractionFunc(); // something like that (only for 1:1 mapping!)
        sout << " tab ";
        for(int i=0; i<nb; ++i)
	  sout << ic->getBeadId(i)+1 << " ";
        sout << "   1.00000  0.00000" << " # " << ic->getName() << endl;
        //sout << "   1.00000  0.00000" << " # " << ic->getGroup() << endl;
    }
    if(sout.str()!="") out << n_entries << endl << sout.str();
}

void DLPTopolApp::WriteMolecularType(ostream &out, Molecule &cg, int nummol)
{
    out << cg.getName() << endl;
    out << "nummols " << nummol << endl;

    WriteMoleculeAtoms(out, cg);
    WriteMoleculeInteractions(out, cg);

    out << "finish" << endl;
}

int main(int argc, char** argv)
{
    DLPTopolApp app;
    return app.Exec(argc, argv);
}

<commit_msg>csg_dlptopol: fixed --help<commit_after>/*
 * Copyright 2009-2011 The VOTCA Development Team (http://www.votca.org)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <iostream>
#include <fstream>
#include <boost/format.hpp>
#include <votca/csg/csgapplication.h>

using namespace votca::csg;
using namespace std;
using boost::format;

/**
    \brief class for writing dlpoly topology files

    This class encapsulates the dlpoly topology writing functions

*/

class DLPTopolApp
    : public CsgApplication
{
public:
    string ProgramName() { return "csg_dlptopol"; }
    void HelpText(ostream &out) {
        out << "Create a dlpoly topology template based on an existing (atomistic) topology "
	    << "and a mapping xml-file.\n"
	    << "NOTE: the created template file needs to be inspected and amended by the user!\n\n"
	    << "Examples:\n"
	    << "* csg_dlptopol --top .dlpf --out .dlpf --cg cg-map.xml     convent FIELD to FIELD_CGV using cg-map.xml\n"
	    << "* csg_dlptopol --top FA-dlpoly.dlpf --out CG-dlpoly.dlpf --cg cg-map.xml   transforms FA-dlpoly.dlpf to CG-dlpoly.dlpf\n"
	    << "* csg_dlptopol --top FA-gromacs.tpr --out FA-dlpoly.dlpf --no-map          transforms FA-gromacs.tpr to FA-dlpoly.dlpf";
    }

    bool DoMapping(void) { return true; }

    void Initialize(void);
    bool EvaluateOptions(void) {
        CsgApplication::EvaluateOptions();
        CheckRequired("out", "no output topology specified");
        return true;
    }
    bool EvaluateTopology(Topology *top, Topology *top_ref);

protected:
    void WriteMoleculeAtoms(ostream &out, Molecule &cg);
    void WriteMoleculeInteractions(ostream &out, Molecule &cg);
    void WriteVDWInteractions(ostream &out, Molecule &cg);
    void WriteMolecularType(ostream &out, Molecule &cg, int nummol);
};

void DLPTopolApp::Initialize(void)
{
    CsgApplication::Initialize();
    //AddProgramOptions()
    //("top", boost::program_options::value<string>(),
    //"  input topology in any known format:\n  <name>.dlpf for dlpoly, <name>.tpr for gromacs\n  (convention: '.dlpf'='use FIELD')");
    AddProgramOptions()
      ("out", boost::program_options::value<string>(),
       "  output topology in dlpoly format:\n  <name>.dlpf (convention: '.dlpf'='use FIELD_CGV')");
}

bool DLPTopolApp::EvaluateTopology(Topology *top, Topology *top_ref)
{
  // check the file names from options

  string fname=OptionsMap()["top"].as<string>();

  if( fname == ".dlpf" ) {
    fname = "FIELD";
  }

  cout << "input  file-name: " << fname << endl;

  fname=OptionsMap()["out"].as<string>();

#ifdef DEBUG
  cout << "output file-name given: " << fname << endl;
#endif

  if( fname == ".dlpf" ) {
    fname = "FIELD_CGV";
  }

#ifdef DEBUG
  cout << "output file-name actual: " << fname << endl;
#else
  cout << "output file-name: " << fname << endl;
#endif

  // do CG mapping

  MoleculeContainer &mols = top->Molecules();
  MoleculeContainer MolecularTypes;
  MoleculeContainer::iterator iter;

  int prv_mol_number = 1;
  string prv_mol_name;
  vector<int> nummols;

  vector<string> vdw_pairs;

  // find all unique molecular types

  for(iter=mols.begin(); iter!=mols.end(); ++iter) {
    Molecule *mol = *iter;

    // molecules are ignored during the mapping stage
    // i.e. the ignored ones do not enter the CG topology (*top) - ?
    //if( IsIgnored(mol->getName()) ) continue;

    if( mol->getName()==prv_mol_name ) {
      prv_mol_number++;
      continue;
    }

    nummols.push_back(prv_mol_number);
    prv_mol_number = 1;
    prv_mol_name   = mol->getName();

    //#ifdef DEBUG
    cout << "'" << mol->getName() << "' added to CG molecular types" << endl;
    //#endif

    MolecularTypes.push_back(mol);

    // collect unique bead pairs over all molecular types found

    for(int ib1=0; ib1<mol->BeadCount(); ib1++) {
      string bead_name1 = mol->getBead(ib1)->getType()->getName();

      for(int imt=0; imt<MolecularTypes.size(); imt++) {

	for(int ib2=0; ib2<MolecularTypes[imt]->BeadCount(); ib2++) {

	  string bead_name2 = MolecularTypes[imt]->getBead(ib2)->getType()->getName();
	  stringstream ss_bp1,ss_bp2;

	  ss_bp1 << format("%8s%8s" ) % bead_name1 % bead_name2;
	  ss_bp2 << format("%8s%8s" ) % bead_name2 % bead_name1;

          bool is_new_pair=true;

	  for(int ibp=0; ibp<vdw_pairs.size(); ibp++) {
	    if( ss_bp1.str()==vdw_pairs[ibp] || ss_bp2.str()==vdw_pairs[ibp] ) {
	      is_new_pair=false;
	      break;
	    }
	  }
	  if( is_new_pair ) {
	    vdw_pairs.push_back(ss_bp1.str());
#ifdef DEBUG
	    cout << "'" << ss_bp1.str() << "' added to CG vdw pairs" << endl;
#endif
	  }
	}
      }
    }
  }
  nummols.push_back(prv_mol_number);

  if(MolecularTypes.size() > 1)
    cout << "WARNING: creation of topology for multiple molecular types "
      "is experimental at this stage\n";

  ofstream fl;
  fl.open(fname.c_str());

  fl << "From VOTCA with love" << " # check/amend this file if needed!\n";
  fl << "units kJ\n";
  fl << "molecular types " << MolecularTypes.size() << endl;

  for(int i=0; i<MolecularTypes.size(); i++) {
    WriteMolecularType(fl, *(MolecularTypes[i]), nummols[i+1]);
  }

  // vdw seciton (pairwise vdw/CG interactions)

  if( vdw_pairs.size()>0 ) {

    fl << "vdw "<< vdw_pairs.size() << endl;

    for(int ibp=0; ibp<vdw_pairs.size(); ibp++) {
      fl << vdw_pairs[ibp] << " tab   1.00000  0.00000\n";
    }
  }

  fl << "close" << endl;

  cout << "Created template for dlpoly topology - please, check & amend if needed!" << endl;

  fl.close();
  return true;
}

void DLPTopolApp::WriteMoleculeAtoms(ostream &out, Molecule &cg)
{
  out << "atoms " << cg.BeadCount() << endl;
    out << "# name  mass  charge  nrept  ifrozen (optional: ngroup, index, name/type, type/residue, index/res-ID) \n";
    for(int i=0; i<cg.BeadCount(); ++i) {
        Bead *b=cg.getBead(i);

        out << format("%8s  %10f  %10f     1     0     1 %10d  %8s  %8s %10d \n")
            % b->getType()->getName() % b->getM() % b->getQ() % (i+1) % b->getType()->getName() % b->getName() % (i+1);
    }
}

void DLPTopolApp::WriteMoleculeInteractions(ostream &out, Molecule &cg)
{
    InteractionContainer ics=cg.Interactions();
    vector<Interaction *>::iterator iter;

    stringstream sout;

    int n_entries = 0;
    int nb=-1;

    for(iter=ics.begin(); iter!=ics.end(); ++iter) {
        Interaction *ic = *iter;
        if(nb != ic->BeadCount()) {

	    if(sout.str()!="")
	      out << n_entries << endl << sout.str();

	    sout.str("");
	    n_entries = 0;

            nb=ic->BeadCount();
            switch(nb) {
                case 2:
		  out << "bonds ";
                    break;
                case 3:
                    out << "angles ";
                    break;
                case 4:
                    out << "dihedrals ";
                    break;
                default:
                    throw runtime_error(string("cannot handle number of beads in interaction:") +
                            ic->getName());
            }
        }
	n_entries++;
	// to do: is it possible to use bond/angle/dihedral function types for 1:1 mapping? (CG map overwrites ic->Group anyway)
        //sout << ic->getInteractionFunc(); // something like that (only for 1:1 mapping!)
        sout << " tab ";
        for(int i=0; i<nb; ++i)
	  sout << ic->getBeadId(i)+1 << " ";
        sout << "   1.00000  0.00000" << " # " << ic->getName() << endl;
        //sout << "   1.00000  0.00000" << " # " << ic->getGroup() << endl;
    }
    if(sout.str()!="") out << n_entries << endl << sout.str();
}

void DLPTopolApp::WriteMolecularType(ostream &out, Molecule &cg, int nummol)
{
    out << cg.getName() << endl;
    out << "nummols " << nummol << endl;

    WriteMoleculeAtoms(out, cg);
    WriteMoleculeInteractions(out, cg);

    out << "finish" << endl;
}

int main(int argc, char** argv)
{
    DLPTopolApp app;
    return app.Exec(argc, argv);
}

<|endoftext|>


<commit_before>#ifndef TURBO_TOOLSET_EXTENSION_HPP
#define TURBO_TOOLSET_EXTENSION_HPP

#if defined(__GNUC__) || defined(__clang__)
#if defined(__has_builtin)
#if __has_builtin(__builtin_expect)
#define TURBO_LIKELY(x) __builtin_expect(x, 1)
#define TURBO_UNLIKELY(x) __builtin_expect(x, 0)
#endif
#endif
#endif

#if !defined(TURBO_LIKELY)
#define TURBO_LIKELY(x) x
#endif
#if !defined(TURBO_UNLIKELY)
#define TURBO_UNLIKELY(x) x
#endif

#endif
<commit_msg>no point checking for __has_builtin since it's not in Gcc 4.8<commit_after>#ifndef TURBO_TOOLSET_EXTENSION_HPP
#define TURBO_TOOLSET_EXTENSION_HPP

#if defined(__GNUC__) || defined(__clang__)
#define TURBO_LIKELY(x) __builtin_expect(x, 1)
#define TURBO_UNLIKELY(x) __builtin_expect(x, 0)
#endif

#if !defined(TURBO_LIKELY)
#define TURBO_LIKELY(x) x
#endif
#if !defined(TURBO_UNLIKELY)
#define TURBO_UNLIKELY(x) x
#endif

#endif
<|endoftext|>


<commit_before>/**
* \file
* The most simple example.
* install: `yaourt -S aur/libcurlpp`
* [curlpp examples](https://github.com/jpbarrette/curlpp/tree/master/examples)
* [curlpp download wnd write callback example](https://github.com/datacratic/curlpp/blob/master/examples/example11.cpp)
* [Link errors with curlpp](http://stackoverflow.com/questions/33420631/link-errors-with-curlpp)
*/

#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <thread>
#include <mutex>
#include <condition_variable>

// curlpp specific headers
#include <curlpp/cURLpp.hpp>
#include <curlpp/Easy.hpp>
#include <curlpp/Options.hpp>

// pugixml
#include <pugixml.hpp>

// [clean html with C or C++ tidy](http://www.cppfun.com/clean-html-with-c-or-cpp-tidy.htm)
// [tidy 5.2.0-1](https://www.archlinux.org/packages/extra/x86_64/tidy/)
// `pacman -S extra/tidy`

#include <tidy.h>
#include <buffio.h>

std::string cleanhtml(const std::string &html)
{
    // init a tidy document
    TidyDoc tidy_doc=tidyCreate();
    TidyBuffer output_buffer= {0};

    // configure tidy
    // the flags tell tidy to output xml and disable warnings
    bool config_success=tidyOptSetBool(tidy_doc,TidyXmlOut,yes)
                        && tidyOptSetBool(tidy_doc,TidyQuiet,yes)
                        && tidyOptSetBool(tidy_doc,TidyNumEntities,yes)
                        && tidyOptSetBool(tidy_doc,TidyShowWarnings,no);

    int tidy_rescode=-1;

    // parse input
    if(config_success)
        tidy_rescode=tidyParseString(tidy_doc,html.c_str());

    // process html
    if(tidy_rescode>=0)
        tidy_rescode=tidySaveBuffer(tidy_doc,&output_buffer);

    if(tidy_rescode<0)
        throw("tidy has a error: "+tidy_rescode);

    std::string result=(char *)output_buffer.bp;
    tidyBufFree(&output_buffer);
    tidyRelease(tidy_doc);

    return result;
}

using namespace curlpp::options;

bool is_ready(false);
std::mutex m;
std::condition_variable cv;

struct simple_walker: pugi::xml_tree_walker {
    virtual bool for_each(pugi::xml_node& node) {
        for (int i = 0; i < depth(); ++i) std::cout << "  "; // indentation
        std::cout << /* node_types[node.type()] << */ ": name='" << node.name() << "', value='" << node.value() << "'\n";
        return true; // continue traversal
    }
};

void
getWebPage(std::string const& url, std::string& page)
{
    std::unique_lock<std::mutex> lk(m);
    try {
        // That's all that is needed to do cleanup of used resources (RAII style).
        curlpp::Cleanup myCleanup;

        // Our request to be sent.
        curlpp::Easy myRequest;

        // Set the URL.
        myRequest.setOpt(new curlpp::options::Url(url));

        // Send request and get a result.
        // By default the result goes to standard output.

        std::ostringstream os;
        curlpp::options::WriteStream ws(&os);
        myRequest.setOpt(ws);
        myRequest.perform();

        page = std::move(os.str());

        is_ready = true;
        cv.notify_one();

    } catch(curlpp::RuntimeError & e) {
        std::cout << e.what() << std::endl;
    }

    catch(curlpp::LogicError & e) {
        std::cout << e.what() << std::endl;
    }
}

std::vector<std::string>
dirList(std::string const& html)
{
    std::string cleaned_xml{cleanhtml(html)};
    pugi::xml_document doc;
    pugi::xml_parse_result result = doc.load_string(cleaned_xml.c_str());
    std::cout << "html.body.h1: " << doc.child("html").child("body").child("h1").child_value() << std::endl;
    auto table = doc.child("html").child("body").child("table");
    // tag::code[]
    for (auto it = table.begin(); it != table.end(); ++it) {
      std::cout << "TR;";
      auto td = it->child("td").child("a");
      for (pugi::xml_attribute_iterator ait = td.attributes_begin(); ait != td.attributes_end(); ++ait) {
          std::cout << " " << ait->name() << " <=> " << ait->value();
      }
      std::cout << " TD " << td << std::endl;
    }

    std::vector<std::string> ret;
    return ret;
}

std::vector<std::string>
parseManifest(std::string const& manifest_xml)
{
    pugi::xml_document doc;
    pugi::xml_parse_result result = doc.load_string(manifest_xml.c_str());

    if (result) {
        std::cout << "XML parsed without errors\n";
    } else {
        std::cout << "Error description: " << result.description() << "\n";
    }

    std::cout << "version name: " << doc.child("metadata").child("version").child_value() << std::endl;
    std::cout << "lastUpdated: " << doc.child("metadata").child("versioning").child("lastUpdated").child_value() << std::endl;

    std::vector<std::string> ret;
    return ret;
}

void
pull_one_url(std::string const& url, bool maven)
{
    std::unique_lock<std::mutex> lk(m);
    try {
        // That's all that is needed to do cleanup of used resources (RAII style).
        curlpp::Cleanup myCleanup;

        // Our request to be sent.
        curlpp::Easy myRequest;

        // Set the URL.
        myRequest.setOpt(new curlpp::options::Url(url));
        //myRequest.setOpt(new curlpp::options::FtpListOnly(true));

        // Send request and get a result.
        // By default the result goes to standard output.

        std::ostringstream os;
        curlpp::options::WriteStream ws(&os);
        myRequest.setOpt(ws);
        myRequest.perform();
        std::cout << "os: " << cleanhtml(os.str()) << std::endl;

        pugi::xml_document doc;
        pugi::xml_parse_result result = doc.load_string(cleanhtml(os.str()).c_str());

        if (result) {
            std::cout << "XML parsed without errors\n";
        } else {
            std::cout << "Error description: " << result.description() << "\n";
        }

        //simple_walker walker;
        //doc.traverse(walker);

        if (maven) {
            std::cout << "version name: " << doc.child("metadata").child("version").child_value() << std::endl;
            std::cout << "lastUpdated: " << doc.child("metadata").child("versioning").child("lastUpdated").child_value() << std::endl;
        } else {
            std::cout << "html.body.h1: " << doc.child("html").child("body").child("h1").child_value() << std::endl;
            auto table = doc.child("html").child("body").child("table");
            // tag::code[]
            for (auto it = table.begin(); it != table.end(); ++it) {
              std::cout << "TR;";
              auto td = it->child("td").child("a");
              for (pugi::xml_attribute_iterator ait = td.attributes_begin(); ait != td.attributes_end(); ++ait) {
                  std::cout << " " << ait->name() << " <=> " << ait->value();
              }
              std::cout << " TD " << td << std::endl;
            }
            // end::code[]
        }
    } catch(curlpp::RuntimeError & e) {
        std::cout << e.what() << std::endl;
    }

    catch(curlpp::LogicError & e) {
        std::cout << e.what() << std::endl;
    }

    is_ready = true;
    cv.notify_one();
}

int main(int, char **)
{
    std::string url1 = "https://oss.sonatype.org/content/repositories/snapshots/edu/berkeley/cs/chisel3_2.11/";
    std::string url2 = "https://oss.sonatype.org/content/repositories/snapshots/edu/berkeley/cs/chisel3_2.11/3.0-BETA-SNAPSHOT/maven-metadata.xml";
    std::string page;
    //getWebPage(url1, page);
    std::thread t1(getWebPage, url1, std::ref(page));
    std::unique_lock<std::mutex> lk(m);
    while (!is_ready) {
        cv.wait(lk);
    }
    t1.join();
    dirList(page);

    return 0;
}

<commit_msg>2nd level dirlist for curlpp example.<commit_after>/**
* \file
* The most simple example.
* install: `yaourt -S aur/libcurlpp`
* [curlpp examples](https://github.com/jpbarrette/curlpp/tree/master/examples)
* [curlpp download wnd write callback example](https://github.com/datacratic/curlpp/blob/master/examples/example11.cpp)
* [Link errors with curlpp](http://stackoverflow.com/questions/33420631/link-errors-with-curlpp)
*/

#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <thread>
#include <mutex>
#include <condition_variable>

// curlpp specific headers
#include <curlpp/cURLpp.hpp>
#include <curlpp/Easy.hpp>
#include <curlpp/Options.hpp>

// pugixml
#include <pugixml.hpp>

// [clean html with C or C++ tidy](http://www.cppfun.com/clean-html-with-c-or-cpp-tidy.htm)
// [tidy 5.2.0-1](https://www.archlinux.org/packages/extra/x86_64/tidy/)
// `pacman -S extra/tidy`

#include <tidy.h>
#include <buffio.h>

std::string cleanhtml(const std::string &html)
{
    // init a tidy document
    TidyDoc tidy_doc=tidyCreate();
    TidyBuffer output_buffer= {0};

    // configure tidy
    // the flags tell tidy to output xml and disable warnings
    bool config_success=tidyOptSetBool(tidy_doc,TidyXmlOut,yes)
                        && tidyOptSetBool(tidy_doc,TidyQuiet,yes)
                        && tidyOptSetBool(tidy_doc,TidyNumEntities,yes)
                        && tidyOptSetBool(tidy_doc,TidyShowWarnings,no);

    int tidy_rescode=-1;

    // parse input
    if(config_success)
        tidy_rescode=tidyParseString(tidy_doc,html.c_str());

    // process html
    if(tidy_rescode>=0)
        tidy_rescode=tidySaveBuffer(tidy_doc,&output_buffer);

    if(tidy_rescode<0)
        throw("tidy has a error: "+tidy_rescode);

    std::string result=(char *)output_buffer.bp;
    tidyBufFree(&output_buffer);
    tidyRelease(tidy_doc);

    return result;
}

using namespace curlpp::options;

bool is_ready(false);
std::mutex m;
std::condition_variable cv;

struct simple_walker: pugi::xml_tree_walker {
    virtual bool for_each(pugi::xml_node& node) {
        for (int i = 0; i < depth(); ++i) std::cout << "  "; // indentation
        std::cout << /* node_types[node.type()] << */ ": name='" << node.name() << "', value='" << node.value() << "'\n";
        return true; // continue traversal
    }
};

void
getWebPage(std::string const& url, std::string& page)
{
    //std::unique_lock<std::mutex> lk(m);
    try {
        // That's all that is needed to do cleanup of used resources (RAII style).
        curlpp::Cleanup myCleanup;

        // Our request to be sent.
        curlpp::Easy myRequest;

        // Set the URL.
        myRequest.setOpt(new curlpp::options::Url(url));

        // Send request and get a result.
        // By default the result goes to standard output.

        std::ostringstream os;
        curlpp::options::WriteStream ws(&os);
        myRequest.setOpt(ws);
        myRequest.perform();

        page = std::move(os.str());

        is_ready = true;
        cv.notify_one();

    } catch(curlpp::RuntimeError & e) {
        std::cout << e.what() << std::endl;
    }

    catch(curlpp::LogicError & e) {
        std::cout << e.what() << std::endl;
    }
}

std::vector<std::string>
dirList(std::string const& html)
{
    std::string cleaned_xml{cleanhtml(html)};
    pugi::xml_document doc;
    pugi::xml_parse_result result = doc.load_string(cleaned_xml.c_str());
    auto table = doc.child("html").child("body").child("table");
    std::vector<std::string> ret;
    for (auto it = table.begin(); it != table.end(); ++it) {
      auto td = it->child("td").child("a");
      for (pugi::xml_attribute_iterator ait = td.attributes_begin(); ait != td.attributes_end(); ++ait) {
          //std::cout << " " << ait->name() << " <=> " << ait->value() << std::endl;
          ret.emplace_back(ait->value());
      }
    }

    return std::move(ret);
}

std::vector<std::string>
parseManifest(std::string const& manifest_xml)
{
    pugi::xml_document doc;
    pugi::xml_parse_result result = doc.load_string(manifest_xml.c_str());

    if (result) {
        std::cout << "XML parsed without errors\n";
    } else {
        std::cout << "Error description: " << result.description() << "\n";
    }

    std::cout << "version name: " << doc.child("metadata").child("version").child_value() << std::endl;
    std::cout << "lastUpdated: " << doc.child("metadata").child("versioning").child("lastUpdated").child_value() << std::endl;

    std::vector<std::string> ret;
    return ret;
}

int main(int, char **)
{
    std::string url1 = "https://oss.sonatype.org/content/repositories/snapshots/edu/berkeley/cs/chisel3_2.11/";
    std::string url2 = "https://oss.sonatype.org/content/repositories/snapshots/edu/berkeley/cs/chisel3_2.11/3.0-BETA-SNAPSHOT/maven-metadata.xml";
    std::string page;
    //getWebPage(url1, page);
    std::thread t1(getWebPage, url1, std::ref(page));
    //std::unique_lock<std::mutex> lk(m);
    //while (!is_ready) {
    //    cv.wait(lk);
    //}
    t1.join();
    auto dirlist = dirList(page);
    for (auto const& dirent : dirlist) {
        std::cout << dirent << std::endl;
    }
    is_ready = false;
    std::thread t2(getWebPage, dirlist[1], std::ref(page));
    //std::unique_lock<std::mutex> lk2(m);
    //while (!is_ready) {
    //    cv.wait(lk2);
    //}
    t2.join();
    dirlist = dirList(page);
    for (auto const& dirent : dirlist) {
        std::cout << dirent << std::endl;
    }

    return 0;
}

<|endoftext|>


<commit_before>/*
    This file is part of Magnum.

    Copyright © 2010, 2011, 2012, 2013 Vladimír Vondruš <mosra@centrum.cz>

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

#include <sstream>
#include <tuple>
#include <Containers/Array.h>
#include <TestSuite/Tester.h>
#include <Utility/Directory.h>

#include "ColorFormat.h"
#include "Image.h"
#include "Trade/ImageData.h"
#include "TgaImageConverter/TgaImageConverter.h"
#include "TgaImporter/TgaImporter.h"

#include "configure.h"

namespace Magnum { namespace Trade { namespace Test {

class TgaImageConverterTest: public TestSuite::Tester {
    public:
        explicit TgaImageConverterTest();

        void wrongFormat();
        void wrongType();

        void data();
};

namespace {
    constexpr const char originalData[] = {
        1, 2, 3, 2, 3, 4,
        3, 4, 5, 4, 5, 6,
        5, 6, 7, 6, 7, 8
    };

    #ifndef MAGNUM_TARGET_GLES
    const ImageReference2D original(ColorFormat::BGR, ColorType::UnsignedByte, {2, 3}, originalData);
    #else
    const ImageReference2D original(ColorFormat::RGB, ColorType::UnsignedByte, {2, 3}, originalData);
    #endif
}

TgaImageConverterTest::TgaImageConverterTest() {
    addTests({&TgaImageConverterTest::wrongFormat,
              &TgaImageConverterTest::wrongType,

              &TgaImageConverterTest::data});
}

void TgaImageConverterTest::wrongFormat() {
    ImageReference2D image(ColorFormat::RG, ColorType::UnsignedByte, {}, nullptr);

    std::ostringstream out;
    Error::setOutput(&out);

    const auto data = TgaImageConverter().exportToData(image);
    CORRADE_VERIFY(!data);
    CORRADE_COMPARE(out.str(), "Trade::TgaImageConverter::convertToData(): unsupported image format ColorFormat::RG\n");
}

void TgaImageConverterTest::wrongType() {
    ImageReference2D image(ColorFormat::Red, ColorType::Float, {}, nullptr);

    std::ostringstream out;
    Error::setOutput(&out);

    const auto data = TgaImageConverter().exportToData(image);
    CORRADE_VERIFY(!data);
    CORRADE_COMPARE(out.str(), "Trade::TgaImageConverter::convertToData(): unsupported image type ColorType::Float\n");
}

void TgaImageConverterTest::data() {
    const auto data = TgaImageConverter().exportToData(original);

    TgaImporter importer;
    CORRADE_VERIFY(importer.openData(data));
    std::optional<Trade::ImageData2D> converted = importer.image2D(0);
    CORRADE_VERIFY(converted);

    CORRADE_COMPARE(converted->size(), Vector2i(2, 3));
    #ifndef MAGNUM_TARGET_GLES
    CORRADE_COMPARE(converted->format(), ColorFormat::BGR);
    #else
    CORRADE_COMPARE(converted->format(), ColorFormat::RGB);
    #endif
    CORRADE_COMPARE(converted->type(), ColorType::UnsignedByte);
    CORRADE_COMPARE(std::string(reinterpret_cast<const char*>(converted->data()), 2*3*3),
                    std::string(reinterpret_cast<const char*>(original.data()), 2*3*3));
}

}}}

CORRADE_TEST_MAIN(Magnum::Trade::Test::TgaImageConverterTest)
<commit_msg>GCC 4.6 compatibility: can't use both constexpr and const.<commit_after>/*
    This file is part of Magnum.

    Copyright © 2010, 2011, 2012, 2013 Vladimír Vondruš <mosra@centrum.cz>

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

#include <sstream>
#include <tuple>
#include <Containers/Array.h>
#include <TestSuite/Tester.h>
#include <Utility/Directory.h>

#include "ColorFormat.h"
#include "Image.h"
#include "Trade/ImageData.h"
#include "TgaImageConverter/TgaImageConverter.h"
#include "TgaImporter/TgaImporter.h"

#include "configure.h"

namespace Magnum { namespace Trade { namespace Test {

class TgaImageConverterTest: public TestSuite::Tester {
    public:
        explicit TgaImageConverterTest();

        void wrongFormat();
        void wrongType();

        void data();
};

namespace {
    constexpr char originalData[] = {
        1, 2, 3, 2, 3, 4,
        3, 4, 5, 4, 5, 6,
        5, 6, 7, 6, 7, 8
    };

    #ifndef MAGNUM_TARGET_GLES
    const ImageReference2D original(ColorFormat::BGR, ColorType::UnsignedByte, {2, 3}, originalData);
    #else
    const ImageReference2D original(ColorFormat::RGB, ColorType::UnsignedByte, {2, 3}, originalData);
    #endif
}

TgaImageConverterTest::TgaImageConverterTest() {
    addTests({&TgaImageConverterTest::wrongFormat,
              &TgaImageConverterTest::wrongType,

              &TgaImageConverterTest::data});
}

void TgaImageConverterTest::wrongFormat() {
    ImageReference2D image(ColorFormat::RG, ColorType::UnsignedByte, {}, nullptr);

    std::ostringstream out;
    Error::setOutput(&out);

    const auto data = TgaImageConverter().exportToData(image);
    CORRADE_VERIFY(!data);
    CORRADE_COMPARE(out.str(), "Trade::TgaImageConverter::convertToData(): unsupported image format ColorFormat::RG\n");
}

void TgaImageConverterTest::wrongType() {
    ImageReference2D image(ColorFormat::Red, ColorType::Float, {}, nullptr);

    std::ostringstream out;
    Error::setOutput(&out);

    const auto data = TgaImageConverter().exportToData(image);
    CORRADE_VERIFY(!data);
    CORRADE_COMPARE(out.str(), "Trade::TgaImageConverter::convertToData(): unsupported image type ColorType::Float\n");
}

void TgaImageConverterTest::data() {
    const auto data = TgaImageConverter().exportToData(original);

    TgaImporter importer;
    CORRADE_VERIFY(importer.openData(data));
    std::optional<Trade::ImageData2D> converted = importer.image2D(0);
    CORRADE_VERIFY(converted);

    CORRADE_COMPARE(converted->size(), Vector2i(2, 3));
    #ifndef MAGNUM_TARGET_GLES
    CORRADE_COMPARE(converted->format(), ColorFormat::BGR);
    #else
    CORRADE_COMPARE(converted->format(), ColorFormat::RGB);
    #endif
    CORRADE_COMPARE(converted->type(), ColorType::UnsignedByte);
    CORRADE_COMPARE(std::string(reinterpret_cast<const char*>(converted->data()), 2*3*3),
                    std::string(reinterpret_cast<const char*>(original.data()), 2*3*3));
}

}}}

CORRADE_TEST_MAIN(Magnum::Trade::Test::TgaImageConverterTest)
<|endoftext|>