<commit_before>#include <string>
#include <boost/random/additive_combine.hpp>
#include <stan/io/dump.hpp>
#include <test/unit/mcmc/hmc/mock_hmc.hpp>
#include <stan/mcmc/hmc/hamiltonians/dense_e_metric.hpp>
#include <test/test-models/good/mcmc/hmc/hamiltonians/funnel.hpp>
#include <stan/callbacks/stream_logger.hpp>
#include <test/unit/util.hpp>
#include <gtest/gtest.h>
#include <iostream>

typedef boost::ecuyer1988 rng_t;

TEST(McmcDenseEMetric, sample_p) {
  rng_t base_rng(0);

  Eigen::VectorXd q(2);
  q(0) = 5;
  q(1) = 1;

  Eigen::MatrixXd  m(2,2);
  m(0, 0) = 3.0;
  m(1, 0) = -2.0;
  m(0, 1) = -2.0;
  m(1, 1) = 4.0;

  Eigen::MatrixXd  m_inv(2,2);
  double det_inv = 1.0 / (3.0 * 4.0 - (-2.0) * (-2.0));
  m_inv(0, 0) = m(1, 1) * det_inv;
  m_inv(1, 0) = -1.0 * m(1, 0) * det_inv;
  m_inv(0, 1) = -1.0 * m(0, 1) * det_inv;
  m_inv(1, 1) = m(0, 0) * det_inv;

  stan::mcmc::mock_model model(q.size());

  stan::mcmc::dense_e_metric<stan::mcmc::mock_model, rng_t> metric(model);
  stan::mcmc::dense_e_point z(q.size());
  z.set_metric(m_inv);

  int n_samples = 1000;

  Eigen::MatrixXd sample_cov(2,2);
  sample_cov(0, 0) = 0.0;
  sample_cov(0, 1) = 0.0;
  sample_cov(1, 0) = 0.0;
  sample_cov(1, 1) = 0.0;

  for (int i = 0; i < n_samples; ++i) {
    metric.sample_p(z, base_rng);
    sample_cov(0, 0) += z.p[0] * z.p[0] / (n_samples + 0.0);
    sample_cov(0, 1) += z.p[0] * z.p[1] / (n_samples + 0.0);
    sample_cov(1, 0) += z.p[1] * z.p[0] / (n_samples + 0.0);
    sample_cov(1, 1) += z.p[1] * z.p[1] / (n_samples + 0.0);
  }

  Eigen::MatrixXd var(2,2);
  var(0, 0) = 2 * m(0, 0);
  var(1, 0) = m(1, 0) * m(1, 0) + m(1, 1) * m(0, 0);
  var(0, 1) = m(0, 1) * m(0, 1) + m(1, 1) * m(0, 0);
  var(1, 1) = 2 * m(1, 1);

  // Covariance matrix within 5sigma of expected value (comes from a Wishart distribution)
  EXPECT_TRUE(std::fabs(m(0, 0)   - sample_cov(0, 0)) < 5.0 * sqrt(var(0, 0) / n_samples));
  EXPECT_TRUE(std::fabs(m(1, 0)   - sample_cov(1, 0)) < 5.0 * sqrt(var(1, 0) / n_samples));
  EXPECT_TRUE(std::fabs(m(0, 1)   - sample_cov(0, 1)) < 5.0 * sqrt(var(0, 1) / n_samples));
  EXPECT_TRUE(std::fabs(m(1, 1)   - sample_cov(1, 1)) < 5.0 * sqrt(var(1, 1) / n_samples));
}

TEST(McmcDenseEMetric, gradients) {
  rng_t base_rng(0);

  Eigen::VectorXd q = Eigen::VectorXd::Ones(11);

  stan::mcmc::dense_e_point z(q.size());
  z.q = q;
  z.p.setOnes();

  std::fstream data_stream(std::string("").c_str(), std::fstream::in);
  stan::io::dump data_var_context(data_stream);
  data_stream.close();

  std::stringstream model_output;
  std::stringstream debug, info, warn, error, fatal;
  stan::callbacks::stream_logger logger(debug, info, warn, error, fatal);

  funnel_model_namespace::funnel_model model(data_var_context, &model_output);

  stan::mcmc::dense_e_metric<funnel_model_namespace::funnel_model, rng_t> metric(model);

  double epsilon = 1e-6;

  metric.init(z, logger);
  Eigen::VectorXd g1 = metric.dtau_dq(z, logger);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.q(i) += epsilon;
    metric.update_potential(z, logger);
    delta += metric.tau(z);

    z.q(i) -= 2 * epsilon;
    metric.update_potential(z, logger);
    delta -= metric.tau(z);

    z.q(i) += epsilon;
    metric.update_potential(z, logger);

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g1(i), epsilon);
  }

  Eigen::VectorXd g2 = metric.dtau_dp(z);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.p(i) += epsilon;
    delta += metric.tau(z);

    z.p(i) -= 2 * epsilon;
    delta -= metric.tau(z);

    z.p(i) += epsilon;

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g2(i), epsilon);
  }

  Eigen::VectorXd g3 = metric.dphi_dq(z, logger);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.q(i) += epsilon;
    metric.update_potential(z, logger);
    delta += metric.phi(z);

    z.q(i) -= 2 * epsilon;
    metric.update_potential(z, logger);
    delta -= metric.phi(z);

    z.q(i) += epsilon;
    metric.update_potential(z, logger);

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g3(i), epsilon);

  }

  EXPECT_EQ("", model_output.str());
  EXPECT_EQ("", debug.str());
  EXPECT_EQ("", info.str());
  EXPECT_EQ("", warn.str());
  EXPECT_EQ("", error.str());
  EXPECT_EQ("", fatal.str());
}

TEST(McmcDenseEMetric, streams) {
  stan::test::capture_std_streams();

  rng_t base_rng(0);

  Eigen::VectorXd q(2);
  q(0) = 5;
  q(1) = 1;

  stan::mcmc::mock_model model(q.size());

  // typedef to use within Google Test macros
  typedef stan::mcmc::dense_e_metric<stan::mcmc::mock_model, rng_t> dense_e;

  EXPECT_NO_THROW(dense_e metric(model));

  stan::test::reset_std_streams();
  EXPECT_EQ("", stan::test::cout_ss.str());
  EXPECT_EQ("", stan::test::cerr_ss.str());
}
<commit_msg>Eigen foo + removing some unnecessy code<commit_after>#include <string>
#include <boost/random/additive_combine.hpp>
#include <stan/io/dump.hpp>
#include <test/unit/mcmc/hmc/mock_hmc.hpp>
#include <stan/mcmc/hmc/hamiltonians/dense_e_metric.hpp>
#include <test/test-models/good/mcmc/hmc/hamiltonians/funnel.hpp>
#include <stan/callbacks/stream_logger.hpp>
#include <test/unit/util.hpp>
#include <gtest/gtest.h>
#include <iostream>

typedef boost::ecuyer1988 rng_t;

TEST(McmcDenseEMetric, sample_p) {
  rng_t base_rng(0);

  Eigen::Matrix2d  m(2,2);
  m(0, 0) = 3.0;
  m(1, 0) = -2.0;
  m(0, 1) = -2.0;
  m(1, 1) = 4.0;

  Eigen::Matrix2d  m_inv = m.inverse();

  stan::mcmc::mock_model model(2);

  stan::mcmc::dense_e_metric<stan::mcmc::mock_model, rng_t> metric(model);
  stan::mcmc::dense_e_point z(2);
  z.set_metric(m_inv);

  int n_samples = 1000;

  Eigen::Matrix2d sample_cov(2,2);
  sample_cov(0, 0) = 0.0;
  sample_cov(0, 1) = 0.0;
  sample_cov(1, 0) = 0.0;
  sample_cov(1, 1) = 0.0;

  for (int i = 0; i < n_samples; ++i) {
    metric.sample_p(z, base_rng);
    sample_cov(0, 0) += z.p[0] * z.p[0] / (n_samples + 0.0);
    sample_cov(0, 1) += z.p[0] * z.p[1] / (n_samples + 0.0);
    sample_cov(1, 0) += z.p[1] * z.p[0] / (n_samples + 0.0);
    sample_cov(1, 1) += z.p[1] * z.p[1] / (n_samples + 0.0);
  }

  Eigen::Matrix2d var(2,2);
  var(0, 0) = 2 * m(0, 0);
  var(1, 0) = m(1, 0) * m(1, 0) + m(1, 1) * m(0, 0);
  var(0, 1) = m(0, 1) * m(0, 1) + m(1, 1) * m(0, 0);
  var(1, 1) = 2 * m(1, 1);

  // Covariance matrix within 5sigma of expected value (comes from a Wishart distribution)
  EXPECT_TRUE(std::fabs(m(0, 0)   - sample_cov(0, 0)) < 5.0 * sqrt(var(0, 0) / n_samples));
  EXPECT_TRUE(std::fabs(m(1, 0)   - sample_cov(1, 0)) < 5.0 * sqrt(var(1, 0) / n_samples));
  EXPECT_TRUE(std::fabs(m(0, 1)   - sample_cov(0, 1)) < 5.0 * sqrt(var(0, 1) / n_samples));
  EXPECT_TRUE(std::fabs(m(1, 1)   - sample_cov(1, 1)) < 5.0 * sqrt(var(1, 1) / n_samples));
}

TEST(McmcDenseEMetric, gradients) {
  rng_t base_rng(0);

  Eigen::VectorXd q = Eigen::VectorXd::Ones(11);

  stan::mcmc::dense_e_point z(q.size());
  z.q = q;
  z.p.setOnes();

  std::fstream data_stream(std::string("").c_str(), std::fstream::in);
  stan::io::dump data_var_context(data_stream);
  data_stream.close();

  std::stringstream model_output;
  std::stringstream debug, info, warn, error, fatal;
  stan::callbacks::stream_logger logger(debug, info, warn, error, fatal);

  funnel_model_namespace::funnel_model model(data_var_context, &model_output);

  stan::mcmc::dense_e_metric<funnel_model_namespace::funnel_model, rng_t> metric(model);

  double epsilon = 1e-6;

  metric.init(z, logger);
  Eigen::VectorXd g1 = metric.dtau_dq(z, logger);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.q(i) += epsilon;
    metric.update_potential(z, logger);
    delta += metric.tau(z);

    z.q(i) -= 2 * epsilon;
    metric.update_potential(z, logger);
    delta -= metric.tau(z);

    z.q(i) += epsilon;
    metric.update_potential(z, logger);

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g1(i), epsilon);
  }

  Eigen::VectorXd g2 = metric.dtau_dp(z);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.p(i) += epsilon;
    delta += metric.tau(z);

    z.p(i) -= 2 * epsilon;
    delta -= metric.tau(z);

    z.p(i) += epsilon;

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g2(i), epsilon);
  }

  Eigen::VectorXd g3 = metric.dphi_dq(z, logger);

  for (int i = 0; i < z.q.size(); ++i) {

    double delta = 0;

    z.q(i) += epsilon;
    metric.update_potential(z, logger);
    delta += metric.phi(z);

    z.q(i) -= 2 * epsilon;
    metric.update_potential(z, logger);
    delta -= metric.phi(z);

    z.q(i) += epsilon;
    metric.update_potential(z, logger);

    delta /= 2 * epsilon;

    EXPECT_NEAR(delta, g3(i), epsilon);

  }

  EXPECT_EQ("", model_output.str());
  EXPECT_EQ("", debug.str());
  EXPECT_EQ("", info.str());
  EXPECT_EQ("", warn.str());
  EXPECT_EQ("", error.str());
  EXPECT_EQ("", fatal.str());
}

TEST(McmcDenseEMetric, streams) {
  stan::test::capture_std_streams();

  rng_t base_rng(0);

  Eigen::VectorXd q(2);
  q(0) = 5;
  q(1) = 1;

  stan::mcmc::mock_model model(q.size());

  // typedef to use within Google Test macros
  typedef stan::mcmc::dense_e_metric<stan::mcmc::mock_model, rng_t> dense_e;

  EXPECT_NO_THROW(dense_e metric(model));

  stan::test::reset_std_streams();
  EXPECT_EQ("", stan::test::cout_ss.str());
  EXPECT_EQ("", stan::test::cerr_ss.str());
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 Mikhail Baranov
 * Copyright (c) 2015 Victor Gaydov
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <CppUTest/TestHarness.h>

#include <set>

#include "roc_core/log.h"

#include "roc_fec/encoder.h"
#include "roc_fec/decoder.h"

#include "roc_fec/ldpc_block_encoder.h"
#include "roc_fec/ldpc_block_decoder.h"

#include "roc_packet/iaudio_packet.h"
#include "roc_packet/packet_queue.h"
#include "roc_packet/interleaver.h"

#include "roc_rtp/parser.h"
#include "roc_rtp/composer.h"

#include "roc_config/config.h"

namespace roc {
namespace test {

using namespace fec;
using namespace packet;

namespace {

const size_t N_DATA_PACKETS = ROC_CONFIG_DEFAULT_FEC_BLOCK_DATA_PACKETS;
const size_t N_FEC_PACKETS = ROC_CONFIG_DEFAULT_FEC_BLOCK_REDUNDANT_PACKETS;

const size_t N_SAMPLES = ROC_CONFIG_DEFAULT_PACKET_SAMPLES;
const size_t N_CH = 2;

const int LEFT = (1 << 0);
const int RIGHT = (1 << 1);

const double EPSILON = 0.0001;

typedef LDPC_BlockEncoder BlockEncoder;
typedef LDPC_BlockDecoder BlockDecoder;

// Divides packets from Encoder into two queues: data and fec packets,
// as needed for Decoder.
class PacketDispatcher : public IPacketWriter {
public:
    PacketDispatcher()
        : packet_num_(0) {
        reset();
    }

    virtual ~PacketDispatcher() {
    }

    virtual void write(const IPacketPtr& p) {
        if (lost_packet_nums_.find(packet_num_) != lost_packet_nums_.end()) {
            ++packet_num_;
            return;
        }

        if (p->type() == IAudioPacket::Type) {
            data_queue_.write(p);
        } else if (IFECPacket::Type) {
            fec_queue_.write(p);
        }

        if (++packet_num_ >= N_DATA_PACKETS + N_FEC_PACKETS) {
            packet_num_ = 0;
        }
    }

    IPacketReader& get_data_reader() {
        return data_queue_;
    }

    IPacketReader& get_fec_reader() {
        return fec_queue_;
    }

    size_t get_data_size() {
        return data_queue_.size();
    }

    size_t get_fec_size() {
        return fec_queue_.size();
    }

    //! Clears both queues.
    void reset() {
        const size_t data_packets_n = data_queue_.size();
        const size_t fec_packets_n = fec_queue_.size();

        for (size_t i = 0; i < data_packets_n; ++i) {
            data_queue_.read();
        }

        for (size_t i = 0; i < fec_packets_n; ++i) {
            fec_queue_.read();
        }

        packet_num_ = 0;
        lost_packet_nums_.clear();
    }

    void lose(const size_t n) {
        lost_packet_nums_.insert(n);
    }

private:
    size_t packet_num_;

    PacketQueue data_queue_;
    PacketQueue fec_queue_;

    std::set<size_t> lost_packet_nums_;
};

} // namespace

TEST_GROUP(fec_codec_integration) {
    rtp::Composer composer;
    PacketDispatcher pckt_disp;

    IPacketPtr data_packets[N_DATA_PACKETS];

    void setup() {
        fill_all_packets(0, N_DATA_PACKETS);
    }

    void fill_all_packets(size_t sn, size_t n_pkts) {
        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            data_packets[i] = fill_one_packet(sn + i, n_pkts);
        }
    }

    IPacketPtr fill_one_packet(size_t sn, size_t n_pkts) {
        IPacketPtr packet = composer.compose(IAudioPacket::Type);
        CHECK(packet);

        sample_t samples[N_SAMPLES * N_CH] = {};

        for (size_t n = 0; n < N_SAMPLES * N_CH; n += N_CH) {
            sample_t s =
                sample_t(N_SAMPLES * N_CH * sn + n) / (N_SAMPLES * N_CH * n_pkts);

            samples[n + 0] = +s;
            samples[n + 1] = -s;
        }

        IAudioPacket* audio_packet = (IAudioPacket*)packet.get();
        audio_packet->set_seqnum((seqnum_t)sn);
        audio_packet->set_size(LEFT | RIGHT, N_SAMPLES);
        audio_packet->write_samples(LEFT | RIGHT, 0, samples, N_SAMPLES);

        return packet;
    }

    void check_audio_packet(IPacketConstPtr packet, size_t sn, size_t n_pkts) {
        CHECK(packet);
        const IAudioPacket* audio_packet = (const IAudioPacket*)packet.get();

        sample_t left[N_SAMPLES] = {};
        sample_t right[N_SAMPLES] = {};

        LONGS_EQUAL((seqnum_t)sn, audio_packet->seqnum());

        CHECK(audio_packet->num_samples() == N_SAMPLES);
        CHECK(audio_packet->channels() == (LEFT | RIGHT));

        CHECK(audio_packet->read_samples(LEFT, 0, left, N_SAMPLES) == N_SAMPLES);
        CHECK(audio_packet->read_samples(RIGHT, 0, right, N_SAMPLES) == N_SAMPLES);

        for (size_t n = 0; n < N_SAMPLES; ++n) {
            sample_t s =
                sample_t(N_SAMPLES * N_CH * sn + n * N_CH) / (N_SAMPLES * N_CH * n_pkts);

            DOUBLES_EQUAL(+s, left[n], EPSILON);
            DOUBLES_EQUAL(-s, right[n], EPSILON);
        }
    }
};

TEST(fec_codec_integration, encode) {
    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
    CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_DATA_PACKETS);
    }
}

TEST(fec_codec_integration, 1_loss) {
    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    pckt_disp.lose(11);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    LONGS_EQUAL(N_DATA_PACKETS - 1, pckt_disp.get_data_size());
    LONGS_EQUAL(N_FEC_PACKETS, pckt_disp.get_fec_size());

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_DATA_PACKETS);
    }
}

TEST(fec_codec_integration, multiblocks_1_loss) {
    enum { N_BLKS = 40 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        size_t lost_sq = size_t(-1);
        if (block_num != 5 && block_num != 21 && block_num != 22) {
            lost_sq = (block_num + 1) % (N_DATA_PACKETS + N_FEC_PACKETS);
            pckt_disp.lose(lost_sq);
        }

        fill_all_packets(N_DATA_PACKETS * block_num, N_DATA_PACKETS * N_BLKS);

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            encoder.write(data_packets[i]);
        }

        if (lost_sq == size_t(-1)) {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);
        } else if (lost_sq < N_DATA_PACKETS) {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS - 1);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);
        } else {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS - 1);
        }

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            IPacketConstPtr p = decoder.read();
            CHECK(p);
            check_audio_packet(p, N_DATA_PACKETS * block_num + i,
                               N_DATA_PACKETS * N_BLKS);
        }

        pckt_disp.reset();
    }
}

TEST(fec_codec_integration, interleaver) {
    enum { N_PACKETS = N_DATA_PACKETS * 30 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Interleaver intrl(pckt_disp);

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    IPacketPtr many_packets[N_PACKETS];

    for (size_t i = 0; i < N_PACKETS; ++i) {
        many_packets[i] = fill_one_packet(i, N_PACKETS);
        encoder.write(many_packets[i]);
    }

    intrl.flush();

    for (size_t i = 0; i < N_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_PACKETS);
    }
}

TEST(fec_codec_integration, decoding_when_multiple_blocks_in_queue) {
    enum { N_BLKS = 3 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        fill_all_packets(N_DATA_PACKETS * block_num, N_DATA_PACKETS * N_BLKS);

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            encoder.write(data_packets[i]);
        }
    }

    CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS * N_BLKS);
    CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS * N_BLKS);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            IPacketConstPtr p = decoder.read();
            CHECK(p);
            check_audio_packet(p, N_DATA_PACKETS * block_num + i,
                               N_DATA_PACKETS * N_BLKS);
        }

        pckt_disp.reset();
    }
}

IGNORE_TEST(fec_codec_integration, decoding_late_packet) {
    // TODO
}

IGNORE_TEST(fec_codec_integration, get_packets_before_marker_bit) {
    // TODO
}

IGNORE_TEST(fec_codec_integration, repair_wrong_source_or_seqnum) {
    // TODO
}

} // namespace test
} // namespace roc
<commit_msg>New test: fec_codec_integration decoding_late_packet<commit_after>/*
 * Copyright (c) 2015 Mikhail Baranov
 * Copyright (c) 2015 Victor Gaydov
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <CppUTest/TestHarness.h>

#include <set>

#include "roc_core/log.h"

#include "roc_fec/encoder.h"
#include "roc_fec/decoder.h"

#include "roc_fec/ldpc_block_encoder.h"
#include "roc_fec/ldpc_block_decoder.h"

#include "roc_packet/iaudio_packet.h"
#include "roc_packet/packet_queue.h"
#include "roc_packet/interleaver.h"

#include "roc_rtp/parser.h"
#include "roc_rtp/composer.h"

#include "roc_config/config.h"

namespace roc {
namespace test {

using namespace fec;
using namespace packet;

namespace {

const size_t N_DATA_PACKETS = ROC_CONFIG_DEFAULT_FEC_BLOCK_DATA_PACKETS;
const size_t N_FEC_PACKETS = ROC_CONFIG_DEFAULT_FEC_BLOCK_REDUNDANT_PACKETS;

const size_t N_SAMPLES = ROC_CONFIG_DEFAULT_PACKET_SAMPLES;
const size_t N_CH = 2;

const int LEFT = (1 << 0);
const int RIGHT = (1 << 1);

const double EPSILON = 0.0001;

typedef LDPC_BlockEncoder BlockEncoder;
typedef LDPC_BlockDecoder BlockDecoder;

// Divides packets from Encoder into two queues: data and fec packets,
// as needed for Decoder.
class PacketDispatcher : public IPacketWriter {
public:
    PacketDispatcher()
        : packet_num_(0) {
        reset();
    }

    virtual ~PacketDispatcher() {
    }

    virtual void write(const IPacketPtr& p) {
        if (lost_packet_nums_.find(packet_num_) != lost_packet_nums_.end()) {
            ++packet_num_;
            return;
        }

        if (p->type() == IAudioPacket::Type) {
            data_queue_.write(p);
        } else if (IFECPacket::Type) {
            fec_queue_.write(p);
        }

        if (++packet_num_ >= N_DATA_PACKETS + N_FEC_PACKETS) {
            packet_num_ = 0;
        }
    }

    IPacketReader& get_data_reader() {
        return data_queue_;
    }

    IPacketReader& get_fec_reader() {
        return fec_queue_;
    }

    size_t get_data_size() {
        return data_queue_.size();
    }

    size_t get_fec_size() {
        return fec_queue_.size();
    }

    //! Clears both queues.
    void reset() {
        const size_t data_packets_n = data_queue_.size();
        const size_t fec_packets_n = fec_queue_.size();

        for (size_t i = 0; i < data_packets_n; ++i) {
            data_queue_.read();
        }

        for (size_t i = 0; i < fec_packets_n; ++i) {
            fec_queue_.read();
        }

        packet_num_ = 0;
        lost_packet_nums_.clear();
    }

    void lose(const size_t n) {
        lost_packet_nums_.insert(n);
    }

private:
    size_t packet_num_;

    PacketQueue data_queue_;
    PacketQueue fec_queue_;

    std::set<size_t> lost_packet_nums_;
};

} // namespace

TEST_GROUP(fec_codec_integration) {
    rtp::Composer composer;
    PacketDispatcher pckt_disp;

    IPacketPtr data_packets[N_DATA_PACKETS];

    void setup() {
        fill_all_packets(0, N_DATA_PACKETS);
    }

    void fill_all_packets(size_t sn, size_t n_pkts) {
        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            data_packets[i] = fill_one_packet(sn + i, n_pkts);
        }
    }

    IPacketPtr fill_one_packet(size_t sn, size_t n_pkts) {
        IPacketPtr packet = composer.compose(IAudioPacket::Type);
        CHECK(packet);

        sample_t samples[N_SAMPLES * N_CH] = {};

        for (size_t n = 0; n < N_SAMPLES * N_CH; n += N_CH) {
            sample_t s =
                sample_t(N_SAMPLES * N_CH * sn + n) / (N_SAMPLES * N_CH * n_pkts);

            samples[n + 0] = +s;
            samples[n + 1] = -s;
        }

        IAudioPacket* audio_packet = (IAudioPacket*)packet.get();
        audio_packet->set_seqnum((seqnum_t)sn);
        audio_packet->set_size(LEFT | RIGHT, N_SAMPLES);
        audio_packet->write_samples(LEFT | RIGHT, 0, samples, N_SAMPLES);

        return packet;
    }

    void check_audio_packet(IPacketConstPtr packet, size_t sn, size_t n_pkts) {
        CHECK(packet);
        const IAudioPacket* audio_packet = (const IAudioPacket*)packet.get();

        sample_t left[N_SAMPLES] = {};
        sample_t right[N_SAMPLES] = {};

        LONGS_EQUAL((seqnum_t)sn, audio_packet->seqnum());

        CHECK(audio_packet->num_samples() == N_SAMPLES);
        CHECK(audio_packet->channels() == (LEFT | RIGHT));

        CHECK(audio_packet->read_samples(LEFT, 0, left, N_SAMPLES) == N_SAMPLES);
        CHECK(audio_packet->read_samples(RIGHT, 0, right, N_SAMPLES) == N_SAMPLES);

        for (size_t n = 0; n < N_SAMPLES; ++n) {
            sample_t s =
                sample_t(N_SAMPLES * N_CH * sn + n * N_CH) / (N_SAMPLES * N_CH * n_pkts);

            DOUBLES_EQUAL(+s, left[n], EPSILON);
            DOUBLES_EQUAL(-s, right[n], EPSILON);
        }
    }
};

TEST(fec_codec_integration, encode) {
    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
    CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_DATA_PACKETS);
    }
}

TEST(fec_codec_integration, 1_loss) {
    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    pckt_disp.lose(11);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    LONGS_EQUAL(N_DATA_PACKETS - 1, pckt_disp.get_data_size());
    LONGS_EQUAL(N_FEC_PACKETS, pckt_disp.get_fec_size());

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_DATA_PACKETS);
    }
}

TEST(fec_codec_integration, multiblocks_1_loss) {
    enum { N_BLKS = 40 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        size_t lost_sq = size_t(-1);
        if (block_num != 5 && block_num != 21 && block_num != 22) {
            lost_sq = (block_num + 1) % (N_DATA_PACKETS + N_FEC_PACKETS);
            pckt_disp.lose(lost_sq);
        }

        fill_all_packets(N_DATA_PACKETS * block_num, N_DATA_PACKETS * N_BLKS);

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            encoder.write(data_packets[i]);
        }

        if (lost_sq == size_t(-1)) {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);
        } else if (lost_sq < N_DATA_PACKETS) {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS - 1);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS);
        } else {
            CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
            CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS - 1);
        }

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            IPacketConstPtr p = decoder.read();
            CHECK(p);
            check_audio_packet(p, N_DATA_PACKETS * block_num + i,
                               N_DATA_PACKETS * N_BLKS);
        }

        pckt_disp.reset();
    }
}

TEST(fec_codec_integration, interleaver) {
    enum { N_PACKETS = N_DATA_PACKETS * 30 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Interleaver intrl(pckt_disp);

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    IPacketPtr many_packets[N_PACKETS];

    for (size_t i = 0; i < N_PACKETS; ++i) {
        many_packets[i] = fill_one_packet(i, N_PACKETS);
        encoder.write(many_packets[i]);
    }

    intrl.flush();

    for (size_t i = 0; i < N_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i, N_PACKETS);
    }
}

TEST(fec_codec_integration, decoding_when_multiple_blocks_in_queue) {
    enum { N_BLKS = 3 };

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        fill_all_packets(N_DATA_PACKETS * block_num, N_DATA_PACKETS * N_BLKS);

        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            encoder.write(data_packets[i]);
        }
    }

    CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS * N_BLKS);
    CHECK(pckt_disp.get_fec_size() == N_FEC_PACKETS * N_BLKS);

    for (size_t block_num = 0; block_num < N_BLKS; ++block_num) {
        for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
            IPacketConstPtr p = decoder.read();
            CHECK(p);
            check_audio_packet(p, N_DATA_PACKETS * block_num + i,
                               N_DATA_PACKETS * N_BLKS);
        }

        pckt_disp.reset();
    }
}

TEST(fec_codec_integration, decoding_late_packet)
{
    // 1. Fill all packets in block except one lost packet.
    // 2. Read first part of block till lost packet.
    // 3. Receive one missing packet.
    // 4. Read and check latter block part.

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    fill_all_packets( 0, N_DATA_PACKETS);
    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        // Loosing packet #10
        if( i == 10)
            continue;
        encoder.write(data_packets[i]);
    }
    CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS-1);

    // Check 0-9 packets.
    for (size_t i = 0; i < 10; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i,
                           N_DATA_PACKETS);
    }

    // Receive packet #10
    encoder.write(data_packets[10]);

    for (size_t i = 10; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i,
                           N_DATA_PACKETS);
    }
}

IGNORE_TEST(fec_codec_integration, get_packets_before_marker_bit) {

    // 1. Fill second half of block and whole block with one loss after. So that there is 10-19 and 20-39 seqnums in packet queue.
    // 2. Check that we've got every packet including lost one.

    BlockEncoder block_encoder;
    BlockDecoder block_decoder;
    rtp::Parser parser;

    Encoder encoder(block_encoder, pckt_disp, composer);
    Decoder decoder(block_decoder, pckt_disp.get_data_reader(),
                    pckt_disp.get_fec_reader(), parser);

    fill_all_packets( 0, N_DATA_PACKETS * 2);
    for (size_t i = 10; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    fill_all_packets( N_DATA_PACKETS, N_DATA_PACKETS * 2);
    pckt_disp.lose(3);

    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        encoder.write(data_packets[i]);
    }

    for (size_t i = 10; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i,
                           N_DATA_PACKETS * 2);
    }
    // CHECK(pckt_disp.get_data_size() == N_DATA_PACKETS);
    for (size_t i = 0; i < N_DATA_PACKETS; ++i) {
        IPacketConstPtr p = decoder.read();
        CHECK(p);
        check_audio_packet(p, i + N_DATA_PACKETS,
                           N_DATA_PACKETS * 2);
    }
}

IGNORE_TEST(fec_codec_integration, repair_wrong_source_or_seqnum) {
}

} // namespace test
} // namespace roc
<|endoftext|>


<commit_before>#include <aleph/math/Bootstrap.hh>

#include <aleph/persistenceDiagrams/PersistenceDiagram.hh>
#include <aleph/persistenceDiagrams/PersistenceIndicatorFunction.hh>

#include <aleph/persistenceDiagrams/io/Raw.hh>

#include <algorithm>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

#include <cmath>

using DataType                     = double;
using PersistenceDiagram           = aleph::PersistenceDiagram<DataType>;
using PersistenceIndicatorFunction = decltype( aleph::persistenceIndicatorFunction( PersistenceDiagram() ) );
using Image                        = PersistenceIndicatorFunction::Image;

auto meanCalculation = [] ( auto begin, auto end )
{
  using T  = typename std::iterator_traits<decltype(begin)>::value_type;
  auto sum = std::accumulate( begin, end, T() );

  return sum / static_cast<double>( std::distance(begin, end) );
};

unsigned index( unsigned int samples, double alpha )
{
  // This accounts for rounding and works regardless of whether
  // the product samples * alpha is an integer or not. Note the
  // offset of -1. It is required because, say, the 100th value
  // is at index 99 of the vector.
  return static_cast<unsigned>( std::ceil( samples * alpha ) ) - 1;
}

int main( int argc, char** argv )
{
  std::vector<PersistenceIndicatorFunction> persistenceIndicatorFunctions;
  persistenceIndicatorFunctions.reserve( argc - 1 );

  for( int i = 1; i < argc; i++ )
  {
    std::cerr << "* Processing '" << argv[i] << "'...";

    auto D = aleph::io::load<DataType>( argv[i] );

    D.removeDiagonal();
    D.removeUnpaired();

    persistenceIndicatorFunctions.emplace_back( aleph::persistenceIndicatorFunction( D ) );

    std::cerr << "finished\n";
  }

  // TODO: make configurable
  unsigned numBootstrapSamples = 50;

  std::vector<PersistenceIndicatorFunction> meanReplicates;
  meanReplicates.reserve( numBootstrapSamples );

  aleph::math::Bootstrap bootstrap;

  bootstrap.makeReplicates( numBootstrapSamples,
                            persistenceIndicatorFunctions.begin(), persistenceIndicatorFunctions.end(),
                            meanCalculation,
                            std::back_inserter( meanReplicates ) );

  auto empiricalMean = meanCalculation( persistenceIndicatorFunctions.begin(), persistenceIndicatorFunctions.end() );

  std::vector<Image> theta;
  theta.reserve( numBootstrapSamples );

  for( auto&& meanReplicate: meanReplicates )
  {
    auto n = persistenceIndicatorFunctions.size();
    auto f = std::sqrt( n ) * ( meanReplicate - empiricalMean );
    f      = f.abs();

    theta.emplace_back( f.sup() );
  }

  std::sort( theta.begin(), theta.end() );

  auto alpha    = 0.05;
  auto quantile = theta.at( index( numBootstrapSamples, alpha / 2 ) );
  auto fLower   = empiricalMean - quantile / std::sqrt( persistenceIndicatorFunctions.size() );
  auto fUpper   = empiricalMean + quantile / std::sqrt( persistenceIndicatorFunctions.size() );

  std::ofstream out( "/tmp/Mean_plus_confidence.txt" );

  out << empiricalMean << "\n\n"
      << fUpper        << "\n\n"
      << fLower        << "\n";

}
<commit_msg>Fixed another implicit cast<commit_after>#include <aleph/math/Bootstrap.hh>

#include <aleph/persistenceDiagrams/PersistenceDiagram.hh>
#include <aleph/persistenceDiagrams/PersistenceIndicatorFunction.hh>

#include <aleph/persistenceDiagrams/io/Raw.hh>

#include <algorithm>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

#include <cmath>

using DataType                     = double;
using PersistenceDiagram           = aleph::PersistenceDiagram<DataType>;
using PersistenceIndicatorFunction = decltype( aleph::persistenceIndicatorFunction( PersistenceDiagram() ) );
using Image                        = PersistenceIndicatorFunction::Image;

auto meanCalculation = [] ( auto begin, auto end )
{
  using T  = typename std::iterator_traits<decltype(begin)>::value_type;
  auto sum = std::accumulate( begin, end, T() );

  return sum / static_cast<double>( std::distance(begin, end) );
};

unsigned index( unsigned int samples, double alpha )
{
  // This accounts for rounding and works regardless of whether
  // the product samples * alpha is an integer or not. Note the
  // offset of -1. It is required because, say, the 100th value
  // is at index 99 of the vector.
  return static_cast<unsigned>( std::ceil( samples * alpha ) ) - 1;
}

int main( int argc, char** argv )
{
  std::vector<PersistenceIndicatorFunction> persistenceIndicatorFunctions;
  persistenceIndicatorFunctions.reserve( static_cast<std::size_t>( argc - 1 ) );

  for( int i = 1; i < argc; i++ )
  {
    std::cerr << "* Processing '" << argv[i] << "'...";

    auto D = aleph::io::load<DataType>( argv[i] );

    D.removeDiagonal();
    D.removeUnpaired();

    persistenceIndicatorFunctions.emplace_back( aleph::persistenceIndicatorFunction( D ) );

    std::cerr << "finished\n";
  }

  // TODO: make configurable
  unsigned numBootstrapSamples = 50;

  std::vector<PersistenceIndicatorFunction> meanReplicates;
  meanReplicates.reserve( numBootstrapSamples );

  aleph::math::Bootstrap bootstrap;

  bootstrap.makeReplicates( numBootstrapSamples,
                            persistenceIndicatorFunctions.begin(), persistenceIndicatorFunctions.end(),
                            meanCalculation,
                            std::back_inserter( meanReplicates ) );

  auto empiricalMean = meanCalculation( persistenceIndicatorFunctions.begin(), persistenceIndicatorFunctions.end() );

  std::vector<Image> theta;
  theta.reserve( numBootstrapSamples );

  for( auto&& meanReplicate: meanReplicates )
  {
    auto n = persistenceIndicatorFunctions.size();
    auto f = std::sqrt( n ) * ( meanReplicate - empiricalMean );
    f      = f.abs();

    theta.emplace_back( f.sup() );
  }

  std::sort( theta.begin(), theta.end() );

  auto alpha    = 0.05;
  auto quantile = theta.at( index( numBootstrapSamples, alpha / 2 ) );
  auto fLower   = empiricalMean - quantile / std::sqrt( persistenceIndicatorFunctions.size() );
  auto fUpper   = empiricalMean + quantile / std::sqrt( persistenceIndicatorFunctions.size() );

  std::ofstream out( "/tmp/Mean_plus_confidence.txt" );

  out << empiricalMean << "\n\n"
      << fUpper        << "\n\n"
      << fLower        << "\n";

}
<|endoftext|>


<commit_before>// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: laszlocsomor@google.com (Laszlo Csomor)
//
// Implementation for long-path-aware open/mkdir/etc. on Windows.
//
// These functions convert the input path to an absolute Windows path
// with "\\?\" prefix if necessary, then pass that to _wopen/_wmkdir/etc.
// (declared in <io.h>) respectively. This allows working with files/directories
// whose paths are longer than MAX_PATH (260 chars).
//
// This file is only used on Windows, it's empty on other platforms.

#if defined(_WIN32)

// Comment this out to fall back to using the ANSI versions (open, mkdir, ...)
// instead of the Unicode ones (_wopen, _wmkdir, ...). Doing so can be useful to
// debug failing tests if that's caused by the long path support.
#define SUPPORT_LONGPATHS

#include <ctype.h>
#include <direct.h>
#include <errno.h>
#include <fcntl.h>
#include <io.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <wctype.h>
#include <windows.h>

#include <google/protobuf/stubs/io_win32.h>
#include <google/protobuf/stubs/scoped_ptr.h>

#include <cassert>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

namespace google {
namespace protobuf {
namespace internal {
namespace win32 {
namespace {

using std::string;
using std::wstring;

template <typename char_type>
struct CharTraits {
  static bool is_alpha(char_type ch);
};

template <>
struct CharTraits<char> {
  static bool is_alpha(char ch) { return isalpha(ch); }
};

template <>
struct CharTraits<wchar_t> {
  static bool is_alpha(wchar_t ch) { return iswalpha(ch); }
};

// Returns true if the path starts with a drive letter, e.g. "c:".
// Note that this won't check for the "\" after the drive letter, so this also
// returns true for "c:foo" (which is "c:\${PWD}\foo").
// This check requires that a path not have a longpath prefix ("\\?\").
template <typename char_type>
bool has_drive_letter(const char_type* ch) {
  return CharTraits<char_type>::is_alpha(ch[0]) && ch[1] == ':';
}

// Returns true if the path starts with a longpath prefix ("\\?\").
template <typename char_type>
bool has_longpath_prefix(const char_type* path) {
  return path[0] == '\\' && path[1] == '\\' && path[2] == '?' &&
         path[3] == '\\';
}

template <typename char_type>
bool is_separator(char_type c) {
  return c == '/' || c == '\\';
}

// Returns true if the path starts with a drive specifier (e.g. "c:\").
template <typename char_type>
bool is_path_absolute(const char_type* path) {
  return has_drive_letter(path) && is_separator(path[2]);
}

template <typename char_type>
bool is_drive_relative(const char_type* path) {
  return has_drive_letter(path) && (path[2] == 0 || !is_separator(path[2]));
}

template <typename char_type>
void replace_directory_separators(char_type* p) {
  for (; *p; ++p) {
    if (*p == '/') {
      *p = '\\';
    }
  }
}

string join_paths(const string& path1, const string& path2) {
  if (path1.empty() || is_path_absolute(path2.c_str()) ||
      has_longpath_prefix(path2.c_str())) {
    return path2;
  }
  if (path2.empty()) {
    return path1;
  }

  if (is_separator(path1[path1.size() - 1])) {
    return is_separator(path2[0]) ? (path1 + path2.substr(1))
                                       : (path1 + path2);
  } else {
    return is_separator(path2[0]) ? (path1 + path2)
                                       : (path1 + '\\' + path2);
  }
}

string normalize(string path) {
  if (has_longpath_prefix(path.c_str())) {
    path = path.substr(4);
  }

  static const string dot(".");
  static const string dotdot("..");
  const string& cpath = path;

  std::vector<string> segments;
  int segment_start = -1;
  // Find the path segments in `path` (separated by "/").
  for (int i = 0;; ++i) {
    if (!is_separator(cpath[i]) && cpath[i] != '\0') {
      // The current character does not end a segment, so start one unless it's
      // already started.
      if (segment_start < 0) {
        segment_start = i;
      }
    } else if (segment_start >= 0 && i > segment_start) {
      // The current character is "/" or "\0", so this ends a segment.
      // Add that to `segments` if there's anything to add; handle "." and "..".
      string segment(cpath, segment_start, i - segment_start);
      segment_start = -1;
      if (segment == dotdot) {
        if (!segments.empty() &&
            (!has_drive_letter(segments[0].c_str()) || segments.size() > 1)) {
          segments.pop_back();
        }
      } else if (segment != dot && !segment.empty()) {
        segments.push_back(segment);
      }
    }
    if (cpath[i] == '\0') {
      break;
    }
  }

  // Handle the case when `path` is just a drive specifier (or some degenerate
  // form of it, e.g. "c:\..").
  if (segments.size() == 1 && segments[0].size() == 2 &&
      has_drive_letter(segments[0].c_str())) {
    return segments[0] + '\\';
  }

  // Join all segments.
  bool first = true;
  std::ostringstream result;
  for (int i = 0; i < segments.size(); ++i) {
    if (!first) {
      result << '\\';
    }
    first = false;
    result << segments[i];
  }
  // Preserve trailing separator if the input contained it.
  if (!cpath.empty() && is_separator(cpath[cpath.size() - 1])) {
    result << '\\';
  }
  return result.str();
}

WCHAR* as_wstring(const string& s) {
  int len = ::MultiByteToWideChar(CP_UTF8, 0, s.c_str(), s.size(), NULL, 0);
  WCHAR* result = new WCHAR[len + 1];
  ::MultiByteToWideChar(CP_UTF8, 0, s.c_str(), s.size(), result, len + 1);
  result[len] = 0;
  return result;
}

void as_wchar_path(const string& path, wstring* wchar_path) {
  scoped_array<WCHAR> wbuf(as_wstring(path));
  replace_directory_separators(wbuf.get());
  wchar_path->assign(wbuf.get());
}

bool as_windows_path(const string& path, wstring* result) {
  if (path.empty()) {
    result->clear();
    return true;
  }
  if (is_separator(path[0]) || is_drive_relative(path.c_str())) {
    return false;
  }

  string mutable_path = path;
  if (!is_path_absolute(mutable_path.c_str()) &&
      !has_longpath_prefix(mutable_path.c_str())) {
    char cwd[MAX_PATH];
    ::GetCurrentDirectoryA(MAX_PATH, cwd);
    mutable_path = join_paths(cwd, mutable_path);
  }
  as_wchar_path(normalize(mutable_path), result);
  if (!has_longpath_prefix(result->c_str())) {
    // Add the "\\?\" prefix unconditionally. This way we prevent the Win32 API
    // from processing the path and "helpfully" removing trailing dots from the
    // path, for example.
    // See https://github.com/bazelbuild/bazel/issues/2935
    *result = wstring(L"\\\\?\\") + *result;
  }
  return true;
}

}  // namespace

int open(const char* path, int flags, int mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wopen(wpath.c_str(), flags, mode);
#else
  return ::_open(path, flags, mode);
#endif
}

int mkdir(const char* path, int _mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wmkdir(wpath.c_str());
#else   // not SUPPORT_LONGPATHS
  return ::_mkdir(path);
#endif  // not SUPPORT_LONGPATHS
}

int access(const char* path, int mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_waccess(wpath.c_str(), mode);
#else
  return ::_access(path, mode);
#endif
}

int chdir(const char* path) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wchdir(wpath.c_str());
#else
  return ::_chdir(path);
#endif
}

int stat(const char* path, struct _stat* buffer) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wstat(wpath.c_str(), buffer);
#else   // not SUPPORT_LONGPATHS
  return ::_stat(path, buffer);
#endif  // not SUPPORT_LONGPATHS
}

FILE* fopen(const char* path, const char* mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return NULL;
  }
  scoped_array<WCHAR> wmode(as_wstring(mode));
  return ::_wfopen(wpath.c_str(), wmode.get());
#else
  return ::fopen(path, mode);
#endif
}

int close(int fd) { return ::close(fd); }

int dup(int fd) { return ::_dup(fd); }

int dup2(int fd1, int fd2) { return ::_dup2(fd1, fd2); }

int read(int fd, void* buffer, size_t size) {
  return ::_read(fd, buffer, size);
}

int setmode(int fd, int mode) { return ::_setmode(fd, mode); }

int write(int fd, const void* buffer, size_t size) {
  return ::_write(fd, buffer, size);
}

wstring testonly_path_to_winpath(const string& path) {
  wstring wpath;
  as_windows_path(path, &wpath);
  return wpath;
}

}  // namespace win32
}  // namespace internal
}  // namespace protobuf
}  // namespace google

#endif  // defined(_WIN32)

<commit_msg>use const char* instead of const std::string& in normalize()<commit_after>// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: laszlocsomor@google.com (Laszlo Csomor)
//
// Implementation for long-path-aware open/mkdir/etc. on Windows.
//
// These functions convert the input path to an absolute Windows path
// with "\\?\" prefix if necessary, then pass that to _wopen/_wmkdir/etc.
// (declared in <io.h>) respectively. This allows working with files/directories
// whose paths are longer than MAX_PATH (260 chars).
//
// This file is only used on Windows, it's empty on other platforms.

#if defined(_WIN32)

// Comment this out to fall back to using the ANSI versions (open, mkdir, ...)
// instead of the Unicode ones (_wopen, _wmkdir, ...). Doing so can be useful to
// debug failing tests if that's caused by the long path support.
#define SUPPORT_LONGPATHS

#include <ctype.h>
#include <direct.h>
#include <errno.h>
#include <fcntl.h>
#include <io.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <wctype.h>
#include <windows.h>

#include <google/protobuf/stubs/io_win32.h>
#include <google/protobuf/stubs/scoped_ptr.h>

#include <cassert>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

namespace google {
namespace protobuf {
namespace internal {
namespace win32 {
namespace {

using std::string;
using std::wstring;

template <typename char_type>
struct CharTraits {
  static bool is_alpha(char_type ch);
};

template <>
struct CharTraits<char> {
  static bool is_alpha(char ch) { return isalpha(ch); }
};

template <>
struct CharTraits<wchar_t> {
  static bool is_alpha(wchar_t ch) { return iswalpha(ch); }
};

// Returns true if the path starts with a drive letter, e.g. "c:".
// Note that this won't check for the "\" after the drive letter, so this also
// returns true for "c:foo" (which is "c:\${PWD}\foo").
// This check requires that a path not have a longpath prefix ("\\?\").
template <typename char_type>
bool has_drive_letter(const char_type* ch) {
  return CharTraits<char_type>::is_alpha(ch[0]) && ch[1] == ':';
}

// Returns true if the path starts with a longpath prefix ("\\?\").
template <typename char_type>
bool has_longpath_prefix(const char_type* path) {
  return path[0] == '\\' && path[1] == '\\' && path[2] == '?' &&
         path[3] == '\\';
}

template <typename char_type>
bool is_separator(char_type c) {
  return c == '/' || c == '\\';
}

// Returns true if the path starts with a drive specifier (e.g. "c:\").
template <typename char_type>
bool is_path_absolute(const char_type* path) {
  return has_drive_letter(path) && is_separator(path[2]);
}

template <typename char_type>
bool is_drive_relative(const char_type* path) {
  return has_drive_letter(path) && (path[2] == 0 || !is_separator(path[2]));
}

template <typename char_type>
void replace_directory_separators(char_type* p) {
  for (; *p; ++p) {
    if (*p == '/') {
      *p = '\\';
    }
  }
}

string join_paths(const string& path1, const string& path2) {
  if (path1.empty() || is_path_absolute(path2.c_str()) ||
      has_longpath_prefix(path2.c_str())) {
    return path2;
  }
  if (path2.empty()) {
    return path1;
  }

  if (is_separator(path1[path1.size() - 1])) {
    return is_separator(path2[0]) ? (path1 + path2.substr(1))
                                       : (path1 + path2);
  } else {
    return is_separator(path2[0]) ? (path1 + path2)
                                       : (path1 + '\\' + path2);
  }
}

string normalize(string path) {
  if (has_longpath_prefix(path.c_str())) {
    path = path.substr(4);
  }

  static const string dot(".");
  static const string dotdot("..");
  const char *p = path.c_str();

  std::vector<string> segments;
  int segment_start = -1;
  // Find the path segments in `path` (separated by "/").
  for (int i = 0;; ++i) {
    if (!is_separator(p[i]) && p[i] != '\0') {
      // The current character does not end a segment, so start one unless it's
      // already started.
      if (segment_start < 0) {
        segment_start = i;
      }
    } else if (segment_start >= 0 && i > segment_start) {
      // The current character is "/" or "\0", so this ends a segment.
      // Add that to `segments` if there's anything to add; handle "." and "..".
      string segment(p, segment_start, i - segment_start);
      segment_start = -1;
      if (segment == dotdot) {
        if (!segments.empty() &&
            (!has_drive_letter(segments[0].c_str()) || segments.size() > 1)) {
          segments.pop_back();
        }
      } else if (segment != dot && !segment.empty()) {
        segments.push_back(segment);
      }
    }
    if (p[i] == '\0') {
      break;
    }
  }

  // Handle the case when `path` is just a drive specifier (or some degenerate
  // form of it, e.g. "c:\..").
  if (segments.size() == 1 && segments[0].size() == 2 &&
      has_drive_letter(segments[0].c_str())) {
    return segments[0] + '\\';
  }

  // Join all segments.
  bool first = true;
  std::ostringstream result;
  for (int i = 0; i < segments.size(); ++i) {
    if (!first) {
      result << '\\';
    }
    first = false;
    result << segments[i];
  }
  // Preserve trailing separator if the input contained it.
  if (!path.empty() && is_separator(p[path.size() - 1])) {
    result << '\\';
  }
  return result.str();
}

WCHAR* as_wstring(const string& s) {
  int len = ::MultiByteToWideChar(CP_UTF8, 0, s.c_str(), s.size(), NULL, 0);
  WCHAR* result = new WCHAR[len + 1];
  ::MultiByteToWideChar(CP_UTF8, 0, s.c_str(), s.size(), result, len + 1);
  result[len] = 0;
  return result;
}

void as_wchar_path(const string& path, wstring* wchar_path) {
  scoped_array<WCHAR> wbuf(as_wstring(path));
  replace_directory_separators(wbuf.get());
  wchar_path->assign(wbuf.get());
}

bool as_windows_path(const string& path, wstring* result) {
  if (path.empty()) {
    result->clear();
    return true;
  }
  if (is_separator(path[0]) || is_drive_relative(path.c_str())) {
    return false;
  }

  string mutable_path = path;
  if (!is_path_absolute(mutable_path.c_str()) &&
      !has_longpath_prefix(mutable_path.c_str())) {
    char cwd[MAX_PATH];
    ::GetCurrentDirectoryA(MAX_PATH, cwd);
    mutable_path = join_paths(cwd, mutable_path);
  }
  as_wchar_path(normalize(mutable_path), result);
  if (!has_longpath_prefix(result->c_str())) {
    // Add the "\\?\" prefix unconditionally. This way we prevent the Win32 API
    // from processing the path and "helpfully" removing trailing dots from the
    // path, for example.
    // See https://github.com/bazelbuild/bazel/issues/2935
    *result = wstring(L"\\\\?\\") + *result;
  }
  return true;
}

}  // namespace

int open(const char* path, int flags, int mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wopen(wpath.c_str(), flags, mode);
#else
  return ::_open(path, flags, mode);
#endif
}

int mkdir(const char* path, int _mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wmkdir(wpath.c_str());
#else   // not SUPPORT_LONGPATHS
  return ::_mkdir(path);
#endif  // not SUPPORT_LONGPATHS
}

int access(const char* path, int mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_waccess(wpath.c_str(), mode);
#else
  return ::_access(path, mode);
#endif
}

int chdir(const char* path) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wchdir(wpath.c_str());
#else
  return ::_chdir(path);
#endif
}

int stat(const char* path, struct _stat* buffer) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return -1;
  }
  return ::_wstat(wpath.c_str(), buffer);
#else   // not SUPPORT_LONGPATHS
  return ::_stat(path, buffer);
#endif  // not SUPPORT_LONGPATHS
}

FILE* fopen(const char* path, const char* mode) {
#ifdef SUPPORT_LONGPATHS
  wstring wpath;
  if (!as_windows_path(path, &wpath)) {
    errno = ENOENT;
    return NULL;
  }
  scoped_array<WCHAR> wmode(as_wstring(mode));
  return ::_wfopen(wpath.c_str(), wmode.get());
#else
  return ::fopen(path, mode);
#endif
}

int close(int fd) { return ::close(fd); }

int dup(int fd) { return ::_dup(fd); }

int dup2(int fd1, int fd2) { return ::_dup2(fd1, fd2); }

int read(int fd, void* buffer, size_t size) {
  return ::_read(fd, buffer, size);
}

int setmode(int fd, int mode) { return ::_setmode(fd, mode); }

int write(int fd, const void* buffer, size_t size) {
  return ::_write(fd, buffer, size);
}

wstring testonly_path_to_winpath(const string& path) {
  wstring wpath;
  as_windows_path(path, &wpath);
  return wpath;
}

}  // namespace win32
}  // namespace internal
}  // namespace protobuf
}  // namespace google

#endif  // defined(_WIN32)

<|endoftext|>


<commit_before>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Id$
 * $Log$
 * Revision 1.4  2002/12/18 14:17:55  gareth
 * Fix to bug #13438. When you eant a vector that calls delete[] on its members you should use RefArrayVectorOf.
 *
 * Revision 1.3  2002/11/04 14:53:28  tng
 * C++ Namespace Support.
 *
 * Revision 1.2  2002/09/24 19:44:40  tng
 * Performance: use XMLString::equals instead of XMLString::compareString
 *
 * Revision 1.1.1.1  2002/02/01 22:22:41  peiyongz
 * sane_include
 *
 * Revision 1.7  2001/11/13 13:21:59  tng
 * Schema fix: Entities should be unparsed.
 *
 * Revision 1.6  2001/10/09 20:50:27  peiyongz
 * init(): take 1 arg
 *
 * Revision 1.5  2001/09/27 13:51:25  peiyongz
 * DTV Reorganization: ctor/init created to be used by derived class
 *
 * Revision 1.4  2001/09/24 21:39:29  peiyongz
 * DTV Reorganization: checkValueSpace()
 *
 * Revision 1.3  2001/08/14 22:11:56  peiyongz
 * new exception message added
 *
 * Revision 1.2  2001/07/24 21:23:39  tng
 * Schema: Use DatatypeValidator for ID/IDREF/ENTITY/ENTITIES/NOTATION.
 *
 * Revision 1.1  2001/07/06 20:19:52  peiyongz
 * ENTITYDTV
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/datatype/ENTITYDatatypeValidator.hpp>
#include <xercesc/validators/datatype/InvalidDatatypeValueException.hpp>
#include <xercesc/framework/XMLEntityDecl.hpp>

XERCES_CPP_NAMESPACE_BEGIN

// ---------------------------------------------------------------------------
//  Constructors and Destructor
// ---------------------------------------------------------------------------
ENTITYDatatypeValidator::ENTITYDatatypeValidator()
:StringDatatypeValidator(0, 0, 0, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{}

ENTITYDatatypeValidator::ENTITYDatatypeValidator(
                          DatatypeValidator*            const baseValidator
                        , RefHashTableOf<KVStringPair>* const facets
                        , RefArrayVectorOf<XMLCh>*           const enums
                        , const int                           finalSet)
:StringDatatypeValidator(baseValidator, facets, finalSet, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{
    init(enums);
}

ENTITYDatatypeValidator::~ENTITYDatatypeValidator()
{}

DatatypeValidator* ENTITYDatatypeValidator::newInstance(
                                      RefHashTableOf<KVStringPair>* const facets
                                    , RefArrayVectorOf<XMLCh>*           const enums
                                    , const int                           finalSet)
{
    return (DatatypeValidator*) new ENTITYDatatypeValidator(this, facets, enums, finalSet);
}

// -----------------------------------------------------------------------
// Compare methods
// -----------------------------------------------------------------------
int ENTITYDatatypeValidator::compare(const XMLCh* const lValue
                                   , const XMLCh* const rValue)
{
    return ( XMLString::equals(lValue, rValue)? 0 : -1);
}

void ENTITYDatatypeValidator::validate(const XMLCh* const content)
{
    // use StringDatatypeValidator (which in turn, invoke
    // the baseValidator) to validate content against
    // facets if any.
    //
    StringDatatypeValidator::validate(content);

    //
    // parse the entity iff an EntityDeclPool is provided
    //
    if (fEntityDeclPool)
    {
        DTDEntityDecl* decl = fEntityDeclPool->getByKey(content);

        if (!decl                ||
            (!decl->isUnparsed())  )
        {
            ThrowXML1(InvalidDatatypeValueException
                    , XMLExcepts::VALUE_ENTITY_Invalid
                    , content);
        }

    }

    return;
}

void ENTITYDatatypeValidator::checkValueSpace(const XMLCh* const content)
{
    //
    // 3.3.11 check must: "NCName"
    //
    if ( !XMLString::isValidNCName(content))
    {
        ThrowXML1(InvalidDatatypeValueException
                , XMLExcepts::VALUE_Invalid_NCName
                , content);
    }

}

XERCES_CPP_NAMESPACE_END

 /**
  * End of file ENTITYDatatypeValidator.cpp
  */
<commit_msg>Throw an invalid datatype value exception if fEntityDeclPool is NULL.<commit_after>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Id$
 * $Log$
 * Revision 1.5  2003/01/06 19:40:05  knoaman
 * Throw an invalid datatype value exception if fEntityDeclPool is NULL.
 *
 * Revision 1.4  2002/12/18 14:17:55  gareth
 * Fix to bug #13438. When you eant a vector that calls delete[] on its members you should use RefArrayVectorOf.
 *
 * Revision 1.3  2002/11/04 14:53:28  tng
 * C++ Namespace Support.
 *
 * Revision 1.2  2002/09/24 19:44:40  tng
 * Performance: use XMLString::equals instead of XMLString::compareString
 *
 * Revision 1.1.1.1  2002/02/01 22:22:41  peiyongz
 * sane_include
 *
 * Revision 1.7  2001/11/13 13:21:59  tng
 * Schema fix: Entities should be unparsed.
 *
 * Revision 1.6  2001/10/09 20:50:27  peiyongz
 * init(): take 1 arg
 *
 * Revision 1.5  2001/09/27 13:51:25  peiyongz
 * DTV Reorganization: ctor/init created to be used by derived class
 *
 * Revision 1.4  2001/09/24 21:39:29  peiyongz
 * DTV Reorganization: checkValueSpace()
 *
 * Revision 1.3  2001/08/14 22:11:56  peiyongz
 * new exception message added
 *
 * Revision 1.2  2001/07/24 21:23:39  tng
 * Schema: Use DatatypeValidator for ID/IDREF/ENTITY/ENTITIES/NOTATION.
 *
 * Revision 1.1  2001/07/06 20:19:52  peiyongz
 * ENTITYDTV
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/datatype/ENTITYDatatypeValidator.hpp>
#include <xercesc/validators/datatype/InvalidDatatypeValueException.hpp>
#include <xercesc/framework/XMLEntityDecl.hpp>

XERCES_CPP_NAMESPACE_BEGIN

// ---------------------------------------------------------------------------
//  Constructors and Destructor
// ---------------------------------------------------------------------------
ENTITYDatatypeValidator::ENTITYDatatypeValidator()
:StringDatatypeValidator(0, 0, 0, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{}

ENTITYDatatypeValidator::ENTITYDatatypeValidator(
                          DatatypeValidator*            const baseValidator
                        , RefHashTableOf<KVStringPair>* const facets
                        , RefArrayVectorOf<XMLCh>*           const enums
                        , const int                           finalSet)
:StringDatatypeValidator(baseValidator, facets, finalSet, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{
    init(enums);
}

ENTITYDatatypeValidator::~ENTITYDatatypeValidator()
{}

DatatypeValidator* ENTITYDatatypeValidator::newInstance(
                                      RefHashTableOf<KVStringPair>* const facets
                                    , RefArrayVectorOf<XMLCh>*           const enums
                                    , const int                           finalSet)
{
    return (DatatypeValidator*) new ENTITYDatatypeValidator(this, facets, enums, finalSet);
}

// -----------------------------------------------------------------------
// Compare methods
// -----------------------------------------------------------------------
int ENTITYDatatypeValidator::compare(const XMLCh* const lValue
                                   , const XMLCh* const rValue)
{
    return ( XMLString::equals(lValue, rValue)? 0 : -1);
}

void ENTITYDatatypeValidator::validate(const XMLCh* const content)
{
    // use StringDatatypeValidator (which in turn, invoke
    // the baseValidator) to validate content against
    // facets if any.
    //
    StringDatatypeValidator::validate(content);

    //
    // parse the entity iff an EntityDeclPool is provided
    //
    if (fEntityDeclPool)
    {
        DTDEntityDecl* decl = fEntityDeclPool->getByKey(content);

        if (!decl                ||
            (!decl->isUnparsed())  )
        {
            ThrowXML1(InvalidDatatypeValueException
                    , XMLExcepts::VALUE_ENTITY_Invalid
                    , content);
        }

    }
    else {
        ThrowXML1
        (
            InvalidDatatypeValueException
            , XMLExcepts::VALUE_ENTITY_Invalid
            , content
        );
    }

    return;
}

void ENTITYDatatypeValidator::checkValueSpace(const XMLCh* const content)
{
    //
    // 3.3.11 check must: "NCName"
    //
    if ( !XMLString::isValidNCName(content))
    {
        ThrowXML1(InvalidDatatypeValueException
                , XMLExcepts::VALUE_Invalid_NCName
                , content);
    }

}

XERCES_CPP_NAMESPACE_END

 /**
  * End of file ENTITYDatatypeValidator.cpp
  */
<|endoftext|>


<commit_before>//
// normalize_manifest
//
// Copyright (c) 2013 Genome Research Ltd.
//

// Author: Iain Bancarz <ib5@sanger.ac.uk>
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
// 3. Neither the name of the Genome Research Ltd nor the names of its contributors
// may be used to endorse or promote products derived from software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR WARRANTIES,
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. EVENT SHALL GENOME RESEARCH LTD. BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
// THE POSSIBILITY OF SUCH DAMAGE.
//

// Executable to normalize a .bpm.csv manifest to the Illumina TOP strand
// Write normalized manifest to given output

#include <getopt.h>
#include <iostream>
#include "Manifest.h"

using namespace std;

int main(int argc, char *argv[])
{

  string manfile = "/nfs/new_illumina_geno04/call/HumanOmniExpress-12v1_A.bpm.csv";
  string outpath = "normalized.bpm.csv";
  cout << "Hello, world!" << endl;
  Manifest *manifest = new Manifest();
  manifest->open(manfile);
  cout << "Finished reading manifest!" << endl;
  cout << manifest->get_chromosome_for_SNP("rs10000049") << endl;
  manifest->write(outpath);

}
<commit_msg>Remove unnecessary stdout chatter<commit_after>//
// normalize_manifest
//
// Copyright (c) 2013 Genome Research Ltd.
//

// Author: Iain Bancarz <ib5@sanger.ac.uk>
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
// 3. Neither the name of the Genome Research Ltd nor the names of its contributors
// may be used to endorse or promote products derived from software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR WARRANTIES,
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. EVENT SHALL GENOME RESEARCH LTD. BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
// THE POSSIBILITY OF SUCH DAMAGE.
//

// Executable to normalize a .bpm.csv manifest to the Illumina TOP strand
// Write normalized manifest to given output

#include <getopt.h>
#include <iostream>
#include "Manifest.h"

using namespace std;

int main(int argc, char *argv[])
{

  string manfile = "/nfs/new_illumina_geno04/call/HumanOmniExpress-12v1_A.bpm.csv";
  string outpath = "normalized.bpm.csv";
  Manifest *manifest = new Manifest();
  manifest->open(manfile);
  cout << "Finished reading manifest: " << manfile << endl;
  manifest->write(outpath);

}
<|endoftext|>