<commit_before>#include <sstream>
#include <iostream>
#include <vector>
#include <cassert>
#include <cmath>
#include <tr1/memory>

#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "hg_sampler.h"
#include "sentence_metadata.h"
#include "scorer.h"
#include "verbose.h"
#include "viterbi.h"
#include "hg.h"
#include "prob.h"
#include "kbest.h"
#include "ff_register.h"
#include "decoder.h"
#include "filelib.h"
#include "fdict.h"
#include "weights.h"
#include "sparse_vector.h"
#include "sampler.h"

using namespace std;
namespace po = boost::program_options;

bool invert_score;
std::tr1::shared_ptr<MT19937> rng;

void RandomPermutation(int len, vector<int>* p_ids) {
  vector<int>& ids = *p_ids;
  ids.resize(len);
  for (int i = 0; i < len; ++i) ids[i] = i;
  for (int i = len; i > 0; --i) {
    int j = rng->next() * i;
    if (j == i) i--;
    swap(ids[i-1], ids[j]);
  }
}

bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("input_weights,w",po::value<string>(),"Input feature weights file")
        ("source,i",po::value<string>(),"Source file for development set")
        ("passes,p", po::value<int>()->default_value(15), "Number of passes through the training data")
        ("reference,r",po::value<vector<string> >(), "[REQD] Reference translation(s) (tokenized text file)")
        ("mt_metric,m",po::value<string>()->default_value("ibm_bleu"), "Scoring metric (ibm_bleu, nist_bleu, koehn_bleu, ter, combi)")
        ("max_step_size,C", po::value<double>()->default_value(0.01), "regularization strength (C)")
        ("mt_metric_scale,s", po::value<double>()->default_value(1.0), "Amount to scale MT loss function by")
        ("k_best_size,k", po::value<int>()->default_value(250), "Size of hypothesis list to search for oracles")
        ("sample_forest,f", "Instead of a k-best list, sample k hypotheses from the decoder's forest")
        ("sample_forest_unit_weight_vector,x", "Before sampling (must use -f option), rescale the weight vector used so it has unit length; this may improve the quality of the samples")
        ("random_seed,S", po::value<uint32_t>(), "Random seed (if not specified, /dev/random will be used)")
        ("decoder_config,c",po::value<string>(),"Decoder configuration file");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);

  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || !conf->count("input_weights") || !conf->count("source") || !conf->count("decoder_config") || !conf->count("reference")) {
    cerr << dcmdline_options << endl;
    return false;
  }
  return true;
}

static const double kMINUS_EPSILON = -1e-6;

struct HypothesisInfo {
  SparseVector<double> features;
  double mt_metric;
};

struct GoodBadOracle {
  std::tr1::shared_ptr<HypothesisInfo> good;
  std::tr1::shared_ptr<HypothesisInfo> bad;
};

struct TrainingObserver : public DecoderObserver {
  TrainingObserver(const int k, const DocScorer& d, bool sf, vector<GoodBadOracle>* o) : ds(d), oracles(*o), kbest_size(k), sample_forest(sf) {}
  const DocScorer& ds;
  vector<GoodBadOracle>& oracles;
  std::tr1::shared_ptr<HypothesisInfo> cur_best;
  const int kbest_size;
  const bool sample_forest;

  const HypothesisInfo& GetCurrentBestHypothesis() const {
    return *cur_best;
  }

  virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg) {
    UpdateOracles(smeta.GetSentenceID(), *hg);
  }

  std::tr1::shared_ptr<HypothesisInfo> MakeHypothesisInfo(const SparseVector<double>& feats, const double score) {
    std::tr1::shared_ptr<HypothesisInfo> h(new HypothesisInfo);
    h->features = feats;
    h->mt_metric = score;
    return h;
  }

  void UpdateOracles(int sent_id, const Hypergraph& forest) {
    std::tr1::shared_ptr<HypothesisInfo>& cur_good = oracles[sent_id].good;
    std::tr1::shared_ptr<HypothesisInfo>& cur_bad = oracles[sent_id].bad;
    cur_bad.reset();  // TODO get rid of??

    if (sample_forest) {
      vector<WordID> cur_prediction;
      ViterbiESentence(forest, &cur_prediction);
      float sentscore = ds[sent_id]->ScoreCandidate(cur_prediction)->ComputeScore();
      cur_best = MakeHypothesisInfo(ViterbiFeatures(forest), sentscore);

      vector<HypergraphSampler::Hypothesis> samples;
      HypergraphSampler::sample_hypotheses(forest, kbest_size, &*rng, &samples);
      for (unsigned i = 0; i < samples.size(); ++i) {
        sentscore = ds[sent_id]->ScoreCandidate(samples[i].words)->ComputeScore();
        if (invert_score) sentscore *= -1.0;
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(samples[i].fmap, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(samples[i].fmap, sentscore);
      }
    } else {
      KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(forest, kbest_size);
      for (int i = 0; i < kbest_size; ++i) {
        const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
          kbest.LazyKthBest(forest.nodes_.size() - 1, i);
        if (!d) break;
        float sentscore = ds[sent_id]->ScoreCandidate(d->yield)->ComputeScore();
        if (invert_score) sentscore *= -1.0;
        // cerr << TD::GetString(d->yield) << " ||| " << d->score << " ||| " << sentscore << endl;
        if (i == 0)
          cur_best = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(d->feature_values, sentscore);
      }
      //cerr << "GOOD: " << cur_good->mt_metric << endl;
      //cerr << " CUR: " << cur_best->mt_metric << endl;
      //cerr << " BAD: " << cur_bad->mt_metric << endl;
    }
  }
};

void ReadTrainingCorpus(const string& fname, vector<string>* c) {
  ReadFile rf(fname);
  istream& in = *rf.stream();
  string line;
  while(in) {
    getline(in, line);
    if (!in) break;
    c->push_back(line);
  }
}

bool ApproxEqual(double a, double b) {
  if (a == b) return true;
  return (fabs(a-b)/fabs(b)) < 0.000001;
}

int main(int argc, char** argv) {
  register_feature_functions();
  SetSilent(true);  // turn off verbose decoder output

  po::variables_map conf;
  if (!InitCommandLine(argc, argv, &conf)) return 1;

  if (conf.count("random_seed"))
    rng.reset(new MT19937(conf["random_seed"].as<uint32_t>()));
  else
    rng.reset(new MT19937);
  const bool sample_forest = conf.count("sample_forest") > 0;
  const bool sample_forest_unit_weight_vector = conf.count("sample_forest_unit_weight_vector") > 0;
  if (sample_forest_unit_weight_vector && !sample_forest) {
    cerr << "Cannot --sample_forest_unit_weight_vector without --sample_forest" << endl;
    return 1;
  }
  vector<string> corpus;
  ReadTrainingCorpus(conf["source"].as<string>(), &corpus);
  const string metric_name = conf["mt_metric"].as<string>();
  ScoreType type = ScoreTypeFromString(metric_name);
  if (type == TER) {
    invert_score = true;
  } else {
    invert_score = false;
  }
  DocScorer ds(type, conf["reference"].as<vector<string> >(), "");
  cerr << "Loaded " << ds.size() << " references for scoring with " << metric_name << endl;
  if (ds.size() != corpus.size()) {
    cerr << "Mismatched number of references (" << ds.size() << ") and sources (" << corpus.size() << ")\n";
    return 1;
  }

  ReadFile ini_rf(conf["decoder_config"].as<string>());
  Decoder decoder(ini_rf.stream());

  // load initial weights
  vector<weight_t>& dense_weights = decoder.CurrentWeightVector();
  SparseVector<weight_t> lambdas;
  Weights::InitFromFile(conf["input_weights"].as<string>(), &dense_weights);
  Weights::InitSparseVector(dense_weights, &lambdas);

  const double max_step_size = conf["max_step_size"].as<double>();
  const double mt_metric_scale = conf["mt_metric_scale"].as<double>();

  assert(corpus.size() > 0);
  vector<GoodBadOracle> oracles(corpus.size());

  TrainingObserver observer(conf["k_best_size"].as<int>(), ds, sample_forest, &oracles);
  int cur_sent = 0;
  int lcount = 0;
  int normalizer = 0;
  double tot_loss = 0;
  int dots = 0;
  int cur_pass = 0;
  SparseVector<double> tot;
  tot += lambdas;          // initial weights
  normalizer++;            // count for initial weights
  int max_iteration = conf["passes"].as<int>() * corpus.size();
  string msg = "# MIRA tuned weights";
  string msga = "# MIRA tuned weights AVERAGED";
  vector<int> order;
  RandomPermutation(corpus.size(), &order);
  while (lcount <= max_iteration) {
    lambdas.init_vector(&dense_weights);
    if ((cur_sent * 40 / corpus.size()) > dots) { ++dots; cerr << '.'; }
    if (corpus.size() == cur_sent) {
      cerr << " [AVG METRIC LAST PASS=" << (tot_loss / corpus.size()) << "]\n";
      Weights::ShowLargestFeatures(dense_weights);
      cur_sent = 0;
      tot_loss = 0;
      dots = 0;
      ostringstream os;
      os << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << ".gz";
      SparseVector<double> x = tot;
      x /= normalizer;
      ostringstream sa;
      sa << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << "-avg.gz";
      x.init_vector(&dense_weights);
      Weights::WriteToFile(os.str(), dense_weights, true, &msg);
      ++cur_pass;
      RandomPermutation(corpus.size(), &order);
    }
    if (cur_sent == 0) {
      cerr << "PASS " << (lcount / corpus.size() + 1) << endl;
    }
    decoder.SetId(order[cur_sent]);
    double sc = 1.0;
    if (sample_forest_unit_weight_vector) {
      sc = lambdas.l2norm();
      if (sc > 0) {
        for (unsigned i = 0; i < dense_weights.size(); ++i)
          dense_weights[i] /= sc;
      }
    }
    decoder.Decode(corpus[order[cur_sent]], &observer);  // update oracles
    if (sc && sc != 1.0) {
      for (unsigned i = 0; i < dense_weights.size(); ++i)
        dense_weights[i] *= sc;
    }
    const HypothesisInfo& cur_hyp = observer.GetCurrentBestHypothesis();
    const HypothesisInfo& cur_good = *oracles[order[cur_sent]].good;
    const HypothesisInfo& cur_bad = *oracles[order[cur_sent]].bad;
    tot_loss += cur_hyp.mt_metric;
    if (!ApproxEqual(cur_hyp.mt_metric, cur_good.mt_metric)) {
      const double loss = cur_bad.features.dot(dense_weights) - cur_good.features.dot(dense_weights) +
          mt_metric_scale * (cur_good.mt_metric - cur_bad.mt_metric);
      //cerr << "LOSS: " << loss << endl;
      if (loss > 0.0) {
        SparseVector<double> diff = cur_good.features;
        diff -= cur_bad.features;
        double step_size = loss / diff.l2norm_sq();
        //cerr << loss << " " << step_size << " " << diff << endl;
        if (step_size > max_step_size) step_size = max_step_size;
        lambdas += (cur_good.features * step_size);
        lambdas -= (cur_bad.features * step_size);
        //cerr << "L: " << lambdas << endl;
      }
    }
    tot += lambdas;
    ++normalizer;
    ++lcount;
    ++cur_sent;
  }
  cerr << endl;
  Weights::WriteToFile("weights.mira-final.gz", dense_weights, true, &msg);
  tot /= normalizer;
  tot.init_vector(dense_weights);
  msg = "# MIRA tuned weights (averaged vector)";
  Weights::WriteToFile("weights.mira-final-avg.gz", dense_weights, true, &msg);
  cerr << "Optimization complete.\nAVERAGED WEIGHTS: weights.mira-final-avg.gz\n";
  return 0;
}

<commit_msg>switch to new score interface for mira<commit_after>#include <sstream>
#include <iostream>
#include <vector>
#include <cassert>
#include <cmath>
#include <tr1/memory>

#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "stringlib.h"
#include "hg_sampler.h"
#include "sentence_metadata.h"
#include "ns.h"
#include "ns_docscorer.h"
#include "verbose.h"
#include "viterbi.h"
#include "hg.h"
#include "prob.h"
#include "kbest.h"
#include "ff_register.h"
#include "decoder.h"
#include "filelib.h"
#include "fdict.h"
#include "weights.h"
#include "sparse_vector.h"
#include "sampler.h"

using namespace std;
namespace po = boost::program_options;

bool invert_score;
std::tr1::shared_ptr<MT19937> rng;

void RandomPermutation(int len, vector<int>* p_ids) {
  vector<int>& ids = *p_ids;
  ids.resize(len);
  for (int i = 0; i < len; ++i) ids[i] = i;
  for (int i = len; i > 0; --i) {
    int j = rng->next() * i;
    if (j == i) i--;
    swap(ids[i-1], ids[j]);
  }
}

bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("input_weights,w",po::value<string>(),"Input feature weights file")
        ("source,i",po::value<string>(),"Source file for development set")
        ("passes,p", po::value<int>()->default_value(15), "Number of passes through the training data")
        ("reference,r",po::value<vector<string> >(), "[REQD] Reference translation(s) (tokenized text file)")
        ("mt_metric,m",po::value<string>()->default_value("ibm_bleu"), "Scoring metric (ibm_bleu, nist_bleu, koehn_bleu, ter, combi)")
        ("max_step_size,C", po::value<double>()->default_value(0.01), "regularization strength (C)")
        ("mt_metric_scale,s", po::value<double>()->default_value(1.0), "Amount to scale MT loss function by")
        ("k_best_size,k", po::value<int>()->default_value(250), "Size of hypothesis list to search for oracles")
        ("sample_forest,f", "Instead of a k-best list, sample k hypotheses from the decoder's forest")
        ("sample_forest_unit_weight_vector,x", "Before sampling (must use -f option), rescale the weight vector used so it has unit length; this may improve the quality of the samples")
        ("random_seed,S", po::value<uint32_t>(), "Random seed (if not specified, /dev/random will be used)")
        ("decoder_config,c",po::value<string>(),"Decoder configuration file");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);

  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || !conf->count("input_weights") || !conf->count("source") || !conf->count("decoder_config") || !conf->count("reference")) {
    cerr << dcmdline_options << endl;
    return false;
  }
  return true;
}

static const double kMINUS_EPSILON = -1e-6;

struct HypothesisInfo {
  SparseVector<double> features;
  double mt_metric;
};

struct GoodBadOracle {
  std::tr1::shared_ptr<HypothesisInfo> good;
  std::tr1::shared_ptr<HypothesisInfo> bad;
};

struct TrainingObserver : public DecoderObserver {
  TrainingObserver(const int k, const DocumentScorer& d, const EvaluationMetric& m, bool sf, vector<GoodBadOracle>* o) : ds(d), metric(m), oracles(*o), kbest_size(k), sample_forest(sf) {}
  const DocumentScorer& ds;
  const EvaluationMetric& metric;
  vector<GoodBadOracle>& oracles;
  std::tr1::shared_ptr<HypothesisInfo> cur_best;
  const int kbest_size;
  const bool sample_forest;

  const HypothesisInfo& GetCurrentBestHypothesis() const {
    return *cur_best;
  }

  virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg) {
    UpdateOracles(smeta.GetSentenceID(), *hg);
  }

  std::tr1::shared_ptr<HypothesisInfo> MakeHypothesisInfo(const SparseVector<double>& feats, const double score) {
    std::tr1::shared_ptr<HypothesisInfo> h(new HypothesisInfo);
    h->features = feats;
    h->mt_metric = score;
    return h;
  }

  void UpdateOracles(int sent_id, const Hypergraph& forest) {
    std::tr1::shared_ptr<HypothesisInfo>& cur_good = oracles[sent_id].good;
    std::tr1::shared_ptr<HypothesisInfo>& cur_bad = oracles[sent_id].bad;
    cur_bad.reset();  // TODO get rid of??

    if (sample_forest) {
      vector<WordID> cur_prediction;
      ViterbiESentence(forest, &cur_prediction);
      SufficientStats sstats;
      ds[sent_id]->Evaluate(cur_prediction, &sstats);
      float sentscore = metric.ComputeScore(sstats);
      cur_best = MakeHypothesisInfo(ViterbiFeatures(forest), sentscore);

      vector<HypergraphSampler::Hypothesis> samples;
      HypergraphSampler::sample_hypotheses(forest, kbest_size, &*rng, &samples);
      for (unsigned i = 0; i < samples.size(); ++i) {
        ds[sent_id]->Evaluate(samples[i].words, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(samples[i].fmap, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(samples[i].fmap, sentscore);
      }
    } else {
      KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(forest, kbest_size);
      SufficientStats sstats;
      for (int i = 0; i < kbest_size; ++i) {
        const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
          kbest.LazyKthBest(forest.nodes_.size() - 1, i);
        if (!d) break;
        ds[sent_id]->Evaluate(d->yield, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        // cerr << TD::GetString(d->yield) << " ||| " << d->score << " ||| " << sentscore << endl;
        if (i == 0)
          cur_best = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(d->feature_values, sentscore);
      }
      //cerr << "GOOD: " << cur_good->mt_metric << endl;
      //cerr << " CUR: " << cur_best->mt_metric << endl;
      //cerr << " BAD: " << cur_bad->mt_metric << endl;
    }
  }
};

void ReadTrainingCorpus(const string& fname, vector<string>* c) {
  ReadFile rf(fname);
  istream& in = *rf.stream();
  string line;
  while(in) {
    getline(in, line);
    if (!in) break;
    c->push_back(line);
  }
}

bool ApproxEqual(double a, double b) {
  if (a == b) return true;
  return (fabs(a-b)/fabs(b)) < 0.000001;
}

int main(int argc, char** argv) {
  register_feature_functions();
  SetSilent(true);  // turn off verbose decoder output

  po::variables_map conf;
  if (!InitCommandLine(argc, argv, &conf)) return 1;

  if (conf.count("random_seed"))
    rng.reset(new MT19937(conf["random_seed"].as<uint32_t>()));
  else
    rng.reset(new MT19937);
  const bool sample_forest = conf.count("sample_forest") > 0;
  const bool sample_forest_unit_weight_vector = conf.count("sample_forest_unit_weight_vector") > 0;
  if (sample_forest_unit_weight_vector && !sample_forest) {
    cerr << "Cannot --sample_forest_unit_weight_vector without --sample_forest" << endl;
    return 1;
  }
  vector<string> corpus;
  ReadTrainingCorpus(conf["source"].as<string>(), &corpus);

  string metric_name = UppercaseString(conf["evaluation_metric"].as<string>());
  if (metric_name == "COMBI") {
    cerr << "WARNING: 'combi' metric is no longer supported, switching to 'COMB:TER=-0.5;IBM_BLEU=0.5'\n";
    metric_name = "COMB:TER=-0.5;IBM_BLEU=0.5";
  } else if (metric_name == "BLEU") {
    cerr << "WARNING: 'BLEU' is ambiguous, assuming 'IBM_BLEU'\n";
    metric_name = "IBM_BLEU";
  }
  EvaluationMetric* metric = EvaluationMetric::Instance(metric_name);
  DocumentScorer ds(metric, conf["reference"].as<vector<string> >());
  cerr << "Loaded " << ds.size() << " references for scoring with " << metric_name << endl;
  invert_score = metric->IsErrorMetric();

  if (ds.size() != corpus.size()) {
    cerr << "Mismatched number of references (" << ds.size() << ") and sources (" << corpus.size() << ")\n";
    return 1;
  }

  ReadFile ini_rf(conf["decoder_config"].as<string>());
  Decoder decoder(ini_rf.stream());

  // load initial weights
  vector<weight_t>& dense_weights = decoder.CurrentWeightVector();
  SparseVector<weight_t> lambdas;
  Weights::InitFromFile(conf["input_weights"].as<string>(), &dense_weights);
  Weights::InitSparseVector(dense_weights, &lambdas);

  const double max_step_size = conf["max_step_size"].as<double>();
  const double mt_metric_scale = conf["mt_metric_scale"].as<double>();

  assert(corpus.size() > 0);
  vector<GoodBadOracle> oracles(corpus.size());

  TrainingObserver observer(conf["k_best_size"].as<int>(), ds, *metric, sample_forest, &oracles);
  int cur_sent = 0;
  int lcount = 0;
  int normalizer = 0;
  double tot_loss = 0;
  int dots = 0;
  int cur_pass = 0;
  SparseVector<double> tot;
  tot += lambdas;          // initial weights
  normalizer++;            // count for initial weights
  int max_iteration = conf["passes"].as<int>() * corpus.size();
  string msg = "# MIRA tuned weights";
  string msga = "# MIRA tuned weights AVERAGED";
  vector<int> order;
  RandomPermutation(corpus.size(), &order);
  while (lcount <= max_iteration) {
    lambdas.init_vector(&dense_weights);
    if ((cur_sent * 40 / corpus.size()) > dots) { ++dots; cerr << '.'; }
    if (corpus.size() == cur_sent) {
      cerr << " [AVG METRIC LAST PASS=" << (tot_loss / corpus.size()) << "]\n";
      Weights::ShowLargestFeatures(dense_weights);
      cur_sent = 0;
      tot_loss = 0;
      dots = 0;
      ostringstream os;
      os << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << ".gz";
      SparseVector<double> x = tot;
      x /= normalizer;
      ostringstream sa;
      sa << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << "-avg.gz";
      x.init_vector(&dense_weights);
      Weights::WriteToFile(os.str(), dense_weights, true, &msg);
      ++cur_pass;
      RandomPermutation(corpus.size(), &order);
    }
    if (cur_sent == 0) {
      cerr << "PASS " << (lcount / corpus.size() + 1) << endl;
    }
    decoder.SetId(order[cur_sent]);
    double sc = 1.0;
    if (sample_forest_unit_weight_vector) {
      sc = lambdas.l2norm();
      if (sc > 0) {
        for (unsigned i = 0; i < dense_weights.size(); ++i)
          dense_weights[i] /= sc;
      }
    }
    decoder.Decode(corpus[order[cur_sent]], &observer);  // update oracles
    if (sc && sc != 1.0) {
      for (unsigned i = 0; i < dense_weights.size(); ++i)
        dense_weights[i] *= sc;
    }
    const HypothesisInfo& cur_hyp = observer.GetCurrentBestHypothesis();
    const HypothesisInfo& cur_good = *oracles[order[cur_sent]].good;
    const HypothesisInfo& cur_bad = *oracles[order[cur_sent]].bad;
    tot_loss += cur_hyp.mt_metric;
    if (!ApproxEqual(cur_hyp.mt_metric, cur_good.mt_metric)) {
      const double loss = cur_bad.features.dot(dense_weights) - cur_good.features.dot(dense_weights) +
          mt_metric_scale * (cur_good.mt_metric - cur_bad.mt_metric);
      //cerr << "LOSS: " << loss << endl;
      if (loss > 0.0) {
        SparseVector<double> diff = cur_good.features;
        diff -= cur_bad.features;
        double step_size = loss / diff.l2norm_sq();
        //cerr << loss << " " << step_size << " " << diff << endl;
        if (step_size > max_step_size) step_size = max_step_size;
        lambdas += (cur_good.features * step_size);
        lambdas -= (cur_bad.features * step_size);
        //cerr << "L: " << lambdas << endl;
      }
    }
    tot += lambdas;
    ++normalizer;
    ++lcount;
    ++cur_sent;
  }
  cerr << endl;
  Weights::WriteToFile("weights.mira-final.gz", dense_weights, true, &msg);
  tot /= normalizer;
  tot.init_vector(dense_weights);
  msg = "# MIRA tuned weights (averaged vector)";
  Weights::WriteToFile("weights.mira-final-avg.gz", dense_weights, true, &msg);
  cerr << "Optimization complete.\nAVERAGED WEIGHTS: weights.mira-final-avg.gz\n";
  return 0;
}

<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: gsiconv.cxx,v $
 *
 *  $Revision: 1.4 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-09 15:00:09 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/
#include <stdio.h>
#include <tools/fsys.hxx>
#include <tools/stream.hxx>

// local includes
#include "utf8conv.hxx"

#define GSI_FILE_UNKNOWN        0x0000
#define GSI_FILE_OLDSTYLE       0x0001
#define GSI_FILE_L10NFRAMEWORK  0x0002

/*****************************************************************************/
USHORT GetGSIFileType( SvStream &rStream )
/*****************************************************************************/
{
    USHORT nFileType = GSI_FILE_UNKNOWN;

    ULONG nPos( rStream.Tell());
    rStream.Seek( STREAM_SEEK_TO_BEGIN );

    ByteString sLine;
    while( !rStream.IsEof() && !sLine.Len())
        rStream.ReadLine( sLine );

    if( sLine.Len()) {
        if( sLine.Search( "($$)" ) != STRING_NOTFOUND )
            nFileType = GSI_FILE_OLDSTYLE;
        else
            nFileType = GSI_FILE_L10NFRAMEWORK;
    }

    rStream.Seek( nPos );

    return nFileType;
}

/*****************************************************************************/
ByteString GetGSILineId( const ByteString &rLine, USHORT nFileType )
/*****************************************************************************/
{
    ByteString sId;
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            sId = rLine;
            sId.SearchAndReplaceAll( "($$)", "\t" );
            sId = sId.GetToken( 0, '\t' );
          break;

        case GSI_FILE_L10NFRAMEWORK:
            sId = rLine.GetToken( 0, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 1, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 4, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 5, '\t' );
          break;
    }
    return sId;
}

/*****************************************************************************/
ByteString GetGSILineLangId( const ByteString &rLine, USHORT nFileType )
/*****************************************************************************/
{
    ByteString sLangId;
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            sLangId = rLine;
            sLangId.SearchAndReplaceAll( "($$)", "\t" );
            sLangId = sLangId.GetToken( 2, '\t' );
          break;

        case GSI_FILE_L10NFRAMEWORK:
            sLangId = rLine.GetToken( 9, '\t' );
          break;
    }
    return sLangId;
}

/*****************************************************************************/
void ConvertGSILine( BOOL bToUTF8, ByteString &rLine,
        rtl_TextEncoding nEncoding, USHORT nFileType )
/*****************************************************************************/
{
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            if ( bToUTF8 )
                rLine = UTF8Converter::ConvertToUTF8( rLine, nEncoding );
            else
                rLine = UTF8Converter::ConvertFromUTF8( rLine, nEncoding );
        break;

        case GSI_FILE_L10NFRAMEWORK: {
            ByteString sConverted;
            for ( USHORT i = 0; i < rLine.GetTokenCount( '\t' ); i++ ) {
                ByteString sToken = rLine.GetToken( i, '\t' );
                if (( i > 9 ) && ( i < 14 )) {
                    if( bToUTF8 )
                        sToken = UTF8Converter::ConvertToUTF8( sToken, nEncoding );
                    else
                        sToken = UTF8Converter::ConvertFromUTF8( sToken, nEncoding );
                }
                if ( i )
                    sConverted += "\t";
                sConverted += sToken;
            }
            rLine = sConverted;
        }
        break;
    }
}

/*****************************************************************************/
void Help()
/*****************************************************************************/
{
    fprintf( stdout, "\n" );
    fprintf( stdout, "gsiconv (c)1999 by StarOffice Entwicklungs GmbH\n" );
    fprintf( stdout, "===============================================\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "gsiconv converts strings in GSI-Files (Gutschmitt Interface) from or to UTF-8\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Syntax: gsiconv (-t|-f langid charset)|(-p n) filename\n" );
    fprintf( stdout, "Switches: -t   => conversion from charset to UTF-8\n" );
    fprintf( stdout, "          -f   => conversion from UTF-8 to charset\n" );
    fprintf( stdout, "          -p n => creates several files with ca. n lines\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Allowed charsets:\n" );
    fprintf( stdout, "          MS_932  => Japanese\n" );
    fprintf( stdout, "          MS_936  => Chinese Simplified\n" );
    fprintf( stdout, "          MS_949  => Korean\n" );
    fprintf( stdout, "          MS_950  => Chinese Traditional\n" );
    fprintf( stdout, "          MS_1250 => East Europe\n" );
    fprintf( stdout, "          MS_1251 => Cyrillic\n" );
    fprintf( stdout, "          MS_1252 => West Europe\n" );
    fprintf( stdout, "          MS_1253 => Greek\n" );
    fprintf( stdout, "          MS_1254 => Turkish\n" );
    fprintf( stdout, "          MS_1255 => Hebrew\n" );
    fprintf( stdout, "          MS_1256 => Arabic\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Allowed langids:\n" );
    fprintf( stdout, "          1  => ENGLISH_US\n" );
    fprintf( stdout, "          3  => PORTUGUESE \n" );
    fprintf( stdout, "          4  => GERMAN_DE (new german style)\n" );
    fprintf( stdout, "          7  => RUSSIAN\n" );
    fprintf( stdout, "          30 => GREEK\n" );
    fprintf( stdout, "          31 => DUTCH\n" );
    fprintf( stdout, "          33 => FRENCH\n" );
    fprintf( stdout, "          34 => SPANISH\n" );
    fprintf( stdout, "          35 => FINNISH\n" );
    fprintf( stdout, "          36 => HUNGARIAN\n" );
    fprintf( stdout, "          39 => ITALIAN\n" );
    fprintf( stdout, "          42 => CZECH\n" );
    fprintf( stdout, "          44 => ENGLISH (UK)\n" );
    fprintf( stdout, "          45 => DANISH\n" );
    fprintf( stdout, "          46 => SWEDISH\n" );
    fprintf( stdout, "          47 => NORWEGIAN\n" );
    fprintf( stdout, "          49 => GERMAN (old german style)\n" );
    fprintf( stdout, "          55 => PORTUGUESE_BRAZILIAN\n" );
    fprintf( stdout, "          81 => JAPANESE\n" );
    fprintf( stdout, "          82 => KOREAN\n" );
    fprintf( stdout, "          86 => CHINESE_SIMPLIFIED\n" );
    fprintf( stdout, "          88 => CHINESE_TRADITIONAL\n" );
    fprintf( stdout, "          90 => TURKISH\n" );
    fprintf( stdout, "          96 => ARABIC\n" );
    fprintf( stdout, "          97 => HEBREW\n" );
    fprintf( stdout, "\n" );
}

/*****************************************************************************/
#if defined( UNX ) || defined( MAC )
int main( int argc, char *argv[] )
#else
int _cdecl main( int argc, char *argv[] )
#endif
/*****************************************************************************/
{
    if (( argc != 5 ) && ( argc != 4 )) {
        Help();
        exit ( 0 );
    }

    if ( argc == 4 ) {
        if ( ByteString( argv[ 1 ] ) == "-p" ) {

            DirEntry aSource = DirEntry( String( argv[ 3 ], RTL_TEXTENCODING_ASCII_US ));
            if ( !aSource.Exists()) {
                fprintf( stderr, "\nERROR: GSI-File %s not found!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 2 );
            }

            DirEntry aOutput( aSource );

            String sBase = aOutput.GetBase();
            String sExt = aOutput.GetExtension();

            String sGSI( argv[ 3 ], RTL_TEXTENCODING_ASCII_US );
            SvFileStream aGSI( sGSI, STREAM_STD_READ  );
            if ( !aGSI.IsOpen()) {
                fprintf( stderr, "\nERROR: Could not open GSI-File %s!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 3 );
            }

            USHORT nFileType( GetGSIFileType( aGSI ));

            ULONG nMaxLines = (ULONG) ByteString( argv[ 2 ] ).ToInt64();
            if ( !nMaxLines ) {
                fprintf( stderr, "\nERROR: Linecount must be at least 1!\n\n" );
                exit ( 3 );
            }

            ByteString sGSILine;
            ByteString sOldId;
            ULONG nLine = 0;
            ULONG nOutputFile = 1;

            String sOutput( sBase );
            sOutput += String( "_", RTL_TEXTENCODING_ASCII_US );
            sOutput += String::CreateFromInt64( nOutputFile );
            if ( sExt.Len()) {
                sOutput += String( ".", RTL_TEXTENCODING_ASCII_US );
                sOutput += sExt;
            }
            nOutputFile ++;

            aOutput.SetName( sOutput );
            SvFileStream aOutputStream( aOutput.GetFull(), STREAM_STD_WRITE | STREAM_TRUNC );

            while ( !aGSI.IsEof()) {

                aGSI.ReadLine( sGSILine );
                ByteString sId( GetGSILineId( sGSILine, nFileType ));

                nLine++;

                if (( nLine >= nMaxLines ) && ( sId != sOldId )) {
                    aOutputStream.Close();

                    ByteString sText( aOutput.GetFull(), gsl_getSystemTextEncoding());
                    sText += " with ";
                    sText += ByteString::CreateFromInt64( nLine );
                    sText += " lines written.";

                    fprintf( stdout, "%s\n", sText.GetBuffer());
                    String sOutput( sBase );
                    sOutput += String( "_", RTL_TEXTENCODING_ASCII_US );
                    sOutput += String::CreateFromInt64( nOutputFile );
                    if ( sExt.Len()) {
                        sOutput += String( ".", RTL_TEXTENCODING_ASCII_US );
                        sOutput += sExt;
                    }
                    nOutputFile ++;

                    aOutput.SetName( sOutput );

                    aOutputStream.Open( aOutput.GetFull(), STREAM_STD_WRITE | STREAM_TRUNC );
                    nLine = 0;
                }

                aOutputStream.WriteLine( sGSILine );

                sOldId = sId;
            }

            aGSI.Close();
            aOutputStream.Close();

            ByteString sText( aOutput.GetFull(), RTL_TEXTENCODING_ASCII_US );
            sText += " with ";
            sText += ByteString::CreateFromInt64( nLine );
            sText += " lines written.";
        }
        else {
            Help();
            exit( 1 );
        }
    }
    else {
        if ( ByteString( argv[ 1 ] ) == "-t" || ByteString( argv[ 1 ] ) == "-f" ) {
            rtl_TextEncoding nEncoding;

            ByteString sCurLangId( argv[ 2 ] );

            ByteString sCharset( argv[ 3 ] );
            sCharset.ToUpperAscii();

            if      ( sCharset == "MS_932" )    nEncoding = RTL_TEXTENCODING_MS_932;
            else if ( sCharset == "MS_936" )    nEncoding = RTL_TEXTENCODING_MS_936;
            else if ( sCharset == "MS_949" )    nEncoding = RTL_TEXTENCODING_MS_949;
            else if ( sCharset == "MS_950" )    nEncoding = RTL_TEXTENCODING_MS_950;
            else if ( sCharset == "MS_1250" )   nEncoding = RTL_TEXTENCODING_MS_1250;
            else if ( sCharset == "MS_1251" )   nEncoding = RTL_TEXTENCODING_MS_1251;
            else if ( sCharset == "MS_1252" )   nEncoding = RTL_TEXTENCODING_MS_1252;
            else if ( sCharset == "MS_1253" )   nEncoding = RTL_TEXTENCODING_MS_1253;
            else if ( sCharset == "MS_1254" )   nEncoding = RTL_TEXTENCODING_MS_1254;
            else if ( sCharset == "MS_1255" )   nEncoding = RTL_TEXTENCODING_MS_1255;
            else if ( sCharset == "MS_1256" )   nEncoding = RTL_TEXTENCODING_MS_1256;
            else if ( sCharset == "MS_1257" )   nEncoding = RTL_TEXTENCODING_MS_1257;
            else if ( sCharset == "UTF8" )      nEncoding = RTL_TEXTENCODING_UTF8;

            else {
                Help();
                exit ( 1 );
            }

            DirEntry aSource = DirEntry( String( argv[ 4 ], RTL_TEXTENCODING_ASCII_US ));
            if ( !aSource.Exists()) {
                fprintf( stderr, "\nERROR: GSI-File %s not found!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 2 );
            }

            String sGSI( argv[ 4 ], RTL_TEXTENCODING_ASCII_US );
            SvFileStream aGSI( sGSI, STREAM_STD_READ );
            if ( !aGSI.IsOpen()) {
                fprintf( stderr, "\nERROR: Could not open GSI-File %s!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 3 );
            }
            USHORT nFileType( GetGSIFileType( aGSI ));

            ByteString sGSILine;
            while ( !aGSI.IsEof()) {

                aGSI.ReadLine( sGSILine );
                ByteString sLangId( GetGSILineLangId( sGSILine, nFileType ));
                if ( sLangId == sCurLangId )
                    ConvertGSILine(( ByteString( argv[ 1 ] ) == "-t" ), sGSILine, nEncoding, nFileType );

                fprintf( stdout, "%s\n", sGSILine.GetBuffer());
            }

            aGSI.Close();
        }
        else {
            Help();
            exit( 1 );
        }
    }
    return 0;
}
<commit_msg>INTEGRATION: CWS warnings01 (1.4.16); FILE MERGED 2005/11/07 12:46:42 ihi 1.4.16.1: #i57362# Remove warnings<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: gsiconv.cxx,v $
 *
 *  $Revision: 1.5 $
 *
 *  last change: $Author: hr $ $Date: 2006-06-19 17:22:22 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/
#include <stdio.h>
#include <tools/fsys.hxx>
#include <tools/stream.hxx>

// local includes
#include "utf8conv.hxx"

#define GSI_FILE_UNKNOWN        0x0000
#define GSI_FILE_OLDSTYLE       0x0001
#define GSI_FILE_L10NFRAMEWORK  0x0002

/*****************************************************************************/
USHORT GetGSIFileType( SvStream &rStream )
/*****************************************************************************/
{
    USHORT nFileType = GSI_FILE_UNKNOWN;

    ULONG nPos( rStream.Tell());
    rStream.Seek( STREAM_SEEK_TO_BEGIN );

    ByteString sLine;
    while( !rStream.IsEof() && !sLine.Len())
        rStream.ReadLine( sLine );

    if( sLine.Len()) {
        if( sLine.Search( "($$)" ) != STRING_NOTFOUND )
            nFileType = GSI_FILE_OLDSTYLE;
        else
            nFileType = GSI_FILE_L10NFRAMEWORK;
    }

    rStream.Seek( nPos );

    return nFileType;
}

/*****************************************************************************/
ByteString GetGSILineId( const ByteString &rLine, USHORT nFileType )
/*****************************************************************************/
{
    ByteString sId;
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            sId = rLine;
            sId.SearchAndReplaceAll( "($$)", "\t" );
            sId = sId.GetToken( 0, '\t' );
          break;

        case GSI_FILE_L10NFRAMEWORK:
            sId = rLine.GetToken( 0, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 1, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 4, '\t' );
            sId += "\t";
            sId += rLine.GetToken( 5, '\t' );
          break;
    }
    return sId;
}

/*****************************************************************************/
ByteString GetGSILineLangId( const ByteString &rLine, USHORT nFileType )
/*****************************************************************************/
{
    ByteString sLangId;
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            sLangId = rLine;
            sLangId.SearchAndReplaceAll( "($$)", "\t" );
            sLangId = sLangId.GetToken( 2, '\t' );
          break;

        case GSI_FILE_L10NFRAMEWORK:
            sLangId = rLine.GetToken( 9, '\t' );
          break;
    }
    return sLangId;
}

/*****************************************************************************/
void ConvertGSILine( BOOL bToUTF8, ByteString &rLine,
        rtl_TextEncoding nEncoding, USHORT nFileType )
/*****************************************************************************/
{
    switch ( nFileType ) {
        case GSI_FILE_OLDSTYLE:
            if ( bToUTF8 )
                rLine = UTF8Converter::ConvertToUTF8( rLine, nEncoding );
            else
                rLine = UTF8Converter::ConvertFromUTF8( rLine, nEncoding );
        break;

        case GSI_FILE_L10NFRAMEWORK: {
            ByteString sConverted;
            for ( USHORT i = 0; i < rLine.GetTokenCount( '\t' ); i++ ) {
                ByteString sToken = rLine.GetToken( i, '\t' );
                if (( i > 9 ) && ( i < 14 )) {
                    if( bToUTF8 )
                        sToken = UTF8Converter::ConvertToUTF8( sToken, nEncoding );
                    else
                        sToken = UTF8Converter::ConvertFromUTF8( sToken, nEncoding );
                }
                if ( i )
                    sConverted += "\t";
                sConverted += sToken;
            }
            rLine = sConverted;
        }
        break;
    }
}

/*****************************************************************************/
void Help()
/*****************************************************************************/
{
    fprintf( stdout, "\n" );
    fprintf( stdout, "gsiconv (c)1999 by StarOffice Entwicklungs GmbH\n" );
    fprintf( stdout, "===============================================\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "gsiconv converts strings in GSI-Files (Gutschmitt Interface) from or to UTF-8\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Syntax: gsiconv (-t|-f langid charset)|(-p n) filename\n" );
    fprintf( stdout, "Switches: -t   => conversion from charset to UTF-8\n" );
    fprintf( stdout, "          -f   => conversion from UTF-8 to charset\n" );
    fprintf( stdout, "          -p n => creates several files with ca. n lines\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Allowed charsets:\n" );
    fprintf( stdout, "          MS_932  => Japanese\n" );
    fprintf( stdout, "          MS_936  => Chinese Simplified\n" );
    fprintf( stdout, "          MS_949  => Korean\n" );
    fprintf( stdout, "          MS_950  => Chinese Traditional\n" );
    fprintf( stdout, "          MS_1250 => East Europe\n" );
    fprintf( stdout, "          MS_1251 => Cyrillic\n" );
    fprintf( stdout, "          MS_1252 => West Europe\n" );
    fprintf( stdout, "          MS_1253 => Greek\n" );
    fprintf( stdout, "          MS_1254 => Turkish\n" );
    fprintf( stdout, "          MS_1255 => Hebrew\n" );
    fprintf( stdout, "          MS_1256 => Arabic\n" );
    fprintf( stdout, "\n" );
    fprintf( stdout, "Allowed langids:\n" );
    fprintf( stdout, "          1  => ENGLISH_US\n" );
    fprintf( stdout, "          3  => PORTUGUESE \n" );
    fprintf( stdout, "          4  => GERMAN_DE (new german style)\n" );
    fprintf( stdout, "          7  => RUSSIAN\n" );
    fprintf( stdout, "          30 => GREEK\n" );
    fprintf( stdout, "          31 => DUTCH\n" );
    fprintf( stdout, "          33 => FRENCH\n" );
    fprintf( stdout, "          34 => SPANISH\n" );
    fprintf( stdout, "          35 => FINNISH\n" );
    fprintf( stdout, "          36 => HUNGARIAN\n" );
    fprintf( stdout, "          39 => ITALIAN\n" );
    fprintf( stdout, "          42 => CZECH\n" );
    fprintf( stdout, "          44 => ENGLISH (UK)\n" );
    fprintf( stdout, "          45 => DANISH\n" );
    fprintf( stdout, "          46 => SWEDISH\n" );
    fprintf( stdout, "          47 => NORWEGIAN\n" );
    fprintf( stdout, "          49 => GERMAN (old german style)\n" );
    fprintf( stdout, "          55 => PORTUGUESE_BRAZILIAN\n" );
    fprintf( stdout, "          81 => JAPANESE\n" );
    fprintf( stdout, "          82 => KOREAN\n" );
    fprintf( stdout, "          86 => CHINESE_SIMPLIFIED\n" );
    fprintf( stdout, "          88 => CHINESE_TRADITIONAL\n" );
    fprintf( stdout, "          90 => TURKISH\n" );
    fprintf( stdout, "          96 => ARABIC\n" );
    fprintf( stdout, "          97 => HEBREW\n" );
    fprintf( stdout, "\n" );
}

/*****************************************************************************/
#if defined( UNX ) || defined( MAC )
int main( int argc, char *argv[] )
#else
int _cdecl main( int argc, char *argv[] )
#endif
/*****************************************************************************/
{
    if (( argc != 5 ) && ( argc != 4 )) {
        Help();
        exit ( 0 );
    }

    if ( argc == 4 ) {
        if ( ByteString( argv[ 1 ] ) == "-p" ) {

            DirEntry aSource = DirEntry( String( argv[ 3 ], RTL_TEXTENCODING_ASCII_US ));
            if ( !aSource.Exists()) {
                fprintf( stderr, "\nERROR: GSI-File %s not found!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 2 );
            }

            DirEntry aOutput( aSource );

            String sBase = aOutput.GetBase();
            String sExt = aOutput.GetExtension();

            String sGSI( argv[ 3 ], RTL_TEXTENCODING_ASCII_US );
            SvFileStream aGSI( sGSI, STREAM_STD_READ  );
            if ( !aGSI.IsOpen()) {
                fprintf( stderr, "\nERROR: Could not open GSI-File %s!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 3 );
            }

            USHORT nFileType( GetGSIFileType( aGSI ));

            ULONG nMaxLines = (ULONG) ByteString( argv[ 2 ] ).ToInt64();
            if ( !nMaxLines ) {
                fprintf( stderr, "\nERROR: Linecount must be at least 1!\n\n" );
                exit ( 3 );
            }

            ByteString sGSILine;
            ByteString sOldId;
            ULONG nLine = 0;
            ULONG nOutputFile = 1;

            String sOutput( sBase );
            sOutput += String( "_", RTL_TEXTENCODING_ASCII_US );
            sOutput += String::CreateFromInt64( nOutputFile );
            if ( sExt.Len()) {
                sOutput += String( ".", RTL_TEXTENCODING_ASCII_US );
                sOutput += sExt;
            }
            nOutputFile ++;

            aOutput.SetName( sOutput );
            SvFileStream aOutputStream( aOutput.GetFull(), STREAM_STD_WRITE | STREAM_TRUNC );

            while ( !aGSI.IsEof()) {

                aGSI.ReadLine( sGSILine );
                ByteString sId( GetGSILineId( sGSILine, nFileType ));

                nLine++;

                if (( nLine >= nMaxLines ) && ( sId != sOldId )) {
                    aOutputStream.Close();

                    ByteString sText( aOutput.GetFull(), gsl_getSystemTextEncoding());
                    sText += " with ";
                    sText += ByteString::CreateFromInt64( nLine );
                    sText += " lines written.";

                    fprintf( stdout, "%s\n", sText.GetBuffer());
                    String sOutput1( sBase );
                    sOutput1 += String( "_", RTL_TEXTENCODING_ASCII_US );
                    sOutput1 += String::CreateFromInt64( nOutputFile );
                    if ( sExt.Len()) {
                        sOutput1 += String( ".", RTL_TEXTENCODING_ASCII_US );
                        sOutput1 += sExt;
                    }
                    nOutputFile ++;

                    aOutput.SetName( sOutput1 );

                    aOutputStream.Open( aOutput.GetFull(), STREAM_STD_WRITE | STREAM_TRUNC );
                    nLine = 0;
                }

                aOutputStream.WriteLine( sGSILine );

                sOldId = sId;
            }

            aGSI.Close();
            aOutputStream.Close();

            ByteString sText( aOutput.GetFull(), RTL_TEXTENCODING_ASCII_US );
            sText += " with ";
            sText += ByteString::CreateFromInt64( nLine );
            sText += " lines written.";
        }
        else {
            Help();
            exit( 1 );
        }
    }
    else {
        if ( ByteString( argv[ 1 ] ) == "-t" || ByteString( argv[ 1 ] ) == "-f" ) {
            rtl_TextEncoding nEncoding;

            ByteString sCurLangId( argv[ 2 ] );

            ByteString sCharset( argv[ 3 ] );
            sCharset.ToUpperAscii();

            if      ( sCharset == "MS_932" )    nEncoding = RTL_TEXTENCODING_MS_932;
            else if ( sCharset == "MS_936" )    nEncoding = RTL_TEXTENCODING_MS_936;
            else if ( sCharset == "MS_949" )    nEncoding = RTL_TEXTENCODING_MS_949;
            else if ( sCharset == "MS_950" )    nEncoding = RTL_TEXTENCODING_MS_950;
            else if ( sCharset == "MS_1250" )   nEncoding = RTL_TEXTENCODING_MS_1250;
            else if ( sCharset == "MS_1251" )   nEncoding = RTL_TEXTENCODING_MS_1251;
            else if ( sCharset == "MS_1252" )   nEncoding = RTL_TEXTENCODING_MS_1252;
            else if ( sCharset == "MS_1253" )   nEncoding = RTL_TEXTENCODING_MS_1253;
            else if ( sCharset == "MS_1254" )   nEncoding = RTL_TEXTENCODING_MS_1254;
            else if ( sCharset == "MS_1255" )   nEncoding = RTL_TEXTENCODING_MS_1255;
            else if ( sCharset == "MS_1256" )   nEncoding = RTL_TEXTENCODING_MS_1256;
            else if ( sCharset == "MS_1257" )   nEncoding = RTL_TEXTENCODING_MS_1257;
            else if ( sCharset == "UTF8" )      nEncoding = RTL_TEXTENCODING_UTF8;

            else {
                Help();
                exit ( 1 );
            }

            DirEntry aSource = DirEntry( String( argv[ 4 ], RTL_TEXTENCODING_ASCII_US ));
            if ( !aSource.Exists()) {
                fprintf( stderr, "\nERROR: GSI-File %s not found!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 2 );
            }

            String sGSI( argv[ 4 ], RTL_TEXTENCODING_ASCII_US );
            SvFileStream aGSI( sGSI, STREAM_STD_READ );
            if ( !aGSI.IsOpen()) {
                fprintf( stderr, "\nERROR: Could not open GSI-File %s!\n\n", ByteString( argv[ 3 ] ).GetBuffer());
                exit ( 3 );
            }
            USHORT nFileType( GetGSIFileType( aGSI ));

            ByteString sGSILine;
            while ( !aGSI.IsEof()) {

                aGSI.ReadLine( sGSILine );
                ByteString sLangId( GetGSILineLangId( sGSILine, nFileType ));
                if ( sLangId == sCurLangId )
                    ConvertGSILine(( ByteString( argv[ 1 ] ) == "-t" ), sGSILine, nEncoding, nFileType );

                fprintf( stdout, "%s\n", sGSILine.GetBuffer());
            }

            aGSI.Close();
        }
        else {
            Help();
            exit( 1 );
        }
    }
    return 0;
}
<|endoftext|>


<commit_before>/**
 * projectM -- Milkdrop-esque visualisation SDK
 * Copyright (C)2003-2004 projectM Team
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 * See 'LICENSE.txt' included within this release
 *
 */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef WIN32
#include "win32-dirent.h"
#else
#include <dirent.h>
#endif /** WIN32 */
#include <time.h>

#include "Preset.hpp"
#include "Parser.hpp"
#include "ParamUtils.hpp"
#include "InitCondUtils.hpp"
#include "fatal.h"
#include <iostream>

Preset::Preset(const std::string & filename, const PresetInputs & presetInputs, PresetOutputs & presetOutputs):
    builtinParams(presetInputs, presetOutputs),
    file_path(filename),
    m_presetOutputs(presetOutputs)
{

  clearMeshChecks();

  initialize(filename);

}

Preset::~Preset()
{

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<InitCond> >(init_cond_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<InitCond> >(per_frame_init_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<PerPixelEqn> >(per_pixel_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<PerFrameEqn> >(per_frame_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<Param> >(user_param_tree);

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {
    	delete(*pos);
  }

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {
    	delete(*pos);
  }

}

/* Adds a per pixel equation according to its string name. This
   will be used only by the parser */

int Preset::add_per_pixel_eqn(char * name, GenExpr * gen_expr)
{

  PerPixelEqn * per_pixel_eqn = NULL;
  int index;
  Param * param = NULL;

  assert(gen_expr);
  assert(name);

  if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: per pixel equation (name = \"%s\")\n", name);

  if (!strncmp(name, "dx", strlen("dx")))
    this->m_presetOutputs.dx_is_mesh = true;
  else if (!strncmp(name, "dy", strlen("dy")))
    this->m_presetOutputs.dy_is_mesh = true;
  else if (!strncmp(name, "cx", strlen("cx")))
    this->m_presetOutputs.cx_is_mesh = true;
  else if (!strncmp(name, "cy", strlen("cy")))
    this->m_presetOutputs.cy_is_mesh = true;
  else if (!strncmp(name, "zoom", strlen("zoom")))
    this->m_presetOutputs.zoom_is_mesh = true;
  else if (!strncmp(name, "zoomexp", strlen("zoomexp")))
    this->m_presetOutputs.zoomexp_is_mesh = true;
  else if (!strncmp(name, "rot", strlen("rot")))
    this->m_presetOutputs.rot_is_mesh= true;
  else if (!strncmp(name, "sx", strlen("sx")))
    this->m_presetOutputs.sx_is_mesh = true;
  else if (!strncmp(name, "sy", strlen("sy")))
    this->m_presetOutputs.sy_is_mesh = true;

  /* Search for the parameter so we know what matrix the per pixel equation is referencing */

  param = ParamUtils::find(name, &this->builtinParams, &this->user_param_tree);
  if ( !param )
  {
    if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: failed to allocate a new parameter!\n");
    return PROJECTM_FAILURE;
  }

  index = per_pixel_eqn_tree.size();

  /* Create the per pixel equation given the index, parameter, and general expression */
  if ((per_pixel_eqn = new PerPixelEqn(index, param, gen_expr)) == NULL)
  {
    if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: failed to create new per pixel equation!\n");
    return PROJECTM_FAILURE;

  }

  /* Insert the per pixel equation into the preset per pixel database */
  std::pair<std::map<int, PerPixelEqn*>::iterator, bool> inserteeOption = per_pixel_eqn_tree.insert
      (std::make_pair(per_pixel_eqn->index, per_pixel_eqn));

  if (!inserteeOption.second)
  {
    printf("failed to add per pixel eqn!\n");
    delete(per_pixel_eqn);
    return PROJECTM_FAILURE;
  }

  /* Done */
  return PROJECTM_SUCCESS;
}

void Preset::evalCustomShapeInitConditions()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos) {
    assert(*pos);
    (*pos)->evalInitConds();
  }
}

void Preset::evalCustomWaveInitConditions()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos) {
    assert(*pos);
   (*pos)->evalInitConds();
}
}

void Preset::evalCustomWavePerFrameEquations()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {

    std::map<std::string, InitCond*> & init_cond_tree = (*pos)->init_cond_tree;
    for (std::map<std::string, InitCond*>::iterator _pos = init_cond_tree.begin(); _pos != init_cond_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }

    std::map<int, PerFrameEqn*> & per_frame_eqn_tree = (*pos)->per_frame_eqn_tree;
    for (std::map<int, PerFrameEqn*>::iterator _pos = per_frame_eqn_tree.begin(); _pos != per_frame_eqn_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
  }

}

void Preset::evalCustomShapePerFrameEquations()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {

    std::map<std::string, InitCond*> & init_cond_tree = (*pos)->init_cond_tree;
    for (std::map<std::string, InitCond*>::iterator _pos = init_cond_tree.begin(); _pos != init_cond_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
    std::map<int, PerFrameEqn*> & per_frame_eqn_tree = (*pos)->per_frame_eqn_tree;
    for (std::map<int, PerFrameEqn*>::iterator _pos = per_frame_eqn_tree.begin(); _pos != per_frame_eqn_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
  }

}

void Preset::evalPerFrameInitEquations()
{

  for (std::map<std::string, InitCond*>::iterator pos = per_frame_init_eqn_tree.begin(); pos != per_frame_init_eqn_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

}

void Preset::evalPerFrameEquations()
{

  for (std::map<std::string, InitCond*>::iterator pos = init_cond_tree.begin(); pos != init_cond_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

  for (std::map<int, PerFrameEqn*>::iterator pos = per_frame_eqn_tree.begin(); pos != per_frame_eqn_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

}

void Preset::initialize(const std::string & pathname)
{

  // Clear equation trees
  /// @slow unnecessary if we ensure this method is private
  init_cond_tree.clear();
  user_param_tree.clear();
  per_frame_eqn_tree.clear();
  per_pixel_eqn_tree.clear();
  per_frame_init_eqn_tree.clear();

  /* Set initial index values */
  this->per_pixel_eqn_string_index = 0;
  this->per_frame_eqn_string_index = 0;
  this->per_frame_init_eqn_string_index = 0;

  /* Clear string buffers */
  /// @bug replace with ostringstream?
  memset(this->per_pixel_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  memset(this->per_frame_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  memset(this->per_frame_init_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  int retval;

  std::cerr << "[Preset] loading file \"" << pathname << "\"..." << std::endl;

  if ((retval = loadPresetFile(pathname)) < 0)
  {

     std::cerr << "[Preset] failed to load file \"" <<
      pathname << "\"!" << std::endl;

    /// @bug how should we handle this problem? a well define exception?
    throw retval;
  }

  /* It's kind of ugly to reset these values here. Should definitely be placed in the parser somewhere */
  this->per_frame_eqn_count = 0;
  this->per_frame_init_eqn_count = 0;

  this->loadBuiltinParamsUnspecInitConds();
  this->loadCustomWaveUnspecInitConds();
  this->loadCustomShapeUnspecInitConds();
}

void Preset::loadBuiltinParamsUnspecInitConds() {

  InitCondUtils::LoadUnspecInitCond loadUnspecInitCond(this->init_cond_tree, this->per_frame_init_eqn_tree);

  this->builtinParams.traverse(loadUnspecInitCond);

}

void Preset::loadCustomWaveUnspecInitConds()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {
    assert(*pos);
    (*pos)->loadUnspecInitConds();
  }

}

void Preset::loadCustomShapeUnspecInitConds()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {
    assert(*pos);
    (*pos)->loadUnspecInitConds();
  }
}

void Preset::evaluateFrame()
{

  /* Evaluate all equation objects in same order as the renderer */

  evalPerFrameInitEquations();
  evalPerFrameEquations();
  evalPerPixelEqns();
  evalCustomWaveInitConditions();
  evalCustomShapeInitConditions();
  evalCustomWavePerFrameEquations();
  evalCustomShapePerFrameEquations();

  // Setup pointers of the custom waves and shapes to the preset outputs instance
  /// @slow an extra O(N) per frame, could do this during eval
  m_presetOutputs.customWaves = PresetOutputs::cwave_container(customWaves);
  m_presetOutputs.customShapes = PresetOutputs::cshape_container(customShapes);

}

// Evaluates all per-pixel equations
void Preset::evalPerPixelEqns()
{

  /* Evaluate all per pixel equations in the tree datastructure */
  for (std::map<int, PerPixelEqn*>::iterator pos = per_pixel_eqn_tree.begin();
       pos != per_pixel_eqn_tree.end(); ++pos)
    pos->second->evaluate();

}

/* loadPresetFile: private function that loads a specific preset denoted
   by the given pathname */
int Preset::loadPresetFile(std::string pathname)
{

  FILE * fs;
  int retval;
  int lineno;
  line_mode_t line_mode;

  /* Open the file corresponding to pathname */
  if ((fs = fopen(pathname.c_str(), "rb")) == 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: loading of file %s failed!\n", pathname);
#endif
    return PROJECTM_ERROR;
  }

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: file stream \"%s\" opened successfully\n", pathname);
#endif

  /* Parse any comments */
  if (Parser::parse_top_comment(fs) < 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: no left bracket found...\n");
#endif
    fclose(fs);
    return PROJECTM_FAILURE;
  }

  /* Parse the preset name and a left bracket */
  char tmp_name[MAX_TOKEN_SIZE];

  if (Parser::parse_preset_name(fs, tmp_name) < 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: loading of preset name in file \"%s\" failed\n", pathname);
#endif
    fclose(fs);
    return PROJECTM_ERROR;
  }
  name = std::string(tmp_name);

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: preset \"%s\" parsed\n", this->name);
#endif

  /* Parse each line until end of file */
  lineno = 0;
#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: beginning line parsing...\n");
#endif
  while ((retval = Parser::parse_line(fs, this)) != EOF)
  {
    if (retval == PROJECTM_PARSE_ERROR)
    {
      line_mode = NORMAL_LINE_MODE;
#if defined(PRESET_DEBUG) && defined(DEBUG)
      DWRITE( "loadPresetFile: parse error in file \"%s\": line %d\n", pathname,lineno);
#endif

    }
    lineno++;
  }

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE("loadPresetFile: finished line parsing successfully\n");
#endif

  /* Now the preset has been loaded.
     Evaluation calls can be made at appropiate
     times in the frame loop */

  fclose(fs);
#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE("loadPresetFile: file \"%s\" closed, preset ready\n", pathname);
#endif
  return PROJECTM_SUCCESS;
}

<commit_msg>clearing the preset outputs struct now in preset init<commit_after>/**
 * projectM -- Milkdrop-esque visualisation SDK
 * Copyright (C)2003-2004 projectM Team
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 * See 'LICENSE.txt' included within this release
 *
 */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef WIN32
#include "win32-dirent.h"
#else
#include <dirent.h>
#endif /** WIN32 */
#include <time.h>

#include "Preset.hpp"
#include "Parser.hpp"
#include "ParamUtils.hpp"
#include "InitCondUtils.hpp"
#include "fatal.h"
#include <iostream>

Preset::Preset(const std::string & filename, const PresetInputs & presetInputs, PresetOutputs & presetOutputs):
    builtinParams(presetInputs, presetOutputs),
    file_path(filename),
    m_presetOutputs(presetOutputs)
{

  m_presetOutputs.customWaves.clear();
  m_presetOutputs.customShapes.clear();
  clearMeshChecks();

  initialize(filename);

}

Preset::~Preset()
{

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<InitCond> >(init_cond_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<InitCond> >(per_frame_init_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<PerPixelEqn> >(per_pixel_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<PerFrameEqn> >(per_frame_eqn_tree);

  Algorithms::traverse<Algorithms::TraverseFunctors::DeleteFunctor<Param> >(user_param_tree);

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {
    	delete(*pos);
  }

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {
    	delete(*pos);
  }

}

/* Adds a per pixel equation according to its string name. This
   will be used only by the parser */

int Preset::add_per_pixel_eqn(char * name, GenExpr * gen_expr)
{

  PerPixelEqn * per_pixel_eqn = NULL;
  int index;
  Param * param = NULL;

  assert(gen_expr);
  assert(name);

  if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: per pixel equation (name = \"%s\")\n", name);

  if (!strncmp(name, "dx", strlen("dx")))
    this->m_presetOutputs.dx_is_mesh = true;
  else if (!strncmp(name, "dy", strlen("dy")))
    this->m_presetOutputs.dy_is_mesh = true;
  else if (!strncmp(name, "cx", strlen("cx")))
    this->m_presetOutputs.cx_is_mesh = true;
  else if (!strncmp(name, "cy", strlen("cy")))
    this->m_presetOutputs.cy_is_mesh = true;
  else if (!strncmp(name, "zoom", strlen("zoom")))
    this->m_presetOutputs.zoom_is_mesh = true;
  else if (!strncmp(name, "zoomexp", strlen("zoomexp")))
    this->m_presetOutputs.zoomexp_is_mesh = true;
  else if (!strncmp(name, "rot", strlen("rot")))
    this->m_presetOutputs.rot_is_mesh= true;
  else if (!strncmp(name, "sx", strlen("sx")))
    this->m_presetOutputs.sx_is_mesh = true;
  else if (!strncmp(name, "sy", strlen("sy")))
    this->m_presetOutputs.sy_is_mesh = true;

  /* Search for the parameter so we know what matrix the per pixel equation is referencing */

  param = ParamUtils::find(name, &this->builtinParams, &this->user_param_tree);
  if ( !param )
  {
    if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: failed to allocate a new parameter!\n");
    return PROJECTM_FAILURE;
  }

  index = per_pixel_eqn_tree.size();

  /* Create the per pixel equation given the index, parameter, and general expression */
  if ((per_pixel_eqn = new PerPixelEqn(index, param, gen_expr)) == NULL)
  {
    if (PER_PIXEL_EQN_DEBUG) printf("add_per_pixel_eqn: failed to create new per pixel equation!\n");
    return PROJECTM_FAILURE;

  }

  /* Insert the per pixel equation into the preset per pixel database */
  std::pair<std::map<int, PerPixelEqn*>::iterator, bool> inserteeOption = per_pixel_eqn_tree.insert
      (std::make_pair(per_pixel_eqn->index, per_pixel_eqn));

  if (!inserteeOption.second)
  {
    printf("failed to add per pixel eqn!\n");
    delete(per_pixel_eqn);
    return PROJECTM_FAILURE;
  }

  /* Done */
  return PROJECTM_SUCCESS;
}

void Preset::evalCustomShapeInitConditions()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos) {
    assert(*pos);
    (*pos)->evalInitConds();
  }
}

void Preset::evalCustomWaveInitConditions()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos) {
    assert(*pos);
   (*pos)->evalInitConds();
}
}

void Preset::evalCustomWavePerFrameEquations()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {

    std::map<std::string, InitCond*> & init_cond_tree = (*pos)->init_cond_tree;
    for (std::map<std::string, InitCond*>::iterator _pos = init_cond_tree.begin(); _pos != init_cond_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }

    std::map<int, PerFrameEqn*> & per_frame_eqn_tree = (*pos)->per_frame_eqn_tree;
    for (std::map<int, PerFrameEqn*>::iterator _pos = per_frame_eqn_tree.begin(); _pos != per_frame_eqn_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
  }

}

void Preset::evalCustomShapePerFrameEquations()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {

    std::map<std::string, InitCond*> & init_cond_tree = (*pos)->init_cond_tree;
    for (std::map<std::string, InitCond*>::iterator _pos = init_cond_tree.begin(); _pos != init_cond_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
    std::map<int, PerFrameEqn*> & per_frame_eqn_tree = (*pos)->per_frame_eqn_tree;
    for (std::map<int, PerFrameEqn*>::iterator _pos = per_frame_eqn_tree.begin(); _pos != per_frame_eqn_tree.end(); ++_pos)
    {
      assert(_pos->second);
      _pos->second->evaluate();
    }
  }

}

void Preset::evalPerFrameInitEquations()
{

  for (std::map<std::string, InitCond*>::iterator pos = per_frame_init_eqn_tree.begin(); pos != per_frame_init_eqn_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

}

void Preset::evalPerFrameEquations()
{

  for (std::map<std::string, InitCond*>::iterator pos = init_cond_tree.begin(); pos != init_cond_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

  for (std::map<int, PerFrameEqn*>::iterator pos = per_frame_eqn_tree.begin(); pos != per_frame_eqn_tree.end(); ++pos)
  {
    assert(pos->second);
    pos->second->evaluate();
  }

}

void Preset::initialize(const std::string & pathname)
{

  // Clear equation trees
  /// @slow unnecessary if we ensure this method is private
  init_cond_tree.clear();
  user_param_tree.clear();
  per_frame_eqn_tree.clear();
  per_pixel_eqn_tree.clear();
  per_frame_init_eqn_tree.clear();

  /* Set initial index values */
  this->per_pixel_eqn_string_index = 0;
  this->per_frame_eqn_string_index = 0;
  this->per_frame_init_eqn_string_index = 0;

  /* Clear string buffers */
  /// @bug replace with ostringstream?
  memset(this->per_pixel_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  memset(this->per_frame_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  memset(this->per_frame_init_eqn_string_buffer, 0, STRING_BUFFER_SIZE);
  int retval;

  std::cerr << "[Preset] loading file \"" << pathname << "\"..." << std::endl;

  if ((retval = loadPresetFile(pathname)) < 0)
  {

     std::cerr << "[Preset] failed to load file \"" <<
      pathname << "\"!" << std::endl;

    /// @bug how should we handle this problem? a well define exception?
    throw retval;
  }

  /* It's kind of ugly to reset these values here. Should definitely be placed in the parser somewhere */
  this->per_frame_eqn_count = 0;
  this->per_frame_init_eqn_count = 0;

  this->loadBuiltinParamsUnspecInitConds();
  this->loadCustomWaveUnspecInitConds();
  this->loadCustomShapeUnspecInitConds();
}

void Preset::loadBuiltinParamsUnspecInitConds() {

  InitCondUtils::LoadUnspecInitCond loadUnspecInitCond(this->init_cond_tree, this->per_frame_init_eqn_tree);

  this->builtinParams.traverse(loadUnspecInitCond);

}

void Preset::loadCustomWaveUnspecInitConds()
{

  for (PresetOutputs::cwave_container::iterator pos = customWaves.begin(); pos != customWaves.end(); ++pos)
  {
    assert(*pos);
    (*pos)->loadUnspecInitConds();
  }

}

void Preset::loadCustomShapeUnspecInitConds()
{

  for (PresetOutputs::cshape_container::iterator pos = customShapes.begin(); pos != customShapes.end(); ++pos)
  {
    assert(*pos);
    (*pos)->loadUnspecInitConds();
  }
}

void Preset::evaluateFrame()
{

  /* Evaluate all equation objects in same order as the renderer */

  evalPerFrameInitEquations();
  evalPerFrameEquations();
  evalPerPixelEqns();
  evalCustomWaveInitConditions();
  evalCustomShapeInitConditions();
  evalCustomWavePerFrameEquations();
  evalCustomShapePerFrameEquations();

  // Setup pointers of the custom waves and shapes to the preset outputs instance
  /// @slow an extra O(N) per frame, could do this during eval
  m_presetOutputs.customWaves = PresetOutputs::cwave_container(customWaves);
  m_presetOutputs.customShapes = PresetOutputs::cshape_container(customShapes);

}

// Evaluates all per-pixel equations
void Preset::evalPerPixelEqns()
{

  /* Evaluate all per pixel equations in the tree datastructure */
  for (std::map<int, PerPixelEqn*>::iterator pos = per_pixel_eqn_tree.begin();
       pos != per_pixel_eqn_tree.end(); ++pos)
    pos->second->evaluate();

}

/* loadPresetFile: private function that loads a specific preset denoted
   by the given pathname */
int Preset::loadPresetFile(std::string pathname)
{

  FILE * fs;
  int retval;
  int lineno;
  line_mode_t line_mode;

  /* Open the file corresponding to pathname */
  if ((fs = fopen(pathname.c_str(), "rb")) == 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: loading of file %s failed!\n", pathname);
#endif
    return PROJECTM_ERROR;
  }

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: file stream \"%s\" opened successfully\n", pathname);
#endif

  /* Parse any comments */
  if (Parser::parse_top_comment(fs) < 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: no left bracket found...\n");
#endif
    fclose(fs);
    return PROJECTM_FAILURE;
  }

  /* Parse the preset name and a left bracket */
  char tmp_name[MAX_TOKEN_SIZE];

  if (Parser::parse_preset_name(fs, tmp_name) < 0)
  {
#if defined(PRESET_DEBUG) && defined(DEBUG)
    DWRITE( "loadPresetFile: loading of preset name in file \"%s\" failed\n", pathname);
#endif
    fclose(fs);
    return PROJECTM_ERROR;
  }
  name = std::string(tmp_name);

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: preset \"%s\" parsed\n", this->name);
#endif

  /* Parse each line until end of file */
  lineno = 0;
#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE( "loadPresetFile: beginning line parsing...\n");
#endif
  while ((retval = Parser::parse_line(fs, this)) != EOF)
  {
    if (retval == PROJECTM_PARSE_ERROR)
    {
      line_mode = NORMAL_LINE_MODE;
#if defined(PRESET_DEBUG) && defined(DEBUG)
      DWRITE( "loadPresetFile: parse error in file \"%s\": line %d\n", pathname,lineno);
#endif

    }
    lineno++;
  }

#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE("loadPresetFile: finished line parsing successfully\n");
#endif

  /* Now the preset has been loaded.
     Evaluation calls can be made at appropiate
     times in the frame loop */

  fclose(fs);
#if defined(PRESET_DEBUG) && defined(DEBUG)
  DWRITE("loadPresetFile: file \"%s\" closed, preset ready\n", pathname);
#endif
  return PROJECTM_SUCCESS;
}

<|endoftext|>


<commit_before>// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2016 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#if defined(HAVE_CONFIG_H)
#include "config/bitcoin-config.h"
#endif

#include "chainparams.h"
#include "clientversion.h"
#include "compat.h"
#include "rpc/server.h"
#include "init.h"
#include "noui.h"
#include "scheduler.h"
#include "util.h"
#include "httpserver.h"
#include "httprpc.h"
#include "utilstrencodings.h"

#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
#include <boost/thread.hpp>

#include <stdio.h>

/* Introduction text for doxygen: */

/*! \mainpage Developer documentation
 *
 * \section intro_sec Introduction
 *
 * This is the developer documentation of the reference client for an experimental new digital currency called Bitcoin (https://www.bitcoin.org/),
 * which enables instant payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate
 * with no central authority: managing transactions and issuing money are carried out collectively by the network.
 *
 * The software is a community-driven open source project, released under the MIT license.
 *
 * \section Navigation
 * Use the buttons Namespaces, Classes or Files at the top of the page to start navigating the code.
 */

void WaitForShutdown(boost::thread_group* threadGroup)
{
    bool fShutdown = ShutdownRequested();
    // Tell the main threads to shutdown.
    while (!fShutdown)
    {
        MilliSleep(200);
        fShutdown = ShutdownRequested();
    }
    if (threadGroup)
    {
        Interrupt(*threadGroup);
        threadGroup->join_all();
    }
}

//////////////////////////////////////////////////////////////////////////////
//
// Start
//
bool AppInit(int argc, char* argv[])
{
    boost::thread_group threadGroup;
    CScheduler scheduler;

    bool fRet = false;

    //
    // Parameters
    //
    // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main()
    ParseParameters(argc, argv);

    // Process help and version before taking care about datadir
    if (IsArgSet("-?") || IsArgSet("-h") ||  IsArgSet("-help") || IsArgSet("-version"))
    {
        std::string strUsage = strprintf(_("%s Daemon"), _(PACKAGE_NAME)) + " " + _("version") + " " + FormatFullVersion() + "\n";

        if (IsArgSet("-version"))
        {
            strUsage += FormatParagraph(LicenseInfo());
        }
        else
        {
            strUsage += "\n" + _("Usage:") + "\n" +
                  "  millcoind [options]                     " + strprintf(_("Start %s Daemon"), _(PACKAGE_NAME)) + "\n";

            strUsage += "\n" + HelpMessage(HMM_BITCOIND);
        }

        fprintf(stdout, "%s", strUsage.c_str());
        return true;
    }

    try
    {
        if (!boost::filesystem::is_directory(GetDataDir(false)))
        {
            fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", GetArg("-datadir", "").c_str());
            return false;
        }
        try
        {
            ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME));
        } catch (const std::exception& e) {
            fprintf(stderr,"Error reading configuration file: %s\n", e.what());
            return false;
        }
        // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause)
        try {
            SelectParams(ChainNameFromCommandLine());
        } catch (const std::exception& e) {
            fprintf(stderr, "Error: %s\n", e.what());
            return false;
        }

        // Command-line RPC
        bool fCommandLine = false;
        for (int i = 1; i < argc; i++)
            if (!IsSwitchChar(argv[i][0]) && !boost::algorithm::istarts_with(argv[i], "millcoin:"))
                fCommandLine = true;

        if (fCommandLine)
        {
            fprintf(stderr, "Error: There is no RPC client functionality in millcoind anymore. Use the millcoin-cli utility instead.\n");
            exit(EXIT_FAILURE);
        }
        // -server defaults to true for bitcoind but not for the GUI so do this here
        SoftSetBoolArg("-server", true);
        // Set this early so that parameter interactions go to console
        InitLogging();
        InitParameterInteraction();
        if (!AppInitBasicSetup())
        {
            // InitError will have been called with detailed error, which ends up on console
            exit(1);
        }
        if (!AppInitParameterInteraction())
        {
            // InitError will have been called with detailed error, which ends up on console
            exit(1);
        }
        if (!AppInitSanityChecks())
        {
            // InitError will have been called with detailed error, which ends up on console
            exit(1);
        }
        if (GetBoolArg("-daemon", false))
        {
#if HAVE_DECL_DAEMON
            fprintf(stdout, "Millcoin server starting\n");

            // Daemonize
            if (daemon(1, 0)) { // don't chdir (1), do close FDs (0)
                fprintf(stderr, "Error: daemon() failed: %s\n", strerror(errno));
                return false;
            }
#else
            fprintf(stderr, "Error: -daemon is not supported on this operating system\n");
            return false;
#endif // HAVE_DECL_DAEMON
        }

        fRet = AppInitMain(threadGroup, scheduler);
    }
    catch (const std::exception& e) {
        PrintExceptionContinue(&e, "AppInit()");
    } catch (...) {
        PrintExceptionContinue(NULL, "AppInit()");
    }

    if (!fRet)
    {
        Interrupt(threadGroup);
        // threadGroup.join_all(); was left out intentionally here, because we didn't re-test all of
        // the startup-failure cases to make sure they don't result in a hang due to some
        // thread-blocking-waiting-for-another-thread-during-startup case
    } else {
        WaitForShutdown(&threadGroup);
    }
    Shutdown();

    return fRet;
}

int main(int argc, char* argv[])
{
    SetupEnvironment();

    // Connect bitcoind signal handlers
    noui_connect();

    return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE);
}
<commit_msg>Delete bitcoind.cpp<commit_after><|endoftext|>