<commit_before>#include "Manager.h"
#include "Timer.h"
#include "SearchNormal.h"
#include "SentenceStats.h"

using namespace std;

namespace Moses
{
/**
 * Organizing main function
 *
 * /param source input sentence
 * /param transOptColl collection of translation options to be used for this sentence
 */
SearchNormal::SearchNormal(Manager& manager, const InputType &source, const TranslationOptionCollection &transOptColl)
  :Search(manager)
  ,m_source(source)
  ,m_hypoStackColl(source.GetSize() + 1)
  ,interrupted_flag(0)
  ,m_transOptColl(transOptColl)
{
  VERBOSE(1, "Translating: " << m_source << endl);
  const StaticData &staticData = StaticData::Instance();

  // only if constraint decoding (having to match a specified output)
  // long sentenceID = source.GetTranslationId();

  // initialize the stacks: create data structure and set limits
  std::vector < HypothesisStackNormal >::iterator iterStack;
  for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) {
    HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal(m_manager);
    sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize(),
                                        staticData.GetMinHypoStackDiversity());
    sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());

    m_hypoStackColl[ind] = sourceHypoColl;
  }
}

SearchNormal::~SearchNormal()
{
  RemoveAllInColl(m_hypoStackColl);
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void SearchNormal::ProcessSentence()
{
  const StaticData &staticData = StaticData::Instance();
  SentenceStats &stats = m_manager.GetSentenceStats();

  // initial seed hypothesis: nothing translated, no words produced
  Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt);
  m_hypoStackColl[0]->AddPrune(hypo);

  // go through each stack
  std::vector < HypothesisStack* >::iterator iterStack;
  for (iterStack = m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack) {
    // check if decoding ran out of time
    double _elapsed_time = GetUserTime();
    if (_elapsed_time > staticData.GetTimeoutThreshold()) {
      VERBOSE(1,"Decoding is out of time (" << _elapsed_time << "," << staticData.GetTimeoutThreshold() << ")" << std::endl);
      interrupted_flag = 1;
      return;
    }
    HypothesisStackNormal &sourceHypoColl = *static_cast<HypothesisStackNormal*>(*iterStack);

    // the stack is pruned before processing (lazy pruning):
    VERBOSE(3,"processing hypothesis from next stack");
    IFVERBOSE(2) {
      stats.StartTimeStack();
    }
    sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
    VERBOSE(3,std::endl);
    sourceHypoColl.CleanupArcList();
    IFVERBOSE(2) {
      stats.StopTimeStack();
    }

    // go through each hypothesis on the stack and try to expand it
    HypothesisStackNormal::const_iterator iterHypo;
    for (iterHypo = sourceHypoColl.begin() ; iterHypo != sourceHypoColl.end() ; ++iterHypo) {
      Hypothesis &hypothesis = **iterHypo;
      ProcessOneHypothesis(hypothesis); // expand the hypothesis
    }
    // some logging
    IFVERBOSE(2) {
      OutputHypoStackSize();
    }

    // this stack is fully expanded;
    actual_hypoStack = &sourceHypoColl;
  }
}

/** Find all translation options to expand one hypothesis, trigger expansion
 * this is mostly a check for overlap with already covered words, and for
 * violation of reordering limits.
 * \param hypothesis hypothesis to be expanded upon
 */
void SearchNormal::ProcessOneHypothesis(const Hypothesis &hypothesis)
{
  // since we check for reordering limits, its good to have that limit handy
  int maxDistortion = StaticData::Instance().GetMaxDistortion();
  bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;

  // no limit of reordering: only check for overlap
  if (maxDistortion < 0) {
    const WordsBitmap hypoBitmap	= hypothesis.GetWordsBitmap();
    const size_t hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
                                    , sourceSize			= m_source.GetSize();

    for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
      size_t maxSize = sourceSize - startPos;
      size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
      maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;

      for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
        // basic checks
        // there have to be translation options
        if (m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos)).size() == 0 ||
            // no overlap with existing words
            hypoBitmap.Overlap(WordsRange(startPos, endPos)) ||
            // specified reordering constraints (set with -monotone-at-punctuation or xml)
            !m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) ) {
          continue;
        }

        //TODO: does this method include incompatible WordLattice hypotheses?
        ExpandAllHypotheses(hypothesis, startPos, endPos);
      }
    }

    return; // done with special case (no reordering limit)
  }

  // if there are reordering limits, make sure it is not violated
  // the coverage bitmap is handy here (and the position of the first gap)
  const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
  const size_t	hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
                                  , sourceSize			= m_source.GetSize();

  // MAIN LOOP. go through each possible range
  for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
    // don't bother expanding phrases if the first position is already taken
    if(hypoBitmap.GetValue(startPos))
      continue;

    WordsRange prevRange = hypothesis.GetCurrSourceWordsRange();

    size_t maxSize = sourceSize - startPos;
    size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
    maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
    size_t closestLeft = hypoBitmap.GetEdgeToTheLeftOf(startPos);
    if (isWordLattice) {
      // first question: is there a path from the closest translated word to the left
      // of the hypothesized extension to the start of the hypothesized extension?
      // long version: is there anything to our left? is it farther left than where we're starting anyway? can we get to it?
      // closestLeft is exclusive: a value of 3 means 2 is covered, our arc is currently ENDING at 3 and can start at 3 implicitly
      if (closestLeft != 0 && closestLeft != startPos && !m_source.CanIGetFromAToB(closestLeft, startPos)) {
        continue;
      }
      if (prevRange.GetStartPos() != NOT_FOUND &&
          prevRange.GetStartPos() > startPos && !m_source.CanIGetFromAToB(startPos, prevRange.GetStartPos())) {
        continue;
      }
    }

    WordsRange currentStartRange(startPos, startPos);
    if(m_source.ComputeDistortionDistance(prevRange, currentStartRange) > maxDistortion)
      continue;

    for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
      // basic checks
      WordsRange extRange(startPos, endPos);
      // there have to be translation options
      if (m_transOptColl.GetTranslationOptionList(extRange).size() == 0 ||
          // no overlap with existing words
          hypoBitmap.Overlap(extRange) ||
          // specified reordering constraints (set with -monotone-at-punctuation or xml)
          !m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) || //
          // connection in input word lattice
          (isWordLattice && !m_source.IsCoveragePossible(extRange))) {
        continue;
      }

      // ask second question here:
      // we already know we can get to our starting point from the closest thing to the left. We now ask the follow up:
      // can we get from our end to the closest thing on the right?
      // long version: is anything to our right? is it farther right than our (inclusive) end? can our end reach it?
      bool leftMostEdge = (hypoFirstGapPos == startPos);

      // closest right definition:
      size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(endPos);
      if (isWordLattice) {
        //if (!leftMostEdge && closestRight != endPos && closestRight != sourceSize && !m_source.CanIGetFromAToB(endPos, closestRight + 1)) {
        if (closestRight != endPos && ((closestRight + 1) < sourceSize) && !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
          continue;
        }
      }

      // any length extension is okay if starting at left-most edge
      if (leftMostEdge) {
        ExpandAllHypotheses(hypothesis, startPos, endPos);
      }
      // starting somewhere other than left-most edge, use caution
      else {
        // the basic idea is this: we would like to translate a phrase starting
        // from a position further right than the left-most open gap. The
        // distortion penalty for the following phrase will be computed relative
        // to the ending position of the current extension, so we ask now what
        // its maximum value will be (which will always be the value of the
        // hypothesis starting at the left-most edge).  If this value is less than
        // the distortion limit, we don't allow this extension to be made.
        WordsRange bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
        int required_distortion =
          m_source.ComputeDistortionDistance(extRange, bestNextExtension);

        if (required_distortion > maxDistortion) {
          continue;
        }

        // everything is fine, we're good to go
        ExpandAllHypotheses(hypothesis, startPos, endPos);

      }
    }
  }
}

/**
 * Expand a hypothesis given a list of translation options
 * \param hypothesis hypothesis to be expanded upon
 * \param startPos first word position of span covered
 * \param endPos last word position of span covered
 */

void SearchNormal::ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos)
{
  // early discarding: check if hypothesis is too bad to build
  // this idea is explained in (Moore&Quirk, MT Summit 2007)
  float expectedScore = 0.0f;
  if (StaticData::Instance().UseEarlyDiscarding()) {
    // expected score is based on score of current hypothesis
    expectedScore = hypothesis.GetScore();

    // add new future score estimate
    expectedScore += m_transOptColl.GetFutureScore().CalcFutureScore( hypothesis.GetWordsBitmap(), startPos, endPos );
  }

  // loop through all translation options
  const TranslationOptionList &transOptList = m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
  TranslationOptionList::const_iterator iter;
  for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter) {
    ExpandHypothesis(hypothesis, **iter, expectedScore);
  }
}

/**
 * Expand one hypothesis with a translation option.
 * this involves initial creation, scoring and adding it to the proper stack
 * \param hypothesis hypothesis to be expanded upon
 * \param transOpt translation option (phrase translation)
 *        that is applied to create the new hypothesis
 * \param expectedScore base score for early discarding
 *        (base hypothesis score plus future score estimation)
 */
void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt, float expectedScore)
{
  const StaticData &staticData = StaticData::Instance();
  SentenceStats &stats = m_manager.GetSentenceStats();

  Hypothesis *newHypo;
  if (! staticData.UseEarlyDiscarding()) {
    // simple build, no questions asked
    IFVERBOSE(2) {
      stats.StartTimeBuildHyp();
    }
    newHypo = hypothesis.CreateNext(transOpt);
    IFVERBOSE(2) {
      stats.StopTimeBuildHyp();
    }
    if (newHypo==NULL) return;
    newHypo->Evaluate(m_transOptColl.GetFutureScore());
  } else
    // early discarding: check if hypothesis is too bad to build
  {
    // worst possible score may have changed -> recompute
    size_t wordsTranslated = hypothesis.GetWordsBitmap().GetNumWordsCovered() + transOpt.GetSize();
    float allowedScore = m_hypoStackColl[wordsTranslated]->GetWorstScore();
    if (staticData.GetMinHypoStackDiversity()) {
      WordsBitmapID id = hypothesis.GetWordsBitmap().GetIDPlus(transOpt.GetStartPos(), transOpt.GetEndPos());
      float allowedScoreForBitmap = m_hypoStackColl[wordsTranslated]->GetWorstScoreForBitmap( id );
      allowedScore = std::min( allowedScore, allowedScoreForBitmap );
    }
    allowedScore += staticData.GetEarlyDiscardingThreshold();

    // add expected score of translation option
    expectedScore += transOpt.GetFutureScore();

    // check if transOpt score push it already below limit
    if (expectedScore < allowedScore) {
      IFVERBOSE(2) {
        stats.AddNotBuilt();
      }
      return;
    }

    // build the hypothesis without scoring
    IFVERBOSE(2) {
      stats.StartTimeBuildHyp();
    }
    newHypo = hypothesis.CreateNext(transOpt);
    if (newHypo==NULL) return;
    IFVERBOSE(2) {
      stats.StopTimeBuildHyp();
    }

    // ... and check if that is below the limit
    if (expectedScore < allowedScore) {
      IFVERBOSE(2) {
        stats.AddEarlyDiscarded();
      }
      FREEHYPO( newHypo );
      return;
    }

  }

  // logging for the curious
  IFVERBOSE(3) {
    newHypo->PrintHypothesis();
  }

  // add to hypothesis stack
  size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
  IFVERBOSE(2) {
    stats.StartTimeStack();
  }
  m_hypoStackColl[wordsTranslated]->AddPrune(newHypo);
  IFVERBOSE(2) {
    stats.StopTimeStack();
  }
}

const std::vector < HypothesisStack* >& SearchNormal::GetHypothesisStacks() const
{
  return m_hypoStackColl;
}

/**
 * Find best hypothesis on the last stack.
 * This is the end point of the best translation, which can be traced back from here
 */
const Hypothesis *SearchNormal::GetBestHypothesis() const
{
  if (interrupted_flag == 0) {
    const HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(m_hypoStackColl.back());
    return hypoColl.GetBestHypothesis();
  } else {
    const HypothesisStackNormal &hypoColl = *actual_hypoStack;
    return hypoColl.GetBestHypothesis();
  }
}

/**
 * Logging of hypothesis stack sizes
 */
void SearchNormal::OutputHypoStackSize()
{
  std::vector < HypothesisStack* >::const_iterator iterStack = m_hypoStackColl.begin();
  TRACE_ERR( "Stack sizes: " << (int)(*iterStack)->size());
  for (++iterStack; iterStack != m_hypoStackColl.end() ; ++iterStack) {
    TRACE_ERR( ", " << (int)(*iterStack)->size());
  }
  TRACE_ERR( endl);
}

}
<commit_msg>debugging<commit_after>#include "Manager.h"
#include "Timer.h"
#include "SearchNormal.h"
#include "SentenceStats.h"

using namespace std;

namespace Moses
{
/**
 * Organizing main function
 *
 * /param source input sentence
 * /param transOptColl collection of translation options to be used for this sentence
 */
SearchNormal::SearchNormal(Manager& manager, const InputType &source, const TranslationOptionCollection &transOptColl)
  :Search(manager)
  ,m_source(source)
  ,m_hypoStackColl(source.GetSize() + 1)
  ,interrupted_flag(0)
  ,m_transOptColl(transOptColl)
{
  VERBOSE(1, "Translating: " << m_source << endl);
  const StaticData &staticData = StaticData::Instance();

  // only if constraint decoding (having to match a specified output)
  // long sentenceID = source.GetTranslationId();

  // initialize the stacks: create data structure and set limits
  std::vector < HypothesisStackNormal >::iterator iterStack;
  for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) {
    HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal(m_manager);
    sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize(),
                                        staticData.GetMinHypoStackDiversity());
    sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());

    m_hypoStackColl[ind] = sourceHypoColl;
  }
}

SearchNormal::~SearchNormal()
{
  RemoveAllInColl(m_hypoStackColl);
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void SearchNormal::ProcessSentence()
{
  const StaticData &staticData = StaticData::Instance();
  SentenceStats &stats = m_manager.GetSentenceStats();

  // initial seed hypothesis: nothing translated, no words produced
  Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt);
  m_hypoStackColl[0]->AddPrune(hypo);

  // go through each stack
  size_t stackNo = 0;
  std::vector < HypothesisStack* >::iterator iterStack;
  for (iterStack = m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack) {
    // check if decoding ran out of time
    double _elapsed_time = GetUserTime();
    if (_elapsed_time > staticData.GetTimeoutThreshold()) {
      VERBOSE(1,"Decoding is out of time (" << _elapsed_time << "," << staticData.GetTimeoutThreshold() << ")" << std::endl);
      interrupted_flag = 1;
      return;
    }
    HypothesisStackNormal &sourceHypoColl = *static_cast<HypothesisStackNormal*>(*iterStack);

    // the stack is pruned before processing (lazy pruning):
    VERBOSE(3,"processing hypothesis from next stack");
    IFVERBOSE(2) {
      stats.StartTimeStack();
    }
    sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
    VERBOSE(3,std::endl);
    sourceHypoColl.CleanupArcList();
    IFVERBOSE(2) {
      stats.StopTimeStack();
    }

    // go through each hypothesis on the stack and try to expand it
    HypothesisStackNormal::const_iterator iterHypo;
    for (iterHypo = sourceHypoColl.begin() ; iterHypo != sourceHypoColl.end() ; ++iterHypo) {
      Hypothesis &hypothesis = **iterHypo;
      ProcessOneHypothesis(hypothesis); // expand the hypothesis
    }
    // some logging
    IFVERBOSE(2) {
      OutputHypoStackSize();
    }
    OutputHypoStack(stackNo);

    // this stack is fully expanded;
    actual_hypoStack = &sourceHypoColl;

    ++stackNo;
  }
}

/** Find all translation options to expand one hypothesis, trigger expansion
 * this is mostly a check for overlap with already covered words, and for
 * violation of reordering limits.
 * \param hypothesis hypothesis to be expanded upon
 */
void SearchNormal::ProcessOneHypothesis(const Hypothesis &hypothesis)
{
  // since we check for reordering limits, its good to have that limit handy
  int maxDistortion = StaticData::Instance().GetMaxDistortion();
  bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;

  // no limit of reordering: only check for overlap
  if (maxDistortion < 0) {
    const WordsBitmap hypoBitmap	= hypothesis.GetWordsBitmap();
    const size_t hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
                                    , sourceSize			= m_source.GetSize();

    for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
      size_t maxSize = sourceSize - startPos;
      size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
      maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;

      for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
        // basic checks
        // there have to be translation options
        if (m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos)).size() == 0 ||
            // no overlap with existing words
            hypoBitmap.Overlap(WordsRange(startPos, endPos)) ||
            // specified reordering constraints (set with -monotone-at-punctuation or xml)
            !m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) ) {
          continue;
        }

        //TODO: does this method include incompatible WordLattice hypotheses?
        ExpandAllHypotheses(hypothesis, startPos, endPos);
      }
    }

    return; // done with special case (no reordering limit)
  }

  // if there are reordering limits, make sure it is not violated
  // the coverage bitmap is handy here (and the position of the first gap)
  const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
  const size_t	hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
                                  , sourceSize			= m_source.GetSize();

  // MAIN LOOP. go through each possible range
  for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) {
    // don't bother expanding phrases if the first position is already taken
    if(hypoBitmap.GetValue(startPos))
      continue;

    WordsRange prevRange = hypothesis.GetCurrSourceWordsRange();

    size_t maxSize = sourceSize - startPos;
    size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
    maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
    size_t closestLeft = hypoBitmap.GetEdgeToTheLeftOf(startPos);
    if (isWordLattice) {
      // first question: is there a path from the closest translated word to the left
      // of the hypothesized extension to the start of the hypothesized extension?
      // long version: is there anything to our left? is it farther left than where we're starting anyway? can we get to it?
      // closestLeft is exclusive: a value of 3 means 2 is covered, our arc is currently ENDING at 3 and can start at 3 implicitly
      if (closestLeft != 0 && closestLeft != startPos && !m_source.CanIGetFromAToB(closestLeft, startPos)) {
        continue;
      }
      if (prevRange.GetStartPos() != NOT_FOUND &&
          prevRange.GetStartPos() > startPos && !m_source.CanIGetFromAToB(startPos, prevRange.GetStartPos())) {
        continue;
      }
    }

    WordsRange currentStartRange(startPos, startPos);
    if(m_source.ComputeDistortionDistance(prevRange, currentStartRange) > maxDistortion)
      continue;

    for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
      // basic checks
      WordsRange extRange(startPos, endPos);
      // there have to be translation options
      if (m_transOptColl.GetTranslationOptionList(extRange).size() == 0 ||
          // no overlap with existing words
          hypoBitmap.Overlap(extRange) ||
          // specified reordering constraints (set with -monotone-at-punctuation or xml)
          !m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) || //
          // connection in input word lattice
          (isWordLattice && !m_source.IsCoveragePossible(extRange))) {
        continue;
      }

      // ask second question here:
      // we already know we can get to our starting point from the closest thing to the left. We now ask the follow up:
      // can we get from our end to the closest thing on the right?
      // long version: is anything to our right? is it farther right than our (inclusive) end? can our end reach it?
      bool leftMostEdge = (hypoFirstGapPos == startPos);

      // closest right definition:
      size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(endPos);
      if (isWordLattice) {
        //if (!leftMostEdge && closestRight != endPos && closestRight != sourceSize && !m_source.CanIGetFromAToB(endPos, closestRight + 1)) {
        if (closestRight != endPos && ((closestRight + 1) < sourceSize) && !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
          continue;
        }
      }

      // any length extension is okay if starting at left-most edge
      if (leftMostEdge) {
        ExpandAllHypotheses(hypothesis, startPos, endPos);
      }
      // starting somewhere other than left-most edge, use caution
      else {
        // the basic idea is this: we would like to translate a phrase starting
        // from a position further right than the left-most open gap. The
        // distortion penalty for the following phrase will be computed relative
        // to the ending position of the current extension, so we ask now what
        // its maximum value will be (which will always be the value of the
        // hypothesis starting at the left-most edge).  If this value is less than
        // the distortion limit, we don't allow this extension to be made.
        WordsRange bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
        int required_distortion =
          m_source.ComputeDistortionDistance(extRange, bestNextExtension);

        if (required_distortion > maxDistortion) {
          continue;
        }

        // everything is fine, we're good to go
        ExpandAllHypotheses(hypothesis, startPos, endPos);

      }
    }
  }
}

/**
 * Expand a hypothesis given a list of translation options
 * \param hypothesis hypothesis to be expanded upon
 * \param startPos first word position of span covered
 * \param endPos last word position of span covered
 */

void SearchNormal::ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos)
{
  // early discarding: check if hypothesis is too bad to build
  // this idea is explained in (Moore&Quirk, MT Summit 2007)
  float expectedScore = 0.0f;
  if (StaticData::Instance().UseEarlyDiscarding()) {
    // expected score is based on score of current hypothesis
    expectedScore = hypothesis.GetScore();

    // add new future score estimate
    expectedScore += m_transOptColl.GetFutureScore().CalcFutureScore( hypothesis.GetWordsBitmap(), startPos, endPos );
  }

  // loop through all translation options
  const TranslationOptionList &transOptList = m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
  TranslationOptionList::const_iterator iter;
  for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter) {
    ExpandHypothesis(hypothesis, **iter, expectedScore);
  }
}

/**
 * Expand one hypothesis with a translation option.
 * this involves initial creation, scoring and adding it to the proper stack
 * \param hypothesis hypothesis to be expanded upon
 * \param transOpt translation option (phrase translation)
 *        that is applied to create the new hypothesis
 * \param expectedScore base score for early discarding
 *        (base hypothesis score plus future score estimation)
 */
void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt, float expectedScore)
{
  const StaticData &staticData = StaticData::Instance();
  SentenceStats &stats = m_manager.GetSentenceStats();

  Hypothesis *newHypo;
  if (! staticData.UseEarlyDiscarding()) {
    // simple build, no questions asked
    IFVERBOSE(2) {
      stats.StartTimeBuildHyp();
    }
    newHypo = hypothesis.CreateNext(transOpt);
    IFVERBOSE(2) {
      stats.StopTimeBuildHyp();
    }
    if (newHypo==NULL) return;
    newHypo->Evaluate(m_transOptColl.GetFutureScore());
  } else
    // early discarding: check if hypothesis is too bad to build
  {
    // worst possible score may have changed -> recompute
    size_t wordsTranslated = hypothesis.GetWordsBitmap().GetNumWordsCovered() + transOpt.GetSize();
    float allowedScore = m_hypoStackColl[wordsTranslated]->GetWorstScore();
    if (staticData.GetMinHypoStackDiversity()) {
      WordsBitmapID id = hypothesis.GetWordsBitmap().GetIDPlus(transOpt.GetStartPos(), transOpt.GetEndPos());
      float allowedScoreForBitmap = m_hypoStackColl[wordsTranslated]->GetWorstScoreForBitmap( id );
      allowedScore = std::min( allowedScore, allowedScoreForBitmap );
    }
    allowedScore += staticData.GetEarlyDiscardingThreshold();

    // add expected score of translation option
    expectedScore += transOpt.GetFutureScore();

    // check if transOpt score push it already below limit
    if (expectedScore < allowedScore) {
      IFVERBOSE(2) {
        stats.AddNotBuilt();
      }
      return;
    }

    // build the hypothesis without scoring
    IFVERBOSE(2) {
      stats.StartTimeBuildHyp();
    }
    newHypo = hypothesis.CreateNext(transOpt);
    if (newHypo==NULL) return;
    IFVERBOSE(2) {
      stats.StopTimeBuildHyp();
    }

    // ... and check if that is below the limit
    if (expectedScore < allowedScore) {
      IFVERBOSE(2) {
        stats.AddEarlyDiscarded();
      }
      FREEHYPO( newHypo );
      return;
    }

  }

  // logging for the curious
  IFVERBOSE(3) {
    newHypo->PrintHypothesis();
  }

  // add to hypothesis stack
  size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
  IFVERBOSE(2) {
    stats.StartTimeStack();
  }
  m_hypoStackColl[wordsTranslated]->AddPrune(newHypo);
  IFVERBOSE(2) {
    stats.StopTimeStack();
  }
}

const std::vector < HypothesisStack* >& SearchNormal::GetHypothesisStacks() const
{
  return m_hypoStackColl;
}

/**
 * Find best hypothesis on the last stack.
 * This is the end point of the best translation, which can be traced back from here
 */
const Hypothesis *SearchNormal::GetBestHypothesis() const
{
  if (interrupted_flag == 0) {
    const HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(m_hypoStackColl.back());
    return hypoColl.GetBestHypothesis();
  } else {
    const HypothesisStackNormal &hypoColl = *actual_hypoStack;
    return hypoColl.GetBestHypothesis();
  }
}

/**
 * Logging of hypothesis stack sizes
 */
void SearchNormal::OutputHypoStackSize()
{
  std::vector < HypothesisStack* >::const_iterator iterStack = m_hypoStackColl.begin();
  TRACE_ERR( "Stack sizes: " << (int)(*iterStack)->size());
  for (++iterStack; iterStack != m_hypoStackColl.end() ; ++iterStack) {
    TRACE_ERR( ", " << (int)(*iterStack)->size());
  }
  TRACE_ERR( endl);
}

void SearchNormal::OutputHypoStack(int stack)
{
  if (stack >= 0) {
    TRACE_ERR( "Stack " << stack << ": " << endl << m_hypoStackColl[stack] << endl);
  } else {
    // all stacks
    int i = 0;
    vector < HypothesisStack* >::iterator iterStack;
    for (iterStack = m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack) {
      HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(*iterStack);
      TRACE_ERR( "Stack " << i++ << ": " << endl << hypoColl << endl);
    }
  }
}

}
<|endoftext|>


<commit_before>/**
 * @file logging.cpp
 * @brief Logging class
 *
 * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
 *
 * This file is part of the MEGA SDK - Client Access Engine.
 *
 * Applications using the MEGA API must present a valid application key
 * and comply with the the rules set forth in the Terms of Service.
 *
 * The MEGA SDK is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * @copyright Simplified (2-clause) BSD License.
 *
 * You should have received a copy of the license along with this
 * program.
 */

#include "mega/logging.h"

namespace mega {

// static member initialization
OutputMap SimpleLogger::outputs;
Logger *SimpleLogger::logger = NULL;
char SimpleLogger::base64Handle[14];

// by the default, display logs with level equal or less than logInfo
enum LogLevel SimpleLogger::logCurrentLevel = logInfo;

SimpleLogger::SimpleLogger(enum LogLevel ll, char const* filename, int line)
{
    ostr << "[" << getTime() << "] ";
    ostr << "[" << toStr(ll) << "] ";
    ostr << filename << ":" << line << " ";

    if (logger)
    {
        t = getTime();
        std::ostringstream oss;
        oss << filename;
        if(line >= 0)
        {
            oss << ":" << line;
        }
        fname = oss.str();
    }
}

SimpleLogger::~SimpleLogger()
{
    OutputStreams::iterator iter;
    OutputStreams vec;

    if (logger)
        logger->log(t.c_str(), level, fname.c_str(), ostr.str().c_str());

    if (lineBreak)
        ostr << std::endl;

    vec = getOutput(level);

    for (iter = vec.begin(); iter != vec.end(); iter++)
    {
        **iter << ostr.str();
    }
}

std::string SimpleLogger::getTime()
{
    char ts[50];
    time_t t = time(NULL);

    if (!strftime(ts, sizeof(ts), "%H:%M:%S", gmtime(&t))) {
        ts[0] = '\0';
    }
    return ts;
}

void SimpleLogger::flush()
{
    for (int i = logFatal; i < logMax; i++)
    {
        OutputStreams::iterator iter;
        OutputStreams vec;

        vec = outputs[static_cast<LogLevel>(i)];;

        for (iter = vec.begin(); iter != vec.end(); iter++)
        {
            std::ostream *os = *iter;
            os->flush();
        }
    }
}

} // namespace
<commit_msg>Restore initialization of log level<commit_after>/**
 * @file logging.cpp
 * @brief Logging class
 *
 * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
 *
 * This file is part of the MEGA SDK - Client Access Engine.
 *
 * Applications using the MEGA API must present a valid application key
 * and comply with the the rules set forth in the Terms of Service.
 *
 * The MEGA SDK is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * @copyright Simplified (2-clause) BSD License.
 *
 * You should have received a copy of the license along with this
 * program.
 */

#include "mega/logging.h"

namespace mega {

// static member initialization
OutputMap SimpleLogger::outputs;
Logger *SimpleLogger::logger = NULL;
char SimpleLogger::base64Handle[14];

// by the default, display logs with level equal or less than logInfo
enum LogLevel SimpleLogger::logCurrentLevel = logInfo;

SimpleLogger::SimpleLogger(enum LogLevel ll, char const* filename, int line)
{
    this->level = ll;

    ostr << "[" << getTime() << "] ";
    ostr << "[" << toStr(ll) << "] ";
    ostr << filename << ":" << line << " ";

    if (logger)
    {
        t = getTime();
        std::ostringstream oss;
        oss << filename;
        if(line >= 0)
        {
            oss << ":" << line;
        }
        fname = oss.str();
    }
}

SimpleLogger::~SimpleLogger()
{
    OutputStreams::iterator iter;
    OutputStreams vec;

    if (logger)
        logger->log(t.c_str(), level, fname.c_str(), ostr.str().c_str());

    if (lineBreak)
        ostr << std::endl;

    vec = getOutput(level);

    for (iter = vec.begin(); iter != vec.end(); iter++)
    {
        **iter << ostr.str();
    }
}

std::string SimpleLogger::getTime()
{
    char ts[50];
    time_t t = time(NULL);

    if (!strftime(ts, sizeof(ts), "%H:%M:%S", gmtime(&t))) {
        ts[0] = '\0';
    }
    return ts;
}

void SimpleLogger::flush()
{
    for (int i = logFatal; i < logMax; i++)
    {
        OutputStreams::iterator iter;
        OutputStreams vec;

        vec = outputs[static_cast<LogLevel>(i)];;

        for (iter = vec.begin(); iter != vec.end(); iter++)
        {
            std::ostream *os = *iter;
            os->flush();
        }
    }
}

} // namespace
<|endoftext|>


<commit_before>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <algorithm>
#include <stdlib.h>
#include "util/check.hh"
#include "util/exception.hh"
#include "util/tokenize_piece.hh"

#include "TargetPhrase.h"
#include "GenerationDictionary.h"
#include "LM/Base.h"
#include "StaticData.h"
#include "ScoreComponentCollection.h"
#include "Util.h"
#include "AlignmentInfoCollection.h"
#include "InputPath.h"

using namespace std;

namespace Moses
{
TargetPhrase::TargetPhrase( std::string out_string)
  :Phrase(0)
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{

  //ACAT
  const StaticData &staticData = StaticData::Instance();
  CreateFromString(Output, staticData.GetInputFactorOrder(), out_string, staticData.GetFactorDelimiter(), NULL);
}

TargetPhrase::TargetPhrase()
  :Phrase()
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{
}

TargetPhrase::TargetPhrase(const Phrase &phrase)
  : Phrase(phrase)
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{
}

TargetPhrase::TargetPhrase(const TargetPhrase &copy)
  : Phrase(copy)
  , m_fullScore(copy.m_fullScore)
  , m_futureScore(copy.m_futureScore)
  , m_scoreBreakdown(copy.m_scoreBreakdown)
  , m_alignTerm(copy.m_alignTerm)
  , m_alignNonTerm(copy.m_alignNonTerm)
{
  if (copy.m_lhsTarget) {
    m_lhsTarget = new Word(*copy.m_lhsTarget);
  } else {
    m_lhsTarget = NULL;
  }

  if (copy.m_ruleSource) {
	  m_lhsTarget = new Word(*copy.m_ruleSource);
  }
  else {
	  m_ruleSource = NULL;
  }
}

TargetPhrase::~TargetPhrase()
{
  //cerr << "m_lhsTarget=" << m_lhsTarget << endl;

  delete m_lhsTarget;
  delete m_ruleSource;
}

#ifdef HAVE_PROTOBUF
void TargetPhrase::WriteToRulePB(hgmert::Rule* pb) const
{
  pb->add_trg_words("[X,1]");
  for (size_t pos = 0 ; pos < GetSize() ; pos++)
    pb->add_trg_words(GetWord(pos)[0]->GetString());
}
#endif

void TargetPhrase::Evaluate(const Phrase &source)
{
  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
  Evaluate(source, ffs);
}

void TargetPhrase::Evaluate(const Phrase &source, const std::vector<FeatureFunction*> &ffs)
{
  if (ffs.size()) {
    const StaticData &staticData = StaticData::Instance();
    ScoreComponentCollection futureScoreBreakdown;
    for (size_t i = 0; i < ffs.size(); ++i) {
      const FeatureFunction &ff = *ffs[i];
      if (! staticData.IsFeatureFunctionIgnored( ff )) {
        ff.Evaluate(source, *this, m_scoreBreakdown, futureScoreBreakdown);
      }
    }

    float weightedScore = m_scoreBreakdown.GetWeightedScore();
    m_futureScore += futureScoreBreakdown.GetWeightedScore();
    m_fullScore = weightedScore + m_futureScore;

  }
}

void TargetPhrase::Evaluate(const InputType &input, const InputPath &inputPath)
{
  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();

  for (size_t i = 0; i < ffs.size(); ++i) {
    const FeatureFunction &ff = *ffs[i];
    ff.Evaluate(input, inputPath, m_scoreBreakdown);
  }
}

void TargetPhrase::SetXMLScore(float score)
{
  const StaticData &staticData = StaticData::Instance();
  const FeatureFunction* prod = staticData.GetPhraseDictionaries()[0];
  size_t numScores = prod->GetNumScoreComponents();
  vector <float> scoreVector(numScores,score/numScores);

  m_scoreBreakdown.Assign(prod, scoreVector);
}

void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString)
{
  AlignmentInfo::CollType alignTerm, alignNonTerm;
  for (util::TokenIter<util::AnyCharacter, true> token(alignString, util::AnyCharacter(" \t")); token; ++token) {
    util::TokenIter<util::SingleCharacter, false> dash(*token, util::SingleCharacter('-'));

    char *endptr;
    size_t sourcePos = strtoul(dash->data(), &endptr, 10);
    UTIL_THROW_IF(endptr != dash->data() + dash->size(), util::ErrnoException, "Error parsing alignment" << *dash);
    ++dash;
    size_t targetPos = strtoul(dash->data(), &endptr, 10);
    UTIL_THROW_IF(endptr != dash->data() + dash->size(), util::ErrnoException, "Error parsing alignment" << *dash);
    UTIL_THROW_IF(++dash, util::Exception, "Extra gunk in alignment " << *token);

    if (GetWord(targetPos).IsNonTerminal()) {
      alignNonTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
    } else {
      alignTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
    }
  }
  SetAlignTerm(alignTerm);
  SetAlignNonTerm(alignNonTerm);

}

void TargetPhrase::SetAlignTerm(const AlignmentInfo::CollType &coll)
{
  const AlignmentInfo *alignmentInfo = AlignmentInfoCollection::Instance().Add(coll);
  m_alignTerm = alignmentInfo;

}

void TargetPhrase::SetAlignNonTerm(const AlignmentInfo::CollType &coll)
{
  const AlignmentInfo *alignmentInfo = AlignmentInfoCollection::Instance().Add(coll);
  m_alignNonTerm = alignmentInfo;
}

void TargetPhrase::SetSparseScore(const FeatureFunction* translationScoreProducer, const StringPiece &sparseString)
{
  m_scoreBreakdown.Assign(translationScoreProducer, sparseString.as_string());
}

void TargetPhrase::Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec)
{
  Phrase::MergeFactors(copy, factorVec);
  m_scoreBreakdown.Merge(copy.GetScoreBreakdown());
  m_futureScore += copy.m_futureScore;
  m_fullScore += copy.m_fullScore;
}

void TargetPhrase::SetProperties(const StringPiece &str)
{
	if (str.size() == 0) {
		return;
	}

  vector<string> toks;
  TokenizeMultiCharSeparator(toks, str.as_string(), "{{");
  for (size_t i = 0; i < toks.size(); ++i) {
	  string &tok = toks[i];
	  if (tok.empty()) {
		  continue;
	  }
	  size_t endPos = tok.rfind("}");

	  tok = tok.substr(0, endPos - 1);

	  vector<string> keyValue = TokenizeFirstOnly(tok, " ");
	  CHECK(keyValue.size() == 2);
	  SetProperty(keyValue[0], keyValue[1]);
  }
}

void TargetPhrase::GetProperty(const std::string &key, std::string &value, bool &found) const
{
  std::map<std::string, std::string>::const_iterator iter;
  iter = m_properties.find(key);
  if (iter == m_properties.end()) {
    found = false;
  } else {
    found = true;
    value = iter->second;
  }
}

void TargetPhrase::SetRuleSource(const Phrase &ruleSource) const
{
  if (m_ruleSource == NULL) {
	  m_ruleSource = new Phrase(ruleSource);
  }
}

void swap(TargetPhrase &first, TargetPhrase &second)
{
  first.SwapWords(second);
  std::swap(first.m_fullScore, second.m_fullScore);
  std::swap(first.m_futureScore, second.m_futureScore);
  swap(first.m_scoreBreakdown, second.m_scoreBreakdown);
  std::swap(first.m_alignTerm, second.m_alignTerm);
  std::swap(first.m_alignNonTerm, second.m_alignNonTerm);
  std::swap(first.m_lhsTarget, second.m_lhsTarget);
}

TO_STRING_BODY(TargetPhrase);

std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp)
{
  if (tp.m_lhsTarget) {
    os << *tp.m_lhsTarget<< " -> ";
  }
  os << static_cast<const Phrase&>(tp) << ":" << flush;
  os << tp.GetAlignNonTerm() << flush;
  os << ": c=" << tp.m_fullScore << flush;
  os << " " << tp.m_scoreBreakdown << flush;

  return os;
}

}

<commit_msg>add option to store rule source in target phrase<commit_after>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <algorithm>
#include <stdlib.h>
#include "util/check.hh"
#include "util/exception.hh"
#include "util/tokenize_piece.hh"

#include "TargetPhrase.h"
#include "GenerationDictionary.h"
#include "LM/Base.h"
#include "StaticData.h"
#include "ScoreComponentCollection.h"
#include "Util.h"
#include "AlignmentInfoCollection.h"
#include "InputPath.h"

using namespace std;

namespace Moses
{
TargetPhrase::TargetPhrase( std::string out_string)
  :Phrase(0)
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{

  //ACAT
  const StaticData &staticData = StaticData::Instance();
  CreateFromString(Output, staticData.GetInputFactorOrder(), out_string, staticData.GetFactorDelimiter(), NULL);
}

TargetPhrase::TargetPhrase()
  :Phrase()
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{
}

TargetPhrase::TargetPhrase(const Phrase &phrase)
  : Phrase(phrase)
  , m_fullScore(0.0)
  , m_futureScore(0.0)
  , m_alignTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_alignNonTerm(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo())
  , m_lhsTarget(NULL)
  , m_ruleSource(NULL)
{
}

TargetPhrase::TargetPhrase(const TargetPhrase &copy)
  : Phrase(copy)
  , m_fullScore(copy.m_fullScore)
  , m_futureScore(copy.m_futureScore)
  , m_scoreBreakdown(copy.m_scoreBreakdown)
  , m_alignTerm(copy.m_alignTerm)
  , m_alignNonTerm(copy.m_alignNonTerm)
{
  if (copy.m_lhsTarget) {
    m_lhsTarget = new Word(*copy.m_lhsTarget);
  } else {
    m_lhsTarget = NULL;
  }

  if (copy.m_ruleSource) {
	  m_ruleSource = new Phrase(*copy.m_ruleSource);
  }
  else {
	  m_ruleSource = NULL;
  }
}

TargetPhrase::~TargetPhrase()
{
  //cerr << "m_lhsTarget=" << m_lhsTarget << endl;

  delete m_lhsTarget;
  delete m_ruleSource;
}

#ifdef HAVE_PROTOBUF
void TargetPhrase::WriteToRulePB(hgmert::Rule* pb) const
{
  pb->add_trg_words("[X,1]");
  for (size_t pos = 0 ; pos < GetSize() ; pos++)
    pb->add_trg_words(GetWord(pos)[0]->GetString());
}
#endif

void TargetPhrase::Evaluate(const Phrase &source)
{
  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
  Evaluate(source, ffs);
}

void TargetPhrase::Evaluate(const Phrase &source, const std::vector<FeatureFunction*> &ffs)
{
  if (ffs.size()) {
    const StaticData &staticData = StaticData::Instance();
    ScoreComponentCollection futureScoreBreakdown;
    for (size_t i = 0; i < ffs.size(); ++i) {
      const FeatureFunction &ff = *ffs[i];
      if (! staticData.IsFeatureFunctionIgnored( ff )) {
        ff.Evaluate(source, *this, m_scoreBreakdown, futureScoreBreakdown);
      }
    }

    float weightedScore = m_scoreBreakdown.GetWeightedScore();
    m_futureScore += futureScoreBreakdown.GetWeightedScore();
    m_fullScore = weightedScore + m_futureScore;

  }
}

void TargetPhrase::Evaluate(const InputType &input, const InputPath &inputPath)
{
  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();

  for (size_t i = 0; i < ffs.size(); ++i) {
    const FeatureFunction &ff = *ffs[i];
    ff.Evaluate(input, inputPath, m_scoreBreakdown);
  }
}

void TargetPhrase::SetXMLScore(float score)
{
  const StaticData &staticData = StaticData::Instance();
  const FeatureFunction* prod = staticData.GetPhraseDictionaries()[0];
  size_t numScores = prod->GetNumScoreComponents();
  vector <float> scoreVector(numScores,score/numScores);

  m_scoreBreakdown.Assign(prod, scoreVector);
}

void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString)
{
  AlignmentInfo::CollType alignTerm, alignNonTerm;
  for (util::TokenIter<util::AnyCharacter, true> token(alignString, util::AnyCharacter(" \t")); token; ++token) {
    util::TokenIter<util::SingleCharacter, false> dash(*token, util::SingleCharacter('-'));

    char *endptr;
    size_t sourcePos = strtoul(dash->data(), &endptr, 10);
    UTIL_THROW_IF(endptr != dash->data() + dash->size(), util::ErrnoException, "Error parsing alignment" << *dash);
    ++dash;
    size_t targetPos = strtoul(dash->data(), &endptr, 10);
    UTIL_THROW_IF(endptr != dash->data() + dash->size(), util::ErrnoException, "Error parsing alignment" << *dash);
    UTIL_THROW_IF(++dash, util::Exception, "Extra gunk in alignment " << *token);

    if (GetWord(targetPos).IsNonTerminal()) {
      alignNonTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
    } else {
      alignTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
    }
  }
  SetAlignTerm(alignTerm);
  SetAlignNonTerm(alignNonTerm);

}

void TargetPhrase::SetAlignTerm(const AlignmentInfo::CollType &coll)
{
  const AlignmentInfo *alignmentInfo = AlignmentInfoCollection::Instance().Add(coll);
  m_alignTerm = alignmentInfo;

}

void TargetPhrase::SetAlignNonTerm(const AlignmentInfo::CollType &coll)
{
  const AlignmentInfo *alignmentInfo = AlignmentInfoCollection::Instance().Add(coll);
  m_alignNonTerm = alignmentInfo;
}

void TargetPhrase::SetSparseScore(const FeatureFunction* translationScoreProducer, const StringPiece &sparseString)
{
  m_scoreBreakdown.Assign(translationScoreProducer, sparseString.as_string());
}

void TargetPhrase::Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec)
{
  Phrase::MergeFactors(copy, factorVec);
  m_scoreBreakdown.Merge(copy.GetScoreBreakdown());
  m_futureScore += copy.m_futureScore;
  m_fullScore += copy.m_fullScore;
}

void TargetPhrase::SetProperties(const StringPiece &str)
{
	if (str.size() == 0) {
		return;
	}

  vector<string> toks;
  TokenizeMultiCharSeparator(toks, str.as_string(), "{{");
  for (size_t i = 0; i < toks.size(); ++i) {
	  string &tok = toks[i];
	  if (tok.empty()) {
		  continue;
	  }
	  size_t endPos = tok.rfind("}");

	  tok = tok.substr(0, endPos - 1);

	  vector<string> keyValue = TokenizeFirstOnly(tok, " ");
	  CHECK(keyValue.size() == 2);
	  SetProperty(keyValue[0], keyValue[1]);
  }
}

void TargetPhrase::GetProperty(const std::string &key, std::string &value, bool &found) const
{
  std::map<std::string, std::string>::const_iterator iter;
  iter = m_properties.find(key);
  if (iter == m_properties.end()) {
    found = false;
  } else {
    found = true;
    value = iter->second;
  }
}

void TargetPhrase::SetRuleSource(const Phrase &ruleSource) const
{
  if (m_ruleSource == NULL) {
	  m_ruleSource = new Phrase(ruleSource);
  }
}

void swap(TargetPhrase &first, TargetPhrase &second)
{
  first.SwapWords(second);
  std::swap(first.m_fullScore, second.m_fullScore);
  std::swap(first.m_futureScore, second.m_futureScore);
  swap(first.m_scoreBreakdown, second.m_scoreBreakdown);
  std::swap(first.m_alignTerm, second.m_alignTerm);
  std::swap(first.m_alignNonTerm, second.m_alignNonTerm);
  std::swap(first.m_lhsTarget, second.m_lhsTarget);
}

TO_STRING_BODY(TargetPhrase);

std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp)
{
  if (tp.m_lhsTarget) {
    os << *tp.m_lhsTarget<< " -> ";
  }
  os << static_cast<const Phrase&>(tp) << ":" << flush;
  os << tp.GetAlignNonTerm() << flush;
  os << ": c=" << tp.m_fullScore << flush;
  os << " " << tp.m_scoreBreakdown << flush;

  return os;
}

}

<|endoftext|>


<commit_before>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <ctime>
#include <cmath>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <utils.hpp>
#include <Transpose.hpp>

typedef float dataType;
std::string typeName("float");

int main(int argc, char *argv[]) {
  bool printCode = false;
  bool printData = false;
	unsigned int clPlatformID = 0;
	unsigned int clDeviceID = 0;
  unsigned int nrThreads = 0;
  unsigned int padding = 0;
  unsigned int vector = 0;
  unsigned int M = 0;
  unsigned int N = 0;
	long long unsigned int wrongItems = 0;

	try {
    isa::utils::ArgumentList args(argc, argv);
    printCode = args.getSwitch("-print_code");
    printData = args.getSwitch("-print_data");
		clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
		clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    vector = args.getSwitchArgument< unsigned int >("-vector");
    nrThreads = args.getSwitchArgument< unsigned int >("-threads");
    M = args.getSwitchArgument< unsigned int >("-M");
    N = args.getSwitchArgument< unsigned int >("-N");
	} catch  ( isa::utils::SwitchNotFound &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }catch ( std::exception &err ) {
    std::cerr << "Usage: " << argv[0] << " [-print_code] [-print_data] -opencl_platform ... -opencl_device ... -padding ... -vector ... -threads ... -M ... -N ..." << std::endl;
		return 1;
	}

	// Initialize OpenCL
	cl::Context * clContext = new cl::Context();
	std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
	std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
	std::vector< std::vector< cl::CommandQueue > > * clQueues = new std::vector< std::vector < cl::CommandQueue > >();

  isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, clContext, clDevices, clQueues);

	// Allocate memory
  std::vector< dataType > input = std::vector< dataType >(M * isa::utils::pad(N, padding));
  cl::Buffer input_d;
  std::vector< dataType > output = std::vector< dataType >(N * isa::utils::pad(M, padding));
  cl::Buffer output_d;
  std::vector< dataType > output_c = std::vector< dataType >(N * isa::utils::pad(M, padding));
  try {
    input_d = cl::Buffer(*clContext, CL_MEM_READ_ONLY, input.size() * sizeof(dataType), NULL, NULL);
    output_d = cl::Buffer(*clContext, CL_MEM_WRITE_ONLY, output.size() * sizeof(dataType), NULL, NULL);
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error allocating memory: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

	srand(time(NULL));
  for ( unsigned int m = 0; m < M; m++ ) {
    for ( unsigned int n = 0; n < N; n++ ) {
      input[(m * isa::utils::pad(N, padding)) + n] = static_cast< dataType >(rand() % 10);
    }
	}

  // Copy data structures to device
  try {
    clQueues->at(clDeviceID)[0].enqueueWriteBuffer(input_d, CL_FALSE, 0, input.size() * sizeof(dataType), reinterpret_cast< void * >(input.data()));
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error H2D transfer: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

  // Generate kernel
  cl::Kernel * kernel;
  std::string * code = isa::OpenCL::getTransposeOpenCL(nrThreads, M, N, padding, vector, typeName);
  if ( printCode ) {
    std::cout << *code << std::endl;
  }

  try {
    kernel = isa::OpenCL::compile("transpose", *code, "-cl-mad-enable -Werror", *clContext, clDevices->at(clDeviceID));
  } catch ( isa::OpenCL::OpenCLError &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  // Run OpenCL kernel and CPU control
  try {
    cl::NDRange global(M, std::ceil(static_cast< double >(N) / nrThreads));
    cl::NDRange local(nrThreads, 1);

    kernel->setArg(0, input_d);
    kernel->setArg(1, output_d);

    clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, NULL, NULL);
    isa::OpenCL::transpose(M, N, padding, input, output_c);
    clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(dataType), reinterpret_cast< void * >(output.data()));
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

  if ( printData ) {
    for ( unsigned int m = 0; m < M; m++ ) {
      for ( unsigned int n = 0; n < N; n++ ) {
        std::cout << input[(m * isa::utils::pad(N, padding)) + n] << " ";
      }
      std::cout << std::endl;
    }
    std::cout << std::endl;
  }
  for ( unsigned int n = 0; n < N; n++ ) {
    for ( unsigned int m = 0; m < M; m++ ) {
      if ( ! isa::utils::same(output_c[(n * isa::utils::pad(M, padding)) + m], output[(n * isa::utils::pad(M, padding)) + m]) ) {
        wrongItems++;
      }
      if ( printData ) {
        std::cout << output[(n * isa::utils::pad(M, padding)) + m] << " ";
      }
    }
    if ( printData ) {
      std::cout << std::endl;
    }
  }
  if ( printData ) {
    std::cout << std::endl;
  }

  if ( wrongItems > 0 ) {
    std::cout << "Wrong samples: " << wrongItems << " (" << (wrongItems * 100.0) / (static_cast< long long unsigned int >(M) * N) << "%)." << std::endl;
  } else {
    std::cout << "TEST PASSED." << std::endl;
  }

	return 0;
}

<commit_msg>Padding the NDRange.<commit_after>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <ctime>
#include <cmath>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <utils.hpp>
#include <Transpose.hpp>

typedef float dataType;
std::string typeName("float");

int main(int argc, char *argv[]) {
  bool printCode = false;
  bool printData = false;
	unsigned int clPlatformID = 0;
	unsigned int clDeviceID = 0;
  unsigned int nrThreads = 0;
  unsigned int padding = 0;
  unsigned int vector = 0;
  unsigned int M = 0;
  unsigned int N = 0;
	long long unsigned int wrongItems = 0;

	try {
    isa::utils::ArgumentList args(argc, argv);
    printCode = args.getSwitch("-print_code");
    printData = args.getSwitch("-print_data");
		clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
		clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    vector = args.getSwitchArgument< unsigned int >("-vector");
    nrThreads = args.getSwitchArgument< unsigned int >("-threads");
    M = args.getSwitchArgument< unsigned int >("-M");
    N = args.getSwitchArgument< unsigned int >("-N");
	} catch  ( isa::utils::SwitchNotFound &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }catch ( std::exception &err ) {
    std::cerr << "Usage: " << argv[0] << " [-print_code] [-print_data] -opencl_platform ... -opencl_device ... -padding ... -vector ... -threads ... -M ... -N ..." << std::endl;
		return 1;
	}

	// Initialize OpenCL
	cl::Context * clContext = new cl::Context();
	std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
	std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
	std::vector< std::vector< cl::CommandQueue > > * clQueues = new std::vector< std::vector < cl::CommandQueue > >();

  isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, clContext, clDevices, clQueues);

	// Allocate memory
  std::vector< dataType > input = std::vector< dataType >(M * isa::utils::pad(N, padding));
  cl::Buffer input_d;
  std::vector< dataType > output = std::vector< dataType >(N * isa::utils::pad(M, padding));
  cl::Buffer output_d;
  std::vector< dataType > output_c = std::vector< dataType >(N * isa::utils::pad(M, padding));
  try {
    input_d = cl::Buffer(*clContext, CL_MEM_READ_ONLY, input.size() * sizeof(dataType), NULL, NULL);
    output_d = cl::Buffer(*clContext, CL_MEM_WRITE_ONLY, output.size() * sizeof(dataType), NULL, NULL);
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error allocating memory: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

	srand(time(NULL));
  for ( unsigned int m = 0; m < M; m++ ) {
    for ( unsigned int n = 0; n < N; n++ ) {
      input[(m * isa::utils::pad(N, padding)) + n] = static_cast< dataType >(rand() % 10);
    }
	}

  // Copy data structures to device
  try {
    clQueues->at(clDeviceID)[0].enqueueWriteBuffer(input_d, CL_FALSE, 0, input.size() * sizeof(dataType), reinterpret_cast< void * >(input.data()));
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error H2D transfer: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

  // Generate kernel
  cl::Kernel * kernel;
  std::string * code = isa::OpenCL::getTransposeOpenCL(nrThreads, M, N, padding, vector, typeName);
  if ( printCode ) {
    std::cout << *code << std::endl;
  }

  try {
    kernel = isa::OpenCL::compile("transpose", *code, "-cl-mad-enable -Werror", *clContext, clDevices->at(clDeviceID));
  } catch ( isa::OpenCL::OpenCLError &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  // Run OpenCL kernel and CPU control
  try {
    cl::NDRange global(isa::utils::pad(M, padding), std::ceil(static_cast< double >(isa::utils::pad(N, padding)) / nrThreads));
    cl::NDRange local(nrThreads, 1);

    kernel->setArg(0, input_d);
    kernel->setArg(1, output_d);

    clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, NULL, NULL);
    isa::OpenCL::transpose(M, N, padding, input, output_c);
    clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(dataType), reinterpret_cast< void * >(output.data()));
  } catch ( cl::Error &err ) {
    std::cerr << "OpenCL error: " << isa::utils::toString< cl_int >(err.err()) << "." << std::endl;
    return 1;
  }

  if ( printData ) {
    for ( unsigned int m = 0; m < M; m++ ) {
      for ( unsigned int n = 0; n < N; n++ ) {
        std::cout << input[(m * isa::utils::pad(N, padding)) + n] << " ";
      }
      std::cout << std::endl;
    }
    std::cout << std::endl;
  }
  for ( unsigned int n = 0; n < N; n++ ) {
    for ( unsigned int m = 0; m < M; m++ ) {
      if ( ! isa::utils::same(output_c[(n * isa::utils::pad(M, padding)) + m], output[(n * isa::utils::pad(M, padding)) + m]) ) {
        wrongItems++;
      }
      if ( printData ) {
        std::cout << output[(n * isa::utils::pad(M, padding)) + m] << " ";
      }
    }
    if ( printData ) {
      std::cout << std::endl;
    }
  }
  if ( printData ) {
    std::cout << std::endl;
  }

  if ( wrongItems > 0 ) {
    std::cout << "Wrong samples: " << wrongItems << " (" << (wrongItems * 100.0) / (static_cast< long long unsigned int >(M) * N) << "%)." << std::endl;
  } else {
    std::cout << "TEST PASSED." << std::endl;
  }

	return 0;
}

<|endoftext|>


<commit_before>#include "Test.h"
#include "SkRandom.h"
#include "SkRefCnt.h"
#include "SkTSearch.h"
#include "SkTSort.h"
#include "SkUtils.h"

class RefClass : public SkRefCnt {
public:
    RefClass(int n) : fN(n) {}
    int get() const { return fN; }

private:
    int fN;
};

static void test_refptr(skiatest::Reporter* reporter) {
    RefClass* r0 = new RefClass(0);

    SkRefPtr<RefClass> rc0;
    REPORTER_ASSERT(reporter, rc0.get() == NULL);
    REPORTER_ASSERT(reporter, !rc0);

    SkRefPtr<RefClass> rc1;
    REPORTER_ASSERT(reporter, rc0 == rc1);
    REPORTER_ASSERT(reporter, rc0.get() != r0);

    rc0 = r0;
    REPORTER_ASSERT(reporter, rc0);
    REPORTER_ASSERT(reporter, rc0 != rc1);
    REPORTER_ASSERT(reporter, rc0.get() == r0);

    rc1 = rc0;
    REPORTER_ASSERT(reporter, rc1);
    REPORTER_ASSERT(reporter, rc0 == rc1);
    REPORTER_ASSERT(reporter, rc0.get() == r0);

    rc0 = NULL;
    REPORTER_ASSERT(reporter, rc0.get() == NULL);
    REPORTER_ASSERT(reporter, !rc0);
    REPORTER_ASSERT(reporter, rc0 != rc1);

    r0->unref();
}

static void test_autounref(skiatest::Reporter* reporter) {
    RefClass obj(0);
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());

    SkAutoTUnref<RefClass> tmp(&obj);
    REPORTER_ASSERT(reporter, &obj == tmp.get());
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());

    REPORTER_ASSERT(reporter, &obj == tmp.detach());
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());
    REPORTER_ASSERT(reporter, NULL == tmp.detach());
    REPORTER_ASSERT(reporter, NULL == tmp.get());

    obj.ref();
    REPORTER_ASSERT(reporter, 2 == obj.getRefCnt());
    {
        SkAutoTUnref<RefClass> tmp2(&obj);
    }
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());
}

//////////////////////////////////////////////////////////////////////////////

#define kSEARCH_COUNT   91

static void test_search(skiatest::Reporter* reporter) {
    int         i, array[kSEARCH_COUNT];
    SkRandom    rand;

    for (i = 0; i < kSEARCH_COUNT; i++) {
        array[i] = rand.nextS();
    }

    SkTHeapSort<int>(array, kSEARCH_COUNT);
    // make sure we got sorted properly
    for (i = 1; i < kSEARCH_COUNT; i++) {
        REPORTER_ASSERT(reporter, array[i-1] <= array[i]);
    }

    // make sure we can find all of our values
    for (i = 0; i < kSEARCH_COUNT; i++) {
        int index = SkTSearch<int>(array, kSEARCH_COUNT, array[i], sizeof(int));
        REPORTER_ASSERT(reporter, index == i);
    }

    // make sure that random values are either found, or the correct
    // insertion index is returned
    for (i = 0; i < 10000; i++) {
        int value = rand.nextS();
        int index = SkTSearch<int>(array, kSEARCH_COUNT, value, sizeof(int));

        if (index >= 0) {
            REPORTER_ASSERT(reporter,
                            index < kSEARCH_COUNT && array[index] == value);
        } else {
            index = ~index;
            REPORTER_ASSERT(reporter, index <= kSEARCH_COUNT);
            if (index < kSEARCH_COUNT) {
                REPORTER_ASSERT(reporter, value < array[index]);
                if (index > 0) {
                    REPORTER_ASSERT(reporter, value > array[index - 1]);
                }
            } else {
                // we should append the new value
                REPORTER_ASSERT(reporter, value > array[kSEARCH_COUNT - 1]);
            }
        }
    }
}

static void test_utf16(skiatest::Reporter* reporter) {
    static const SkUnichar gUni[] = {
        0x10000, 0x18080, 0x20202, 0xFFFFF, 0x101234
    };

    uint16_t buf[2];

    for (size_t i = 0; i < SK_ARRAY_COUNT(gUni); i++) {
        size_t count = SkUTF16_FromUnichar(gUni[i], buf);
        REPORTER_ASSERT(reporter, count == 2);
        size_t count2 = SkUTF16_CountUnichars(buf, 2);
        REPORTER_ASSERT(reporter, count2 == 1);
        const uint16_t* ptr = buf;
        SkUnichar c = SkUTF16_NextUnichar(&ptr);
        REPORTER_ASSERT(reporter, c == gUni[i]);
        REPORTER_ASSERT(reporter, ptr - buf == 2);
    }
}

static void TestUTF(skiatest::Reporter* reporter) {
    static const struct {
        const char* fUtf8;
        SkUnichar   fUni;
    } gTest[] = {
        { "a",                  'a' },
        { "\x7f",               0x7f },
        { "\xC2\x80",           0x80 },
        { "\xC3\x83",           (3 << 6) | 3    },
        { "\xDF\xBF",           0x7ff },
        { "\xE0\xA0\x80",       0x800 },
        { "\xE0\xB0\xB8",       0xC38 },
        { "\xE3\x83\x83",       (3 << 12) | (3 << 6) | 3    },
        { "\xEF\xBF\xBF",       0xFFFF },
        { "\xF0\x90\x80\x80",   0x10000 },
        { "\xF3\x83\x83\x83",   (3 << 18) | (3 << 12) | (3 << 6) | 3    }
    };

    for (size_t i = 0; i < SK_ARRAY_COUNT(gTest); i++) {
        const char* p = gTest[i].fUtf8;
        int         n = SkUTF8_CountUnichars(p);
        SkUnichar   u0 = SkUTF8_ToUnichar(gTest[i].fUtf8);
        SkUnichar   u1 = SkUTF8_NextUnichar(&p);

        REPORTER_ASSERT(reporter, n == 1);
        REPORTER_ASSERT(reporter, u0 == u1);
        REPORTER_ASSERT(reporter, u0 == gTest[i].fUni);
        REPORTER_ASSERT(reporter,
                        p - gTest[i].fUtf8 == (int)strlen(gTest[i].fUtf8));
    }

    test_utf16(reporter);
    test_search(reporter);
    test_refptr(reporter);
    test_autounref(reporter);
}

#include "TestClassDef.h"
DEFINE_TESTCLASS("Utils", UtfTestClass, TestUTF)
<commit_msg>tiny document-only change<commit_after>#include "Test.h"
#include "SkRandom.h"
#include "SkRefCnt.h"
#include "SkTSearch.h"
#include "SkTSort.h"
#include "SkUtils.h"

class RefClass : public SkRefCnt {
public:
    RefClass(int n) : fN(n) {}
    int get() const { return fN; }

private:
    int fN;
};

static void test_refptr(skiatest::Reporter* reporter) {
    RefClass* r0 = new RefClass(0);

    SkRefPtr<RefClass> rc0;
    REPORTER_ASSERT(reporter, rc0.get() == NULL);
    REPORTER_ASSERT(reporter, !rc0);

    SkRefPtr<RefClass> rc1;
    REPORTER_ASSERT(reporter, rc0 == rc1);
    REPORTER_ASSERT(reporter, rc0.get() != r0);

    rc0 = r0;
    REPORTER_ASSERT(reporter, rc0);
    REPORTER_ASSERT(reporter, rc0 != rc1);
    REPORTER_ASSERT(reporter, rc0.get() == r0);

    rc1 = rc0;
    REPORTER_ASSERT(reporter, rc1);
    REPORTER_ASSERT(reporter, rc0 == rc1);
    REPORTER_ASSERT(reporter, rc0.get() == r0);

    rc0 = NULL;
    REPORTER_ASSERT(reporter, rc0.get() == NULL);
    REPORTER_ASSERT(reporter, !rc0);
    REPORTER_ASSERT(reporter, rc0 != rc1);

    r0->unref();
}

static void test_autounref(skiatest::Reporter* reporter) {
    RefClass obj(0);
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());

    SkAutoTUnref<RefClass> tmp(&obj);
    REPORTER_ASSERT(reporter, &obj == tmp.get());
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());

    REPORTER_ASSERT(reporter, &obj == tmp.detach());
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());
    REPORTER_ASSERT(reporter, NULL == tmp.detach());
    REPORTER_ASSERT(reporter, NULL == tmp.get());

    obj.ref();
    REPORTER_ASSERT(reporter, 2 == obj.getRefCnt());
    {
        SkAutoTUnref<RefClass> tmp2(&obj);
    }
    REPORTER_ASSERT(reporter, 1 == obj.getRefCnt());
}

/////////////////////////////////////////////////////////////////////////////

#define kSEARCH_COUNT   91

static void test_search(skiatest::Reporter* reporter) {
    int         i, array[kSEARCH_COUNT];
    SkRandom    rand;

    for (i = 0; i < kSEARCH_COUNT; i++) {
        array[i] = rand.nextS();
    }

    SkTHeapSort<int>(array, kSEARCH_COUNT);
    // make sure we got sorted properly
    for (i = 1; i < kSEARCH_COUNT; i++) {
        REPORTER_ASSERT(reporter, array[i-1] <= array[i]);
    }

    // make sure we can find all of our values
    for (i = 0; i < kSEARCH_COUNT; i++) {
        int index = SkTSearch<int>(array, kSEARCH_COUNT, array[i], sizeof(int));
        REPORTER_ASSERT(reporter, index == i);
    }

    // make sure that random values are either found, or the correct
    // insertion index is returned
    for (i = 0; i < 10000; i++) {
        int value = rand.nextS();
        int index = SkTSearch<int>(array, kSEARCH_COUNT, value, sizeof(int));

        if (index >= 0) {
            REPORTER_ASSERT(reporter,
                            index < kSEARCH_COUNT && array[index] == value);
        } else {
            index = ~index;
            REPORTER_ASSERT(reporter, index <= kSEARCH_COUNT);
            if (index < kSEARCH_COUNT) {
                REPORTER_ASSERT(reporter, value < array[index]);
                if (index > 0) {
                    REPORTER_ASSERT(reporter, value > array[index - 1]);
                }
            } else {
                // we should append the new value
                REPORTER_ASSERT(reporter, value > array[kSEARCH_COUNT - 1]);
            }
        }
    }
}

static void test_utf16(skiatest::Reporter* reporter) {
    static const SkUnichar gUni[] = {
        0x10000, 0x18080, 0x20202, 0xFFFFF, 0x101234
    };

    uint16_t buf[2];

    for (size_t i = 0; i < SK_ARRAY_COUNT(gUni); i++) {
        size_t count = SkUTF16_FromUnichar(gUni[i], buf);
        REPORTER_ASSERT(reporter, count == 2);
        size_t count2 = SkUTF16_CountUnichars(buf, 2);
        REPORTER_ASSERT(reporter, count2 == 1);
        const uint16_t* ptr = buf;
        SkUnichar c = SkUTF16_NextUnichar(&ptr);
        REPORTER_ASSERT(reporter, c == gUni[i]);
        REPORTER_ASSERT(reporter, ptr - buf == 2);
    }
}

static void TestUTF(skiatest::Reporter* reporter) {
    static const struct {
        const char* fUtf8;
        SkUnichar   fUni;
    } gTest[] = {
        { "a",                  'a' },
        { "\x7f",               0x7f },
        { "\xC2\x80",           0x80 },
        { "\xC3\x83",           (3 << 6) | 3    },
        { "\xDF\xBF",           0x7ff },
        { "\xE0\xA0\x80",       0x800 },
        { "\xE0\xB0\xB8",       0xC38 },
        { "\xE3\x83\x83",       (3 << 12) | (3 << 6) | 3    },
        { "\xEF\xBF\xBF",       0xFFFF },
        { "\xF0\x90\x80\x80",   0x10000 },
        { "\xF3\x83\x83\x83",   (3 << 18) | (3 << 12) | (3 << 6) | 3    }
    };

    for (size_t i = 0; i < SK_ARRAY_COUNT(gTest); i++) {
        const char* p = gTest[i].fUtf8;
        int         n = SkUTF8_CountUnichars(p);
        SkUnichar   u0 = SkUTF8_ToUnichar(gTest[i].fUtf8);
        SkUnichar   u1 = SkUTF8_NextUnichar(&p);

        REPORTER_ASSERT(reporter, n == 1);
        REPORTER_ASSERT(reporter, u0 == u1);
        REPORTER_ASSERT(reporter, u0 == gTest[i].fUni);
        REPORTER_ASSERT(reporter,
                        p - gTest[i].fUtf8 == (int)strlen(gTest[i].fUtf8));
    }

    test_utf16(reporter);
    test_search(reporter);
    test_refptr(reporter);
    test_autounref(reporter);
}

#include "TestClassDef.h"
DEFINE_TESTCLASS("Utils", UtfTestClass, TestUTF)
<|endoftext|>


<commit_before>// Copyright (c) 2011-2020 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#if defined(HAVE_CONFIG_H)
#include <config/bitcoin-config.h>
#endif

#include <mapport.h>

#include <clientversion.h>
#include <logging.h>
#include <net.h>
#include <netaddress.h>
#include <netbase.h>
#include <threadinterrupt.h>
#include <util/system.h>

#ifdef USE_UPNP
#include <miniupnpc/miniupnpc.h>
#include <miniupnpc/upnpcommands.h>
#include <miniupnpc/upnperrors.h>
// The minimum supported miniUPnPc API version is set to 10. This keeps compatibility
// with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages.
static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed");
#endif

#include <cassert>
#include <chrono>
#include <functional>
#include <string>
#include <thread>

#ifdef USE_UPNP
static CThreadInterrupt g_upnp_interrupt;
static std::thread g_upnp_thread;

using namespace std::chrono_literals;
static constexpr auto PORT_MAPPING_REANNOUNCE_PERIOD{20min};

static void ThreadMapPort()
{
    std::string port = strprintf("%u", GetListenPort());
    const char * multicastif = nullptr;
    const char * minissdpdpath = nullptr;
    struct UPNPDev * devlist = nullptr;
    char lanaddr[64];

    int error = 0;
#if MINIUPNPC_API_VERSION < 14
    devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, &error);
#else
    devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error);
#endif

    struct UPNPUrls urls;
    struct IGDdatas data;
    int r;

    r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr));
    if (r == 1)
    {
        if (fDiscover) {
            char externalIPAddress[40];
            r = UPNP_GetExternalIPAddress(urls.controlURL, data.first.servicetype, externalIPAddress);
            if (r != UPNPCOMMAND_SUCCESS) {
                LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r);
            } else {
                if (externalIPAddress[0]) {
                    CNetAddr resolved;
                    if (LookupHost(externalIPAddress, resolved, false)) {
                        LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString());
                        AddLocal(resolved, LOCAL_UPNP);
                    }
                } else {
                    LogPrintf("UPnP: GetExternalIPAddress failed.\n");
                }
            }
        }

        std::string strDesc = PACKAGE_NAME " " + FormatFullVersion();

        do {
            r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0");

            if (r != UPNPCOMMAND_SUCCESS) {
                LogPrintf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", port, port, lanaddr, r, strupnperror(r));
            } else {
                LogPrintf("UPnP Port Mapping successful.\n");
            }
        } while (g_upnp_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD));

        r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0);
        LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r);
        freeUPNPDevlist(devlist); devlist = nullptr;
        FreeUPNPUrls(&urls);
    } else {
        LogPrintf("No valid UPnP IGDs found\n");
        freeUPNPDevlist(devlist); devlist = nullptr;
        if (r != 0)
            FreeUPNPUrls(&urls);
    }
}

void StartMapPort()
{
    if (!g_upnp_thread.joinable()) {
        assert(!g_upnp_interrupt);
        g_upnp_thread = std::thread((std::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort)));
    }
}

void InterruptMapPort()
{
    if(g_upnp_thread.joinable()) {
        g_upnp_interrupt();
    }
}

void StopMapPort()
{
    if(g_upnp_thread.joinable()) {
        g_upnp_thread.join();
        g_upnp_interrupt.reset();
    }
}

#else
void StartMapPort()
{
    // Intentionally left blank.
}
void InterruptMapPort()
{
    // Intentionally left blank.
}
void StopMapPort()
{
    // Intentionally left blank.
}
#endif
<commit_msg>net: Keep trying to use UPnP when -upnp=1<commit_after>// Copyright (c) 2011-2020 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#if defined(HAVE_CONFIG_H)
#include <config/bitcoin-config.h>
#endif

#include <mapport.h>

#include <clientversion.h>
#include <logging.h>
#include <net.h>
#include <netaddress.h>
#include <netbase.h>
#include <threadinterrupt.h>
#include <util/system.h>

#ifdef USE_UPNP
#include <miniupnpc/miniupnpc.h>
#include <miniupnpc/upnpcommands.h>
#include <miniupnpc/upnperrors.h>
// The minimum supported miniUPnPc API version is set to 10. This keeps compatibility
// with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages.
static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed");
#endif

#include <cassert>
#include <chrono>
#include <functional>
#include <string>
#include <thread>

#ifdef USE_UPNP
static CThreadInterrupt g_upnp_interrupt;
static std::thread g_upnp_thread;

using namespace std::chrono_literals;
static constexpr auto PORT_MAPPING_REANNOUNCE_PERIOD{20min};
static constexpr auto PORT_MAPPING_RETRY_PERIOD{5min};

static bool ProcessUpnp()
{
    bool ret = false;
    std::string port = strprintf("%u", GetListenPort());
    const char * multicastif = nullptr;
    const char * minissdpdpath = nullptr;
    struct UPNPDev * devlist = nullptr;
    char lanaddr[64];

    int error = 0;
#if MINIUPNPC_API_VERSION < 14
    devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, &error);
#else
    devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error);
#endif

    struct UPNPUrls urls;
    struct IGDdatas data;
    int r;

    r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr));
    if (r == 1)
    {
        if (fDiscover) {
            char externalIPAddress[40];
            r = UPNP_GetExternalIPAddress(urls.controlURL, data.first.servicetype, externalIPAddress);
            if (r != UPNPCOMMAND_SUCCESS) {
                LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r);
            } else {
                if (externalIPAddress[0]) {
                    CNetAddr resolved;
                    if (LookupHost(externalIPAddress, resolved, false)) {
                        LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString());
                        AddLocal(resolved, LOCAL_UPNP);
                    }
                } else {
                    LogPrintf("UPnP: GetExternalIPAddress failed.\n");
                }
            }
        }

        std::string strDesc = PACKAGE_NAME " " + FormatFullVersion();

        do {
            r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0");

            if (r != UPNPCOMMAND_SUCCESS) {
                ret = false;
                LogPrintf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", port, port, lanaddr, r, strupnperror(r));
                break;
            } else {
                ret = true;
                LogPrintf("UPnP Port Mapping successful.\n");
            }
        } while (g_upnp_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD));
        g_upnp_interrupt.reset();

        r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0);
        LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r);
        freeUPNPDevlist(devlist); devlist = nullptr;
        FreeUPNPUrls(&urls);
    } else {
        LogPrintf("No valid UPnP IGDs found\n");
        freeUPNPDevlist(devlist); devlist = nullptr;
        if (r != 0)
            FreeUPNPUrls(&urls);
    }

    return ret;
}

static void ThreadMapPort()
{
    do {
        if (ProcessUpnp()) return;
    } while (g_upnp_interrupt.sleep_for(PORT_MAPPING_RETRY_PERIOD));
}

void StartMapPort()
{
    if (!g_upnp_thread.joinable()) {
        assert(!g_upnp_interrupt);
        g_upnp_thread = std::thread((std::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort)));
    }
}

void InterruptMapPort()
{
    if(g_upnp_thread.joinable()) {
        g_upnp_interrupt();
    }
}

void StopMapPort()
{
    if(g_upnp_thread.joinable()) {
        g_upnp_thread.join();
        g_upnp_interrupt.reset();
    }
}

#else
void StartMapPort()
{
    // Intentionally left blank.
}
void InterruptMapPort()
{
    // Intentionally left blank.
}
void StopMapPort()
{
    // Intentionally left blank.
}
#endif
<|endoftext|>


<commit_before>#include "matcher.h"

#include <cassert>
#include <ctime>
#include <regex.h>
#include <sstream>
#include <sys/time.h>
#include <vector>

#include "exceptions.h"
#include "logger.h"
#include "utils.h"

namespace newsboat {

Matchable::Matchable() {}
Matchable::~Matchable() {}

Matcher::Matcher() {}

Matcher::Matcher(const std::string& expr)
	: exp(expr)
{
	parse(expr);
}

const std::string& Matcher::get_expression()
{
	return exp;
}

bool Matcher::parse(const std::string& expr)
{
	struct timeval tv1, tv2;
	gettimeofday(&tv1, nullptr);

	errmsg = "";

	bool b = p.parse_string(expr);

	if (!b) {
		errmsg = utils::wstr2str(p.get_error());
	}

	gettimeofday(&tv2, nullptr);
	unsigned long diff =
		(((tv2.tv_sec - tv1.tv_sec) * 1000000) + tv2.tv_usec) -
		tv1.tv_usec;
	LOG(Level::DEBUG,
		"Matcher::parse: parsing `%s' took %lu µs (success = %d)",
		expr,
		diff,
		b ? 1 : 0);

	return b;
}

bool Matcher::matches(Matchable* item)
{
	/*
	 * with this method, every class that is derived from Matchable can be
	 * matched against a filter expression that was previously passed to the
	 * class with the parse() method.
	 *
	 * This makes it easy to use the Matcher virtually everywhere, since C++
	 * allows multiple inheritance (i.e. deriving from Matchable can even be
	 * used in class hierarchies), and deriving from Matchable means that
	 * you only have to implement two methods has_attribute() and
	 * get_attribute().
	 *
	 * The whole matching code is speed-critical, as the matching happens on
	 * a lot of different occassions, and slow matching can be easily
	 * measured (and felt by the user) on slow computers with a lot of items
	 * to match.
	 */
	bool retval = false;
	if (item) {
		ScopeMeasure m1("Matcher::matches");
		retval = matches_r(p.get_root(), item);
	}
	return retval;
}

bool Matcher::matchop_lt(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::istringstream islit(e->literal);
	std::istringstream isatt(item->get_attribute(e->name));
	int ilit, iatt;
	islit >> ilit;
	isatt >> iatt;
	return iatt < ilit;
}

bool Matcher::matchop_between(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::vector<std::string> lit = utils::tokenize(e->literal, ":");
	std::istringstream isatt(item->get_attribute(e->name));
	int att;
	isatt >> att;
	if (lit.size() < 2)
		return false;
	std::istringstream is1(lit[0]), is2(lit[1]);
	int i1, i2;
	is1 >> i1;
	is2 >> i2;
	if (i1 > i2) {
		int tmp = i1;
		i1 = i2;
		i2 = tmp;
	}
	return (att >= i1 && att <= i2);
}

bool Matcher::matchop_gt(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::istringstream islit(e->literal);
	std::istringstream isatt(item->get_attribute(e->name));
	int ilit, iatt;
	islit >> ilit;
	isatt >> iatt;
	return iatt > ilit;
}

bool Matcher::matchop_rxeq(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	if (!e->regex) {
		e->regex = new regex_t;
		int err;
		if ((err = regcomp(e->regex,
			     e->literal.c_str(),
			     REG_EXTENDED | REG_ICASE | REG_NOSUB)) != 0) {
			char buf[1024];
			regerror(err, e->regex, buf, sizeof(buf));
			throw MatcherException(
				MatcherException::Type::INVALID_REGEX,
				e->literal,
				buf);
		}
	}
	if (regexec(e->regex,
		    item->get_attribute(e->name).c_str(),
		    0,
		    nullptr,
		    0) == 0)
		return true;
	return false;
}

bool Matcher::matchop_cont(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::vector<std::string> elements =
		utils::tokenize(item->get_attribute(e->name), " ");
	std::string literal = e->literal;
	for (const auto& elem : elements) {
		if (literal == elem) {
			return true;
		}
	}
	return false;
}

bool Matcher::matchop_eq(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name)) {
		LOG(Level::WARN,
			"Matcher::matches_r: attribute %s not available",
			e->name);
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	}
	return (item->get_attribute(e->name) == e->literal);
}

bool Matcher::matches_r(expression* e, Matchable* item)
{
	if (e) {
		switch (e->op) {
		/* the operator "and" and "or" simply connect two different
		 * subexpressions */
		case LOGOP_AND:
			return matches_r(e->l, item) &&
				matches_r(e->r, item); // short-circuit
						       // evaulation in C ->
						       // short circuit
						       // evaluation in the
						       // filter language

		case LOGOP_OR:
			return matches_r(e->l, item) ||
				matches_r(e->r, item); // same here

		/* while the other operator connect an attribute with a value */
		case MATCHOP_EQ:
			return matchop_eq(e, item);

		case MATCHOP_NE:
			return !matchop_eq(e, item);

		case MATCHOP_LT:
			return matchop_lt(e, item);

		case MATCHOP_BETWEEN:
			return matchop_between(e, item);

		case MATCHOP_GT:
			return matchop_gt(e, item);

		case MATCHOP_LE:
			return !matchop_gt(e, item);

		case MATCHOP_GE:
			return !matchop_lt(e, item);

		case MATCHOP_RXEQ:
			return matchop_rxeq(e, item);

		case MATCHOP_RXNE:
			return !matchop_rxeq(e, item);

		case MATCHOP_CONTAINS:
			return matchop_cont(e, item);

		case MATCHOP_CONTAINSNOT:
			return !matchop_cont(e, item);
		}
		return false;
	} else {
		return true; // shouldn't happen
	}
}

const std::string& Matcher::get_parse_error()
{
	return errmsg;
}

} // namespace newsboat
<commit_msg>Deallocate regex_t if regex compilation failed<commit_after>#include "matcher.h"

#include <cassert>
#include <ctime>
#include <regex.h>
#include <sstream>
#include <sys/time.h>
#include <vector>

#include "exceptions.h"
#include "logger.h"
#include "utils.h"

namespace newsboat {

Matchable::Matchable() {}
Matchable::~Matchable() {}

Matcher::Matcher() {}

Matcher::Matcher(const std::string& expr)
	: exp(expr)
{
	parse(expr);
}

const std::string& Matcher::get_expression()
{
	return exp;
}

bool Matcher::parse(const std::string& expr)
{
	struct timeval tv1, tv2;
	gettimeofday(&tv1, nullptr);

	errmsg = "";

	bool b = p.parse_string(expr);

	if (!b) {
		errmsg = utils::wstr2str(p.get_error());
	}

	gettimeofday(&tv2, nullptr);
	unsigned long diff =
		(((tv2.tv_sec - tv1.tv_sec) * 1000000) + tv2.tv_usec) -
		tv1.tv_usec;
	LOG(Level::DEBUG,
		"Matcher::parse: parsing `%s' took %lu µs (success = %d)",
		expr,
		diff,
		b ? 1 : 0);

	return b;
}

bool Matcher::matches(Matchable* item)
{
	/*
	 * with this method, every class that is derived from Matchable can be
	 * matched against a filter expression that was previously passed to the
	 * class with the parse() method.
	 *
	 * This makes it easy to use the Matcher virtually everywhere, since C++
	 * allows multiple inheritance (i.e. deriving from Matchable can even be
	 * used in class hierarchies), and deriving from Matchable means that
	 * you only have to implement two methods has_attribute() and
	 * get_attribute().
	 *
	 * The whole matching code is speed-critical, as the matching happens on
	 * a lot of different occassions, and slow matching can be easily
	 * measured (and felt by the user) on slow computers with a lot of items
	 * to match.
	 */
	bool retval = false;
	if (item) {
		ScopeMeasure m1("Matcher::matches");
		retval = matches_r(p.get_root(), item);
	}
	return retval;
}

bool Matcher::matchop_lt(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::istringstream islit(e->literal);
	std::istringstream isatt(item->get_attribute(e->name));
	int ilit, iatt;
	islit >> ilit;
	isatt >> iatt;
	return iatt < ilit;
}

bool Matcher::matchop_between(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::vector<std::string> lit = utils::tokenize(e->literal, ":");
	std::istringstream isatt(item->get_attribute(e->name));
	int att;
	isatt >> att;
	if (lit.size() < 2)
		return false;
	std::istringstream is1(lit[0]), is2(lit[1]);
	int i1, i2;
	is1 >> i1;
	is2 >> i2;
	if (i1 > i2) {
		int tmp = i1;
		i1 = i2;
		i2 = tmp;
	}
	return (att >= i1 && att <= i2);
}

bool Matcher::matchop_gt(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::istringstream islit(e->literal);
	std::istringstream isatt(item->get_attribute(e->name));
	int ilit, iatt;
	islit >> ilit;
	isatt >> iatt;
	return iatt > ilit;
}

bool Matcher::matchop_rxeq(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	if (!e->regex) {
		e->regex = new regex_t;
		int err;
		if ((err = regcomp(e->regex,
			     e->literal.c_str(),
			     REG_EXTENDED | REG_ICASE | REG_NOSUB)) != 0) {
			delete e->regex;
			e->regex = nullptr;
			char buf[1024];
			regerror(err, e->regex, buf, sizeof(buf));
			throw MatcherException(
				MatcherException::Type::INVALID_REGEX,
				e->literal,
				buf);
		}
	}
	if (regexec(e->regex,
		    item->get_attribute(e->name).c_str(),
		    0,
		    nullptr,
		    0) == 0)
		return true;
	return false;
}

bool Matcher::matchop_cont(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name))
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	std::vector<std::string> elements =
		utils::tokenize(item->get_attribute(e->name), " ");
	std::string literal = e->literal;
	for (const auto& elem : elements) {
		if (literal == elem) {
			return true;
		}
	}
	return false;
}

bool Matcher::matchop_eq(expression* e, Matchable* item)
{
	if (!item->has_attribute(e->name)) {
		LOG(Level::WARN,
			"Matcher::matches_r: attribute %s not available",
			e->name);
		throw MatcherException(
			MatcherException::Type::ATTRIB_UNAVAIL, e->name);
	}
	return (item->get_attribute(e->name) == e->literal);
}

bool Matcher::matches_r(expression* e, Matchable* item)
{
	if (e) {
		switch (e->op) {
		/* the operator "and" and "or" simply connect two different
		 * subexpressions */
		case LOGOP_AND:
			return matches_r(e->l, item) &&
				matches_r(e->r, item); // short-circuit
						       // evaulation in C ->
						       // short circuit
						       // evaluation in the
						       // filter language

		case LOGOP_OR:
			return matches_r(e->l, item) ||
				matches_r(e->r, item); // same here

		/* while the other operator connect an attribute with a value */
		case MATCHOP_EQ:
			return matchop_eq(e, item);

		case MATCHOP_NE:
			return !matchop_eq(e, item);

		case MATCHOP_LT:
			return matchop_lt(e, item);

		case MATCHOP_BETWEEN:
			return matchop_between(e, item);

		case MATCHOP_GT:
			return matchop_gt(e, item);

		case MATCHOP_LE:
			return !matchop_gt(e, item);

		case MATCHOP_GE:
			return !matchop_lt(e, item);

		case MATCHOP_RXEQ:
			return matchop_rxeq(e, item);

		case MATCHOP_RXNE:
			return !matchop_rxeq(e, item);

		case MATCHOP_CONTAINS:
			return matchop_cont(e, item);

		case MATCHOP_CONTAINSNOT:
			return !matchop_cont(e, item);
		}
		return false;
	} else {
		return true; // shouldn't happen
	}
}

const std::string& Matcher::get_parse_error()
{
	return errmsg;
}

} // namespace newsboat
<|endoftext|>