<commit_before>/*
 * Manager.cpp
 *
 *  Created on: 23 Oct 2015
 *      Author: hieu
 */
#include <boost/foreach.hpp>
#include <cstdlib>
#include <vector>
#include <sstream>
#include "../System.h"
#include "../TranslationModel/PhraseTable.h"
#include "Manager.h"
#include "InputPath.h"
#include "Hypothesis.h"
#include "TargetPhraseImpl.h"
#include "ActiveChart.h"
#include "Sentence.h"

using namespace std;

namespace Moses2
{
namespace SCFG
{

Manager::Manager(System &sys, const TranslationTask &task,
    const std::string &inputStr, long translationId)
:ManagerBase(sys, task, inputStr, translationId)
{

}

Manager::~Manager()
{

}

void Manager::Decode()
{
  // init pools etc
  //cerr << "START InitPools()" << endl;
  InitPools();
  //cerr << "START ParseInput()" << endl;

  FactorCollection &vocab = system.GetVocab();
  m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr,
      m_translationId);

  const SCFG::Sentence &sentence = static_cast<const SCFG::Sentence&>(GetInput());

  size_t inputSize = sentence.GetSize();
  //cerr << "inputSize=" << inputSize << endl;

  m_inputPaths.Init(sentence, *this);
  //cerr << "CREATED m_inputPaths" << endl;

  m_stacks.Init(*this, inputSize);
  //cerr << "CREATED m_stacks" << endl;

  for (int startPos = inputSize - 1; startPos >= 0; --startPos) {
    //cerr << endl << "startPos=" << startPos << endl;
    SCFG::InputPath &initPath = *m_inputPaths.GetMatrix().GetValue(startPos, 0);

    //cerr << "BEFORE InitActiveChart=" << initPath.Debug(system) << endl;
    InitActiveChart(initPath);
    //cerr << "AFTER InitActiveChart=" << initPath.Debug(system) << endl;

    int maxPhraseSize = inputSize - startPos + 1;
    for (int phraseSize = 1; phraseSize < maxPhraseSize; ++phraseSize) {
      //cerr << endl << "phraseSize=" << phraseSize << endl;
      SCFG::InputPath &path = *m_inputPaths.GetMatrix().GetValue(startPos, phraseSize);

      Stack &stack = m_stacks.GetStack(startPos, phraseSize);

      //cerr << "BEFORE LOOKUP path=" << path.Debug(system) << endl;
      Lookup(path);
      //cerr << "AFTER LOOKUP path="  << path.Debug(system) << endl;
      Decode(path, stack);
      //cerr << "AFTER DECODE path=" << path.Debug(system) << endl;

      LookupUnary(path);
      //cerr << "AFTER LookupUnary path=" << path.Debug(system) << endl;

      //cerr << "#rules=" << path.GetNumRules() << endl;
    }
  }

  //m_stacks.OutputStacks();
}

void Manager::InitActiveChart(SCFG::InputPath &path)
{
   size_t numPt = system.mappings.size();
   //cerr << "numPt=" << numPt << endl;

   for (size_t i = 0; i < numPt; ++i) {
     const PhraseTable &pt = *system.mappings[i];
     //cerr << "START InitActiveChart" << endl;
     pt.InitActiveChart(GetPool(), *this, path);
     //cerr << "FINISHED InitActiveChart" << endl;
   }
}

void Manager::Lookup(SCFG::InputPath &path)
{
  size_t numPt = system.mappings.size();
  //cerr << "numPt=" << numPt << endl;

  for (size_t i = 0; i < numPt; ++i) {
    const PhraseTable &pt = *system.mappings[i];
    size_t maxChartSpan = system.maxChartSpans[i];
    pt.Lookup(GetPool(), *this, maxChartSpan, m_stacks, path);
  }

  /*
  size_t tpsNum = path.targetPhrases.GetSize();
  if (tpsNum) {
    cerr << tpsNum << " " << path << endl;
  }
  */
}

void Manager::LookupUnary(SCFG::InputPath &path)
{
  size_t numPt = system.mappings.size();
  //cerr << "numPt=" << numPt << endl;

  for (size_t i = 0; i < numPt; ++i) {
    const PhraseTable &pt = *system.mappings[i];
    pt.LookupUnary(GetPool(), *this, m_stacks, path);
  }

  /*
  size_t tpsNum = path.targetPhrases.GetSize();
  if (tpsNum) {
    cerr << tpsNum << " " << path << endl;
  }
  */
}

///////////////////////////////////////////////////////////////
// CUBE-PRUNING
///////////////////////////////////////////////////////////////
void Manager::Decode(SCFG::InputPath &path, Stack &stack)
{
  // clear cube pruning data
  //std::vector<QueueItem*> &container = Container(m_queue);
  //container.clear();
  Recycler<HypothesisBase*> &hypoRecycler = GetHypoRecycle();
  while (!m_queue.empty()) {
    QueueItem *item = m_queue.top();
    m_queue.pop();
    // recycle unused hypos from queue
    Hypothesis *hypo = item->hypo;
    hypoRecycler.Recycle(hypo);

    // recycle queue item
    m_queueItemRecycler.push_back(item);
  }

  m_seenPositions.clear();

  // init queue
  BOOST_FOREACH(const InputPath::Coll::value_type &valPair, *path.targetPhrases) {
    const SymbolBind &symbolBind = valPair.first;
    const SCFG::TargetPhrases &tps = *valPair.second;
    CreateQueue(path, symbolBind, tps);
  }

  // MAIN LOOP
  size_t pops = 0;
  while (!m_queue.empty() && pops < system.options.cube.pop_limit) {
    //cerr << "pops=" << pops << endl;
    QueueItem *item = m_queue.top();
    m_queue.pop();

    // add hypo to stack
    Hypothesis *hypo = item->hypo;

    //cerr << "hypo=" << *hypo << " " << endl;
    stack.Add(hypo, GetHypoRecycle(), arcLists);
    //cerr << "Added " << *hypo << " " << endl;

    item->CreateNext(GetSystemPool(), GetPool(), *this, m_queue, m_seenPositions, path);
    //cerr << "Created next " << endl;
    m_queueItemRecycler.push_back(item);

    ++pops;
  }

}

void Manager::CreateQueue(
    const SCFG::InputPath &path,
    const SymbolBind &symbolBind,
    const SCFG::TargetPhrases &tps)
{
  QueueItem *item = QueueItem::Create(GetPool(), *this);
  item->Init(GetPool(), symbolBind, tps);
  for (size_t i = 0; i < symbolBind.coll.size(); ++i) {
    const SymbolBindElement &ele = symbolBind.coll[i];
    if (ele.hypos) {
      item->AddHypos(*ele.hypos);
    }
  }
  item->CreateHypo(GetSystemPool(), *this, path, symbolBind);

  //cerr << "hypo=" << item->hypo->Debug(system) << endl;

  m_queue.push(item);
}

///////////////////////////////////////////////////////////////
// NON CUBE-PRUNING
///////////////////////////////////////////////////////////////
/*
void Manager::Decode(SCFG::InputPath &path, Stack &stack)
{
  //cerr << "path=" << path << endl;

  boost::unordered_map<SCFG::SymbolBind, SCFG::TargetPhrases*>::const_iterator iterOuter;
  for (iterOuter = path.targetPhrases->begin(); iterOuter != path.targetPhrases->end(); ++iterOuter) {
    const SCFG::SymbolBind &symbolBind = iterOuter->first;

    const SCFG::TargetPhrases &tps = *iterOuter->second;
    //cerr << "symbolBind=" << symbolBind << " tps=" << tps.GetSize() << endl;

    SCFG::TargetPhrases::const_iterator iter;
    for (iter = tps.begin(); iter != tps.end(); ++iter) {
      const SCFG::TargetPhraseImpl &tp = **iter;
      //cerr << "tp=" << tp << endl;
      ExpandHypo(path, symbolBind, tp, stack);
    }
  }
}
*/

void Manager::ExpandHypo(
    const SCFG::InputPath &path,
    const SCFG::SymbolBind &symbolBind,
    const SCFG::TargetPhraseImpl &tp,
    Stack &stack)
{
  Recycler<HypothesisBase*> &hypoRecycler = GetHypoRecycle();

  std::vector<const SymbolBindElement*> ntEles = symbolBind.GetNTElements();
  Vector<size_t> prevHyposIndices(GetPool(), symbolBind.numNT);
  assert(ntEles.size() == symbolBind.numNT);
  //cerr << "ntEles:" << ntEles.size() << endl;

  size_t ind = 0;
  while (IncrPrevHypoIndices(prevHyposIndices, ind, ntEles)) {
    SCFG::Hypothesis *hypo = SCFG::Hypothesis::Create(GetSystemPool(), *this);
    hypo->Init(*this, path, symbolBind, tp, prevHyposIndices);
    hypo->EvaluateWhenApplied();

    stack.Add(hypo, hypoRecycler, arcLists);

    ++ind;
  }
}

bool Manager::IncrPrevHypoIndices(
    Vector<size_t> &prevHyposIndices,
    size_t ind,
    const std::vector<const SymbolBindElement*> ntEles)
{
  if (ntEles.size() == 0) {
    // no nt. Do the 1st
    return ind ? false : true;
  }

  size_t numHypos = 0;

  //cerr << "IncrPrevHypoIndices:" << ind << " " << ntEles.size() << " ";
  for (size_t i = 0; i < ntEles.size() - 1; ++i) {
    const SymbolBindElement &ele = *ntEles[i];
    const Hypotheses &hypos = *ele.hypos;
    numHypos = hypos.size();

    std::div_t divRet = std::div((int)ind, (int)numHypos);
    ind = divRet.quot;

    size_t hypoInd = divRet.rem;
    prevHyposIndices[i] = hypoInd;
    //cerr << "(" << i << "," << ind << "," << numHypos << "," << hypoInd << ")";
  }

  // last
  prevHyposIndices.back() = ind;

  // check if last is over limit
  const SymbolBindElement &ele = *ntEles.back();
  const Hypotheses &hypos = *ele.hypos;
  numHypos = hypos.size();

  //cerr << "(" << (ntEles.size() - 1) << "," << ind << "," << numHypos << ","  << ind << ")";
  //cerr << endl;

  if (ind >= numHypos) {
    return false;
  }
  else {
    return true;
  }
}

std::string Manager::OutputBest() const
{
  stringstream out;
  const Stack &lastStack = m_stacks.GetLastStack();
  const Hypothesis *bestHypo = lastStack.GetBestHypo(*this, const_cast<ArcLists&>(arcLists));

  if (bestHypo) {
    if (system.options.output.ReportHypoScore) {
      out << bestHypo->GetScores().GetTotalScore() << " ";
    }

    //cerr << "BEST TRANSLATION: " << bestHypo << bestHypo->Debug(system) << endl;
    //cerr << " " << out.str() << endl;
    bestHypo->OutputToStream(out);
  }
  else {
    if (system.options.output.ReportHypoScore) {
      out << "0 ";
    }

    //cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
  }

  out << endl;
  return out.str();
}

} // namespace
}

<commit_msg>don't output <s> </s><commit_after>/*
 * Manager.cpp
 *
 *  Created on: 23 Oct 2015
 *      Author: hieu
 */
#include <boost/foreach.hpp>
#include <cstdlib>
#include <vector>
#include <sstream>
#include "../System.h"
#include "../TranslationModel/PhraseTable.h"
#include "Manager.h"
#include "InputPath.h"
#include "Hypothesis.h"
#include "TargetPhraseImpl.h"
#include "ActiveChart.h"
#include "Sentence.h"

using namespace std;

namespace Moses2
{
namespace SCFG
{

Manager::Manager(System &sys, const TranslationTask &task,
    const std::string &inputStr, long translationId)
:ManagerBase(sys, task, inputStr, translationId)
{

}

Manager::~Manager()
{

}

void Manager::Decode()
{
  // init pools etc
  //cerr << "START InitPools()" << endl;
  InitPools();
  //cerr << "START ParseInput()" << endl;

  FactorCollection &vocab = system.GetVocab();
  m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr,
      m_translationId);

  const SCFG::Sentence &sentence = static_cast<const SCFG::Sentence&>(GetInput());

  size_t inputSize = sentence.GetSize();
  //cerr << "inputSize=" << inputSize << endl;

  m_inputPaths.Init(sentence, *this);
  //cerr << "CREATED m_inputPaths" << endl;

  m_stacks.Init(*this, inputSize);
  //cerr << "CREATED m_stacks" << endl;

  for (int startPos = inputSize - 1; startPos >= 0; --startPos) {
    //cerr << endl << "startPos=" << startPos << endl;
    SCFG::InputPath &initPath = *m_inputPaths.GetMatrix().GetValue(startPos, 0);

    //cerr << "BEFORE InitActiveChart=" << initPath.Debug(system) << endl;
    InitActiveChart(initPath);
    //cerr << "AFTER InitActiveChart=" << initPath.Debug(system) << endl;

    int maxPhraseSize = inputSize - startPos + 1;
    for (int phraseSize = 1; phraseSize < maxPhraseSize; ++phraseSize) {
      //cerr << endl << "phraseSize=" << phraseSize << endl;
      SCFG::InputPath &path = *m_inputPaths.GetMatrix().GetValue(startPos, phraseSize);

      Stack &stack = m_stacks.GetStack(startPos, phraseSize);

      //cerr << "BEFORE LOOKUP path=" << path.Debug(system) << endl;
      Lookup(path);
      //cerr << "AFTER LOOKUP path="  << path.Debug(system) << endl;
      Decode(path, stack);
      //cerr << "AFTER DECODE path=" << path.Debug(system) << endl;

      LookupUnary(path);
      //cerr << "AFTER LookupUnary path=" << path.Debug(system) << endl;

      //cerr << "#rules=" << path.GetNumRules() << endl;
    }
  }

  //m_stacks.OutputStacks();
}

void Manager::InitActiveChart(SCFG::InputPath &path)
{
   size_t numPt = system.mappings.size();
   //cerr << "numPt=" << numPt << endl;

   for (size_t i = 0; i < numPt; ++i) {
     const PhraseTable &pt = *system.mappings[i];
     //cerr << "START InitActiveChart" << endl;
     pt.InitActiveChart(GetPool(), *this, path);
     //cerr << "FINISHED InitActiveChart" << endl;
   }
}

void Manager::Lookup(SCFG::InputPath &path)
{
  size_t numPt = system.mappings.size();
  //cerr << "numPt=" << numPt << endl;

  for (size_t i = 0; i < numPt; ++i) {
    const PhraseTable &pt = *system.mappings[i];
    size_t maxChartSpan = system.maxChartSpans[i];
    pt.Lookup(GetPool(), *this, maxChartSpan, m_stacks, path);
  }

  /*
  size_t tpsNum = path.targetPhrases.GetSize();
  if (tpsNum) {
    cerr << tpsNum << " " << path << endl;
  }
  */
}

void Manager::LookupUnary(SCFG::InputPath &path)
{
  size_t numPt = system.mappings.size();
  //cerr << "numPt=" << numPt << endl;

  for (size_t i = 0; i < numPt; ++i) {
    const PhraseTable &pt = *system.mappings[i];
    pt.LookupUnary(GetPool(), *this, m_stacks, path);
  }

  /*
  size_t tpsNum = path.targetPhrases.GetSize();
  if (tpsNum) {
    cerr << tpsNum << " " << path << endl;
  }
  */
}

///////////////////////////////////////////////////////////////
// CUBE-PRUNING
///////////////////////////////////////////////////////////////
void Manager::Decode(SCFG::InputPath &path, Stack &stack)
{
  // clear cube pruning data
  //std::vector<QueueItem*> &container = Container(m_queue);
  //container.clear();
  Recycler<HypothesisBase*> &hypoRecycler = GetHypoRecycle();
  while (!m_queue.empty()) {
    QueueItem *item = m_queue.top();
    m_queue.pop();
    // recycle unused hypos from queue
    Hypothesis *hypo = item->hypo;
    hypoRecycler.Recycle(hypo);

    // recycle queue item
    m_queueItemRecycler.push_back(item);
  }

  m_seenPositions.clear();

  // init queue
  BOOST_FOREACH(const InputPath::Coll::value_type &valPair, *path.targetPhrases) {
    const SymbolBind &symbolBind = valPair.first;
    const SCFG::TargetPhrases &tps = *valPair.second;
    CreateQueue(path, symbolBind, tps);
  }

  // MAIN LOOP
  size_t pops = 0;
  while (!m_queue.empty() && pops < system.options.cube.pop_limit) {
    //cerr << "pops=" << pops << endl;
    QueueItem *item = m_queue.top();
    m_queue.pop();

    // add hypo to stack
    Hypothesis *hypo = item->hypo;

    //cerr << "hypo=" << *hypo << " " << endl;
    stack.Add(hypo, GetHypoRecycle(), arcLists);
    //cerr << "Added " << *hypo << " " << endl;

    item->CreateNext(GetSystemPool(), GetPool(), *this, m_queue, m_seenPositions, path);
    //cerr << "Created next " << endl;
    m_queueItemRecycler.push_back(item);

    ++pops;
  }

}

void Manager::CreateQueue(
    const SCFG::InputPath &path,
    const SymbolBind &symbolBind,
    const SCFG::TargetPhrases &tps)
{
  QueueItem *item = QueueItem::Create(GetPool(), *this);
  item->Init(GetPool(), symbolBind, tps);
  for (size_t i = 0; i < symbolBind.coll.size(); ++i) {
    const SymbolBindElement &ele = symbolBind.coll[i];
    if (ele.hypos) {
      item->AddHypos(*ele.hypos);
    }
  }
  item->CreateHypo(GetSystemPool(), *this, path, symbolBind);

  //cerr << "hypo=" << item->hypo->Debug(system) << endl;

  m_queue.push(item);
}

///////////////////////////////////////////////////////////////
// NON CUBE-PRUNING
///////////////////////////////////////////////////////////////
/*
void Manager::Decode(SCFG::InputPath &path, Stack &stack)
{
  //cerr << "path=" << path << endl;

  boost::unordered_map<SCFG::SymbolBind, SCFG::TargetPhrases*>::const_iterator iterOuter;
  for (iterOuter = path.targetPhrases->begin(); iterOuter != path.targetPhrases->end(); ++iterOuter) {
    const SCFG::SymbolBind &symbolBind = iterOuter->first;

    const SCFG::TargetPhrases &tps = *iterOuter->second;
    //cerr << "symbolBind=" << symbolBind << " tps=" << tps.GetSize() << endl;

    SCFG::TargetPhrases::const_iterator iter;
    for (iter = tps.begin(); iter != tps.end(); ++iter) {
      const SCFG::TargetPhraseImpl &tp = **iter;
      //cerr << "tp=" << tp << endl;
      ExpandHypo(path, symbolBind, tp, stack);
    }
  }
}
*/

void Manager::ExpandHypo(
    const SCFG::InputPath &path,
    const SCFG::SymbolBind &symbolBind,
    const SCFG::TargetPhraseImpl &tp,
    Stack &stack)
{
  Recycler<HypothesisBase*> &hypoRecycler = GetHypoRecycle();

  std::vector<const SymbolBindElement*> ntEles = symbolBind.GetNTElements();
  Vector<size_t> prevHyposIndices(GetPool(), symbolBind.numNT);
  assert(ntEles.size() == symbolBind.numNT);
  //cerr << "ntEles:" << ntEles.size() << endl;

  size_t ind = 0;
  while (IncrPrevHypoIndices(prevHyposIndices, ind, ntEles)) {
    SCFG::Hypothesis *hypo = SCFG::Hypothesis::Create(GetSystemPool(), *this);
    hypo->Init(*this, path, symbolBind, tp, prevHyposIndices);
    hypo->EvaluateWhenApplied();

    stack.Add(hypo, hypoRecycler, arcLists);

    ++ind;
  }
}

bool Manager::IncrPrevHypoIndices(
    Vector<size_t> &prevHyposIndices,
    size_t ind,
    const std::vector<const SymbolBindElement*> ntEles)
{
  if (ntEles.size() == 0) {
    // no nt. Do the 1st
    return ind ? false : true;
  }

  size_t numHypos = 0;

  //cerr << "IncrPrevHypoIndices:" << ind << " " << ntEles.size() << " ";
  for (size_t i = 0; i < ntEles.size() - 1; ++i) {
    const SymbolBindElement &ele = *ntEles[i];
    const Hypotheses &hypos = *ele.hypos;
    numHypos = hypos.size();

    std::div_t divRet = std::div((int)ind, (int)numHypos);
    ind = divRet.quot;

    size_t hypoInd = divRet.rem;
    prevHyposIndices[i] = hypoInd;
    //cerr << "(" << i << "," << ind << "," << numHypos << "," << hypoInd << ")";
  }

  // last
  prevHyposIndices.back() = ind;

  // check if last is over limit
  const SymbolBindElement &ele = *ntEles.back();
  const Hypotheses &hypos = *ele.hypos;
  numHypos = hypos.size();

  //cerr << "(" << (ntEles.size() - 1) << "," << ind << "," << numHypos << ","  << ind << ")";
  //cerr << endl;

  if (ind >= numHypos) {
    return false;
  }
  else {
    return true;
  }
}

std::string Manager::OutputBest() const
{
  string outStr;
  const Stack &lastStack = m_stacks.GetLastStack();
  const Hypothesis *bestHypo = lastStack.GetBestHypo(*this, const_cast<ArcLists&>(arcLists));

  if (bestHypo) {
    //cerr << "BEST TRANSLATION: " << bestHypo << bestHypo->Debug(system) << endl;
    //cerr << " " << out.str() << endl;
    stringstream out;
    bestHypo->OutputToStream(out);

    outStr = out.str();
    outStr = outStr.substr(4);
    outStr = outStr.substr(0, outStr.size() - 6);

    if (system.options.output.ReportHypoScore) {
      outStr = SPrint(bestHypo->GetScores().GetTotalScore()) + " " + outStr;
    }
  }
  else {
    if (system.options.output.ReportHypoScore) {
      outStr = "0 ";
    }

    //cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
  }

  outStr += "\n";
  return outStr;
}

} // namespace
}

<|endoftext|>


<commit_before>/* Copyright 2019 Argonne National Laboratory
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "am_mpi.h"

static MPI_Win g_am_win = MPI_WIN_NULL;
static void *g_am_base = NULL;
static __thread int thread_id = 0;
static __thread int am_seq = 0;
static Realm::atomic<unsigned int> num_threads(0);
static unsigned char buf_recv_list[AM_BUF_COUNT][1024];
static unsigned char *buf_recv = buf_recv_list[0];
static MPI_Request req_recv_list[AM_BUF_COUNT];
static int n_am_mult_recv = 5;
static int pre_initialized;
static int node_size;
static int node_this;
static MPI_Comm comm_medium;
int i_recv_list = 0;

namespace Realm {
namespace MPI {

#define AM_MSG_HEADER_SIZE 4 * sizeof(int)

void AM_Init(int *p_node_this, int *p_node_size)
{
    char *s;

    MPI_Initialized(&pre_initialized);
    if (pre_initialized) {
        int mpi_thread_model;
        MPI_Query_thread(&mpi_thread_model);
        assert(mpi_thread_model == MPI_THREAD_MULTIPLE);
    } else {
        int mpi_thread_model;
        MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &mpi_thread_model);
        assert(mpi_thread_model == MPI_THREAD_MULTIPLE);
    }
    MPI_Comm_size(MPI_COMM_WORLD, &node_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &node_this);
    *p_node_size = node_size;
    *p_node_this = node_this;

    s = getenv("AM_MULT_RECV");
    if (s) {
        n_am_mult_recv = atoi(s);
    }
    for (int  i = 0; i<n_am_mult_recv; i++) {
        CHECK_MPI( MPI_Irecv(buf_recv_list[i], 1024, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &req_recv_list[i]) );
    }
    MPI_Comm_dup(MPI_COMM_WORLD, &comm_medium);
}

void AM_Finalize()
{
    MPI_Request req_final;

    CHECK_MPI( MPI_Ibarrier(MPI_COMM_WORLD, &req_final) );
    while (1) {
        int is_done;
        MPI_Status status;
        CHECK_MPI( MPI_Test(&req_final, &is_done, &status) );
        if (req_final == MPI_REQUEST_NULL) {
            break;
        }
        AMPoll();
    }

    AMPoll_cancel();
    CHECK_MPI( MPI_Comm_free(&comm_medium) );

    if (!pre_initialized) {
        MPI_Finalize();
    }
}

void AM_init_long_messages(MPI_Win win, void *am_base)
{
    g_am_win = win;
    g_am_base = am_base;
}

void AMPoll()
{
    struct AM_msg *msg;
    int tn_src;

    while (1) {
        int got_am;
        MPI_Status status;
        CHECK_MPI( MPI_Test(&req_recv_list[i_recv_list], &got_am, &status) );
        if (!got_am) {
            break;
        }
        msg = (struct AM_msg *) buf_recv;

        tn_src = status.MPI_SOURCE;

        char *header;
        char *payload;
        int payload_type = 0;
        if (msg->type == 0) {
            header = msg->stuff;
            payload = msg->stuff + msg->header_size;
        } else if (msg->type == 1) {
            header = msg->stuff + 4;

            int msg_tag = *(int32_t *)(msg->stuff);
            payload = (char *) malloc(msg->payload_size);
            payload_type = 1;    // need_free;
            CHECK_MPI( MPI_Recv(payload, msg->payload_size, MPI_BYTE, tn_src, msg_tag, comm_medium, &status) );
        } else if (msg->type == 2) {
            int offset = *(int32_t *)(msg->stuff);
            header = msg->stuff + 4;
            payload = (char *) g_am_base + offset;
        }

        if (msg->msgid != 0x7fff) {
            Realm::ActiveMessageHandlerTable::MessageHandler handler = Realm::activemsg_handler_table.lookup_message_handler(msg->msgid);
            (*handler) (tn_src, header, payload, msg->payload_size);
        }

        if (payload_type == 1) {
            free(payload);
        }
        CHECK_MPI( MPI_Irecv(buf_recv_list[i_recv_list], 1024, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &req_recv_list[i_recv_list]) );
        i_recv_list = (i_recv_list + 1) % n_am_mult_recv;
        buf_recv = buf_recv_list[i_recv_list];
    }

}

void AMPoll_cancel()
{
    for (int  i = 0; i<n_am_mult_recv; i++) {
        CHECK_MPI( MPI_Cancel(&req_recv_list[i]) );
    }
}

void AMSend(int tgt, int msgid, int header_size, int payload_size, const char *header, const char *payload, int has_dest, MPI_Aint dest)
{
    char buf_send[1024];

    struct AM_msg *msg = (struct AM_msg *)(buf_send);
    msg->msgid = msgid;
    msg->header_size = header_size;
    msg->payload_size = payload_size;
    char *msg_header = msg->stuff;

    if (has_dest) {
        assert(g_am_win);
        CHECK_MPI( MPI_Put(payload, payload_size, MPI_BYTE, tgt, dest, payload_size, MPI_BYTE, g_am_win) );
        CHECK_MPI( MPI_Win_flush(tgt, g_am_win) );

        msg->type = 2;
        *((int32_t *) msg_header) = (int32_t) dest;
        memcpy(msg_header + 4, header, header_size);
        int n = AM_MSG_HEADER_SIZE + 4 + header_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_BYTE, tgt, 0x1, MPI_COMM_WORLD) );
    } else if (AM_MSG_HEADER_SIZE + header_size + payload_size < 1024) {
        msg->type = 0;
        if (header_size > 0) {
            memcpy(msg_header, header, header_size);
        }
        if (payload_size > 0) {
            memcpy(msg_header + header_size, payload, payload_size);
        }
        int n = AM_MSG_HEADER_SIZE + header_size + payload_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_CHAR, tgt, 0x1, MPI_COMM_WORLD) );
    } else {
        msg->type = 1;
        int msg_tag = 0x0;
        if (thread_id == 0) {
            thread_id = num_threads.fetch_add_acqrel(1) + 1;
        }
        am_seq = (am_seq + 1) & 0x1f;
        msg_tag = (thread_id << 10) + am_seq;
        *((int32_t *) msg_header) = (int32_t) msg_tag;
        memcpy(msg_header + 4, header, header_size);
        int n = AM_MSG_HEADER_SIZE + 4 + header_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_BYTE, tgt, 0x1, MPI_COMM_WORLD) );
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(payload, payload_size, MPI_BYTE, tgt, msg_tag, comm_medium) );

    }
}

} /* namespace MPI */
} /* namespace Realm */
<commit_msg>realm: add code path for invalid mpi message types<commit_after>/* Copyright 2019 Argonne National Laboratory
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "am_mpi.h"

static MPI_Win g_am_win = MPI_WIN_NULL;
static void *g_am_base = NULL;
static __thread int thread_id = 0;
static __thread int am_seq = 0;
static Realm::atomic<unsigned int> num_threads(0);
static unsigned char buf_recv_list[AM_BUF_COUNT][1024];
static unsigned char *buf_recv = buf_recv_list[0];
static MPI_Request req_recv_list[AM_BUF_COUNT];
static int n_am_mult_recv = 5;
static int pre_initialized;
static int node_size;
static int node_this;
static MPI_Comm comm_medium;
int i_recv_list = 0;

namespace Realm {
namespace MPI {

#define AM_MSG_HEADER_SIZE 4 * sizeof(int)

void AM_Init(int *p_node_this, int *p_node_size)
{
    char *s;

    MPI_Initialized(&pre_initialized);
    if (pre_initialized) {
        int mpi_thread_model;
        MPI_Query_thread(&mpi_thread_model);
        assert(mpi_thread_model == MPI_THREAD_MULTIPLE);
    } else {
        int mpi_thread_model;
        MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &mpi_thread_model);
        assert(mpi_thread_model == MPI_THREAD_MULTIPLE);
    }
    MPI_Comm_size(MPI_COMM_WORLD, &node_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &node_this);
    *p_node_size = node_size;
    *p_node_this = node_this;

    s = getenv("AM_MULT_RECV");
    if (s) {
        n_am_mult_recv = atoi(s);
    }
    for (int  i = 0; i<n_am_mult_recv; i++) {
        CHECK_MPI( MPI_Irecv(buf_recv_list[i], 1024, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &req_recv_list[i]) );
    }
    MPI_Comm_dup(MPI_COMM_WORLD, &comm_medium);
}

void AM_Finalize()
{
    MPI_Request req_final;

    CHECK_MPI( MPI_Ibarrier(MPI_COMM_WORLD, &req_final) );
    while (1) {
        int is_done;
        MPI_Status status;
        CHECK_MPI( MPI_Test(&req_final, &is_done, &status) );
        if (req_final == MPI_REQUEST_NULL) {
            break;
        }
        AMPoll();
    }

    AMPoll_cancel();
    CHECK_MPI( MPI_Comm_free(&comm_medium) );

    if (!pre_initialized) {
        MPI_Finalize();
    }
}

void AM_init_long_messages(MPI_Win win, void *am_base)
{
    g_am_win = win;
    g_am_base = am_base;
}

void AMPoll()
{
    struct AM_msg *msg;
    int tn_src;

    while (1) {
        int got_am;
        MPI_Status status;
        CHECK_MPI( MPI_Test(&req_recv_list[i_recv_list], &got_am, &status) );
        if (!got_am) {
            break;
        }
        msg = (struct AM_msg *) buf_recv;

        tn_src = status.MPI_SOURCE;

        char *header;
        char *payload;
        int payload_type = 0;
        if (msg->type == 0) {
            header = msg->stuff;
            payload = msg->stuff + msg->header_size;
        } else if (msg->type == 1) {
            header = msg->stuff + 4;

            int msg_tag = *(int32_t *)(msg->stuff);
            payload = (char *) malloc(msg->payload_size);
            payload_type = 1;    // need_free;
            CHECK_MPI( MPI_Recv(payload, msg->payload_size, MPI_BYTE, tn_src, msg_tag, comm_medium, &status) );
        } else if (msg->type == 2) {
            int offset = *(int32_t *)(msg->stuff);
            header = msg->stuff + 4;
            payload = (char *) g_am_base + offset;
        } else {
            assert(0 && "invalid message type");
            header = 0;
            payload = 0;
            msg->msgid = 0x7fff; // skips handler below
        }

        if (msg->msgid != 0x7fff) {
            Realm::ActiveMessageHandlerTable::MessageHandler handler = Realm::activemsg_handler_table.lookup_message_handler(msg->msgid);
            (*handler) (tn_src, header, payload, msg->payload_size);
        }

        if (payload_type == 1) {
            free(payload);
        }
        CHECK_MPI( MPI_Irecv(buf_recv_list[i_recv_list], 1024, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &req_recv_list[i_recv_list]) );
        i_recv_list = (i_recv_list + 1) % n_am_mult_recv;
        buf_recv = buf_recv_list[i_recv_list];
    }

}

void AMPoll_cancel()
{
    for (int  i = 0; i<n_am_mult_recv; i++) {
        CHECK_MPI( MPI_Cancel(&req_recv_list[i]) );
    }
}

void AMSend(int tgt, int msgid, int header_size, int payload_size, const char *header, const char *payload, int has_dest, MPI_Aint dest)
{
    char buf_send[1024];

    struct AM_msg *msg = (struct AM_msg *)(buf_send);
    msg->msgid = msgid;
    msg->header_size = header_size;
    msg->payload_size = payload_size;
    char *msg_header = msg->stuff;

    if (has_dest) {
        assert(g_am_win);
        CHECK_MPI( MPI_Put(payload, payload_size, MPI_BYTE, tgt, dest, payload_size, MPI_BYTE, g_am_win) );
        CHECK_MPI( MPI_Win_flush(tgt, g_am_win) );

        msg->type = 2;
        *((int32_t *) msg_header) = (int32_t) dest;
        memcpy(msg_header + 4, header, header_size);
        int n = AM_MSG_HEADER_SIZE + 4 + header_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_BYTE, tgt, 0x1, MPI_COMM_WORLD) );
    } else if (AM_MSG_HEADER_SIZE + header_size + payload_size < 1024) {
        msg->type = 0;
        if (header_size > 0) {
            memcpy(msg_header, header, header_size);
        }
        if (payload_size > 0) {
            memcpy(msg_header + header_size, payload, payload_size);
        }
        int n = AM_MSG_HEADER_SIZE + header_size + payload_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_CHAR, tgt, 0x1, MPI_COMM_WORLD) );
    } else {
        msg->type = 1;
        int msg_tag = 0x0;
        if (thread_id == 0) {
            thread_id = num_threads.fetch_add_acqrel(1) + 1;
        }
        am_seq = (am_seq + 1) & 0x1f;
        msg_tag = (thread_id << 10) + am_seq;
        *((int32_t *) msg_header) = (int32_t) msg_tag;
        memcpy(msg_header + 4, header, header_size);
        int n = AM_MSG_HEADER_SIZE + 4 + header_size;
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(buf_send, n, MPI_BYTE, tgt, 0x1, MPI_COMM_WORLD) );
        assert(tgt != node_this);
        CHECK_MPI( MPI_Send(payload, payload_size, MPI_BYTE, tgt, msg_tag, comm_medium) );

    }
}

} /* namespace MPI */
} /* namespace Realm */
<|endoftext|>


<commit_before>// Copyright 2017-2020 The Verible Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "verilog/analysis/verilog_project.h"

#include <iostream>
#include <string>
#include <vector>

#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "absl/types/optional.h"
#include "common/text/text_structure.h"
#include "common/util/file_util.h"
#include "common/util/logging.h"
#include "verilog/analysis/verilog_analyzer.h"

namespace verilog {

// All files we process with the verilog project, essentially applications that
// build a symbol table (project-tool, kythe-indexer) only benefit from
// processing the same sequence of tokens a synthesis tool sees.
static constexpr verilog::VerilogPreprocess::Config kPreprocessConfig{
    .filter_branches = true,
};

VerilogSourceFile::VerilogSourceFile(absl::string_view referenced_path,
                                     const absl::Status& status)
    : referenced_path_(referenced_path), status_(status) {}

absl::Status VerilogSourceFile::Open() {
  // Don't re-open.  analyzed_structure_ should be set/written once only.
  if (state_ != State::kInitialized) return status_;

  // Load file contents.
  std::string content;
  status_ = verible::file::GetContents(ResolvedPath(), &content);
  if (!status_.ok()) return status_;

  // TODO(fangism): std::move or memory-map to avoid a short-term copy.
  analyzed_structure_ = std::make_unique<VerilogAnalyzer>(
      content, ResolvedPath(), kPreprocessConfig);
  state_ = State::kOpened;
  // status_ is Ok here.
  return status_;
}

absl::Status VerilogSourceFile::Parse() {
  // Parsed state is cached.
  if (state_ == State::kParsed) return status_;

  // Open file and load contents if not already done.
  status_ = Open();
  if (!status_.ok()) return status_;

  // Lex, parse, populate underlying TextStructureView.
  const absl::Time analyze_start = absl::Now();
  status_ = analyzed_structure_->Analyze();
  LOG(INFO) << "Analyzed " << ResolvedPath() << " in "
            << absl::ToInt64Milliseconds(absl::Now() - analyze_start) << "ms";
  state_ = State::kParsed;
  return status_;
}

const verible::TextStructureView* VerilogSourceFile::GetTextStructure() const {
  if (analyzed_structure_ == nullptr) return nullptr;
  return &analyzed_structure_->Data();
}

std::vector<std::string> VerilogSourceFile::ErrorMessages() const {
  std::vector<std::string> result;
  if (!analyzed_structure_) return result;
  result = analyzed_structure_->LinterTokenErrorMessages(false);
  return result;
}

std::ostream& operator<<(std::ostream& stream,
                         const VerilogSourceFile& source) {
  stream << "referenced path: " << source.ReferencedPath() << std::endl;
  stream << "resolved path: " << source.ResolvedPath() << std::endl;
  stream << "corpus: " << source.Corpus() << std::endl;
  const auto status = source.Status();
  stream << "status: " << (status.ok() ? "ok" : status.message()) << std::endl;
  const auto* text_structure = source.GetTextStructure();
  stream << "have text structure? "
         << ((text_structure != nullptr) ? "yes" : "no") << std::endl;
  return stream;
}

absl::Status InMemoryVerilogSourceFile::Open() {
  analyzed_structure_ = ABSL_DIE_IF_NULL(std::make_unique<VerilogAnalyzer>(
      contents_for_open_, ResolvedPath(), kPreprocessConfig));
  state_ = State::kOpened;
  status_ = absl::OkStatus();
  return status_;
}

absl::Status ParsedVerilogSourceFile::Open() {
  state_ = State::kOpened;
  status_ = absl::OkStatus();
  return status_;
}

absl::Status ParsedVerilogSourceFile::Parse() {
  state_ = State::kParsed;
  status_ = absl::OkStatus();
  return status_;
}

const verible::TextStructureView* ParsedVerilogSourceFile::GetTextStructure()
    const {
  return text_structure_;
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenFile(
    absl::string_view referenced_filename, absl::string_view resolved_filename,
    absl::string_view corpus) {
  const auto inserted = files_.emplace(
      referenced_filename, absl::make_unique<VerilogSourceFile>(
                               referenced_filename, resolved_filename, corpus));
  CHECK(inserted.second);  // otherwise, would have already returned above
  const auto file_iter = inserted.first;
  VerilogSourceFile& file(*file_iter->second);

  // Read the file's contents.
  const absl::Status status = file.Open();
  if (!status.ok()) return status;

  // NOTE: string view maps don't support removal operation. The following block
  // is valid only if files won't be removed from the project.
  if (populate_string_maps_) {
    const absl::string_view contents(file.GetTextStructure()->Contents());

    // Register the file's contents range in string_view_map_.
    string_view_map_.must_emplace(contents);

    // Map the start of the file's contents to its corresponding owner
    // VerilogSourceFile.
    const auto map_inserted =
        buffer_to_analyzer_map_.emplace(contents.begin(), file_iter);
    CHECK(map_inserted.second);
  }

  return &file;
}

bool VerilogProject::RemoveRegisteredFile(
    absl::string_view referenced_filename) {
  CHECK(!populate_string_maps_)
      << "Removing of files added to string maps is not supported! Disable "
         "populating string maps.";
  if (files_.erase(std::string(referenced_filename)) == 1) {
    LOG(INFO) << "Removed " << referenced_filename << " from the project.";
    return true;
  }
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    if (files_.erase(resolved_filename) == 1) {
      LOG(INFO) << "Removed " << resolved_filename << " from the project.";
      return true;
    }
  }
  return false;
}

VerilogSourceFile* VerilogProject::LookupRegisteredFileInternal(
    absl::string_view referenced_filename) const {
  const auto opened_file = FindOpenedFile(referenced_filename);
  if (opened_file) {
    if (!opened_file->ok()) {
      return nullptr;
    }
    return opened_file->value();
  }

  // Check if this is already opened include file
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return opened_file->ok() ? opened_file->value() : nullptr;
    }
  }
  return nullptr;
}

absl::optional<absl::StatusOr<VerilogSourceFile*>>
VerilogProject::FindOpenedFile(absl::string_view filename) const {
  const auto found = files_.find(filename);
  if (found != files_.end()) {
    const auto status = found->second->Status();
    if (!status.ok()) return status;
    return found->second.get();
  }
  return absl::nullopt;
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenTranslationUnit(
    absl::string_view referenced_filename) {
  // Check for a pre-existing entry to avoid duplicate files.
  {
    const auto opened_file = FindOpenedFile(referenced_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Locate the file among the base paths.
  const std::string resolved_filename =
      verible::file::JoinPath(TranslationUnitRoot(), referenced_filename);
  // Check if this is already opened file
  {
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  return OpenFile(referenced_filename, resolved_filename, Corpus());
}

absl::Status VerilogProject::IncludeFileNotFoundError(
    absl::string_view referenced_filename) const {
  return absl::NotFoundError(absl::StrCat(
      "Unable to find '", referenced_filename,
      "' among the included paths: ", absl::StrJoin(include_paths_, ", ")));
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenIncludedFile(
    absl::string_view referenced_filename) {
  VLOG(1) << __FUNCTION__ << ", referenced: " << referenced_filename;
  // Check for a pre-existing entry to avoid duplicate files.
  {
    const auto opened_file = FindOpenedFile(referenced_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Check if this is already opened include file
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Locate the file among the base paths.
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    if (verible::file::FileExists(resolved_filename).ok()) {
      VLOG(2) << "File '" << resolved_filename << "' exists. Resolved from '"
              << referenced_filename << "'";
      return OpenFile(referenced_filename, resolved_filename, Corpus());
    }
    VLOG(2) << "Checked for file'" << resolved_filename
            << "', but not found. Resolved from '" << referenced_filename
            << "'";
  }

  // Not found in any path.  Cache this status.
  const auto inserted = files_.emplace(
      referenced_filename,
      absl::make_unique<VerilogSourceFile>(
          referenced_filename, IncludeFileNotFoundError(referenced_filename)));
  CHECK(inserted.second) << "Not-found file should have been recorded as such.";
  return inserted.first->second->Status();
}

void VerilogProject::AddVirtualFile(absl::string_view resolved_filename,
                                    absl::string_view content) {
  const auto inserted = files_.emplace(
      resolved_filename, absl::make_unique<InMemoryVerilogSourceFile>(
                             resolved_filename, content, /*corpus=*/""));
  CHECK(inserted.second);
}

const VerilogSourceFile* VerilogProject::LookupFileOrigin(
    absl::string_view content_substring) const {
  // Look for corresponding source text (superstring) buffer start.
  const auto found_superstring = string_view_map_.find(content_substring);
  if (found_superstring == string_view_map_.end()) return nullptr;
  const absl::string_view::const_iterator buffer_start =
      found_superstring->first;

  // Reverse-lookup originating file based on buffer start.
  const auto found_file = buffer_to_analyzer_map_.find(buffer_start);
  if (found_file == buffer_to_analyzer_map_.end()) return nullptr;

  const VerilogSourceFile* file = found_file->second->second.get();
  return file;
}

}  // namespace verilog
<commit_msg>require enabling string view population before looking up the file origin<commit_after>// Copyright 2017-2020 The Verible Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "verilog/analysis/verilog_project.h"

#include <iostream>
#include <string>
#include <vector>

#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "absl/types/optional.h"
#include "common/text/text_structure.h"
#include "common/util/file_util.h"
#include "common/util/logging.h"
#include "verilog/analysis/verilog_analyzer.h"

namespace verilog {

// All files we process with the verilog project, essentially applications that
// build a symbol table (project-tool, kythe-indexer) only benefit from
// processing the same sequence of tokens a synthesis tool sees.
static constexpr verilog::VerilogPreprocess::Config kPreprocessConfig{
    .filter_branches = true,
};

VerilogSourceFile::VerilogSourceFile(absl::string_view referenced_path,
                                     const absl::Status& status)
    : referenced_path_(referenced_path), status_(status) {}

absl::Status VerilogSourceFile::Open() {
  // Don't re-open.  analyzed_structure_ should be set/written once only.
  if (state_ != State::kInitialized) return status_;

  // Load file contents.
  std::string content;
  status_ = verible::file::GetContents(ResolvedPath(), &content);
  if (!status_.ok()) return status_;

  // TODO(fangism): std::move or memory-map to avoid a short-term copy.
  analyzed_structure_ = std::make_unique<VerilogAnalyzer>(
      content, ResolvedPath(), kPreprocessConfig);
  state_ = State::kOpened;
  // status_ is Ok here.
  return status_;
}

absl::Status VerilogSourceFile::Parse() {
  // Parsed state is cached.
  if (state_ == State::kParsed) return status_;

  // Open file and load contents if not already done.
  status_ = Open();
  if (!status_.ok()) return status_;

  // Lex, parse, populate underlying TextStructureView.
  const absl::Time analyze_start = absl::Now();
  status_ = analyzed_structure_->Analyze();
  LOG(INFO) << "Analyzed " << ResolvedPath() << " in "
            << absl::ToInt64Milliseconds(absl::Now() - analyze_start) << "ms";
  state_ = State::kParsed;
  return status_;
}

const verible::TextStructureView* VerilogSourceFile::GetTextStructure() const {
  if (analyzed_structure_ == nullptr) return nullptr;
  return &analyzed_structure_->Data();
}

std::vector<std::string> VerilogSourceFile::ErrorMessages() const {
  std::vector<std::string> result;
  if (!analyzed_structure_) return result;
  result = analyzed_structure_->LinterTokenErrorMessages(false);
  return result;
}

std::ostream& operator<<(std::ostream& stream,
                         const VerilogSourceFile& source) {
  stream << "referenced path: " << source.ReferencedPath() << std::endl;
  stream << "resolved path: " << source.ResolvedPath() << std::endl;
  stream << "corpus: " << source.Corpus() << std::endl;
  const auto status = source.Status();
  stream << "status: " << (status.ok() ? "ok" : status.message()) << std::endl;
  const auto* text_structure = source.GetTextStructure();
  stream << "have text structure? "
         << ((text_structure != nullptr) ? "yes" : "no") << std::endl;
  return stream;
}

absl::Status InMemoryVerilogSourceFile::Open() {
  analyzed_structure_ = ABSL_DIE_IF_NULL(std::make_unique<VerilogAnalyzer>(
      contents_for_open_, ResolvedPath(), kPreprocessConfig));
  state_ = State::kOpened;
  status_ = absl::OkStatus();
  return status_;
}

absl::Status ParsedVerilogSourceFile::Open() {
  state_ = State::kOpened;
  status_ = absl::OkStatus();
  return status_;
}

absl::Status ParsedVerilogSourceFile::Parse() {
  state_ = State::kParsed;
  status_ = absl::OkStatus();
  return status_;
}

const verible::TextStructureView* ParsedVerilogSourceFile::GetTextStructure()
    const {
  return text_structure_;
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenFile(
    absl::string_view referenced_filename, absl::string_view resolved_filename,
    absl::string_view corpus) {
  const auto inserted = files_.emplace(
      referenced_filename, absl::make_unique<VerilogSourceFile>(
                               referenced_filename, resolved_filename, corpus));
  CHECK(inserted.second);  // otherwise, would have already returned above
  const auto file_iter = inserted.first;
  VerilogSourceFile& file(*file_iter->second);

  // Read the file's contents.
  const absl::Status status = file.Open();
  if (!status.ok()) return status;

  // NOTE: string view maps don't support removal operation. The following block
  // is valid only if files won't be removed from the project.
  if (populate_string_maps_) {
    const absl::string_view contents(file.GetTextStructure()->Contents());

    // Register the file's contents range in string_view_map_.
    string_view_map_.must_emplace(contents);

    // Map the start of the file's contents to its corresponding owner
    // VerilogSourceFile.
    const auto map_inserted =
        buffer_to_analyzer_map_.emplace(contents.begin(), file_iter);
    CHECK(map_inserted.second);
  }

  return &file;
}

bool VerilogProject::RemoveRegisteredFile(
    absl::string_view referenced_filename) {
  CHECK(!populate_string_maps_)
      << "Removing of files added to string maps is not supported! Disable "
         "populating string maps.";
  if (files_.erase(std::string(referenced_filename)) == 1) {
    LOG(INFO) << "Removed " << referenced_filename << " from the project.";
    return true;
  }
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    if (files_.erase(resolved_filename) == 1) {
      LOG(INFO) << "Removed " << resolved_filename << " from the project.";
      return true;
    }
  }
  return false;
}

VerilogSourceFile* VerilogProject::LookupRegisteredFileInternal(
    absl::string_view referenced_filename) const {
  const auto opened_file = FindOpenedFile(referenced_filename);
  if (opened_file) {
    if (!opened_file->ok()) {
      return nullptr;
    }
    return opened_file->value();
  }

  // Check if this is already opened include file
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return opened_file->ok() ? opened_file->value() : nullptr;
    }
  }
  return nullptr;
}

absl::optional<absl::StatusOr<VerilogSourceFile*>>
VerilogProject::FindOpenedFile(absl::string_view filename) const {
  const auto found = files_.find(filename);
  if (found != files_.end()) {
    const auto status = found->second->Status();
    if (!status.ok()) return status;
    return found->second.get();
  }
  return absl::nullopt;
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenTranslationUnit(
    absl::string_view referenced_filename) {
  // Check for a pre-existing entry to avoid duplicate files.
  {
    const auto opened_file = FindOpenedFile(referenced_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Locate the file among the base paths.
  const std::string resolved_filename =
      verible::file::JoinPath(TranslationUnitRoot(), referenced_filename);
  // Check if this is already opened file
  {
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  return OpenFile(referenced_filename, resolved_filename, Corpus());
}

absl::Status VerilogProject::IncludeFileNotFoundError(
    absl::string_view referenced_filename) const {
  return absl::NotFoundError(absl::StrCat(
      "Unable to find '", referenced_filename,
      "' among the included paths: ", absl::StrJoin(include_paths_, ", ")));
}

absl::StatusOr<VerilogSourceFile*> VerilogProject::OpenIncludedFile(
    absl::string_view referenced_filename) {
  VLOG(1) << __FUNCTION__ << ", referenced: " << referenced_filename;
  // Check for a pre-existing entry to avoid duplicate files.
  {
    const auto opened_file = FindOpenedFile(referenced_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Check if this is already opened include file
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    const auto opened_file = FindOpenedFile(resolved_filename);
    if (opened_file) {
      return *opened_file;
    }
  }

  // Locate the file among the base paths.
  for (const auto& include_path : include_paths_) {
    const std::string resolved_filename =
        verible::file::JoinPath(include_path, referenced_filename);
    if (verible::file::FileExists(resolved_filename).ok()) {
      VLOG(2) << "File '" << resolved_filename << "' exists. Resolved from '"
              << referenced_filename << "'";
      return OpenFile(referenced_filename, resolved_filename, Corpus());
    }
    VLOG(2) << "Checked for file'" << resolved_filename
            << "', but not found. Resolved from '" << referenced_filename
            << "'";
  }

  // Not found in any path.  Cache this status.
  const auto inserted = files_.emplace(
      referenced_filename,
      absl::make_unique<VerilogSourceFile>(
          referenced_filename, IncludeFileNotFoundError(referenced_filename)));
  CHECK(inserted.second) << "Not-found file should have been recorded as such.";
  return inserted.first->second->Status();
}

void VerilogProject::AddVirtualFile(absl::string_view resolved_filename,
                                    absl::string_view content) {
  const auto inserted = files_.emplace(
      resolved_filename, absl::make_unique<InMemoryVerilogSourceFile>(
                             resolved_filename, content, /*corpus=*/""));
  CHECK(inserted.second);
}

const VerilogSourceFile* VerilogProject::LookupFileOrigin(
    absl::string_view content_substring) const {
  CHECK(populate_string_maps_)
      << "Populating string maps must be enabled for LookupFileOrigin!";
  // Look for corresponding source text (superstring) buffer start.
  const auto found_superstring = string_view_map_.find(content_substring);
  if (found_superstring == string_view_map_.end()) return nullptr;
  const absl::string_view::const_iterator buffer_start =
      found_superstring->first;

  // Reverse-lookup originating file based on buffer start.
  const auto found_file = buffer_to_analyzer_map_.find(buffer_start);
  if (found_file == buffer_to_analyzer_map_.end()) return nullptr;

  const VerilogSourceFile* file = found_file->second->second.get();
  return file;
}

}  // namespace verilog
<|endoftext|>


<commit_before>#include "testApp.h"

bool locked = false;

//--------------------------------------------------------------
void testApp::setup(){
    ofSetFrameRate(30);
    loadCameras();
    rs = new ofxRandomSampler(ipcams.size());
    // initialize connection
    for(int i = 0; i < NUM_CAMERAS; i++) {
        IPCameraDef& cam = getRandomCamera();

        // if your camera uses standard web-based authentication, use this
        ipGrabber[i].setUsername(cam.username);
        ipGrabber[i].setPassword(cam.password);

        // if your camera uses cookies for authentication, use something like this:
        // ipGrabber[i].setCookie("user", cam.username);
        // ipGrabber[i].setCookie("password", cam.password);

        URI uri(cam.uri);
        ipGrabber[i].setURI(uri);
        ipGrabber[i].connect();
        // set up the listener!
        ofAddListener(ipGrabber[i].videoResized, this, &testApp::videoResized);
    }
}

//--------------------------------------------------------------
IPCameraDef& testApp::getRandomCamera() {
    return ipcams[rs->next()]; // get the next camera, sampling w/o replacement
}

//--------------------------------------------------------------
void testApp::loadCameras() {

    // all of these cameras were found using this google query
    // http://www.google.com/search?q=inurl%3A%22axis-cgi%2Fmjpg%22
    // some of the cameras below may no longer be valid.

    // to define a camera with a username / password
    //ipcams.push_back(IPCameraDef("http://148.61.142.228/axis-cgi/mjpg/video.cgi", "username", "password"));

    ipcams.push_back(IPCameraDef("http://148.61.142.228/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://82.79.176.85:8081/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://81.8.151.136:88/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://130.15.110.15/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://80.34.88.249/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://216.8.159.21/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://kassertheatercam.montclair.edu/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://74.94.55.182/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://213.77.33.2:8080/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://129.89.28.32/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://129.171.176.150/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://134.29.208.43/mjpg/video.mjpg?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://134.29.208.43/mjpg/video.mjpg?resolution=320x240"));

}

//--------------------------------------------------------------
void testApp::videoResized(const void * sender, ofResizeEventArgs& arg) {

    ofLog(OF_LOG_VERBOSE, "testApp::videoResized: A camera was resized.");

    // find the camera that sent the resize event changed
    for(int i = 0; i < NUM_CAMERAS; i++) {
        if(sender == &ipGrabber[i]) {
            stringstream ss;
            ss << "testApp::videoResized: ";
            ss << "Camera connected to: " << ipGrabber[i].getURI() + " ";
            ss << "New DIM = " << arg.width << "/" << arg.height;
            ofLog(OF_LOG_VERBOSE, ss.str());
        }
    }
    ofLog(OF_LOG_VERBOSE, "Unable to locate the camera.  Very odd.");
}

//--------------------------------------------------------------
void testApp::update(){
    // update the cameras
    for(int i = 0; i < NUM_CAMERAS; i++) {
        ipGrabber[i].update();
    }
}

//--------------------------------------------------------------
void testApp::draw(){
	ofBackground(0,0,0);

	ofSetHexColor(0xffffff);

    int row = 0;
    int col = 0;

    int x = 0;
    int y = 0;

    int w = 320;
    int h = 240;

    float totalKBPS = 0;
    float totalFPS = 0;

    for(int i = 0; i < NUM_CAMERAS; i++) {
        x = col * w;
        y = row * h;

        // draw in a grid
        row = (row + 1) % NUM_ROWS;
        if(row == 0) {
            col = (col + 1) % NUM_COLS;
        }

        ofPushMatrix();
        ofTranslate(x,y);
        ofSetColor(255,255,255,255);
        ipGrabber[i].draw(0,0,w,h); // draw the camera

        ofEnableAlphaBlending();

        // draw the info box
        ofSetColor(0,0,0,127);
        ofRect(5,h-47,w-10,42);

        float kbps = ipGrabber[i].getBitRate() / (8 * 1000.0);
        totalKBPS+=kbps;

        float fps = ipGrabber[i].getFrameRate();
        totalFPS+=fps;

        ofSetColor(255);
        ofDrawBitmapString("HOST: " + ipGrabber[i].getHost(), 10, h-34);
        ofDrawBitmapString(" FPS: " + ofToString(fps, 2,7,' '), 10, h-22);
        ofDrawBitmapString("KB/S: " + ofToString(kbps, 2,7,' '), 10, h-10);

        ofDisableAlphaBlending();

        ofPopMatrix();
    }

    // keep track of some totals
    float avgFPS = totalFPS / NUM_CAMERAS;
    float avgKBPS = totalKBPS / NUM_CAMERAS;

    ofEnableAlphaBlending();
    ofSetColor(0,80);
    ofRect(5,5, 150, 40);

    ofSetColor(255);
    ofDrawBitmapString(" AVG FPS: " + ofToString(avgFPS,2,7,' '), 10,17);
    ofDrawBitmapString("AVG KB/S: " + ofToString(avgKBPS,2,7,' '), 10,29);
    ofDrawBitmapString("TOT KB/S: " + ofToString(totalKBPS,2,7,' '), 10,41);
    ofDisableAlphaBlending();

}

//--------------------------------------------------------------
void testApp::keyPressed  (int key){
    if(key == ' ') {

        rs->reset();
        // initialize connection
        for(int i = 0; i < NUM_CAMERAS; i++) {
            ipGrabber[i].waitForDisconnect();
            // we may need to wait a sec here
            IPCameraDef& cam = getRandomCamera();
            ipGrabber[i].setUsername(cam.username);
            ipGrabber[i].setPassword(cam.password);
            URI uri(cam.uri);
            ipGrabber[i].setURI(uri);
            ipGrabber[i].connect();
        }
    }
}

//--------------------------------------------------------------
void testApp::keyReleased(int key){

}

//--------------------------------------------------------------
void testApp::mouseMoved(int x, int y ){
}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::mouseReleased(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void testApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void testApp::dragEvent(ofDragInfo dragInfo){

}
<commit_msg>Fix data rate calc and units.<commit_after>#include "testApp.h"

bool locked = false;

//--------------------------------------------------------------
void testApp::setup(){
    ofSetFrameRate(30);
    loadCameras();
    rs = new ofxRandomSampler(ipcams.size());
    // initialize connection
    for(int i = 0; i < NUM_CAMERAS; i++) {
        IPCameraDef& cam = getRandomCamera();

        // if your camera uses standard web-based authentication, use this
        ipGrabber[i].setUsername(cam.username);
        ipGrabber[i].setPassword(cam.password);

        // if your camera uses cookies for authentication, use something like this:
        // ipGrabber[i].setCookie("user", cam.username);
        // ipGrabber[i].setCookie("password", cam.password);

        URI uri(cam.uri);
        ipGrabber[i].setURI(uri);
        ipGrabber[i].connect();
        // set up the listener!
        ofAddListener(ipGrabber[i].videoResized, this, &testApp::videoResized);
    }
}

//--------------------------------------------------------------
IPCameraDef& testApp::getRandomCamera() {
    return ipcams[rs->next()]; // get the next camera, sampling w/o replacement
}

//--------------------------------------------------------------
void testApp::loadCameras() {

    // all of these cameras were found using this google query
    // http://www.google.com/search?q=inurl%3A%22axis-cgi%2Fmjpg%22
    // some of the cameras below may no longer be valid.

    // to define a camera with a username / password
    //ipcams.push_back(IPCameraDef("http://148.61.142.228/axis-cgi/mjpg/video.cgi", "username", "password"));

    ipcams.push_back(IPCameraDef("http://148.61.142.228/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://82.79.176.85:8081/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://81.8.151.136:88/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://130.15.110.15/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://80.34.88.249/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://216.8.159.21/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://kassertheatercam.montclair.edu/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://74.94.55.182/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://213.77.33.2:8080/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://129.89.28.32/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://129.171.176.150/axis-cgi/mjpg/video.cgi?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://134.29.208.43/mjpg/video.mjpg?resolution=320x240"));
    ipcams.push_back(IPCameraDef("http://134.29.208.43/mjpg/video.mjpg?resolution=320x240"));

}

//--------------------------------------------------------------
void testApp::videoResized(const void * sender, ofResizeEventArgs& arg) {

    ofLog(OF_LOG_VERBOSE, "testApp::videoResized: A camera was resized.");

    // find the camera that sent the resize event changed
    for(int i = 0; i < NUM_CAMERAS; i++) {
        if(sender == &ipGrabber[i]) {
            stringstream ss;
            ss << "testApp::videoResized: ";
            ss << "Camera connected to: " << ipGrabber[i].getURI() + " ";
            ss << "New DIM = " << arg.width << "/" << arg.height;
            ofLog(OF_LOG_VERBOSE, ss.str());
        }
    }
    ofLog(OF_LOG_VERBOSE, "Unable to locate the camera.  Very odd.");
}

//--------------------------------------------------------------
void testApp::update(){
    // update the cameras
    for(int i = 0; i < NUM_CAMERAS; i++) {
        ipGrabber[i].update();
    }
}

//--------------------------------------------------------------
void testApp::draw(){
	ofBackground(0,0,0);

	ofSetHexColor(0xffffff);

    int row = 0;
    int col = 0;

    int x = 0;
    int y = 0;

    int w = 320;
    int h = 240;

    float totalKbps = 0;
    float totalFPS = 0;

    for(int i = 0; i < NUM_CAMERAS; i++) {
        x = col * w;
        y = row * h;

        // draw in a grid
        row = (row + 1) % NUM_ROWS;
        if(row == 0) {
            col = (col + 1) % NUM_COLS;
        }

        ofPushMatrix();
        ofTranslate(x,y);
        ofSetColor(255,255,255,255);
        ipGrabber[i].draw(0,0,w,h); // draw the camera

        ofEnableAlphaBlending();

        // draw the info box
        ofSetColor(0,0,0,127);
        ofRect(5,h-47,w-10,42);

        float kbps = ipGrabber[i].getBitRate() / 1000.0f; // kilobits / second, not kibibits / second
        totalKbps+=kbps;

        float fps = ipGrabber[i].getFrameRate();
        totalFPS+=fps;

        ofSetColor(255);
        ofDrawBitmapString("HOST: " + ipGrabber[i].getHost(), 10, h-34);
        ofDrawBitmapString(" FPS: " + ofToString(fps, 2,7,' '), 10, h-22);
        ofDrawBitmapString("Kb/S: " + ofToString(kbps, 2,7,' '), 10, h-10);

        ofDisableAlphaBlending();

        ofPopMatrix();
    }

    // keep track of some totals
    float avgFPS = totalFPS / NUM_CAMERAS;
    float avgKbps = totalKbps / NUM_CAMERAS;

    ofEnableAlphaBlending();
    ofSetColor(0,80);
    ofRect(5,5, 150, 40);

    ofSetColor(255);
    ofDrawBitmapString(" AVG FPS: " + ofToString(avgFPS,2,7,' '), 10,17);
    ofDrawBitmapString("AVG Kb/S: " + ofToString(avgKbps,2,7,' '), 10,29);
    ofDrawBitmapString("TOT Kb/S: " + ofToString(totalKbps,2,7,' '), 10,41);
    ofDisableAlphaBlending();

}

//--------------------------------------------------------------
void testApp::keyPressed  (int key){
    if(key == ' ') {

        rs->reset();
        // initialize connection
        for(int i = 0; i < NUM_CAMERAS; i++) {
            ipGrabber[i].waitForDisconnect();
            // we may need to wait a sec here
            IPCameraDef& cam = getRandomCamera();
            ipGrabber[i].setUsername(cam.username);
            ipGrabber[i].setPassword(cam.password);
            URI uri(cam.uri);
            ipGrabber[i].setURI(uri);
            ipGrabber[i].connect();
        }
    }
}

//--------------------------------------------------------------
void testApp::keyReleased(int key){

}

//--------------------------------------------------------------
void testApp::mouseMoved(int x, int y ){
}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::mouseReleased(int x, int y, int butipGrabbern){

}

//--------------------------------------------------------------
void testApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void testApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void testApp::dragEvent(ofDragInfo dragInfo){

}
<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <gtk/gtk.h>

#include "views/focus/accelerator_handler.h"

#include "views/focus/focus_manager.h"
#include "views/widget/widget_gtk.h"
#include "views/window/window_gtk.h"

namespace views {

AcceleratorHandler::AcceleratorHandler() : last_key_pressed_(0) {
}

bool AcceleratorHandler::Dispatch(GdkEvent* event) {
  if (event->type != GDK_KEY_PRESS && event->type != GDK_KEY_RELEASE) {
    gtk_main_do_event(event);
    return true;
  }

  GdkEventKey* key_event = reinterpret_cast<GdkEventKey*>(event);
  // Let's retrieve the focus manager for the GdkWindow.
  GdkWindow* window = gdk_window_get_toplevel(key_event->window);
  gpointer ptr;
  gdk_window_get_user_data(window, &ptr);
  DCHECK(ptr);  // The top-level window is expected to always be associated
                // with the top-level gtk widget.
  WindowGtk* widget =
      WidgetGtk::GetWindowForNative(reinterpret_cast<GtkWidget*>(ptr));
  if (!widget) {
    // During dnd we get events for windows we don't control (such as the
    // window being dragged).
    return true;
  }
  FocusManager* focus_manager = widget->GetFocusManager();
  if (!focus_manager) {
    NOTREACHED();
    return true;
  }

  if (event->type == GDK_KEY_PRESS) {
    KeyEvent view_key_event(key_event, true);
    // FocusManager::OnKeyPressed and OnKeyReleased return false if this
    // message has been consumed and should not be propagated further.
    if (!focus_manager->OnKeyEvent(view_key_event)) {
      last_key_pressed_ = key_event->keyval;
      return true;
    }
  }

  // Key release, make sure to filter-out the key release for key press consumed
  // as accelerators to avoid unpaired key release.
  if (event->type == GDK_KEY_RELEASE &&
      key_event->keyval == last_key_pressed_) {
    last_key_pressed_ = 0;
    return true;
  }

  gtk_main_do_event(event);
  return true;
}

}  // namespace views
<commit_msg>Fixes bug in previous change. In AcceleratorHandler if there is no WidgetGtk we should dispatch the event, not eat it.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <gtk/gtk.h>

#include "views/focus/accelerator_handler.h"

#include "views/focus/focus_manager.h"
#include "views/widget/widget_gtk.h"
#include "views/window/window_gtk.h"

namespace views {

AcceleratorHandler::AcceleratorHandler() : last_key_pressed_(0) {
}

bool AcceleratorHandler::Dispatch(GdkEvent* event) {
  if (event->type != GDK_KEY_PRESS && event->type != GDK_KEY_RELEASE) {
    gtk_main_do_event(event);
    return true;
  }

  GdkEventKey* key_event = reinterpret_cast<GdkEventKey*>(event);
  // Let's retrieve the focus manager for the GdkWindow.
  GdkWindow* window = gdk_window_get_toplevel(key_event->window);
  gpointer ptr;
  gdk_window_get_user_data(window, &ptr);
  DCHECK(ptr);  // The top-level window is expected to always be associated
                // with the top-level gtk widget.
  WindowGtk* widget =
      WidgetGtk::GetWindowForNative(reinterpret_cast<GtkWidget*>(ptr));
  if (!widget) {
    // During dnd we get events for windows we don't control (such as the
    // window being dragged).
    gtk_main_do_event(event);
    return true;
  }
  FocusManager* focus_manager = widget->GetFocusManager();
  if (!focus_manager) {
    NOTREACHED();
    return true;
  }

  if (event->type == GDK_KEY_PRESS) {
    KeyEvent view_key_event(key_event, true);
    // FocusManager::OnKeyPressed and OnKeyReleased return false if this
    // message has been consumed and should not be propagated further.
    if (!focus_manager->OnKeyEvent(view_key_event)) {
      last_key_pressed_ = key_event->keyval;
      return true;
    }
  }

  // Key release, make sure to filter-out the key release for key press consumed
  // as accelerators to avoid unpaired key release.
  if (event->type == GDK_KEY_RELEASE &&
      key_event->keyval == last_key_pressed_) {
    last_key_pressed_ = 0;
    return true;
  }

  gtk_main_do_event(event);
  return true;
}

}  // namespace views
<|endoftext|>


<commit_before>/*
 * @file: perceptron_main.cpp
 * @author: Udit Saxena
 *
 * This program runs the Simple Perceptron Classifier.
 *
 * Perceptrons are simple single-layer binary classifiers, which solve linearly
 * separable problems with a linear decision boundary.
 */

#include <mlpack/core.hpp>
#include "perceptron.hpp"

using namespace mlpack;
using namespace mlpack::perceptron;
using namespace std;
using namespace arma;

PROGRAM_INFO("Perceptron",
    "This program implements a perceptron, which is a single level "
    "Neural Network. The perceptron makes its predictions based on "
    "a linear predictor function combining a set of weights with the feature "
    "vector.\n"
    "The perceptron learning rule is able to converge, given enough iterations "
    "using the --iterations (-i) parameter, if the data supplied is "
    "linearly separable. "
    "\n"
    "The Perceptron is parameterized by a matrix of weight vectors which "
    "denotes the numerical weights of the Neural Network."
    "\n"
    "This program allows training of a perceptron, and then application of "
    "the learned perceptron to a test dataset.  To train a perceptron, "
    "a training dataset must be passed to --train_file (-t).  Labels can either "
    "be present as the last dimension of the training dataset, or given "
    "explicitly with the --labels_file (-l) parameter."
    "\n"
    "A test file is given through the --test_file (-T) parameter.  The "
    "predicted labels for the test set will be stored in the file specified by "
    "the --output_file (-o) parameter."
    );

// Necessary parameters
PARAM_STRING_REQ("train_file", "A file containing the training set.", "t");
PARAM_STRING("labels_file", "A file containing labels for the training set.",
  "l","");
PARAM_STRING_REQ("test_file", "A file containing the test set.", "T");

// Optional parameters.
PARAM_STRING("output", "The file in which the predicted labels for the test set"
    " will be written.", "o", "output.csv");
PARAM_INT("iterations","The maximum number of iterations the perceptron is "
  "to be run", "i", 1000);

int main(int argc, char** argv)
{
  CLI::ParseCommandLine(argc, argv);

  // Get reference dataset filename.
  const string trainingDataFilename = CLI::GetParam<string>("train_file");
  mat trainingData;
  data::Load(trainingDataFilename, trainingData, true);

  const string labelsFilename = CLI::GetParam<string>("labels_file");
  // Load labels.
  mat labelsIn;

  // Did the user pass in labels?
  if (CLI::HasParam("labels_file"))
  {
    // Load labels.
    const string labelsFilename = CLI::GetParam<string>("labels_file");
    data::Load(labelsFilename, labelsIn, true);
  }
  else
  {
    // Use the last row of the training data as the labels.
    Log::Info << "Using the last dimension of training set as labels." << endl;
    labelsIn = trainingData.row(trainingData.n_rows - 1).t();
    trainingData.shed_row(trainingData.n_rows - 1);
  }

  // Do the labels need to be transposed?
  if (labelsIn.n_rows == 1)
  {
    labelsIn = labelsIn.t();
  }

  // Normalize the labels.
  Col<size_t> labels;
  vec mappings;
  data::NormalizeLabels(labelsIn.unsafe_col(0), labels, mappings);

  // Load test dataset.
  const string testingDataFilename = CLI::GetParam<string>("test_file");
  mat testingData;
  data::Load(testingDataFilename, testingData, true);
  if (testingData.n_rows != trainingData.n_rows)
  {
    Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") "
        << "must be the same as training data (" << trainingData.n_rows - 1
        << ")!" << std::endl;
  }

  int iterations = CLI::GetParam<int>("iterations");

  // Create and train the classifier.
  Timer::Start("Training");
  Perceptron<> p(trainingData, labels.t(), max(labels) + 1, iterations);
  Timer::Stop("Training");

  // Time the running of the Perceptron Classifier.
  Row<size_t> predictedLabels(testingData.n_cols);
  Timer::Start("Testing");
  p.Classify(testingData, predictedLabels);
  Timer::Stop("Testing");

  // Un-normalize labels to prepare output.
  vec results;
  data::RevertLabels(predictedLabels.t(), mappings, results);

  // saving the predictedLabels in the transposed manner in output
  const string outputFilename = CLI::GetParam<string>("output");
  data::Save(outputFilename, results, true, false);
}
<commit_msg>Refactor perceptron program to load/save models.<commit_after>/*
 * @file: perceptron_main.cpp
 * @author: Udit Saxena
 *
 * This program runs the Simple Perceptron Classifier.
 *
 * Perceptrons are simple single-layer binary classifiers, which solve linearly
 * separable problems with a linear decision boundary.
 */

#include <mlpack/core.hpp>
#include "perceptron.hpp"

using namespace mlpack;
using namespace mlpack::perceptron;
using namespace std;
using namespace arma;

PROGRAM_INFO("Perceptron",
    "This program implements a perceptron, which is a single level neural "
    "network. The perceptron makes its predictions based on a linear predictor "
    "function combining a set of weights with the feature vector.  The "
    "perceptron learning rule is able to converge, given enough iterations "
    "using the --max_iterations (-i) parameter, if the data supplied is "
    "linearly separable.  The perceptron is parameterized by a matrix of weight"
    " vectors that denote the numerical weights of the neural network."
    "\n\n"
    "This program allows loading a perceptron from a model (-i) or training a "
    "perceptron given training data (-t), or both those things at once.  In "
    "addition, this program allows classification on a test dataset (-T) and "
    "will save the classification results to the given output file (-o).  The "
    "perceptron model itself may be saved with a file specified using the -m "
    "option."
    "\n\n"
    "The training data given with the -t option should have class labels as its"
    " last dimension (so, if the training data is in CSV format, labels should "
    "be the last column).  Alternately, the -l (--labels_file) option may be "
    "used to specify a separate file of labels."
    "\n\n"
    "All these options make it easy to train a perceptron, and then re-use that"
    " perceptron for later classification.  The invocation below trains a "
    "perceptron on 'training_data.csv' (and 'training_labels.csv)' and saves "
    "the model to 'perceptron.xml'."
    "\n\n"
    "$ perceptron -t training_data.csv -l training_labels.csv -m perceptron.csv"
    "\n\n"
    "Then, this model can be re-used for classification on 'test_data.csv'.  "
    "The example below does precisely that, saving the predicted classes to "
    "'predictions.csv'."
    "\n\n"
    "$ perceptron -i perceptron.xml -T test_data.csv -o predictions.csv"
    "\n\n"
    "Note that all of the options may be specified at once: predictions may be "
    "calculated right after training a model, and model training can occur even"
    " if an existing perceptron model is passed with -i (--input_model).  "
    "However, note that the number of classes and the dimensionality of all "
    "data must match.  So you cannot pass a perceptron model trained on 2 "
    "classes and then re-train with a 4-class dataset.  Similarly, attempting "
    "classification on a 3-dimensional dataset with a perceptron that has been "
    "trained on 8 dimensions will cause an error."
    );

// Training parameters.
PARAM_STRING("training_file", "A file containing the training set.", "t", "");
PARAM_STRING("labels_file", "A file containing labels for the training set.",
  "l", "");
PARAM_INT("max_iterations","The maximum number of iterations the perceptron is "
  "to be run", "M", 1000);

// Model loading/saving.
PARAM_STRING("input_model", "File containing input perceptron model.", "i", "");
PARAM_STRING("output_model", "File to save trained perceptron model to.", "m",
    "");

// Testing/classification parameters.
PARAM_STRING("test_file", "A file containing the test set.", "T", "");
PARAM_STRING("output_file", "The file in which the predicted labels for the "
    "test set will be written.", "o", "output.csv");

// When we save a model, we must also save the class mappings.  So we use this
// auxiliary structure to store both the perceptron and the mapping, and we'll
// save this.
class PerceptronModel
{
 private:
  Perceptron<>& p;
  arma::vec& map;

 public:
  PerceptronModel(Perceptron<>& p, arma::vec& map) : p(p), map(map) { }

  template<typename Archive>
  void Serialize(Archive& ar, const unsigned int /* version */)
  {
    ar & data::CreateNVP(p, "perceptron");
    ar & data::CreateNVP(map, "mappings");
  }
};

int main(int argc, char** argv)
{
  CLI::ParseCommandLine(argc, argv);

  // First, get all parameters and validate them.
  const string trainingDataFile = CLI::GetParam<string>("training_file");
  const string labelsFile = CLI::GetParam<string>("labels_file");
  const string inputModelFile = CLI::GetParam<string>("input_model");
  const string testDataFile = CLI::GetParam<string>("test_file");
  const string outputModelFile = CLI::GetParam<string>("output_model");
  const string outputFile = CLI::GetParam<string>("output_file");
  const size_t maxIterations = (size_t) CLI::GetParam<int>("max_iterations");

  // We must either load a model or train a model.
  if (inputModelFile == "" && trainingDataFile == "")
    Log::Fatal << "Either an input model must be specified with --input_model "
        << "or training data must be given (--training_file)!" << endl;

  // If the user isn't going to save the output model or any predictions, we
  // should issue a warning.
  if (outputModelFile == "" && testDataFile == "")
    Log::Warn << "Output will not be saved!  (Neither --test_file nor "
        << "--output_model are specified.)" << endl;

  // Now, load our model, if there is one.
  Perceptron<>* p = NULL;
  arma::vec mappings;
  if (inputModelFile != "")
  {
    Log::Info << "Loading saved perceptron from model file '" << inputModelFile
        << "'." << endl;

    // The parameters here are invalid, but we are about to load the model
    // anyway...
    p = new Perceptron<>(0, 0);
    PerceptronModel pm(*p, mappings); // Also load class mappings.
    data::Load(inputModelFile, "perceptron_model", pm, true);
  }

  // Next, load the training data and labels (if they have been given).
  if (trainingDataFile != "")
  {
    Log::Info << "Training perceptron on dataset '" << trainingDataFile;
    if (labelsFile != "")
      Log::Info << "' with labels in '" << labelsFile << "'";
    else
      Log::Info << "'";
    Log::Info << " for a maximum of " << maxIterations << " iterations."
        << endl;

    mat trainingData;
    data::Load(trainingDataFile, trainingData, true);

    // Load labels.
    mat labelsIn;

    // Did the user pass in labels?
    if (CLI::HasParam("labels_file"))
    {
      // Load labels.
      const string labelsFile = CLI::GetParam<string>("labels_file");
      data::Load(labelsFile, labelsIn, true);
    }
    else
    {
      // Use the last row of the training data as the labels.
      Log::Info << "Using the last dimension of training set as labels."
          << endl;
      labelsIn = trainingData.row(trainingData.n_rows - 1).t();
      trainingData.shed_row(trainingData.n_rows - 1);
    }

    // Do the labels need to be transposed?
    if (labelsIn.n_rows == 1)
      labelsIn = labelsIn.t();

    // Normalize the labels.
    Col<size_t> labels;
    data::NormalizeLabels(labelsIn.unsafe_col(0), labels, mappings);

    // Now, if we haven't already created a perceptron, do it.  Otherwise, make
    // sure the dimensions are right, then continue training.
    if (p == NULL)
    {
      // Create and train the classifier.
      Timer::Start("training");
      p = new Perceptron<>(trainingData, labels.t(), max(labels) + 1,
          maxIterations);
      Timer::Stop("training");
    }
    else
    {
      // Check dimensionality.
      if (p->Weights().n_rows != trainingData.n_rows)
      {
        Log::Fatal << "Perceptron from '" << inputModelFile << "' is built on "
            << "data with " << p->Weights().n_rows << " dimensions, but data in"
            << " '" << trainingDataFile << "' has " << trainingData.n_rows
            << "dimensions!" << endl;
      }

      // Check the number of labels.
      if (max(labels) + 1 > p->Weights().n_cols)
      {
        Log::Fatal << "Perceptron from '" << inputModelFile << "' has "
            << p->Weights().n_cols << " classes, but the training data has "
            << max(labels) + 1 << " classes!" << endl;
      }

      // Now train.
      Timer::Start("training");
      p->MaxIterations() = maxIterations;
      p->Train(trainingData, labels.t());
      Timer::Stop("training");
    }
  }

  // Now, the training procedure is complete.  Do we have any test data?
  if (testDataFile != "")
  {
    Log::Info << "Classifying dataset '" << testDataFile << "'." << endl;
    mat testData;
    data::Load(testDataFile, testData, true);

    if (testData.n_rows != p->Weights().n_rows)
    {
      Log::Fatal << "Test data dimensionality (" << testData.n_rows << ") must "
          << "be the same as the dimensionality of the perceptron ("
          << p->Weights().n_rows << ")!" << endl;
    }

    // Time the running of the perceptron classifier.
    Row<size_t> predictedLabels(testData.n_cols);
    Timer::Start("testing");
    p->Classify(testData, predictedLabels);
    Timer::Stop("testing");

    // Un-normalize labels to prepare output.
    vec results;
    data::RevertLabels(predictedLabels.t(), mappings, results);

    // Save the predictedLabels, but we have to transpose them.
    data::Save(outputFile, results, false /* non-fatal */, false);
  }

  // Lastly, do we need to save the output model?
  if (outputModelFile != "")
  {
    PerceptronModel pm(*p, mappings);
    data::Save(outputModelFile, "perceptron_model", pm);
  }

  // Clean up memory.
  delete p;
}
<|endoftext|>


<commit_before>/*
 * Mickey R7RS Scheme
 *
 * Copyright (C) 2011-2012 Christian Stigen Larsen <csl@sublevel3.org>
 * http://csl.sublevel3.org                              _
 *                                                        \
 * Distributed under the LGPL 2.1; see LICENSE            /\
 * Please post bugfixes and suggestions to the author.   /  \_
 *
 */

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "exceptions.h"
#include "print.h"
#include "tokenizer.h"
#include "util.h"

static const char* source = NULL;
static bool inside_string = false;
static bool fold_case_flag = false;
static int line = 1;

int current_line_number()
{
  return line;
}

static inline void checkline(const char ch)
{
  if ( ch == '\n' )
    ++line;
}

bool fold_case()
{
  return fold_case_flag;
}

void set_source(const char* program)
{
  source = program;
  inside_string = false;
  line = 1;
}

static bool not_pipe(const char* s)
{
  return *s!='\0' && *s!='|';
}

static bool string_or_non_delimiter(const char* s)
{
  char ch = *s;

  // ignore next datum symbol #; is a token
  if ( s[0]=='#' && s[1]==';' )
    return false;

  bool open_paren = (ch=='('        /* normal paren */
      || (s[0]=='#' && s[1]=='(')   /* vector form #( ... ) */
      || (s[0]=='#' && s[1]=='u' && /* bytevector form #u8( ... ) */
          s[2]=='8' && s[3]=='('));

  if ( ch == '\"' )
    inside_string = !inside_string;

  return ch!='\0'
    && (inside_string? true :
          !open_paren && ch!=')' && !isspace(ch));
}

static const char* skip_space(const char* s)
{
  while ( isspace(*s) ) {
    checkline(*s);
    ++s;
  }

  return s;
}

static const char* copy_while(
    char *dest, const char* src, bool (*while_expr)(const char*))
{
  while ( while_expr(src) )
    *dest++ = *src++;

  *dest = '\0';
  return src;
}

const char* get_token()
{
  // mutatable return buffer
  static char token[256];

  for ( ;; ) {
    token[0] = token[1] = '\0';
    source = skip_space(source);

    // comment? skip to end of line
    if ( *source == ';' ) {
      while ( *source != '\n' ) ++source;
      continue;
    }

    // hash-bang or similar? skip to end of line
    // TODO: Properly handle reader directives like case-folding, etc.
    if ( source[0]=='#' && source[1]=='!' ) {

      // skip to end of line
      const char *start = source;
      while ( *source != '\n' ) ++source;

      if ( !strncmp("#!fold-case", start, source - start) )
        fold_case_flag = true;
      else if ( !strncmp("#!no-fold-case", start, source - start) )
        fold_case_flag = false;

      continue;
    }

    // block-comments?
    if ( source[0]=='#' && source[1]=='|' ) {
      // match nested pairs
      source += 2;
      for ( int n=1; n && *source; ++source ) {
             if ( source[0]=='#' && source[1]=='|' ) { ++source; ++n; }
        else if ( source[0]=='|' && source[1]=='#' ) { ++source; --n; }
      }
      continue;
    }

    // vector form "#( ... )"
    if ( source[0]=='#' && source[1]=='(' ) {
      strcpy(token, "#(");
      source += 2;
      return token;
    }

    // bytevector form "#u8( ... )"
    if ( source[0]=='#' && source[1]=='u' &&
         source[2]=='8' && source[3]=='(' )
    {
      strcpy(token, "#u8(");
      source += 4;
      return token;
    }

    // ignore-next-datum form "#;"
    if ( source[0]=='#' && source[1]==';' ) {
      strcpy(token, "#;");
      source += 2;
      return token;
    }

    if ( char_in(*source, "()'") )
      // tokens ( and )
      token[0] = *source++;
    else {
      // long-form-symbol w/format "|foo bar baz|"
      if ( source[0]=='|' ) {
        const char* start = source;
        token[0]='|';
        source = copy_while(token+1, source+1, not_pipe);

        if ( *source =='|' )
          ++source;
        else
          raise(parser_exception(format(
            "Invalid |long symbol| on line %d: %s\n", line, start)));

        const size_t l = strlen(token);
        token[l] = '|';
        token[l+1] = '\0';
      } else
        // other tokens
        source = copy_while(token, source, string_or_non_delimiter);
    }

    // emit NULL when finished
    return !empty(token) ? token : NULL;
  }
}
<commit_msg>Bugfix: Check for end of input<commit_after>/*
 * Mickey R7RS Scheme
 *
 * Copyright (C) 2011-2012 Christian Stigen Larsen <csl@sublevel3.org>
 * http://csl.sublevel3.org                              _
 *                                                        \
 * Distributed under the LGPL 2.1; see LICENSE            /\
 * Please post bugfixes and suggestions to the author.   /  \_
 *
 */

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "exceptions.h"
#include "print.h"
#include "tokenizer.h"
#include "util.h"

static const char* source = NULL;
static bool inside_string = false;
static bool fold_case_flag = false;
static int line = 1;

int current_line_number()
{
  return line;
}

static inline void checkline(const char ch)
{
  if ( ch == '\n' )
    ++line;
}

bool fold_case()
{
  return fold_case_flag;
}

void set_source(const char* program)
{
  source = program;
  inside_string = false;
  line = 1;
}

static bool not_pipe(const char* s)
{
  return *s!='\0' && *s!='|';
}

static bool string_or_non_delimiter(const char* s)
{
  char ch = *s;

  // ignore next datum symbol #; is a token
  if ( s[0]=='#' && s[1]==';' )
    return false;

  bool open_paren = (ch=='('        /* normal paren */
      || (s[0]=='#' && s[1]=='(')   /* vector form #( ... ) */
      || (s[0]=='#' && s[1]=='u' && /* bytevector form #u8( ... ) */
          s[2]=='8' && s[3]=='('));

  if ( ch == '\"' )
    inside_string = !inside_string;

  return ch!='\0'
    && (inside_string? true :
          !open_paren && ch!=')' && !isspace(ch));
}

static const char* skip_space(const char* s)
{
  while ( isspace(*s) ) {
    checkline(*s);
    ++s;
  }

  return s;
}

static const char* copy_while(
    char *dest, const char* src, bool (*while_expr)(const char*))
{
  while ( while_expr(src) )
    *dest++ = *src++;

  *dest = '\0';
  return src;
}

const char* get_token()
{
  // mutatable return buffer
  static char token[256];

  for ( ;; ) {
    token[0] = token[1] = '\0';
    source = skip_space(source);

    // comment? skip to end of line
    if ( *source == ';' ) {
      while ( *source != '\n' ) {
        ++source;

        if ( *source == '\0' )
          return NULL;
      }

      continue;
    }

    // hash-bang or similar? skip to end of line
    // TODO: Properly handle reader directives like case-folding, etc.
    if ( source[0]=='#' && source[1]=='!' ) {

      // skip to end of line
      const char *start = source;
      while ( *source != '\n' ) ++source;

      if ( !strncmp("#!fold-case", start, source - start) )
        fold_case_flag = true;
      else if ( !strncmp("#!no-fold-case", start, source - start) )
        fold_case_flag = false;

      continue;
    }

    // block-comments?
    if ( source[0]=='#' && source[1]=='|' ) {
      // match nested pairs
      source += 2;
      for ( int n=1; n && *source; ++source ) {
             if ( source[0]=='#' && source[1]=='|' ) { ++source; ++n; }
        else if ( source[0]=='|' && source[1]=='#' ) { ++source; --n; }
      }
      continue;
    }

    // vector form "#( ... )"
    if ( source[0]=='#' && source[1]=='(' ) {
      strcpy(token, "#(");
      source += 2;
      return token;
    }

    // bytevector form "#u8( ... )"
    if ( source[0]=='#' && source[1]=='u' &&
         source[2]=='8' && source[3]=='(' )
    {
      strcpy(token, "#u8(");
      source += 4;
      return token;
    }

    // ignore-next-datum form "#;"
    if ( source[0]=='#' && source[1]==';' ) {
      strcpy(token, "#;");
      source += 2;
      return token;
    }

    if ( char_in(*source, "()'") )
      // tokens ( and )
      token[0] = *source++;
    else {
      // long-form-symbol w/format "|foo bar baz|"
      if ( source[0]=='|' ) {
        const char* start = source;
        token[0]='|';
        source = copy_while(token+1, source+1, not_pipe);

        if ( *source =='|' )
          ++source;
        else
          raise(parser_exception(format(
            "Invalid |long symbol| on line %d: %s\n", line, start)));

        const size_t l = strlen(token);
        token[l] = '|';
        token[l+1] = '\0';
      } else
        // other tokens
        source = copy_while(token, source, string_or_non_delimiter);
    }

    // emit NULL when finished
    return !empty(token) ? token : NULL;
  }
}
<|endoftext|>