<commit_before>#ifndef HASH_TRIE
#define HASH_TRIE

#include "term_hash_table.hpp"
#include "doc_list.hpp"
#include <vector>
#include <string>
#include <math.h>
#include <iostream>

NS_IZENELIB_AM_BEGIN

template<
  uint32_t ENTRY_SIZE = 1000000,
  uint32_t NODE_SIZE_RATE = 10 // it indicate the reducing rate of hash table entries number for different levels in trie.
  >
class HashTrie
{
#define NODE_ENTRY_SIZE_TABLE_SIZE 4

  typedef DocList<> doc_list_t;

  TermHashTable *root_;
  TermHashTable* buf_node_;
  FILE* f_;
  FILE* doc_f_;
  uint32_t node_entry_size_[NODE_ENTRY_SIZE_TABLE_SIZE];
  uint32_t node_num_;

  void insert_(const std::vector<uint64_t>& terms, std::size_t start,  uint32_t docid)
  {
    TermHashTable* node = root_;
    for (std::size_t i=start; i<terms.size(); ++i)
    {
      bool is_new = false;
      char* p = node->insert(terms[i], is_new);
      Term t(p);
      if (is_new)//new term
      {
        if (i == terms.size()-1)//the last one
        {
          //add doc id
          if (t.get_doc_list() == (uint64_t)-1)
            t.set_doc_list((uint64_t)(new doc_list_t(docid)));
          else
            ((doc_list_t*)t.get_doc_list())->append(docid);

          continue;
        }

        node = new TermHashTable(node_entry_size_[i-start+1>=NODE_ENTRY_SIZE_TABLE_SIZE? NODE_ENTRY_SIZE_TABLE_SIZE-1: i-start+1]);
        ++node_num_;
        t.set_child((uint64_t)node);
        t.set_loaded(1);
        continue;
      }

      if (i == terms.size()-1)//the last one
      {
        //add doc id
        if (t.get_doc_list() == (uint64_t)-1)
          t.set_doc_list((uint64_t)(new doc_list_t(docid)));
        else
          ((doc_list_t*)t.get_doc_list())->append(docid);

        continue;
      }

      node = (TermHashTable*)t.get_child();
      if ((uint64_t)node == (uint64_t)-1)
      {
        node = new TermHashTable(node_entry_size_[i-start+1>=NODE_ENTRY_SIZE_TABLE_SIZE? NODE_ENTRY_SIZE_TABLE_SIZE-1: i-start+1]);
        ++node_num_;
        t.set_child((uint64_t)node);
        t.set_loaded(1);
      }
    }
  }

  uint64_t size_(TermHashTable* tb)
  {
    uint64_t r= tb->size();
    ++node_num_;

    //std::cout<<"*****************\n";
    //std::cout<<*tb<<std::endl;
    //std::cout<<tb->size()<<std::endl;

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      //std::cout<<(*i).term()<<std::endl;

      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;
      r += size_((TermHashTable*)((*i).get_child()));
    }
    //std::cout<<"-----------------\n";

    return r;
  }

  void release_(TermHashTable* tb)
  {
    if (tb == NULL)
      return;

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;
      release_((TermHashTable*)((*i).get_child()));
    }

    delete tb;
    ++node_num_;
  }

  void save_(TermHashTable* tb)
  {
    uint64_t pos1 = ftell(f_);
    tb->save(f_);

    bool f = false;
    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;

      f = true;
      uint64_t pos = ftell(f_);

      save_((TermHashTable*)((*i).get_child()));
      (*i).set_child(pos);
      (*i).set_loaded(0);

      uint64_t list = (*i).get_doc_list();
      if (list !=(uint64_t)-1)
      {
        (*i).set_doc_list(ftell(doc_f_));
        //std::cout<<ftell(doc_f_)<<std::endl;
        ((doc_list_t*)list)->save(doc_f_);
      }
    }

    if (f)
    {
      uint64_t pos2 = ftell(f_);
      fseek(f_, pos1, SEEK_SET);
      tb->save(f_);
      fseek(f_, pos2, SEEK_SET);
    }

    delete tb;
  }

  void load_(TermHashTable* tb, uint64_t addr)
  {
    tb->load(f_, addr);

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      uint64_t child = (*i).get_child();

      if ((*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;

      TermHashTable* tht = new TermHashTable();
      ++node_num_;
      load_(tht, child);
      (*i).set_child((uint64_t)tht);
      (*i).set_loaded(1);

      if ((*i).get_doc_list() == (uint64_t)-1)
        continue;

      doc_list_t* list = new doc_list_t(doc_f_,(*i).get_doc_list());
      (*i).set_doc_list((uint64_t)list);
    }
  }

  inline void load()
  {
    uint32_t nn = 0;
    node_num_ = 1;

    fseek(f_, 0, SEEK_SET);

    assert(fread(&nn, sizeof(uint32_t), 1, f_)==1);

    root_ = new TermHashTable(ENTRY_SIZE);
    load_(root_, ftell(f_));
    assert(node_num_ == nn);
    //std::cout<<"node num loaded: "<<node_num_<<std::endl;
  }

  void get_docs_(uint64_t tb, bool loaded, doc_list_t** list)
  {
    if (tb == (uint64_t)-1)
        return;

    TermHashTable* node = NULL;
    if (loaded)
      node = (TermHashTable* )tb;
    else
    {
      node = buf_node_;
      node->load(f_, tb);
    }

    std::vector<uint64_t> children;
    std::vector<bool> loads;

    for (TermHashTable::const_iterator i=node->begin(); i!=node->end(); ++i)
    {
      children.push_back((*i).get_child());
      loads.push_back((*i).get_loaded());

      if ((*i).get_doc_list() == (uint64_t)-1)
        continue;

      if (*list == NULL)
        *list = new doc_list_t(*(doc_list_t*)(*i).get_doc_list());
      else
        (*list)->append(*(doc_list_t*)(*i).get_doc_list());
    }

    std::vector<bool>::const_iterator j=loads.begin();
    for (std::vector<uint64_t>::const_iterator i=children.begin(); i!=children.end(); ++i, ++j)
      get_docs_(*i,*j, list);
  }

public:
  HashTrie(const char* nm)
  {
    node_entry_size_[0] = ENTRY_SIZE;
    for (uint32_t i=1; i<NODE_ENTRY_SIZE_TABLE_SIZE; ++i)
    {
      node_entry_size_[i] = (uint32_t)(sqrt((double)node_entry_size_[i-1])*i/(NODE_ENTRY_SIZE_TABLE_SIZE*NODE_SIZE_RATE)+0.5);
      if(node_entry_size_[i]==0)
        node_entry_size_[i] = 1;
      //std::cout<<node_entry_size_[i]<<std::endl;
    }

    buf_node_ = new TermHashTable();

    node_num_ = 1;

    f_ = fopen(nm, "r+");
    if (f_ == NULL)
    {
      f_ = fopen(nm, "w+");
      assert(fwrite(&node_num_, sizeof(uint32_t), 1, f_)==1);
      root_ = new TermHashTable(ENTRY_SIZE);
    }
    else
      load();

    if (f_ == NULL)
    {
      std::cout<<"Can't create file: "<<nm<<std::endl;
      return;
    }

    std::string str = nm;
    str += ".doc";
    doc_f_ = fopen(str.c_str(), "r+");
    if (doc_f_ == NULL)
    {
      doc_f_ = fopen(str.c_str(), "w+");
    }

    if (doc_f_ == NULL)
    {
      std::cout<<"Can't create file: "<<str<<std::endl;
      return;
    }
  }

  ~HashTrie()
  {
    //std::cout<<"deleting trie!\n";
    fclose(f_);
    uint32_t nn = node_num_;
    node_num_ = 0;
    release_(root_);
    assert(nn == node_num_);
    //std::cout<<"deleting buf_node_!\n";
    delete buf_node_;
  }

  inline uint32_t node_num()const
  {
    return node_num_;
  }

  void insert(const std::vector<uint64_t>& terms, uint32_t docid)
  {
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      insert_(terms, i, docid);
    }

    //std::cout<<*root_<<std::endl;
  }

  uint64_t size()
  {
    uint32_t nn = node_num_;
    node_num_ = 0;
    uint64_t s = size_(root_);
    assert(nn==node_num_);

    return s;
  }

  uint32_t get_freq(const std::vector<uint64_t>& terms)
  {
    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t(node->find(terms[i]));
      if (t.is_null())
        return 0;

      if (i == terms.size()-1)//last one
      {
        return t.get_freq();
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }

      node = buf_node_;
      node->load(f_, t.get_child());
    }

    return 0;
  }

  bool get_suffix(const std::vector<uint64_t>& terms, std::vector<uint64_t>& suffixs, std::vector<uint32_t>& counts)
  {
    suffixs.clear();
    counts.clear();

    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t( node->find(terms[i]));
      if (t.is_null())
        return false;

      if (i == terms.size()-1)//last one
      {
        if (t.get_loaded())
          node = (TermHashTable* )t.get_child();
        else
        {
          node = buf_node_;
          node->load(f_, t.get_child());
        }

        break;
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }
      node = buf_node_;
      node->load(f_, t.get_child());
    }

    for (TermHashTable::const_iterator i=node->begin(); i!=node->end(); ++i)
    {
      suffixs.push_back((*i).get_id());
      counts.push_back((*i).get_freq());
    }

    return true;
  }

  bool get_docs(const std::vector<uint64_t>& terms, std::vector<uint32_t>& docs)
  {
    docs.clear();
    doc_list_t* list = NULL;

    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t( node->find(terms[i]));
      if (t.is_null())
        return false;

      if (i == terms.size()-1)//last one
      {
        if (t.get_doc_list()!= (uint64_t)-1)
        {
          list = new doc_list_t(*(doc_list_t*)t.get_doc_list());
        }

        get_docs_(t.get_child(), t.get_loaded(), &list);

        if (list == NULL)
          docs.clear();
        else
          list->assign(docs);

        return true;
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }
      node = buf_node_;
      node->load(f_, t.get_child());
    }

    return false;
  }

  inline void save()
  {
    fseek(f_, 0, SEEK_SET);
    assert(fwrite(&node_num_, sizeof(uint32_t), 1, f_)==1);
    save_(root_);
    fflush(f_);
    root_ = NULL;
    node_num_ = 0;
  }

}
;
NS_IZENELIB_AM_END
#endif

/**
   TgPhraseExtractor(const std::string& path);
		~TgPhraseExtractor();
		void addSentence(const std::vector<termid_t>& termList, const std::vector<loc_t>& positionList, docid_t docId);
		bool analysis();
		labelid_t getPhraseId(const std::vector<termid_t>& termList);
		count_t freq(labelid_t labelId);
		bool getDocIdList(labelid_t labelId, std::vector<docid_t>& docIdList);
		private:
			SuffixNodeList begin();
			bool getAllPrefix(const std::vector<termid_t>& termList, std::vector<termid_t>& prefixList, std::vector<count_t>& countList);
			bool getAllSuffix(const std::vector<termid_t>& termList, std::vector<termid_t>& suffixList, std::vector<count_t>& countList);
 **/
<commit_msg>trie-indexer<commit_after>#ifndef HASH_TRIE
#define HASH_TRIE

#include "term_hash_table.hpp"
#include "doc_list.hpp"
#include <vector>
#include <string>
#include <math.h>
#include <iostream>

NS_IZENELIB_AM_BEGIN

template<
  uint32_t ENTRY_SIZE = 1000000,
  uint32_t NODE_SIZE_RATE = 10 // it indicate the reducing rate of hash table entries number for different levels in trie.
  >
class HashTrie
{
#define NODE_ENTRY_SIZE_TABLE_SIZE 4

  typedef DocList<> doc_list_t;

  TermHashTable *root_;
  TermHashTable* buf_node_;
  FILE* f_;
  FILE* doc_f_;
  uint32_t node_entry_size_[NODE_ENTRY_SIZE_TABLE_SIZE];
  uint32_t node_num_;

  void insert_(const std::vector<uint64_t>& terms, std::size_t start,  uint32_t docid)
  {
    TermHashTable* node = root_;
    for (std::size_t i=start; i<terms.size(); ++i)
    {
      bool is_new = false;
      char* p = node->insert(terms[i], is_new);
      Term t(p);
      if (is_new)//new term
      {
        if (i == terms.size()-1)//the last one
        {
          //add doc id
          if (t.get_doc_list() == (uint64_t)-1)
            t.set_doc_list((uint64_t)(new doc_list_t(docid)));
          else
            ((doc_list_t*)t.get_doc_list())->append(docid);

          continue;
        }

        node = new TermHashTable(node_entry_size_[i-start+1>=NODE_ENTRY_SIZE_TABLE_SIZE? NODE_ENTRY_SIZE_TABLE_SIZE-1: i-start+1]);
        ++node_num_;
        t.set_child((uint64_t)node);
        t.set_loaded(1);
        continue;
      }

      if (i == terms.size()-1)//the last one
      {
        //add doc id
        if (t.get_doc_list() == (uint64_t)-1)
          t.set_doc_list((uint64_t)(new doc_list_t(docid)));
        else
          ((doc_list_t*)t.get_doc_list())->append(docid);

        continue;
      }

      node = (TermHashTable*)t.get_child();
      if ((uint64_t)node == (uint64_t)-1)
      {
        node = new TermHashTable(node_entry_size_[i-start+1>=NODE_ENTRY_SIZE_TABLE_SIZE? NODE_ENTRY_SIZE_TABLE_SIZE-1: i-start+1]);
        ++node_num_;
        t.set_child((uint64_t)node);
        t.set_loaded(1);
      }
    }
  }

  uint64_t size_(TermHashTable* tb)
  {
    uint64_t r= tb->size();
    ++node_num_;

    //std::cout<<"*****************\n";
    //std::cout<<*tb<<std::endl;
    //std::cout<<tb->size()<<std::endl;

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      //std::cout<<(*i).term()<<std::endl;

      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;
      r += size_((TermHashTable*)((*i).get_child()));
    }
    //std::cout<<"-----------------\n";

    return r;
  }

  void release_(TermHashTable* tb)
  {
    if (tb == NULL)
      return;

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;
      release_((TermHashTable*)((*i).get_child()));
    }

    delete tb;
    ++node_num_;
  }

  void save_(TermHashTable* tb)
  {
    uint64_t pos1 = ftell(f_);
    tb->save(f_);

    bool f = false;
    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      if (!(*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;

      f = true;
      uint64_t pos = ftell(f_);

      save_((TermHashTable*)((*i).get_child()));
      (*i).set_child(pos);
      (*i).set_loaded(0);

      uint64_t list = (*i).get_doc_list();
      if (list !=(uint64_t)-1)
      {
        (*i).set_doc_list(ftell(doc_f_));
        //std::cout<<ftell(doc_f_)<<std::endl;
        ((doc_list_t*)list)->save(doc_f_);
      }
    }

    if (f)
    {
      uint64_t pos2 = ftell(f_);
      fseek(f_, pos1, SEEK_SET);
      tb->save(f_);
      fseek(f_, pos2, SEEK_SET);
    }

    delete tb;
  }

  void load_(TermHashTable* tb, uint64_t addr)
  {
    tb->load(f_, addr);

    for (TermHashTable::const_iterator i=tb->begin();i!=tb->end();++i)
    {
      uint64_t child = (*i).get_child();

      if ((*i).get_loaded() || (*i).get_child()==(uint64_t)-1)
        continue;

      TermHashTable* tht = new TermHashTable();
      ++node_num_;
      load_(tht, child);
      (*i).set_child((uint64_t)tht);
      (*i).set_loaded(1);

      if ((*i).get_doc_list() == (uint64_t)-1)
        continue;

      doc_list_t* list = new doc_list_t(doc_f_,(*i).get_doc_list());
      (*i).set_doc_list((uint64_t)list);
    }
  }

  inline void load()
  {
    uint32_t nn = 0;
    node_num_ = 1;

    fseek(f_, 0, SEEK_SET);

    assert(fread(&nn, sizeof(uint32_t), 1, f_)==1);

    root_ = new TermHashTable(ENTRY_SIZE);
    load_(root_, ftell(f_));
    assert(node_num_ == nn);
    //std::cout<<"node num loaded: "<<node_num_<<std::endl;
  }

  void get_docs_(uint64_t tb, bool loaded, doc_list_t** list)
  {
    if (tb == (uint64_t)-1)
        return;

    TermHashTable* node = NULL;
    if (loaded)
      node = (TermHashTable* )tb;
    else
    {
      node = buf_node_;
      node->load(f_, tb);
    }

    std::vector<uint64_t> children;
    std::vector<bool> loads;

    for (TermHashTable::const_iterator i=node->begin(); i!=node->end(); ++i)
    {
      children.push_back((*i).get_child());
      loads.push_back((*i).get_loaded());

      if ((*i).get_doc_list() == (uint64_t)-1)
        continue;

      if (*list == NULL)
        *list = new doc_list_t(*(doc_list_t*)(*i).get_doc_list());
      else
        (*list)->append(*(doc_list_t*)(*i).get_doc_list());
    }

    std::vector<bool>::const_iterator j=loads.begin();
    for (std::vector<uint64_t>::const_iterator i=children.begin(); i!=children.end(); ++i, ++j)
      get_docs_(*i,*j, list);
  }

public:
  HashTrie(const char* nm)
  {
    node_entry_size_[0] = ENTRY_SIZE;
    for (uint32_t i=1; i<NODE_ENTRY_SIZE_TABLE_SIZE; ++i)
    {
      node_entry_size_[i] = (uint32_t)(sqrt((double)node_entry_size_[i-1])*i/(NODE_ENTRY_SIZE_TABLE_SIZE*NODE_SIZE_RATE)+0.5);
      if(node_entry_size_[i]==0)
        node_entry_size_[i] = 1;
      //std::cout<<node_entry_size_[i]<<std::endl;
    }

    buf_node_ = new TermHashTable();

    node_num_ = 1;

    f_ = fopen(nm, "r+");
    if (f_ == NULL)
    {
      f_ = fopen(nm, "w+");
      assert(fwrite(&node_num_, sizeof(uint32_t), 1, f_)==1);
      root_ = new TermHashTable(ENTRY_SIZE);
    }
    else
      load();

    if (f_ == NULL)
    {
      std::cout<<"Can't create file: "<<nm<<std::endl;
      return;
    }

    std::string str = nm;
    str += ".doc";
    doc_f_ = fopen(str.c_str(), "r+");
    if (doc_f_ == NULL)
    {
      doc_f_ = fopen(str.c_str(), "w+");
    }

    if (doc_f_ == NULL)
    {
      std::cout<<"Can't create file: "<<str<<std::endl;
      return;
    }
  }

  ~HashTrie()
  {
    //std::cout<<"deleting trie!\n";
    fclose(f_);
    uint32_t nn = node_num_;
    node_num_ = 0;
    release_(root_);
    assert(nn == node_num_);
    //std::cout<<"deleting buf_node_!\n";
    delete buf_node_;
  }

  inline uint32_t node_num()const
  {
    return node_num_;
  }

  void insert(const std::vector<uint64_t>& terms, uint32_t docid)
  {
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      insert_(terms, i, docid);
    }

    //std::cout<<*root_<<std::endl;
  }

  uint64_t size()
  {
    uint32_t nn = node_num_;
    node_num_ = 0;
    uint64_t s = size_(root_);
    assert(nn==node_num_);

    return s;
  }

  uint32_t get_freq(const std::vector<uint64_t>& terms)
  {
    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t(node->find(terms[i]));
      if (t.is_null())
        return 0;

      if (i == terms.size()-1)//last one
      {
        return t.get_freq();
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }

      node = buf_node_;
      node->load(f_, t.get_child());
    }

    return 0;
  }

  bool get_suffix(const std::vector<uint64_t>& terms, std::vector<uint64_t>& suffixs, std::vector<uint32_t>& counts)
  {
    suffixs.clear();
    counts.clear();

    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t( node->find(terms[i]));
      if (t.is_null())
        return false;

      if (i == terms.size()-1)//last one
      {
        if (t.get_loaded())
          node = (TermHashTable* )t.get_child();
        else
        {
          node = buf_node_;
          node->load(f_, t.get_child());
        }

        break;
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }
      node = buf_node_;
      node->load(f_, t.get_child());
    }

    for (TermHashTable::const_iterator i=node->begin(); i!=node->end(); ++i)
    {
      suffixs.push_back((*i).get_id());
      counts.push_back((*i).get_freq());
    }

    return true;
  }

  bool get_docs(const std::vector<uint64_t>& terms, std::vector<uint32_t>& docs)
  {
    docs.clear();
    doc_list_t* list = NULL;

    TermHashTable* node = root_;
    for (std::size_t i=0; i<terms.size(); ++i)
    {
      Term t( node->find(terms[i]));
      if (t.is_null())
        return false;

      if (i == terms.size()-1)//last one
      {
        if (t.get_doc_list()!= (uint64_t)-1)
        {
          list = new doc_list_t(*(doc_list_t*)t.get_doc_list());
        }

        get_docs_(t.get_child(), t.get_loaded(), &list);

        if (list == NULL)
          docs.clear();
        else
          list->assign(docs);

        return true;
      }

      if (t.get_loaded())
      {
        node = (TermHashTable* )t.get_child();
        continue;
      }
      node = buf_node_;
      node->load(f_, t.get_child());
    }

    return false;
  }

  inline void save()
  {
    fseek(f_, 0, SEEK_SET);
    assert(fwrite(&node_num_, sizeof(uint32_t), 1, f_)==1);
    save_(root_);
    fflush(f_);
    root_ = NULL;
    node_num_ = 0;
  }

  const TermHashTable* get_root()const
  {
    return root_;
  }

}
;
NS_IZENELIB_AM_END
#endif

/**
   TgPhraseExtractor(const std::string& path);
		~TgPhraseExtractor();
		void addSentence(const std::vector<termid_t>& termList, const std::vector<loc_t>& positionList, docid_t docId);
		bool analysis();
		labelid_t getPhraseId(const std::vector<termid_t>& termList);
		count_t freq(labelid_t labelId);
		bool getDocIdList(labelid_t labelId, std::vector<docid_t>& docIdList);
		private:
			SuffixNodeList begin();
			bool getAllPrefix(const std::vector<termid_t>& termList, std::vector<termid_t>& prefixList, std::vector<count_t>& countList);
			bool getAllSuffix(const std::vector<termid_t>& termList, std::vector<termid_t>& suffixList, std::vector<count_t>& countList);
 **/
<|endoftext|>


<commit_before>#include "ByteBuffer.h"

ByteBuffer::ByteBuffer(bufsize_t v_size, bufsize_t v_padding)
    : content(NULL), contentSize(v_size), padding(v_padding)
{
    if (v_size == 0) throw BufferException("Zero size requested");

    this->content =  allocContent(v_size, v_padding);
    this->contentSize = v_size;
}

ByteBuffer::ByteBuffer(AbstractByteBuffer *v_parent, offset_t v_offset, bufsize_t v_size, bufsize_t v_padding)
{
    if (v_parent == NULL) throw BufferException("Cannot make subBuffer for NULL buffer!");
    if (v_size == 0) throw BufferException("Cannot make 0 size buffer!");

    bufsize_t parentSize = v_parent->getContentSize();

    bufsize_t copySize = v_size < parentSize ? v_size : parentSize;
    bufsize_t allocSize = v_size > parentSize ? v_size : parentSize;

    BYTE *bContent = v_parent->getContentAt(v_offset, copySize);
    if (bContent == NULL) throw BufferException("Cannot make Buffer for NULL content!");

    this->content =  allocContent(allocSize, v_padding);
    this->contentSize = allocSize;

    memcpy(this->content, bContent, copySize);
    TRACE();
}

BYTE* ByteBuffer::allocContent(bufsize_t v_size, bufsize_t v_padding)
{
    if (v_size == 0) throw BufferException("Zero size requested");
    bufsize_t allocSize = v_size + v_padding;
    BYTE* content =  (BYTE*) calloc(sizeof(BYTE), allocSize);
    if (content == NULL) throw BufferException("Cannot allocate buffer of size: 0x" + QString::number(allocSize, 16));

    return content;
}

bool ByteBuffer::resize(bufsize_t newSize)
{
    BYTE *newContent = NULL;
    try {
        newContent = allocContent(newSize, this->padding);
    } catch(BufferException &e) {
        newContent = NULL;
    }
    if (newContent == NULL) return false;

    BYTE *oldContent = this->content;
    bufsize_t oldSize = this->contentSize;
    bufsize_t copySize = newSize < oldSize ? newSize : oldSize;

    memcpy(newContent, oldContent, copySize);

    this->content =  newContent;
    this->contentSize = newSize;
    delete [] oldContent;
}

ByteBuffer::~ByteBuffer()
{
    free(this->content);
    TRACE();
}

<commit_msg>[BUGFIX] In ByteBuffer::resize - added missing returns<commit_after>#include "ByteBuffer.h"

ByteBuffer::ByteBuffer(bufsize_t v_size, bufsize_t v_padding)
    : content(NULL), contentSize(v_size), padding(v_padding)
{
    if (v_size == 0) throw BufferException("Zero size requested");

    this->content =  allocContent(v_size, v_padding);
    this->contentSize = v_size;
}

ByteBuffer::ByteBuffer(AbstractByteBuffer *v_parent, offset_t v_offset, bufsize_t v_size, bufsize_t v_padding)
{
    if (v_parent == NULL) throw BufferException("Cannot make subBuffer for NULL buffer!");
    if (v_size == 0) throw BufferException("Cannot make 0 size buffer!");

    bufsize_t parentSize = v_parent->getContentSize();

    bufsize_t copySize = v_size < parentSize ? v_size : parentSize;
    bufsize_t allocSize = v_size > parentSize ? v_size : parentSize;

    BYTE *bContent = v_parent->getContentAt(v_offset, copySize);
    if (bContent == NULL) throw BufferException("Cannot make Buffer for NULL content!");

    this->content =  allocContent(allocSize, v_padding);
    this->contentSize = allocSize;

    memcpy(this->content, bContent, copySize);
    TRACE();
}

BYTE* ByteBuffer::allocContent(bufsize_t v_size, bufsize_t v_padding)
{
    if (v_size == 0) throw BufferException("Zero size requested");
    bufsize_t allocSize = v_size + v_padding;
    BYTE* content =  (BYTE*) calloc(sizeof(BYTE), allocSize);
    if (content == NULL) throw BufferException("Cannot allocate buffer of size: 0x" + QString::number(allocSize, 16));

    return content;
}

bool ByteBuffer::resize(bufsize_t newSize)
{
    if (newSize == this->contentSize) return true;

    BYTE *newContent = NULL;
    try {
        newContent = allocContent(newSize, this->padding);
    } catch(BufferException &e) {
        newContent = NULL;
    }
    if (newContent == NULL) return false;

    BYTE *oldContent = this->content;
    bufsize_t oldSize = this->contentSize;
    bufsize_t copySize = newSize < oldSize ? newSize : oldSize;

    memcpy(newContent, oldContent, copySize);

    this->content =  newContent;
    this->contentSize = newSize;
    delete [] oldContent;
    return true;
}

ByteBuffer::~ByteBuffer()
{
    free(this->content);
    TRACE();
}

<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <memory>

#include "webrtc/modules/desktop_capture/cropped_desktop_frame.h"

#include "webrtc/rtc_base/checks.h"
#include "webrtc/rtc_base/constructormagic.h"

namespace webrtc {

// A DesktopFrame that is a sub-rect of another DesktopFrame.
class CroppedDesktopFrame : public DesktopFrame {
 public:
  CroppedDesktopFrame(std::unique_ptr<DesktopFrame> frame,
                      const DesktopRect& rect);

 private:
  const std::unique_ptr<DesktopFrame> frame_;

  RTC_DISALLOW_COPY_AND_ASSIGN(CroppedDesktopFrame);
};

std::unique_ptr<DesktopFrame> CreateCroppedDesktopFrame(
    std::unique_ptr<DesktopFrame> frame,
    const DesktopRect& rect) {
  RTC_DCHECK(frame);

  if (!DesktopRect::MakeSize(frame->size()).ContainsRect(rect)) {
    return nullptr;
  }

  if (frame->size().equals(rect.size())) {
    return frame;
  }

  return std::unique_ptr<DesktopFrame>(
      new CroppedDesktopFrame(std::move(frame), rect));
}

CroppedDesktopFrame::CroppedDesktopFrame(std::unique_ptr<DesktopFrame> frame,
                                         const DesktopRect& rect)
    : DesktopFrame(frame->size(),
                   frame->stride(),
                   frame->GetFrameDataAtPos(rect.top_left()),
                   frame->shared_memory()),
      frame_(std::move(frame)) {
  MoveFrameInfoFrom(frame_.get());
  set_top_left(frame_->top_left().add(rect.top_left()));
  mutable_updated_region()->IntersectWith(rect);
  mutable_updated_region()->Translate(-rect.left(), -rect.top());
}

}  // namespace webrtc
<commit_msg>CroppedDesktopFrame wrongly sets the DesktopFrame::size()<commit_after>/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <memory>

#include "webrtc/modules/desktop_capture/cropped_desktop_frame.h"

#include "webrtc/rtc_base/checks.h"
#include "webrtc/rtc_base/constructormagic.h"

namespace webrtc {

// A DesktopFrame that is a sub-rect of another DesktopFrame.
class CroppedDesktopFrame : public DesktopFrame {
 public:
  CroppedDesktopFrame(std::unique_ptr<DesktopFrame> frame,
                      const DesktopRect& rect);

 private:
  const std::unique_ptr<DesktopFrame> frame_;

  RTC_DISALLOW_COPY_AND_ASSIGN(CroppedDesktopFrame);
};

std::unique_ptr<DesktopFrame> CreateCroppedDesktopFrame(
    std::unique_ptr<DesktopFrame> frame,
    const DesktopRect& rect) {
  RTC_DCHECK(frame);

  if (!DesktopRect::MakeSize(frame->size()).ContainsRect(rect)) {
    return nullptr;
  }

  if (frame->size().equals(rect.size())) {
    return frame;
  }

  return std::unique_ptr<DesktopFrame>(
      new CroppedDesktopFrame(std::move(frame), rect));
}

CroppedDesktopFrame::CroppedDesktopFrame(std::unique_ptr<DesktopFrame> frame,
                                         const DesktopRect& rect)
    : DesktopFrame(rect.size(),
                   frame->stride(),
                   frame->GetFrameDataAtPos(rect.top_left()),
                   frame->shared_memory()),
      frame_(std::move(frame)) {
  MoveFrameInfoFrom(frame_.get());
  set_top_left(frame_->top_left().add(rect.top_left()));
  mutable_updated_region()->IntersectWith(rect);
  mutable_updated_region()->Translate(-rect.left(), -rect.top());
}

}  // namespace webrtc
<|endoftext|>


<commit_before>// Tests for GDAL's VSI virtual file system.
//
//   http://trac.osgeo.org/gdal/browser/trunk/gdal/port/cpl_vsil.cpp
//
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <algorithm>
#include <memory>

#include "gunit.h"
#include "gcore/gdal_priv.h"
#include "port/cpl_vsi_virtual.h"

using std::fill;

namespace {

// Create a fake handler for testing that provides a stat
// that always succeeds and an open that always fails.
class DummyFilesystem : public VSIFilesystemHandler {
 public:
  DummyFilesystem();
  virtual VSIVirtualHandle *Open(const char *path, const char *access);
  virtual int Stat(const char *path, VSIStatBufL *stat, int nFlags);

  // Record what happened during the test.
  string open_last_path_;
  string open_last_access_;
  string stat_last_path_;
};

DummyFilesystem::DummyFilesystem() {}

VSIVirtualHandle *DummyFilesystem::Open(const char *path, const char *access) {
  open_last_path_ = path;
  open_last_access_ = access;
  return nullptr;
}

int DummyFilesystem::Stat(const char *path, VSIStatBufL *stat, int nFlags) {
  stat_last_path_ = path;
  memset(stat, 0, sizeof(*stat));

  // Giving a size and mode is enough to fake that the file exists.
  stat->st_size = 1000;
  stat->st_mode = 33184;

  return 0;
}

// Tests adding a driver and calling open.
TEST(CplVsiFileManagerTest, FailToOpenExistingFile) {
  GDALAllRegister();

  // handler will be deleted by the VSIFilesystemHandler destructor.
  DummyFilesystem *dummy_handler(new DummyFilesystem);
  VSIFileManager::InstallHandler("/dummy/", dummy_handler);

  const string path("/dummy/foo.tif");

  GDALDataset *foo =
      reinterpret_cast<GDALDataset*>(GDALOpen(path.c_str(), GA_ReadOnly));

  ASSERT_EQ(nullptr, foo);
  // Also searches for /dummy/foo.tif.xml in GDAL 2.*.
  ASSERT_TRUE(dummy_handler->stat_last_path_.find(path) != std::string::npos);
  ASSERT_TRUE(dummy_handler->open_last_path_.find(path) != std::string::npos);
  ASSERT_STREQ("rb", dummy_handler->open_last_access_.c_str());
}

// Tests the VSI functions for non-existent file paths.
TEST(CplVSIFunctions, VSIFilesAndPathLocalDoesNotExist) {
  ASSERT_EQ(nullptr, VSIReadDir("/does/not/exist0"));
  ASSERT_EQ(nullptr, VSIReadDirRecursive("/does/not/exist1"));
  ASSERT_EQ(-1, VSIMkdir("/does/not/exist2", 0755));
  ASSERT_EQ(-1, VSIUnlink("/does/not/exist3"));
  ASSERT_EQ(-1, VSIRmdir("/does/not/exist4"));

  std::unique_ptr<VSIStatBufL> stat_buf(new VSIStatBufL);

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(-1, VSIStatL("/does/not/exist5", stat_buf.get()));
  ASSERT_EQ(0, stat_buf->st_mode);
  ASSERT_EQ(0, stat_buf->st_size);

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(-1, VSIStatExL("/does/not/exist6", stat_buf.get(), 0));
  ASSERT_EQ(0, stat_buf->st_mode);
  ASSERT_EQ(0, stat_buf->st_size);
}

// Tests most of the file operations by creating a directory and a file.
TEST(CplVSIFunctions, VSIFileAndDirOpsLocal) {
  const string temp_dir("/vsimem");

  std::unique_ptr<VSIStatBufL> stat_buf(new VSIStatBufL);
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(temp_dir.c_str(), stat_buf.get()));

  const string src_dir(temp_dir + "/foo");
  const string dst_dir(temp_dir + "/bar");
  ASSERT_EQ(0, VSIMkdir(src_dir.c_str(), 0755));
  ASSERT_EQ(0, VSIRename(src_dir.c_str(), dst_dir.c_str()));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(dst_dir.c_str(), stat_buf.get()));
  ASSERT_EQ(stat_buf->st_mode, 040755);

  const int case_sensitive = VSIIsCaseSensitiveFS(temp_dir.c_str());
  ASSERT_TRUE((0 == case_sensitive || 1 == case_sensitive));

  const string filename(dst_dir + "/baz.txt");

  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);

  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(-1, VSIFSeekL(file, -1, SEEK_SET));
  ASSERT_EQ(-1, VSIFSeekL(file, -1, SEEK_CUR));
  ASSERT_EQ(-1, VSIFSeekL(file, -1, SEEK_END));

  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_SET));
  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_CUR));
  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_END));

  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(3, VSIFWriteL("012", 1, 3, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(3, VSIFTellL(file));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(3, stat_buf->st_size);

  VSIRewindL(file);
  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(0, VSIFCloseL(file));

  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);

  char buf[43] = {};
  ASSERT_EQ(1, VSIFReadL(buf, 1, 1, file));
  ASSERT_STREQ("0", buf);
  ASSERT_EQ(false, bool(VSIFEofL(file)));

  ASSERT_EQ(2, VSIFReadL(buf, 1, 10, file));
  ASSERT_EQ(true, bool(VSIFEofL(file)));

  ASSERT_EQ(0, VSIFSeekL(file, -2, SEEK_CUR));
  ASSERT_EQ(false, bool(VSIFEofL(file)));
  ASSERT_EQ(1, VSIFTellL(file));

  // TODO(schwehr): Why does this work this way?
  // Seeking past the end.
  ASSERT_EQ(0, VSIFSeekL(file, 5, SEEK_CUR));
  ASSERT_EQ(6, VSIFTellL(file));
  ASSERT_EQ(false, bool(VSIFEofL(file)));

  ASSERT_EQ(0, VSIFCloseL(file));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(3, stat_buf->st_size);

  // Cannot truncate when opened read-only.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(-1, VSIFTruncateL(file, 1));
  ASSERT_EQ(0, VSIFCloseL(file));

  file = VSIFOpenL(filename.c_str(), "r+");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(3, VSIFReadL(buf, 1, 10, file));
  ASSERT_EQ(true, bool(VSIFEofL(file)));
  ASSERT_EQ(0, VSIFTruncateL(file, 1));
  ASSERT_EQ(true, bool(VSIFEofL(file)));
  ASSERT_EQ(0, VSIFFlushL(file));
  // Truncate does not modify the offset of open files.
  ASSERT_EQ(3, VSIFTellL(file));
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(1, stat_buf->st_size);

  ASSERT_EQ(1, VSIFWriteL("3", 1, 1, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(4, VSIFTellL(file));

  VSIRewindL(file);
  ASSERT_EQ(0, VSIFTellL(file));
  ASSERT_EQ(4, VSIFReadL(buf, 1, 10, file));

  ASSERT_STREQ("0", buf);
  ASSERT_EQ('\0', buf[1]);
  ASSERT_EQ('\0', buf[2]);
  ASSERT_STREQ("3", buf+3);

  // Truncate can also grow a file.
  ASSERT_EQ(0, VSIFTruncateL(file, 42));
  ASSERT_EQ(0, VSIFFlushL(file));
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(0, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(42, stat_buf->st_size);

  // Truncate does not move the file index forward or backwards.
  ASSERT_EQ(4, VSIFTellL(file));
  VSIRewindL(file);
  ASSERT_EQ(42, VSIFReadL(buf, 1, 43, file));
  ASSERT_STREQ("0", buf);
  ASSERT_EQ('\0', buf[1]);
  ASSERT_EQ('\0', buf[2]);
  ASSERT_STREQ("3", buf+3);
  ASSERT_EQ('\0', buf[23]);
  ASSERT_EQ('\0', buf[24]);

  ASSERT_EQ(0, VSIFCloseL(file));

  // Fail to rmdir with contents.
  ASSERT_EQ(-1, VSIRmdir(dst_dir.c_str()));

  ASSERT_EQ(0, VSIUnlink(filename.c_str()));
  ASSERT_EQ(0, VSIRmdir(dst_dir.c_str()));
}

// Test reading more than is available in a file on a normal filesystem.
// Most likely uses Read from cpl_vsil_unix_stdio_64.cpp.
TEST(CplVSIFunctions, VSIReadTooMuchLocalFileSystem) {
  const string temp_dir("/vsimem");
  const string filename(temp_dir + "/short.txt");
  const string contents("012\n\n");
  const int length = contents.length();

  // Write a small amount of data.
  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(length, VSIFWriteL(contents.c_str(), 1, length, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;

  const int buf_size = 1024;
  char buf[buf_size] = {};

  // Read the file normally.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, length, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;
  fill(buf, buf + buf_size, 0);

  // Try to read way too much.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, buf_size, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;
  fill(buf, buf + buf_size, 0);

  // Try to read way too much with swapped size and count.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  // 1, 1024 -> 1024, 1
  // It will be unable to read 1 block of 1024 bytes.
  EXPECT_EQ(0, VSIFReadL(buf, buf_size, 1, file));
  // The read reports nothing read, but buf is set.
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
}

// Test reading more than is available in a memory filesystem.
TEST(CplVSIFunctions, VSIFReadLTooMuchVsimem) {
  const string filename("/vsimem/short.txt");
  const string contents("\n\nabc");
  const int length = contents.length();

  // Write a small amount of data.
  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(length, VSIFWriteL(contents.c_str(), 1, length, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;

  const int buf_size = 1024;
  char buf[buf_size] = {};

  // Try to read way too much.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, buf_size, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
}

// TODO(schwehr): Test the rest of cpl_vsil.cpp.

}  // namespace
<commit_msg>Minor fixes for cpl_vsil_test.cc<commit_after>// Tests for GDAL's VSI virtual file system.
//
//   http://trac.osgeo.org/gdal/browser/trunk/gdal/port/cpl_vsil.cpp
//
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <algorithm>
#include <memory>
#include <string>
#include <vector>

#include "gcore/gdal.h"
#include "gcore/gdal_priv.h"
#include "gunit.h"
#include "port/cpl_vsi.h"
#include "port/cpl_vsi_virtual.h"

using std::fill;

namespace {

// Create a fake handler for testing that provides a stat
// that always succeeds and an open that always fails.
class DummyFilesystem : public VSIFilesystemHandler {
 public:
  DummyFilesystem();
  virtual VSIVirtualHandle *Open(const char *path, const char *access);
  virtual int Stat(const char *path, VSIStatBufL *stat, int nFlags);

  // Record what happened during the test.
  string open_last_path_;
  string open_last_access_;
  string stat_last_path_;
};

DummyFilesystem::DummyFilesystem() {}

VSIVirtualHandle *DummyFilesystem::Open(const char *path, const char *access) {
  open_last_path_ = path;
  open_last_access_ = access;
  return nullptr;
}

int DummyFilesystem::Stat(const char *path, VSIStatBufL *stat, int nFlags) {
  stat_last_path_ = path;
  memset(stat, 0, sizeof(*stat));

  // Giving a size and mode is enough to fake that the file exists.
  stat->st_size = 1000;
  stat->st_mode = 33184;

  return 0;
}

// Tests adding a driver and calling open.
TEST(CplVsiFileManagerTest, FailToOpenExistingFile) {
  GDALAllRegister();

  // handler will be deleted by the VSIFilesystemHandler destructor.
  DummyFilesystem *dummy_handler(new DummyFilesystem);
  VSIFileManager::InstallHandler("/dummy/", dummy_handler);

  const string path("/dummy/foo.tif");

  GDALDataset *foo =
      reinterpret_cast<GDALDataset*>(GDALOpen(path.c_str(), GA_ReadOnly));

  ASSERT_EQ(nullptr, foo);
  // Also searches for /dummy/foo.tif.xml in GDAL 2.*.
  ASSERT_TRUE(dummy_handler->stat_last_path_.find(path) != std::string::npos);
  ASSERT_TRUE(dummy_handler->open_last_path_.find(path) != std::string::npos);
  ASSERT_STREQ("rb", dummy_handler->open_last_access_.c_str());
}

// Tests the VSI functions for non-existent file paths.
TEST(CplVSIFunctions, VSIFilesAndPathLocalDoesNotExist) {
  ASSERT_EQ(nullptr, VSIReadDir("/does/not/exist0"));
  ASSERT_EQ(nullptr, VSIReadDirRecursive("/does/not/exist1"));
  ASSERT_EQ(-1, VSIMkdir("/does/not/exist2", 0755));
  ASSERT_EQ(-1, VSIUnlink("/does/not/exist3"));
  ASSERT_EQ(-1, VSIRmdir("/does/not/exist4"));

  std::unique_ptr<VSIStatBufL> stat_buf(new VSIStatBufL);

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(-1, VSIStatL("/does/not/exist5", stat_buf.get()));
  ASSERT_EQ(0, stat_buf->st_mode);
  ASSERT_EQ(0, stat_buf->st_size);

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(-1, VSIStatExL("/does/not/exist6", stat_buf.get(), 0));
  ASSERT_EQ(0, stat_buf->st_mode);
  ASSERT_EQ(0, stat_buf->st_size);
}

// Tests most of the file operations by creating a directory and a file.
TEST(CplVSIFunctions, VSIFileAndDirOpsLocal) {
  const string temp_dir(FLAGS_test_tmpdir);

  constexpr int kSuccess = 0;
  constexpr int kFailure = -1;

  std::unique_ptr<VSIStatBufL> stat_buf(new VSIStatBufL);
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(temp_dir.c_str(), stat_buf.get()));

  const string src_dir(temp_dir + "/foo");
  const string dst_dir(temp_dir + "/bar");
  ASSERT_EQ(kSuccess, VSIMkdir(src_dir.c_str(), 0755));
  ASSERT_EQ(kSuccess, VSIRename(src_dir.c_str(), dst_dir.c_str()));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(dst_dir.c_str(), stat_buf.get()));
  ASSERT_EQ(stat_buf->st_mode, 040755);

  const int case_sensitive = VSIIsCaseSensitiveFS(temp_dir.c_str());
  ASSERT_TRUE((0 == case_sensitive || 1 == case_sensitive));

  const string filename(dst_dir + "/baz.txt");

  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);

  ASSERT_EQ(0, VSIFTellL(file));

  // Cannot seek to before the file or to
  // std::numeric_limits<vsi_l_offset>::max()
  ASSERT_EQ(kFailure, VSIFSeekL(file, -1, SEEK_SET));
  ASSERT_EQ(kFailure, VSIFSeekL(file, -1, SEEK_CUR));
  ASSERT_EQ(kFailure, VSIFSeekL(file, -1, SEEK_END));

  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_SET));
  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_CUR));
  ASSERT_EQ(0, VSIFSeekL(file, 0, SEEK_END));

  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(3, VSIFWriteL("012", 1, 3, file));
  ASSERT_EQ(kSuccess, VSIFFlushL(file));
  ASSERT_EQ(3, VSIFTellL(file));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(3, stat_buf->st_size);

  VSIRewindL(file);
  ASSERT_EQ(0, VSIFTellL(file));

  ASSERT_EQ(kSuccess, VSIFCloseL(file));

  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);

  char buf[43] = {};
  ASSERT_EQ(1, VSIFReadL(buf, 1, 1, file));
  ASSERT_STREQ("0", buf);
  ASSERT_EQ(false, bool(VSIFEofL(file)));

  ASSERT_EQ(2, VSIFReadL(buf, 1, 10, file));
  ASSERT_EQ(true, bool(VSIFEofL(file)));

  ASSERT_EQ(kSuccess, VSIFSeekL(file, -2, SEEK_CUR));
  ASSERT_EQ(false, bool(VSIFEofL(file)));
  ASSERT_EQ(1, VSIFTellL(file));

  // TODO(schwehr): Why does this work this way?
  // Seeking past the end.
  ASSERT_EQ(kSuccess, VSIFSeekL(file, 5, SEEK_CUR));
  ASSERT_EQ(6, VSIFTellL(file));
  ASSERT_EQ(false, bool(VSIFEofL(file)));

  ASSERT_EQ(kSuccess, VSIFCloseL(file));

  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(3, stat_buf->st_size);

  // Cannot truncate when opened read-only.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(-1, VSIFTruncateL(file, 1));
  ASSERT_EQ(kSuccess, VSIFCloseL(file));

  file = VSIFOpenL(filename.c_str(), "r+");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(3, VSIFReadL(buf, 1, 10, file));
  ASSERT_EQ(true, bool(VSIFEofL(file)));
  ASSERT_EQ(kSuccess, VSIFTruncateL(file, 1));
  ASSERT_EQ(true, bool(VSIFEofL(file)));
  ASSERT_EQ(kSuccess, VSIFFlushL(file));
  // Truncate does not modify the offset of open files.
  ASSERT_EQ(3, VSIFTellL(file));
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(1, stat_buf->st_size);

  ASSERT_EQ(1, VSIFWriteL("3", 1, 1, file));
  ASSERT_EQ(kSuccess, VSIFFlushL(file));
  ASSERT_EQ(4, VSIFTellL(file));

  VSIRewindL(file);
  ASSERT_EQ(0, VSIFTellL(file));
  ASSERT_EQ(4, VSIFReadL(buf, 1, 10, file));

  ASSERT_STREQ("0", buf);
  ASSERT_EQ('\0', buf[1]);
  ASSERT_EQ('\0', buf[2]);
  ASSERT_STREQ("3", buf+3);

  // Truncate can also grow a file.
  ASSERT_EQ(kSuccess, VSIFTruncateL(file, 42));
  ASSERT_EQ(kSuccess, VSIFFlushL(file));
  memset(stat_buf.get(), 0, sizeof(VSIStatBufL));
  ASSERT_EQ(kSuccess, VSIStatL(filename.c_str(), stat_buf.get()));
  ASSERT_EQ(42, stat_buf->st_size);

  // Truncate does not move the file index forward or backwards.
  ASSERT_EQ(4, VSIFTellL(file));
  VSIRewindL(file);
  ASSERT_EQ(42, VSIFReadL(buf, 1, 43, file));
  ASSERT_STREQ("0", buf);
  ASSERT_EQ('\0', buf[1]);
  ASSERT_EQ('\0', buf[2]);
  ASSERT_STREQ("3", buf+3);
  ASSERT_EQ('\0', buf[23]);
  ASSERT_EQ('\0', buf[24]);

  ASSERT_EQ(kSuccess, VSIFCloseL(file));

  // Fail to rmdir with contents.
  ASSERT_EQ(-1, VSIRmdir(dst_dir.c_str()));

  ASSERT_EQ(kSuccess, VSIUnlink(filename.c_str()));
  ASSERT_EQ(kSuccess, VSIRmdir(dst_dir.c_str()));
}

// Test reading more than is available in a file on a normal filesystem.
// Most likely uses Read from cpl_vsil_unix_stdio_64.cpp.
TEST(CplVSIFunctions, VSIReadTooMuchLocalFileSystem) {
  const string temp_dir(FLAGS_test_tmpdir);
  const string filename(temp_dir + "/short.txt");
  const string contents("012\n\n");
  const int length = contents.length();

  // Write a small amount of data.
  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(length, VSIFWriteL(contents.c_str(), 1, length, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;

  const int buf_size = 1024;
  char buf[buf_size] = {};

  // Read the file normally.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, length, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;
  fill(buf, buf + buf_size, 0);

  // Try to read way too much.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, buf_size, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;
  fill(buf, buf + buf_size, 0);

  // Try to read way too much with swapped size and count.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  // 1, 1024 -> 1024, 1
  // It will be unable to read 1 block of 1024 bytes.
  EXPECT_EQ(0, VSIFReadL(buf, buf_size, 1, file));
  // The read reports nothing read, but buf is set.
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
}

// Test reading more than is available in a memory filesystem.
TEST(CplVSIFunctions, VSIFReadLTooMuchVsimem) {
  const string filename("/vsimem/short.txt");
  const string contents("\n\nabc");
  const int length = contents.length();

  // Write a small amount of data.
  VSILFILE *file = VSIFOpenL(filename.c_str(), "w");
  ASSERT_NE(nullptr, file);
  ASSERT_EQ(length, VSIFWriteL(contents.c_str(), 1, length, file));
  ASSERT_EQ(0, VSIFFlushL(file));
  ASSERT_EQ(0, VSIFCloseL(file));
  file = nullptr;

  const int buf_size = 1024;
  char buf[buf_size] = {};

  // Try to read way too much.
  file = VSIFOpenL(filename.c_str(), "r");
  ASSERT_NE(nullptr, file);
  EXPECT_EQ(length, VSIFReadL(buf, 1, buf_size, file));
  EXPECT_STREQ(contents.c_str(), buf);
  ASSERT_EQ(0, VSIFCloseL(file));
}

// TODO(schwehr): Test the rest of cpl_vsil.cpp.

}  // namespace
<|endoftext|>


<commit_before>/**
 * \file
 * \brief DynamicThread class header
 *
 * \author Copyright (C) 2015 Kamil Szczygiel http://www.distortec.com http://www.freddiechopin.info
 *
 * \par License
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
 * distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * \date 2015-11-09
 */

#ifndef INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_
#define INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_

#include "distortos/DynamicSignalsReceiver.hpp"
#include "distortos/Thread.hpp"

#include "distortos/memory/storageDeleter.hpp"

namespace distortos
{

/**
 * \brief DynamicThreadParameters struct is a helper with parameters for DynamicThread's constructor
 *
 * This struct is a replacement for overloads of DynamicThread's constructor and makeDynamicThread() which -
 * unfortunately - cannot be used, as they would lead to compilation errors due to ambiguity.
 */

struct DynamicThreadParameters
{
	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] canReceiveSignalss selects whether reception of signals is enabled (true) or disabled (false) for
	 * this thread
	 * \param [in] queuedSignalss is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActionss is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicyy is the scheduling policy of the thread
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const bool canReceiveSignalss,
			const size_t queuedSignalss, const size_t signalActionss, const uint8_t priorityy,
			const SchedulingPolicy schedulingPolicyy) :
			stackSize{stackSizee},
			canReceiveSignals{canReceiveSignalss},
			queuedSignals{queuedSignalss},
			signalActions{signalActionss},
			priority{priorityy},
			schedulingPolicy{schedulingPolicyy}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] canReceiveSignalss selects whether reception of signals is enabled (true) or disabled (false) for
	 * this thread
	 * \param [in] queuedSignalss is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActionss is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const bool canReceiveSignalss,
			const size_t queuedSignalss, const size_t signalActionss, const uint8_t priorityy) :
			stackSize{stackSizee},
			canReceiveSignals{canReceiveSignalss},
			queuedSignals{queuedSignalss},
			signalActions{signalActionss},
			priority{priorityy},
			schedulingPolicy{SchedulingPolicy::RoundRobin}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicyy is the scheduling policy of the thread
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const uint8_t priorityy,
			const SchedulingPolicy schedulingPolicyy) :
			stackSize{stackSizee},
			canReceiveSignals{},
			queuedSignals{},
			signalActions{},
			priority{priorityy},
			schedulingPolicy{schedulingPolicyy}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const uint8_t priorityy) :
			stackSize{stackSizee},
			canReceiveSignals{},
			queuedSignals{},
			signalActions{},
			priority{priorityy},
			schedulingPolicy{SchedulingPolicy::RoundRobin}
	{

	}

	/// size of stack, bytes
	const size_t stackSize;

	/// selects whether reception of signals is enabled (true) or disabled (false) for this thread
	const bool canReceiveSignals;

	/// max number of queued signals for this thread, relevant only if \a canReceiveSignals == true, 0 to disable
	/// queuing of signals for this thread
	const size_t queuedSignals;

	/// max number of different SignalAction objects for this thread, relevant only if \a canReceiveSignals == true, 0
	/// to disable catching of signals for this thread
	const size_t signalActions;

	/// thread's priority, 0 - lowest, UINT8_MAX - highest
	const uint8_t priority;

	/// scheduling policy of the thread
	SchedulingPolicy schedulingPolicy;
};

/**
 * \brief DynamicThread class is a templated interface for thread that has dynamic storage for stack and internal
 * DynamicSignalsReceiver object.
 *
 * \param Function is the function that will be executed in separate thread
 * \param Args are the arguments for \a Function
 */

template<typename Function, typename... Args>
class DynamicThread : public Thread<Function, Args...>
{
public:

	/// base of DynamicThread
	using Base = Thread<Function, Args...>;

	/**
	 * \brief DynamicThread's constructor
	 *
	 * \param [in] stackSize is the size of stack, bytes
	 * \param [in] canReceiveSignals selects whether reception of signals is enabled (true) or disabled (false) for this
	 * thread
	 * \param [in] queuedSignals is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActions is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priority is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicy is the scheduling policy of the thread
	 * \param [in] function is a function that will be executed in separate thread
	 * \param [in] args are arguments for \a function
	 */

	DynamicThread(size_t stackSize, bool canReceiveSignals, size_t queuedSignals, size_t signalActions,
			uint8_t priority, SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args);

	/**
	 * \brief DynamicThread's constructor
	 *
	 * \param [in] parameters is a DynamicThreadParameters struct with thread parameters
	 * \param [in] function is a function that will be executed in separate thread
	 * \param [in] args are arguments for \a function
	 */

	DynamicThread(const DynamicThreadParameters parameters, Function&& function, Args&&... args) :
			DynamicThread{parameters.stackSize, parameters.canReceiveSignals, parameters.queuedSignals,
					parameters.signalActions, parameters.priority, parameters.schedulingPolicy,
					std::forward<Function>(function), std::forward<Args>(args)...}
	{

	}

	DynamicThread(const DynamicThread&) = delete;
	DynamicThread(DynamicThread&&) = default;
	const DynamicThread& operator=(const DynamicThread&) = delete;
	DynamicThread& operator=(DynamicThread&&) = delete;

private:

	/// internal DynamicSignalsReceiver object
	DynamicSignalsReceiver dynamicSignalsReceiver_;

	/// bound function object
	std::function<void()> boundFunction_;
};

template<typename Function, typename... Args>
DynamicThread<Function, Args...>::DynamicThread(const size_t stackSize, const bool canReceiveSignals,
		const size_t queuedSignals, const size_t signalActions, const uint8_t priority,
		const SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args) :
		Base{{new uint8_t[stackSize], memory::storageDeleter<uint8_t>}, stackSize, priority, schedulingPolicy,
				canReceiveSignals == true ? &dynamicSignalsReceiver_ : nullptr, std::forward<Function>(function),
				std::forward<Args>(args)...},
		dynamicSignalsReceiver_{canReceiveSignals == true ? queuedSignals : 0,
				canReceiveSignals == true ? signalActions : 0},
		boundFunction_{std::bind(std::forward<Function>(function), std::forward<Args>(args)...)}
{

}

/**
 * \brief Helper factory function to make DynamicThread object with deduced template arguments
 *
 * \param Function is the function that will be executed
 * \param Args are the arguments for \a Function
 *
 * \param [in] stackSize is the size of stack, bytes
 * \param [in] canReceiveSignals selects whether reception of signals is enabled (true) or disabled (false) for this
 * thread
 * \param [in] queuedSignals is the max number of queued signals for this thread, relevant only if
 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
 * \param [in] signalActions is the max number of different SignalAction objects for this thread, relevant only if
 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
 * \param [in] priority is the thread's priority, 0 - lowest, UINT8_MAX - highest
 * \param [in] schedulingPolicy is the scheduling policy of the thread
 * \param [in] function is a function that will be executed in separate thread
 * \param [in] args are arguments for \a function
 *
 * \return DynamicThread object with deduced template arguments
 */

template<typename Function, typename... Args>
DynamicThread<Function, Args...> makeDynamicThread(const size_t stackSize, const bool canReceiveSignals,
		const size_t queuedSignals, const size_t signalActions, const uint8_t priority,
		const SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args)
{
	return {stackSize, canReceiveSignals, queuedSignals, signalActions, priority, schedulingPolicy,
			std::forward<Function>(function), std::forward<Args>(args)...};
}

/**
 * \brief Helper factory function to make DynamicThread object with deduced template arguments
 *
 * \param Function is the function that will be executed
 * \param Args are the arguments for \a Function
 *
 * \param [in] parameters is a DynamicThreadParameters struct with thread parameters
 * \param [in] function is a function that will be executed in separate thread
 * \param [in] args are arguments for \a function
 *
 * \return DynamicThread object with deduced template arguments
 */

template<typename Function, typename... Args>
DynamicThread<Function, Args...> makeDynamicThread(const DynamicThreadParameters parameters, Function&& function,
		Args&&... args)
{
	return {parameters, std::forward<Function>(function), std::forward<Args>(args)...};
}

}	// namespace distortos

#endif	// INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_
<commit_msg>DynamicThread: add override of run() virtual function<commit_after>/**
 * \file
 * \brief DynamicThread class header
 *
 * \author Copyright (C) 2015 Kamil Szczygiel http://www.distortec.com http://www.freddiechopin.info
 *
 * \par License
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
 * distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * \date 2015-11-09
 */

#ifndef INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_
#define INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_

#include "distortos/DynamicSignalsReceiver.hpp"
#include "distortos/Thread.hpp"

#include "distortos/memory/storageDeleter.hpp"

namespace distortos
{

/**
 * \brief DynamicThreadParameters struct is a helper with parameters for DynamicThread's constructor
 *
 * This struct is a replacement for overloads of DynamicThread's constructor and makeDynamicThread() which -
 * unfortunately - cannot be used, as they would lead to compilation errors due to ambiguity.
 */

struct DynamicThreadParameters
{
	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] canReceiveSignalss selects whether reception of signals is enabled (true) or disabled (false) for
	 * this thread
	 * \param [in] queuedSignalss is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActionss is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicyy is the scheduling policy of the thread
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const bool canReceiveSignalss,
			const size_t queuedSignalss, const size_t signalActionss, const uint8_t priorityy,
			const SchedulingPolicy schedulingPolicyy) :
			stackSize{stackSizee},
			canReceiveSignals{canReceiveSignalss},
			queuedSignals{queuedSignalss},
			signalActions{signalActionss},
			priority{priorityy},
			schedulingPolicy{schedulingPolicyy}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] canReceiveSignalss selects whether reception of signals is enabled (true) or disabled (false) for
	 * this thread
	 * \param [in] queuedSignalss is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActionss is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const bool canReceiveSignalss,
			const size_t queuedSignalss, const size_t signalActionss, const uint8_t priorityy) :
			stackSize{stackSizee},
			canReceiveSignals{canReceiveSignalss},
			queuedSignals{queuedSignalss},
			signalActions{signalActionss},
			priority{priorityy},
			schedulingPolicy{SchedulingPolicy::RoundRobin}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicyy is the scheduling policy of the thread
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const uint8_t priorityy,
			const SchedulingPolicy schedulingPolicyy) :
			stackSize{stackSizee},
			canReceiveSignals{},
			queuedSignals{},
			signalActions{},
			priority{priorityy},
			schedulingPolicy{schedulingPolicyy}
	{

	}

	/**
	 * \brief DynamicThreadParameters's constructor
	 *
	 * \param [in] stackSizee is the size of stack, bytes
	 * \param [in] priorityy is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 */

	constexpr DynamicThreadParameters(const size_t stackSizee, const uint8_t priorityy) :
			stackSize{stackSizee},
			canReceiveSignals{},
			queuedSignals{},
			signalActions{},
			priority{priorityy},
			schedulingPolicy{SchedulingPolicy::RoundRobin}
	{

	}

	/// size of stack, bytes
	const size_t stackSize;

	/// selects whether reception of signals is enabled (true) or disabled (false) for this thread
	const bool canReceiveSignals;

	/// max number of queued signals for this thread, relevant only if \a canReceiveSignals == true, 0 to disable
	/// queuing of signals for this thread
	const size_t queuedSignals;

	/// max number of different SignalAction objects for this thread, relevant only if \a canReceiveSignals == true, 0
	/// to disable catching of signals for this thread
	const size_t signalActions;

	/// thread's priority, 0 - lowest, UINT8_MAX - highest
	const uint8_t priority;

	/// scheduling policy of the thread
	SchedulingPolicy schedulingPolicy;
};

/**
 * \brief DynamicThread class is a templated interface for thread that has dynamic storage for stack and internal
 * DynamicSignalsReceiver object.
 *
 * \param Function is the function that will be executed in separate thread
 * \param Args are the arguments for \a Function
 */

template<typename Function, typename... Args>
class DynamicThread : public Thread<Function, Args...>
{
public:

	/// base of DynamicThread
	using Base = Thread<Function, Args...>;

	/**
	 * \brief DynamicThread's constructor
	 *
	 * \param [in] stackSize is the size of stack, bytes
	 * \param [in] canReceiveSignals selects whether reception of signals is enabled (true) or disabled (false) for this
	 * thread
	 * \param [in] queuedSignals is the max number of queued signals for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
	 * \param [in] signalActions is the max number of different SignalAction objects for this thread, relevant only if
	 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
	 * \param [in] priority is the thread's priority, 0 - lowest, UINT8_MAX - highest
	 * \param [in] schedulingPolicy is the scheduling policy of the thread
	 * \param [in] function is a function that will be executed in separate thread
	 * \param [in] args are arguments for \a function
	 */

	DynamicThread(size_t stackSize, bool canReceiveSignals, size_t queuedSignals, size_t signalActions,
			uint8_t priority, SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args);

	/**
	 * \brief DynamicThread's constructor
	 *
	 * \param [in] parameters is a DynamicThreadParameters struct with thread parameters
	 * \param [in] function is a function that will be executed in separate thread
	 * \param [in] args are arguments for \a function
	 */

	DynamicThread(const DynamicThreadParameters parameters, Function&& function, Args&&... args) :
			DynamicThread{parameters.stackSize, parameters.canReceiveSignals, parameters.queuedSignals,
					parameters.signalActions, parameters.priority, parameters.schedulingPolicy,
					std::forward<Function>(function), std::forward<Args>(args)...}
	{

	}

	DynamicThread(const DynamicThread&) = delete;
	DynamicThread(DynamicThread&&) = default;
	const DynamicThread& operator=(const DynamicThread&) = delete;
	DynamicThread& operator=(DynamicThread&&) = delete;

private:

	/**
	 * \brief Thread's internal function.
	 *
	 * Executes bound function object.
	 */

	virtual void run() override
	{
		boundFunction_();
	}

	/// internal DynamicSignalsReceiver object
	DynamicSignalsReceiver dynamicSignalsReceiver_;

	/// bound function object
	std::function<void()> boundFunction_;
};

template<typename Function, typename... Args>
DynamicThread<Function, Args...>::DynamicThread(const size_t stackSize, const bool canReceiveSignals,
		const size_t queuedSignals, const size_t signalActions, const uint8_t priority,
		const SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args) :
		Base{{new uint8_t[stackSize], memory::storageDeleter<uint8_t>}, stackSize, priority, schedulingPolicy,
				canReceiveSignals == true ? &dynamicSignalsReceiver_ : nullptr, std::forward<Function>(function),
				std::forward<Args>(args)...},
		dynamicSignalsReceiver_{canReceiveSignals == true ? queuedSignals : 0,
				canReceiveSignals == true ? signalActions : 0},
		boundFunction_{std::bind(std::forward<Function>(function), std::forward<Args>(args)...)}
{

}

/**
 * \brief Helper factory function to make DynamicThread object with deduced template arguments
 *
 * \param Function is the function that will be executed
 * \param Args are the arguments for \a Function
 *
 * \param [in] stackSize is the size of stack, bytes
 * \param [in] canReceiveSignals selects whether reception of signals is enabled (true) or disabled (false) for this
 * thread
 * \param [in] queuedSignals is the max number of queued signals for this thread, relevant only if
 * \a canReceiveSignals == true, 0 to disable queuing of signals for this thread
 * \param [in] signalActions is the max number of different SignalAction objects for this thread, relevant only if
 * \a canReceiveSignals == true, 0 to disable catching of signals for this thread
 * \param [in] priority is the thread's priority, 0 - lowest, UINT8_MAX - highest
 * \param [in] schedulingPolicy is the scheduling policy of the thread
 * \param [in] function is a function that will be executed in separate thread
 * \param [in] args are arguments for \a function
 *
 * \return DynamicThread object with deduced template arguments
 */

template<typename Function, typename... Args>
DynamicThread<Function, Args...> makeDynamicThread(const size_t stackSize, const bool canReceiveSignals,
		const size_t queuedSignals, const size_t signalActions, const uint8_t priority,
		const SchedulingPolicy schedulingPolicy, Function&& function, Args&&... args)
{
	return {stackSize, canReceiveSignals, queuedSignals, signalActions, priority, schedulingPolicy,
			std::forward<Function>(function), std::forward<Args>(args)...};
}

/**
 * \brief Helper factory function to make DynamicThread object with deduced template arguments
 *
 * \param Function is the function that will be executed
 * \param Args are the arguments for \a Function
 *
 * \param [in] parameters is a DynamicThreadParameters struct with thread parameters
 * \param [in] function is a function that will be executed in separate thread
 * \param [in] args are arguments for \a function
 *
 * \return DynamicThread object with deduced template arguments
 */

template<typename Function, typename... Args>
DynamicThread<Function, Args...> makeDynamicThread(const DynamicThreadParameters parameters, Function&& function,
		Args&&... args)
{
	return {parameters, std::forward<Function>(function), std::forward<Args>(args)...};
}

}	// namespace distortos

#endif	// INCLUDE_DISTORTOS_DYNAMICTHREAD_HPP_
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// TODO(wesm): LLVM 7 produces pesky C4244 that disable pragmas around the LLVM
// includes seem to not fix as with LLVM 6
#if defined(_MSC_VER)
#pragma warning(disable : 4244)
#endif

#include "gandiva/engine.h"

#include <iostream>
#include <memory>
#include <mutex>
#include <sstream>
#include <string>
#include <unordered_set>
#include <utility>

#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4141)
#pragma warning(disable : 4146)
#pragma warning(disable : 4244)
#pragma warning(disable : 4267)
#pragma warning(disable : 4624)
#endif

#include <llvm/Analysis/Passes.h>
#include <llvm/Analysis/TargetTransformInfo.h>
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/LLVMContext.h>
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/Verifier.h>
#include <llvm/Linker/Linker.h>
#include <llvm/MC/SubtargetFeature.h>
#include <llvm/Support/DynamicLibrary.h>
#include <llvm/Support/Host.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_ostream.h>
#include <llvm/Transforms/IPO.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#include <llvm/Transforms/InstCombine/InstCombine.h>
#include <llvm/Transforms/Scalar.h>
#include <llvm/Transforms/Scalar/GVN.h>
#include <llvm/Transforms/Utils.h>
#include <llvm/Transforms/Vectorize.h>

#if defined(_MSC_VER)
#pragma warning(pop)
#endif

#include "gandiva/configuration.h"
#include "gandiva/decimal_ir.h"
#include "gandiva/exported_funcs_registry.h"

#include "arrow/util/make_unique.h"

namespace gandiva {

extern const unsigned char kPrecompiledBitcode[];
extern const size_t kPrecompiledBitcodeSize;

std::once_flag llvm_init_once_flag;
static bool llvm_init = false;

void Engine::InitOnce() {
  DCHECK_EQ(llvm_init, false);

  llvm::InitializeNativeTarget();
  llvm::InitializeNativeTargetAsmPrinter();
  llvm::InitializeNativeTargetAsmParser();
  llvm::InitializeNativeTargetDisassembler();
  llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr);

  llvm_init = true;
}

Engine::Engine(const std::shared_ptr<Configuration>& conf,
               std::unique_ptr<llvm::LLVMContext> ctx,
               std::unique_ptr<llvm::ExecutionEngine> engine, llvm::Module* module)
    : context_(std::move(ctx)),
      execution_engine_(std::move(engine)),
      ir_builder_(arrow::internal::make_unique<llvm::IRBuilder<>>(*context_)),
      module_(module),
      types_(*context_),
      optimize_(conf->optimize()) {}

Status Engine::Init() {
  // Add mappings for functions that can be accessed from LLVM/IR module.
  AddGlobalMappings();

  ARROW_RETURN_NOT_OK(LoadPreCompiledIR());
  ARROW_RETURN_NOT_OK(DecimalIR::AddFunctions(this));

  return Status::OK();
}

/// factory method to construct the engine.
Status Engine::Make(const std::shared_ptr<Configuration>& conf,
                    std::unique_ptr<Engine>* out) {
  std::call_once(llvm_init_once_flag, InitOnce);

  auto ctx = arrow::internal::make_unique<llvm::LLVMContext>();
  auto module = arrow::internal::make_unique<llvm::Module>("codegen", *ctx);

  // Capture before moving, ExecutionEngine does not allow retrieving the
  // original Module.
  auto module_ptr = module.get();

  auto opt_level =
      conf->optimize() ? llvm::CodeGenOpt::Aggressive : llvm::CodeGenOpt::None;
  // Note that the lifetime of the error string is not captured by the
  // ExecutionEngine but only for the lifetime of the builder. Found by
  // inspecting LLVM sources.
  std::string builder_error;
  std::unique_ptr<llvm::ExecutionEngine> exec_engine{
      llvm::EngineBuilder(std::move(module))
          .setMCPU(llvm::sys::getHostCPUName())
          .setEngineKind(llvm::EngineKind::JIT)
          .setOptLevel(opt_level)
          .setErrorStr(&builder_error)
          .create()};

  if (exec_engine == nullptr) {
    return Status::CodeGenError("Could not instantiate llvm::ExecutionEngine: ",
                                builder_error);
  }

  std::unique_ptr<Engine> engine{
      new Engine(conf, std::move(ctx), std::move(exec_engine), module_ptr)};
  ARROW_RETURN_NOT_OK(engine->Init());
  *out = std::move(engine);
  return Status::OK();
}

// This method was modified from its original version for a part of MLIR
// Original source from
// https://github.com/llvm/llvm-project/blob/9f2ce5b915a505a5488a5cf91bb0a8efa9ddfff7/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
// The original copyright notice follows.

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

static void SetDataLayout(llvm::Module* module) {
  auto target_triple = llvm::sys::getDefaultTargetTriple();
  std::string error_message;
  auto target = llvm::TargetRegistry::lookupTarget(target_triple, error_message);
  if (!target) {
    return;
  }

  std::string cpu(llvm::sys::getHostCPUName());
  llvm::SubtargetFeatures features;
  llvm::StringMap<bool> host_features;

  if (llvm::sys::getHostCPUFeatures(host_features)) {
    for (auto& f : host_features) {
      features.AddFeature(f.first(), f.second);
    }
  }

  std::unique_ptr<llvm::TargetMachine> machine(
      target->createTargetMachine(target_triple, cpu, features.getString(), {}, {}));

  module->setDataLayout(machine->createDataLayout());
}
// end of the mofified method from MLIR

// Handling for pre-compiled IR libraries.
Status Engine::LoadPreCompiledIR() {
  auto bitcode = llvm::StringRef(reinterpret_cast<const char*>(kPrecompiledBitcode),
                                 kPrecompiledBitcodeSize);

  /// Read from file into memory buffer.
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer_or_error =
      llvm::MemoryBuffer::getMemBuffer(bitcode, "precompiled", false);

  ARROW_RETURN_IF(!buffer_or_error,
                  Status::CodeGenError("Could not load module from IR: ",
                                       buffer_or_error.getError().message()));

  std::unique_ptr<llvm::MemoryBuffer> buffer = move(buffer_or_error.get());

  /// Parse the IR module.
  llvm::Expected<std::unique_ptr<llvm::Module>> module_or_error =
      llvm::getOwningLazyBitcodeModule(move(buffer), *context());
  if (!module_or_error) {
    // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds
    // (ARROW-5148)
    std::string str;
    llvm::raw_string_ostream stream(str);
    stream << module_or_error.takeError();
    return Status::CodeGenError(stream.str());
  }
  std::unique_ptr<llvm::Module> ir_module = move(module_or_error.get());

  // set dataLayout
  SetDataLayout(ir_module.get());

  ARROW_RETURN_IF(llvm::verifyModule(*ir_module, &llvm::errs()),
                  Status::CodeGenError("verify of IR Module failed"));
  ARROW_RETURN_IF(llvm::Linker::linkModules(*module_, move(ir_module)),
                  Status::CodeGenError("failed to link IR Modules"));

  return Status::OK();
}

// Get rid of all functions that don't need to be compiled.
// This helps in reducing the overall compilation time. This pass is trivial,
// and is always done since the number of functions in gandiva is very high.
// (Adapted from Apache Impala)
//
// Done by marking all the unused functions as internal, and then, running
// a pass for dead code elimination.
Status Engine::RemoveUnusedFunctions() {
  // Setup an optimiser pipeline
  std::unique_ptr<llvm::legacy::PassManager> pass_manager(
      new llvm::legacy::PassManager());

  std::unordered_set<std::string> used_functions;
  used_functions.insert(functions_to_compile_.begin(), functions_to_compile_.end());

  pass_manager->add(
      llvm::createInternalizePass([&used_functions](const llvm::GlobalValue& func) {
        return (used_functions.find(func.getName().str()) != used_functions.end());
      }));
  pass_manager->add(llvm::createGlobalDCEPass());
  pass_manager->run(*module_);
  return Status::OK();
}

// Optimise and compile the module.
Status Engine::FinalizeModule() {
  ARROW_RETURN_NOT_OK(RemoveUnusedFunctions());

  if (optimize_) {
    // misc passes to allow for inlining, vectorization, ..
    std::unique_ptr<llvm::legacy::PassManager> pass_manager(
        new llvm::legacy::PassManager());

    llvm::TargetIRAnalysis target_analysis =
        execution_engine_->getTargetMachine()->getTargetIRAnalysis();
    pass_manager->add(llvm::createTargetTransformInfoWrapperPass(target_analysis));
    pass_manager->add(llvm::createFunctionInliningPass());
    pass_manager->add(llvm::createInstructionCombiningPass());
    pass_manager->add(llvm::createPromoteMemoryToRegisterPass());
    pass_manager->add(llvm::createGVNPass());
    pass_manager->add(llvm::createNewGVNPass());
    pass_manager->add(llvm::createCFGSimplificationPass());
    pass_manager->add(llvm::createLoopVectorizePass());
    pass_manager->add(llvm::createSLPVectorizerPass());
    pass_manager->add(llvm::createGlobalOptimizerPass());

    // run the optimiser
    llvm::PassManagerBuilder pass_builder;
    pass_builder.OptLevel = 3;
    pass_builder.populateModulePassManager(*pass_manager);
    pass_manager->run(*module_);
  }

  ARROW_RETURN_IF(llvm::verifyModule(*module_, &llvm::errs()),
                  Status::CodeGenError("Module verification failed after optimizer"));

  // do the compilation
  execution_engine_->finalizeObject();
  module_finalized_ = true;

  return Status::OK();
}

void* Engine::CompiledFunction(llvm::Function* irFunction) {
  DCHECK(module_finalized_);
  return execution_engine_->getPointerToFunction(irFunction);
}

void Engine::AddGlobalMappingForFunc(const std::string& name, llvm::Type* ret_type,
                                     const std::vector<llvm::Type*>& args,
                                     void* function_ptr) {
  constexpr bool is_var_arg = false;
  auto prototype = llvm::FunctionType::get(ret_type, args, is_var_arg);
  constexpr auto linkage = llvm::GlobalValue::ExternalLinkage;
  auto fn = llvm::Function::Create(prototype, linkage, name, module());
  execution_engine_->addGlobalMapping(fn, function_ptr);
}

void Engine::AddGlobalMappings() { ExportedFuncsRegistry::AddMappings(this); }

std::string Engine::DumpIR() {
  std::string ir;
  llvm::raw_string_ostream stream(ir);
  module_->print(stream, nullptr);
  return ir;
}

}  // namespace gandiva
<commit_msg>ARROW-12154: [C++][Gandiva] Fix gandiva crash in certain OS/CPU combinations<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// TODO(wesm): LLVM 7 produces pesky C4244 that disable pragmas around the LLVM
// includes seem to not fix as with LLVM 6
#if defined(_MSC_VER)
#pragma warning(disable : 4244)
#endif

#include "gandiva/engine.h"

#include <iostream>
#include <memory>
#include <mutex>
#include <sstream>
#include <string>
#include <unordered_set>
#include <utility>

#include "arrow/util/logging.h"

#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4141)
#pragma warning(disable : 4146)
#pragma warning(disable : 4244)
#pragma warning(disable : 4267)
#pragma warning(disable : 4624)
#endif

#include <llvm/Analysis/Passes.h>
#include <llvm/Analysis/TargetTransformInfo.h>
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/LLVMContext.h>
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/Verifier.h>
#include <llvm/Linker/Linker.h>
#include <llvm/MC/SubtargetFeature.h>
#include <llvm/Support/DynamicLibrary.h>
#include <llvm/Support/Host.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_ostream.h>
#include <llvm/Transforms/IPO.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#include <llvm/Transforms/InstCombine/InstCombine.h>
#include <llvm/Transforms/Scalar.h>
#include <llvm/Transforms/Scalar/GVN.h>
#include <llvm/Transforms/Utils.h>
#include <llvm/Transforms/Vectorize.h>

#if defined(_MSC_VER)
#pragma warning(pop)
#endif

#include "arrow/util/make_unique.h"
#include "gandiva/configuration.h"
#include "gandiva/decimal_ir.h"
#include "gandiva/exported_funcs_registry.h"

namespace gandiva {

extern const unsigned char kPrecompiledBitcode[];
extern const size_t kPrecompiledBitcodeSize;

std::once_flag llvm_init_once_flag;
static bool llvm_init = false;
static llvm::StringRef cpu_name;
static llvm::SmallVector<std::string, 10> cpu_attrs;

void Engine::InitOnce() {
  DCHECK_EQ(llvm_init, false);

  llvm::InitializeNativeTarget();
  llvm::InitializeNativeTargetAsmPrinter();
  llvm::InitializeNativeTargetAsmParser();
  llvm::InitializeNativeTargetDisassembler();
  llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr);

  cpu_name = llvm::sys::getHostCPUName();
  llvm::StringMap<bool> host_features;
  std::string cpu_attrs_str;
  if (llvm::sys::getHostCPUFeatures(host_features)) {
    for (auto& f : host_features) {
      std::string attr = f.second ? std::string("+") + f.first().str()
                                  : std::string("-") + f.first().str();
      cpu_attrs.push_back(attr);
      cpu_attrs_str += " " + attr;
    }
  }
  ARROW_LOG(INFO) << "Detected CPU Name : " << cpu_name.str();
  ARROW_LOG(INFO) << "Detected CPU Features:" << cpu_attrs_str;
  llvm_init = true;
}

Engine::Engine(const std::shared_ptr<Configuration>& conf,
               std::unique_ptr<llvm::LLVMContext> ctx,
               std::unique_ptr<llvm::ExecutionEngine> engine, llvm::Module* module)
    : context_(std::move(ctx)),
      execution_engine_(std::move(engine)),
      ir_builder_(arrow::internal::make_unique<llvm::IRBuilder<>>(*context_)),
      module_(module),
      types_(*context_),
      optimize_(conf->optimize()) {}

Status Engine::Init() {
  // Add mappings for functions that can be accessed from LLVM/IR module.
  AddGlobalMappings();

  ARROW_RETURN_NOT_OK(LoadPreCompiledIR());
  ARROW_RETURN_NOT_OK(DecimalIR::AddFunctions(this));

  return Status::OK();
}

/// factory method to construct the engine.
Status Engine::Make(const std::shared_ptr<Configuration>& conf,
                    std::unique_ptr<Engine>* out) {
  std::call_once(llvm_init_once_flag, InitOnce);

  auto ctx = arrow::internal::make_unique<llvm::LLVMContext>();
  auto module = arrow::internal::make_unique<llvm::Module>("codegen", *ctx);

  // Capture before moving, ExecutionEngine does not allow retrieving the
  // original Module.
  auto module_ptr = module.get();

  auto opt_level =
      conf->optimize() ? llvm::CodeGenOpt::Aggressive : llvm::CodeGenOpt::None;

  // Note that the lifetime of the error string is not captured by the
  // ExecutionEngine but only for the lifetime of the builder. Found by
  // inspecting LLVM sources.
  std::string builder_error;
  std::unique_ptr<llvm::ExecutionEngine> exec_engine{
      llvm::EngineBuilder(std::move(module))
          .setMCPU(cpu_name)
          .setMAttrs(cpu_attrs)
          .setEngineKind(llvm::EngineKind::JIT)
          .setOptLevel(opt_level)
          .setErrorStr(&builder_error)
          .create()};

  if (exec_engine == nullptr) {
    return Status::CodeGenError("Could not instantiate llvm::ExecutionEngine: ",
                                builder_error);
  }

  std::unique_ptr<Engine> engine{
      new Engine(conf, std::move(ctx), std::move(exec_engine), module_ptr)};
  ARROW_RETURN_NOT_OK(engine->Init());
  *out = std::move(engine);
  return Status::OK();
}

// This method was modified from its original version for a part of MLIR
// Original source from
// https://github.com/llvm/llvm-project/blob/9f2ce5b915a505a5488a5cf91bb0a8efa9ddfff7/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
// The original copyright notice follows.

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

static void SetDataLayout(llvm::Module* module) {
  auto target_triple = llvm::sys::getDefaultTargetTriple();
  std::string error_message;
  auto target = llvm::TargetRegistry::lookupTarget(target_triple, error_message);
  if (!target) {
    return;
  }

  std::string cpu(llvm::sys::getHostCPUName());
  llvm::SubtargetFeatures features;
  llvm::StringMap<bool> host_features;

  if (llvm::sys::getHostCPUFeatures(host_features)) {
    for (auto& f : host_features) {
      features.AddFeature(f.first(), f.second);
    }
  }

  std::unique_ptr<llvm::TargetMachine> machine(
      target->createTargetMachine(target_triple, cpu, features.getString(), {}, {}));

  module->setDataLayout(machine->createDataLayout());
}
// end of the mofified method from MLIR

// Handling for pre-compiled IR libraries.
Status Engine::LoadPreCompiledIR() {
  auto bitcode = llvm::StringRef(reinterpret_cast<const char*>(kPrecompiledBitcode),
                                 kPrecompiledBitcodeSize);

  /// Read from file into memory buffer.
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer_or_error =
      llvm::MemoryBuffer::getMemBuffer(bitcode, "precompiled", false);

  ARROW_RETURN_IF(!buffer_or_error,
                  Status::CodeGenError("Could not load module from IR: ",
                                       buffer_or_error.getError().message()));

  std::unique_ptr<llvm::MemoryBuffer> buffer = move(buffer_or_error.get());

  /// Parse the IR module.
  llvm::Expected<std::unique_ptr<llvm::Module>> module_or_error =
      llvm::getOwningLazyBitcodeModule(move(buffer), *context());
  if (!module_or_error) {
    // NOTE: llvm::handleAllErrors() fails linking with RTTI-disabled LLVM builds
    // (ARROW-5148)
    std::string str;
    llvm::raw_string_ostream stream(str);
    stream << module_or_error.takeError();
    return Status::CodeGenError(stream.str());
  }
  std::unique_ptr<llvm::Module> ir_module = move(module_or_error.get());

  // set dataLayout
  SetDataLayout(ir_module.get());

  ARROW_RETURN_IF(llvm::verifyModule(*ir_module, &llvm::errs()),
                  Status::CodeGenError("verify of IR Module failed"));
  ARROW_RETURN_IF(llvm::Linker::linkModules(*module_, move(ir_module)),
                  Status::CodeGenError("failed to link IR Modules"));

  return Status::OK();
}

// Get rid of all functions that don't need to be compiled.
// This helps in reducing the overall compilation time. This pass is trivial,
// and is always done since the number of functions in gandiva is very high.
// (Adapted from Apache Impala)
//
// Done by marking all the unused functions as internal, and then, running
// a pass for dead code elimination.
Status Engine::RemoveUnusedFunctions() {
  // Setup an optimiser pipeline
  std::unique_ptr<llvm::legacy::PassManager> pass_manager(
      new llvm::legacy::PassManager());

  std::unordered_set<std::string> used_functions;
  used_functions.insert(functions_to_compile_.begin(), functions_to_compile_.end());

  pass_manager->add(
      llvm::createInternalizePass([&used_functions](const llvm::GlobalValue& func) {
        return (used_functions.find(func.getName().str()) != used_functions.end());
      }));
  pass_manager->add(llvm::createGlobalDCEPass());
  pass_manager->run(*module_);
  return Status::OK();
}

// Optimise and compile the module.
Status Engine::FinalizeModule() {
  ARROW_RETURN_NOT_OK(RemoveUnusedFunctions());

  if (optimize_) {
    // misc passes to allow for inlining, vectorization, ..
    std::unique_ptr<llvm::legacy::PassManager> pass_manager(
        new llvm::legacy::PassManager());

    llvm::TargetIRAnalysis target_analysis =
        execution_engine_->getTargetMachine()->getTargetIRAnalysis();
    pass_manager->add(llvm::createTargetTransformInfoWrapperPass(target_analysis));
    pass_manager->add(llvm::createFunctionInliningPass());
    pass_manager->add(llvm::createInstructionCombiningPass());
    pass_manager->add(llvm::createPromoteMemoryToRegisterPass());
    pass_manager->add(llvm::createGVNPass());
    pass_manager->add(llvm::createNewGVNPass());
    pass_manager->add(llvm::createCFGSimplificationPass());
    pass_manager->add(llvm::createLoopVectorizePass());
    pass_manager->add(llvm::createSLPVectorizerPass());
    pass_manager->add(llvm::createGlobalOptimizerPass());

    // run the optimiser
    llvm::PassManagerBuilder pass_builder;
    pass_builder.OptLevel = 3;
    pass_builder.populateModulePassManager(*pass_manager);
    pass_manager->run(*module_);
  }

  ARROW_RETURN_IF(llvm::verifyModule(*module_, &llvm::errs()),
                  Status::CodeGenError("Module verification failed after optimizer"));

  // do the compilation
  execution_engine_->finalizeObject();
  module_finalized_ = true;

  return Status::OK();
}

void* Engine::CompiledFunction(llvm::Function* irFunction) {
  DCHECK(module_finalized_);
  return execution_engine_->getPointerToFunction(irFunction);
}

void Engine::AddGlobalMappingForFunc(const std::string& name, llvm::Type* ret_type,
                                     const std::vector<llvm::Type*>& args,
                                     void* function_ptr) {
  constexpr bool is_var_arg = false;
  auto prototype = llvm::FunctionType::get(ret_type, args, is_var_arg);
  constexpr auto linkage = llvm::GlobalValue::ExternalLinkage;
  auto fn = llvm::Function::Create(prototype, linkage, name, module());
  execution_engine_->addGlobalMapping(fn, function_ptr);
}

void Engine::AddGlobalMappings() { ExportedFuncsRegistry::AddMappings(this); }

std::string Engine::DumpIR() {
  std::string ir;
  llvm::raw_string_ostream stream(ir);
  module_->print(stream, nullptr);
  return ir;
}

}  // namespace gandiva
<|endoftext|>