<commit_before>#include <ecl/map.hpp>

#include <iostream>
#include <cstdlib>
#include <string>
#include <ostream>

enum class E
{
      e1
    , e2
    , e3
    , e4
    , e5
    , e6
    , e7
    , e8
    , e9
    , e10
};

std::ostream& operator<<(std::ostream& s, E e)
{
    switch(e)
    {
        case E::e1 : s << "e1" ; break;
        case E::e2 : s << "e2" ; break;
        case E::e3 : s << "e3" ; break;
        case E::e4 : s << "e4" ; break;
        case E::e5 : s << "e5" ; break;
        case E::e6 : s << "e6" ; break;
        case E::e7 : s << "e7" ; break;
        case E::e8 : s << "e8" ; break;
        case E::e9 : s << "e9" ; break;
        case E::e10: s << "e10"; break;
        default    :             break;
    }

    return s;
}

using key_type   = E;
using value_type = std::uint32_t;

using map_t      = ecl::map<key_type, value_type, 8>;

 auto m0 = ecl::create_map<key_type, value_type>
 (
       std::make_pair(E::e1, 1u)
     , std::make_pair(E::e2, 2u)
     , std::make_pair(E::e3, 4u)
     , std::make_pair(E::e3, 3u) // overwrite E::e3 element
 );

template<typename T>
void fill_map_ilist(std::string prefix, T& m)
{
    std::cout << ">>> Insert to map" << std::endl;
    m.insert(
    {
          { E::e1  , 1  }
        , { E::e2  , 3  }
        , { E::e3  , 5  }
        , { E::e4  , 7  }
        , { E::e5  , 9  }
        , { E::e6  , 11 }
        , { E::e7  , 13 }
        , { E::e8  , 15 }
        , { E::e9  , 17 }
        , { E::e10 , 19 }
    });

    dump_map(prefix, m);
}

template<typename T>
void fill_map_operator(std::string prefix, T& m)
{
    std::cout << ">>> Assign via operator[]" << std::endl;

    m[E::e1]  = 5;
    m[E::e2]  = 10;
    m[E::e3]  = 15;
    m[E::e4]  = 20;
    m[E::e5]  = 25;
    m[E::e6]  = 30;
    m[E::e7]  = 35;
    m[E::e8]  = 40;
    m[E::e9]  = 45;
    m[E::e10] = 50;

    dump_map(prefix, m);
//    m.print();
}

template<typename T>
void dump_map(std::string prefix, T& m)
{
    std::cout << ">>> Dump map" << std::endl;
    std::cout << prefix << " map elements count: " << m.size() << std::endl;

    std::cout << prefix << " range-for interation over map: ";
    for(auto& p : m)
    {
        std::cout << "(" << p.first << ":" << p.second << ") ";
    }
    std::cout << std::endl;

    std::cout << prefix << ".at(E::e1):  " << m.at(E::e1)  << std::endl;
    std::cout << prefix << ".at(E::e2):  " << m.at(E::e2)  << std::endl;
    std::cout << prefix << ".at(E::e3):  " << m.at(E::e3)  << std::endl;
    std::cout << prefix << ".at(E::e4):  " << m.at(E::e4)  << std::endl;
    std::cout << prefix << ".at(E::e5):  " << m.at(E::e5)  << std::endl;
    std::cout << prefix << ".at(E::e6):  " << m.at(E::e6)  << std::endl;
    std::cout << prefix << ".at(E::e7):  " << m.at(E::e7)  << std::endl;
    std::cout << prefix << ".at(E::e8):  " << m.at(E::e8)  << std::endl;
    std::cout << prefix << ".at(E::e9):  " << m.at(E::e9)  << std::endl;
    std::cout << prefix << ".at(E::e10): " << m.at(E::e10) << std::endl;
}

template<typename T>
void erase_map_by_key(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by keys" << std::endl;
    std::cout << prefix << ".erase(E::e1) : " << m.erase(E::e1)  << std::endl;
    std::cout << prefix << ".erase(E::e2) : " << m.erase(E::e2)  << std::endl;
    std::cout << prefix << ".erase(E::e3) : " << m.erase(E::e3)  << std::endl;
    std::cout << prefix << ".erase(E::e4) : " << m.erase(E::e4)  << std::endl;
    std::cout << prefix << ".erase(E::e5) : " << m.erase(E::e5)  << std::endl;
    std::cout << prefix << ".erase(E::e6) : " << m.erase(E::e6)  << std::endl;
    std::cout << prefix << ".erase(E::e7) : " << m.erase(E::e7)  << std::endl;
    std::cout << prefix << ".erase(E::e8) : " << m.erase(E::e8)  << std::endl;
    std::cout << prefix << ".erase(E::e9) : " << m.erase(E::e9)  << std::endl;
    std::cout << prefix << ".erase(E::e10): " << m.erase(E::e10) << std::endl;

    dump_map(prefix, m);
}

template<typename T>
void erase_map_by_irange(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by iterator range" << std::endl;

    m.erase(m.begin(), m.end());

    dump_map(prefix, m);
}

template<typename T>
void erase_map_by_iterator(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by iterator" << std::endl;
    typename T::const_iterator it = m.begin();
                   std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;

//    it = m.begin();
//                      std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;

    dump_map(prefix, m);
}

int main(int, char**, char**)
{
    map_t m1
    {
          { E::e1  , 2  }
        , { E::e2  , 4  }
        , { E::e3  , 6  }
        , { E::e4  , 8  }
        , { E::e5  , 10 }
        , { E::e6  , 12 }
        , { E::e7  , 14 }
        , { E::e8  , 16 }
        , { E::e9  , 18 }
        , { E::e10 , 20 }
    };
    dump_map("m1", m1);
    erase_map_by_key("m1", m1);

    fill_map_ilist("m1", m1);
    erase_map_by_irange("m1", m1);
//    map_t m1;

    fill_map_operator("m1", m1);
    erase_map_by_iterator("m1", m1);

    return 0;
}
<commit_msg>Map example: erase via iterator assignment and iterator increment.<commit_after>#include <ecl/map.hpp>

#include <iostream>
#include <cstdlib>
#include <string>
#include <ostream>

enum class E
{
      e1
    , e2
    , e3
    , e4
    , e5
    , e6
    , e7
    , e8
    , e9
    , e10
};

std::ostream& operator<<(std::ostream& s, E e)
{
    switch(e)
    {
        case E::e1 : s << "e1" ; break;
        case E::e2 : s << "e2" ; break;
        case E::e3 : s << "e3" ; break;
        case E::e4 : s << "e4" ; break;
        case E::e5 : s << "e5" ; break;
        case E::e6 : s << "e6" ; break;
        case E::e7 : s << "e7" ; break;
        case E::e8 : s << "e8" ; break;
        case E::e9 : s << "e9" ; break;
        case E::e10: s << "e10"; break;
        default    :             break;
    }

    return s;
}

using key_type   = E;
using value_type = std::uint32_t;

using map_t      = ecl::map<key_type, value_type, 8>;

 auto m0 = ecl::create_map<key_type, value_type>
 (
       std::make_pair(E::e1, 1u)
     , std::make_pair(E::e2, 2u)
     , std::make_pair(E::e3, 4u)
     , std::make_pair(E::e3, 3u) // overwrite E::e3 element
 );

template<typename T>
void fill_map_ilist(std::string prefix, T& m)
{
    std::cout << ">>> Insert to map" << std::endl;
    m.insert(
    {
          { E::e1  , 1  }
        , { E::e2  , 3  }
        , { E::e3  , 5  }
        , { E::e4  , 7  }
        , { E::e5  , 9  }
        , { E::e6  , 11 }
        , { E::e7  , 13 }
        , { E::e8  , 15 }
        , { E::e9  , 17 }
        , { E::e10 , 19 }
    });

    dump_map(prefix, m);
}

template<typename T>
void fill_map_operator(std::string prefix, T& m)
{
    std::cout << ">>> Assign via operator[]" << std::endl;

    m[E::e1]  = 5;
    m[E::e2]  = 10;
    m[E::e3]  = 15;
    m[E::e4]  = 20;
    m[E::e5]  = 25;
    m[E::e6]  = 30;
    m[E::e7]  = 35;
    m[E::e8]  = 40;
    m[E::e9]  = 45;
    m[E::e10] = 50;

    dump_map(prefix, m);
//    m.print();
}

template<typename T>
void dump_map(std::string prefix, T& m)
{
    std::cout << ">>> Dump map" << std::endl;
    std::cout << prefix << " map elements count: " << m.size() << std::endl;

    std::cout << prefix << " range-for interation over map: ";
    for(auto& p : m)
    {
        std::cout << "(" << p.first << ":" << p.second << ") ";
    }
    std::cout << std::endl;

    std::cout << prefix << ".at(E::e1):  " << m.at(E::e1)  << std::endl;
    std::cout << prefix << ".at(E::e2):  " << m.at(E::e2)  << std::endl;
    std::cout << prefix << ".at(E::e3):  " << m.at(E::e3)  << std::endl;
    std::cout << prefix << ".at(E::e4):  " << m.at(E::e4)  << std::endl;
    std::cout << prefix << ".at(E::e5):  " << m.at(E::e5)  << std::endl;
    std::cout << prefix << ".at(E::e6):  " << m.at(E::e6)  << std::endl;
    std::cout << prefix << ".at(E::e7):  " << m.at(E::e7)  << std::endl;
    std::cout << prefix << ".at(E::e8):  " << m.at(E::e8)  << std::endl;
    std::cout << prefix << ".at(E::e9):  " << m.at(E::e9)  << std::endl;
    std::cout << prefix << ".at(E::e10): " << m.at(E::e10) << std::endl;
}

template<typename T>
void erase_map_by_key(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by keys" << std::endl;
    std::cout << prefix << ".erase(E::e1) : " << m.erase(E::e1)  << std::endl;
    std::cout << prefix << ".erase(E::e2) : " << m.erase(E::e2)  << std::endl;
    std::cout << prefix << ".erase(E::e3) : " << m.erase(E::e3)  << std::endl;
    std::cout << prefix << ".erase(E::e4) : " << m.erase(E::e4)  << std::endl;
    std::cout << prefix << ".erase(E::e5) : " << m.erase(E::e5)  << std::endl;
    std::cout << prefix << ".erase(E::e6) : " << m.erase(E::e6)  << std::endl;
    std::cout << prefix << ".erase(E::e7) : " << m.erase(E::e7)  << std::endl;
    std::cout << prefix << ".erase(E::e8) : " << m.erase(E::e8)  << std::endl;
    std::cout << prefix << ".erase(E::e9) : " << m.erase(E::e9)  << std::endl;
    std::cout << prefix << ".erase(E::e10): " << m.erase(E::e10) << std::endl;

    dump_map(prefix, m);
}

template<typename T>
void erase_map_by_irange(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by iterator range" << std::endl;

    m.erase(m.begin(), m.end());

    dump_map(prefix, m);
}

template<typename T>
void erase_map_by_iterator_increment(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by iterator (increment)" << std::endl;
    typename T::const_iterator it = m.begin();
                   std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;
    m.erase(it++); std::cout << it->first << ":" << it->second << std::endl;

//    it = m.begin();
//                      std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
//    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;

    dump_map(prefix, m);
}

template<typename T>
void erase_map_by_iterator_assign_return(std::string prefix, T& m)
{
    std::cout << ">>> Erase map by iterator (assign return)" << std::endl;
    typename T::const_iterator it = m.begin();
                      std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;
    it = m.erase(it); std::cout << it->first << ":" << it->second << std::endl;

    dump_map(prefix, m);
}

int main(int, char**, char**)
{
    map_t m1
    {
          { E::e1  , 2  }
        , { E::e2  , 4  }
        , { E::e3  , 6  }
        , { E::e4  , 8  }
        , { E::e5  , 10 }
        , { E::e6  , 12 }
        , { E::e7  , 14 }
        , { E::e8  , 16 }
        , { E::e9  , 18 }
        , { E::e10 , 20 }
    };
    dump_map("m1", m1);
    erase_map_by_key("m1", m1);

    fill_map_ilist("m1", m1);
    erase_map_by_irange("m1", m1);
//    map_t m1;

    fill_map_operator("m1", m1);
    erase_map_by_iterator_increment("m1", m1);

    fill_map_operator("m1", m1);
    erase_map_by_iterator_assign_return("m1", m1);

    return 0;
}
<|endoftext|>


<commit_before>#include <cstddef>
#include <cstdlib>
#include <hls_video.h>

#include "Accel.h"
#include "AccelSchedule.h"
#include "AccelTest.h"
#include "Dense.h"
#include "ZipIO.h"
#include "ParamIO.h"
#include "DataIO.h"
#include "Timer.h"

int main(int argc, char** argv) {
  if (argc < 2) {
    printf ("Give number of character to produce as 1st arg\n");
    printf ("Give the initial srting as 2nd arg optional\n");
    return 0;
  }
  const unsigned n_char = std::stoi(argv[1]);
  bool Init = false;
  char* str_init = NULL;

  if (argc == 3) {
    Init = true;
    str_init = argv[2];   // *ML: cant loas text with ' '
    printf("* Initial string is %s\n", str_init);
  }
  // print some config numbers
  printf ("* WT_WORDS   = %u\n", WT_WORDS);
  printf ("* BIAS_WORDS = %u\n", BIAS_WORDS);

  // Load input data
  //printf ("## Loading input data ##\n");
  // ML: hidden state can be initialized by a given string

  // Load parameters
  printf ("## Loading parameters ##\n");
  Params params(get_root_dir() + "/params/rnn_parameters.zip");

  // ---------------------------------------------------------------------
  // allocate and binarize all weights
  // ---------------------------------------------------------------------
  Word* wt[N_LAYERS];
  Word* b[N_LAYERS];

  for (unsigned l = 0; l < N_LAYERS; ++l) {
    const unsigned M = M_tab[l];
    const unsigned N = N_tab[l];

    if (layer_is_rnn(l+1)) {
      wt[l] = new Word[(M+N)*4*N / WORD_SIZE];
      b[l] = new Word[4*N / WORD_SIZE];
    }
    else {
      wt[l] = new Word[M*N / WORD_SIZE];    // ML: RNN layers
      b[l] = new Word[N / WORD_SIZE];
    }
    if (layer_is_rnn(l+1)) {
      for (unsigned w_l = 0; w_l < N_W_LAYERS; ++w_l) {
        // ML: set in_to weight and hid_to weight
        const float* weights_in = params.float_data(widx_tab[l*N_W_LAYERS*2 + 2*w_l]);
        const float* weights_hid = params.float_data(widx_tab[l*N_W_LAYERS*2 + 2*w_l +1]);
        set_rnn_weight_array(wt[l], weights_in, weights_hid, l+1, w_l);
        // ML: set bias
        const float* bias = params.float_data(bidx_tab[l*N_W_LAYERS + w_l]);
        set_rnn_bias_array(b[l], bias, l+1, w_l);
      }
    } else {

      const float* weights = params.float_data(widx_tab[16]);
      set_dense_weight_array(wt[l], weights, l+1);
      const float* bias = params.float_data(bidx_tab[8]);
      set_dense_bias_array(b[l], bias, l+1);
    }

  }

  // ---------------------------------------------------------------------
  // // compute accelerator schedule (divides up weights)
  // ---------------------------------------------------------------------
  AccelSchedule layer_sched[N_LAYERS];
  for (unsigned l = 0; l < N_LAYERS; ++l) {
    compute_accel_schedule(
        wt[l], b[l],
        M_tab[l], N_tab[l], T_tab[l],
        layer_sched[l], l
    );
  }

  // allocate memories for data i/o for the accelerator
  Word* data_i  = (Word*) MEM_ALLOC( DMEM_WORDS * sizeof(Word) );   // ML: need to be modified!
  Word* data_o  = (Word*) MEM_ALLOC( DMEM_O_WORDS * sizeof(Word) );
  if (!data_i || !data_o) {
    fprintf (stderr, "**** ERROR: Alloc failed in %s\n", __FILE__);
    return (-2);
  }

  unsigned n_errors = 0;

  printf("## Initialing the RNN\n");

  if (Init) {
    unsigned i = 0;
    while (str_init[i] != '\0') {
      set_char_to_word(data_i, str_init[i]);

      for (unsigned l = 1; l <= 3; ++l) {
        const unsigned M = M_tab[l-1];
        const unsigned N = N_tab[l-1];

        dense_layer(
          data_i, data_o,
          l-1,
          (l==1) ? 1 : 0,    // input_words
          layer_sched[l][0].n_inputs,
          layer_sched[l][0].n_outputs,
          layer_sched[l][0].wt,
          layer_sched[l][0].b
        );
      }
      i++;
    }
  }

  printf ("## Running RNN for %d characters\n", n_char);

  //--------------------------------------------------------------
  // Run RNN
  //--------------------------------------------------------------

  // ML: load an arbitrary input character [1, 0. 0, ..., 0]
  for (unsigned i = 0; i < VOCAB_SIZE/DATA_PER_WORD; ++i) {
    if (i == 0) {
      data_i[i] = 0;
      DATA start_seed = 1;
      data_i[i](15,0) = start_seed(15,0);
    } else {
      data_i[i] = 0;
    }
  }

  for (unsigned n = 0; n < n_char; ++n) {

    //------------------------------------------------------------
    // Execute RNN layers
    //------------------------------------------------------------
    for (unsigned l = 1; l <= 3; ++l) {
      const unsigned M = M_tab[l-1];
      const unsigned N = N_tab[l-1];

      dense_layer(
        data_i, data_o,
        l-1,
        (n==0 && l==1 && (~Init)) ? 1 : 0,    // input_words
        layer_sched[l][0].n_inputs,
        layer_sched[l][0].n_outputs,
        layer_sched[l][0].wt,
        layer_sched[l][0].b
      );
    }

    //------------------------------------------------------------
    // Execute the prediciton
    //------------------------------------------------------------
    int prediction = 0;
    int max = -512; // ML: may shoulb be less

    for (unsigned i = 0; i < VOCAB_SIZE; i++) {
      DATA temp;
      int add = i / DATA_PER_WORD;
      int off = i % DATA_PER_WORD;
      temp(15,0) = data_o[add]((off+1)*16-1,off*16);
      if (temp.to_int() > max) {
        max = temp;
        prediction = i;
      }
    }

    assert(prediction >= 0 && prediction <= 63);

    std::cout<<vocab[prediction];

  }

  printf("\n");
  /*printf ("\n");
  printf ("Errors: %u (%4.2f%%)\n", n_errors, float(n_errors)*100/n_imgs);
  printf ("\n");
  printf ("Total accel runtime = %10.4f seconds\n", total_time());
  printf ("\n");*/

  MEM_FREE( data_o );
  MEM_FREE( data_i );
  for (unsigned n = 0; n < N_LAYERS; ++n) {
    delete[] wt[n];
    delete[] b[n];
  }
  return 0;
}
<commit_msg>fixed a bug<commit_after>#include <cstddef>
#include <cstdlib>
#include <hls_video.h>

#include "Accel.h"
#include "AccelSchedule.h"
#include "AccelTest.h"
#include "Dense.h"
#include "ZipIO.h"
#include "ParamIO.h"
#include "DataIO.h"
#include "Timer.h"

int main(int argc, char** argv) {
  if (argc < 2) {
    printf ("Give number of character to produce as 1st arg\n");
    printf ("Give the initial srting as 2nd arg optional\n");
    return 0;
  }
  const unsigned n_char = std::stoi(argv[1]);
  bool Init = false;
  char* str_init = NULL;

  if (argc == 3) {
    Init = true;
    str_init = argv[2];   // *ML: cant loas text with ' '
    printf("* Initial string is %s\n", str_init);
  }
  // print some config numbers
  printf ("* WT_WORDS   = %u\n", WT_WORDS);
  printf ("* BIAS_WORDS = %u\n", BIAS_WORDS);

  // Load input data
  //printf ("## Loading input data ##\n");
  // ML: hidden state can be initialized by a given string

  // Load parameters
  printf ("## Loading parameters ##\n");
  Params params(get_root_dir() + "/params/rnn_parameters.zip");

  // ---------------------------------------------------------------------
  // allocate and binarize all weights
  // ---------------------------------------------------------------------
  Word* wt[N_LAYERS];
  Word* b[N_LAYERS];

  for (unsigned l = 0; l < N_LAYERS; ++l) {
    const unsigned M = M_tab[l];
    const unsigned N = N_tab[l];

    if (layer_is_rnn(l+1)) {
      wt[l] = new Word[(M+N)*4*N / WORD_SIZE];
      b[l] = new Word[4*N / WORD_SIZE];
    }
    else {
      wt[l] = new Word[M*N / WORD_SIZE];    // ML: RNN layers
      b[l] = new Word[N / WORD_SIZE];
    }
    if (layer_is_rnn(l+1)) {
      for (unsigned w_l = 0; w_l < N_W_LAYERS; ++w_l) {
        // ML: set in_to weight and hid_to weight
        const float* weights_in = params.float_data(widx_tab[l*N_W_LAYERS*2 + 2*w_l]);
        const float* weights_hid = params.float_data(widx_tab[l*N_W_LAYERS*2 + 2*w_l +1]);
        set_rnn_weight_array(wt[l], weights_in, weights_hid, l+1, w_l);
        // ML: set bias
        const float* bias = params.float_data(bidx_tab[l*N_W_LAYERS + w_l]);
        set_rnn_bias_array(b[l], bias, l+1, w_l);
      }
    } else {

      const float* weights = params.float_data(widx_tab[16]);
      set_dense_weight_array(wt[l], weights, l+1);
      const float* bias = params.float_data(bidx_tab[8]);
      set_dense_bias_array(b[l], bias, l+1);
    }

  }

  // ---------------------------------------------------------------------
  // // compute accelerator schedule (divides up weights)
  // ---------------------------------------------------------------------
  AccelSchedule layer_sched[N_LAYERS];
  for (unsigned l = 0; l < N_LAYERS; ++l) {
    compute_accel_schedule(
        wt[l], b[l],
        M_tab[l], N_tab[l], T_tab[l],
        layer_sched[l], l
    );
  }

  // allocate memories for data i/o for the accelerator
  Word* data_i  = (Word*) MEM_ALLOC( DMEM_WORDS * sizeof(Word) );   // ML: need to be modified!
  Word* data_o  = (Word*) MEM_ALLOC( DMEM_O_WORDS * sizeof(Word) );
  if (!data_i || !data_o) {
    fprintf (stderr, "**** ERROR: Alloc failed in %s\n", __FILE__);
    return (-2);
  }

  unsigned n_errors = 0;

  printf("## Initialing the RNN\n");

  if (Init) {
    unsigned i = 0;
    while (str_init[i] != '\0') {
      set_char_to_word(data_i, str_init[i]);

      for (unsigned l = 1; l <= 3; ++l) {
        const unsigned M = M_tab[l-1];
        const unsigned N = N_tab[l-1];

        dense_layer(
          data_i, data_o,
          l-1,
          (l==1) ? 1 : 0,    // input_words
          layer_sched[l-1][0].n_inputs,
          layer_sched[l-1][0].n_outputs,
          layer_sched[l-1][0].wt,
          layer_sched[l-1][0].b
        );
      }
      i++;
    }
  }

  printf ("## Running RNN for %d characters\n", n_char);

  //--------------------------------------------------------------
  // Run RNN
  //--------------------------------------------------------------

  // ML: load an arbitrary input character [1, 0. 0, ..., 0]
  for (unsigned i = 0; i < VOCAB_SIZE/DATA_PER_WORD; ++i) {
    if (i == 0) {
      data_i[i] = 0;
      DATA start_seed = 1;
      data_i[i](15,0) = start_seed(15,0);
    } else {
      data_i[i] = 0;
    }
  }

  for (unsigned n = 0; n < n_char; ++n) {

    //------------------------------------------------------------
    // Execute RNN layers
    //------------------------------------------------------------
    for (unsigned l = 1; l <= 3; ++l) {
      const unsigned M = M_tab[l-1];
      const unsigned N = N_tab[l-1];

      dense_layer(
        data_i, data_o,
        l-1,
        (n==0 && l==1 && (~Init)) ? 1 : 0,    // input_words
        layer_sched[l-1][0].n_inputs,
        layer_sched[l-1][0].n_outputs,
        layer_sched[l-1][0].wt,
        layer_sched[l-1][0].b
      );
    }

    //------------------------------------------------------------
    // Execute the prediciton
    //------------------------------------------------------------
    int prediction = 0;
    int max = -512; // ML: may shoulb be less

    for (unsigned i = 0; i < VOCAB_SIZE; i++) {
      DATA temp;
      int add = i / DATA_PER_WORD;
      int off = i % DATA_PER_WORD;
      temp(15,0) = data_o[add]((off+1)*16-1,off*16);
      if (temp.to_int() > max) {
        max = temp;
        prediction = i;
      }
    }

    assert(prediction >= 0 && prediction <= 63);

    std::cout<<vocab[prediction];

  }

  printf("\n");
  /*printf ("\n");
  printf ("Errors: %u (%4.2f%%)\n", n_errors, float(n_errors)*100/n_imgs);
  printf ("\n");
  printf ("Total accel runtime = %10.4f seconds\n", total_time());
  printf ("\n");*/

  MEM_FREE( data_o );
  MEM_FREE( data_i );
  for (unsigned n = 0; n < N_LAYERS; ++n) {
    delete[] wt[n];
    delete[] b[n];
  }
  return 0;
}
<|endoftext|>


<commit_before>/** \file    restart_zts
 *  \brief   Restart the docker container with the Zotero Translation Server
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2020, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <sstream>
#include "ExecUtil.h"
#include "StringUtil.h"
#include "WebUtil.h"
#include "util.h"

namespace {

void SendHeaders() {
    std::cout << "Content-Type: text/html; charset=utf-8\r\n\r\n"
              << "\n";
}

void SendTrailer() {
    std::cout << "\n";

}

void DisplayRestartButton() {
    SendHeaders();
    std::cout << "
Restart Zotero Translation Server Service
\n"
              << "<form action=\"\" method=\"post\">\n"
              << "\t<input type=\"submit\" name=\"action\" value=\"Restart\">\n"
              << "</form>\n";
    SendTrailer();
}

bool IsRestartActionPresent(std::multimap<std::string, std::string> *cgi_args) {
    WebUtil::GetAllCgiArgs(cgi_args);
    const auto key_and_value(cgi_args->find("action"));
    return key_and_value != cgi_args->cend() and key_and_value->second == "Restart";
}

void OutputZTSStatus() {
    const std::string tmp_output(FileUtil::AutoTempFile().getFilePath());
    ExecUtil::ExecOrDie("/usr/bin/sudo", { "systemctl", "status", "zts" }, "" /*stdin*/,
                        tmp_output, tmp_output);

    std::ifstream zts_output_file(tmp_output);
    if (not zts_output_file)
        LOG_ERROR("Could not open " + tmp_output + " for reading\n");
    std::stringstream zts_output_istream;
    zts_output_istream << zts_output_file.rdbuf();
    std::cout << StringUtil::ReplaceString("\n", "
", zts_output_istream.str());
}

void RestartZTS() {
    SendHeaders();
    std::cout << "
Attempting restart of ZTS...
\n" << std::endl;
    bool log_no_decorations_old(logger->getLogNoDecorations());
    bool log_strip_call_site_old(logger->getLogStripCallSite());
    logger->setLogNoDecorations(true);
    logger->setLogStripCallSite(true);
    logger->redirectOutput(STDOUT_FILENO);
    try {
        ExecUtil::ExecOrDie("/usr/bin/sudo", { "systemctl", "restart", "zts" });
        OutputZTSStatus();
    } catch (const std::runtime_error &error) {
        std::cerr << error.what();
    }
    std::cout << "
Done...</h>\n";
    SendTrailer();
    logger->redirectOutput(STDERR_FILENO);
    logger->setLogNoDecorations(log_no_decorations_old);
    logger->setLogStripCallSite(log_strip_call_site_old);
}

} // end unnamed namespace

int Main(int /*argc*/, char */*argv*/[]) {
    std::multimap<std::string, std::string> cgi_args;
    if (not IsRestartActionPresent(&cgi_args)) {
        DisplayRestartButton();
        return EXIT_SUCCESS;
    }
    RestartZTS();
    return EXIT_SUCCESS;
}

<commit_msg>Add configureable possibility to switch a different translator branch<commit_after>/** \file    restart_zts
 *  \brief   Restart the docker container with the Zotero Translation Server
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2020,2021, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <functional>
#include <iostream>
#include <sstream>
#include "ExecUtil.h"
#include "IniFile.h"
#include "StringUtil.h"
#include "WebUtil.h"
#include "util.h"

namespace {

const std::string ZTS_RESTART_CONFIG("/usr/local/var/lib/tuelib/zts_restart.conf");
const std::string ZTS_TRANSLATORS_DIR("/tmp/translators");

void SendHeaders() {
    std::cout << "Content-Type: text/html; charset=utf-8\r\n\r\n"
              << "\n";
}

void SendTrailer() {
    std::cout << "\n";

}

struct TranslatorsLocationConfig {
    std::string name_;
    std::string url_;
    std::string local_path_;
    std::string branch_;
    TranslatorsLocationConfig(std::string name = "", std::string url = "",
        std::string local_path = "", std::string branch = "") :
        name_(name), url_(url), local_path_(local_path), branch_(branch)
        {}
};

void GetTranslatorLocationConfigs(const IniFile &ini_file,
                               std::vector<TranslatorsLocationConfig> * translators_location_configs) {
    translators_location_configs->clear();
    const std::string location_prefix("Repo_");
    for (const auto &section : ini_file) {
        if (StringUtil::StartsWith(section.getSectionName(), location_prefix)) {
            TranslatorsLocationConfig translators_location_config;
            translators_location_config.name_ = section.getSectionName().substr(location_prefix.length());
            translators_location_config.url_  = section.getString("url");
            translators_location_config.local_path_  = section.getString("local_path");
            translators_location_config.branch_ = section.getString("branch");
            translators_location_configs->emplace_back(translators_location_config);
        }
    }
}

void DisplayRestartAndSelectButtons(const  std::vector<TranslatorsLocationConfig> &translators_location_configs) {
    SendHeaders();
    std::cout << "
Restart Zotero Translation Server Service
\n"
              << "<form action=\"\" method=\"post\">\n";
    for (const auto &translators_location_config : translators_location_configs)
        std::cout  << "\t<input type=\"submit\" name=\"action\" value=\"" + translators_location_config.name_  +"\">\n";
    std::cout << "\t<input type=\"submit\" name=\"action\" value=\"Restart\">\n"
              << "</form>\n";
    SendTrailer();
}

bool IsRestartActionPresent(const std::multimap<std::string, std::string> &cgi_args) {
    const auto key_and_value(cgi_args.find("action"));
    return key_and_value != cgi_args.cend() and key_and_value->second == "Restart";
}

void ExecuteAndDumpMessages(const std::string &command, const std::vector<std::string> &args) {
    auto auto_temp_file((FileUtil::AutoTempFile()));
    const std::string tmp_output(auto_temp_file.getFilePath());
    ExecUtil::ExecOrDie(command, args, "" /*stdin*/,
                        tmp_output, tmp_output);
    std::ifstream output_file(tmp_output);
    if (not output_file)
        LOG_ERROR("Could not open " + tmp_output + " for reading\n");
    std::stringstream output_istream;
    output_istream << output_file.rdbuf();
    std::cout << StringUtil::ReplaceString("\n", "
", output_istream.str());
}

template<typename Function>
void ExecuteAndSendStatus(const std::string &message, Function function) {
    SendHeaders();
    std::cout << message << std::endl;
    bool log_no_decorations_old(logger->getLogNoDecorations());
    bool log_strip_call_site_old(logger->getLogStripCallSite());
    logger->setLogNoDecorations(true);
    logger->setLogStripCallSite(true);
    logger->redirectOutput(STDOUT_FILENO);
    try {
        function();
    } catch (const std::runtime_error &error) {
        std::cerr << error.what();
    }
    std::cout << "
Done...
\n";
    SendTrailer();
    logger->redirectOutput(STDERR_FILENO);
    logger->setLogNoDecorations(log_no_decorations_old);
    logger->setLogStripCallSite(log_strip_call_site_old);
}

void RestartZTS() {
    auto closure =  []{
                        ExecUtil::ExecOrDie("/usr/bin/sudo", { "systemctl", "restart", "zts" });
                        ExecuteAndDumpMessages("/usr/bin/sudo", { "systemctl", "status", "zts" });
                      };
    ExecuteAndSendStatus("
Trying to restart ZTS Server
", closure);
}

void RelinkTranslatorDirectory(const TranslatorsLocationConfig &translators_location_config) {
     auto closure = [&]{ExecuteAndDumpMessages("/usr/bin/sudo",
                            { "ln" , "--symbolic", "--force", "--no-dereference",
                              translators_location_config.local_path_, ZTS_TRANSLATORS_DIR});};
     ExecuteAndSendStatus("
Switching to branch " + translators_location_config.name_ + "
",
                           closure);
}

bool GetSwitchBranch(const std::multimap<std::string, std::string> &cgi_args,
                     std::vector<TranslatorsLocationConfig> translators_location_configs,
                     TranslatorsLocationConfig * const translator_location_config) {
     const auto key_and_value(cgi_args.find("action"));
     if (key_and_value == cgi_args.end())
         return false;
     const std::string target(key_and_value->second);
     auto match(std::find_if(translators_location_configs.begin(),
                             translators_location_configs.end(),
                             [&target](const TranslatorsLocationConfig &target_obj)
                                      {return target_obj.name_ == target;}));
     if (match == translators_location_configs.end()) {
         std::cout << "NO MATCH";
         return false;
     }
     *translator_location_config = *match;
     return true;
}

} // end unnamed namespace

int Main(int /*argc*/, char */*argv*/[]) {
    std::multimap<std::string, std::string> cgi_args;
    WebUtil::GetAllCgiArgs(&cgi_args);
    IniFile ini_file(ZTS_RESTART_CONFIG);
    std::vector<TranslatorsLocationConfig> translators_location_configs;
    GetTranslatorLocationConfigs(ini_file, &translators_location_configs);
    if (IsRestartActionPresent(cgi_args)) {
        RestartZTS();
        return EXIT_SUCCESS;
    }

    TranslatorsLocationConfig translators_location_config;
    if (GetSwitchBranch(cgi_args, translators_location_configs, &translators_location_config)) {
        RelinkTranslatorDirectory(translators_location_config);
        return EXIT_SUCCESS;
    }

    DisplayRestartAndSelectButtons(translators_location_configs);
    return EXIT_SUCCESS;
}

<|endoftext|>


<commit_before>/****************************************************************/
/*               DO NOT MODIFY THIS HEADER                      */
/* MOOSE - Multiphysics Object Oriented Simulation Environment  */
/*                                                              */
/*           (c) 2010 Battelle Energy Alliance, LLC             */
/*                   ALL RIGHTS RESERVED                        */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

#include "InternalVolume.h"

template <>
InputParameters validParams<InternalVolume>()
{
  InputParameters params = validParams<SideIntegral>();
  params.set<bool>("use_displaced_mesh") = true;
  return params;
}

InternalVolume::InternalVolume(const std::string & name,
                               InputParameters parameters)
  : SideIntegral( name, parameters )
{}

Real
InternalVolume::computeQpIntegral()
{
  return -_q_point[_qp](0)*_normals[_qp](0);
}
<commit_msg>Minor changes<commit_after>/****************************************************************/
/*               DO NOT MODIFY THIS HEADER                      */
/* MOOSE - Multiphysics Object Oriented Simulation Environment  */
/*                                                              */
/*           (c) 2010 Battelle Energy Alliance, LLC             */
/*                   ALL RIGHTS RESERVED                        */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

#include "InternalVolume.h"

template <>
InputParameters validParams<InternalVolume>()
{
  InputParameters params = validParams<SideIntegral>();
  params.set<bool>("use_displaced_mesh") = true;
  return params;
}

InternalVolume::InternalVolume(const std::string & name,
                               InputParameters parameters)
  : SideIntegral( name, parameters )
{}

//    /              /
//   |              |
//   |  div(F) dV = | F dot n dS
//   |              |
//  / V            / dS
//
// with
//   F = a field
//   n = the normal at the surface
//   V = the volume of the domain
//   S = the surface of the domain
//
// If we choose F as [x 0 0]^T, then
//   div(F) = 1.
// So,
//
//    /       /
//   |       |
//   |  dV = | x * n[0] dS
//   |       |
//  / V     / dS
//
// That is, the volume of the domain is the integral over the surface of the domain
// of the x position of the surface times the x-component of the normal of the
// surface.
//

Real
InternalVolume::computeQpIntegral()
{
  return -_q_point[_qp](0)*_normals[_qp](0);
}
<|endoftext|>


<commit_before>/** \brief Utility for deleting partial or entire MARC records based on an input list.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015-2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <vector>
#include <cstdlib>
#include "BSZUtil.h"
#include "FileUtil.h"
#include "MARC.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[--input-format=(marc-21|marc-xml)] [--output-format=(marc-21|marc-xml)]"
            " deletion_list input_marc21 output_marc21 entire_record_deletion_log\n"
            "Record ID's of records that were deleted and not merely modified will be written to \"entire_record_deletion_log\".");
}

/** \brief Deletes LOK sections if their pseudo tags are found in "local_deletion_ids"
 *  \return True if at least one local section has been deleted, else false.
 */
bool DeleteLocalSections(const std::unordered_set <std::string> &local_deletion_ids, MARC::Record * const record) {
    bool modified(false);
    std::vector<MARC::Record::iterator> local_block_starts_for_deletion;

    for (const auto &local_block_start : record->findStartOfAllLocalDataBlocks()) {
        const auto _001_range(record->getLocalTagRange("001", local_block_start));

        if (_001_range.size() != 1)
            LOG_ERROR("Every local data block has to have exactly one 001 field. (Record: "
                      + record->getControlNumber() + ", First field in local block was: "
                      + local_block_start->toString() + " - Found "
                      + std::to_string(_001_range.size()) + ".)");
        const MARC::Subfields subfields(_001_range.front().getSubfields());
        const std::string subfield_contents(subfields.getFirstSubfieldWithCode('0'));
        if (not StringUtil::StartsWith(subfield_contents, "001 ")
            or local_deletion_ids.find(subfield_contents.substr(4)) == local_deletion_ids.end())
            continue;

        local_block_starts_for_deletion.emplace_back(local_block_start);
        modified = true;
    }
    record->deleteLocalBlocks(local_block_starts_for_deletion);

    return modified;
}

void ProcessRecords(const std::unordered_set <std::string> &title_deletion_ids,
                    const std::unordered_set <std::string> &local_deletion_ids, MARC::Reader * const marc_reader,
                    MARC::Writer * const marc_writer, File * const entire_record_deletion_log)
{
    unsigned total_record_count(0), deleted_record_count(0), modified_record_count(0);
    while (MARC::Record record = marc_reader->read()) {
        ++total_record_count;

        if (title_deletion_ids.find(record.getControlNumber()) != title_deletion_ids.end()) {
            ++deleted_record_count;
            (*entire_record_deletion_log) << record.getControlNumber() << '\n';
        } else { // Look for local (LOK) data sets that may need to be deleted.
            if (DeleteLocalSections(local_deletion_ids, &record))
                ++modified_record_count;
            marc_writer->write(record);
        }
    }

    std::cerr << "Read " << total_record_count << " records.\n";
    std::cerr << "Deleted " << deleted_record_count << " records.\n";
    std::cerr << "Modified " << modified_record_count << " records.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc < 5)
        Usage();

    const auto reader_type(MARC::GetOptionalReaderType(&argc, &argv, 1));
    const auto writer_type(MARC::GetOptionalWriterType(&argc, &argv, 1));

    if (argc != 5)
        Usage();

    const auto deletion_list(FileUtil::OpenInputFileOrDie(argv[1]));
    std::unordered_set<std::string> title_deletion_ids, local_deletion_ids;
    BSZUtil::ExtractDeletionIds(deletion_list.get(), &title_deletion_ids, &local_deletion_ids);

    const auto marc_reader(MARC::Reader::Factory(argv[2], reader_type));
    const auto marc_writer(MARC::Writer::Factory(argv[3], writer_type));
    const auto entire_record_deletion_log(FileUtil::OpenForAppendingOrDie(argv[4]));

    ProcessRecords(title_deletion_ids, local_deletion_ids, marc_reader.get(), marc_writer.get(), entire_record_deletion_log.get());

    return EXIT_SUCCESS;
}
<commit_msg>Added a documentation link.<commit_after>/** \brief Utility for deleting partial or entire MARC records based on an input list.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *  \documentation https://wiki.bsz-bw.de/doku.php?id=v-team:daten:datendienste:sekkor under "Löschungen".
 *
 *  \copyright 2015-2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <vector>
#include <cstdlib>
#include "BSZUtil.h"
#include "FileUtil.h"
#include "MARC.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[--input-format=(marc-21|marc-xml)] [--output-format=(marc-21|marc-xml)]"
            " deletion_list input_marc21 output_marc21 entire_record_deletion_log\n"
            "Record ID's of records that were deleted and not merely modified will be written to \"entire_record_deletion_log\".");
}

/** \brief Deletes LOK sections if their pseudo tags are found in "local_deletion_ids"
 *  \return True if at least one local section has been deleted, else false.
 */
bool DeleteLocalSections(const std::unordered_set <std::string> &local_deletion_ids, MARC::Record * const record) {
    bool modified(false);
    std::vector<MARC::Record::iterator> local_block_starts_for_deletion;

    for (const auto &local_block_start : record->findStartOfAllLocalDataBlocks()) {
        const auto _001_range(record->getLocalTagRange("001", local_block_start));

        if (_001_range.size() != 1)
            LOG_ERROR("Every local data block has to have exactly one 001 field. (Record: "
                      + record->getControlNumber() + ", First field in local block was: "
                      + local_block_start->toString() + " - Found "
                      + std::to_string(_001_range.size()) + ".)");
        const MARC::Subfields subfields(_001_range.front().getSubfields());
        const std::string subfield_contents(subfields.getFirstSubfieldWithCode('0'));
        if (not StringUtil::StartsWith(subfield_contents, "001 ")
            or local_deletion_ids.find(subfield_contents.substr(4)) == local_deletion_ids.end())
            continue;

        local_block_starts_for_deletion.emplace_back(local_block_start);
        modified = true;
    }
    record->deleteLocalBlocks(local_block_starts_for_deletion);

    return modified;
}

void ProcessRecords(const std::unordered_set <std::string> &title_deletion_ids,
                    const std::unordered_set <std::string> &local_deletion_ids, MARC::Reader * const marc_reader,
                    MARC::Writer * const marc_writer, File * const entire_record_deletion_log)
{
    unsigned total_record_count(0), deleted_record_count(0), modified_record_count(0);
    while (MARC::Record record = marc_reader->read()) {
        ++total_record_count;

        if (title_deletion_ids.find(record.getControlNumber()) != title_deletion_ids.end()) {
            ++deleted_record_count;
            (*entire_record_deletion_log) << record.getControlNumber() << '\n';
        } else { // Look for local (LOK) data sets that may need to be deleted.
            if (DeleteLocalSections(local_deletion_ids, &record))
                ++modified_record_count;
            marc_writer->write(record);
        }
    }

    std::cerr << "Read " << total_record_count << " records.\n";
    std::cerr << "Deleted " << deleted_record_count << " records.\n";
    std::cerr << "Modified " << modified_record_count << " records.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 5)
        Usage();

    const auto reader_type(MARC::GetOptionalReaderType(&argc, &argv, 1));
    const auto writer_type(MARC::GetOptionalWriterType(&argc, &argv, 1));

    if (argc != 5)
        Usage();

    const auto deletion_list(FileUtil::OpenInputFileOrDie(argv[1]));
    std::unordered_set<std::string> title_deletion_ids, local_deletion_ids;
    BSZUtil::ExtractDeletionIds(deletion_list.get(), &title_deletion_ids, &local_deletion_ids);

    const auto marc_reader(MARC::Reader::Factory(argv[2], reader_type));
    const auto marc_writer(MARC::Writer::Factory(argv[3], writer_type));
    const auto entire_record_deletion_log(FileUtil::OpenForAppendingOrDie(argv[4]));

    ProcessRecords(title_deletion_ids, local_deletion_ids, marc_reader.get(), marc_writer.get(), entire_record_deletion_log.get());

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>#include <pybind11/pybind11.h>

#include <pybind11/stl.h> // for conversions between c++ and python collections
#include <pybind11/numpy.h> // support for numpy arrays
#include "trajectory.h"
#include "raster.h"
#include "planning.h"

namespace py = pybind11;

/** Converts a numpy array to a vector */
template<class T>
std::vector<T> as_vector(py::array_t<T, py::array::c_style | py::array::forcecast> array) {
    std::vector<T> data(array.size());
    for(ssize_t x=0; x<array.shape(0); x++) {
        for(ssize_t y=0; y<array.shape(1); y++) {
            data[x + y*array.shape(0)] = *(array.data(x, y));
        }
    }
    return data;
}

/** Converts a vector to a 2D numpy array. */
template<class T>
py::array_t<T> as_nparray(std::vector<T> vec, size_t x_width, size_t y_height) {
    ASSERT(vec.size() == x_width * y_height)
    py::array_t<T, py::array::c_style | py::array::forcecast> array(std::vector<size_t> {x_width, y_height});
    auto s_x_width = static_cast<ssize_t>(x_width);
    auto s_y_height = static_cast<ssize_t>(y_height);
    for(ssize_t x=0; x<s_x_width; x++) {
        for (ssize_t y = 0; y < s_y_height; y++) {
            *(array.mutable_data(x, y)) = vec[x + y*x_width];
        }
    }
    return array;
}

PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>);

PYBIND11_MODULE(uav_planning, m) {
    m.doc() = "Python module for UAV trajectory planning";

    srand(0);

    py::class_<DRaster>(m, "DRaster")
            .def(py::init([](py::array_t<double, py::array::c_style | py::array::forcecast> arr,
                             double x_offset, double y_offset, double cell_width) {
                return new DRaster(as_vector<double>(arr), arr.shape(0), arr.shape(1), x_offset, y_offset, cell_width);
            }))
            .def("as_numpy", [](DRaster& self) {
                return as_nparray<double>(self.data, self.x_width, self.y_height);
            })
            .def_readonly("x_offset", &DRaster::x_offset)
            .def_readonly("y_offset", &DRaster::y_offset)
            .def_readonly("cell_width", &DRaster::cell_width);

    py::class_<LRaster>(m, "LRaster")
            .def(py::init([](py::array_t<long, py::array::c_style | py::array::forcecast> arr,
                             double x_offset, double y_offset, double cell_width) {
                return new LRaster(as_vector<long>(arr), arr.shape(0), arr.shape(1), x_offset, y_offset, cell_width);
            }))
            .def("as_numpy", [](LRaster& self) {
                return as_nparray<long>(self.data, self.x_width, self.y_height);
            })
            .def_readonly("x_offset", &LRaster::x_offset)
            .def_readonly("y_offset", &LRaster::y_offset)
            .def_readonly("cell_width", &LRaster::cell_width);

    py::class_<TimeWindow>(m, "TimeWindow")
            .def(py::init<const double, const double>(),
                 py::arg("start"), py::arg("end"))
            .def_readonly("start", &TimeWindow::start)
            .def_readonly("end", &TimeWindow::end)
            .def("__repr__",
                 [](const TimeWindow &tw) {
                     std::stringstream repr;
                     repr << "TimeWindow(" << tw.start << ", " << tw.end << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](TimeWindow &self) {
                return py::make_tuple(self.start, self.end);
            } );

    py::class_<Cell>(m, "Cell")
            .def(py::init<const size_t, const size_t>(),
                 py::arg("x"), py::arg("y"))
            .def_readonly("x", &Cell::x)
            .def_readonly("y", &Cell::y)
            .def("__repr__",
                 [](const Cell &c) {
                     std::stringstream repr;
                     repr << "Cell(" << c.x << ", " << c.y << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Cell &self) {
                return py::make_tuple(self.x, self.y);
            } );

    py::class_<Position>(m, "Position2d")
            .def(py::init<double, double>(),
                 py::arg("x"), py::arg("y"))
            .def_readonly("x", &Position::x)
            .def_readonly("y", &Position::y)
            .def("__repr__",
                 [](const Position &p) {
                     std::stringstream repr;
                     repr << "Position2d(" << p.x << ", " << p.y << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position &self) {
                return py::make_tuple(self.x, self.y);
            } );

    py::class_<Position3d>(m, "Position")
            .def(py::init<double, double, double>(),
                 py::arg("x"), py::arg("y"), py::arg("z"))
            .def_readonly("x", &Position3d::x)
            .def_readonly("y", &Position3d::y)
            .def_readonly("z", &Position3d::z)
            .def("__repr__",
                 [](const Position3d &p) {
                     std::stringstream repr;
                     repr << "Position(" << p.x << ", " << p.y << ", " << p.z << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position3d &self) {
                return py::make_tuple(self.x, self.y, self.z);
            } );

    py::class_<PositionTime>(m, "Position2dTime")
            .def(py::init<Position, double>(),
                 py::arg("point"), py::arg("time"))
            .def_readonly("pt", &PositionTime::pt)
            .def_readonly("y", &PositionTime::time)
            .def("__repr__",
                 [](const PositionTime &p) {
                     std::stringstream repr;
                     repr << "Position2dTime(" << p.pt.x << ", " << p.pt.y << ", " << p.time << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](PositionTime &self) {
                return py::make_tuple(py::make_tuple(self.pt.x, self.pt.y), self.time);
            } );

    py::class_<Position3dTime>(m, "PositionTime")
            .def(py::init<Position3d, double>(),
                 py::arg("point"), py::arg("time"))
            .def_readonly("pt", &Position3dTime::pt)
            .def_readonly("y", &Position3dTime::time)
            .def("__repr__",
                 [](const Position3dTime &p) {
                     std::stringstream repr;
                     repr << "PositionTime(" << p.pt.x << ", " << p.pt.y << ", " << p.pt.z << ", " << p.time << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position3dTime &self) {
                return py::make_tuple(py::make_tuple(self.pt.x, self.pt.y, self.pt.z), self.time);
            } );

    py::class_<IsochroneCluster, shared_ptr<IsochroneCluster>>(m, "IsochroneCluster")
            .def(py::init<const TimeWindow&, vector<Cell>>(), py::arg("tw") , py::arg("cell_vector"))
            .def_readonly("time_window", &IsochroneCluster::time_window)
            .def_readonly("cells", &IsochroneCluster::cells);

    py::class_<FireData>(m, "FireData")
            .def(py::init<DRaster&, DDiscreteRaster&>(), py::arg("ignitions"), py::arg("elevation"))
            .def_readonly("ignitions", &FireData::ignitions)
            .def_readonly("traversal_end", &FireData::traversal_end)
            .def_readonly("propagation_directions", &FireData::propagation_directions)
            .def_readonly("elevation", &FireData::elevation);
//            .def_readonly_static("isochrone_timespan", &FireData::isochrone_timespan)
//            .def_readonly("isochrones", &FireData::isochrones);

    py::class_<Waypoint3d>(m, "Waypoint")
            .def(py::init<const double, const double, const double, const double>(),
                 py::arg("x"), py::arg("y"), py::arg("z"), py::arg("direction"))
            .def_readonly("x", &Waypoint3d::x)
            .def_readonly("y", &Waypoint3d::y)
            .def_readonly("z", &Waypoint3d::z)
            .def_readonly("dir", &Waypoint3d::dir)
            .def("__repr__", &Waypoint3d::to_string);

    py::class_<Segment3d>(m, "Segment")
            .def(py::init<const Waypoint3d, const double>())
            .def(py::init<const Waypoint3d, const Waypoint3d>())
            .def_readonly("start", &Segment3d::start)
            .def_readonly("end", &Segment3d::end)
            .def_readonly("length", &Segment3d::length)
            .def("__repr__", &Segment3d::to_string);

    py::class_<UAV>(m, "UAV")
            .def(py::init<const double, const double, const double>())
            .def_readonly("min_turn_radius", &UAV::min_turn_radius)
            .def_readonly("max_air_speed", &UAV::max_air_speed)
            .def_readonly("max_pitch_angle", &UAV::max_pitch_angle)
            .def("travel_distance", (double (UAV::*)(const Waypoint3d &, const Waypoint3d &) const)&UAV::travel_distance, py::arg("origin"), py::arg("destination"))
            .def("travel_distance", (double (UAV::*)(const Waypoint &, const Waypoint &) const)&UAV::travel_distance, py::arg("origin"), py::arg("destination"))
            .def("travel_time", (double (UAV::*)(const Waypoint3d &, const Waypoint3d &) const)&UAV::travel_time, py::arg("origin"), py::arg("destination"))
            .def("travel_time", (double (UAV::*)(const Waypoint &, const Waypoint &) const)&UAV::travel_time, py::arg("origin"), py::arg("destination"));

    py::class_<Trajectory>(m, "Trajectory")
            .def(py::init<const TrajectoryConfig&>())
            .def("start_time", (double (Trajectory::*)() const)&Trajectory::start_time)
            .def("end_time", (double (Trajectory::*)() const)&Trajectory::end_time)
            .def_readonly("segments", &Trajectory::traj)
            .def("length", &Trajectory::length)
            .def("duration", &Trajectory::duration)
            .def("as_waypoints", &Trajectory::as_waypoints)
            .def("sampled", &Trajectory::sampled, py::arg("step_size") = 1)
            .def("with_waypoint_at_end", &Trajectory::with_waypoint_at_end)
            .def("__repr__", &Trajectory::to_string);

    py::class_<TrajectoryConfig>(m, "TrajectoryConfig")
            .def(py::init<UAV, Waypoint3d, Waypoint3d, double, double>())
            .def_readonly("uav", &TrajectoryConfig::uav)
            .def_readonly("max_flight_time", &TrajectoryConfig::max_flight_time)
            .def_static("build", [](UAV uav, double start_time, double max_flight_time) -> TrajectoryConfig {
                return TrajectoryConfig(uav, start_time, max_flight_time);
            }, "Constructor", py::arg("uav"), py::arg("start_time") = 0,
                        py::arg("max_flight_time") = std::numeric_limits<double>::max());

    py::class_<Plan>(m, "Plan")
            .def_readonly("trajectories", &Plan::trajectories)
            .def("utility", &Plan::utility)
            .def("duration", &Plan::duration)
            .def_readonly("firedata", &Plan::firedata)
            .def("observations", &Plan::observations);

    py::class_<SearchResult>(m, "SearchResult")
            .def("initial_plan", &SearchResult::initial)
            .def("final_plan", &SearchResult::final)
            .def_readonly("intermediate_plans", &SearchResult::intermediate_plans)
            .def_readonly("planning_time", &SearchResult::planning_time)
            .def_readonly("preprocessing_time", &SearchResult::preprocessing_time);

    m.def("make_plan_vns", [](UAV uav, DRaster ignitions, DRaster elevation, double min_time, double max_time,
                              double max_flight_time, size_t save_every, bool save_improvements,
                              size_t discrete_elevation_interval=1) -> SearchResult {
        auto fire_data = make_shared<FireData>(ignitions, DDiscreteRaster(std::move(elevation),
                                                                          discrete_elevation_interval));
        TrajectoryConfig conf(uav, min_time, max_flight_time);
        Plan p(vector<TrajectoryConfig> { conf }, fire_data, TimeWindow{min_time, max_time});

        DefaultVnsSearch vns;

        auto res = vns.search(p, 0, save_every, save_improvements);
        return res;
    }, py::arg("uav"), py::arg("ignitions"), py::arg("elevation"), py::arg("min_time"), py::arg("max_time"),
          py::arg("max_flight_time"), py::arg("save_every") = 0, py::arg("save_improvements") = false,
          py::arg("discrete_elevation_interval") = 1);

    m.def("plan_vns", [](vector<TrajectoryConfig> configs, DRaster ignitions, DRaster elevation, double min_time,
                         double max_time, size_t save_every, bool save_improvements=false,
                         size_t discrete_elevation_interval=1) -> SearchResult {
        auto time = []() {
            struct timeval tp;
            gettimeofday(&tp, NULL);
            return (double) tp.tv_sec + ((double)(tp.tv_usec / 1000) /1000.);
        };

        printf("Processing firedata data\n");
        double preprocessing_start = time();
        auto fire_data = make_shared<FireData>(ignitions, DDiscreteRaster(std::move(elevation),
                                                                          discrete_elevation_interval));
        double preprocessing_end = time();

        printf("Building initial plan\n");
        Plan p(configs, fire_data, TimeWindow{min_time, max_time});

        printf("Planning\n");
        DefaultVnsSearch vns;
        const double planning_start = time();
        auto res = vns.search(p, 0, save_every, save_improvements);
        const double planning_end = time();
        printf("Plan found\n");
        res.planning_time = planning_end - planning_start;
        res.preprocessing_time = preprocessing_end - preprocessing_start;
        return res;
    }, py::arg("trajectory_configs"), py::arg("ignitions"), py::arg("elevation"), py::arg("min_time"),
          py::arg("max_time"), py::arg("save_every") = 0, py::arg("save_improvements") = false,
          py::arg("discrete_elevation_interval") = 1, py::call_guard<py::gil_scoped_release>());
}
<commit_msg>Make trajectory configuration available from python.<commit_after>#include <pybind11/pybind11.h>

#include <pybind11/stl.h> // for conversions between c++ and python collections
#include <pybind11/numpy.h> // support for numpy arrays
#include "trajectory.h"
#include "raster.h"
#include "planning.h"

namespace py = pybind11;

/** Converts a numpy array to a vector */
template<class T>
std::vector<T> as_vector(py::array_t<T, py::array::c_style | py::array::forcecast> array) {
    std::vector<T> data(array.size());
    for(ssize_t x=0; x<array.shape(0); x++) {
        for(ssize_t y=0; y<array.shape(1); y++) {
            data[x + y*array.shape(0)] = *(array.data(x, y));
        }
    }
    return data;
}

/** Converts a vector to a 2D numpy array. */
template<class T>
py::array_t<T> as_nparray(std::vector<T> vec, size_t x_width, size_t y_height) {
    ASSERT(vec.size() == x_width * y_height)
    py::array_t<T, py::array::c_style | py::array::forcecast> array(std::vector<size_t> {x_width, y_height});
    auto s_x_width = static_cast<ssize_t>(x_width);
    auto s_y_height = static_cast<ssize_t>(y_height);
    for(ssize_t x=0; x<s_x_width; x++) {
        for (ssize_t y = 0; y < s_y_height; y++) {
            *(array.mutable_data(x, y)) = vec[x + y*x_width];
        }
    }
    return array;
}

PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>);

PYBIND11_MODULE(uav_planning, m) {
    m.doc() = "Python module for UAV trajectory planning";

    srand(0);

    py::class_<DRaster>(m, "DRaster")
            .def(py::init([](py::array_t<double, py::array::c_style | py::array::forcecast> arr,
                             double x_offset, double y_offset, double cell_width) {
                return new DRaster(as_vector<double>(arr), arr.shape(0), arr.shape(1), x_offset, y_offset, cell_width);
            }))
            .def("as_numpy", [](DRaster& self) {
                return as_nparray<double>(self.data, self.x_width, self.y_height);
            })
            .def_readonly("x_offset", &DRaster::x_offset)
            .def_readonly("y_offset", &DRaster::y_offset)
            .def_readonly("cell_width", &DRaster::cell_width);

    py::class_<LRaster>(m, "LRaster")
            .def(py::init([](py::array_t<long, py::array::c_style | py::array::forcecast> arr,
                             double x_offset, double y_offset, double cell_width) {
                return new LRaster(as_vector<long>(arr), arr.shape(0), arr.shape(1), x_offset, y_offset, cell_width);
            }))
            .def("as_numpy", [](LRaster& self) {
                return as_nparray<long>(self.data, self.x_width, self.y_height);
            })
            .def_readonly("x_offset", &LRaster::x_offset)
            .def_readonly("y_offset", &LRaster::y_offset)
            .def_readonly("cell_width", &LRaster::cell_width);

    py::class_<TimeWindow>(m, "TimeWindow")
            .def(py::init<const double, const double>(),
                 py::arg("start"), py::arg("end"))
            .def_readonly("start", &TimeWindow::start)
            .def_readonly("end", &TimeWindow::end)
            .def("__repr__",
                 [](const TimeWindow &tw) {
                     std::stringstream repr;
                     repr << "TimeWindow(" << tw.start << ", " << tw.end << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](TimeWindow &self) {
                return py::make_tuple(self.start, self.end);
            } );

    py::class_<Cell>(m, "Cell")
            .def(py::init<const size_t, const size_t>(),
                 py::arg("x"), py::arg("y"))
            .def_readonly("x", &Cell::x)
            .def_readonly("y", &Cell::y)
            .def("__repr__",
                 [](const Cell &c) {
                     std::stringstream repr;
                     repr << "Cell(" << c.x << ", " << c.y << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Cell &self) {
                return py::make_tuple(self.x, self.y);
            } );

    py::class_<Position>(m, "Position2d")
            .def(py::init<double, double>(),
                 py::arg("x"), py::arg("y"))
            .def_readonly("x", &Position::x)
            .def_readonly("y", &Position::y)
            .def("__repr__",
                 [](const Position &p) {
                     std::stringstream repr;
                     repr << "Position2d(" << p.x << ", " << p.y << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position &self) {
                return py::make_tuple(self.x, self.y);
            } );

    py::class_<Position3d>(m, "Position")
            .def(py::init<double, double, double>(),
                 py::arg("x"), py::arg("y"), py::arg("z"))
            .def_readonly("x", &Position3d::x)
            .def_readonly("y", &Position3d::y)
            .def_readonly("z", &Position3d::z)
            .def("__repr__",
                 [](const Position3d &p) {
                     std::stringstream repr;
                     repr << "Position(" << p.x << ", " << p.y << ", " << p.z << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position3d &self) {
                return py::make_tuple(self.x, self.y, self.z);
            } );

    py::class_<PositionTime>(m, "Position2dTime")
            .def(py::init<Position, double>(),
                 py::arg("point"), py::arg("time"))
            .def_readonly("pt", &PositionTime::pt)
            .def_readonly("y", &PositionTime::time)
            .def("__repr__",
                 [](const PositionTime &p) {
                     std::stringstream repr;
                     repr << "Position2dTime(" << p.pt.x << ", " << p.pt.y << ", " << p.time << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](PositionTime &self) {
                return py::make_tuple(py::make_tuple(self.pt.x, self.pt.y), self.time);
            } );

    py::class_<Position3dTime>(m, "PositionTime")
            .def(py::init<Position3d, double>(),
                 py::arg("point"), py::arg("time"))
            .def_readonly("pt", &Position3dTime::pt)
            .def_readonly("y", &Position3dTime::time)
            .def("__repr__",
                 [](const Position3dTime &p) {
                     std::stringstream repr;
                     repr << "PositionTime(" << p.pt.x << ", " << p.pt.y << ", " << p.pt.z << ", " << p.time << ")";
                     return repr.str();
                 }
            )
            .def("as_tuple", [](Position3dTime &self) {
                return py::make_tuple(py::make_tuple(self.pt.x, self.pt.y, self.pt.z), self.time);
            } );

    py::class_<IsochroneCluster, shared_ptr<IsochroneCluster>>(m, "IsochroneCluster")
            .def(py::init<const TimeWindow&, vector<Cell>>(), py::arg("tw") , py::arg("cell_vector"))
            .def_readonly("time_window", &IsochroneCluster::time_window)
            .def_readonly("cells", &IsochroneCluster::cells);

    py::class_<FireData>(m, "FireData")
            .def(py::init<DRaster&, DDiscreteRaster&>(), py::arg("ignitions"), py::arg("elevation"))
            .def_readonly("ignitions", &FireData::ignitions)
            .def_readonly("traversal_end", &FireData::traversal_end)
            .def_readonly("propagation_directions", &FireData::propagation_directions)
            .def_readonly("elevation", &FireData::elevation);
//            .def_readonly_static("isochrone_timespan", &FireData::isochrone_timespan)
//            .def_readonly("isochrones", &FireData::isochrones);

    py::class_<Waypoint3d>(m, "Waypoint")
            .def(py::init<const double, const double, const double, const double>(),
                 py::arg("x"), py::arg("y"), py::arg("z"), py::arg("direction"))
            .def_readonly("x", &Waypoint3d::x)
            .def_readonly("y", &Waypoint3d::y)
            .def_readonly("z", &Waypoint3d::z)
            .def_readonly("dir", &Waypoint3d::dir)
            .def("__repr__", &Waypoint3d::to_string);

    py::class_<Segment3d>(m, "Segment")
            .def(py::init<const Waypoint3d, const double>())
            .def(py::init<const Waypoint3d, const Waypoint3d>())
            .def_readonly("start", &Segment3d::start)
            .def_readonly("end", &Segment3d::end)
            .def_readonly("length", &Segment3d::length)
            .def("__repr__", &Segment3d::to_string);

    py::class_<UAV>(m, "UAV")
            .def(py::init<const double, const double, const double>())
            .def_readonly("min_turn_radius", &UAV::min_turn_radius)
            .def_readonly("max_air_speed", &UAV::max_air_speed)
            .def_readonly("max_pitch_angle", &UAV::max_pitch_angle)
            .def("travel_distance", (double (UAV::*)(const Waypoint3d &, const Waypoint3d &) const)&UAV::travel_distance, py::arg("origin"), py::arg("destination"))
            .def("travel_distance", (double (UAV::*)(const Waypoint &, const Waypoint &) const)&UAV::travel_distance, py::arg("origin"), py::arg("destination"))
            .def("travel_time", (double (UAV::*)(const Waypoint3d &, const Waypoint3d &) const)&UAV::travel_time, py::arg("origin"), py::arg("destination"))
            .def("travel_time", (double (UAV::*)(const Waypoint &, const Waypoint &) const)&UAV::travel_time, py::arg("origin"), py::arg("destination"));

    py::class_<Trajectory>(m, "Trajectory")
            .def(py::init<const TrajectoryConfig&>())
            .def_readonly("conf", &Trajectory::conf)
            .def("start_time", (double (Trajectory::*)() const)&Trajectory::start_time)
            .def("end_time", (double (Trajectory::*)() const)&Trajectory::end_time)
            .def_readonly("segments", &Trajectory::traj)
            .def("length", &Trajectory::length)
            .def("duration", &Trajectory::duration)
            .def("as_waypoints", &Trajectory::as_waypoints)
            .def("sampled", &Trajectory::sampled, py::arg("step_size") = 1)
            .def("with_waypoint_at_end", &Trajectory::with_waypoint_at_end)
            .def("__repr__", &Trajectory::to_string);

    py::class_<TrajectoryConfig>(m, "TrajectoryConfig")
            .def(py::init<UAV, Waypoint3d, Waypoint3d, double, double>())
            .def_readonly("uav", &TrajectoryConfig::uav)
            .def_readonly("max_flight_time", &TrajectoryConfig::max_flight_time)
            .def_static("build", [](UAV uav, double start_time, double max_flight_time) -> TrajectoryConfig {
                return TrajectoryConfig(uav, start_time, max_flight_time);
            }, "Constructor", py::arg("uav"), py::arg("start_time") = 0,
                        py::arg("max_flight_time") = std::numeric_limits<double>::max());

    py::class_<Plan>(m, "Plan")
            .def_readonly("trajectories", &Plan::trajectories)
            .def("utility", &Plan::utility)
            .def("duration", &Plan::duration)
            .def_readonly("firedata", &Plan::firedata)
            .def("observations", &Plan::observations);

    py::class_<SearchResult>(m, "SearchResult")
            .def("initial_plan", &SearchResult::initial)
            .def("final_plan", &SearchResult::final)
            .def_readonly("intermediate_plans", &SearchResult::intermediate_plans)
            .def_readonly("planning_time", &SearchResult::planning_time)
            .def_readonly("preprocessing_time", &SearchResult::preprocessing_time);

    m.def("make_plan_vns", [](UAV uav, DRaster ignitions, DRaster elevation, double min_time, double max_time,
                              double max_flight_time, size_t save_every, bool save_improvements,
                              size_t discrete_elevation_interval=1) -> SearchResult {
        auto fire_data = make_shared<FireData>(ignitions, DDiscreteRaster(std::move(elevation),
                                                                          discrete_elevation_interval));
        TrajectoryConfig conf(uav, min_time, max_flight_time);
        Plan p(vector<TrajectoryConfig> { conf }, fire_data, TimeWindow{min_time, max_time});

        DefaultVnsSearch vns;

        auto res = vns.search(p, 0, save_every, save_improvements);
        return res;
    }, py::arg("uav"), py::arg("ignitions"), py::arg("elevation"), py::arg("min_time"), py::arg("max_time"),
          py::arg("max_flight_time"), py::arg("save_every") = 0, py::arg("save_improvements") = false,
          py::arg("discrete_elevation_interval") = 1);

    m.def("plan_vns", [](vector<TrajectoryConfig> configs, DRaster ignitions, DRaster elevation, double min_time,
                         double max_time, size_t save_every, bool save_improvements=false,
                         size_t discrete_elevation_interval=1) -> SearchResult {
        auto time = []() {
            struct timeval tp;
            gettimeofday(&tp, NULL);
            return (double) tp.tv_sec + ((double)(tp.tv_usec / 1000) /1000.);
        };

        printf("Processing firedata data\n");
        double preprocessing_start = time();
        auto fire_data = make_shared<FireData>(ignitions, DDiscreteRaster(std::move(elevation),
                                                                          discrete_elevation_interval));
        double preprocessing_end = time();

        printf("Building initial plan\n");
        Plan p(configs, fire_data, TimeWindow{min_time, max_time});

        printf("Planning\n");
        DefaultVnsSearch vns;
        const double planning_start = time();
        auto res = vns.search(p, 0, save_every, save_improvements);
        const double planning_end = time();
        printf("Plan found\n");
        res.planning_time = planning_end - planning_start;
        res.preprocessing_time = preprocessing_end - preprocessing_start;
        return res;
    }, py::arg("trajectory_configs"), py::arg("ignitions"), py::arg("elevation"), py::arg("min_time"),
          py::arg("max_time"), py::arg("save_every") = 0, py::arg("save_improvements") = false,
          py::arg("discrete_elevation_interval") = 1, py::call_guard<py::gil_scoped_release>());
}
<|endoftext|>


<commit_before>/** \brief Utility for replacing generating up-to-date authority MARC collections.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <cstdio>
#include <cstdlib>
#include <dirent.h>
#include "Compiler.h"
#include "BSZUtil.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcUtil.h"
#include "MarcWriter.h"
#include "RegexMatcher.h"
#include "util.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " deletion_list reference_records source_records target_records\n"
              << "       Replaces all records in \"source_records\" that have an identical control number\n"
              << "       as a record in \"reference_records\" with the corresponding record in\n"
              << "       \"reference_records\".  The file with the replacements as well as any records\n"
              << "       that could not be replaced is the output file \"target_records\".\n"
              << "       \"deletion_list\", \"reference_records\", and \"source_records\" must all be regular\n"
              << "       expressions containing \\d\\d\\d\\d\\d\\d stading in for YYMMDD.  Additionally\n"
              << "       \"target_records\" must also contain the YYMMDD patternNo  (No other metacharacters\n"
              << "       than \\d should probably be used.)\n\n";
    std::exit(EXIT_FAILURE);
}

/** \param path_regex  A PCRE regex that must contain a \d\d\d\d\d\d subexpression standing in for YYYYMMDD.
 *  \return Either the most recent file or the empty string if no files matched the regex.
 */
std::string GetMostRecentFile(const std::string &path_regex) {
    if (unlikely(path_regex.find("\\d\\d\\d\\d\\d\\d") == std::string::npos))
        Error("in GetMostRecentFile: regex \"" + path_regex + "\" does not contain \\d\\d\\d\\d\\d\\d!");

    std::string filename, directory;
    FileUtil::DirnameAndBasename(path_regex, &filename, &directory);

    std::string err_msg;
    RegexMatcher *matcher(RegexMatcher::RegexMatcherFactory(filename, &err_msg));
    if (unlikely(matcher == nullptr))
        Error("in GetMostRecentFile: failed to compile regex \"" + filename + "\"! (" + err_msg + ")");

    DIR * const directory_stream(::opendir(directory.c_str()));
    if (unlikely(directory_stream == nullptr))
        Error("in GetMostRecentFile: opendir(" + directory + ") failed(" + std::string(::strerror(errno)) + ")");

    std::string most_recent_file;
    struct dirent *entry;
    while ((entry = ::readdir(directory_stream)) != nullptr) {
        if ((entry->d_type == DT_REG or entry->d_type == DT_UNKNOWN) and matcher->matched(entry->d_name)) {
            std::string dir_entry(entry->d_name);
            if (dir_entry > most_recent_file)
                most_recent_file.swap(dir_entry);
        }
    }
    ::closedir(directory_stream);
    delete matcher;

    return most_recent_file;
}

// Copies records from "marc_reader" to "marc_writer", skipping those whose ID's are found in
// "delete_full_record_ids".
void EraseRecords(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                  const std::unordered_set <std::string> &delete_full_record_ids)
{
    std::cout << "Eliminating records listed in a deletion list...\n";

    unsigned total_record_count(0), deletion_count(0);
    while (const MarcRecord record = marc_reader->read()) {
        ++total_record_count;

        if (delete_full_record_ids.find(record.getControlNumber()) == delete_full_record_ids.cend())
            ++deletion_count;
        else
            marc_writer->write(record);
    }

    std::cout << "Read " << total_record_count << " records and dropped " << deletion_count << " records.\n";
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
        Usage();

    try {
        const std::string MARC_TEMP_FILENAME("/tmp/update_authority_data.temp.mrc");

        const std::string MARC_TARGET_FILENAME(argv[4]);
        const std::string MARC_TARGET_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_TARGET_FILENAME));

        const std::string MARC_SOURCE_FILENAME(GetMostRecentFile(argv[3]));
        const std::string MARC_SOURCE_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_SOURCE_FILENAME));

        const std::string DELETION_LIST_FILENAME(GetMostRecentFile(argv[1]));
        const std::string DELETION_LIST_DATE(BSZUtil::ExtractDateFromFilenameOrDie(DELETION_LIST_FILENAME));

        if (DELETION_LIST_DATE >= MARC_SOURCE_DATE) {
            std::unique_ptr<File> deletion_list_file(FileUtil::OpenInputFileOrDie(DELETION_LIST_FILENAME));
            std::unordered_set <std::string> delete_full_record_ids, local_deletion_ids;
            BSZUtil::ExtractDeletionIds(deletion_list_file.get(), &delete_full_record_ids, &local_deletion_ids);

            std::unique_ptr<MarcReader> marc_source_reader(MarcReader::Factory(MARC_SOURCE_FILENAME));
            std::unique_ptr<MarcWriter> marc_temp_writer(MarcWriter::Factory(MARC_TEMP_FILENAME));
            EraseRecords(marc_source_reader.get(), marc_temp_writer.get(), delete_full_record_ids);
        } else
            FileUtil::CopyOrDie(MARC_SOURCE_FILENAME, MARC_TEMP_FILENAME);

        const std::string MARC_REFERENCE_FILENAME(GetMostRecentFile(argv[2]));
        const std::string MARC_REFERENCE_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_REFERENCE_FILENAME));
        if (MARC_REFERENCE_DATE >= MARC_SOURCE_DATE) {
            const std::string REPLACE_MARC_RECORDS_PATH("/usr/local/bin/replace_marc_records");
            if (ExecUtil::Exec(REPLACE_MARC_RECORDS_PATH,
                               { MARC_REFERENCE_FILENAME, MARC_TEMP_FILENAME, MARC_TARGET_FILENAME }) != 0)
                Error("failed to execute \"" + REPLACE_MARC_RECORDS_PATH + "\"!");
        } else
            FileUtil::CopyOrDie(MARC_TEMP_FILENAME, MARC_TARGET_FILENAME);
    } catch (const std::exception &e) {
        Error("Caught exception: " + std::string(e.what()));
    }
}
<commit_msg>Added a sanity check.<commit_after>/** \brief Utility for replacing generating up-to-date authority MARC collections.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <cstdio>
#include <cstdlib>
#include <dirent.h>
#include "Compiler.h"
#include "BSZUtil.h"
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcUtil.h"
#include "MarcWriter.h"
#include "RegexMatcher.h"
#include "util.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " deletion_list reference_records source_records target_records\n"
              << "       Replaces all records in \"source_records\" that have an identical control number\n"
              << "       as a record in \"reference_records\" with the corresponding record in\n"
              << "       \"reference_records\".  The file with the replacements as well as any records\n"
              << "       that could not be replaced is the output file \"target_records\".\n"
              << "       \"deletion_list\", \"reference_records\", and \"source_records\" must all be regular\n"
              << "       expressions containing \\d\\d\\d\\d\\d\\d stading in for YYMMDD.  Additionally\n"
              << "       \"target_records\" must also contain the YYMMDD patternNo  (No other metacharacters\n"
              << "       than \\d should probably be used.)\n\n";
    std::exit(EXIT_FAILURE);
}

/** \param path_regex  A PCRE regex that must contain a \d\d\d\d\d\d subexpression standing in for YYYYMMDD.
 *  \return Either the most recent file or the empty string if no files matched the regex.
 */
std::string GetMostRecentFile(const std::string &path_regex) {
    if (unlikely(path_regex.find("\\d\\d\\d\\d\\d\\d") == std::string::npos))
        Error("in GetMostRecentFile: regex \"" + path_regex + "\" does not contain \\d\\d\\d\\d\\d\\d!");

    std::string filename, directory;
    FileUtil::DirnameAndBasename(path_regex, &filename, &directory);

    std::string err_msg;
    RegexMatcher *matcher(RegexMatcher::RegexMatcherFactory(filename, &err_msg));
    if (unlikely(matcher == nullptr))
        Error("in GetMostRecentFile: failed to compile regex \"" + filename + "\"! (" + err_msg + ")");

    DIR * const directory_stream(::opendir(directory.c_str()));
    if (unlikely(directory_stream == nullptr))
        Error("in GetMostRecentFile: opendir(" + directory + ") failed(" + std::string(::strerror(errno)) + ")");

    std::string most_recent_file;
    struct dirent *entry;
    while ((entry = ::readdir(directory_stream)) != nullptr) {
        if ((entry->d_type == DT_REG or entry->d_type == DT_UNKNOWN) and matcher->matched(entry->d_name)) {
            std::string dir_entry(entry->d_name);
            if (dir_entry > most_recent_file)
                most_recent_file.swap(dir_entry);
        }
    }
    ::closedir(directory_stream);
    delete matcher;

    return most_recent_file;
}

// Copies records from "marc_reader" to "marc_writer", skipping those whose ID's are found in
// "delete_full_record_ids".
void EraseRecords(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                  const std::unordered_set <std::string> &delete_full_record_ids)
{
    std::cout << "Eliminating records listed in a deletion list...\n";

    unsigned total_record_count(0), deletion_count(0);
    while (const MarcRecord record = marc_reader->read()) {
        ++total_record_count;

        if (delete_full_record_ids.find(record.getControlNumber()) == delete_full_record_ids.cend())
            ++deletion_count;
        else
            marc_writer->write(record);
    }

    std::cout << "Read " << total_record_count << " records and dropped " << deletion_count << " records.\n";
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
        Usage();

    try {
        const std::string MARC_TEMP_FILENAME("/tmp/update_authority_data.temp.mrc");

        const std::string MARC_TARGET_FILENAME(argv[4]);
        const std::string MARC_TARGET_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_TARGET_FILENAME));

        const std::string MARC_SOURCE_FILENAME(GetMostRecentFile(argv[3]));
        const std::string MARC_SOURCE_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_SOURCE_FILENAME));

        if (MARC_TARGET_DATE >= MARC_SOURCE_DATE) {
            std::cout << "Nothing to be done!\n";
            return EXIT_SUCCESS;
        }

        const std::string DELETION_LIST_FILENAME(GetMostRecentFile(argv[1]));
        const std::string DELETION_LIST_DATE(BSZUtil::ExtractDateFromFilenameOrDie(DELETION_LIST_FILENAME));

        if (DELETION_LIST_DATE >= MARC_SOURCE_DATE) {
            std::unique_ptr<File> deletion_list_file(FileUtil::OpenInputFileOrDie(DELETION_LIST_FILENAME));
            std::unordered_set <std::string> delete_full_record_ids, local_deletion_ids;
            BSZUtil::ExtractDeletionIds(deletion_list_file.get(), &delete_full_record_ids, &local_deletion_ids);

            std::unique_ptr<MarcReader> marc_source_reader(MarcReader::Factory(MARC_SOURCE_FILENAME));
            std::unique_ptr<MarcWriter> marc_temp_writer(MarcWriter::Factory(MARC_TEMP_FILENAME));
            EraseRecords(marc_source_reader.get(), marc_temp_writer.get(), delete_full_record_ids);
        } else
            FileUtil::CopyOrDie(MARC_SOURCE_FILENAME, MARC_TEMP_FILENAME);

        const std::string MARC_REFERENCE_FILENAME(GetMostRecentFile(argv[2]));
        const std::string MARC_REFERENCE_DATE(BSZUtil::ExtractDateFromFilenameOrDie(MARC_REFERENCE_FILENAME));
        if (MARC_REFERENCE_DATE >= MARC_SOURCE_DATE) {
            const std::string REPLACE_MARC_RECORDS_PATH("/usr/local/bin/replace_marc_records");
            if (ExecUtil::Exec(REPLACE_MARC_RECORDS_PATH,
                               { MARC_REFERENCE_FILENAME, MARC_TEMP_FILENAME, MARC_TARGET_FILENAME }) != 0)
                Error("failed to execute \"" + REPLACE_MARC_RECORDS_PATH + "\"!");
        } else
            FileUtil::CopyOrDie(MARC_TEMP_FILENAME, MARC_TARGET_FILENAME);
    } catch (const std::exception &e) {
        Error("Caught exception: " + std::string(e.what()));
    }
}
<|endoftext|>


<commit_before>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include "LanguageModelFactory.h"
#include "UserMessage.h"
#include "TypeDef.h"
#include "FactorCollection.h"

// include appropriate header
#ifdef LM_SRI
#  include "LanguageModelSRI.h"
#endif
#ifdef LM_IRST
#  include "LanguageModelIRST.h"
#endif

#include "LanguageModelInternal.h"
#include "LanguageModelSkip.h"
#include "LanguageModelJoint.h"

namespace LanguageModelFactory
{

	LanguageModel* CreateLanguageModel(LMImplementation lmImplementation, const std::vector<FactorType> &factorTypes
                                   , size_t nGramOrder, const std::string &languageModelFile, float weight, FactorCollection &factorCollection)
	{
	  LanguageModel *lm = NULL;
	  switch (lmImplementation)
	  {
	  	case SRI:
				#ifdef LM_SRI
				  lm = new LanguageModelSRI(true);
			  #elif LM_IRST
	     		lm = new LanguageModelIRST(true);
				#else
					lm = new LanguageModelInternal(true);
			  #endif
			  break;
			case IRST:
				#ifdef LM_IRST
	     		lm = new LanguageModelIRST(true);
			  #elif LM_IRST
	     		lm = new LanguageModelSRI(true);
				#else
					lm = new LanguageModelInternal(true);
			  #endif
				break;
			case Skip:
				#ifdef LM_SRI
	     		lm = new LanguageModelSkip(new LanguageModelSRI(false), true);
				#elif LM_SRI
	     			lm = new LanguageModelSkip(new LanguageModelIRST(false), true);
				#else
	     			lm = new LanguageModelSkip(new LanguageModelInternal(false), true);
				#endif
				break;
			case Joint:
				#ifdef LM_SRI
	     		lm = new LanguageModelJoint(new LanguageModelSRI(false), true);
				#elif LM_IRST
		     		lm = new LanguageModelJoint(new LanguageModelIRST(false), true);
				#else
		     		lm = new LanguageModelJoint(new LanguageModelInternal(false), true);
				#endif
				break;
	  }

	  if (lm == NULL)
	  {
	  	UserMessage::Add("Language model type unknown. Probably not compiled into library");
	  }
	  else
	  {
	  	switch (lm->GetLMType())
	  	{
	  	case SingleFactor:
	  		static_cast<LanguageModelSingleFactor*>(lm)->Load(languageModelFile, factorCollection, factorTypes[0], weight, nGramOrder);
	  		break;
	  	case MultiFactor:
	  		 static_cast<LanguageModelMultiFactor*>(lm)->Load(languageModelFile, factorCollection, factorTypes, weight, nGramOrder);
	  			break;
	  	}
	  }

	  return lm;
	}
}
<commit_msg>added internal, x-platform LM which exactly mimics SRILM<commit_after>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include "LanguageModelFactory.h"
#include "UserMessage.h"
#include "TypeDef.h"
#include "FactorCollection.h"

// include appropriate header
#ifdef LM_SRI
#  include "LanguageModelSRI.h"
#endif
#ifdef LM_IRST
#  include "LanguageModelIRST.h"
#endif

#include "LanguageModelInternal.h"
#include "LanguageModelSkip.h"
#include "LanguageModelJoint.h"

namespace LanguageModelFactory
{

	LanguageModel* CreateLanguageModel(LMImplementation lmImplementation, const std::vector<FactorType> &factorTypes
                                   , size_t nGramOrder, const std::string &languageModelFile, float weight, FactorCollection &factorCollection)
	{
	  LanguageModel *lm = NULL;
	  switch (lmImplementation)
	  {
	  	case SRI:
				#ifdef LM_SRI
				  lm = new LanguageModelSRI(true);
				#elif LM_INTERNAL
					lm = new LanguageModelInternal(true);
			  #endif
			  break;
			case IRST:
				#ifdef LM_IRST
	     		lm = new LanguageModelIRST(true);
			  #elif LM_SRI
	     		lm = new LanguageModelSRI(true);
				#elif LM_INTERNAL
					lm = new LanguageModelInternal(true);
			  #endif
				break;
			case Skip:
				#ifdef LM_SRI
	     		lm = new LanguageModelSkip(new LanguageModelSRI(false), true);
				#elif LM_INTERNAL
	     			lm = new LanguageModelSkip(new LanguageModelInternal(false), true);
				#endif
				break;
			case Joint:
				#ifdef LM_SRI
	     		lm = new LanguageModelJoint(new LanguageModelSRI(false), true);
				#elif LM_INTERNAL
		     		lm = new LanguageModelJoint(new LanguageModelInternal(false), true);
				#endif
				break;
	  }

	  if (lm == NULL)
	  {
	  	UserMessage::Add("Language model type unknown. Probably not compiled into library");
	  }
	  else
	  {
	  	switch (lm->GetLMType())
	  	{
	  	case SingleFactor:
	  		static_cast<LanguageModelSingleFactor*>(lm)->Load(languageModelFile, factorCollection, factorTypes[0], weight, nGramOrder);
	  		break;
	  	case MultiFactor:
	  		 static_cast<LanguageModelMultiFactor*>(lm)->Load(languageModelFile, factorCollection, factorTypes, weight, nGramOrder);
	  			break;
	  	}
	  }

	  return lm;
	}
}
<|endoftext|>