<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#pragma once

#include "vast/concept/parseable/vast/json.hpp"
#include "vast/defaults.hpp"
#include "vast/detail/line_range.hpp"
#include "vast/error.hpp"
#include "vast/event.hpp"
#include "vast/format/multi_layout_reader.hpp"
#include "vast/format/ostream_writer.hpp"
#include "vast/fwd.hpp"
#include "vast/json.hpp"
#include "vast/logger.hpp"
#include "vast/schema.hpp"

#include <caf/expected.hpp>
#include <caf/fwd.hpp>

namespace vast::format::json {

class writer : public ostream_writer {
public:
  using defaults = vast::defaults::export_::json;

  using super = ostream_writer;

  using super::super;

  caf::error write(const table_slice& x) override;

  const char* name() const override;
};

/// Adds a JSON object to a table slice builder according to a given layout.
/// @param builder The builder to add the JSON object to.
/// @param xs The JSON object to add to *builder.
/// @param layout The record type describing *xs*.
/// @returns An error iff the operation failed.
caf::error add(table_slice_builder& builder, const vast::json::object& xs,
               const record_type& layout);

/// @relates reader
struct default_selector {
  caf::optional<record_type> operator()(const vast::json::object&) {
    return layout;
  }

  caf::error schema(vast::schema sch) {
    auto entry = *sch.begin();
    if (!caf::holds_alternative<record_type>(entry))
      return make_error(ec::invalid_configuration,
                        "only record_types supported for json schema");
    layout = flatten(caf::get<record_type>(entry));
    return caf::none;
  }

  vast::schema schema() const {
    vast::schema result;
    if (layout)
      result.add(*layout);
    return result;
  }

  static const char* name() {
    return "json-reader";
  }

  caf::optional<record_type> layout = caf::none;
};

/// A reader for JSON data. It operates with a *selector* to determine the
/// mapping of JSON object to the appropriate record type in the schema.
template <class Selector = default_selector>
class reader final : public multi_layout_reader {
public:
  using super = multi_layout_reader;

  /// Constructs a JSON reader.
  /// @param table_slice_type The ID for table slice type to build.
  /// @param options Additional options.
  /// @param in The stream of JSON objects.
  explicit reader(caf::atom_value table_slice_type,
                  const caf::settings& options,
                  std::unique_ptr<std::istream> in = nullptr);

  void reset(std::unique_ptr<std::istream> in);

  caf::error schema(vast::schema sch) override;

  vast::schema schema() const override;

  const char* name() const override;

protected:
  caf::error read_impl(size_t max_events, size_t max_slice_size,
                       consumer& f) override;

private:
  using iterator_type = std::string_view::const_iterator;

  Selector selector_;
  std::unique_ptr<std::istream> input_;
  std::unique_ptr<detail::line_range> lines_;
  caf::optional<size_t> proto_field_;
  std::vector<size_t> port_fields_;
};

// -- implementation ----------------------------------------------------------

template <class Selector>
reader<Selector>::reader(caf::atom_value table_slice_type,
                         const caf::settings& /*options*/,
                         std::unique_ptr<std::istream> in)
  : super(table_slice_type) {
  if (in != nullptr)
    reset(std::move(in));
}

template <class Selector>
void reader<Selector>::reset(std::unique_ptr<std::istream> in) {
  VAST_ASSERT(in != nullptr);
  input_ = std::move(in);
  lines_ = std::make_unique<detail::line_range>(*input_);
}

template <class Selector>
caf::error reader<Selector>::schema(vast::schema s) {
  return selector_.schema(std::move(s));
}

template <class Selector>
vast::schema reader<Selector>::schema() const {
  return selector_.schema();
}

template <class Selector>
const char* reader<Selector>::name() const {
  return Selector::name();
}

template <class Selector>
caf::error reader<Selector>::read_impl(size_t max_events, size_t max_slice_size,
                                       consumer& cons) {
  VAST_ASSERT(max_events > 0);
  VAST_ASSERT(max_slice_size > 0);
  size_t produced = 0;
  for (; produced < max_events; lines_->next()) {
    // EOF check.
    if (lines_->done())
      return finish(cons, make_error(ec::end_of_input, "input exhausted"));
    auto& line = lines_->get();
    vast::json j;
    if (!parsers::json(line, j)) {
      VAST_WARNING(this, "failed to parse", line);
      continue;
    }
    auto xs = caf::get_if<vast::json::object>(&j);
    if (!xs)
      return make_error(ec::type_clash, "not a json object");
    auto layout = selector_(*xs);
    if (!layout)
      continue;
    auto bptr = builder(*layout);
    if (bptr == nullptr)
      return make_error(ec::parse_error, "unable to get a builder");
    if (auto err = add(*bptr, *xs, *layout)) {
      err.context() += caf::make_message("line", lines_->line_number());
      return finish(cons, err);
    }
    produced++;
    if (bptr->rows() == max_slice_size)
      if (auto err = finish(cons, bptr))
        return err;
  }
  return finish(cons);
}

} // namespace vast::format::json
<commit_msg>Print the input data when JSON parsing failed<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#pragma once

#include "vast/concept/parseable/vast/json.hpp"
#include "vast/defaults.hpp"
#include "vast/detail/line_range.hpp"
#include "vast/error.hpp"
#include "vast/event.hpp"
#include "vast/format/multi_layout_reader.hpp"
#include "vast/format/ostream_writer.hpp"
#include "vast/fwd.hpp"
#include "vast/json.hpp"
#include "vast/logger.hpp"
#include "vast/schema.hpp"

#include <caf/expected.hpp>
#include <caf/fwd.hpp>

namespace vast::format::json {

class writer : public ostream_writer {
public:
  using defaults = vast::defaults::export_::json;

  using super = ostream_writer;

  using super::super;

  caf::error write(const table_slice& x) override;

  const char* name() const override;
};

/// Adds a JSON object to a table slice builder according to a given layout.
/// @param builder The builder to add the JSON object to.
/// @param xs The JSON object to add to *builder.
/// @param layout The record type describing *xs*.
/// @returns An error iff the operation failed.
caf::error add(table_slice_builder& builder, const vast::json::object& xs,
               const record_type& layout);

/// @relates reader
struct default_selector {
  caf::optional<record_type> operator()(const vast::json::object&) {
    return layout;
  }

  caf::error schema(vast::schema sch) {
    auto entry = *sch.begin();
    if (!caf::holds_alternative<record_type>(entry))
      return make_error(ec::invalid_configuration,
                        "only record_types supported for json schema");
    layout = flatten(caf::get<record_type>(entry));
    return caf::none;
  }

  vast::schema schema() const {
    vast::schema result;
    if (layout)
      result.add(*layout);
    return result;
  }

  static const char* name() {
    return "json-reader";
  }

  caf::optional<record_type> layout = caf::none;
};

/// A reader for JSON data. It operates with a *selector* to determine the
/// mapping of JSON object to the appropriate record type in the schema.
template <class Selector = default_selector>
class reader final : public multi_layout_reader {
public:
  using super = multi_layout_reader;

  /// Constructs a JSON reader.
  /// @param table_slice_type The ID for table slice type to build.
  /// @param options Additional options.
  /// @param in The stream of JSON objects.
  explicit reader(caf::atom_value table_slice_type,
                  const caf::settings& options,
                  std::unique_ptr<std::istream> in = nullptr);

  void reset(std::unique_ptr<std::istream> in);

  caf::error schema(vast::schema sch) override;

  vast::schema schema() const override;

  const char* name() const override;

protected:
  caf::error read_impl(size_t max_events, size_t max_slice_size,
                       consumer& f) override;

private:
  using iterator_type = std::string_view::const_iterator;

  Selector selector_;
  std::unique_ptr<std::istream> input_;
  std::unique_ptr<detail::line_range> lines_;
  caf::optional<size_t> proto_field_;
  std::vector<size_t> port_fields_;
};

// -- implementation ----------------------------------------------------------

template <class Selector>
reader<Selector>::reader(caf::atom_value table_slice_type,
                         const caf::settings& /*options*/,
                         std::unique_ptr<std::istream> in)
  : super(table_slice_type) {
  if (in != nullptr)
    reset(std::move(in));
}

template <class Selector>
void reader<Selector>::reset(std::unique_ptr<std::istream> in) {
  VAST_ASSERT(in != nullptr);
  input_ = std::move(in);
  lines_ = std::make_unique<detail::line_range>(*input_);
}

template <class Selector>
caf::error reader<Selector>::schema(vast::schema s) {
  return selector_.schema(std::move(s));
}

template <class Selector>
vast::schema reader<Selector>::schema() const {
  return selector_.schema();
}

template <class Selector>
const char* reader<Selector>::name() const {
  return Selector::name();
}

template <class Selector>
caf::error reader<Selector>::read_impl(size_t max_events, size_t max_slice_size,
                                       consumer& cons) {
  VAST_ASSERT(max_events > 0);
  VAST_ASSERT(max_slice_size > 0);
  size_t produced = 0;
  for (; produced < max_events; lines_->next()) {
    // EOF check.
    if (lines_->done())
      return finish(cons, make_error(ec::end_of_input, "input exhausted"));
    auto& line = lines_->get();
    vast::json j;
    if (!parsers::json(line, j)) {
      VAST_WARNING(this, "failed to parse line", lines_->line_number(), ":",
                   line);
      continue;
    }
    auto xs = caf::get_if<vast::json::object>(&j);
    if (!xs)
      return make_error(ec::type_clash, "not a json object");
    auto layout = selector_(*xs);
    if (!layout)
      continue;
    auto bptr = builder(*layout);
    if (bptr == nullptr)
      return make_error(ec::parse_error, "unable to get a builder");
    if (auto err = add(*bptr, *xs, *layout)) {
      err.context() += caf::make_message("line", lines_->line_number());
      return finish(cons, err);
    }
    produced++;
    if (bptr->rows() == max_slice_size)
      if (auto err = finish(cons, bptr))
        return err;
  }
  return finish(cons);
}

} // namespace vast::format::json
<|endoftext|>


<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#pragma once

#include <cstdint>
#include <chrono>
#include <vector>

#include "vast/logger.hpp"

#include <caf/behavior.hpp>
#include <caf/event_based_actor.hpp>
#include <caf/stateful_actor.hpp>

#include "vast/concept/printable/stream.hpp"
#include "vast/concept/printable/vast/uuid.hpp"
#include "vast/event.hpp"
#include "vast/format/writer.hpp"
#include "vast/system/accountant.hpp"
#include "vast/system/atoms.hpp"
#include "vast/system/instrumentation.hpp"
#include "vast/system/query_status.hpp"

namespace vast::system {

// The base class for SINK actors.
template <class Writer>
struct sink_state {
  std::chrono::steady_clock::duration flush_interval = std::chrono::seconds(1);
  std::chrono::steady_clock::time_point last_flush;
  uint64_t processed = 0;
  uint64_t max_events = 0;
  caf::event_based_actor* self;
  accountant_type accountant;
  vast::system::measurement measurement;
  Writer writer;
  const char* name = "writer";

  sink_state(caf::event_based_actor* self_ptr) : self(self_ptr) {
    // nop
  }

  void send_report() {
    if (accountant && measurement.events > 0) {
      auto r = performance_report{{{std::string{name}, measurement}}};
      measurement = {};
      self->send(accountant, std::move(r));
    }
  }
};

template <class Writer>
caf::behavior sink(caf::stateful_actor<sink_state<Writer>>* self,
                   Writer&& writer, uint64_t max_events) {
  static_assert(std::is_base_of_v<format::writer, Writer>);
  using namespace std::chrono;
  auto& st = self->state;
  st.writer = std::move(writer);
  st.name = st.writer.name();
  st.last_flush = steady_clock::now();
  if (max_events > 0) {
    VAST_DEBUG(self, "caps event export at", max_events, "events");
    st.max_events = max_events;
  }
  self->set_exit_handler(
    [=](const caf::exit_msg& msg) {
      self->state.send_report();
      self->quit(msg.reason);
    }
  );
  return {
    [=](std::vector<event>& xs) {
      VAST_DEBUG(self, "got:", xs.size(), "events from",
                 self->current_sender());
      auto& st = self->state;
      auto reached_max_events = [&] {
        VAST_INFO(self, "reached max_events:", st.max_events, "events");
        st.writer.flush();
        st.send_report();
        self->quit();
      };
      // Drop excess elements.
      auto remaining = st.max_events - st.processed;
      if (remaining == 0)
        return reached_max_events();
      if (xs.size() > remaining)
        xs.resize(remaining);
      // Handle events.
      auto t = timer::start(st.measurement);
      if (auto err = st.writer.write(xs)) {
        VAST_ERROR(self, self->system().render(err));
        self->quit(std::move(err));
        return;
      }
      t.stop(xs.size());
      // Stop when reaching configured limit.
      st.processed += xs.size();
      if (st.processed >= st.max_events)
        return reached_max_events();
      // Force flush if necessary.
      auto now = steady_clock::now();
      if (now - st.last_flush > st.flush_interval) {
        st.writer.flush();
        st.last_flush = now;
        st.send_report();
      }
    },
    [=](const uuid& id, const query_status&) {
      VAST_IGNORE_UNUSED(id);
      VAST_DEBUG(self, "got query statistics from", id);
    },
    [=](limit_atom, uint64_t max) {
      VAST_DEBUG(self, "caps event export at", max, "events");
      if (self->state.processed < max)
        self->state.max_events = max;
      else
        VAST_WARNING(self, "ignores new max_events of", max,
                     "(already processed", self->state.processed, " events)");
    },
    [=](accountant_type accountant) {
      VAST_DEBUG(self, "sets accountant to", accountant);
      auto& st = self->state;
      st.accountant = std::move(accountant);
      self->send(st.accountant, announce_atom::value, st.name);
    },
  };
}

} // namespace vast::system
<commit_msg>Fix handling of default max-events setting<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#pragma once

#include <cstdint>
#include <chrono>
#include <vector>

#include "vast/logger.hpp"

#include <caf/behavior.hpp>
#include <caf/event_based_actor.hpp>
#include <caf/stateful_actor.hpp>

#include "vast/concept/printable/stream.hpp"
#include "vast/concept/printable/vast/uuid.hpp"
#include "vast/event.hpp"
#include "vast/format/writer.hpp"
#include "vast/system/accountant.hpp"
#include "vast/system/atoms.hpp"
#include "vast/system/instrumentation.hpp"
#include "vast/system/query_status.hpp"

namespace vast::system {

// The base class for SINK actors.
template <class Writer>
struct sink_state {
  std::chrono::steady_clock::duration flush_interval = std::chrono::seconds(1);
  std::chrono::steady_clock::time_point last_flush;
  uint64_t processed = 0;
  uint64_t max_events = 0;
  caf::event_based_actor* self;
  accountant_type accountant;
  vast::system::measurement measurement;
  Writer writer;
  const char* name = "writer";

  sink_state(caf::event_based_actor* self_ptr) : self(self_ptr) {
    // nop
  }

  void send_report() {
    if (accountant && measurement.events > 0) {
      auto r = performance_report{{{std::string{name}, measurement}}};
      measurement = {};
      self->send(accountant, std::move(r));
    }
  }
};

template <class Writer>
caf::behavior sink(caf::stateful_actor<sink_state<Writer>>* self,
                   Writer&& writer, uint64_t max_events) {
  static_assert(std::is_base_of_v<format::writer, Writer>);
  using namespace std::chrono;
  auto& st = self->state;
  st.writer = std::move(writer);
  st.name = st.writer.name();
  st.last_flush = steady_clock::now();
  if (max_events > 0) {
    VAST_DEBUG(self, "caps event export at", max_events, "events");
    st.max_events = max_events;
  } else {
    // Interpret 0 as infinite.
    st.max_events = std::numeric_limits<uint64_t>::max();
  }
  self->set_exit_handler(
    [=](const caf::exit_msg& msg) {
      self->state.send_report();
      self->quit(msg.reason);
    }
  );
  return {
    [=](std::vector<event>& xs) {
      VAST_DEBUG(self, "got:", xs.size(), "events from",
                 self->current_sender());
      auto& st = self->state;
      auto reached_max_events = [&] {
        VAST_INFO(self, "reached max_events:", st.max_events, "events");
        st.writer.flush();
        st.send_report();
        self->quit();
      };
      // Drop excess elements.
      auto remaining = st.max_events - st.processed;
      if (remaining == 0)
        return reached_max_events();
      if (xs.size() > remaining)
        xs.resize(remaining);
      // Handle events.
      auto t = timer::start(st.measurement);
      if (auto err = st.writer.write(xs)) {
        VAST_ERROR(self, self->system().render(err));
        self->quit(std::move(err));
        return;
      }
      t.stop(xs.size());
      // Stop when reaching configured limit.
      st.processed += xs.size();
      if (st.processed >= st.max_events)
        return reached_max_events();
      // Force flush if necessary.
      auto now = steady_clock::now();
      if (now - st.last_flush > st.flush_interval) {
        st.writer.flush();
        st.last_flush = now;
        st.send_report();
      }
    },
    [=](const uuid& id, const query_status&) {
      VAST_IGNORE_UNUSED(id);
      VAST_DEBUG(self, "got query statistics from", id);
    },
    [=](limit_atom, uint64_t max) {
      VAST_DEBUG(self, "caps event export at", max, "events");
      if (self->state.processed < max)
        self->state.max_events = max;
      else
        VAST_WARNING(self, "ignores new max_events of", max,
                     "(already processed", self->state.processed, " events)");
    },
    [=](accountant_type accountant) {
      VAST_DEBUG(self, "sets accountant to", accountant);
      auto& st = self->state;
      st.accountant = std::move(accountant);
      self->send(st.accountant, announce_atom::value, st.name);
    },
  };
}

} // namespace vast::system
<|endoftext|>


<commit_before>/**
 * @file utils/matrix.hpp
 * @brief Utility methods for OpenCV matrices
 * @author Paolo D'Apice
 */

#ifndef VIS_TEST_UTILS_MATRIX_HPP
#define VIS_TEST_UTILS_MATRIX_HPP

#include <armadillo>
#include <opencv2/core/core.hpp>

namespace test {

/// @return \c true if matrices are equals.
static
bool
equals(const cv::Mat& a, const cv::Mat& b) {
    if (a.size() != b.size()) return false;
    cv::Mat diff = (a != b);
    return cv::countNonZero(diff) == 0 ? true : false;
}

/// @return \c true if matrices are equals.
template <typename T>
bool
equals(const arma::Mat<T>& a, const arma::Mat<T>& b) {
    if (a.n_rows != b.n_rows or a.n_cols != b.n_cols) return false;
    return all(vectorise(a == b));
}

/// @return \c true if matrix minimum and maximum matches the given values.
static
bool
hasMinMax(const cv::Mat& in, double min, double max) {
    double min_, max_;
    cv::minMaxIdx(in, &min_, &max_);
    return ( (max >= max_) and (min <= min_));
}

} /* namespace test */

#endif /* VIS_TEST_UTILS_MATRIX_HPP */

<commit_msg>Fix linker error<commit_after>/**
 * @file utils/matrix.hpp
 * @brief Utility methods for OpenCV matrices
 * @author Paolo D'Apice
 */

#ifndef VIS_TEST_UTILS_MATRIX_HPP
#define VIS_TEST_UTILS_MATRIX_HPP

#include <armadillo>
#include <opencv2/core/core.hpp>

namespace test {

/// @return \c true if matrices are equals.
inline
bool
equals(const cv::Mat& a, const cv::Mat& b) {
    if (a.size() != b.size()) return false;
    cv::Mat diff = (a != b);
    return cv::countNonZero(diff) == 0 ? true : false;
}

/// @return \c true if matrices are equals.
template <typename T>
bool
equals(const arma::Mat<T>& a, const arma::Mat<T>& b) {
    if (a.n_rows != b.n_rows or a.n_cols != b.n_cols) return false;
    return all(vectorise(a == b));
}

/// @return \c true if matrix minimum and maximum matches the given values.
inline
bool
hasMinMax(const cv::Mat& in, double min, double max) {
    double min_, max_;
    cv::minMaxIdx(in, &min_, &max_);
    return ( (max >= max_) and (min <= min_));
}

} /* namespace test */

#endif /* VIS_TEST_UTILS_MATRIX_HPP */

<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: dicimp.hxx,v $
 *
 *  $Revision: 1.7 $
 *
 *  last change: $Author: vg $ $Date: 2007-05-25 12:22:01 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _LINGUISTIC_DICIMP_HXX_
#define _LINGUISTIC_DICIMP_HXX_

#include <com/sun/star/linguistic2/XDictionary1.hpp>
#include <com/sun/star/linguistic2/XDictionary.hpp>
#include <com/sun/star/frame/XStorable.hpp>
#include <com/sun/star/lang/XServiceInfo.hpp>

#include <uno/lbnames.h>            // CPPU_CURRENT_LANGUAGE_BINDING_NAME macro, which specify the environment type
#include <cppuhelper/implbase3.hxx> // helper for implementations
#include <cppuhelper/implbase1.hxx> // helper for implementations

#ifndef _CPPUHELPER_INTERFACECONTAINER_H_
#include <cppuhelper/interfacecontainer.h>
#endif

#ifndef _STRING_HXX
#include <tools/string.hxx>
#endif

#ifndef _STREAM_HXX
#include <tools/stream.hxx>
#endif

#include "misc.hxx"

///////////////////////////////////////////////////////////////////////////

#define DIC_MAX_ENTRIES     2000

INT16           ReadDicVersion( SvStream *pStream, USHORT &nLng, BOOL &bNeg );
const String    GetDicExtension();

///////////////////////////////////////////////////////////////////////////

class DictionaryNeo :
    public ::cppu::WeakImplHelper3
    <
        ::com::sun::star::linguistic2::XDictionary1,
        ::com::sun::star::linguistic2::XDictionary,
        ::com::sun::star::frame::XStorable
    >
{

    ::cppu::OInterfaceContainerHelper                           aDicEvtListeners;
    ::com::sun::star::uno::Sequence<
        ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > > aEntries;
    ::rtl::OUString                                             aDicName;
    ::rtl::OUString                                             aMainURL;
    ::com::sun::star::linguistic2::DictionaryType               eDicType;
    INT16                                                       nCount;
    INT16                                                       nLanguage;
    INT16                                                       nDicVersion;
    BOOL                                                        bNeedEntries;
    BOOL                                                        bIsModified;
    BOOL                                                        bIsActive;
    BOOL                                                        bIsReadonly;

    // disallow copy-constructor and assignment-operator for now
    DictionaryNeo(const DictionaryNeo &);
    DictionaryNeo & operator = (const DictionaryNeo &);

    void                    launchEvent(INT16 nEvent,
        ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > xEntry);

    ULONG                   loadEntries(const ::rtl::OUString &rMainURL);
    ULONG                   saveEntries(const ::rtl::OUString &rMainURL);
    int                     cmpDicEntry(const ::rtl::OUString &rWord1,
                                        const ::rtl::OUString &rWord2,
                                        BOOL bSimilarOnly = FALSE);
    BOOL                    seekEntry(const ::rtl::OUString &rWord, INT32 *pPos,
                                        BOOL bSimilarOnly = FALSE);
    BOOL                    isSorted();

    BOOL                    addEntry_Impl(const ::com::sun::star::uno::Reference<
        ::com::sun::star::linguistic2::XDictionaryEntry > xDicEntry,
                                          BOOL bIsLoadEntries = FALSE);

public:
    DictionaryNeo();
    DictionaryNeo(const ::rtl::OUString &rName, INT16 nLang,
                    ::com::sun::star::linguistic2::DictionaryType eType,
                      const ::rtl::OUString &rMainURL);
    virtual ~DictionaryNeo();

    // XNamed
    virtual ::rtl::OUString SAL_CALL
        getName()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setName( const ::rtl::OUString& aName )
            throw(::com::sun::star::uno::RuntimeException);

    // XDictionary1 (same as XDictionary but for sal_Int16 as language)
    // only the different ones are listed
    virtual sal_Int16 SAL_CALL
        getLanguage()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setLanguage( sal_Int16 nLang )
            throw(::com::sun::star::uno::RuntimeException);

    // XDictionary
    virtual ::com::sun::star::linguistic2::DictionaryType SAL_CALL
        getDictionaryType()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setActive( sal_Bool bActivate )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isActive()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Int16 SAL_CALL
        getCount()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::lang::Locale SAL_CALL
        getLocale()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setLocale( const ::com::sun::star::lang::Locale& aLocale )
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > SAL_CALL
        getEntry( const ::rtl::OUString& aWord )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        addEntry( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEntry >& xDicEntry )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        add( const ::rtl::OUString& aWord, sal_Bool bIsNegative,
                const ::rtl::OUString& aRplcText )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        remove( const ::rtl::OUString& aWord )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isFull()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::uno::Sequence<
            ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEntry > > SAL_CALL
        getEntries()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        clear()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        addDictionaryEventListener( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEventListener >& xListener )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        removeDictionaryEventListener( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEventListener >& xListener )
            throw(::com::sun::star::uno::RuntimeException);

    // XStorable
    virtual sal_Bool SAL_CALL
        hasLocation()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::rtl::OUString SAL_CALL
        getLocation()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isReadonly()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        store()
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        storeAsURL( const ::rtl::OUString& aURL,
                const ::com::sun::star::uno::Sequence<
                    ::com::sun::star::beans::PropertyValue >& aArgs )
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        storeToURL( const ::rtl::OUString& aURL,
                const ::com::sun::star::uno::Sequence<
                    ::com::sun::star::beans::PropertyValue >& aArgs )
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
};

///////////////////////////////////////////////////////////////////////////

class DicEntry :
    public cppu::WeakImplHelper1
    <
        ::com::sun::star::linguistic2::XDictionaryEntry
    >
{
    ::rtl::OUString aDicWord,       // including hyphen positions represented by "="
                    aReplacement;   // including hyphen positions represented by "="
    BOOL            bIsNegativ;

    // disallow copy-constructor and assignment-operator for now
    DicEntry(const DicEntry &);
    DicEntry & operator = (const DicEntry &);

    void            splitDicFileWord(const ::rtl::OUString &rDicFileWord,
                                     ::rtl::OUString &rDicWord,
                                     ::rtl::OUString &rReplacement);

public:
    DicEntry();
    DicEntry(const ::rtl::OUString &rDicFileWord, BOOL bIsNegativ);
    DicEntry(const ::rtl::OUString &rDicWord, BOOL bIsNegativ,
             const ::rtl::OUString &rRplcText);
    virtual ~DicEntry();

    // XDictionaryEntry
    virtual ::rtl::OUString SAL_CALL
        getDictionaryWord() throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isNegative() throw(::com::sun::star::uno::RuntimeException);
    virtual ::rtl::OUString SAL_CALL
        getReplacementText() throw(::com::sun::star::uno::RuntimeException);
};

///////////////////////////////////////////////////////////////////////////

#endif

<commit_msg>INTEGRATION: CWS tl41_DEV300 (1.7.10); FILE MERGED 2008/02/14 15:31:05 tl 1.7.10.1: #i80286# make dictionaries read-only if not in writeable path<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: dicimp.hxx,v $
 *
 *  $Revision: 1.8 $
 *
 *  last change: $Author: obo $ $Date: 2008-03-25 16:25:25 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _LINGUISTIC_DICIMP_HXX_
#define _LINGUISTIC_DICIMP_HXX_

#include <com/sun/star/linguistic2/XDictionary1.hpp>
#include <com/sun/star/linguistic2/XDictionary.hpp>
#include <com/sun/star/frame/XStorable.hpp>
#include <com/sun/star/lang/XServiceInfo.hpp>

#include <uno/lbnames.h>            // CPPU_CURRENT_LANGUAGE_BINDING_NAME macro, which specify the environment type
#include <cppuhelper/implbase3.hxx> // helper for implementations
#include <cppuhelper/implbase1.hxx> // helper for implementations

#ifndef _CPPUHELPER_INTERFACECONTAINER_H_
#include <cppuhelper/interfacecontainer.h>
#endif

#ifndef _STRING_HXX
#include <tools/string.hxx>
#endif

#ifndef _STREAM_HXX
#include <tools/stream.hxx>
#endif

#include "misc.hxx"

///////////////////////////////////////////////////////////////////////////

#define DIC_MAX_ENTRIES     2000

INT16           ReadDicVersion( SvStream *pStream, USHORT &nLng, BOOL &bNeg );
const String    GetDicExtension();

///////////////////////////////////////////////////////////////////////////

class DictionaryNeo :
    public ::cppu::WeakImplHelper3
    <
        ::com::sun::star::linguistic2::XDictionary1,
        ::com::sun::star::linguistic2::XDictionary,
        ::com::sun::star::frame::XStorable
    >
{

    ::cppu::OInterfaceContainerHelper                           aDicEvtListeners;
    ::com::sun::star::uno::Sequence<
        ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > > aEntries;
    ::rtl::OUString                                             aDicName;
    ::rtl::OUString                                             aMainURL;
    ::com::sun::star::linguistic2::DictionaryType               eDicType;
    INT16                                                       nCount;
    INT16                                                       nLanguage;
    INT16                                                       nDicVersion;
    BOOL                                                        bNeedEntries;
    BOOL                                                        bIsModified;
    BOOL                                                        bIsActive;
    BOOL                                                        bIsReadonly;

    // disallow copy-constructor and assignment-operator for now
    DictionaryNeo(const DictionaryNeo &);
    DictionaryNeo & operator = (const DictionaryNeo &);

    void                    launchEvent(INT16 nEvent,
        ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > xEntry);

    ULONG                   loadEntries(const ::rtl::OUString &rMainURL);
    ULONG                   saveEntries(const ::rtl::OUString &rMainURL);
    int                     cmpDicEntry(const ::rtl::OUString &rWord1,
                                        const ::rtl::OUString &rWord2,
                                        BOOL bSimilarOnly = FALSE);
    BOOL                    seekEntry(const ::rtl::OUString &rWord, INT32 *pPos,
                                        BOOL bSimilarOnly = FALSE);
    BOOL                    isSorted();

    BOOL                    addEntry_Impl(const ::com::sun::star::uno::Reference<
        ::com::sun::star::linguistic2::XDictionaryEntry > xDicEntry,
                                          BOOL bIsLoadEntries = FALSE);

public:
    DictionaryNeo();
    DictionaryNeo(const ::rtl::OUString &rName, INT16 nLang,
                    ::com::sun::star::linguistic2::DictionaryType eType,
                    const ::rtl::OUString &rMainURL,
                    BOOL bWriteable );
    virtual ~DictionaryNeo();

    // XNamed
    virtual ::rtl::OUString SAL_CALL
        getName()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setName( const ::rtl::OUString& aName )
            throw(::com::sun::star::uno::RuntimeException);

    // XDictionary1 (same as XDictionary but for sal_Int16 as language)
    // only the different ones are listed
    virtual sal_Int16 SAL_CALL
        getLanguage()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setLanguage( sal_Int16 nLang )
            throw(::com::sun::star::uno::RuntimeException);

    // XDictionary
    virtual ::com::sun::star::linguistic2::DictionaryType SAL_CALL
        getDictionaryType()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setActive( sal_Bool bActivate )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isActive()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Int16 SAL_CALL
        getCount()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::lang::Locale SAL_CALL
        getLocale()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        setLocale( const ::com::sun::star::lang::Locale& aLocale )
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::uno::Reference<
            ::com::sun::star::linguistic2::XDictionaryEntry > SAL_CALL
        getEntry( const ::rtl::OUString& aWord )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        addEntry( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEntry >& xDicEntry )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        add( const ::rtl::OUString& aWord, sal_Bool bIsNegative,
                const ::rtl::OUString& aRplcText )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        remove( const ::rtl::OUString& aWord )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isFull()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::com::sun::star::uno::Sequence<
            ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEntry > > SAL_CALL
        getEntries()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        clear()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        addDictionaryEventListener( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEventListener >& xListener )
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        removeDictionaryEventListener( const ::com::sun::star::uno::Reference<
                ::com::sun::star::linguistic2::XDictionaryEventListener >& xListener )
            throw(::com::sun::star::uno::RuntimeException);

    // XStorable
    virtual sal_Bool SAL_CALL
        hasLocation()
            throw(::com::sun::star::uno::RuntimeException);
    virtual ::rtl::OUString SAL_CALL
        getLocation()
            throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isReadonly()
            throw(::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        store()
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        storeAsURL( const ::rtl::OUString& aURL,
                const ::com::sun::star::uno::Sequence<
                    ::com::sun::star::beans::PropertyValue >& aArgs )
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
    virtual void SAL_CALL
        storeToURL( const ::rtl::OUString& aURL,
                const ::com::sun::star::uno::Sequence<
                    ::com::sun::star::beans::PropertyValue >& aArgs )
            throw(::com::sun::star::io::IOException,
                  ::com::sun::star::uno::RuntimeException);
};

///////////////////////////////////////////////////////////////////////////

class DicEntry :
    public cppu::WeakImplHelper1
    <
        ::com::sun::star::linguistic2::XDictionaryEntry
    >
{
    ::rtl::OUString aDicWord,       // including hyphen positions represented by "="
                    aReplacement;   // including hyphen positions represented by "="
    BOOL            bIsNegativ;

    // disallow copy-constructor and assignment-operator for now
    DicEntry(const DicEntry &);
    DicEntry & operator = (const DicEntry &);

    void            splitDicFileWord(const ::rtl::OUString &rDicFileWord,
                                     ::rtl::OUString &rDicWord,
                                     ::rtl::OUString &rReplacement);

public:
    DicEntry();
    DicEntry(const ::rtl::OUString &rDicFileWord, BOOL bIsNegativ);
    DicEntry(const ::rtl::OUString &rDicWord, BOOL bIsNegativ,
             const ::rtl::OUString &rRplcText);
    virtual ~DicEntry();

    // XDictionaryEntry
    virtual ::rtl::OUString SAL_CALL
        getDictionaryWord() throw(::com::sun::star::uno::RuntimeException);
    virtual sal_Bool SAL_CALL
        isNegative() throw(::com::sun::star::uno::RuntimeException);
    virtual ::rtl::OUString SAL_CALL
        getReplacementText() throw(::com::sun::star::uno::RuntimeException);
};

///////////////////////////////////////////////////////////////////////////

#endif

<|endoftext|>


<commit_before>#include <iostream>

#include "filelib.h"
#include "decoder.h"
#include "ff_register.h"
#include "verbose.h"

using namespace std;

int main(int argc, char** argv) {
  register_feature_functions();
  Decoder decoder(argc, argv);

  const string input = decoder.GetConf()["input"].as<string>();
  const bool show_feature_dictionary = decoder.GetConf().count("show_feature_dictionary");
  if (!SILENT) cerr << "Reading input from " << ((input == "-") ? "STDIN" : input.c_str()) << endl;
  ReadFile in_read(input);
  istream *in = in_read.stream();
  assert(*in);

  string buf;
#ifdef CP_TIME
    clock_t time_cp(0);//, end_cp;
#endif
  while(*in) {
    getline(*in, buf);
    if (buf.empty()) continue;
    decoder.Decode(buf);
  }
#ifdef CP_TIME
    cerr << "Time required for Cube Pruning execution: "
    << CpTime::Get()
    << " seconds." << "\n\n";
#endif
  if (show_feature_dictionary) {
    int num = FD::NumFeats();
    for (int i = 1; i < num; ++i) {
      cout << FD::Convert(i) << endl;
    }
  }
  return 0;
}

<commit_msg>add timing stats for Kenneth<commit_after>#include <iostream>

#include "filelib.h"
#include "decoder.h"
#include "ff_register.h"
#include "verbose.h"
#include "timing_stats.h"

using namespace std;

int main(int argc, char** argv) {
  register_feature_functions();
  Decoder decoder(argc, argv);

  const string input = decoder.GetConf()["input"].as<string>();
  const bool show_feature_dictionary = decoder.GetConf().count("show_feature_dictionary");
  if (!SILENT) cerr << "Reading input from " << ((input == "-") ? "STDIN" : input.c_str()) << endl;
  ReadFile in_read(input);
  istream *in = in_read.stream();
  assert(*in);

  string buf;
#ifdef CP_TIME
    clock_t time_cp(0);//, end_cp;
#endif
  while(*in) {
    getline(*in, buf);
    if (buf.empty()) continue;
    decoder.Decode(buf);
  }
  Timer::Summarize();
#ifdef CP_TIME
    cerr << "Time required for Cube Pruning execution: "
    << CpTime::Get()
    << " seconds." << "\n\n";
#endif
  if (show_feature_dictionary) {
    int num = FD::NumFeats();
    for (int i = 1; i < num; ++i) {
      cout << FD::Convert(i) << endl;
    }
  }
  return 0;
}

<|endoftext|>


<commit_before>#include "hg.h"
#include "lazy.h"
#include "fdict.h"
#include "tdict.h"

#include "lm/enumerate_vocab.hh"
#include "lm/model.hh"
#include "search/config.hh"
#include "search/context.hh"
#include "search/edge.hh"
#include "search/vertex.hh"
#include "search/vertex_generator.hh"
#include "util/exception.hh"

#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>

#include <iostream>
#include <vector>

namespace {

struct MapVocab : public lm::EnumerateVocab {
  public:
    MapVocab() {}

    // Do not call after Lookup.
    void Add(lm::WordIndex index, const StringPiece &str) {
      const WordID cdec_id = TD::Convert(str.as_string());
      if (cdec_id >= out_.size()) out_.resize(cdec_id + 1);
      out_[cdec_id] = index;
    }

    // Assumes Add has been called and will never be called again.
    lm::WordIndex FromCDec(WordID id) const {
      return out_[out_.size() > id ? id : 0];
    }

  private:
    std::vector<lm::WordIndex> out_;
};

class LazyBase {
  public:
    LazyBase(const std::vector<weight_t> &weights) :
      cdec_weights_(weights),
      config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) {
      std::cerr << "Weights KLanguageModel " << config_.GetWeights().LM() << " KLanguageModel_OOV " << config_.GetWeights().OOV() << " WordPenalty " << config_.GetWeights().WordPenalty()  << std::endl;
    }

    virtual ~LazyBase() {}

    virtual void Search(const Hypergraph &hg) const = 0;

    static LazyBase *Load(const char *model_file, const std::vector<weight_t> &weights);

  protected:
    lm::ngram::Config GetConfig() {
      lm::ngram::Config ret;
      ret.enumerate_vocab = &vocab_;
      return ret;
    }

    MapVocab vocab_;

    const std::vector<weight_t> &cdec_weights_;

    const search::Config config_;
};

template <class Model> class Lazy : public LazyBase {
  public:
    Lazy(const char *model_file, const std::vector<weight_t> &weights) : LazyBase(weights), m_(model_file, GetConfig()) {}

    void Search(const Hypergraph &hg) const;

  private:
    void ConvertEdge(const search::Context<Model> &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const;

    const Model m_;
};

LazyBase *LazyBase::Load(const char *model_file, const std::vector<weight_t> &weights) {
  lm::ngram::ModelType model_type;
  if (!lm::ngram::RecognizeBinary(model_file, model_type)) model_type = lm::ngram::PROBING;
  switch (model_type) {
    case lm::ngram::PROBING:
      return new Lazy<lm::ngram::ProbingModel>(model_file, weights);
    case lm::ngram::REST_PROBING:
      return new Lazy<lm::ngram::RestProbingModel>(model_file, weights);
    default:
      UTIL_THROW(util::Exception, "Sorry this lm type isn't supported yet.");
  }
}

template <class Model> void Lazy<Model>::Search(const Hypergraph &hg) const {
  boost::scoped_array<search::Vertex> out_vertices(new search::Vertex[hg.nodes_.size()]);
  boost::scoped_array<search::Edge> out_edges(new search::Edge[hg.edges_.size()]);

  search::Context<Model> context(config_, m_);

  for (unsigned int i = 0; i < hg.nodes_.size(); ++i) {
    search::Vertex &out_vertex = out_vertices[i];
    const Hypergraph::EdgesVector &down_edges = hg.nodes_[i].in_edges_;
    for (unsigned int j = 0; j < down_edges.size(); ++j) {
      unsigned int edge_index = down_edges[j];
      ConvertEdge(context, i == hg.nodes_.size() - 1, out_vertices.get(), hg.edges_[edge_index], out_edges[edge_index]);
      out_vertex.Add(out_edges[edge_index]);
    }
    out_vertex.FinishedAdding();
    search::VertexGenerator(context, out_vertex);
  }
  search::PartialVertex top = out_vertices[hg.nodes_.size() - 1].RootPartial();
  if (top.Empty()) {
    std::cout << "NO PATH FOUND";
  } else {
    search::PartialVertex continuation;
    while (!top.Complete()) {
      top.Split(continuation);
      top = continuation;
    }
    std::cout << top.End().Bound() << std::endl;
  }
}

// TODO: get weights into here somehow.
template <class Model> void Lazy<Model>::ConvertEdge(const search::Context<Model> &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const {
  const std::vector<WordID> &e = in.rule_->e();
  std::vector<lm::WordIndex> words;
  unsigned int terminals = 0;
  for (std::vector<WordID>::const_iterator word = e.begin(); word != e.end(); ++word) {
    if (*word <= 0) {
      out.Add(vertices[in.tail_nodes_[-*word]]);
      words.push_back(lm::kMaxWordIndex);
    } else {
      ++terminals;
      words.push_back(vocab_.FromCDec(*word));
    }
  }

  if (final) {
    words.push_back(m_.GetVocabulary().EndSentence());
  }

  float additive = in.rule_->GetFeatureValues().dot(cdec_weights_);
  UTIL_THROW_IF(isnan(additive), util::Exception, "Bad dot product");
  additive -= terminals * context.GetWeights().WordPenalty() * static_cast<float>(terminals) / M_LN10;

  out.InitRule().Init(context, additive, words, final);
}

boost::scoped_ptr<LazyBase> AwfulGlobalLazy;

} // namespace

void PassToLazy(const char *model_file, const std::vector<weight_t> &weights, const Hypergraph &hg) {
  if (!AwfulGlobalLazy.get()) {
    AwfulGlobalLazy.reset(LazyBase::Load(model_file, weights));
  }
  AwfulGlobalLazy->Search(hg);
}
<commit_msg>Fix word penalty<commit_after>#include "hg.h"
#include "lazy.h"
#include "fdict.h"
#include "tdict.h"

#include "lm/enumerate_vocab.hh"
#include "lm/model.hh"
#include "search/config.hh"
#include "search/context.hh"
#include "search/edge.hh"
#include "search/vertex.hh"
#include "search/vertex_generator.hh"
#include "util/exception.hh"

#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>

#include <iostream>
#include <vector>

namespace {

struct MapVocab : public lm::EnumerateVocab {
  public:
    MapVocab() {}

    // Do not call after Lookup.
    void Add(lm::WordIndex index, const StringPiece &str) {
      const WordID cdec_id = TD::Convert(str.as_string());
      if (cdec_id >= out_.size()) out_.resize(cdec_id + 1);
      out_[cdec_id] = index;
    }

    // Assumes Add has been called and will never be called again.
    lm::WordIndex FromCDec(WordID id) const {
      return out_[out_.size() > id ? id : 0];
    }

  private:
    std::vector<lm::WordIndex> out_;
};

class LazyBase {
  public:
    LazyBase(const std::vector<weight_t> &weights) :
      cdec_weights_(weights),
      config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) {
      std::cerr << "Weights KLanguageModel " << config_.GetWeights().LM() << " KLanguageModel_OOV " << config_.GetWeights().OOV() << " WordPenalty " << config_.GetWeights().WordPenalty()  << std::endl;
    }

    virtual ~LazyBase() {}

    virtual void Search(const Hypergraph &hg) const = 0;

    static LazyBase *Load(const char *model_file, const std::vector<weight_t> &weights);

  protected:
    lm::ngram::Config GetConfig() {
      lm::ngram::Config ret;
      ret.enumerate_vocab = &vocab_;
      return ret;
    }

    MapVocab vocab_;

    const std::vector<weight_t> &cdec_weights_;

    const search::Config config_;
};

template <class Model> class Lazy : public LazyBase {
  public:
    Lazy(const char *model_file, const std::vector<weight_t> &weights) : LazyBase(weights), m_(model_file, GetConfig()) {}

    void Search(const Hypergraph &hg) const;

  private:
    void ConvertEdge(const search::Context<Model> &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const;

    const Model m_;
};

LazyBase *LazyBase::Load(const char *model_file, const std::vector<weight_t> &weights) {
  lm::ngram::ModelType model_type;
  if (!lm::ngram::RecognizeBinary(model_file, model_type)) model_type = lm::ngram::PROBING;
  switch (model_type) {
    case lm::ngram::PROBING:
      return new Lazy<lm::ngram::ProbingModel>(model_file, weights);
    case lm::ngram::REST_PROBING:
      return new Lazy<lm::ngram::RestProbingModel>(model_file, weights);
    default:
      UTIL_THROW(util::Exception, "Sorry this lm type isn't supported yet.");
  }
}

void PrintFinal(const Hypergraph &hg, const search::Edge *edge_base, const search::Final &final) {
  const std::vector<WordID> &words = hg.edges_[&final.From() - edge_base].rule_->e();
  boost::array<const search::Final*, search::kMaxArity>::const_iterator child(final.Children().begin());
  for (std::vector<WordID>::const_iterator i = words.begin(); i != words.end(); ++i) {
    if (*i > 0) {
      std::cout << TD::Convert(*i) << ' ';
    } else {
      PrintFinal(hg, edge_base, **child++);
    }
  }
}

template <class Model> void Lazy<Model>::Search(const Hypergraph &hg) const {
  boost::scoped_array<search::Vertex> out_vertices(new search::Vertex[hg.nodes_.size()]);
  boost::scoped_array<search::Edge> out_edges(new search::Edge[hg.edges_.size()]);
  search::Context<Model> context(config_, m_);

  for (unsigned int i = 0; i < hg.nodes_.size(); ++i) {
    search::Vertex &out_vertex = out_vertices[i];
    const Hypergraph::EdgesVector &down_edges = hg.nodes_[i].in_edges_;
    for (unsigned int j = 0; j < down_edges.size(); ++j) {
      unsigned int edge_index = down_edges[j];
      ConvertEdge(context, i == hg.nodes_.size() - 1, out_vertices.get(), hg.edges_[edge_index], out_edges[edge_index]);
      out_vertex.Add(out_edges[edge_index]);
    }
    out_vertex.FinishedAdding();
    search::VertexGenerator(context, out_vertex);
  }
  search::PartialVertex top = out_vertices[hg.nodes_.size() - 1].RootPartial();
  if (top.Empty()) {
    std::cout << "NO PATH FOUND";
  } else {
    search::PartialVertex continuation;
    while (!top.Complete()) {
      top.Split(continuation);
      top = continuation;
    }
    PrintFinal(hg, out_edges.get(), top.End());
    std::cout << "||| " << top.End().Bound() << std::endl;
  }
}

// TODO: get weights into here somehow.
template <class Model> void Lazy<Model>::ConvertEdge(const search::Context<Model> &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const {
  const std::vector<WordID> &e = in.rule_->e();
  std::vector<lm::WordIndex> words;
  unsigned int terminals = 0;
  for (std::vector<WordID>::const_iterator word = e.begin(); word != e.end(); ++word) {
    if (*word <= 0) {
      out.Add(vertices[in.tail_nodes_[-*word]]);
      words.push_back(lm::kMaxWordIndex);
    } else {
      ++terminals;
      words.push_back(vocab_.FromCDec(*word));
    }
  }

  if (final) {
    words.push_back(m_.GetVocabulary().EndSentence());
  }

  float additive = in.rule_->GetFeatureValues().dot(cdec_weights_);
  UTIL_THROW_IF(isnan(additive), util::Exception, "Bad dot product");
  additive -= static_cast<float>(terminals) * context.GetWeights().WordPenalty() / M_LN10;

  out.InitRule().Init(context, additive, words, final);
}

boost::scoped_ptr<LazyBase> AwfulGlobalLazy;

} // namespace

void PassToLazy(const char *model_file, const std::vector<weight_t> &weights, const Hypergraph &hg) {
  if (!AwfulGlobalLazy.get()) {
    AwfulGlobalLazy.reset(LazyBase::Load(model_file, weights));
  }
  AwfulGlobalLazy->Search(hg);
}
<|endoftext|>


<commit_before>#include "ylikuutio_string.hpp"

// Include standard headers
#include <cstdio>   // std::FILE, std::fclose, std::fopen, std::fread, std::getchar, std::printf etc.
#include <cstring>  // std::memcmp, std::strcmp, std::strlen, std::strncmp
#include <iostream> // std::cout, std::cin, std::cerr
#include <list>     // std::list
#include <stdint.h> // uint32_t etc.
#include <sstream>  // std::istringstream, std::ostringstream, std::stringstream
#include <string>   // std::string
#include <vector>   // std::vector

namespace string
{
    bool check_and_report_if_some_string_matches(
            const char* const base_pointer,
            const char* const data_pointer,
            const uint64_t data_size,
            const std::vector<std::string> identifier_strings_vector)
    {
        for (std::string identifier_string : identifier_strings_vector)
        {
            const char* const identifier_string_char = identifier_string.c_str();

            if (data_pointer + identifier_string.length() > base_pointer + data_size)
            {
                // If current `identifier_string` can't fit in the memory region,
                // proceed to the next `identifier_string`, if there is any left.
                continue;
            }

            if (std::strncmp(data_pointer, identifier_string_char, std::strlen(identifier_string_char)) == 0)
            {
                return true;
            }
        }
        return false;
    }

    void extract_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const dest_base_pointer,
            char* dest_data_pointer,
            const uint64_t dest_data_size,
            const char* const char_end_string)
    {
        while (src_data_pointer < src_base_pointer + src_data_size &&
                dest_data_pointer + 1 < dest_base_pointer + dest_data_size &&
                std::strncmp(src_data_pointer, char_end_string, std::strlen(char_end_string)) != 0)
        {
            strncpy(dest_data_pointer++, src_data_pointer++, 1);
        }
        *dest_data_pointer = '\0';
    }

    void extract_string_with_several_endings(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const dest_base_pointer,
            char* dest_data_pointer,
            const uint64_t dest_data_size,
            const char* const char_end_string)
    {
        // This function copies characters from `src_mem_pointer` until a character matches.

        while (src_data_pointer < src_base_pointer + src_data_size &&
                dest_data_pointer + 1 < dest_base_pointer + dest_data_size)
        {
            uint32_t n_of_ending_characters = std::strlen(char_end_string);
            const char* end_char_pointer;
            end_char_pointer = char_end_string;

            // Check if current character is any of the ending characters.
            while (*end_char_pointer != '\0')
            {
                if (std::strncmp(src_data_pointer, end_char_pointer, 1) == 0)
                {
                    *dest_data_pointer = '\0';
                    return;
                }
                end_char_pointer++;
            }

            // OK, current character is not any of the ending characters.
            // Copy it and advance the pointers accordingly.
            strncpy(dest_data_pointer++, src_data_pointer++, 1);
        }
        *dest_data_pointer = '\0';
        return;
    }

    int32_t extract_int32_t_value_from_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const char_end_string,
            const char* const description)
    {
        char char_number_buffer[1024];
        string::extract_string_with_several_endings(
                src_base_pointer,
                ++src_data_pointer,
                src_data_size,
                char_number_buffer,
                char_number_buffer,
                sizeof(char_number_buffer),
                char_end_string);

        uint32_t value = std::atoi(char_number_buffer);

        if (description != nullptr)
        {
            std::printf("%s: %d\n", description, value);
        }
        return value;
    }

    float extract_float_value_from_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const char_end_string,
            const char* const description)
    {
        char char_number_buffer[1024];
        string::extract_string_with_several_endings(
                src_base_pointer,
                ++src_data_pointer,
                src_data_size,
                char_number_buffer,
                char_number_buffer,
                sizeof(char_number_buffer),
                char_end_string);

        float value = std::atof(char_number_buffer);

        if (description != nullptr)
        {
            std::printf("%s: %f\n", description, value);
        }
        return value;
    }

    int32_t extract_unicode_value_from_string(const char*& unicode_char_pointer)
    {
        if (*unicode_char_pointer == '\0')
        {
            unicode_char_pointer++;
            std::cerr << "Error: Unicode can not begin with \\0!\n";
            return 0xdfff; // invalid unicode!
        }

        if (*unicode_char_pointer != '&')
        {
            // it's just a character, so return its value,
            // and advance to the next character.
            return (int32_t) *unicode_char_pointer++;
        }

        if (*++unicode_char_pointer != '#')
        {
            // not valid format, must begin `"&#x"`.
            unicode_char_pointer++;
            std::cerr << "Error: Unicode string format not supported!\n";
            return 0xdfff; // invalid unicode!
        }

        if (*++unicode_char_pointer != 'x')
        {
            // not valid format, must begin `"&#x"`.
            unicode_char_pointer++;
            std::cerr << "Error: Unicode string format not supported!\n";
            return 0xdfff; // invalid unicode!
        }

        // valid format.
        std::string hex_string;

        // unicode string beginning with '&'
        while (*++unicode_char_pointer != ';')
        {
            if (*unicode_char_pointer == '\0')
            {
                std::cerr << "Error: Null character \\0 reached before end of Unicode string!\n";
                return 0xdfff; // invalid unicode!
            }

            char current_char = *unicode_char_pointer;
            hex_string.append(unicode_char_pointer);
        }

        // Advance to the next character.
        unicode_char_pointer++;

        // convert hexadecimal string to signed integer.
        // http://stackoverflow.com/questions/1070497/c-convert-hex-string-to-signed-integer/1070499#1070499
        uint32_t unicode_value;
        std::stringstream unicode_stringstream;
        unicode_stringstream << std::hex << hex_string;
        unicode_stringstream >> unicode_value;
        return unicode_value;
    }

    std::string convert_std_list_char_to_std_string(const std::list<char>& std_list_char)
    {
        std::string my_string;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            my_string.push_back(*it);
        }

        return my_string;
    }

    std::string convert_std_list_char_to_std_string(
            const std::list<char>& std_list_char,
            const uint32_t first_line_length,
            const uint32_t line_length)
    {
        std::string my_string;
        uint32_t remaining_characters_on_this_line = first_line_length;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            if (remaining_characters_on_this_line == 0)
            {
                my_string.push_back('\\');
                my_string.push_back('n');
                remaining_characters_on_this_line = line_length;
            }
            my_string.push_back(*it);
            remaining_characters_on_this_line--;
        }
        return my_string;
    }

    std::vector<std::string> convert_std_list_char_to_std_vector_std_string(
            const std::list<char>& std_list_char,
            const uint32_t line_length)
    {
        std::vector<std::string> my_vector;
        std::string my_string;
        uint32_t remaining_characters_on_this_line = line_length;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            if (remaining_characters_on_this_line == 0)
            {
                my_vector.push_back(my_string);
                my_string.clear();
                remaining_characters_on_this_line = line_length;
            }
            my_string.push_back(*it);
            remaining_characters_on_this_line--;
        }

        if (my_string.size() > 0)
        {
            my_vector.push_back(my_string);
        }

        return my_vector;
    }

    bool check_if_float_string(const std::string& my_string)
    {
        int32_t maximum_safe_length_for_float_string = 38;
        return check_if_floating_point_string(my_string, maximum_safe_length_for_float_string);
    }

    bool check_if_double_string(const std::string& my_string)
    {
        int32_t maximum_safe_length_for_double_string = 308;
        return check_if_floating_point_string(my_string, maximum_safe_length_for_double_string);
    }

    bool check_if_floating_point_string(const std::string& my_string, int32_t safe_number_of_chars)
    {
        int32_t n_chars = 0;

        if (my_string.empty())
        {
            return false;
        }

        bool is_dot_found = false;

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789.-
            //
            // Besides, there may be only 1 dot. The dot may not be the last character.
            // Besides, there may be only 1 minus sign. The minus sign must be the first character.

            if (my_string.at(i) == '-' && i == 0)
            {
                // Minus sign is OK.
                continue;
            }

            if (my_string.at(i) == '.')
            {
                if (is_dot_found)
                {
                    // Two or more dots.
                    return false;
                }

                if (i == my_string.length() - 1)
                {
                    // Last character is dot.
                    return false;
                }

                // OK, decimal point here.
                is_dot_found = true;
                continue;
            }

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }

            if (!is_dot_found)
            {
                if (++n_chars > safe_number_of_chars)
                {
                    // Too many characters, maximum safe number is 38 characters for float, 308 for double.
                    return false;
                }
            }
        }

        return true;
    }

    bool check_if_signed_integer_string(const std::string& my_string)
    {
        if (my_string.empty())
        {
            return false;
        }

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789
            if (my_string.at(i) == '-' && i == 0)
            {
                // Minus sign is OK.
                continue;
            }

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }
        }

        return true;
    }

    bool check_if_unsigned_integer_string(const std::string& my_string)
    {
        if (my_string.empty())
        {
            return false;
        }

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }
        }

        return true;
    }
}
<commit_msg>`extract_int32_t_value_from_string`: `atoi` -> `strtoul`.<commit_after>#include "ylikuutio_string.hpp"

// Include standard headers
#include <cstdio>   // std::FILE, std::fclose, std::fopen, std::fread, std::getchar, std::printf etc.
#include <cstring>  // std::memcmp, std::strcmp, std::strlen, std::strncmp
#include <iostream> // std::cout, std::cin, std::cerr
#include <list>     // std::list
#include <stdint.h> // uint32_t etc.
#include <sstream>  // std::istringstream, std::ostringstream, std::stringstream
#include <string>   // std::string
#include <vector>   // std::vector

namespace string
{
    bool check_and_report_if_some_string_matches(
            const char* const base_pointer,
            const char* const data_pointer,
            const uint64_t data_size,
            const std::vector<std::string> identifier_strings_vector)
    {
        for (std::string identifier_string : identifier_strings_vector)
        {
            const char* const identifier_string_char = identifier_string.c_str();

            if (data_pointer + identifier_string.length() > base_pointer + data_size)
            {
                // If current `identifier_string` can't fit in the memory region,
                // proceed to the next `identifier_string`, if there is any left.
                continue;
            }

            if (std::strncmp(data_pointer, identifier_string_char, std::strlen(identifier_string_char)) == 0)
            {
                return true;
            }
        }
        return false;
    }

    void extract_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const dest_base_pointer,
            char* dest_data_pointer,
            const uint64_t dest_data_size,
            const char* const char_end_string)
    {
        while (src_data_pointer < src_base_pointer + src_data_size &&
                dest_data_pointer + 1 < dest_base_pointer + dest_data_size &&
                std::strncmp(src_data_pointer, char_end_string, std::strlen(char_end_string)) != 0)
        {
            strncpy(dest_data_pointer++, src_data_pointer++, 1);
        }
        *dest_data_pointer = '\0';
    }

    void extract_string_with_several_endings(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const dest_base_pointer,
            char* dest_data_pointer,
            const uint64_t dest_data_size,
            const char* const char_end_string)
    {
        // This function copies characters from `src_mem_pointer` until a character matches.

        while (src_data_pointer < src_base_pointer + src_data_size &&
                dest_data_pointer + 1 < dest_base_pointer + dest_data_size)
        {
            uint32_t n_of_ending_characters = std::strlen(char_end_string);
            const char* end_char_pointer;
            end_char_pointer = char_end_string;

            // Check if current character is any of the ending characters.
            while (*end_char_pointer != '\0')
            {
                if (std::strncmp(src_data_pointer, end_char_pointer, 1) == 0)
                {
                    *dest_data_pointer = '\0';
                    return;
                }
                end_char_pointer++;
            }

            // OK, current character is not any of the ending characters.
            // Copy it and advance the pointers accordingly.
            strncpy(dest_data_pointer++, src_data_pointer++, 1);
        }
        *dest_data_pointer = '\0';
        return;
    }

    int32_t extract_int32_t_value_from_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const char_end_string,
            const char* const description)
    {
        char char_number_buffer[1024];
        string::extract_string_with_several_endings(
                src_base_pointer,
                ++src_data_pointer,
                src_data_size,
                char_number_buffer,
                char_number_buffer,
                sizeof(char_number_buffer),
                char_end_string);

        uint32_t value = std::strtoul(char_number_buffer, nullptr, 10); // base 10.

        if (description != nullptr)
        {
            std::printf("%s: %d\n", description, value);
        }
        return value;
    }

    float extract_float_value_from_string(
            const char* const src_base_pointer,
            char*& src_data_pointer,
            const uint64_t src_data_size,
            const char* const char_end_string,
            const char* const description)
    {
        char char_number_buffer[1024];
        string::extract_string_with_several_endings(
                src_base_pointer,
                ++src_data_pointer,
                src_data_size,
                char_number_buffer,
                char_number_buffer,
                sizeof(char_number_buffer),
                char_end_string);

        float value = std::atof(char_number_buffer);

        if (description != nullptr)
        {
            std::printf("%s: %f\n", description, value);
        }
        return value;
    }

    int32_t extract_unicode_value_from_string(const char*& unicode_char_pointer)
    {
        if (*unicode_char_pointer == '\0')
        {
            unicode_char_pointer++;
            std::cerr << "Error: Unicode can not begin with \\0!\n";
            return 0xdfff; // invalid unicode!
        }

        if (*unicode_char_pointer != '&')
        {
            // it's just a character, so return its value,
            // and advance to the next character.
            return (int32_t) *unicode_char_pointer++;
        }

        if (*++unicode_char_pointer != '#')
        {
            // not valid format, must begin `"&#x"`.
            unicode_char_pointer++;
            std::cerr << "Error: Unicode string format not supported!\n";
            return 0xdfff; // invalid unicode!
        }

        if (*++unicode_char_pointer != 'x')
        {
            // not valid format, must begin `"&#x"`.
            unicode_char_pointer++;
            std::cerr << "Error: Unicode string format not supported!\n";
            return 0xdfff; // invalid unicode!
        }

        // valid format.
        std::string hex_string;

        // unicode string beginning with '&'
        while (*++unicode_char_pointer != ';')
        {
            if (*unicode_char_pointer == '\0')
            {
                std::cerr << "Error: Null character \\0 reached before end of Unicode string!\n";
                return 0xdfff; // invalid unicode!
            }

            char current_char = *unicode_char_pointer;
            hex_string.append(unicode_char_pointer);
        }

        // Advance to the next character.
        unicode_char_pointer++;

        // convert hexadecimal string to signed integer.
        // http://stackoverflow.com/questions/1070497/c-convert-hex-string-to-signed-integer/1070499#1070499
        uint32_t unicode_value;
        std::stringstream unicode_stringstream;
        unicode_stringstream << std::hex << hex_string;
        unicode_stringstream >> unicode_value;
        return unicode_value;
    }

    std::string convert_std_list_char_to_std_string(const std::list<char>& std_list_char)
    {
        std::string my_string;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            my_string.push_back(*it);
        }

        return my_string;
    }

    std::string convert_std_list_char_to_std_string(
            const std::list<char>& std_list_char,
            const uint32_t first_line_length,
            const uint32_t line_length)
    {
        std::string my_string;
        uint32_t remaining_characters_on_this_line = first_line_length;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            if (remaining_characters_on_this_line == 0)
            {
                my_string.push_back('\\');
                my_string.push_back('n');
                remaining_characters_on_this_line = line_length;
            }
            my_string.push_back(*it);
            remaining_characters_on_this_line--;
        }
        return my_string;
    }

    std::vector<std::string> convert_std_list_char_to_std_vector_std_string(
            const std::list<char>& std_list_char,
            const uint32_t line_length)
    {
        std::vector<std::string> my_vector;
        std::string my_string;
        uint32_t remaining_characters_on_this_line = line_length;

        for (std::list<char>::const_iterator it = std_list_char.begin(); it != std_list_char.end(); it++)
        {
            if (remaining_characters_on_this_line == 0)
            {
                my_vector.push_back(my_string);
                my_string.clear();
                remaining_characters_on_this_line = line_length;
            }
            my_string.push_back(*it);
            remaining_characters_on_this_line--;
        }

        if (my_string.size() > 0)
        {
            my_vector.push_back(my_string);
        }

        return my_vector;
    }

    bool check_if_float_string(const std::string& my_string)
    {
        int32_t maximum_safe_length_for_float_string = 38;
        return check_if_floating_point_string(my_string, maximum_safe_length_for_float_string);
    }

    bool check_if_double_string(const std::string& my_string)
    {
        int32_t maximum_safe_length_for_double_string = 308;
        return check_if_floating_point_string(my_string, maximum_safe_length_for_double_string);
    }

    bool check_if_floating_point_string(const std::string& my_string, int32_t safe_number_of_chars)
    {
        int32_t n_chars = 0;

        if (my_string.empty())
        {
            return false;
        }

        bool is_dot_found = false;

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789.-
            //
            // Besides, there may be only 1 dot. The dot may not be the last character.
            // Besides, there may be only 1 minus sign. The minus sign must be the first character.

            if (my_string.at(i) == '-' && i == 0)
            {
                // Minus sign is OK.
                continue;
            }

            if (my_string.at(i) == '.')
            {
                if (is_dot_found)
                {
                    // Two or more dots.
                    return false;
                }

                if (i == my_string.length() - 1)
                {
                    // Last character is dot.
                    return false;
                }

                // OK, decimal point here.
                is_dot_found = true;
                continue;
            }

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }

            if (!is_dot_found)
            {
                if (++n_chars > safe_number_of_chars)
                {
                    // Too many characters, maximum safe number is 38 characters for float, 308 for double.
                    return false;
                }
            }
        }

        return true;
    }

    bool check_if_signed_integer_string(const std::string& my_string)
    {
        if (my_string.empty())
        {
            return false;
        }

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789
            if (my_string.at(i) == '-' && i == 0)
            {
                // Minus sign is OK.
                continue;
            }

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }
        }

        return true;
    }

    bool check_if_unsigned_integer_string(const std::string& my_string)
    {
        if (my_string.empty())
        {
            return false;
        }

        for (int32_t i = 0; i < my_string.length(); i++)
        {
            // Each of the characters must be one of the following:
            // 0123456789

            if (my_string.at(i) < '0' || my_string.at(i) > '9')
            {
                return false;
            }
        }

        return true;
    }
}
<|endoftext|>