<commit_before>#include "source/server/admin/prometheus_stats.h"

#include "source/common/common/empty_string.h"
#include "source/common/common/macros.h"
#include "source/common/stats/histogram_impl.h"

#include "absl/strings/str_cat.h"

namespace Envoy {
namespace Server {

namespace {

// TODO(mathetake) replace with re2 for speed and safety,
// and change the signature of sanitizeName so it accepts string_view.
const std::regex& promRegex() { CONSTRUCT_ON_FIRST_USE(std::regex, "[^a-zA-Z0-9_]"); }

/**
 * Take a string and sanitize it according to Prometheus conventions.
 */
std::string sanitizeName(const std::string& name) {
  // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by
  // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/.
  // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix.
  return std::regex_replace(name, promRegex(), "_");
}

/*
 * Determine whether a metric has never been emitted and choose to
 * not show it if we only wanted used metrics.
 */
template <class StatType>
static bool shouldShowMetric(const StatType& metric, const bool used_only,
                             const absl::optional<std::regex>& regex) {
  return ((!used_only || metric.used()) &&
          (!regex.has_value() || std::regex_search(metric.name(), regex.value())));
}

/*
 * Comparator for Stats::Metric that does not require a string representation
 * to make the comparison, for memory efficiency.
 */
struct MetricLessThan {
  bool operator()(const Stats::Metric* a, const Stats::Metric* b) const {
    ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
    return a->constSymbolTable().lessThan(a->statName(), b->statName());
  }
};

/**
 * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting
 * them by tag-extracted metric name, and then outputting them in the correct sorted order into
 * response.
 *
 * @param response The buffer to put the output into.
 * @param used_only Whether to only output stats that are used.
 * @param regex A filter on which stats to output.
 * @param metrics The metrics to output stats for. This must contain all stats of the given type
 *        to be included in the same output.
 * @param generate_output A function which returns the output text for this metric.
 * @param type The name of the prometheus metric type for used in TYPE annotations.
 */
template <class StatType>
uint64_t outputStatType(
    Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
    const std::vector<Stats::RefcountPtr<StatType>>& metrics,
    const std::function<std::string(
        const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
    absl::string_view type, const Stats::CustomStatNamespaces& custom_namespaces) {

  /*
   * From
   * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
   *
   * All lines for a given metric must be provided as one single group, with the optional HELP and
   * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
   * expositions is preferred but not required, i.e. do not sort if the computational cost is
   * prohibitive.
   */

  // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
  // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
  // be sorted before producing the final output to satisfy the "preferred" ordering from the
  // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
  // consistent across calls.
  using StatTypeUnsortedCollection = std::vector<const StatType*>;

  // Return early to avoid crashing when getting the symbol table from the first metric.
  if (metrics.empty()) {
    return 0;
  }

  // There should only be one symbol table for all of the stats in the admin
  // interface. If this assumption changes, the name comparisons in this function
  // will have to change to compare to convert all StatNames to strings before
  // comparison.
  const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();

  // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
  // of the exposition format.
  std::map<Stats::StatName, StatTypeUnsortedCollection, Stats::StatNameLessThan> groups(
      global_symbol_table);

  for (const auto& metric : metrics) {
    ASSERT(&global_symbol_table == &metric->constSymbolTable());

    if (!shouldShowMetric(*metric, used_only, regex)) {
      continue;
    }

    groups[metric->tagExtractedStatName()].push_back(metric.get());
  }

  auto result = groups.size();
  for (auto& group : groups) {
    const absl::optional<std::string> prefixed_tag_extracted_name =
        PrometheusStatsFormatter::metricName(global_symbol_table.toString(group.first),
                                             custom_namespaces);
    if (!prefixed_tag_extracted_name.has_value()) {
      --result;
      continue;
    }
    response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name.value(), type));

    // Sort before producing the final output to satisfy the "preferred" ordering from the
    // prometheus spec: metrics will be sorted by their tags' textual representation, which will
    // be consistent across calls.
    std::sort(group.second.begin(), group.second.end(), MetricLessThan());

    for (const auto& metric : group.second) {
      response.add(generate_output(*metric, prefixed_tag_extracted_name.value()));
    }
    response.add("\n");
  }
  return result;
}

/*
 * Return the prometheus output for a numeric Stat (Counter or Gauge).
 */
template <class StatType>
std::string generateNumericOutput(const StatType& metric,
                                  const std::string& prefixed_tag_extracted_name) {
  const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags());
  return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value());
}

/*
 * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
 * newlines) that contains all the individual bucket counts and sum/count for a single histogram
 * (metric_name plus all tags).
 */
std::string generateHistogramOutput(const Stats::ParentHistogram& histogram,
                                    const std::string& prefixed_tag_extracted_name) {
  const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags());
  const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");

  const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
  Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets();
  const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
  std::string output;
  for (size_t i = 0; i < supported_buckets.size(); ++i) {
    double bucket = supported_buckets[i];
    uint64_t value = computed_buckets[i];
    // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
    // doesn't have a specific modifier to format as a fixed-point value only so we use the
    // 'g' operator which prints the number in general fixed point format or scientific format
    // with precision 50 to round the number up to 32 significant digits in fixed point format
    // which should cover pretty much all cases
    output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name,
                              hist_tags, bucket, value));
  }

  output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name,
                            hist_tags, stats.sampleCount()));
  output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags,
                            stats.sampleSum()));
  output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags,
                            stats.sampleCount()));

  return output;
};

} // namespace

std::string PrometheusStatsFormatter::formattedTags(const std::vector<Stats::Tag>& tags) {
  std::vector<std::string> buf;
  buf.reserve(tags.size());
  for (const Stats::Tag& tag : tags) {
    buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_));
  }
  return absl::StrJoin(buf, ",");
}

absl::optional<std::string>
PrometheusStatsFormatter::metricName(const std::string& extracted_name,
                                     const Stats::CustomStatNamespaces& custom_namespaces) {
  const absl::optional<absl::string_view> custom_namespace_stripped =
      custom_namespaces.stripRegisteredPrefix(extracted_name);
  if (custom_namespace_stripped.has_value()) {
    // This case the name has a custom namespace, and it is a custom metric.
    const std::string sanitized_name = sanitizeName(std::string(custom_namespace_stripped.value()));
    // We expose these metrics without modifying (e.g. without "envoy_"),
    // so we have to check the "user-defined" stat name complies with the Prometheus naming
    // convention. Specifically the name must start with the "[a-zA-Z_]" pattern.
    // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern
    // thanks to sanitizeName above, so the only thing we have to do is check
    // if it does not start with digits.
    if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) {
      return absl::nullopt;
    }
    return sanitized_name;
  }

  // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best
  // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on
  // https://prometheus.io/docs/concepts/data_model/
  return absl::StrCat("envoy_", sanitizeName(extracted_name));
}

// TODO(efimki): Add support of text readouts stats.
uint64_t PrometheusStatsFormatter::statsAsPrometheus(
    const std::vector<Stats::CounterSharedPtr>& counters,
    const std::vector<Stats::GaugeSharedPtr>& gauges,
    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
    const bool used_only, const absl::optional<std::regex>& regex,
    const Stats::CustomStatNamespaces& custom_namespaces) {

  uint64_t metric_name_count = 0;
  metric_name_count += outputStatType<Stats::Counter>(response, used_only, regex, counters,
                                                      generateNumericOutput<Stats::Counter>,
                                                      "counter", custom_namespaces);

  metric_name_count +=
      outputStatType<Stats::Gauge>(response, used_only, regex, gauges,
                                   generateNumericOutput<Stats::Gauge>, "gauge", custom_namespaces);

  metric_name_count += outputStatType<Stats::ParentHistogram>(response, used_only, regex,
                                                              histograms, generateHistogramOutput,
                                                              "histogram", custom_namespaces);

  return metric_name_count;
}

} // namespace Server
} // namespace Envoy
<commit_msg>stats: use re2 for prometheus metrics sanitization. (#18110)<commit_after>#include "source/server/admin/prometheus_stats.h"

#include "source/common/common/empty_string.h"
#include "source/common/common/macros.h"
#include "source/common/common/regex.h"
#include "source/common/stats/histogram_impl.h"

#include "absl/strings/str_cat.h"

namespace Envoy {
namespace Server {

namespace {

const Regex::CompiledGoogleReMatcher& promRegex() {
  CONSTRUCT_ON_FIRST_USE(Regex::CompiledGoogleReMatcher, "[^a-zA-Z0-9_]", false);
}

/**
 * Take a string and sanitize it according to Prometheus conventions.
 */
std::string sanitizeName(const absl::string_view name) {
  // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by
  // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/.
  // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix.
  return promRegex().replaceAll(name, "_");
}

/*
 * Determine whether a metric has never been emitted and choose to
 * not show it if we only wanted used metrics.
 */
template <class StatType>
static bool shouldShowMetric(const StatType& metric, const bool used_only,
                             const absl::optional<std::regex>& regex) {
  return ((!used_only || metric.used()) &&
          (!regex.has_value() || std::regex_search(metric.name(), regex.value())));
}

/*
 * Comparator for Stats::Metric that does not require a string representation
 * to make the comparison, for memory efficiency.
 */
struct MetricLessThan {
  bool operator()(const Stats::Metric* a, const Stats::Metric* b) const {
    ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
    return a->constSymbolTable().lessThan(a->statName(), b->statName());
  }
};

/**
 * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting
 * them by tag-extracted metric name, and then outputting them in the correct sorted order into
 * response.
 *
 * @param response The buffer to put the output into.
 * @param used_only Whether to only output stats that are used.
 * @param regex A filter on which stats to output.
 * @param metrics The metrics to output stats for. This must contain all stats of the given type
 *        to be included in the same output.
 * @param generate_output A function which returns the output text for this metric.
 * @param type The name of the prometheus metric type for used in TYPE annotations.
 */
template <class StatType>
uint64_t outputStatType(
    Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
    const std::vector<Stats::RefcountPtr<StatType>>& metrics,
    const std::function<std::string(
        const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
    absl::string_view type, const Stats::CustomStatNamespaces& custom_namespaces) {

  /*
   * From
   * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
   *
   * All lines for a given metric must be provided as one single group, with the optional HELP and
   * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
   * expositions is preferred but not required, i.e. do not sort if the computational cost is
   * prohibitive.
   */

  // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
  // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
  // be sorted before producing the final output to satisfy the "preferred" ordering from the
  // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
  // consistent across calls.
  using StatTypeUnsortedCollection = std::vector<const StatType*>;

  // Return early to avoid crashing when getting the symbol table from the first metric.
  if (metrics.empty()) {
    return 0;
  }

  // There should only be one symbol table for all of the stats in the admin
  // interface. If this assumption changes, the name comparisons in this function
  // will have to change to compare to convert all StatNames to strings before
  // comparison.
  const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();

  // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
  // of the exposition format.
  std::map<Stats::StatName, StatTypeUnsortedCollection, Stats::StatNameLessThan> groups(
      global_symbol_table);

  for (const auto& metric : metrics) {
    ASSERT(&global_symbol_table == &metric->constSymbolTable());

    if (!shouldShowMetric(*metric, used_only, regex)) {
      continue;
    }

    groups[metric->tagExtractedStatName()].push_back(metric.get());
  }

  auto result = groups.size();
  for (auto& group : groups) {
    const absl::optional<std::string> prefixed_tag_extracted_name =
        PrometheusStatsFormatter::metricName(global_symbol_table.toString(group.first),
                                             custom_namespaces);
    if (!prefixed_tag_extracted_name.has_value()) {
      --result;
      continue;
    }
    response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name.value(), type));

    // Sort before producing the final output to satisfy the "preferred" ordering from the
    // prometheus spec: metrics will be sorted by their tags' textual representation, which will
    // be consistent across calls.
    std::sort(group.second.begin(), group.second.end(), MetricLessThan());

    for (const auto& metric : group.second) {
      response.add(generate_output(*metric, prefixed_tag_extracted_name.value()));
    }
    response.add("\n");
  }
  return result;
}

/*
 * Return the prometheus output for a numeric Stat (Counter or Gauge).
 */
template <class StatType>
std::string generateNumericOutput(const StatType& metric,
                                  const std::string& prefixed_tag_extracted_name) {
  const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags());
  return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value());
}

/*
 * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
 * newlines) that contains all the individual bucket counts and sum/count for a single histogram
 * (metric_name plus all tags).
 */
std::string generateHistogramOutput(const Stats::ParentHistogram& histogram,
                                    const std::string& prefixed_tag_extracted_name) {
  const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags());
  const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");

  const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
  Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets();
  const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
  std::string output;
  for (size_t i = 0; i < supported_buckets.size(); ++i) {
    double bucket = supported_buckets[i];
    uint64_t value = computed_buckets[i];
    // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
    // doesn't have a specific modifier to format as a fixed-point value only so we use the
    // 'g' operator which prints the number in general fixed point format or scientific format
    // with precision 50 to round the number up to 32 significant digits in fixed point format
    // which should cover pretty much all cases
    output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name,
                              hist_tags, bucket, value));
  }

  output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name,
                            hist_tags, stats.sampleCount()));
  output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags,
                            stats.sampleSum()));
  output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags,
                            stats.sampleCount()));

  return output;
};

} // namespace

std::string PrometheusStatsFormatter::formattedTags(const std::vector<Stats::Tag>& tags) {
  std::vector<std::string> buf;
  buf.reserve(tags.size());
  for (const Stats::Tag& tag : tags) {
    buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_));
  }
  return absl::StrJoin(buf, ",");
}

absl::optional<std::string>
PrometheusStatsFormatter::metricName(const std::string& extracted_name,
                                     const Stats::CustomStatNamespaces& custom_namespaces) {
  const absl::optional<absl::string_view> custom_namespace_stripped =
      custom_namespaces.stripRegisteredPrefix(extracted_name);
  if (custom_namespace_stripped.has_value()) {
    // This case the name has a custom namespace, and it is a custom metric.
    const std::string sanitized_name = sanitizeName(custom_namespace_stripped.value());
    // We expose these metrics without modifying (e.g. without "envoy_"),
    // so we have to check the "user-defined" stat name complies with the Prometheus naming
    // convention. Specifically the name must start with the "[a-zA-Z_]" pattern.
    // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern
    // thanks to sanitizeName above, so the only thing we have to do is check
    // if it does not start with digits.
    if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) {
      return absl::nullopt;
    }
    return sanitized_name;
  }

  // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best
  // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on
  // https://prometheus.io/docs/concepts/data_model/
  return absl::StrCat("envoy_", sanitizeName(extracted_name));
}

// TODO(efimki): Add support of text readouts stats.
uint64_t PrometheusStatsFormatter::statsAsPrometheus(
    const std::vector<Stats::CounterSharedPtr>& counters,
    const std::vector<Stats::GaugeSharedPtr>& gauges,
    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
    const bool used_only, const absl::optional<std::regex>& regex,
    const Stats::CustomStatNamespaces& custom_namespaces) {

  uint64_t metric_name_count = 0;
  metric_name_count += outputStatType<Stats::Counter>(response, used_only, regex, counters,
                                                      generateNumericOutput<Stats::Counter>,
                                                      "counter", custom_namespaces);

  metric_name_count +=
      outputStatType<Stats::Gauge>(response, used_only, regex, gauges,
                                   generateNumericOutput<Stats::Gauge>, "gauge", custom_namespaces);

  metric_name_count += outputStatType<Stats::ParentHistogram>(response, used_only, regex,
                                                              histograms, generateHistogramOutput,
                                                              "histogram", custom_namespaces);

  return metric_name_count;
}

} // namespace Server
} // namespace Envoy
<|endoftext|>


<commit_before>/// FFT driver
///
/// (c) Koheron

#ifndef __DRIVERS_FFT_HPP__
#define __DRIVERS_FFT_HPP__

#include <context.hpp>

#include <atomic>
#include <thread>
#include <chrono>
#include <mutex>
#include <cmath>
#include <limits>
#include <array>

#include <boards/alpha/drivers/clock-generator.hpp>
#include <boards/alpha/drivers/ltc2157.hpp>

class FFT
{
  public:
    FFT(Context& ctx_)
    : ctx(ctx_)
    , ctl(ctx.mm.get<mem::control>())
    , sts(ctx.mm.get<mem::status>())
    , psd_map(ctx.mm.get<mem::psd>())
    , demod_map(ctx.mm.get<mem::demod>())
    , clk_gen(ctx.get<ClockGenerator>())
    , ltc2157(ctx.get<Ltc2157>())
    {
        set_input_channel(0);
        set_scale_sch(0);
        set_fft_window(1);
        ctl.set_bit<reg::psd_valid, 0>();
        start_psd_acquisition();
    }

    //////////////////////////////////////
    // Power Spectral Density
    //////////////////////////////////////

    void set_input_channel(uint32_t channel) {
        if (channel >= 2) {
            ctx.log<ERROR>("FFT::set_input_channel invalid channel\n");
            return;
        }

        input_channel = channel;
        ctl.write<reg::psd_input_sel>(channel);
    }

    void set_scale_sch(uint32_t scale_sch) {
        // LSB at 1 for forward FFT
        ctl.write<reg::ctl_fft>(1 + (scale_sch << 1));
    }

    void set_fft_window(uint32_t window_index_) {
        if (window_index_ == 0) {
            window.fill(1.0); // Rectangular
        } else if (window_index_ == 1) {
            set_cosine_sum_window({0.5, 0.5, 0, 0, 0}, 1.0); // Hanning
        } else if (window_index_ == 2) {
            set_cosine_sum_window({1.0, 1.93, 1.29, 0.388, 0.028}, 0.2); // Flat top
        } else if (window_index_ == 3) {
            set_cosine_sum_window({0.35875, 0.48829, 0.14128, 0.01168, 0}, 1.0); // Blackman Harris
        } else {
            ctx.log<ERROR>("Invalid FFT window index \n");
            return;
        }

        set_window_buffer();
        window_index = window_index_;
    }

    // Read averaged spectrum data
    const auto& read_psd_raw() {
        std::lock_guard<std::mutex> lock(mutex);
        return psd_buffer_raw;
    }

    // Return the PSD in W/Hz
    const auto& read_psd() {
        std::lock_guard<std::mutex> lock(mutex);
        return psd_buffer;
    }

    uint32_t get_number_averages() const {
        return prm::n_cycles;
    }

    uint32_t get_fft_size() const {
        return prm::fft_size;
    }

    // Return the raw input value of each ADC channel
    // n_avg: number of averages
    const std::array<int32_t, prm::n_adc> get_adc_raw_data(uint32_t n_avg) {
        if (n_avg <= 1) {
            return { sts.read<reg::adc0, int16_t>(),
                     sts.read<reg::adc1, int16_t>() };
        } else {
            int32_t adc0 = 0;
            int32_t adc1 = 0;

            for (size_t i=0; i<n_avg; i++) {
                adc0 += sts.read<reg::adc0, int16_t>();
                adc1 += sts.read<reg::adc1, int16_t>();
            }

            return { int32_t(std::round(adc0 / double(n_avg))),
                     int32_t(std::round(adc1 / double(n_avg))) };
        }
    }

    //////////////////////////////////////
    // Direct Digital Synthesis
    //////////////////////////////////////

    void set_dds_freq(uint32_t channel, double freq_hz) {
        if (channel >= 2) {
            ctx.log<ERROR>("FFT::set_dds_freq invalid channel\n");
            return;
        }

        if (std::isnan(freq_hz)) {
            ctx.log<ERROR>("FFT::set_dds_freq Frequency is NaN\n");
            return;
        }

        if (freq_hz > fs_adc / 2) {
            freq_hz = fs_adc / 2;
        }

        if (freq_hz < 0.0) {
            freq_hz = 0.0;
        }

        double factor = (uint64_t(1) << 32) / fs_adc;
        ctl.write_reg(reg::phase_incr0 + 4 * channel, uint32_t(factor * freq_hz));
        dds_freq[channel] = freq_hz;
    }

    auto get_control_parameters() {
        return std::make_tuple(dds_freq[0], dds_freq[1], fs_adc, input_channel, W1, W2);
    }

    const auto& get_window_index() const {
        return window_index;
    }

 private:
    Context& ctx;
    Memory<mem::control>& ctl;
    Memory<mem::status>& sts;
    Memory<mem::psd>& psd_map;
    Memory<mem::demod>& demod_map;
    ClockGenerator& clk_gen;
    Ltc2157& ltc2157;

    double fs_adc; // ADC sampling rate (Hz)
    std::array<std::array<float, prm::fft_size/2>, 2> freq_calibration; // Conversion to W/Hz

    std::array<double, prm::fft_size> window;
    double W1, W2; // Window correction factors
    uint32_t window_index;

    uint32_t input_channel = 0;
    std::array<double, 2> dds_freq = {{0.0, 0.0}};

    std::array<float, prm::fft_size/2> psd_buffer_raw;
    std::array<float, prm::fft_size/2> psd_buffer;
    std::thread psd_thread;
    std::mutex mutex;
    std::atomic<bool> psd_acquisition_started{false};
    std::atomic<uint32_t> acq_cycle_index{0};
    void psd_acquisition_thread();
    void start_psd_acquisition();

    // https://en.wikipedia.org/wiki/Window_function
    void set_cosine_sum_window(std::array<double, 5> a, double scaling) {
        double sign;
        for (size_t i=0; i<prm::fft_size; i++) {
            window[i] = 0;
            for (size_t j=0; j<a.size(); j++) {
                j % 2 == 0 ? sign = 1.0 : sign = -1.0;
                window[i] += sign * a[j] * std::cos(2 * M_PI * i * j / double(prm::fft_size - 1));
            }
            window[i] *= scaling;
        }
    }

    // Vectors to convert PSD raw data into W/Hz
    void set_conversion_vectors() {
        constexpr double load = 50.0; // Ohm

        fs_adc = clk_gen.get_adc_sampling_freq();

        auto Hinv = koheron::make_array(
            ltc2157.get_inverse_transfer_function<0, prm::fft_size/2>(fs_adc), // Inverse transfer function ADC channel 0
            ltc2157.get_inverse_transfer_function<1, prm::fft_size/2>(fs_adc)  // Inverse transfer function ADC channel 1
        );

        std::array<double, 2> vin = { ltc2157.get_input_voltage_range(0),
                                      ltc2157.get_input_voltage_range(1) };

        float C0 =  (vin[0] / (2 << 20)) * (vin[0] / (2 << 20)) / prm::n_cycles / fs_adc / load / W2;
        float C1 =  (vin[1] / (2 << 20)) * (vin[1] / (2 << 20)) / prm::n_cycles / fs_adc / load / W2;

        for (unsigned int i=0; i<prm::fft_size/2; i++) {
            freq_calibration[0][i] = C0 * Hinv[0][i];
            freq_calibration[1][i] = C1 * Hinv[1][i];
        }
    }

    void set_window_buffer() {
        std::array<uint32_t, prm::fft_size> window_buffer;
        double res1 = 0;
        double res2 = 0;

        for (size_t i=0; i<prm::fft_size; i++) {
            window_buffer[i] = ((int32_t(32768 * window[i]) + 32768) % 65536) + 32768;
            res1 += window[i];
            res2 += window[i] * window[i];
        }

        demod_map.write_array(window_buffer);

        W1 = (res1 / prm::fft_size) * (res1 / prm::fft_size);
        W2 = res2 / prm::fft_size;
        set_conversion_vectors();
    }

    uint32_t get_cycle_index() {
        return sts.read<reg::cycle_index>();
    }

}; // class FFT

inline void FFT::start_psd_acquisition() {
    if (! psd_acquisition_started) {
        psd_buffer.fill(0);
        psd_thread = std::thread{&FFT::psd_acquisition_thread, this};
        psd_thread.detach();
    }
}

inline void  FFT::psd_acquisition_thread() {
    using namespace std::chrono_literals;

    psd_acquisition_started = true;

    while (psd_acquisition_started) {
        uint32_t cycle_index = get_cycle_index();
        uint32_t previous_cycle_index = cycle_index;

        // Wait for
        while (cycle_index >= previous_cycle_index) {
            auto sleep_time = std::chrono::nanoseconds((prm::n_cycles - cycle_index) * 8192 * 4);
            if (sleep_time > 1ms) {
                std::this_thread::sleep_for(sleep_time);
            }
            previous_cycle_index = cycle_index;
            cycle_index = get_cycle_index();
        }

        {
            std::lock_guard<std::mutex> lock(mutex);
            psd_buffer_raw = psd_map.read_array<float, prm::fft_size/2, 0>();

            if (std::abs(clk_gen.get_adc_sampling_freq() - fs_adc) > std::numeric_limits<double>::round_error()) {
                // If the sampling frequency has changed
                // we recompute the calibration vectors ...
                set_conversion_vectors();

                // ... and we reset the DDS frequencies accordingly
                set_dds_freq(0, dds_freq[0]);
                set_dds_freq(1, dds_freq[1]);
            }

            for (unsigned int i=0; i<prm::fft_size/2; i++) {
                psd_buffer[i] = psd_buffer_raw[i] * freq_calibration[input_channel][i];
            }
        }

        acq_cycle_index = get_cycle_index();
    }
}

#endif // __DRIVERS_FFT_HPP__
<commit_msg>fix path in alpha250 fft.hpp<commit_after>/// FFT driver
///
/// (c) Koheron

#ifndef __DRIVERS_FFT_HPP__
#define __DRIVERS_FFT_HPP__

#include <context.hpp>

#include <atomic>
#include <thread>
#include <chrono>
#include <mutex>
#include <cmath>
#include <limits>
#include <array>

#include <boards/alpha250/drivers/clock-generator.hpp>
#include <boards/alpha250/drivers/ltc2157.hpp>

class FFT
{
  public:
    FFT(Context& ctx_)
    : ctx(ctx_)
    , ctl(ctx.mm.get<mem::control>())
    , sts(ctx.mm.get<mem::status>())
    , psd_map(ctx.mm.get<mem::psd>())
    , demod_map(ctx.mm.get<mem::demod>())
    , clk_gen(ctx.get<ClockGenerator>())
    , ltc2157(ctx.get<Ltc2157>())
    {
        set_input_channel(0);
        set_scale_sch(0);
        set_fft_window(1);
        ctl.set_bit<reg::psd_valid, 0>();
        start_psd_acquisition();
    }

    //////////////////////////////////////
    // Power Spectral Density
    //////////////////////////////////////

    void set_input_channel(uint32_t channel) {
        if (channel >= 2) {
            ctx.log<ERROR>("FFT::set_input_channel invalid channel\n");
            return;
        }

        input_channel = channel;
        ctl.write<reg::psd_input_sel>(channel);
    }

    void set_scale_sch(uint32_t scale_sch) {
        // LSB at 1 for forward FFT
        ctl.write<reg::ctl_fft>(1 + (scale_sch << 1));
    }

    void set_fft_window(uint32_t window_index_) {
        if (window_index_ == 0) {
            window.fill(1.0); // Rectangular
        } else if (window_index_ == 1) {
            set_cosine_sum_window({0.5, 0.5, 0, 0, 0}, 1.0); // Hanning
        } else if (window_index_ == 2) {
            set_cosine_sum_window({1.0, 1.93, 1.29, 0.388, 0.028}, 0.2); // Flat top
        } else if (window_index_ == 3) {
            set_cosine_sum_window({0.35875, 0.48829, 0.14128, 0.01168, 0}, 1.0); // Blackman Harris
        } else {
            ctx.log<ERROR>("Invalid FFT window index \n");
            return;
        }

        set_window_buffer();
        window_index = window_index_;
    }

    // Read averaged spectrum data
    const auto& read_psd_raw() {
        std::lock_guard<std::mutex> lock(mutex);
        return psd_buffer_raw;
    }

    // Return the PSD in W/Hz
    const auto& read_psd() {
        std::lock_guard<std::mutex> lock(mutex);
        return psd_buffer;
    }

    uint32_t get_number_averages() const {
        return prm::n_cycles;
    }

    uint32_t get_fft_size() const {
        return prm::fft_size;
    }

    // Return the raw input value of each ADC channel
    // n_avg: number of averages
    const std::array<int32_t, prm::n_adc> get_adc_raw_data(uint32_t n_avg) {
        if (n_avg <= 1) {
            return { sts.read<reg::adc0, int16_t>(),
                     sts.read<reg::adc1, int16_t>() };
        } else {
            int32_t adc0 = 0;
            int32_t adc1 = 0;

            for (size_t i=0; i<n_avg; i++) {
                adc0 += sts.read<reg::adc0, int16_t>();
                adc1 += sts.read<reg::adc1, int16_t>();
            }

            return { int32_t(std::round(adc0 / double(n_avg))),
                     int32_t(std::round(adc1 / double(n_avg))) };
        }
    }

    //////////////////////////////////////
    // Direct Digital Synthesis
    //////////////////////////////////////

    void set_dds_freq(uint32_t channel, double freq_hz) {
        if (channel >= 2) {
            ctx.log<ERROR>("FFT::set_dds_freq invalid channel\n");
            return;
        }

        if (std::isnan(freq_hz)) {
            ctx.log<ERROR>("FFT::set_dds_freq Frequency is NaN\n");
            return;
        }

        if (freq_hz > fs_adc / 2) {
            freq_hz = fs_adc / 2;
        }

        if (freq_hz < 0.0) {
            freq_hz = 0.0;
        }

        double factor = (uint64_t(1) << 32) / fs_adc;
        ctl.write_reg(reg::phase_incr0 + 4 * channel, uint32_t(factor * freq_hz));
        dds_freq[channel] = freq_hz;
    }

    auto get_control_parameters() {
        return std::make_tuple(dds_freq[0], dds_freq[1], fs_adc, input_channel, W1, W2);
    }

    const auto& get_window_index() const {
        return window_index;
    }

 private:
    Context& ctx;
    Memory<mem::control>& ctl;
    Memory<mem::status>& sts;
    Memory<mem::psd>& psd_map;
    Memory<mem::demod>& demod_map;
    ClockGenerator& clk_gen;
    Ltc2157& ltc2157;

    double fs_adc; // ADC sampling rate (Hz)
    std::array<std::array<float, prm::fft_size/2>, 2> freq_calibration; // Conversion to W/Hz

    std::array<double, prm::fft_size> window;
    double W1, W2; // Window correction factors
    uint32_t window_index;

    uint32_t input_channel = 0;
    std::array<double, 2> dds_freq = {{0.0, 0.0}};

    std::array<float, prm::fft_size/2> psd_buffer_raw;
    std::array<float, prm::fft_size/2> psd_buffer;
    std::thread psd_thread;
    std::mutex mutex;
    std::atomic<bool> psd_acquisition_started{false};
    std::atomic<uint32_t> acq_cycle_index{0};
    void psd_acquisition_thread();
    void start_psd_acquisition();

    // https://en.wikipedia.org/wiki/Window_function
    void set_cosine_sum_window(std::array<double, 5> a, double scaling) {
        double sign;
        for (size_t i=0; i<prm::fft_size; i++) {
            window[i] = 0;
            for (size_t j=0; j<a.size(); j++) {
                j % 2 == 0 ? sign = 1.0 : sign = -1.0;
                window[i] += sign * a[j] * std::cos(2 * M_PI * i * j / double(prm::fft_size - 1));
            }
            window[i] *= scaling;
        }
    }

    // Vectors to convert PSD raw data into W/Hz
    void set_conversion_vectors() {
        constexpr double load = 50.0; // Ohm

        fs_adc = clk_gen.get_adc_sampling_freq();

        auto Hinv = koheron::make_array(
            ltc2157.get_inverse_transfer_function<0, prm::fft_size/2>(fs_adc), // Inverse transfer function ADC channel 0
            ltc2157.get_inverse_transfer_function<1, prm::fft_size/2>(fs_adc)  // Inverse transfer function ADC channel 1
        );

        std::array<double, 2> vin = { ltc2157.get_input_voltage_range(0),
                                      ltc2157.get_input_voltage_range(1) };

        float C0 =  (vin[0] / (2 << 20)) * (vin[0] / (2 << 20)) / prm::n_cycles / fs_adc / load / W2;
        float C1 =  (vin[1] / (2 << 20)) * (vin[1] / (2 << 20)) / prm::n_cycles / fs_adc / load / W2;

        for (unsigned int i=0; i<prm::fft_size/2; i++) {
            freq_calibration[0][i] = C0 * Hinv[0][i];
            freq_calibration[1][i] = C1 * Hinv[1][i];
        }
    }

    void set_window_buffer() {
        std::array<uint32_t, prm::fft_size> window_buffer;
        double res1 = 0;
        double res2 = 0;

        for (size_t i=0; i<prm::fft_size; i++) {
            window_buffer[i] = ((int32_t(32768 * window[i]) + 32768) % 65536) + 32768;
            res1 += window[i];
            res2 += window[i] * window[i];
        }

        demod_map.write_array(window_buffer);

        W1 = (res1 / prm::fft_size) * (res1 / prm::fft_size);
        W2 = res2 / prm::fft_size;
        set_conversion_vectors();
    }

    uint32_t get_cycle_index() {
        return sts.read<reg::cycle_index>();
    }

}; // class FFT

inline void FFT::start_psd_acquisition() {
    if (! psd_acquisition_started) {
        psd_buffer.fill(0);
        psd_thread = std::thread{&FFT::psd_acquisition_thread, this};
        psd_thread.detach();
    }
}

inline void  FFT::psd_acquisition_thread() {
    using namespace std::chrono_literals;

    psd_acquisition_started = true;

    while (psd_acquisition_started) {
        uint32_t cycle_index = get_cycle_index();
        uint32_t previous_cycle_index = cycle_index;

        // Wait for
        while (cycle_index >= previous_cycle_index) {
            auto sleep_time = std::chrono::nanoseconds((prm::n_cycles - cycle_index) * 8192 * 4);
            if (sleep_time > 1ms) {
                std::this_thread::sleep_for(sleep_time);
            }
            previous_cycle_index = cycle_index;
            cycle_index = get_cycle_index();
        }

        {
            std::lock_guard<std::mutex> lock(mutex);
            psd_buffer_raw = psd_map.read_array<float, prm::fft_size/2, 0>();

            if (std::abs(clk_gen.get_adc_sampling_freq() - fs_adc) > std::numeric_limits<double>::round_error()) {
                // If the sampling frequency has changed
                // we recompute the calibration vectors ...
                set_conversion_vectors();

                // ... and we reset the DDS frequencies accordingly
                set_dds_freq(0, dds_freq[0]);
                set_dds_freq(1, dds_freq[1]);
            }

            for (unsigned int i=0; i<prm::fft_size/2; i++) {
                psd_buffer[i] = psd_buffer_raw[i] * freq_calibration[input_channel][i];
            }
        }

        acq_cycle_index = get_cycle_index();
    }
}

#endif // __DRIVERS_FFT_HPP__
<|endoftext|>


<commit_before>#include <stdio.h>
#include <signal.h>
#include <stdlib.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <time.h>
#include <wiringPi.h>
#include "camera.hpp"
#include "../motor.h"

#pragma comment(lib,"opencv_world320.lib")

static const int ROTATE_POWER = 100;
static const int ROTATE_MILLISECONDS = 200;
static const int LEFT_MAX = -100;
static const int RIGHT_MAX = 100;
static const int CENTER_THRESHOLD = 30;//-30~30で直進するようにする
static const double EXIST_THRESHOLD = 0.1;//ゴール存在判定 パーセンテージ
static const int MINIMUM_TIMUOUT = 300;//seconds
static const int MAXIMUM_TIMEOUT = 500;//seconds
static const int DELAY_TIME = 1000;

void handler(int signum);

Camera camera;

int main (void)
{
	signal(SIGINT, handler);
	time_t startTime;
	time(&startTime);
	time_t lastTime;
	time(&lastTime);
	pwm_initialize();
	int forward_count = 0;//連続してforwardした回数
	while(lastTime-startTime<MAXIMUM_TIMEOUT)
	{
		printf("lastTime - startTime %d\n",lastTime - startTime);
		time(&lastTime);
		camera.takePhoto();
	  camera.binarize();
	  double count = camera.countArea();
		if(count < EXIST_THRESHOLD)
		{
			//回転するだけ
			forward_count=0;
			motor_right(ROTATE_POWER);
			delay(ROTATE_MILLISECONDS);
			motor_stop();
			delay(DELAY_TIME);
		}
		else
		{
			//見つけれたら前進
			forward_count++;
			motor_forward(100);
			delay(400);
			motor_stop();
			delay(DELAY_TIME);
		}
		if(lastTime-startTime>MINIMUM_TIMUOUT && forward_count>=3)
		{
			printf("NORMAL TIMEOUT\n");
			return 0;
		}
	}
	printf("MAXIMUM_TIMEOUT\n");
	return 0;
}

void handler(int signum)
{
	motor_stop();
	camera.~Camera();
	delay(100);
	exit(1);
}
<commit_msg>ddbueg<commit_after>#include <stdio.h>
#include <signal.h>
#include <stdlib.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <time.h>
#include <wiringPi.h>
#include "camera.hpp"
#include "../motor.h"

#pragma comment(lib,"opencv_world320.lib")

static const int ROTATE_POWER = 100;
static const int ROTATE_MILLISECONDS = 200;
static const int LEFT_MAX = -100;
static const int RIGHT_MAX = 100;
static const int CENTER_THRESHOLD = 30;//-30~30で直進するようにする
static const double EXIST_THRESHOLD = 0.1;//ゴール存在判定 パーセンテージ
static const int MINIMUM_TIMUOUT = 200;//seconds
static const int MAXIMUM_TIMEOUT = 500;//seconds
static const int DELAY_MILLIS = 1000;
static const int CONTINUOUS_FORWARD = 3;

void handler(int signum);

Camera camera;

int main (void)
{
	signal(SIGINT, handler);
	time_t startTime;
	time(&startTime);
	time_t lastTime;
	time(&lastTime);
	pwm_initialize();
	int forward_count = 0;//連続してforwardした回数
	while(lastTime-startTime<MAXIMUM_TIMEOUT)
	{
		printf("lastTime - startTime %d\n",lastTime - startTime);
		time(&lastTime);
		camera.takePhoto();
	  camera.binarize();
	  double count = camera.countArea();
		if(count < EXIST_THRESHOLD)
		{
			//回転するだけ
			forward_count=0;
			motor_right(ROTATE_POWER);
			delay(ROTATE_MILLISECONDS);
			motor_stop();
			delay(DELAY_MILLIS);
		}
		else
		{
			//見つけれたら前進
			forward_count++;
			motor_forward(100);
			delay(400);
			motor_stop();
			delay(DELAY_MILLIS);
		}
		if(lastTime-startTime>MINIMUM_TIMUOUT && forward_count>=CONTINUOUS_FORWARD)
		{
			printf("NORMAL TIMEOUT\n");
			return 0;
		}
	}
	printf("MAXIMUM_TIMEOUT\n");
	return 0;
}

void handler(int signum)
{
	motor_stop();
	camera.~Camera();
	delay(100);
	exit(1);
}
<|endoftext|>


<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>

#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/bind.hpp>
#include <boost/mem_fn.hpp>
#include <boost/foreach.hpp>

#include "util/metrics.h"
#include "runtime/coordinator.h"
#include "runtime/exec-env.h"

#include "statestore/simple-scheduler.h"
#include "statestore/state-store-subscriber.h"
#include "gen-cpp/Types_types.h"

#include "util/network-util.h"

using namespace std;
using namespace boost;

namespace impala {

static const string LOCAL_ASSIGNMENTS_KEY("simple-scheduler.local-assignments.total");
static const string ASSIGNMENTS_KEY("simple-scheduler.assignments.total");
static const string SCHEDULER_INIT_KEY("simple-scheduler.initialized");

const string SimpleScheduler::IMPALA_MEMBERSHIP_TOPIC("impala-membership");

SimpleScheduler::SimpleScheduler(StateStoreSubscriber* subscriber,
    const string& backend_id, const TNetworkAddress& backend_address,
    Metrics* metrics)
  : metrics_(metrics),
    statestore_subscriber_(subscriber),
    backend_id_(backend_id),
    thrift_serializer_(false),
    total_assignments_(NULL),
    total_local_assignments_(NULL),
    initialised_(NULL),
    update_count_(0) {
  backend_descriptor_.address = backend_address;
  next_nonlocal_host_entry_ = host_map_.begin();
}

SimpleScheduler::SimpleScheduler(const vector<TNetworkAddress>& backends,
                                 Metrics* metrics)
  : metrics_(metrics),
    statestore_subscriber_(NULL),
    thrift_serializer_(false),
    total_assignments_(NULL),
    total_local_assignments_(NULL),
    initialised_(NULL),
    update_count_(0) {
  DCHECK(backends.size() > 0);
  for (int i = 0; i < backends.size(); ++i) {
    vector<string> ipaddrs;
    Status status = HostnameToIpAddrs(backends[i].hostname, &ipaddrs);
    if (!status.ok()) {
      VLOG(1) << "Failed to resolve " << backends[i].hostname << ": "
              << status.GetErrorMsg();
      continue;
    }

    // Try to find a non-localhost address, otherwise just use the
    // first IP address returned.
    string ipaddr = ipaddrs[0];
    if (!FindFirstNonLocalhost(ipaddrs, &ipaddr)) {
      VLOG(1) << "Only localhost addresses found for " << backends[i].hostname;
    }

    HostMap::iterator it = host_map_.find(ipaddr);
    if (it == host_map_.end()) {
      it = host_map_.insert(
          make_pair(ipaddr, list<TNetworkAddress>())).first;
      host_ip_map_[backends[i].hostname] = ipaddr;
    }

    TNetworkAddress backend_address = MakeNetworkAddress(ipaddr, backends[i].port);
    it->second.push_back(backend_address);
  }
  next_nonlocal_host_entry_ = host_map_.begin();
}

Status SimpleScheduler::Init() {
  LOG(INFO) << "Starting simple scheduler";
  if (statestore_subscriber_ != NULL) {
    StateStoreSubscriber::UpdateCallback cb =
        bind<void>(mem_fn(&SimpleScheduler::UpdateMembership), this, _1, _2);
    RETURN_IF_ERROR(
        statestore_subscriber_->AddTopic(IMPALA_MEMBERSHIP_TOPIC, true, cb));
  }
  if (metrics_ != NULL) {
    total_assignments_ =
        metrics_->CreateAndRegisterPrimitiveMetric(ASSIGNMENTS_KEY, 0L);
    total_local_assignments_ =
        metrics_->CreateAndRegisterPrimitiveMetric(LOCAL_ASSIGNMENTS_KEY, 0L);
    initialised_ =
        metrics_->CreateAndRegisterPrimitiveMetric(SCHEDULER_INIT_KEY, true);
  }

  // Figure out what our IP address is, so that each subscriber
  // doesn't have to resolve it on every heartbeat.
  vector<string> ipaddrs;
  const string& hostname = backend_descriptor_.address.hostname;
  Status status = HostnameToIpAddrs(hostname, &ipaddrs);
  if (!status.ok()) {
    VLOG(1) << "Failed to resolve " << hostname << ": " << status.GetErrorMsg();
    return status;
  }
  // Find a non-localhost address for this host; if one can't be
  // found use the first address returned by HostnameToIpAddrs
  string ipaddr = ipaddrs[0];
  if (!FindFirstNonLocalhost(ipaddrs, &ipaddr)) {
    VLOG(3) << "Only localhost addresses found for " << hostname;
  }

  backend_descriptor_.ip_address = ipaddr;
  LOG(INFO) << "Simple-scheduler using " << ipaddr << " as IP address";

  return Status::OK;
}

void SimpleScheduler::UpdateMembership(
    const StateStoreSubscriber::TopicDeltaMap& service_state,
    vector<TTopicUpdate>* topic_updates) {
  ++update_count_;
  // TODO: Work on a copy if possible, or at least do resolution as a separate step
  // First look to see if the topic(s) we're interested in have an update
  StateStoreSubscriber::TopicDeltaMap::const_iterator topic =
      service_state.find(IMPALA_MEMBERSHIP_TOPIC);

  // Copy to work on without holding the map lock
  HostMap host_map_copy;
  HostIpAddressMap host_ip_map_copy;
  bool found_self = false;

  if (topic != service_state.end()) {
    const TTopicDelta& delta = topic->second;
    if (delta.is_delta) {
      // TODO: Handle deltas when the state-store starts sending them
      LOG(WARNING) << "Unexpected delta update from state-store, ignoring as scheduler"
                      " cannot handle deltas";
      return;
    }

    BOOST_FOREACH(const TTopicItem& item, delta.topic_entries) {
      TBackendDescriptor backend_descriptor;
      // Benchmarks have suggested that this method can deserialize
      // ~10m messages per second, so no immediate need to consider optimisation.
      uint32_t len = item.value.size();
      Status status = DeserializeThriftMsg(reinterpret_cast<const uint8_t*>(
          item.value.data()), &len, false, &backend_descriptor);
      if (!status.ok()) {
        VLOG(2) << "Error deserializing topic item with key: " << item.key;
        continue;
      }

      if (item.key == backend_id_) {
        if (backend_descriptor.address == backend_descriptor_.address) {
          found_self = true;
        } else {
          // Someone else has registered this subscriber ID with a
          // different address. We will try to re-register
          // (i.e. overwrite their subscription), but there is likely
          // a configuration problem.
          LOG_EVERY_N(WARNING, 30) << "Duplicate subscriber registration from address: "
                                   << backend_descriptor.address;
        }
      }

      host_map_copy[backend_descriptor.ip_address].push_back(
          backend_descriptor.address);
      host_ip_map_copy[backend_descriptor.address.hostname] =
          backend_descriptor.ip_address;
    }
  }

  // If this impalad is not in our view of the membership list, we
  // should add it and tell the state-store.
  if (!found_self) {
    VLOG(2) << "Registering local backend with state-store";
    topic_updates->push_back(TTopicUpdate());
    TTopicUpdate& update = topic_updates->back();
    update.topic_name = IMPALA_MEMBERSHIP_TOPIC;
    update.topic_updates.push_back(TTopicItem());

    TTopicItem& item = update.topic_updates.back();
    item.key = backend_id_;
    Status status = thrift_serializer_.Serialize(&backend_descriptor_, &item.value);
    if (!status.ok()) {
      LOG(INFO) << "Failed to serialize Impala backend address for state-store topic: "
                << status.GetErrorMsg();
      topic_updates->pop_back();
    }
  }

  {
    lock_guard<mutex> lock(host_map_lock_);
    host_map_.swap(host_map_copy);
    host_ip_map_.swap(host_ip_map_copy);
    next_nonlocal_host_entry_ = host_map_.begin();
  }
}

Status SimpleScheduler::GetHosts(
    const vector<TNetworkAddress>& data_locations, HostList* hostports) {
  hostports->clear();
  for (int i = 0; i < data_locations.size(); ++i) {
    TNetworkAddress backend;
    GetHost(data_locations[i], &backend);
    hostports->push_back(backend);
  }
  DCHECK_EQ(data_locations.size(), hostports->size());
  return Status::OK;
}

Status SimpleScheduler::GetHost(const TNetworkAddress& data_location,
    TNetworkAddress* hostport) {
  lock_guard<mutex> lock(host_map_lock_);
  if (host_map_.size() == 0) {
    return Status("No backends configured");
  }
  bool local_assignment = false;
  HostMap::iterator entry = host_map_.find(data_location.hostname);

  if (entry == host_map_.end()) {
    // host_map_ map ip address to backend but data_location.hostname might be a hostname.
    // Find the ip address of the data_location from host_ip_map_.
    HostIpAddressMap::const_iterator itr = host_ip_map_.find(data_location.hostname);
    if (itr != host_ip_map_.end()) {
      entry = host_map_.find(itr->second);
    }
  }

  if (entry == host_map_.end()) {
    // round robin the ipaddress
    entry = next_nonlocal_host_entry_;
    ++next_nonlocal_host_entry_;
    if (next_nonlocal_host_entry_ == host_map_.end()) {
      next_nonlocal_host_entry_ = host_map_.begin();
    }
  } else {
    local_assignment = true;
  }
  DCHECK(!entry->second.empty());
  // Round-robin between impalads on the same ipaddress.
  // Pick the first one, then move it to the back of the queue
  *hostport = entry->second.front();
  entry->second.pop_front();
  entry->second.push_back(*hostport);

  if (metrics_ != NULL) {
    total_assignments_->Increment(1);
    if (local_assignment) {
      total_local_assignments_->Increment(1L);
    }
  }

  if (VLOG_FILE_IS_ON) {
    stringstream s;
    s << "(" << data_location.hostname << ":" << data_location.port;
    s << " -> " << (hostport->hostname) << ":" << (hostport->port) << ")";
    VLOG_FILE << "SimpleScheduler assignment (data->backend):  " << s.str();
  }
  return Status::OK;
}

void SimpleScheduler::GetAllKnownHosts(HostList* hostports) {
  lock_guard<mutex> lock(host_map_lock_);
  hostports->clear();
  BOOST_FOREACH(const HostMap::value_type& hosts, host_map_) {
    hostports->insert(hostports->end(), hosts.second.begin(), hosts.second.end());
  }
}

}
<commit_msg>Fix expr-test break<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>

#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/bind.hpp>
#include <boost/mem_fn.hpp>
#include <boost/foreach.hpp>

#include "util/metrics.h"
#include "runtime/coordinator.h"
#include "runtime/exec-env.h"

#include "statestore/simple-scheduler.h"
#include "statestore/state-store-subscriber.h"
#include "gen-cpp/Types_types.h"

#include "util/network-util.h"

using namespace std;
using namespace boost;

namespace impala {

static const string LOCAL_ASSIGNMENTS_KEY("simple-scheduler.local-assignments.total");
static const string ASSIGNMENTS_KEY("simple-scheduler.assignments.total");
static const string SCHEDULER_INIT_KEY("simple-scheduler.initialized");

const string SimpleScheduler::IMPALA_MEMBERSHIP_TOPIC("impala-membership");

SimpleScheduler::SimpleScheduler(StateStoreSubscriber* subscriber,
    const string& backend_id, const TNetworkAddress& backend_address,
    Metrics* metrics)
  : metrics_(metrics),
    statestore_subscriber_(subscriber),
    backend_id_(backend_id),
    thrift_serializer_(false),
    total_assignments_(NULL),
    total_local_assignments_(NULL),
    initialised_(NULL),
    update_count_(0) {
  backend_descriptor_.address = backend_address;
  next_nonlocal_host_entry_ = host_map_.begin();
}

SimpleScheduler::SimpleScheduler(const vector<TNetworkAddress>& backends,
                                 Metrics* metrics)
  : metrics_(metrics),
    statestore_subscriber_(NULL),
    thrift_serializer_(false),
    total_assignments_(NULL),
    total_local_assignments_(NULL),
    initialised_(NULL),
    update_count_(0) {
  DCHECK(backends.size() > 0);
  for (int i = 0; i < backends.size(); ++i) {
    vector<string> ipaddrs;
    Status status = HostnameToIpAddrs(backends[i].hostname, &ipaddrs);
    if (!status.ok()) {
      VLOG(1) << "Failed to resolve " << backends[i].hostname << ": "
              << status.GetErrorMsg();
      continue;
    }

    // Try to find a non-localhost address, otherwise just use the
    // first IP address returned.
    string ipaddr = ipaddrs[0];
    if (!FindFirstNonLocalhost(ipaddrs, &ipaddr)) {
      VLOG(1) << "Only localhost addresses found for " << backends[i].hostname;
    }

    HostMap::iterator it = host_map_.find(ipaddr);
    if (it == host_map_.end()) {
      it = host_map_.insert(
          make_pair(ipaddr, list<TNetworkAddress>())).first;
      host_ip_map_[backends[i].hostname] = ipaddr;
    }

    TNetworkAddress backend_address = MakeNetworkAddress(ipaddr, backends[i].port);
    it->second.push_back(backend_address);
  }
  next_nonlocal_host_entry_ = host_map_.begin();
}

Status SimpleScheduler::Init() {
  LOG(INFO) << "Starting simple scheduler";
  if (statestore_subscriber_ != NULL) {
    StateStoreSubscriber::UpdateCallback cb =
        bind<void>(mem_fn(&SimpleScheduler::UpdateMembership), this, _1, _2);
    RETURN_IF_ERROR(
        statestore_subscriber_->AddTopic(IMPALA_MEMBERSHIP_TOPIC, true, cb));
  }
  if (metrics_ != NULL) {
    total_assignments_ =
        metrics_->CreateAndRegisterPrimitiveMetric(ASSIGNMENTS_KEY, 0L);
    total_local_assignments_ =
        metrics_->CreateAndRegisterPrimitiveMetric(LOCAL_ASSIGNMENTS_KEY, 0L);
    initialised_ =
        metrics_->CreateAndRegisterPrimitiveMetric(SCHEDULER_INIT_KEY, true);
  }

  if (statestore_subscriber_ != NULL) {
    // Figure out what our IP address is, so that each subscriber
    // doesn't have to resolve it on every heartbeat.
    vector<string> ipaddrs;
    const string& hostname = backend_descriptor_.address.hostname;
    Status status = HostnameToIpAddrs(hostname, &ipaddrs);
    if (!status.ok()) {
      VLOG(1) << "Failed to resolve " << hostname << ": " << status.GetErrorMsg();
      return status;
    }
    // Find a non-localhost address for this host; if one can't be
    // found use the first address returned by HostnameToIpAddrs
    string ipaddr = ipaddrs[0];
    if (!FindFirstNonLocalhost(ipaddrs, &ipaddr)) {
      VLOG(3) << "Only localhost addresses found for " << hostname;
    }

    backend_descriptor_.ip_address = ipaddr;
    LOG(INFO) << "Simple-scheduler using " << ipaddr << " as IP address";
  }
  return Status::OK;
}

void SimpleScheduler::UpdateMembership(
    const StateStoreSubscriber::TopicDeltaMap& service_state,
    vector<TTopicUpdate>* topic_updates) {
  ++update_count_;
  // TODO: Work on a copy if possible, or at least do resolution as a separate step
  // First look to see if the topic(s) we're interested in have an update
  StateStoreSubscriber::TopicDeltaMap::const_iterator topic =
      service_state.find(IMPALA_MEMBERSHIP_TOPIC);

  // Copy to work on without holding the map lock
  HostMap host_map_copy;
  HostIpAddressMap host_ip_map_copy;
  bool found_self = false;

  if (topic != service_state.end()) {
    const TTopicDelta& delta = topic->second;
    if (delta.is_delta) {
      // TODO: Handle deltas when the state-store starts sending them
      LOG(WARNING) << "Unexpected delta update from state-store, ignoring as scheduler"
                      " cannot handle deltas";
      return;
    }

    BOOST_FOREACH(const TTopicItem& item, delta.topic_entries) {
      TBackendDescriptor backend_descriptor;
      // Benchmarks have suggested that this method can deserialize
      // ~10m messages per second, so no immediate need to consider optimisation.
      uint32_t len = item.value.size();
      Status status = DeserializeThriftMsg(reinterpret_cast<const uint8_t*>(
          item.value.data()), &len, false, &backend_descriptor);
      if (!status.ok()) {
        VLOG(2) << "Error deserializing topic item with key: " << item.key;
        continue;
      }

      if (item.key == backend_id_) {
        if (backend_descriptor.address == backend_descriptor_.address) {
          found_self = true;
        } else {
          // Someone else has registered this subscriber ID with a
          // different address. We will try to re-register
          // (i.e. overwrite their subscription), but there is likely
          // a configuration problem.
          LOG_EVERY_N(WARNING, 30) << "Duplicate subscriber registration from address: "
                                   << backend_descriptor.address;
        }
      }

      host_map_copy[backend_descriptor.ip_address].push_back(
          backend_descriptor.address);
      host_ip_map_copy[backend_descriptor.address.hostname] =
          backend_descriptor.ip_address;
    }
  }

  // If this impalad is not in our view of the membership list, we
  // should add it and tell the state-store.
  if (!found_self) {
    VLOG(2) << "Registering local backend with state-store";
    topic_updates->push_back(TTopicUpdate());
    TTopicUpdate& update = topic_updates->back();
    update.topic_name = IMPALA_MEMBERSHIP_TOPIC;
    update.topic_updates.push_back(TTopicItem());

    TTopicItem& item = update.topic_updates.back();
    item.key = backend_id_;
    Status status = thrift_serializer_.Serialize(&backend_descriptor_, &item.value);
    if (!status.ok()) {
      LOG(INFO) << "Failed to serialize Impala backend address for state-store topic: "
                << status.GetErrorMsg();
      topic_updates->pop_back();
    }
  }

  {
    lock_guard<mutex> lock(host_map_lock_);
    host_map_.swap(host_map_copy);
    host_ip_map_.swap(host_ip_map_copy);
    next_nonlocal_host_entry_ = host_map_.begin();
  }
}

Status SimpleScheduler::GetHosts(
    const vector<TNetworkAddress>& data_locations, HostList* hostports) {
  hostports->clear();
  for (int i = 0; i < data_locations.size(); ++i) {
    TNetworkAddress backend;
    GetHost(data_locations[i], &backend);
    hostports->push_back(backend);
  }
  DCHECK_EQ(data_locations.size(), hostports->size());
  return Status::OK;
}

Status SimpleScheduler::GetHost(const TNetworkAddress& data_location,
    TNetworkAddress* hostport) {
  lock_guard<mutex> lock(host_map_lock_);
  if (host_map_.size() == 0) {
    return Status("No backends configured");
  }
  bool local_assignment = false;
  HostMap::iterator entry = host_map_.find(data_location.hostname);

  if (entry == host_map_.end()) {
    // host_map_ map ip address to backend but data_location.hostname might be a hostname.
    // Find the ip address of the data_location from host_ip_map_.
    HostIpAddressMap::const_iterator itr = host_ip_map_.find(data_location.hostname);
    if (itr != host_ip_map_.end()) {
      entry = host_map_.find(itr->second);
    }
  }

  if (entry == host_map_.end()) {
    // round robin the ipaddress
    entry = next_nonlocal_host_entry_;
    ++next_nonlocal_host_entry_;
    if (next_nonlocal_host_entry_ == host_map_.end()) {
      next_nonlocal_host_entry_ = host_map_.begin();
    }
  } else {
    local_assignment = true;
  }
  DCHECK(!entry->second.empty());
  // Round-robin between impalads on the same ipaddress.
  // Pick the first one, then move it to the back of the queue
  *hostport = entry->second.front();
  entry->second.pop_front();
  entry->second.push_back(*hostport);

  if (metrics_ != NULL) {
    total_assignments_->Increment(1);
    if (local_assignment) {
      total_local_assignments_->Increment(1L);
    }
  }

  if (VLOG_FILE_IS_ON) {
    stringstream s;
    s << "(" << data_location.hostname << ":" << data_location.port;
    s << " -> " << (hostport->hostname) << ":" << (hostport->port) << ")";
    VLOG_FILE << "SimpleScheduler assignment (data->backend):  " << s.str();
  }
  return Status::OK;
}

void SimpleScheduler::GetAllKnownHosts(HostList* hostports) {
  lock_guard<mutex> lock(host_map_lock_);
  hostports->clear();
  BOOST_FOREACH(const HostMap::value_type& hosts, host_map_) {
    hostports->insert(hostports->end(), hosts.second.begin(), hosts.second.end());
  }
}

}
<|endoftext|>


<commit_before>#include <cmath>
#include <cstdlib>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

#include <popt.h>

#include "defs.hh"
#include "StringSet.hh"
#include "FactorEncoder.hh"
#include "Unigrams.hh"

using namespace std;

void assert_single_chars(map<string, flt_type> &vocab,
                         const map<string, flt_type> &chars,
                         flt_type val)
{
    for (auto it = chars.cbegin(); it != chars.cend(); ++it)
        if (vocab.find(it->first) == vocab.end())
            vocab[it->first] = val;
}

int main(int argc, char* argv[]) {

    float cutoff_value = 0.0;
    int n_candidates_per_iter = 5000;
    int max_removals_per_iter = 5000;
    int min_removals_per_iter = 0;
    int target_vocab_size = 50000;
    bool enable_forward_backward = false;
    flt_type one_char_min_lp = -25.0;
    string vocab_fname;
    string corpus_fname;

    // Popt documentation:
    // http://linux.die.net/man/3/popt
    // http://privatemisc.blogspot.fi/2012/12/popt-basic-example.html
    poptContext pc;
    struct poptOption po[] = {
        {"cutoff", 'u', POPT_ARG_FLOAT, &cutoff_value, 11001, NULL, "Cutoff value for each iteration"},
        {"candidates", 'c', POPT_ARG_INT, &n_candidates_per_iter, 11002, NULL, "Number of candidate subwords to try to remove per iteration"},
        {"max_removals", 'a', POPT_ARG_INT, &max_removals_per_iter, 11003, NULL, "Maximum number of removals per iteration"},
        {"min_removals", 'i', POPT_ARG_INT, &min_removals_per_iter, 11004, NULL, "Minimum number of removals per iteration (stopping criterion)"},
        {"vocab_size", 'g', POPT_ARG_INT, &target_vocab_size, 11007, NULL, "Target vocabulary size (stopping criterion)"},
        {"forward_backward", 'f', POPT_ARG_NONE, &enable_forward_backward, 11007, "Use Forward-backward segmentation instead of Viterbi", NULL},
        POPT_AUTOHELP
        {NULL}
    };

    pc = poptGetContext(NULL, argc, (const char **)argv, po, 0);
    poptSetOtherOptionHelp(pc, "[INITIAL VOCABULARY] [TRAINING CORPUS]");

    int val;
    while ((val = poptGetNextOpt(pc)) >= 0)
        continue;

    // poptGetNextOpt returns -1 when the final argument has been parsed
    // otherwise an error occured
    if (val != -1) {
        switch (val) {
        case POPT_ERROR_NOARG:
            cerr << "Argument missing for an option" << endl;
            exit(1);
        case POPT_ERROR_BADOPT:
            cerr << "Option's argument could not be parsed" << endl;
            exit(1);
        case POPT_ERROR_BADNUMBER:
        case POPT_ERROR_OVERFLOW:
            cerr << "Option could not be converted to number" << endl;
            exit(1);
        default:
            cerr << "Unknown error in option processing" << endl;
            exit(1);
        }
    }

    // Handle ARG part of command line
    if (poptPeekArg(pc) != NULL)
        vocab_fname.assign((char*)poptGetArg(pc));
    else {
        cerr << "Initial vocabulary file not set" << endl;
        exit(1);
    }

    if (poptPeekArg(pc) != NULL)
        corpus_fname.assign((char*)poptGetArg(pc));
    else {
        cerr << "Training corpus not set" << endl;
        exit(1);
    }

    cerr << "parameters, initial vocabulary: " << vocab_fname << endl;
    cerr << "parameters, training corpus: " << corpus_fname << endl;
    cerr << "parameters, cutoff: " << setprecision(15) << cutoff_value << endl;
    cerr << "parameters, candidates per iteration: " << n_candidates_per_iter << endl;
    cerr << "parameters, removals per iteration: " << max_removals_per_iter << endl;
    cerr << "below 100k.." << endl;
    cerr << "parameters, removals per iteration: 500" << endl;
    cerr << "parameters, min removals per iteration: " << min_removals_per_iter << endl;
    cerr << "parameters, target vocab size: " << target_vocab_size << endl;
    cerr << "parameters, use forward-backward: " << enable_forward_backward << endl;

    int maxlen;
    map<string, flt_type> all_chars;
    map<string, flt_type> vocab;
    map<string, flt_type> freqs;
    vector<string> sents;

    cerr << "Reading vocabulary " << vocab_fname << endl;
    int retval = Unigrams::read_vocab(vocab_fname, vocab, maxlen);
    if (retval < 0) {
        cerr << "something went wrong reading vocabulary" << endl;
        exit(0);
    }
    cerr << "\t" << "size: " << vocab.size() << endl;
    cerr << "\t" << "maximum string length: " << maxlen << endl;
    for (auto it = vocab.cbegin(); it != vocab.end(); ++it)
        if (it->first.length() == 1)
            all_chars[it->first] = 0.0;

    Unigrams gg;
    if (enable_forward_backward)
        gg.set_segmentation_method(forward_backward);
    else
        gg.set_segmentation_method(viterbi);

    cerr << "Reading training corpus " << corpus_fname << endl;
    Unigrams::read_sents(corpus_fname, sents);

    cerr << "Initial cutoff" << endl;
    flt_type segsents_cost = gg.resegment_sents(sents, vocab, freqs);
    vocab = freqs;
    Unigrams::freqs_to_logprobs(vocab);
    ostringstream tempfname;
    tempfname << "cutoff.0.vocab";
    Unigrams::write_vocab(tempfname.str(), vocab);
    assert_single_chars(vocab, all_chars, one_char_min_lp);

    cerr << "cost: " << segsents_cost << endl;

    flt_type temp_cutoff = 5.0;
    while (true) {
        gg.cutoff(freqs, temp_cutoff);
        cerr << "\tcutoff: " << temp_cutoff << "\t" << "vocabulary size: " << freqs.size() << endl;
        vocab = freqs;
        gg.freqs_to_logprobs(vocab);
        assert_single_chars(vocab, all_chars, one_char_min_lp);

        if (vocab.size() <= target_vocab_size) break;
        segsents_cost = gg.resegment_sents(sents, vocab, freqs);

        vocab = freqs;
        Unigrams::freqs_to_logprobs(vocab);
        ostringstream tempfname;
        tempfname << "cutoff." << temp_cutoff << ".vocab";
        Unigrams::write_vocab(tempfname.str(), vocab);

        cerr << "cost: " << segsents_cost << endl;
        temp_cutoff += 2.5;
    }

    /*
    cerr << "Removing subwords one by one" << endl;
    int itern = 1;
    while (true) {

        cerr << "iteration " << itern << endl;

        if (vocab.size() <= 100000) {
            max_removals_per_iter = 500;
        }

        cerr << "collecting candidate subwords for removal" << endl;
        set<string> candidates;
        if ((int)vocab.size()-n_candidates_per_iter < target_vocab_size) n_candidates_per_iter = (int)vocab.size()-target_vocab_size;
        gg.init_removal_candidates(n_candidates_per_iter, words, vocab, candidates);

        cerr << "ranking candidate subwords" << endl;
        vector<pair<string, flt_type> > removal_scores;
        cost = gg.rank_removal_candidates(words, vocab, candidates, freqs, removal_scores);

        cerr << "starting cost before removing subwords one by one: " << cost << endl;

        // Perform removals one by one
        unsigned int n_removals = 0;
        for (unsigned int i=0; i<removal_scores.size(); i++) {

            if (removal_scores[i].first.length() == 1) continue;
            // Score most probably went to zero already
            if (vocab.find(removal_scores[i].first) == vocab.end()) continue;

            freqs.erase(removal_scores[i].first);
            vocab = freqs;
            Unigrams::freqs_to_logprobs(vocab);
            assert_single_chars(vocab, all_chars, one_char_min_lp);

            n_removals++;

            if (vocab.size() % 5000 == 0) {
                ostringstream vocabfname;
                vocabfname << "iter" << itern << "_" << vocab.size() << ".vocab";
                Unigrams::write_vocab(vocabfname.str(), vocab);
            }

            if (n_removals >= max_removals_per_iter) break;
            if (vocab.size() <= target_vocab_size) break;
        }

        int n_cutoff = gg.cutoff(freqs, cutoff_value);
        vocab = freqs;
        Unigrams::freqs_to_logprobs(vocab);
        assert_single_chars(vocab, all_chars, one_char_min_lp);
        cost = gg.iterate(words, vocab, 2);
        assert_single_chars(vocab, all_chars, one_char_min_lp);

        cerr << "subwords removed in this iteration: " << n_removals << endl;
        cerr << "subwords removed with cutoff this iteration: " << n_cutoff << endl;
        cerr << "current vocabulary size: " << vocab.size() << endl;
        cerr << "likelihood after the removals: " << cost << endl;

        ostringstream vocabfname;
        vocabfname << "iter" << itern << ".vocab";
        Unigrams::write_vocab(vocabfname.str(), vocab);

        itern++;

        if (n_removals < min_removals_per_iter) {
            cerr << "stopping by min_removals_per_iter." << endl;
            break;
        }

        if (vocab.size() <= target_vocab_size) {
            cerr << "stopping by min_vocab_size." << endl;
            break;
        }
    }
    */

    exit(1);
}

<commit_msg>Minor style improvements.<commit_after>#include <cmath>
#include <cstdlib>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

#include <popt.h>

#include "defs.hh"
#include "StringSet.hh"
#include "FactorEncoder.hh"
#include "Unigrams.hh"

using namespace std;

void assert_single_chars(map<string, flt_type> &vocab,
                         const map<string, flt_type> &chars,
                         flt_type val)
{
    for (auto it = chars.cbegin(); it != chars.cend(); ++it)
        if (vocab.find(it->first) == vocab.end())
            vocab[it->first] = val;
}

int main(int argc, char* argv[]) {

    float cutoff_value = 0.0;
    int n_candidates_per_iter = 5000;
    int max_removals_per_iter = 5000;
    int min_removals_per_iter = 0;
    int target_vocab_size = 50000;
    bool enable_forward_backward = false;
    flt_type one_char_min_lp = -25.0;
    string vocab_fname;
    string corpus_fname;

    // Popt documentation:
    // http://linux.die.net/man/3/popt
    // http://privatemisc.blogspot.fi/2012/12/popt-basic-example.html
    poptContext pc;
    struct poptOption po[] = {
        {"cutoff", 'u', POPT_ARG_FLOAT, &cutoff_value, 11001, NULL, "Cutoff value for each iteration"},
        {"candidates", 'c', POPT_ARG_INT, &n_candidates_per_iter, 11002, NULL, "Number of candidate subwords to try to remove per iteration"},
        {"max_removals", 'a', POPT_ARG_INT, &max_removals_per_iter, 11003, NULL, "Maximum number of removals per iteration"},
        {"min_removals", 'i', POPT_ARG_INT, &min_removals_per_iter, 11004, NULL, "Minimum number of removals per iteration (stopping criterion)"},
        {"vocab_size", 'g', POPT_ARG_INT, &target_vocab_size, 11007, NULL, "Target vocabulary size (stopping criterion)"},
        {"forward_backward", 'f', POPT_ARG_NONE, &enable_forward_backward, 11007, "Use Forward-backward segmentation instead of Viterbi", NULL},
        POPT_AUTOHELP
        {NULL}
    };

    pc = poptGetContext(NULL, argc, (const char **)argv, po, 0);
    poptSetOtherOptionHelp(pc, "[INITIAL VOCABULARY] [TRAINING CORPUS]");

    int val;
    while ((val = poptGetNextOpt(pc)) >= 0)
        continue;

    // poptGetNextOpt returns -1 when the final argument has been parsed
    // otherwise an error occured
    if (val != -1) {
        switch (val) {
        case POPT_ERROR_NOARG:
            cerr << "Argument missing for an option" << endl;
            exit(1);
        case POPT_ERROR_BADOPT:
            cerr << "Option's argument could not be parsed" << endl;
            exit(1);
        case POPT_ERROR_BADNUMBER:
        case POPT_ERROR_OVERFLOW:
            cerr << "Option could not be converted to number" << endl;
            exit(1);
        default:
            cerr << "Unknown error in option processing" << endl;
            exit(1);
        }
    }

    // Handle ARG part of command line
    if (poptPeekArg(pc) != NULL)
        vocab_fname.assign((char*)poptGetArg(pc));
    else {
        cerr << "Initial vocabulary file not set" << endl;
        exit(1);
    }

    if (poptPeekArg(pc) != NULL)
        corpus_fname.assign((char*)poptGetArg(pc));
    else {
        cerr << "Training corpus not set" << endl;
        exit(1);
    }

    cerr << "parameters, initial vocabulary: " << vocab_fname << endl;
    cerr << "parameters, training corpus: " << corpus_fname << endl;
    cerr << "parameters, cutoff: " << setprecision(15) << cutoff_value << endl;
    cerr << "parameters, candidates per iteration: " << n_candidates_per_iter << endl;
    cerr << "parameters, removals per iteration: " << max_removals_per_iter << endl;
    cerr << "below 100k.." << endl;
    cerr << "parameters, removals per iteration: 500" << endl;
    cerr << "parameters, min removals per iteration: " << min_removals_per_iter << endl;
    cerr << "parameters, target vocab size: " << target_vocab_size << endl;
    cerr << "parameters, use forward-backward: " << enable_forward_backward << endl;

    int maxlen;
    map<string, flt_type> all_chars;
    map<string, flt_type> vocab;
    map<string, flt_type> freqs;
    vector<string> sents;

    cerr << "Reading vocabulary " << vocab_fname << endl;
    int retval = Unigrams::read_vocab(vocab_fname, vocab, maxlen);
    if (retval < 0) {
        cerr << "something went wrong reading vocabulary" << endl;
        exit(0);
    }
    cerr << "\t" << "size: " << vocab.size() << endl;
    cerr << "\t" << "maximum string length: " << maxlen << endl;
    for (auto it = vocab.cbegin(); it != vocab.end(); ++it)
        if (it->first.length() == 1)
            all_chars[it->first] = 0.0;

    Unigrams gg;
    if (enable_forward_backward)
        gg.set_segmentation_method(forward_backward);
    else
        gg.set_segmentation_method(viterbi);

    cerr << "Reading training corpus " << corpus_fname << endl;
    Unigrams::read_sents(corpus_fname, sents);

    cerr << "Initial cutoff" << endl;
    flt_type cost = gg.resegment_sents(sents, vocab, freqs);
    vocab = freqs;
    Unigrams::freqs_to_logprobs(vocab);
    ostringstream tempfname;
    tempfname << "cutoff.0.vocab";
    Unigrams::write_vocab(tempfname.str(), vocab);
    assert_single_chars(vocab, all_chars, one_char_min_lp);

    cerr << "cost: " << cost << endl;

    flt_type temp_cutoff = 5.0;
    while (true) {
        Unigrams::cutoff(freqs, temp_cutoff);
        cerr << "\tcutoff: " << temp_cutoff << "\t" << "vocabulary size: " << freqs.size() << endl;
        vocab = freqs;
        Unigrams::freqs_to_logprobs(vocab);
        assert_single_chars(vocab, all_chars, one_char_min_lp);

        if (vocab.size() <= target_vocab_size) break;
        cost = gg.resegment_sents(sents, vocab, freqs);

        vocab = freqs;
        Unigrams::freqs_to_logprobs(vocab);
        ostringstream tempfname;
        tempfname << "cutoff." << temp_cutoff << ".vocab";
        Unigrams::write_vocab(tempfname.str(), vocab);

        cerr << "cost: " << cost << endl;
        temp_cutoff += 2.5;
    }

    /*
    cerr << "Removing subwords one by one" << endl;
    int itern = 1;
    while (true) {

        cerr << "iteration " << itern << endl;

        if (vocab.size() <= 100000) {
            max_removals_per_iter = 500;
        }

        cerr << "collecting candidate subwords for removal" << endl;
        set<string> candidates;
        if ((int)vocab.size()-n_candidates_per_iter < target_vocab_size) n_candidates_per_iter = (int)vocab.size()-target_vocab_size;
        gg.init_removal_candidates(n_candidates_per_iter, words, vocab, candidates);

        cerr << "ranking candidate subwords" << endl;
        vector<pair<string, flt_type> > removal_scores;
        cost = gg.rank_removal_candidates(words, vocab, candidates, freqs, removal_scores);

        cerr << "starting cost before removing subwords one by one: " << cost << endl;

        // Perform removals one by one
        unsigned int n_removals = 0;
        for (unsigned int i=0; i<removal_scores.size(); i++) {

            if (removal_scores[i].first.length() == 1) continue;
            // Score most probably went to zero already
            if (vocab.find(removal_scores[i].first) == vocab.end()) continue;

            freqs.erase(removal_scores[i].first);
            vocab = freqs;
            Unigrams::freqs_to_logprobs(vocab);
            assert_single_chars(vocab, all_chars, one_char_min_lp);

            n_removals++;

            if (vocab.size() % 5000 == 0) {
                ostringstream vocabfname;
                vocabfname << "iter" << itern << "_" << vocab.size() << ".vocab";
                Unigrams::write_vocab(vocabfname.str(), vocab);
            }

            if (n_removals >= max_removals_per_iter) break;
            if (vocab.size() <= target_vocab_size) break;
        }

        int n_cutoff = gg.cutoff(freqs, cutoff_value);
        vocab = freqs;
        Unigrams::freqs_to_logprobs(vocab);
        assert_single_chars(vocab, all_chars, one_char_min_lp);
        cost = gg.iterate(words, vocab, 2);
        assert_single_chars(vocab, all_chars, one_char_min_lp);

        cerr << "subwords removed in this iteration: " << n_removals << endl;
        cerr << "subwords removed with cutoff this iteration: " << n_cutoff << endl;
        cerr << "current vocabulary size: " << vocab.size() << endl;
        cerr << "likelihood after the removals: " << cost << endl;

        ostringstream vocabfname;
        vocabfname << "iter" << itern << ".vocab";
        Unigrams::write_vocab(vocabfname.str(), vocab);

        itern++;

        if (n_removals < min_removals_per_iter) {
            cerr << "stopping by min_removals_per_iter." << endl;
            break;
        }

        if (vocab.size() <= target_vocab_size) {
            cerr << "stopping by min_vocab_size." << endl;
            break;
        }
    }
    */

    exit(1);
}

<|endoftext|>


<commit_before>/// @example count_primes.cpp
/// This example shows how to count primes.

#include <primesieve.hpp>
#include <stdint.h>
#include <iostream>

int main()
{
  uint64_t count = primesieve::count_primes(0, 1000);
  std::cout << "Primes below 1000 = " << count << std::endl;

  // use multi-threading for large intervals
  uint64_t stop = 1000000000;
  count = primesieve::parallel_count_primes(0, stop);
  std::cout << "Primes below 10^9 = " << count << std::endl;

  return 0;
}
<commit_msg>Refactoring<commit_after>/// @example count_primes.cpp
/// This example shows how to count primes.

#include <primesieve.hpp>
#include <stdint.h>
#include <iostream>

int main()
{
  uint64_t count = primesieve::count_primes(0, 1000);
  std::cout << "Primes below 1000 = " << count << std::endl;

  uint64_t stop = 1000000000;

  // use multi-threading for large intervals
  count = primesieve::parallel_count_primes(0, stop);
  std::cout << "Primes below 10^9 = " << count << std::endl;

  return 0;
}
<|endoftext|>


<commit_before>#include "Level.hpp"

#include "Enemy.hpp"
#include "Target.hpp"

#include "Random.hpp"

#include <iostream> // delete

Level::Level()
  : Scene()
{
  m_player = new Player;
  m_gameObjects.push_back(m_player);
  /*  Element * e = new Element;
      e->getTransform().setPosition(100, 100);
      e->m_physics.addVelocity(sf::Vector2f(2, 1) );
      e->m_physics.addAcceleration(sf::Vector2f(-0.001f, -.01f) );
      m_gameObjects.push_back(e);
  */
  spawnEnemy();
}

Level::~Level()
{
  delete m_player;
}

void Level::draw(sf::RenderTarget& rt, sf::RenderStates rs) const
{
  Scene::draw(rt, rs);
  for(auto it = m_targets.begin(); it != m_targets.end(); ++it)
    {
      (*it)->draw(rt, rs);
    }
  Scene::drawEntities(rt, rs);
  for(auto it = m_ennemies.begin(); it != m_ennemies.end(); ++it)
    {
      (*it)->draw(rt, rs);
    }
}

void Level::update()
{
  Scene::update();
  updateEnemies();
  updateTargets();
  sf::Time t = m_clock.getElapsedTime();
  if( t.asSeconds() > 0.2f && m_targets.size() < 5 ){
    spawnTarget();
  }
  if(t.asSeconds() > 0.4f ){
    spawnEnemy();
    m_clock.restart();
  }
}

void Level::spawnEnemy()
{
  std::cout << "spawning enemy"<< m_ennemies.size() <<std::endl;
  Enemy * e = new Enemy;
  e->getTransform().setPosition(Random::get(100, 700), 50);
  e->m_physics.addRotationForce(5.0f);
  e->m_physics.addVelocity(sf::Vector2f(Random::get(-2,2), Random::get(4, 7) ) );
  m_ennemies.push_back(e );
}

void Level::spawnTarget()
{
  std::cout << "spawning target"<< m_targets.size() <<std::endl;
  Target * e = new Target;
  e->getTransform().setPosition(Random::get(100, 700), Random::get(50, 550) );
  e->m_physics.addVelocity(sf::Vector2f(Random::get(-10, 10) / 10.f,
                                        Random::get(-10, 10) / 10.f ) );
  e->m_physics.addAcceleration(  e->m_physics.getVelocity() / -100.0f );
  e->m_physics.addGrowth(sf::Vector2f(-0.005f, -0.005f) );
  m_targets.push_back(e);
}

void Level::updateEnemies()
{

  for(auto it = m_ennemies.begin(); it != m_ennemies.end(); ++it)
    {
      (*it)->update();
      if((*it)->getTransform().getPosition().x > 800
         || (*it)->getTransform().getPosition().y > 600  ){
        delete (*it);
        *it = nullptr;
      }
    }
  m_ennemies.remove(nullptr);
}

void Level::updateTargets()
{
  for(auto it = m_targets.begin(); it != m_targets.end(); ++it)
    {
      (*it)->update();
      if((*it)->getTransform().getPosition().x > 800
         || (*it)->getTransform().getPosition().y > 600
         || (*it)->getTransform().getPosition().x < -32
         || (*it)->getTransform().getPosition().y < -32){
        delete (*it);
        *it = nullptr;
      }
    }
  m_targets.remove(nullptr);
}

void Level::handleActions(Controller * c)
{
  /*
    for(auto it = c->getOneTime().begin(); it != c->getOneTime().end(); ++it)
    {
    if(it->second.compare("LEFT") ){
    movePlayer(-1);
    }

    if(it->second.compare("RIGHT") ){
    movePlayer(1);
    }
    }*/
  auto actions = c->getActions();
  while(not actions.empty() )
    {
      Action * a = actions.front();

      if(a->getName().compare(Action::LEFT) == 0){
        movePlayer(-4);
      }

      if(a->getName().compare("RIGHT") == 0){
        movePlayer(4);
      }

      actions.pop();
    }
}

void Level::movePlayer(const short& x)
{
  std::cout<<"moving player H : "<< x << std::endl;
  m_player->getTransform().move(x, 0);
}
<commit_msg>bug fix : replaced action by string for now<commit_after>#include "Level.hpp"

#include "Enemy.hpp"
#include "Target.hpp"

#include "Random.hpp"

#include <iostream> // delete

Level::Level()
  : Scene()
{
  m_player = new Player;
  m_gameObjects.push_back(m_player);
  /*  Element * e = new Element;
      e->getTransform().setPosition(100, 100);
      e->m_physics.addVelocity(sf::Vector2f(2, 1) );
      e->m_physics.addAcceleration(sf::Vector2f(-0.001f, -.01f) );
      m_gameObjects.push_back(e);
  */
  spawnEnemy();
}

Level::~Level()
{
  delete m_player;
}

void Level::draw(sf::RenderTarget& rt, sf::RenderStates rs) const
{
  Scene::draw(rt, rs);
  for(auto it = m_targets.begin(); it != m_targets.end(); ++it)
    {
      (*it)->draw(rt, rs);
    }
  Scene::drawEntities(rt, rs);
  for(auto it = m_ennemies.begin(); it != m_ennemies.end(); ++it)
    {
      (*it)->draw(rt, rs);
    }
}

void Level::update()
{
  Scene::update();
  updateEnemies();
  updateTargets();
  sf::Time t = m_clock.getElapsedTime();
  if( t.asSeconds() > 0.2f && m_targets.size() < 5 ){
    spawnTarget();
  }
  if(t.asSeconds() > 0.4f ){
    spawnEnemy();
    m_clock.restart();
  }
}

void Level::spawnEnemy()
{
  std::cout << "spawning enemy"<< m_ennemies.size() <<std::endl;
  Enemy * e = new Enemy;
  e->getTransform().setPosition(Random::get(100, 700), 50);
  e->m_physics.addRotationForce(5.0f);
  e->m_physics.addVelocity(sf::Vector2f(Random::get(-2,2), Random::get(4, 7) ) );
  m_ennemies.push_back(e );
}

void Level::spawnTarget()
{
  std::cout << "spawning target"<< m_targets.size() <<std::endl;
  Target * e = new Target;
  e->getTransform().setPosition(Random::get(100, 700), Random::get(50, 550) );
  e->m_physics.addVelocity(sf::Vector2f(Random::get(-10, 10) / 10.f,
                                        Random::get(-10, 10) / 10.f ) );
  e->m_physics.addAcceleration(  e->m_physics.getVelocity() / -100.0f );
  e->m_physics.addGrowth(sf::Vector2f(-0.005f, -0.005f) );
  m_targets.push_back(e);
}

void Level::updateEnemies()
{

  for(auto it = m_ennemies.begin(); it != m_ennemies.end(); ++it)
    {
      (*it)->update();
      if((*it)->getTransform().getPosition().x > 800
         || (*it)->getTransform().getPosition().y > 600  ){
        delete (*it);
        *it = nullptr;
      }
    }
  m_ennemies.remove(nullptr);
}

void Level::updateTargets()
{
  for(auto it = m_targets.begin(); it != m_targets.end(); ++it)
    {
      (*it)->update();
      if((*it)->getTransform().getPosition().x > 800
         || (*it)->getTransform().getPosition().y > 600
         || (*it)->getTransform().getPosition().x < -32
         || (*it)->getTransform().getPosition().y < -32){
        delete (*it);
        *it = nullptr;
      }
    }
  m_targets.remove(nullptr);
}

void Level::handleActions(Controller * c)
{
  /*
    for(auto it = c->getOneTime().begin(); it != c->getOneTime().end(); ++it)
    {
    if(it->second.compare("LEFT") ){
    movePlayer(-1);
    }

    if(it->second.compare("RIGHT") ){
    movePlayer(1);
    }
    }*/
  auto actions = c->getActions();
  while(not actions.empty() )
    {
      Action * a = actions.front();

      if(a->getName().compare("LEFT") == 0){
        movePlayer(-4);
      }

      if(a->getName().compare("RIGHT") == 0){
        movePlayer(4);
      }

      actions.pop();
    }
}

void Level::movePlayer(const short& x)
{
  std::cout<<"moving player H : "<< x << std::endl;
  m_player->getTransform().move(x, 0);
}
<|endoftext|>


<commit_before><commit_msg>Analyzer: support substitution of refs through the types map<commit_after><|endoftext|>


<commit_before>/***********************************************************************************************************************
**
** Copyright (c) 2011, 2014 ETH Zurich
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
** following conditions are met:
**
**    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following
**      disclaimer.
**    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
**      following disclaimer in the documentation and/or other materials provided with the distribution.
**    * Neither the name of the ETH Zurich nor the names of its contributors may be used to endorse or promote products
**      derived from this software without specific prior written permission.
**
**
** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
** INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
** DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
** WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**
***********************************************************************************************************************/

#include "oovisualization.h"
#include "SelfTest/src/SelfTestSuite.h"

#include "allOOVisualizations.h"

#include "OOModel/src/allOOModelNodes.h"

#include "VisualizationBase/src/Scene.h"
#include "VisualizationBase/src/node_extensions/Position.h"

Q_EXPORT_PLUGIN2( oovisualization, OOVisualization::OOVisualization )

using namespace OOModel;
using namespace Visualization;

namespace OOVisualization {

Core::InitializationRegistry& itemTypeInitializationRegistry()
{
	static Core::InitializationRegistry r;
	return r;
}

bool OOVisualization::initialize(Core::EnvisionManager&)
{
	// Register extensions
	Project::registerNewExtension<Position>();
	Module::registerNewExtension<Position>();
	Class::registerNewExtension<Position>();
	Method::registerNewExtension<Position>();

	// Register visualizations
	itemTypeInitializationRegistry().initializeAll();
	Scene::defaultRenderer()->registerVisualization(StatementItemList::typeIdStatic(),
			createVisualization<VStatementItemList, StatementItemList>);

	return true;
}

void OOVisualization::unload()
{
}

void OOVisualization::selfTest(QString testid)
{
	if (testid.isEmpty()) SelfTest::TestManager<OOVisualization>::runAllTests().printResultStatistics();
	else SelfTest::TestManager<OOVisualization>::runTest(testid).printResultStatistics();
}

}
<commit_msg>Register public interface only semantic zoom level<commit_after>/***********************************************************************************************************************
**
** Copyright (c) 2011, 2014 ETH Zurich
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
** following conditions are met:
**
**    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following
**      disclaimer.
**    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
**      following disclaimer in the documentation and/or other materials provided with the distribution.
**    * Neither the name of the ETH Zurich nor the names of its contributors may be used to endorse or promote products
**      derived from this software without specific prior written permission.
**
**
** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
** INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
** DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
** WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**
***********************************************************************************************************************/

#include "oovisualization.h"
#include "SelfTest/src/SelfTestSuite.h"

#include "allOOVisualizations.h"

#include "OOModel/src/allOOModelNodes.h"

#include "VisualizationBase/src/Scene.h"
#include "VisualizationBase/src/node_extensions/Position.h"

#include <semantic_zoom/VClassSzPublic.h>

Q_EXPORT_PLUGIN2( oovisualization, OOVisualization::OOVisualization )

using namespace OOModel;
using namespace Visualization;

namespace OOVisualization {

Core::InitializationRegistry& itemTypeInitializationRegistry()
{
	static Core::InitializationRegistry r;
	return r;
}

bool OOVisualization::initialize(Core::EnvisionManager&)
{
	// Register extensions
	Project::registerNewExtension<Position>();
	Module::registerNewExtension<Position>();
	Class::registerNewExtension<Position>();
	Method::registerNewExtension<Position>();

	// Register visualizations
	itemTypeInitializationRegistry().initializeAll();
	Scene::defaultRenderer()->registerVisualization(StatementItemList::typeIdStatic(),
			createVisualization<VStatementItemList, StatementItemList>);

	Scene::defaultRenderer()->registerSemanticZoomLevel("public_interface");
	Scene::defaultRenderer()->registerVisualization(Class::typeIdStatic(), 0, 1,
			createVisualization<VClassSzPublic, Class>);

	return true;
}

void OOVisualization::unload()
{
}

void OOVisualization::selfTest(QString testid)
{
	if (testid.isEmpty()) SelfTest::TestManager<OOVisualization>::runAllTests().printResultStatistics();
	else SelfTest::TestManager<OOVisualization>::runTest(testid).printResultStatistics();
}

}
<|endoftext|>