<commit_before>#include <gflags/gflags.h>
#include <chrono>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "ncode_common/src/common.h"
#include "ncode_common/src/file.h"
#include "ncode_common/src/logging.h"
#include "ncode_common/src/lp/mc_flow.h"
#include "ncode_common/src/net/algorithm.h"
#include "ncode_common/src/net/net_common.h"
#include "ncode_common/src/net/net_gen.h"
#include "ncode_common/src/perfect_hash.h"
#include "ncode_common/src/strutil.h"

DEFINE_string(topology_files, "", "Topology files");

using NodePair = std::pair<nc::net::GraphNodeIndex, nc::net::GraphNodeIndex>;

static constexpr nc::net::Bandwidth kDefaultDemand =
    nc::net::Bandwidth::FromGBitsPerSecond(100);
static constexpr size_t kDefaultFlowCount = 1000;
static constexpr char kSuperSourceId[] = "SuperSource";
static constexpr char kSuperSinkId[] = "SuperSink";

static std::vector<std::string> GetTopologyFiles() {
  std::vector<std::string> out;
  std::vector<std::string> split = nc::Split(FLAGS_topology_files, ",");
  for (const std::string& piece : split) {
    std::vector<std::string> files = nc::Glob(piece);
    out.insert(out.end(), files.begin(), files.end());
  }

  return out;
}

// Routes a set of aggregates on their shortest paths and returns the set of
// links that bottleneck first when all aggregates are grown at the same rate.
static std::pair<nc::net::GraphLinkSet, nc::net::Bandwidth>
BottleneckLinksForAggregates(
    const std::set<NodePair>& aggregates, const nc::net::GraphStorage& graph,
    const std::map<NodePair, nc::net::Walk>& shortest_paths) {
  // Will create a fake traffic matrix and routing configuration that only
  // contains the shortest path for each aggregate. Will then use the
  // oversubscription model to figure out what links will be bottlenecked.
  ctr::TrafficMatrix tm(&graph);
  for (const auto& src_and_dst : aggregates) {
    tm.AddDemand(ctr::AggregateId(src_and_dst),
                 ctr::DemandAndFlowCount(kDefaultDemand, kDefaultFlowCount));
  }

  ctr::RoutingConfiguration routing_config(tm);
  for (const auto& src_and_dst : aggregates) {
    const nc::net::Walk& sp = nc::FindOrDie(shortest_paths, src_and_dst);
    routing_config.AddRouteAndFraction(ctr::AggregateId(src_and_dst),
                                       {ctr::RouteAndFraction(&sp, 1.0)});
  }

  ctr::OverSubModel oversub_model(routing_config);
  const nc::net::GraphLinkMap<double>& links_to_load =
      oversub_model.link_to_load();

  nc::net::GraphLinkSet out;
  for (const auto& link_and_load : links_to_load) {
    double load = *(link_and_load.second);
    if (load > 0.99) {
      out.Insert(link_and_load.first);
    }
  }

  // Need to also figure out what the total flow through the network is.
  nc::net::Bandwidth total;
  const std::map<const nc::net::Walk*, nc::net::Bandwidth>& per_flow_bw_map =
      oversub_model.per_flow_bandwidth_map();
  for (const auto& path_and_bw : per_flow_bw_map) {
    nc::net::Bandwidth bandwidth = path_and_bw.second * kDefaultFlowCount;
    total += bandwidth;
  }

  return {out, total};
}

// Returns true if two nodes are reachable given a set of constraints.
static bool Reachable(nc::net::GraphNodeIndex node_one,
                      nc::net::GraphNodeIndex node_two,
                      const nc::net::GraphStorage& graph,
                      const nc::net::ExclusionSet& exclusion_set) {
  nc::net::GraphNodeSet reachable_from_one =
      nc::net::ReachableNodes(node_one, graph, exclusion_set);
  return reachable_from_one.Contains(node_two);
}

// Returns the flow between the super-source that connects the sources of a
// number of aggregates and another sources and the super-sink that connects the
// destinations of a number of aggregates and another destination.
static nc::net::Bandwidth PairwiseFlow(const std::set<NodePair>& aggregates,
                                       nc::net::GraphNodeIndex src,
                                       nc::net::GraphNodeIndex dst,
                                       const nc::net::GraphStorage& graph) {
  // Will first create super source / sinks.
  nc::net::GraphBuilder graph_builder = graph.ToBuilder();
  for (const auto& src_and_dst : aggregates) {
    const std::string& src_id = graph.GetNode(src_and_dst.first)->id();
    const std::string& dst_id = graph.GetNode(src_and_dst.second)->id();
    graph_builder.AddLink(
        {kSuperSourceId, src_id, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
    graph_builder.AddLink(
        {dst_id, kSuperSinkId, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
  }

  const std::string& src_id = graph.GetNode(src)->id();
  const std::string& dst_id = graph.GetNode(dst)->id();
  graph_builder.AddLink(
      {kSuperSourceId, src_id, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
  graph_builder.AddLink(
      {dst_id, kSuperSinkId, nc::net::Bandwidth::Max(), nc::net::Delay(1)});

  // The flow problem will take link capacities in Mbps.
  nc::net::GraphStorage extended_graph(graph_builder);
  nc::net::GraphLinkMap<double> link_capacities;
  for (nc::net::GraphLinkIndex link : extended_graph.AllLinks()) {
    const nc::net::GraphLink* link_ptr = extended_graph.GetLink(link);
    double capacity = link_ptr->bandwidth().Mbps();
    link_capacities[link] = capacity;
  }
  nc::lp::MaxFlowSingleCommodityFlowProblem flow_problem(link_capacities,
                                                         &extended_graph);
  flow_problem.AddDemand(kSuperSourceId, kSuperSinkId);

  double max_flow_Mbps;
  CHECK(flow_problem.GetMaxFlow(&max_flow_Mbps));
  return nc::net::Bandwidth::FromMBitsPerSecond(max_flow_Mbps);
}

// Returns the fraction of all pairs that are reachable when the shortest path
// between two nodes in excluded from the graph.
static std::pair<double, double> ReachableFraction(
    const std::set<NodePair>& aggregates, const nc::net::GraphStorage& graph,
    const std::map<NodePair, nc::net::Walk>& shortest_paths) {
  double total = 0;
  double unreachable = 0;
  double unreachable_no_alternative = 0;

  nc::net::GraphLinkSet full_links;
  nc::net::Bandwidth total_flow;
  std::tie(full_links, total_flow) =
      BottleneckLinksForAggregates(aggregates, graph, shortest_paths);

  nc::net::ExclusionSet exclusion_set;
  exclusion_set.Links(full_links);

  nc::net::GraphNodeSet all_nodes = graph.AllNodes();
  for (nc::net::GraphNodeIndex src : all_nodes) {
    for (nc::net::GraphNodeIndex dst : all_nodes) {
      if (src == dst) {
        continue;
      }

      if (nc::ContainsKey(aggregates, std::make_pair(src, dst))) {
        continue;
      }

      ++total;
      if (Reachable(src, dst, graph, exclusion_set)) {
        continue;
      }

      ++unreachable;
      nc::net::Bandwidth pairwise_flow =
          PairwiseFlow(aggregates, src, dst, graph);
      if (pairwise_flow == total_flow) {
        ++unreachable_no_alternative;
      }
    }
  }

  return {unreachable / total, unreachable_no_alternative / total};
}

// Returns the aggregates whose shortest paths cross a link for each link.
nc::net::GraphLinkMap<std::set<NodePair>> SPCrossAggregates(
    const nc::net::GraphStorage& graph,
    std::map<NodePair, nc::net::Walk>* shortest_paths) {
  nc::net::GraphLinkMap<std::set<NodePair>> out;

  for (nc::net::GraphNodeIndex src : graph.AllNodes()) {
    nc::net::ShortestPath sp_tree(
        src, graph.AllNodes(), nc::net::ExclusionSet(), graph.AdjacencyList());
    for (nc::net::GraphNodeIndex dst : graph.AllNodes()) {
      if (src == dst) {
        continue;
      }

      std::unique_ptr<nc::net::Walk> shortest_path = sp_tree.GetPath(dst);
      for (nc::net::GraphLinkIndex link_on_path :
           sp_tree.GetPath(dst)->links()) {
        out[link_on_path].emplace(src, dst);
      }

      (*shortest_paths)[{src, dst}] = *shortest_path;
    }
  }

  return out;
}

static void PlotReachableFractions(const nc::net::GraphStorage& graph) {
  std::map<NodePair, nc::net::Walk> shortest_paths;
  nc::net::GraphLinkMap<std::vector<NodePair>> cross_map =
      SPCrossAggregates(graph, &shortest_paths);

  for (const auto& link_and_aggregates : cross_map) {
    nc::net::GraphLinkIndex link = link_and_aggregates.first;
    const std::set<NodePair> aggregates = *(link_and_aggregates.second);

    double unreachable;
    double unreachable_no_alternative;
    std::tie(unreachable, unreachable_no_alternative) =
        ReachableFraction(aggregates, graph, shortest_paths);
    LOG(INFO) << unreachable << " " << unreachable_no_alternative << " "
              << graph.GetLink(link)->ToStringNoPorts();
  }
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);

  std::vector<std::string> topology_files = GetTopologyFiles();
  CHECK(!topology_files.empty());

  std::vector<double> all_links_to_remove;
  for (const std::string& topology_file : topology_files) {
    nc::net::GraphBuilder builder = nc::net::LoadRepetitaOrDie(
        nc::File::ReadFileToStringOrDie(topology_file));
    builder.RemoveMultipleLinks();
    nc::net::GraphStorage graph(builder);
    PlotReachableFractions(graph);
  }
}
<commit_msg>updates to meshy<commit_after>#include <gflags/gflags.h>
#include <stddef.h>
#include <chrono>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "ncode_common/src/common.h"
#include "ncode_common/src/file.h"
#include "ncode_common/src/logging.h"
#include "ncode_common/src/lp/mc_flow.h"
#include "ncode_common/src/map_util.h"
#include "ncode_common/src/net/algorithm.h"
#include "ncode_common/src/net/net_common.h"
#include "ncode_common/src/net/net_gen.h"
#include "ncode_common/src/perfect_hash.h"
#include "ncode_common/src/strutil.h"
#include "ncode_common/src/viz/grapher.h"
#include "common.h"
#include "opt/oversubscription_model.h"

DEFINE_string(topology_files, "", "Topology files");

using NodePair = std::pair<nc::net::GraphNodeIndex, nc::net::GraphNodeIndex>;

static constexpr nc::net::Bandwidth kDefaultDemand =
    nc::net::Bandwidth::FromGBitsPerSecond(100);
static constexpr size_t kDefaultFlowCount = 1000;
static constexpr char kSuperSourceId[] = "SuperSource";
static constexpr char kSuperSinkId[] = "SuperSink";

static std::vector<std::string> GetTopologyFiles() {
  std::vector<std::string> out;
  std::vector<std::string> split = nc::Split(FLAGS_topology_files, ",");
  for (const std::string& piece : split) {
    std::vector<std::string> files = nc::Glob(piece);
    out.insert(out.end(), files.begin(), files.end());
  }

  return out;
}

// Routes a set of aggregates on their shortest paths and returns the set of
// links that bottleneck first when all aggregates are grown at the same rate.
static std::pair<nc::net::GraphLinkSet, nc::net::Bandwidth>
BottleneckLinksForAggregates(
    const std::set<NodePair>& aggregates, const nc::net::GraphStorage& graph,
    const std::map<NodePair, nc::net::Walk>& shortest_paths) {
  // Will create a fake traffic matrix and routing configuration that only
  // contains the shortest path for each aggregate. Will then use the
  // oversubscription model to figure out what links will be bottlenecked.
  ctr::TrafficMatrix tm(&graph);
  for (const auto& src_and_dst : aggregates) {
    tm.AddDemand(ctr::AggregateId(src_and_dst),
                 ctr::DemandAndFlowCount(kDefaultDemand, kDefaultFlowCount));
  }

  ctr::RoutingConfiguration routing_config(tm);
  for (const auto& src_and_dst : aggregates) {
    const nc::net::Walk& sp = nc::FindOrDie(shortest_paths, src_and_dst);
    routing_config.AddRouteAndFraction(ctr::AggregateId(src_and_dst),
                                       {ctr::RouteAndFraction(&sp, 1.0)});
  }

  ctr::OverSubModel oversub_model(routing_config);
  const nc::net::GraphLinkMap<double>& links_to_load =
      oversub_model.link_to_load();

  nc::net::GraphLinkSet out;
  for (const auto& link_and_load : links_to_load) {
    double load = *(link_and_load.second);
    if (load > 0.99) {
      out.Insert(link_and_load.first);
    }
  }

  // Need to also figure out what the total flow through the network is.
  nc::net::Bandwidth total;
  const std::map<const nc::net::Walk*, nc::net::Bandwidth>& per_flow_bw_map =
      oversub_model.per_flow_bandwidth_map();
  for (const auto& path_and_bw : per_flow_bw_map) {
    nc::net::Bandwidth bandwidth = path_and_bw.second * kDefaultFlowCount;
    total += bandwidth;
  }

  return {out, total};
}

// Returns true if two nodes are reachable given a set of constraints.
static bool Reachable(nc::net::GraphNodeIndex node_one,
                      nc::net::GraphNodeIndex node_two,
                      const nc::net::GraphStorage& graph,
                      const nc::net::ExclusionSet& exclusion_set) {
  nc::net::GraphNodeSet reachable_from_one =
      nc::net::ReachableNodes(node_one, graph, exclusion_set);
  return reachable_from_one.Contains(node_two);
}

// Returns the flow between the super-source that connects the sources of a
// number of aggregates and another sources and the super-sink that connects the
// destinations of a number of aggregates and another destination.
static nc::net::Bandwidth PairwiseFlow(const std::set<NodePair>& aggregates,
                                       nc::net::GraphNodeIndex src,
                                       nc::net::GraphNodeIndex dst,
                                       const nc::net::GraphStorage& graph) {
  // Will first create super source / sinks.
  nc::net::GraphBuilder graph_builder = graph.ToBuilder();

  for (const auto& src_and_dst : aggregates) {
    const std::string& src_id = graph.GetNode(src_and_dst.first)->id();
    const std::string& dst_id = graph.GetNode(src_and_dst.second)->id();
    graph_builder.AddLink(
        {kSuperSourceId, src_id, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
    graph_builder.AddLink(
        {dst_id, kSuperSinkId, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
  }

  const std::string& src_id = graph.GetNode(src)->id();
  const std::string& dst_id = graph.GetNode(dst)->id();
  graph_builder.AddLink(
      {kSuperSourceId, src_id, nc::net::Bandwidth::Max(), nc::net::Delay(1)});
  graph_builder.AddLink(
      {dst_id, kSuperSinkId, nc::net::Bandwidth::Max(), nc::net::Delay(1)});

  // The flow problem will take link capacities in Mbps.
  nc::net::GraphStorage extended_graph(graph_builder);
  nc::net::GraphLinkMap<double> link_capacities;
  for (nc::net::GraphLinkIndex link : extended_graph.AllLinks()) {
    const nc::net::GraphLink* link_ptr = extended_graph.GetLink(link);
    double capacity = link_ptr->bandwidth().Mbps();
    link_capacities[link] = capacity;
  }
  nc::lp::MaxFlowSingleCommodityFlowProblem flow_problem(link_capacities,
                                                         &extended_graph);
  flow_problem.AddDemand(kSuperSourceId, kSuperSinkId);

  double max_flow_Mbps;
  CHECK(flow_problem.GetMaxFlow(&max_flow_Mbps));
  return nc::net::Bandwidth::FromMBitsPerSecond(max_flow_Mbps);
}

// Returns the fraction of all pairs that are reachable when the shortest path
// between two nodes in excluded from the graph.
static std::pair<double, double> ReachableFraction(
    const std::set<NodePair>& aggregates, const nc::net::GraphStorage& graph,
    const std::map<NodePair, nc::net::Walk>& shortest_paths) {
  double total = 0;
  double unreachable = 0;
  double unreachable_no_alternative = 0;

  nc::net::GraphLinkSet full_links;
  nc::net::Bandwidth total_flow;
  std::tie(full_links, total_flow) =
      BottleneckLinksForAggregates(aggregates, graph, shortest_paths);

  nc::net::ExclusionSet exclusion_set;
  exclusion_set.Links(full_links);

  nc::net::GraphNodeSet all_nodes = graph.AllNodes();
  for (nc::net::GraphNodeIndex src : all_nodes) {
    for (nc::net::GraphNodeIndex dst : all_nodes) {
      if (src == dst) {
        continue;
      }

      if (nc::ContainsKey(aggregates, std::make_pair(src, dst))) {
        continue;
      }

      ++total;
      if (Reachable(src, dst, graph, exclusion_set)) {
        continue;
      }

      ++unreachable;
      nc::net::Bandwidth pairwise_flow =
          PairwiseFlow(aggregates, src, dst, graph);
      if (pairwise_flow == total_flow) {
        ++unreachable_no_alternative;
      }
    }
  }

  return {unreachable / total, unreachable_no_alternative / total};
}

// Returns the aggregates whose shortest paths cross a link for each link.
nc::net::GraphLinkMap<std::set<NodePair>> SPCrossAggregates(
    const nc::net::GraphStorage& graph,
    std::map<NodePair, nc::net::Walk>* shortest_paths) {
  nc::net::GraphLinkMap<std::set<NodePair>> out;

  for (nc::net::GraphNodeIndex src : graph.AllNodes()) {
    nc::net::ShortestPath sp_tree(
        src, graph.AllNodes(), nc::net::ExclusionSet(), graph.AdjacencyList());
    for (nc::net::GraphNodeIndex dst : graph.AllNodes()) {
      if (src == dst) {
        continue;
      }

      std::unique_ptr<nc::net::Walk> shortest_path = sp_tree.GetPath(dst);
      for (nc::net::GraphLinkIndex link_on_path :
           sp_tree.GetPath(dst)->links()) {
        out[link_on_path].emplace(src, dst);
      }

      (*shortest_paths)[{src, dst}] = *shortest_path;
    }
  }

  return out;
}

static double PlotReachableFractions(const nc::net::GraphStorage& graph,
                                     const std::string& graph_name) {
  std::map<NodePair, nc::net::Walk> shortest_paths;
  nc::net::GraphLinkMap<std::vector<NodePair>> cross_map =
      SPCrossAggregates(graph, &shortest_paths);

  std::vector<double> unreachable_values;
  std::vector<double> unreachable_no_alt_values;
  double total_delta = 0;

  for (const auto& link_and_aggregates : cross_map) {
    nc::net::GraphLinkIndex link = link_and_aggregates.first;
    const std::set<NodePair> aggregates = *(link_and_aggregates.second);

    double unreachable;
    double unreachable_no_alternative;
    std::tie(unreachable, unreachable_no_alternative) =
        ReachableFraction(aggregates, graph, shortest_paths);
    LOG(INFO) << unreachable << " " << unreachable_no_alternative << " "
              << graph.GetLink(link)->ToStringNoPorts();

    unreachable_values.emplace_back(unreachable);
    unreachable_no_alt_values.emplace_back(unreachable_no_alternative);

    total_delta += unreachable - unreachable_no_alternative;
  }

  nc::viz::PythonGrapher grapher(nc::StrCat("meshy_out/g_", graph_name));
  grapher.PlotCDF({}, {{"unreachable", unreachable_values},
                       {"unreachable_no_alt", unreachable_no_alt_values}});
  double weight = graph.NodeCount() * (graph.NodeCount() - 1) / 2.0;
  return total_delta / weight;
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);

  std::vector<std::string> topology_files = GetTopologyFiles();
  CHECK(!topology_files.empty());

  double max_delta = 0;
  std::string topology_with_max_delta;

  std::vector<double> all_links_to_remove;
  for (const std::string& topology_file : topology_files) {
    nc::net::GraphBuilder builder = nc::net::LoadRepetitaOrDie(
        nc::File::ReadFileToStringOrDie(topology_file));
    builder.RemoveMultipleLinks();
    nc::net::GraphStorage graph(builder);

    if (graph.NodeCount() > 50 || graph.NodeCount() < 20) {
      LOG(INFO) << "Skipping " << topology_file;
      continue;
    }

    LOG(INFO) << "Processing " << topology_file << " size "
              << graph.NodeCount();
    std::string filename = nc::File::ExtractFileName(topology_file);
    double delta = PlotReachableFractions(graph, filename);

    if (delta > max_delta) {
      max_delta = delta;
      topology_with_max_delta = filename;
    }
  }

  LOG(INFO) << "max delta " << max_delta << " topology "
            << topology_with_max_delta;
}
<|endoftext|>


<commit_before>#ifndef VG_MINIMIZER_HPP_INCLUDED
#define VG_MINIMIZER_HPP_INCLUDED

/** \file
 * A minimizer index prototype with minimal VG dependencies.
 */

#include <cstdint>
#include <iostream>
#include <limits>
#include <utility>
#include <vector>

#include "types.hpp"

namespace vg {

//------------------------------------------------------------------------------

/**
 * A class that implements the minimizer index as a hash table mapping kmers to sets of pos_t.
 * The hash table uses quadratic probing with power-of-two size.
 * We encode kmers using 2 bits/character and take wang_hash_64() of the encoding. A minimizer
 * is the kmer with the smallest hash in a window of w consecutive kmers and their reverse
 * complements.
 *
 * Index versions:
 *
 *   1  The initial version.
 *
 *   2  Minimizer selection is based on hashes instead of lexicographic order. A sequence and
 *      its reverse complement have the same minimizers, reducing index size by 50%. Not
 *      compatible with version 1.
 *
 *   3  Construction-time hit cap is no longer used. Compatible with version 2.
 */
class MinimizerIndex {
public:
    typedef std::uint64_t key_type;
    typedef std::uint64_t code_type;
    typedef std::uint32_t offset_type;

    // Public constants.
    // Any key is smaller than NO_KEY and NO_VALUE maps to an empty pos_t.
    constexpr static size_t    KMER_LENGTH      = 21;
    constexpr static size_t    WINDOW_LENGTH    = 11;
    constexpr static size_t    KMER_MAX_LENGTH  = 31;
    constexpr static size_t    INITIAL_CAPACITY = 1024;
    constexpr static double    MAX_LOAD_FACTOR  = 0.77;
    constexpr static key_type  NO_KEY           = std::numeric_limits<key_type>::max();
    constexpr static code_type NO_VALUE         = 0;

    union value_type {
        code_type               value;
        std::vector<code_type>* pointer;
    };

    typedef std::pair<key_type, value_type> cell_type;

    static cell_type empty_cell() { return cell_type(NO_KEY, { NO_VALUE }); }

    struct minimizer_type {
        key_type    key;        // Encoded minimizer.
        size_t      hash;       // Hash of the minimizer.
        offset_type offset;     // First/last offset of the kmer for forward/reverse complement.
        bool        is_reverse; // The minimizer is the reverse complement of the kmer.

        /// Is the minimizer empty?
        bool empty() const { return (this->key == NO_KEY); }

        /// Sort by (offset, !is_reverse). When the offsets are equal, a reverse complement
        /// minimizer is earlier in the sequence than a forward minimizer.
        bool operator<(const minimizer_type& another) const {
            return ((this->offset < another.offset) ||
                    (this->offset == another.offset && this->is_reverse > another.is_reverse));
        }

        bool operator==(const minimizer_type& another) const {
            return (this->key == another.key && this->offset == another.offset && this->is_reverse == another.is_reverse);
        }
    };

    struct Header {
        std::uint32_t tag, version;
        std::uint64_t flags;
        size_t        k, w;
        size_t        keys, capacity, max_keys;
        size_t        values;
        size_t        unused1; // This used to be max_occs.
        size_t        unique;
        size_t        unused2; // This used to be frequent.

        constexpr static std::uint32_t TAG = 0x31513151;
        constexpr static std::uint32_t VERSION = 3;
        constexpr static std::uint32_t MIN_VERSION = 2;

        Header();
        Header(size_t kmer_length, size_t window_length);
        void sanitize();
        bool check() const;

        bool operator==(const Header& another) const;
        bool operator!=(const Header& another) const { return !(this->operator==(another)); }
    };

//------------------------------------------------------------------------------

    /// Constructs an index with the default parameters.
    MinimizerIndex();

    /// Constructs an index with the specified parameter values.
    MinimizerIndex(size_t kmer_length, size_t window_length);

    /// Copy constructor.
    MinimizerIndex(const MinimizerIndex& source);

    /// Move constructor.
    MinimizerIndex(MinimizerIndex&& source);

    /// Destructor.
    ~MinimizerIndex();

    /// Swaps the contents of the indexes.
    void swap(MinimizerIndex& another);

    /// Copy assignment.
    MinimizerIndex& operator=(const MinimizerIndex& source);

    /// Move assignment.
    MinimizerIndex& operator=(MinimizerIndex&& source);

    /// Serialize the index to the ostream. Returns the number of bytes written and
    /// true if the serialization was successful.
    std::pair<size_t, bool> serialize(std::ostream& out) const;

    /// Load the index from the istream and return true if successful.
    bool load(std::istream& in);

    /// Equality comparison for testing.
    bool operator==(const MinimizerIndex& another) const;

    /// Inequality comparison for testing.
    bool operator!=(const MinimizerIndex& another) const { return !(this->operator==(another)); }

//------------------------------------------------------------------------------

    /// Returns the minimizer in the window specified by the iterators. If no minimizer
    /// exists (e.g. because all kmers contain invalid characters), the return value is
    /// an empty minimizer.
    minimizer_type minimizer(std::string::const_iterator begin, std::string::const_iterator end) const;

    /// Returns all minimizers in the string specified by the iterators. The return
    /// value is a vector of minimizers sorted by their offsets.
    std::vector<minimizer_type> minimizers(std::string::const_iterator begin, std::string::const_iterator end) const;

    /// Returns all minimizers in the string. The return value is a vector of
    /// minimizers sorted by their offsets.
    std::vector<minimizer_type> minimizers(const std::string& str) const {
        return this->minimizers(str.begin(), str.end());
    }

    /// Inserts the position into the index, using minimizer.key as the key and
    /// minimizer.hash as its hash. Does not insert empty minimizers or positions.
    /// The offset of the position will be truncated to fit in REV_OFFSET bits.
    /// Use minimizer() or minimizers() to get the minimizer and valid_offset() to check
    /// if the offset fits in the available space.
    /// The position should match the orientation of the minimizer: a path label
    /// starting from the position should have the minimizer as its prefix.
    void insert(const minimizer_type& minimizer, const pos_t& pos);

    /// Returns the sorted set of occurrences of the minimizer.
    /// Use minimizer() or minimizers() to get the minimizer.
    /// If the minimizer is in reverse orientation, use reverse_base_pos() to reverse
    /// the reported occurrences.
    std::vector<pos_t> find(const minimizer_type& minimizer) const;

    /// Returns the occurrence count of the minimizer.
    /// Use minimizer() or minimizers() to get the minimizer.
    size_t count(const minimizer_type& minimizer) const;

//------------------------------------------------------------------------------

    /// Length of the kmers in the index.
    size_t k() const { return this->header.k; }

    /// Window length for the minimizers.
    size_t w() const { return this->header.w; }

    /// Number of keys in the index.
    size_t size() const { return this->header.keys; }

    /// Is the index empty.
    bool empty() const { return (this->size() == 0); }

    /// Number of values (minimizer occurrences) in the index.
    size_t values() const { return this->header.values; }

    /// Size of the hash table.
    size_t capacity() const { return this->header.capacity; }

    /// Actual capacity of the hash table. Exceeding it will initiate rehashing.
    size_t max_keys() const { return this->header.max_keys; }

    /// Current load factor of the hash table.
    double load_factor() const { return static_cast<double>(this->size()) / static_cast<double>(this->capacity()); }

    /// Number of minimizers with a single occurrence.
    size_t unique_keys() const { return this->header.unique; }

//------------------------------------------------------------------------------

private:
    Header                 header;
    std::vector<cell_type> hash_table;
    std::vector<bool>      is_pointer;

//------------------------------------------------------------------------------

public:
    // Constants for the encoding between std::string and key_type.
    constexpr static size_t   PACK_WIDTH = 2;
    constexpr static key_type PACK_MASK  = 0x3;

    // Arrays for the encoding between std::string and key_type.
    const static std::vector<unsigned char> CHAR_TO_PACK;
    const static std::vector<char>          PACK_TO_CHAR;
    const static std::vector<key_type>      KMER_MASK;

    // Constants for the encoding between pos_t and code_type.
    constexpr static size_t    ID_OFFSET  = 11;
    constexpr static size_t    REV_OFFSET = 10;
    constexpr static code_type REV_MASK   = 0x400;
    constexpr static code_type OFF_MASK   = 0x3FF;

    /// Is the offset small enough to fit in the low-order bits of the encoding?
    static bool valid_offset(const pos_t& pos) {
        return (offset(pos) <= OFF_MASK);
    }

    /// Encode pos_t as code_type.
    static code_type encode(const pos_t& pos) {
        return (static_cast<code_type>(id(pos)) << ID_OFFSET) |
               (static_cast<code_type>(is_rev(pos)) << REV_OFFSET) |
               (static_cast<code_type>(offset(pos)) & OFF_MASK);
    }

    /// Decode code_type as pos_t.
    static pos_t decode(code_type pos) {
        return make_pos_t(pos >> ID_OFFSET, pos & REV_MASK, pos & OFF_MASK);
    }

//------------------------------------------------------------------------------

private:
    void copy(const MinimizerIndex& source);
    void clear(size_t i);   // Deletes the pointer at hash_table[i].
    void clear();           // Deletes all pointers in the hash table.

    // Find the hash table offset for the key with the given hash value.
    size_t find_offset(key_type key, size_t hash) const;

    // Insert (key, pos) to hash_table[offset], which is assumed to be empty.
    // Rehashing may be necessary.
    void insert(key_type key, code_type pos, size_t offset);

    // Add pos to the list of occurrences of key at hash_table[offset].
    // The occurrences may be deleted.
    void append(key_type key, code_type pos, size_t offset);

    // Does the list of occurrences at hash_table[offset] contain pos?
    bool contains(size_t offset, code_type pos) const;

    // Double the size of the hash table.
    void rehash();
};

//------------------------------------------------------------------------------

} // namespace vg

#endif // VG_MINIMIZER_HPP_INCLUDED
<commit_msg>Use fixed-width types in the header<commit_after>#ifndef VG_MINIMIZER_HPP_INCLUDED
#define VG_MINIMIZER_HPP_INCLUDED

/** \file
 * A minimizer index prototype with minimal VG dependencies.
 */

#include <cstdint>
#include <iostream>
#include <limits>
#include <utility>
#include <vector>

#include "types.hpp"

namespace vg {

//------------------------------------------------------------------------------

/**
 * A class that implements the minimizer index as a hash table mapping kmers to sets of pos_t.
 * The hash table uses quadratic probing with power-of-two size.
 * We encode kmers using 2 bits/character and take wang_hash_64() of the encoding. A minimizer
 * is the kmer with the smallest hash in a window of w consecutive kmers and their reverse
 * complements.
 *
 * Index versions:
 *
 *   1  The initial version.
 *
 *   2  Minimizer selection is based on hashes instead of lexicographic order. A sequence and
 *      its reverse complement have the same minimizers, reducing index size by 50%. Not
 *      compatible with version 1.
 *
 *   3  Construction-time hit cap is no longer used. Compatible with version 2.
 */
class MinimizerIndex {
public:
    typedef std::uint64_t key_type;
    typedef std::uint64_t code_type;
    typedef std::uint32_t offset_type;

    // Public constants.
    // Any key is smaller than NO_KEY and NO_VALUE maps to an empty pos_t.
    constexpr static size_t    KMER_LENGTH      = 21;
    constexpr static size_t    WINDOW_LENGTH    = 11;
    constexpr static size_t    KMER_MAX_LENGTH  = 31;
    constexpr static size_t    INITIAL_CAPACITY = 1024;
    constexpr static double    MAX_LOAD_FACTOR  = 0.77;
    constexpr static key_type  NO_KEY           = std::numeric_limits<key_type>::max();
    constexpr static code_type NO_VALUE         = 0;

    union value_type {
        code_type               value;
        std::vector<code_type>* pointer;
    };

    typedef std::pair<key_type, value_type> cell_type;

    static cell_type empty_cell() { return cell_type(NO_KEY, { NO_VALUE }); }

    struct minimizer_type {
        key_type    key;        // Encoded minimizer.
        size_t      hash;       // Hash of the minimizer.
        offset_type offset;     // First/last offset of the kmer for forward/reverse complement.
        bool        is_reverse; // The minimizer is the reverse complement of the kmer.

        /// Is the minimizer empty?
        bool empty() const { return (this->key == NO_KEY); }

        /// Sort by (offset, !is_reverse). When the offsets are equal, a reverse complement
        /// minimizer is earlier in the sequence than a forward minimizer.
        bool operator<(const minimizer_type& another) const {
            return ((this->offset < another.offset) ||
                    (this->offset == another.offset && this->is_reverse > another.is_reverse));
        }

        bool operator==(const minimizer_type& another) const {
            return (this->key == another.key && this->offset == another.offset && this->is_reverse == another.is_reverse);
        }
    };

    struct Header {
        std::uint32_t tag, version;
        std::uint64_t flags;
        std::uint64_t k, w;
        std::uint64_t keys, capacity, max_keys;
        std::uint64_t values;
        std::uint64_t unused1; // This used to be max_occs.
        std::uint64_t unique;
        std::uint64_t unused2; // This used to be frequent.

        constexpr static std::uint32_t TAG = 0x31513151;
        constexpr static std::uint32_t VERSION = 3;
        constexpr static std::uint32_t MIN_VERSION = 2;

        Header();
        Header(size_t kmer_length, size_t window_length);
        void sanitize();
        bool check() const;

        bool operator==(const Header& another) const;
        bool operator!=(const Header& another) const { return !(this->operator==(another)); }
    };

//------------------------------------------------------------------------------

    /// Constructs an index with the default parameters.
    MinimizerIndex();

    /// Constructs an index with the specified parameter values.
    MinimizerIndex(size_t kmer_length, size_t window_length);

    /// Copy constructor.
    MinimizerIndex(const MinimizerIndex& source);

    /// Move constructor.
    MinimizerIndex(MinimizerIndex&& source);

    /// Destructor.
    ~MinimizerIndex();

    /// Swaps the contents of the indexes.
    void swap(MinimizerIndex& another);

    /// Copy assignment.
    MinimizerIndex& operator=(const MinimizerIndex& source);

    /// Move assignment.
    MinimizerIndex& operator=(MinimizerIndex&& source);

    /// Serialize the index to the ostream. Returns the number of bytes written and
    /// true if the serialization was successful.
    std::pair<size_t, bool> serialize(std::ostream& out) const;

    /// Load the index from the istream and return true if successful.
    bool load(std::istream& in);

    /// Equality comparison for testing.
    bool operator==(const MinimizerIndex& another) const;

    /// Inequality comparison for testing.
    bool operator!=(const MinimizerIndex& another) const { return !(this->operator==(another)); }

//------------------------------------------------------------------------------

    /// Returns the minimizer in the window specified by the iterators. If no minimizer
    /// exists (e.g. because all kmers contain invalid characters), the return value is
    /// an empty minimizer.
    minimizer_type minimizer(std::string::const_iterator begin, std::string::const_iterator end) const;

    /// Returns all minimizers in the string specified by the iterators. The return
    /// value is a vector of minimizers sorted by their offsets.
    std::vector<minimizer_type> minimizers(std::string::const_iterator begin, std::string::const_iterator end) const;

    /// Returns all minimizers in the string. The return value is a vector of
    /// minimizers sorted by their offsets.
    std::vector<minimizer_type> minimizers(const std::string& str) const {
        return this->minimizers(str.begin(), str.end());
    }

    /// Inserts the position into the index, using minimizer.key as the key and
    /// minimizer.hash as its hash. Does not insert empty minimizers or positions.
    /// The offset of the position will be truncated to fit in REV_OFFSET bits.
    /// Use minimizer() or minimizers() to get the minimizer and valid_offset() to check
    /// if the offset fits in the available space.
    /// The position should match the orientation of the minimizer: a path label
    /// starting from the position should have the minimizer as its prefix.
    void insert(const minimizer_type& minimizer, const pos_t& pos);

    /// Returns the sorted set of occurrences of the minimizer.
    /// Use minimizer() or minimizers() to get the minimizer.
    /// If the minimizer is in reverse orientation, use reverse_base_pos() to reverse
    /// the reported occurrences.
    std::vector<pos_t> find(const minimizer_type& minimizer) const;

    /// Returns the occurrence count of the minimizer.
    /// Use minimizer() or minimizers() to get the minimizer.
    size_t count(const minimizer_type& minimizer) const;

//------------------------------------------------------------------------------

    /// Length of the kmers in the index.
    size_t k() const { return this->header.k; }

    /// Window length for the minimizers.
    size_t w() const { return this->header.w; }

    /// Number of keys in the index.
    size_t size() const { return this->header.keys; }

    /// Is the index empty.
    bool empty() const { return (this->size() == 0); }

    /// Number of values (minimizer occurrences) in the index.
    size_t values() const { return this->header.values; }

    /// Size of the hash table.
    size_t capacity() const { return this->header.capacity; }

    /// Actual capacity of the hash table. Exceeding it will initiate rehashing.
    size_t max_keys() const { return this->header.max_keys; }

    /// Current load factor of the hash table.
    double load_factor() const { return static_cast<double>(this->size()) / static_cast<double>(this->capacity()); }

    /// Number of minimizers with a single occurrence.
    size_t unique_keys() const { return this->header.unique; }

//------------------------------------------------------------------------------

private:
    Header                 header;
    std::vector<cell_type> hash_table;
    std::vector<bool>      is_pointer;

//------------------------------------------------------------------------------

public:
    // Constants for the encoding between std::string and key_type.
    constexpr static size_t   PACK_WIDTH = 2;
    constexpr static key_type PACK_MASK  = 0x3;

    // Arrays for the encoding between std::string and key_type.
    const static std::vector<unsigned char> CHAR_TO_PACK;
    const static std::vector<char>          PACK_TO_CHAR;
    const static std::vector<key_type>      KMER_MASK;

    // Constants for the encoding between pos_t and code_type.
    constexpr static size_t    ID_OFFSET  = 11;
    constexpr static size_t    REV_OFFSET = 10;
    constexpr static code_type REV_MASK   = 0x400;
    constexpr static code_type OFF_MASK   = 0x3FF;

    /// Is the offset small enough to fit in the low-order bits of the encoding?
    static bool valid_offset(const pos_t& pos) {
        return (offset(pos) <= OFF_MASK);
    }

    /// Encode pos_t as code_type.
    static code_type encode(const pos_t& pos) {
        return (static_cast<code_type>(id(pos)) << ID_OFFSET) |
               (static_cast<code_type>(is_rev(pos)) << REV_OFFSET) |
               (static_cast<code_type>(offset(pos)) & OFF_MASK);
    }

    /// Decode code_type as pos_t.
    static pos_t decode(code_type pos) {
        return make_pos_t(pos >> ID_OFFSET, pos & REV_MASK, pos & OFF_MASK);
    }

//------------------------------------------------------------------------------

private:
    void copy(const MinimizerIndex& source);
    void clear(size_t i);   // Deletes the pointer at hash_table[i].
    void clear();           // Deletes all pointers in the hash table.

    // Find the hash table offset for the key with the given hash value.
    size_t find_offset(key_type key, size_t hash) const;

    // Insert (key, pos) to hash_table[offset], which is assumed to be empty.
    // Rehashing may be necessary.
    void insert(key_type key, code_type pos, size_t offset);

    // Add pos to the list of occurrences of key at hash_table[offset].
    // The occurrences may be deleted.
    void append(key_type key, code_type pos, size_t offset);

    // Does the list of occurrences at hash_table[offset] contain pos?
    bool contains(size_t offset, code_type pos) const;

    // Double the size of the hash table.
    void rehash();
};

//------------------------------------------------------------------------------

} // namespace vg

#endif // VG_MINIMIZER_HPP_INCLUDED
<|endoftext|>


<commit_before>/*
	Flexisip, a flexible SIP proxy server with media capabilities.
	Copyright (C) 2010-2015  Belledonne Communications SARL, All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as
	published by the Free Software Foundation, either version 3 of the
	License, or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "module.hh"
#include "agent.hh"
#include "log/logmanager.hh"
#include <sofia-sip/tport.h>
#include <sofia-sip/msg_addr.h>
#include <unordered_map>

using namespace std;

typedef struct DosContext {
	uint64_t recv_msg_count_since_last_check;
	double last_check_recv_msg_check_time;
	double packet_count_rate;
} DosContext;

class DoSProtection : public Module, ModuleToolbox {

  private:
	static ModuleInfo<DoSProtection> sInfo;
	int mTimePeriod;
	int mPacketRateLimit;
	int mBanTime;
	bool mAtChecked;
	list<string> mWhiteList;
	unordered_map<string, DosContext> mDosContexts;
	unordered_map<string, DosContext>::iterator mDOSHashtableIterator;

	void onDeclare(GenericStruct *module_config) {
		ConfigItemDescriptor configs[] = {
			{Integer, "time-period", "Number of milliseconds to consider to compute the packet rate", "3000"},
			{Integer, "packet-rate-limit", "Maximum packet rate in packets/seconds,  averaged over [time-period] "
										   "millisecond(s) to consider it as a DoS attack.",
			 "20"},
			{Integer, "ban-time", "Number of minutes to ban the ip/port using iptables (might be less because it justs "
								  "uses the minutes of the clock, not the seconds. So if the unban command is queued "
								  "at 13:11:56 and scheduled and the ban time is 1 minute, it will be executed at "
								  "13:12:00)",
			 "2"},
			config_item_end};
		module_config->get<ConfigBoolean>("enabled")->setDefault("true");
		module_config->addChildrenValues(configs);
	}

	void onLoad(const GenericStruct *mc) {
		mTimePeriod = mc->get<ConfigInt>("time-period")->read();
		mPacketRateLimit = mc->get<ConfigInt>("packet-rate-limit")->read();
		mBanTime = mc->get<ConfigInt>("ban-time")->read();
		mDOSHashtableIterator = mDosContexts.begin();

		GenericStruct *cluster = GenericManager::get()->getRoot()->get<GenericStruct>("cluster");
		mWhiteList = cluster->get<ConfigStringList>("nodes")->read();
		for (auto it = mWhiteList.begin(); it != mWhiteList.end(); ++it) {
			const char *white_ip = (*it).c_str();
			LOGI("IP %s is in DOS protection white list", white_ip);
		}
		LOGI("IP 127.0.0.1 automatically added to DOS protection white list");

		tport_t *primaries = tport_primaries(nta_agent_tports(mAgent->getSofiaAgent()));
		if (primaries == NULL)
			LOGF("No sip transport defined.");
		for (tport_t *tport = primaries; tport != NULL; tport = tport_next(tport)) {
			tport_set_params(tport, TPTAG_DOS(mTimePeriod), TAG_END());
		}
		if (getuid() != 0) {
			LOGE("Flexisip not started with root privileges! iptables commands for DoS protection won't work.");
			return;
		}
	}

	void onUnload() {
	}

	virtual bool isValidNextConfig( const ConfigValue &value ) {
		GenericStruct *module_config = dynamic_cast<GenericStruct *>(value.getParent());
		if (!module_config->get<ConfigBoolean>("enabled")->readNext())
			return true;
		else {

#if __APPLE__
			LOGEN("DosProtection only works on linux hosts. Please disable this module.");
			return false;
#else
			// we only want to check 'at' availability once
			if ( !mAtChecked ){
				mAtChecked = true;
				int at_command = system("which at > /dev/null");
				if( WIFEXITED(at_command) && WEXITSTATUS(at_command) == 0 ) {
					// at command was found, we can be sure that iptables rules will be cleaned up after the required time
					return true;
				} else {
					LOGEN("Couldn't find the commant 'at' in your PATH. DosProtection needs it to be used correctly. Please fix this or disable DosProtection.");
					return false;
				}
			}
			return true;
#endif
		}
	}

	void onIdle() {
		struct timeval now;
		double started_time_in_millis, time_elapsed;

		gettimeofday(&now, NULL);
		started_time_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);

		if (mDOSHashtableIterator == mDosContexts.end()) {
			mDOSHashtableIterator = mDosContexts.begin();
		}
		for (; mDOSHashtableIterator != mDosContexts.end();) {
			double now_in_millis;
			DosContext dos = mDOSHashtableIterator->second;

			gettimeofday(&now, NULL);
			now_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);
			time_elapsed = now_in_millis - dos.last_check_recv_msg_check_time;

			if (time_elapsed >= 3600 * 1000) { // If no message received in the past hour
				mDOSHashtableIterator = mDosContexts.erase(mDOSHashtableIterator);
			} else {
				++mDOSHashtableIterator;
			}

			if (now_in_millis - started_time_in_millis >= 100) { // Do not use more than 100ms to clean the hashtable
				LOGW("Started to clean dos hashtable %fms ago, let's stop for now a continue later",
					 now_in_millis - started_time_in_millis);
				break;
			}
		}
	}

	bool isIpWhiteListed(const char *ip) {
		if (!ip) return true; // If IP is null, is useless to try to add it in iptables...

		if (ip && strcmp(ip, "127.0.0.1") == 0) { // Never ban localhost, used for presence
			return true;
		}

		for (auto it = mWhiteList.begin(); it != mWhiteList.end(); ++it) { // Never ban ips from cluster
			const char *white_ip = (*it).c_str();
			if (white_ip && strcmp(ip, white_ip) == 0) {
				return true;
			}
		}

		return false;
	}

	static void ban_ip_with_iptables(const char *ip, const char *port, const char *protocol, int ban_time) {
		char iptables_cmd[512];
		snprintf(iptables_cmd, sizeof(iptables_cmd), "iptables -w -C INPUT -p %s -s %s -m multiport --sports %s -j DROP",
				 protocol, ip, port);
		if (system(iptables_cmd) == 0) {
			LOGW("IP %s port %s on protocol %s is already in the iptables banned list, skipping...", ip, port, protocol);
		} else {
			snprintf(iptables_cmd, sizeof(iptables_cmd), "iptables -w -A INPUT -p %s -s %s -m multiport --sports %s -j DROP"
					" && echo \"iptables -w -D INPUT -p %s -s %s -m multiport --sports %s -j DROP\" | at -M now +%i minutes",
					protocol, ip, port, protocol, ip, port, ban_time);
			if (system(iptables_cmd) != 0) {
				LOGW("iptables command failed: %s", strerror(errno));
			}
		}
	}

	void onRequest(shared_ptr<RequestSipEvent> &ev) throw (FlexisipException) {
		shared_ptr<tport_t> inTport = ev->getIncomingTport();
		tport_t *tport = inTport.get();

		if (tport == NULL) {
			LOGE("Tport is null, can't check the packet count rate");
			return;
		}

		if (tport_is_udp(tport)) { // Sofia doesn't create a secondary tport for udp, so it will ban the primary and we
								   // don't want that
			shared_ptr<MsgSip> msg = ev->getMsgSip();
			MsgSip *msgSip = msg.get();
			su_sockaddr_t su[1];
			socklen_t len = sizeof su;
			sockaddr *addr = NULL;
			char ip[NI_MAXHOST], port[NI_MAXSERV];
			int err;

			msg_get_address(msgSip->getMsg(), su, &len);
			addr = &(su[0].su_sa);

			if ((err = getnameinfo(addr, len, ip, sizeof(ip), port, sizeof(port), NI_NUMERICHOST | NI_NUMERICSERV)) ==
				0) {
				string id = string(ip) + ":" + string(port);
				struct timeval now;
				DosContext &dosContext = mDosContexts[id];
				double now_in_millis, time_elapsed;

				dosContext.recv_msg_count_since_last_check++;
				gettimeofday(&now, NULL);
				now_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);
				if (dosContext.last_check_recv_msg_check_time == 0) {
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				}

				time_elapsed = now_in_millis - dosContext.last_check_recv_msg_check_time;
				if (time_elapsed < 0) {
					dosContext.packet_count_rate = 0;
					dosContext.recv_msg_count_since_last_check = 0;
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				} else if (time_elapsed >= mTimePeriod) {
					dosContext.packet_count_rate = dosContext.recv_msg_count_since_last_check / time_elapsed * 1000;
					dosContext.recv_msg_count_since_last_check = 0;
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				}

				if (dosContext.packet_count_rate >= mPacketRateLimit) {
					LOGW("Packet count rate (%f) >= limit (%i), blocking ip/port %s/%s on protocol udp for %i minutes",
						 dosContext.packet_count_rate, mPacketRateLimit, ip, port, mBanTime);
					if (!isIpWhiteListed(ip)) {
						ban_ip_with_iptables(ip, port, "udp", mBanTime);
						ev->terminateProcessing(); // the event is discarded
					} else {
						LOGW("IP %s should be banned but wasn't because in white list", ip);
					}
					dosContext.packet_count_rate = 0; // Reset it to not add the iptables rule twice by mistake
				}
			} else {
				LOGW("getnameinfo() failed: %s", gai_strerror(err));
			}
		} else {
			float packet_count_rate = tport_get_packet_count_rate(tport);
			if (packet_count_rate >= mPacketRateLimit) {
				sockaddr *addr = tport_get_address(tport)->ai_addr;
				socklen_t len = tport_get_address(tport)->ai_addrlen;
				char ip[NI_MAXHOST], port[NI_MAXSERV];
				int err;

				if ((err = getnameinfo(addr, len, ip, sizeof(ip), port, sizeof(port),
									   NI_NUMERICHOST | NI_NUMERICSERV)) == 0) {
					LOGW("Packet count rate (%f) >= limit (%i), blocking ip/port %s/%s on protocol tcp for %i minutes",
						 packet_count_rate, mPacketRateLimit, ip, port, mBanTime);
					if (!isIpWhiteListed(ip)) {
						ban_ip_with_iptables(ip, port, "tcp", mBanTime);
						ev->terminateProcessing(); // the event is discarded
					} else {
						LOGW("IP %s should be banned but wasn't because in white list", ip);
					}
					tport_reset_packet_count_rate(tport); // Reset it to not add the iptables rule twice by mistake
				} else {
					LOGW("getnameinfo() failed: %s", gai_strerror(err));
				}
			}
		}
	}

	void onResponse(std::shared_ptr<ResponseSipEvent> &ev) throw (FlexisipException){

	};

  public:
	DoSProtection(Agent *ag) : Module(ag) {
		mAtChecked = false;
	}

	~DoSProtection() {
	}
};

ModuleInfo<DoSProtection>
	DoSProtection::sInfo("DoSProtection",
						 "This module bans user when they are sending too much packets on a given timelapse"
						 "To see the list of currently banned ips/ports, use iptables -L"
						 "You can also check the queue of unban commands using atq",
						 ModuleInfoBase::ModuleOid::DoSProtection);
<commit_msg>Check if iptables does support -w parameter. If not, return an error<commit_after>/*
	Flexisip, a flexible SIP proxy server with media capabilities.
	Copyright (C) 2010-2015  Belledonne Communications SARL, All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as
	published by the Free Software Foundation, either version 3 of the
	License, or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "module.hh"
#include "agent.hh"
#include "log/logmanager.hh"
#include <sofia-sip/tport.h>
#include <sofia-sip/msg_addr.h>
#include <unordered_map>

using namespace std;

typedef struct DosContext {
	uint64_t recv_msg_count_since_last_check;
	double last_check_recv_msg_check_time;
	double packet_count_rate;
} DosContext;

class DoSProtection : public Module, ModuleToolbox {

  private:
	static ModuleInfo<DoSProtection> sInfo;
	int mTimePeriod;
	int mPacketRateLimit;
	int mBanTime;
	bool mAtChecked;
	bool mIptablesVersionChecked;
	list<string> mWhiteList;
	unordered_map<string, DosContext> mDosContexts;
	unordered_map<string, DosContext>::iterator mDOSHashtableIterator;

	void onDeclare(GenericStruct *module_config) {
		ConfigItemDescriptor configs[] = {
			{Integer, "time-period", "Number of milliseconds to consider to compute the packet rate", "3000"},
			{Integer, "packet-rate-limit", "Maximum packet rate in packets/seconds,  averaged over [time-period] "
										   "millisecond(s) to consider it as a DoS attack.",
			 "20"},
			{Integer, "ban-time", "Number of minutes to ban the ip/port using iptables (might be less because it justs "
								  "uses the minutes of the clock, not the seconds. So if the unban command is queued "
								  "at 13:11:56 and scheduled and the ban time is 1 minute, it will be executed at "
								  "13:12:00)",
			 "2"},
			config_item_end};
		module_config->get<ConfigBoolean>("enabled")->setDefault("true");
		module_config->addChildrenValues(configs);
	}

	void onLoad(const GenericStruct *mc) {
		mTimePeriod = mc->get<ConfigInt>("time-period")->read();
		mPacketRateLimit = mc->get<ConfigInt>("packet-rate-limit")->read();
		mBanTime = mc->get<ConfigInt>("ban-time")->read();
		mDOSHashtableIterator = mDosContexts.begin();

		GenericStruct *cluster = GenericManager::get()->getRoot()->get<GenericStruct>("cluster");
		mWhiteList = cluster->get<ConfigStringList>("nodes")->read();
		for (auto it = mWhiteList.begin(); it != mWhiteList.end(); ++it) {
			const char *white_ip = (*it).c_str();
			LOGI("IP %s is in DOS protection white list", white_ip);
		}
		LOGI("IP 127.0.0.1 automatically added to DOS protection white list");

		tport_t *primaries = tport_primaries(nta_agent_tports(mAgent->getSofiaAgent()));
		if (primaries == NULL)
			LOGF("No sip transport defined.");
		for (tport_t *tport = primaries; tport != NULL; tport = tport_next(tport)) {
			tport_set_params(tport, TPTAG_DOS(mTimePeriod), TAG_END());
		}
		if (getuid() != 0) {
			LOGE("Flexisip not started with root privileges! iptables commands for DoS protection won't work.");
			return;
		}
	}

	void onUnload() {
	}

	virtual bool isValidNextConfig( const ConfigValue &value ) {
		GenericStruct *module_config = dynamic_cast<GenericStruct *>(value.getParent());
		if (!module_config->get<ConfigBoolean>("enabled")->readNext())
			return true;
		else {

#if __APPLE__
			LOGEN("DosProtection only works on linux hosts. Please disable this module.");
			return false;
#else
			// we only want to check 'at' availability once
			if ( !mAtChecked ){
				mAtChecked = true;
				int at_command = system("which at > /dev/null");
				if( WIFEXITED(at_command) && WEXITSTATUS(at_command) == 0 ) {
					// at command was found, we can be sure that iptables rules will be cleaned up after the required time
				} else {
					LOGEN("Couldn't find the commant 'at' in your PATH. DosProtection needs it to be used correctly. Please fix this or disable DosProtection.");
					return false;
				}
			}
			if (!mIptablesVersionChecked) {
				mIptablesVersionChecked = true;
				int iptables_command = system("iptables -w -V > /dev/null");
				if (WIFEXITED(iptables_command) && WEXITSTATUS(iptables_command) == 0) {
					// iptables seems to support -w parameter required to allow concurrent usage of iptables
				} else {
					LOGEN("IPtables' -w parameter not understood, which probably means you have a version older than 1.4.20. Please fix this or disable DosProtection.");
					return false;
				}
			}
			return true;
#endif
		}
	}

	void onIdle() {
		struct timeval now;
		double started_time_in_millis, time_elapsed;

		gettimeofday(&now, NULL);
		started_time_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);

		if (mDOSHashtableIterator == mDosContexts.end()) {
			mDOSHashtableIterator = mDosContexts.begin();
		}
		for (; mDOSHashtableIterator != mDosContexts.end();) {
			double now_in_millis;
			DosContext dos = mDOSHashtableIterator->second;

			gettimeofday(&now, NULL);
			now_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);
			time_elapsed = now_in_millis - dos.last_check_recv_msg_check_time;

			if (time_elapsed >= 3600 * 1000) { // If no message received in the past hour
				mDOSHashtableIterator = mDosContexts.erase(mDOSHashtableIterator);
			} else {
				++mDOSHashtableIterator;
			}

			if (now_in_millis - started_time_in_millis >= 100) { // Do not use more than 100ms to clean the hashtable
				LOGW("Started to clean dos hashtable %fms ago, let's stop for now a continue later",
					 now_in_millis - started_time_in_millis);
				break;
			}
		}
	}

	bool isIpWhiteListed(const char *ip) {
		if (!ip) return true; // If IP is null, is useless to try to add it in iptables...

		if (ip && strcmp(ip, "127.0.0.1") == 0) { // Never ban localhost, used for presence
			return true;
		}

		for (auto it = mWhiteList.begin(); it != mWhiteList.end(); ++it) { // Never ban ips from cluster
			const char *white_ip = (*it).c_str();
			if (white_ip && strcmp(ip, white_ip) == 0) {
				return true;
			}
		}

		return false;
	}

	static void ban_ip_with_iptables(const char *ip, const char *port, const char *protocol, int ban_time) {
		char iptables_cmd[512];
		snprintf(iptables_cmd, sizeof(iptables_cmd), "iptables -w -C INPUT -p %s -s %s -m multiport --sports %s -j DROP",
				 protocol, ip, port);
		if (system(iptables_cmd) == 0) {
			LOGW("IP %s port %s on protocol %s is already in the iptables banned list, skipping...", ip, port, protocol);
		} else {
			snprintf(iptables_cmd, sizeof(iptables_cmd), "iptables -w -A INPUT -p %s -s %s -m multiport --sports %s -j DROP"
					" && echo \"iptables -w -D INPUT -p %s -s %s -m multiport --sports %s -j DROP\" | at -M now +%i minutes",
					protocol, ip, port, protocol, ip, port, ban_time);
			if (system(iptables_cmd) != 0) {
				LOGW("iptables command failed: %s", strerror(errno));
			}
		}
	}

	void onRequest(shared_ptr<RequestSipEvent> &ev) throw (FlexisipException) {
		shared_ptr<tport_t> inTport = ev->getIncomingTport();
		tport_t *tport = inTport.get();

		if (tport == NULL) {
			LOGE("Tport is null, can't check the packet count rate");
			return;
		}

		if (tport_is_udp(tport)) { // Sofia doesn't create a secondary tport for udp, so it will ban the primary and we
								   // don't want that
			shared_ptr<MsgSip> msg = ev->getMsgSip();
			MsgSip *msgSip = msg.get();
			su_sockaddr_t su[1];
			socklen_t len = sizeof su;
			sockaddr *addr = NULL;
			char ip[NI_MAXHOST], port[NI_MAXSERV];
			int err;

			msg_get_address(msgSip->getMsg(), su, &len);
			addr = &(su[0].su_sa);

			if ((err = getnameinfo(addr, len, ip, sizeof(ip), port, sizeof(port), NI_NUMERICHOST | NI_NUMERICSERV)) ==
				0) {
				string id = string(ip) + ":" + string(port);
				struct timeval now;
				DosContext &dosContext = mDosContexts[id];
				double now_in_millis, time_elapsed;

				dosContext.recv_msg_count_since_last_check++;
				gettimeofday(&now, NULL);
				now_in_millis = now.tv_sec * 1000 + (now.tv_usec / 1000);
				if (dosContext.last_check_recv_msg_check_time == 0) {
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				}

				time_elapsed = now_in_millis - dosContext.last_check_recv_msg_check_time;
				if (time_elapsed < 0) {
					dosContext.packet_count_rate = 0;
					dosContext.recv_msg_count_since_last_check = 0;
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				} else if (time_elapsed >= mTimePeriod) {
					dosContext.packet_count_rate = dosContext.recv_msg_count_since_last_check / time_elapsed * 1000;
					dosContext.recv_msg_count_since_last_check = 0;
					dosContext.last_check_recv_msg_check_time = now_in_millis;
				}

				if (dosContext.packet_count_rate >= mPacketRateLimit) {
					LOGW("Packet count rate (%f) >= limit (%i), blocking ip/port %s/%s on protocol udp for %i minutes",
						 dosContext.packet_count_rate, mPacketRateLimit, ip, port, mBanTime);
					if (!isIpWhiteListed(ip)) {
						ban_ip_with_iptables(ip, port, "udp", mBanTime);
						ev->terminateProcessing(); // the event is discarded
					} else {
						LOGW("IP %s should be banned but wasn't because in white list", ip);
					}
					dosContext.packet_count_rate = 0; // Reset it to not add the iptables rule twice by mistake
				}
			} else {
				LOGW("getnameinfo() failed: %s", gai_strerror(err));
			}
		} else {
			float packet_count_rate = tport_get_packet_count_rate(tport);
			if (packet_count_rate >= mPacketRateLimit) {
				sockaddr *addr = tport_get_address(tport)->ai_addr;
				socklen_t len = tport_get_address(tport)->ai_addrlen;
				char ip[NI_MAXHOST], port[NI_MAXSERV];
				int err;

				if ((err = getnameinfo(addr, len, ip, sizeof(ip), port, sizeof(port),
									   NI_NUMERICHOST | NI_NUMERICSERV)) == 0) {
					LOGW("Packet count rate (%f) >= limit (%i), blocking ip/port %s/%s on protocol tcp for %i minutes",
						 packet_count_rate, mPacketRateLimit, ip, port, mBanTime);
					if (!isIpWhiteListed(ip)) {
						ban_ip_with_iptables(ip, port, "tcp", mBanTime);
						ev->terminateProcessing(); // the event is discarded
					} else {
						LOGW("IP %s should be banned but wasn't because in white list", ip);
					}
					tport_reset_packet_count_rate(tport); // Reset it to not add the iptables rule twice by mistake
				} else {
					LOGW("getnameinfo() failed: %s", gai_strerror(err));
				}
			}
		}
	}

	void onResponse(std::shared_ptr<ResponseSipEvent> &ev) throw (FlexisipException){

	};

  public:
	DoSProtection(Agent *ag) : Module(ag) {
		mAtChecked = false;
		mIptablesVersionChecked = false;
	}

	~DoSProtection() {
	}
};

ModuleInfo<DoSProtection>
	DoSProtection::sInfo("DoSProtection",
						 "This module bans user when they are sending too much packets on a given timelapse"
						 "To see the list of currently banned ips/ports, use iptables -L"
						 "You can also check the queue of unban commands using atq",
						 ModuleInfoBase::ModuleOid::DoSProtection);
<|endoftext|>


<commit_before>#include "common.hpp"

extern "C"
void __serial_print1(const char*);

template<typename R, typename ...Args>
inline void stubtrace_print(const char* name, R ret, Args&&... args) {
  __serial_print1("<WARNING> Stubbed syscall: ");
  strace_print(name, ret, args...);
}

// Strace for stubbed syscalls.
// calling the syscall, recording return value and always printing
template<typename Fn, typename ...Args>
inline auto stubtrace(Fn func, const char* name, Args&&... args) {
  auto ret = func(args...);
  stubtrace_print(name, ret, args...);
  return ret;
}
<commit_msg>musl: Only print stubbed syscalls if strace is enabled<commit_after>#include "common.hpp"

extern "C"
void __serial_print1(const char*);

template<typename R, typename ...Args>
inline void stubtrace_print(const char* name, R ret, Args&&... args) {
  __serial_print1("<WARNING> Stubbed syscall: ");
  strace_print(name, ret, args...);
}

// Strace for stubbed syscalls.
// calling the syscall, recording return value and only printing when strace is on
template<typename Fn, typename ...Args>
inline auto stubtrace(Fn func, const char* name[[maybe_unused]], Args&&... args) {
  auto ret = func(args...);

  if constexpr (__strace)
    stubtrace_print(name, ret, args...);

  return ret;
}
<|endoftext|>


<commit_before>#include "ofxJamoma.h"
#include "gorgone.h"

void ofxJamoma::setup(void* parent, string name, string masterIp)
{
  mParent = TTPtr(parent);
  mAppLocalName = name;
  mAppRemoteName = "master";
  mAppRemoteName2 = "laser-1";
  mAppRemoteIp = masterIp;
  setupJamomaApp();
  registerJamomaParam();
}

void ofxJamoma::exit()
{
}

void ofxJamoma::setupJamomaApp(){
    TTValue     args, v, out, none;
    TTAddress   address;
    TTErr       err;

    TTLogMessage("\n*** Initialisation of Modular environnement ***\n");
    /////////////////////////////////////////////////////////////////////

    // Init the Modular library (passing the folder path where all the dylibs are)
    TTModularInit("/usr/local/jamoma/lib/jamoma/");

    // Create an application manager
    mApplicationManager = TTObject("ApplicationManager");

    TTLogMessage("\n*** Creation of %s and %s applications ***\n", mAppLocalName.c_str(), mAppRemoteName.c_str());
    ////////////////////////////////////////////////////////////////////////////////

    // Create a local application called "gorgone-1" and get it back
    mApplicationLocal = mApplicationManager.send("ApplicationInstantiateLocal", mAppLocalName.c_str());
    mApplicationRemote = mApplicationManager.send("ApplicationInstantiateDistant", mAppRemoteName.c_str());
    mApplicationRemote2 = mApplicationManager.send("ApplicationInstantiateDistant", mAppRemoteName2.c_str());

    // Get registered application names
    mApplicationManager.get("applicationNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTLogMessage("%s application is well registered into the application manager \n", name.c_str());
    }

    TTLogMessage("\n*** Enable Minuit communication ***\n");
    ////////////////////////////////////////////////////////////////////////

    // Create a Minuit protocol unit
    err = mApplicationManager.send("ProtocolInstantiate", "Minuit", out);

    if (err) {
        TTLogError("Error : can't create Minuit protocol unit \n");
        return;
    }
    else
        mProtocolMinuit = out[0];

    // Get Minuit Protocol attribute names and types
    mProtocolMinuit.get("parameterNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTSymbol type = mProtocolMinuit.attributeType(name);
        TTLogMessage("Minuit %s parameter is a %s \n", name.c_str(), type.c_str());
    }

    // Register mymyRemoteAppApp and myRemoteApp to the Minuit protocol
    mProtocolMinuit.send("ApplicationRegister", mAppLocalName.c_str(), out);
    mProtocolMinuit.send("ApplicationRegister", mAppRemoteName.c_str(), out);
    mProtocolMinuit.send("ApplicationRegister", mAppRemoteName2.c_str(), out);

    // Select gorgone-1 to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppLocalName.c_str(), out);
    mProtocolMinuit.set("port", 9998);
    mProtocolMinuit.set("ip", "127.0.0.1");

    // Select gorgone-1 to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppRemoteName2.c_str(), out);
    mProtocolMinuit.set("port", 9999);
    mProtocolMinuit.set("ip", "127.0.0.1");

    // Select myRemoteApp to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppRemoteName.c_str(), out);
    mProtocolMinuit.set("port", 13579);
    mProtocolMinuit.set("ip", mAppRemoteIp.c_str());

    // Get Minuit parameters for each registered application
    mProtocolMinuit.get("applicationNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);

        mProtocolMinuit.send("ApplicationSelect", name, out);
        TTLogMessage("Minuit setup for %s application : \n", name.c_str());

        mProtocolMinuit.get("ip", v);
        TTSymbol ip = v[0];
        TTLogMessage("- ip = %s \n", ip.c_str());

        mProtocolMinuit.get("port", v);
        TTUInt16 port = v[0];
        TTLogMessage("- port = %d \n", port);
    }

    // Enable Minuit communication
    mProtocolMinuit.send("Run");

    TTLogMessage("\n*** Current Protocol Setup ***\n");
    // Get protocol names
    mApplicationManager.get("protocolNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTLogMessage("%s protocol is well registered into the application manager \n", name.c_str());
    }
}

void ofxJamoma::registerJamomaParam(){
    TTValue     args, v, out, none;
    TTAddress   address;
    TTErr       err;
      TTLogMessage("\n*** Creation and registration of a decimal parameter ***\n");
    /////////////////////////////////////////////////////////

    // Create a decimal parameter data and set its callback function and baton and some attributes
    mTrackEnableParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackEnableParameter);
    mTrackEnableParameter.set("baton", args);
    mTrackEnableParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackEnableParameter.set("type", "boolean");
    mTrackEnableParameter.set("description", "start/stop tracking");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/enable", mTrackEnableParameter);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /tracking/enable address \n");

    else {
        address = out[0];
        TTLogMessage("\n /tracking/enable : effective registration address is %s \n", address.c_str());
    }

        // Create a decimal parameter data and set its callback function and baton and some attributes
    mDrawingEnableParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingEnableParameter);
    mDrawingEnableParameter.set("baton", args);
    mDrawingEnableParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingEnableParameter.set("type", "boolean");
    mDrawingEnableParameter.set("description", "start/stop drawing");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/enable", mDrawingEnableParameter);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /drawing/enable address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/enable : effective registration address is %s \n", address.c_str());
    }

    // Create a new boolean parameter
    mComputeIrisCodeParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mComputeIrisCodeParameter);
    mComputeIrisCodeParameter.set("baton", args);
    mComputeIrisCodeParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mComputeIrisCodeParameter.set("type", "boolean");
    mComputeIrisCodeParameter.set("description", "start/stop drawing");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/computecode", mComputeIrisCodeParameter);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /tracking/computecode address \n");

    else {
        address = out[0];
        TTLogMessage("\n /tracking/computecode : effective registration address is %s \n", address.c_str());
    }

    // Create a new array parameter
    mDrawingCoeffParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingCoeffParameter);
    mDrawingCoeffParameter.set("baton", args);
    mDrawingCoeffParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingCoeffParameter.set("type", "array");
    mDrawingCoeffParameter.set("description", "shape coefficient");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/coeff", mDrawingCoeffParameter);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /drawing/coeff address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/coeff : effective registration address is %s \n", address.c_str());
    }

    // Create a new array return for iris code
    mTrackingIrisCodeReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingIrisCodeReturn);
    mTrackingIrisCodeReturn.set("baton", args);
    mTrackingIrisCodeReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingIrisCodeReturn.set("type", "array");
    mTrackingIrisCodeReturn.set("description", "iris code");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/iriscode", mTrackingIrisCodeReturn);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /tracking/iriscode address \n");

    else {
        address = out[0];
        TTLogMessage("\n /tracking/iriscode : effective registration address is %s \n", address.c_str());
    }

        // Create a new array return for drawing shape
    mDrawingShapeXReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingShapeXReturn);
    mDrawingShapeXReturn.set("baton", args);
    mDrawingShapeXReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingShapeXReturn.set("type", "array");
    mDrawingShapeXReturn.set("description", "drawing shape X");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/shape/x", mDrawingShapeXReturn);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /drawing/shape/x address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/shape/x : effective registration address is %s \n", address.c_str());
    }

            // Create a new array return for drawing shape
    mDrawingShapeYReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingShapeYReturn);
    mDrawingShapeYReturn.set("baton", args);
    mDrawingShapeYReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingShapeYReturn.set("type", "array");
    mDrawingShapeYReturn.set("description", "Drawing shape Y");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/shape/y", mDrawingShapeYReturn);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /drawing/shape/y address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/shape/y : effective registration address is %s \n", address.c_str());
    }

    // Create a new brightness parameter
    mTrackingLedBrightness = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingLedBrightness);
    mTrackingLedBrightness.set("baton", args);
    mTrackingLedBrightness.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingLedBrightness.set("type", "integer");
    mTrackingLedBrightness.set("description", "LED brightness");
    mTrackingLedBrightness.set("rangeBounds", TTValue(0, 1023));
    mTrackingLedBrightness.set("rangeClipmode", "both");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/ledbrightness", mTrackingLedBrightness);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /tracking/ledbrightness address \n");

    else {
        address = out[0];
        TTLogMessage("\n /tracking/ledbrightness : effective registration address is %s \n", address.c_str());
    }

    // Create a new brightness parameter
    mTrackingLaserBrightness = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingLaserBrightness);
    mTrackingLaserBrightness.set("baton", args);
    mTrackingLaserBrightness.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingLaserBrightness.set("type", "decimal");
    mTrackingLaserBrightness.set("description", "laser brightness");
    mTrackingLaserBrightness.set("rangeBounds", TTValue(0., 100.));
    mTrackingLaserBrightness.set("rangeClipmode", "both");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/laserbrightness", mTrackingLaserBrightness);
    err = mApplicationLocal.send("ObjectRegister", args, out);

    if (err)
        TTLogError("Error : can't register data at /drawing/laserbrightness address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/laserbrightness : effective registration address is %s \n", address.c_str());
    }
}

TTErr
DemoAppDataReturnValueCallback(const TTValue& baton, const TTValue& value)
{
    gorgone*  gorgoneApp = (gorgone*)TTPtr(baton[0]);
    TTObject    anObject = baton[1];

  // Reteive which data has been updated
    if (anObject.instance() == gorgoneApp->jamoma.mTrackEnableParameter.instance()) {

        gorgoneApp->bTracking = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mDrawingEnableParameter.instance()) {
        gorgoneApp->bDisplaying = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mComputeIrisCodeParameter.instance()) {
        gorgoneApp->bComputeCode = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mDrawingCoeffParameter.instance()) {
        gorgoneApp->svgInterp.coeff.clear();
        for (int i = 0; i < value.size(); i++)
            gorgoneApp->svgInterp.coeff.push_back(value[i]);
        gorgoneApp->svgInterp.multiInterpolation();

        TTValue x,y;
        vector<ofVec3f> line = gorgoneApp->svgInterp.interpolatedLine;

        for (int i=0; i<line.size(); i++){
          x.push_back(line[i].x);
          y.push_back(line[i].y);
        }

        gorgoneApp->jamoma.mDrawingShapeXReturn.set("value", x);
        gorgoneApp->jamoma.mDrawingShapeYReturn.set("value", y);

        return kTTErrNone;
    }

#ifdef TARGET_RASPBERRY_PI
    if (anObject.instance() == gorgoneApp->jamoma.mTrackingLedBrightness.instance()) {
        gorgoneApp->vidGrabber.led.setBrightness(value[0]);
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mTrackingLaserBrightness.instance()) {
        gorgoneApp->setPwm(value[0]);
        return kTTErrNone;
    }

#endif

    return kTTErrGeneric;
}
<commit_msg>remove deprecated stuff<commit_after>#include "ofxJamoma.h"
#include "gorgone.h"

void ofxJamoma::setup(void* parent, string name, string masterIp)
{
  mParent = TTPtr(parent);
  mAppLocalName = name;
  mAppRemoteName = "master";
  mAppRemoteName2 = "laser-1";
  mAppRemoteIp = masterIp;
  setupJamomaApp();
  registerJamomaParam();
}

void ofxJamoma::exit()
{
}

void ofxJamoma::setupJamomaApp(){
    TTValue     args, v, out, none;
    TTAddress   address;
    TTErr       err;

    TTLogMessage("\n*** Initialisation of Modular environnement ***\n");
    /////////////////////////////////////////////////////////////////////

    // Init the Modular library (passing the folder path where all the dylibs are)
    TTModularInit("/usr/local/jamoma/lib/jamoma/");

    // Create an application manager
    mApplicationManager = TTObject("ApplicationManager");

    TTLogMessage("\n*** Creation of %s and %s applications ***\n", mAppLocalName.c_str(), mAppRemoteName.c_str());
    ////////////////////////////////////////////////////////////////////////////////

    // Create a local application called "gorgone-1" and get it back
    mApplicationLocal = mApplicationManager.send("ApplicationInstantiateLocal", mAppLocalName.c_str());
    mApplicationRemote = mApplicationManager.send("ApplicationInstantiateDistant", mAppRemoteName.c_str());
    mApplicationRemote2 = mApplicationManager.send("ApplicationInstantiateDistant", mAppRemoteName2.c_str());

    // Get registered application names
    mApplicationManager.get("applicationNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTLogMessage("%s application is well registered into the application manager \n", name.c_str());
    }

    TTLogMessage("\n*** Enable Minuit communication ***\n");
    ////////////////////////////////////////////////////////////////////////

    // Create a Minuit protocol unit
    mProtocolMinuit = mApplicationManager.send("ProtocolInstantiate", "Minuit");

    // Get Minuit Protocol attribute names and types
    mProtocolMinuit.get("parameterNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTSymbol type = mProtocolMinuit.attributeType(name);
        TTLogMessage("Minuit %s parameter is a %s \n", name.c_str(), type.c_str());
    }

    // Register mymyRemoteAppApp and myRemoteApp to the Minuit protocol
    mProtocolMinuit.send("ApplicationRegister", mAppLocalName.c_str());
    mProtocolMinuit.send("ApplicationRegister", mAppRemoteName.c_str());
    mProtocolMinuit.send("ApplicationRegister", mAppRemoteName2.c_str());

    // Select gorgone-1 to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppLocalName.c_str());
    mProtocolMinuit.set("port", 9998);
    mProtocolMinuit.set("ip", "127.0.0.1");

    // Select gorgone-1 to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppRemoteName2.c_str());
    mProtocolMinuit.set("port", 9999);
    mProtocolMinuit.set("ip", "127.0.0.1");

    // Select myRemoteApp to set its protocol parameters
    mProtocolMinuit.send("ApplicationSelect", mAppRemoteName.c_str());
    mProtocolMinuit.set("port", 13579);
    mProtocolMinuit.set("ip", mAppRemoteIp.c_str());

    // Get Minuit parameters for each registered application
    mProtocolMinuit.get("applicationNames", out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);

        mProtocolMinuit.send("ApplicationSelect", name);
        TTLogMessage("Minuit setup for %s application : \n", name.c_str());

        mProtocolMinuit.get("ip", v);
        TTSymbol ip = v[0];
        TTLogMessage("- ip = %s \n", ip.c_str());

        mProtocolMinuit.get("port", v);
        TTUInt16 port = v[0];
        TTLogMessage("- port = %d \n", port);
    }

    // Enable Minuit communication
    mProtocolMinuit.send("Run");

    TTLogMessage("\n*** Current Protocol Setup ***\n");
    // Get protocol names
    mApplicationManager.get("protocolNames",out);
    for (TTElementIter it = out.begin() ; it != out.end() ; it++) {
        TTSymbol name = TTElement(*it);
        TTLogMessage("%s protocol is well registered into the application manager \n", name.c_str());
    }
}

void ofxJamoma::registerJamomaParam(){
    TTValue     args, v, out, none;
    TTAddress   address;
    TTErr       err;
      TTLogMessage("\n*** Creation and registration of a decimal parameter ***\n");
    /////////////////////////////////////////////////////////

    // Create a decimal parameter data and set its callback function and baton and some attributes
    mTrackEnableParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackEnableParameter);
    mTrackEnableParameter.set("baton", args);
    mTrackEnableParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackEnableParameter.set("type", "boolean");
    mTrackEnableParameter.set("description", "start/stop tracking");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/enable", mTrackEnableParameter);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /tracking/enable : effective registration address is %s \n", address.c_str());

        // Create a decimal parameter data and set its callback function and baton and some attributes
    mDrawingEnableParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingEnableParameter);
    mDrawingEnableParameter.set("baton", args);
    mDrawingEnableParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingEnableParameter.set("type", "boolean");
    mDrawingEnableParameter.set("description", "start/stop drawing");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/enable", mDrawingEnableParameter);
    out = mApplicationLocal.send("ObjectRegister", args);

    address = out[0];
    TTLogMessage("\n /drawing/enable : effective registration address is %s \n", address.c_str());

    // Create a new boolean parameter
    mComputeIrisCodeParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mComputeIrisCodeParameter);
    mComputeIrisCodeParameter.set("baton", args);
    mComputeIrisCodeParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mComputeIrisCodeParameter.set("type", "boolean");
    mComputeIrisCodeParameter.set("description", "start/stop drawing");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/computecode", mComputeIrisCodeParameter);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /tracking/computecode : effective registration address is %s \n", address.c_str());

    // Create a new array parameter
    mDrawingCoeffParameter = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingCoeffParameter);
    mDrawingCoeffParameter.set("baton", args);
    mDrawingCoeffParameter.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingCoeffParameter.set("type", "array");
    mDrawingCoeffParameter.set("description", "shape coefficient");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/coeff", mDrawingCoeffParameter);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /drawing/coeff : effective registration address is %s \n", address.c_str());

    // Create a new array return for iris code
    mTrackingIrisCodeReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingIrisCodeReturn);
    mTrackingIrisCodeReturn.set("baton", args);
    mTrackingIrisCodeReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingIrisCodeReturn.set("type", "array");
    mTrackingIrisCodeReturn.set("description", "iris code");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/iriscode", mTrackingIrisCodeReturn);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /tracking/iriscode : effective registration address is %s \n", address.c_str());

        // Create a new array return for drawing shape
    mDrawingShapeXReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingShapeXReturn);
    mDrawingShapeXReturn.set("baton", args);
    mDrawingShapeXReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingShapeXReturn.set("type", "array");
    mDrawingShapeXReturn.set("description", "drawing shape X");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/shape/x", mDrawingShapeXReturn);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /drawing/shape/x : effective registration address is %s \n", address.c_str());

    // Create a new array return for drawing shape
    mDrawingShapeYReturn = TTObject("Data", "return");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mDrawingShapeYReturn);
    mDrawingShapeYReturn.set("baton", args);
    mDrawingShapeYReturn.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mDrawingShapeYReturn.set("type", "array");
    mDrawingShapeYReturn.set("description", "Drawing shape Y");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/shape/y", mDrawingShapeYReturn);
    out = mApplicationLocal.send("ObjectRegister", args);

    if (err)
        TTLogError("Error : can't register data at /drawing/shape/y address \n");

    else {
        address = out[0];
        TTLogMessage("\n /drawing/shape/y : effective registration address is %s \n", address.c_str());
    }

    // Create a new brightness parameter
    mTrackingLedBrightness = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingLedBrightness);
    mTrackingLedBrightness.set("baton", args);
    mTrackingLedBrightness.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingLedBrightness.set("type", "integer");
    mTrackingLedBrightness.set("description", "LED brightness");
    mTrackingLedBrightness.set("rangeBounds", TTValue(0, 1023));
    mTrackingLedBrightness.set("rangeClipmode", "both");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/tracking/ledbrightness", mTrackingLedBrightness);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /tracking/ledbrightness : effective registration address is %s \n", address.c_str());

    // Create a new brightness parameter
    mTrackingLaserBrightness = TTObject("Data", "parameter");

    // Setup the callback mechanism to get the value back
    args = TTValue(mParent, mTrackingLaserBrightness);
    mTrackingLaserBrightness.set("baton", args);
    mTrackingLaserBrightness.set("function", TTPtr(&DemoAppDataReturnValueCallback));

    // Setup the data attributes depending of its use inside the application
    mTrackingLaserBrightness.set("type", "decimal");
    mTrackingLaserBrightness.set("description", "laser brightness");
    mTrackingLaserBrightness.set("rangeBounds", TTValue(0., 100.));
    mTrackingLaserBrightness.set("rangeClipmode", "both");

    // Register the parameter data into gorgone-1 at an address
    args = TTValue("/drawing/laserbrightness", mTrackingLaserBrightness);
    out = mApplicationLocal.send("ObjectRegister", args);
    address = out[0];
    TTLogMessage("\n /drawing/laserbrightness : effective registration address is %s \n", address.c_str());
}

TTErr
DemoAppDataReturnValueCallback(const TTValue& baton, const TTValue& value)
{
    gorgone*  gorgoneApp = (gorgone*)TTPtr(baton[0]);
    TTObject    anObject = baton[1];

  // Reteive which data has been updated
    if (anObject.instance() == gorgoneApp->jamoma.mTrackEnableParameter.instance()) {

        gorgoneApp->bTracking = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mDrawingEnableParameter.instance()) {
        gorgoneApp->bDisplaying = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mComputeIrisCodeParameter.instance()) {
        gorgoneApp->bComputeCode = value[0];
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mDrawingCoeffParameter.instance()) {
        gorgoneApp->svgInterp.coeff.clear();
        for (int i = 0; i < value.size(); i++)
            gorgoneApp->svgInterp.coeff.push_back(value[i]);
        gorgoneApp->svgInterp.multiInterpolation();

        TTValue x,y;
        vector<ofVec3f> line = gorgoneApp->svgInterp.interpolatedLine;

        for (int i=0; i<line.size(); i++){
          x.push_back(line[i].x);
          y.push_back(line[i].y);
        }

        gorgoneApp->jamoma.mDrawingShapeXReturn.set("value", x);
        gorgoneApp->jamoma.mDrawingShapeYReturn.set("value", y);

        return kTTErrNone;
    }

#ifdef TARGET_RASPBERRY_PI
    if (anObject.instance() == gorgoneApp->jamoma.mTrackingLedBrightness.instance()) {
        gorgoneApp->vidGrabber.led.setBrightness(value[0]);
        return kTTErrNone;
    }

    if (anObject.instance() == gorgoneApp->jamoma.mTrackingLaserBrightness.instance()) {
        gorgoneApp->setPwm(value[0]);
        return kTTErrNone;
    }

#endif

    return kTTErrGeneric;
}
<|endoftext|>