<commit_before>//
// Copyright (c) 2014,2015,2016,2018 CNRS
// Authors: Florent Lamiraux, Joseph Mirabel, Diane Bury
//
// This file is part of hpp-core
// hpp-core is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-core is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-core  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH
#define HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH

#include <limits>
#include <iterator>

#include <boost/icl/continuous_interval.hpp>
#include <boost/icl/interval_set.hpp>

#include <hpp/fcl/collision_data.h>
#include <hpp/fcl/collision.h>
#include <hpp/pinocchio/body.hh>
#include <hpp/pinocchio/collision-object.hh>
#include <hpp/pinocchio/joint.hh>
#include <hpp/core/deprecated.hh>
#include <hpp/core/fwd.hh>

namespace hpp {
  namespace core {
    namespace continuousValidation {
      /// Computation of collision-free sub-intervals of a path
      ///
      /// This class aims at validating a path for the absence of collision
      /// between two bodies of a robot.
      ///
      /// The interval of definition of the path is successively covered
      /// by intervals where boths bodies are proved to be collision-free.
      /// Each interval is computed by bounding from above the velocity of
      /// all points of body 1 in the reference frame of body 2.
      template <typename ValidationReportTypePtr_t>
      class IntervalValidation
      {
      public:
        virtual bool validateConfiguration(const value_type &t, interval_t &interval,
                                  ValidationReportTypePtr_t &report,
                                  pinocchio::DeviceData& data) = 0;

        /// Set path to validate
        /// \param path path to validate,
        /// \param reverse whether path is validated from end to beginning.
        void path(const PathPtr_t &path, bool reverse)
        {
          path_ = path;
          reverse_ = reverse;
          valid_ = false;
          validInterval_ = interval_set();
          setupPath();
        }

        /// Get path
        PathConstPtr_t path() const
        {
          return path_;
        }

        value_type tolerance() const
        {
          return tolerance_;
        }

        virtual std::string name () const = 0;
        virtual std::ostream& print (std::ostream& os) const = 0;

      protected:
        typedef boost::icl::continuous_interval<value_type> continuous_interval;
        typedef boost::icl::interval_set<value_type> interval_set;
        PathPtr_t path_;
        value_type tolerance_;
        bool reverse_;
        bool refine_;
        bool valid_;
        interval_set validInterval_;
        /// Constructor of interval validation element
        ///
        /// \param tolerance allowed penetration should be positive
        IntervalValidation (value_type tolerance) : tolerance_(tolerance),
          reverse_(false), refine_(true)
        {
          if (tolerance < 0) {
            throw std::runtime_error ("tolerance should be non-negative.");
          }
        }

        IntervalValidation (const IntervalValidation& other) :
          tolerance_(other.tolerance_), refine_(true)
        {
          assert (tolerance_ > 0);
        }

      private:
        virtual void setupPath() = 0;
      }; // class IntervalValidation

      template <typename ValidationReportTypePtr_t>
      inline std::ostream &operator<<(std::ostream &os, const IntervalValidation<ValidationReportTypePtr_t> &b)
      {
        return b.print(os);
      }
    } // namespace continuousValidation
  } // namespace core
} // namespace hpp
#endif // HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH
<commit_msg>In continuous validation, replace assert by exception.<commit_after>//
// Copyright (c) 2014,2015,2016,2018 CNRS
// Authors: Florent Lamiraux, Joseph Mirabel, Diane Bury
//
// This file is part of hpp-core
// hpp-core is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-core is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-core  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH
#define HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH

#include <limits>
#include <iterator>

#include <boost/icl/continuous_interval.hpp>
#include <boost/icl/interval_set.hpp>

#include <hpp/fcl/collision_data.h>
#include <hpp/fcl/collision.h>
#include <hpp/pinocchio/body.hh>
#include <hpp/pinocchio/collision-object.hh>
#include <hpp/pinocchio/joint.hh>
#include <hpp/core/deprecated.hh>
#include <hpp/core/fwd.hh>

namespace hpp {
  namespace core {
    namespace continuousValidation {
      /// Computation of collision-free sub-intervals of a path
      ///
      /// This class aims at validating a path for the absence of collision
      /// between two bodies of a robot.
      ///
      /// The interval of definition of the path is successively covered
      /// by intervals where boths bodies are proved to be collision-free.
      /// Each interval is computed by bounding from above the velocity of
      /// all points of body 1 in the reference frame of body 2.
      template <typename ValidationReportTypePtr_t>
      class IntervalValidation
      {
      public:
        virtual bool validateConfiguration(const value_type &t, interval_t &interval,
                                  ValidationReportTypePtr_t &report,
                                  pinocchio::DeviceData& data) = 0;

        /// Set path to validate
        /// \param path path to validate,
        /// \param reverse whether path is validated from end to beginning.
        void path(const PathPtr_t &path, bool reverse)
        {
          path_ = path;
          reverse_ = reverse;
          valid_ = false;
          validInterval_ = interval_set();
          setupPath();
        }

        /// Get path
        PathConstPtr_t path() const
        {
          return path_;
        }

        value_type tolerance() const
        {
          return tolerance_;
        }

        virtual std::string name () const = 0;
        virtual std::ostream& print (std::ostream& os) const = 0;

      protected:
        typedef boost::icl::continuous_interval<value_type> continuous_interval;
        typedef boost::icl::interval_set<value_type> interval_set;
        PathPtr_t path_;
        value_type tolerance_;
        bool reverse_;
        bool refine_;
        bool valid_;
        interval_set validInterval_;
        /// Constructor of interval validation element
        ///
        /// \param tolerance allowed penetration should be positive
        IntervalValidation (value_type tolerance) : tolerance_(tolerance),
          reverse_(false), refine_(true)
        {
          if (tolerance < 0) {
            throw std::runtime_error ("tolerance should be non-negative.");
          }
        }

        IntervalValidation (const IntervalValidation& other) :
          tolerance_(other.tolerance_), refine_(true)
        {
          if (tolerance_ < 0) {
            throw std::runtime_error ("tolerance should be non-negative.");
          }
        }

      private:
        virtual void setupPath() = 0;
      }; // class IntervalValidation

      template <typename ValidationReportTypePtr_t>
      inline std::ostream &operator<<(std::ostream &os, const IntervalValidation<ValidationReportTypePtr_t> &b)
      {
        return b.print(os);
      }
    } // namespace continuousValidation
  } // namespace core
} // namespace hpp
#endif // HPP_CORE_CONTINUOUS_VALIDATION_INTERVAL_VALIDATION_HH
<|endoftext|>


<commit_before>#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>

#include <iostream>

// Edge Tangent Flow

double Ws(const cv::Point2d &a, const cv::Point2d &b, const int radius) {
  return ((cv::norm(cv::Mat(a), cv::Mat(b)) < radius) ? 1.0 : 0.0);
}

double Wm(const cv::Point2d &a, const cv::Point2d &b, const cv::Mat &mag,
          const double eta = 1.0) {
  const double ga = mag.at<double>(a);
  const double gb = mag.at<double>(b);
  return (0.5 * (1.0 + std::tanh(eta * (gb - ga))));
}

double Wd(const cv::Point2d &a, const cv::Point2d &b, const cv::Mat &tcurr) {
  const cv::Vec2d ta = tcurr.at<cv::Vec2d>(a);
  const cv::Vec2d tb = tcurr.at<cv::Vec2d>(b);
  const double mult = ta.dot(tb);
  return ((mult > 0) ? std::abs(mult) : -1.0 * std::abs(mult));
}

void ETFIteration(cv::Mat &tcurr, const cv::Mat &mag, const int kradius) {
  cv::Mat tnew = cv::Mat::zeros(tcurr.size(), CV_64FC2);
  for (auto y = 0; y < (int)tcurr.size().height; ++y) {
    for (auto x = 0; x < (int)tcurr.size().width; ++x) {
      const auto pa = cv::Point2d(x, y);
      auto sigma = cv::Vec2d(0.0, 0.0);
      for (auto oy = -kradius; oy <= kradius; ++oy) {
        int py = y + oy;
        if (py < 0 || py >= tcurr.size().height)
          continue;
        for (auto ox = -kradius; ox <= kradius; ++ox) {
          int px = x + ox;
          if ((px == 0 && py == 0) || px < 0 || px >= tcurr.size().width)
            continue;
          const auto pb = cv::Point2d(px, py);
          sigma += tcurr.at<cv::Vec2d>(pb) * Ws(pa, pb, kradius) *
                   Wm(pa, pb, mag) * Wd(pa, pb, tcurr);
        }
      }
      cv::normalize(sigma, tnew.at<cv::Vec2d>(pa));
    }
  }
  tcurr = tnew;
}

cv::Mat g_perpendicular(const cv::Mat &gx, const cv::Mat &gy) {
  cv::Mat ng;
  std::vector<cv::Mat> garray = {gy, -1 * gx};
  cv::merge(garray.data(), garray.size(), ng);
  ng.forEach<cv::Vec2d>([](cv::Vec2d &v, const int *pos) {
    auto cp = v;
    cv::normalize(cp, v);
  });
  return ng;
}

cv::Mat ETF(const cv::Mat &gray, const int kernel_radius,
            const int etf_iterations) {
  cv::Mat gx, gy, gm, mag;
  cv::Sobel(gray, gx, CV_64F, 1, 0, 3);
  cv::Sobel(gray, gy, CV_64F, 0, 1, 3);
  cv::magnitude(gx, gy, gm);
  cv::normalize(gm, mag, 1.0, 0.0, cv::NORM_MINMAX);

  cv::imshow("Grad", mag);
  cv::waitKey(100);

  cv::Mat etf = g_perpendicular(gx, gy);
  for (auto i = 0; i < etf_iterations; ++i) {
    cv::Mat etf_vis0, etf_vis1, s[2];
    cv::split(etf, s);
    etf_vis0 = 0.5 * s[0] + 0.5 * s[1];
    // cv::imshow("ETF0-0", s[0]);
    // cv::imshow("ETF0-1", s[1]);
    cv::imshow("ETF0-mean", etf_vis0);

    std::cout << "starting iteration " << i << std::endl;
    ETFIteration(etf, mag, kernel_radius);
    std::cout << "finished iteration " << i << std::endl;

    cv::split(etf, s);
    etf_vis1 = 0.5 * s[0] + 0.5 * s[1];
    cv::imshow("ETF1-0", s[0]);
    cv::imshow("ETF1-1", s[1]);
    cv::imshow("ETF1-mean", etf_vis1);
    cv::waitKey(100);
  }

  return etf;
}

// Flow-based Difference-of-Gaussian

double F(const cv::Point &s, const cv::Mat &gray, const cv::Mat &gauss_dog,
         const cv::Mat &etf, const int delta_q) {
  const cv::Vec2d direction = etf.at<cv::Vec2d>(s);
  const auto perpendicular = cv::Vec2d(direction[1], -direction[0]);

  double integral = 0.0;
  const int half_k = gauss_dog.size().area() / 2;
  for (int k = -half_k; k <= half_k; ++k) {
    const auto offset = k * delta_q * perpendicular;
    const auto target = s + cv::Point(offset);
    if (!target.inside(cv::Rect(cv::Point(0, 0), gray.size())))
      continue;
    integral +=
        gauss_dog.at<double>(k + half_k) * (double)gray.at<uchar>(target);
  }

  return integral;
}

cv::Mat FDOG(const cv::Mat &gray, const cv::Mat &etf, const double p_s,
             const double sigma_c, const double sigma_m, const double thrs,
             const double delta_p, const double delta_q) {
  int p = 2.0 * sigma_m;
  p += (p % 2) ? 0 : 1;
  const auto gauss_kernel_m = cv::getGaussianKernel(p, sigma_m, CV_64F);

  const double sigma_s = 1.6 * sigma_c;
  int q = 2.0 * sigma_c;
  q = (q % 2) ? 0 : 1;
  const auto gauss_kernel_c = cv::getGaussianKernel(q, sigma_c, CV_64F);
  const auto gauss_kernel_s = cv::getGaussianKernel(q, sigma_s, CV_64F);
  const auto gauss_dog_cs = gauss_kernel_c - p_s * gauss_kernel_s; // f(t)

  for (auto y = 0; y < gray.size().height; ++y) {
    for (auto x = 0; x < gray.size().width; ++x) {
      const auto pt_gray = cv::Point(x, y);
      // cx(0)
    }
  }
}

// Coherent Lines Filter

cv::Mat coherentLines(const cv::Mat &img, const int kernel_radius = 5,
                      const int etf_iterations = 5, const double p_s = 0.99,
                      const double sigma_c = 1.0, const double sigma_m = 3.0,
                      const double thrs = 0.2, const double delta_m = 1.0,
                      const double delta_n = 1.0) {
  cv::Mat gray;
  cv::cvtColor(img, gray, CV_BGR2GRAY);
  const auto etf = ETF(gray, kernel_radius, etf_iterations);
  const auto fdog =
      FDOG(gray, etf, p_s, sigma_c, sigma_m, thrs, delta_m, delta_n);
  return cv::Mat();
}

// Sample

int main(int argc, char **argv) {
  if (argc != 2) {
    std::cout << argv[0] << " <path_to_image>" << std::endl;
    return 1;
  }

  auto img = cv::imread(argv[1]);
  cv::imshow("input", img);
  cv::waitKey(100);

  coherentLines(img);
  cv::waitKey(0);

  return 0;
}
<commit_msg>add H(x)<commit_after>#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>

#include <iostream>

// Edge Tangent Flow

double Ws(const cv::Point2d &a, const cv::Point2d &b, const int radius) {
  return ((cv::norm(cv::Mat(a), cv::Mat(b)) < radius) ? 1.0 : 0.0);
}

double Wm(const cv::Point2d &a, const cv::Point2d &b, const cv::Mat &mag,
          const double eta = 1.0) {
  const double ga = mag.at<double>(a);
  const double gb = mag.at<double>(b);
  return (0.5 * (1.0 + std::tanh(eta * (gb - ga))));
}

double Wd(const cv::Point2d &a, const cv::Point2d &b, const cv::Mat &tcurr) {
  const cv::Vec2d ta = tcurr.at<cv::Vec2d>(a);
  const cv::Vec2d tb = tcurr.at<cv::Vec2d>(b);
  const double mult = ta.dot(tb);
  return ((mult > 0) ? std::abs(mult) : -1.0 * std::abs(mult));
}

void ETFIteration(cv::Mat &tcurr, const cv::Mat &mag, const int kradius) {
  cv::Mat tnew = cv::Mat::zeros(tcurr.size(), CV_64FC2);
  for (auto y = 0; y < (int)tcurr.size().height; ++y) {
    for (auto x = 0; x < (int)tcurr.size().width; ++x) {
      const auto pa = cv::Point2d(x, y);
      auto sigma = cv::Vec2d(0.0, 0.0);
      for (auto oy = -kradius; oy <= kradius; ++oy) {
        int py = y + oy;
        if (py < 0 || py >= tcurr.size().height)
          continue;
        for (auto ox = -kradius; ox <= kradius; ++ox) {
          int px = x + ox;
          if ((px == 0 && py == 0) || px < 0 || px >= tcurr.size().width)
            continue;
          const auto pb = cv::Point2d(px, py);
          sigma += tcurr.at<cv::Vec2d>(pb) * Ws(pa, pb, kradius) *
                   Wm(pa, pb, mag) * Wd(pa, pb, tcurr);
        }
      }
      cv::normalize(sigma, tnew.at<cv::Vec2d>(pa));
    }
  }
  tcurr = tnew;
}

cv::Mat g_perpendicular(const cv::Mat &gx, const cv::Mat &gy) {
  cv::Mat ng;
  std::vector<cv::Mat> garray = {gy, -1 * gx};
  cv::merge(garray.data(), garray.size(), ng);
  ng.forEach<cv::Vec2d>([](cv::Vec2d &v, const int *pos) {
    auto cp = v;
    cv::normalize(cp, v);
  });
  return ng;
}

cv::Mat ETF(const cv::Mat &gray, const int kernel_radius,
            const int etf_iterations) {
  cv::Mat gx, gy, gm, mag;
  cv::Sobel(gray, gx, CV_64F, 1, 0, 3);
  cv::Sobel(gray, gy, CV_64F, 0, 1, 3);
  cv::magnitude(gx, gy, gm);
  cv::normalize(gm, mag, 1.0, 0.0, cv::NORM_MINMAX);

  cv::imshow("Grad", mag);
  cv::waitKey(100);

  cv::Mat etf = g_perpendicular(gx, gy);
  for (auto i = 0; i < etf_iterations; ++i) {
    cv::Mat etf_vis0, etf_vis1, s[2];
    cv::split(etf, s);
    etf_vis0 = 0.5 * s[0] + 0.5 * s[1];
    // cv::imshow("ETF0-0", s[0]);
    // cv::imshow("ETF0-1", s[1]);
    cv::imshow("ETF0-mean", etf_vis0);

    std::cout << "starting iteration " << i << std::endl;
    ETFIteration(etf, mag, kernel_radius);
    std::cout << "finished iteration " << i << std::endl;

    cv::split(etf, s);
    etf_vis1 = 0.5 * s[0] + 0.5 * s[1];
    cv::imshow("ETF1-0", s[0]);
    cv::imshow("ETF1-1", s[1]);
    cv::imshow("ETF1-mean", etf_vis1);
    cv::waitKey(100);
  }

  return etf;
}

// Flow-based Difference-of-Gaussian

double F(const cv::Point &s, const cv::Mat &gray, const cv::Mat &gauss_dog,
         const cv::Mat &etf, const int delta_q) {
  const cv::Vec2d direction = etf.at<cv::Vec2d>(s);
  const auto perpendicular = cv::Vec2d(direction[1], -direction[0]);
  double integral = 0.0;
  const int half_k = gauss_dog.size().area() / 2;
  for (int k = -half_k; k <= half_k; ++k) {
    const auto offset = k * delta_q * perpendicular;
    const auto target = s + cv::Point(offset);
    if (!target.inside(cv::Rect(cv::Point(0, 0), gray.size())))
      continue;
    integral +=
        gauss_dog.at<double>(k + half_k) * (double)gray.at<uchar>(target);
  }
  return integral;
}

uchar H(const cv::Point &s, const cv::Mat &gray, const cv::Mat &etf,
        const cv::Mat &gauss_m, const cv::Mat &gauss_dog, const int delta_p,
        const int delta_q, const double thrs) {
  const cv::Vec2d direction = etf.at<cv::Vec2d>(s);
  double integral = 0.0;
  const int half_k = gauss_m.size().area() / 2;
  for (int k = -half_k; k <= half_k; ++k) {
    const auto offset = k * delta_p * direction;
    const auto target = s + cv::Point(offset);
    if (!target.inside(cv::Rect(cv::Point(0, 0), gray.size())))
      continue;
    integral += gauss_m.at<double>(k + half_k) *
                F(target, gray, gauss_dog, etf, delta_q);
  }
  return ((integral < 0 && 1 + std::tanh(integral) < thrs) ? 0 : 1);
}

cv::Mat FDOG(const cv::Mat &gray, const cv::Mat &etf, const double p_s,
             const double sigma_c, const double sigma_m, const double thrs,
             const double delta_p, const double delta_q) {
  int p = 2.0 * sigma_m;
  p += (p % 2) ? 0 : 1;
  const auto gauss_kernel_m = cv::getGaussianKernel(p, sigma_m, CV_64F);

  const double sigma_s = 1.6 * sigma_c;
  int q = 2.0 * sigma_c;
  q = (q % 2) ? 0 : 1;
  const auto gauss_kernel_c = cv::getGaussianKernel(q, sigma_c, CV_64F);
  const auto gauss_kernel_s = cv::getGaussianKernel(q, sigma_s, CV_64F);
  const auto gauss_dog_cs = gauss_kernel_c - p_s * gauss_kernel_s; // f(t)

  cv::Mat response = cv::Mat::zeros(gray.size(), CV_8UC1);
  for (auto y = 0; y < gray.size().height; ++y) {
    for (auto x = 0; x < gray.size().width; ++x) {
      const auto root = cv::Point(x, y);
      response.at<uchar>(root) = H(root, gray, etf, gauss_kernel_m,
                                   gauss_dog_cs, delta_p, delta_q, thrs);
    }
  }
  return response;
}

// Coherent Lines Filter

cv::Mat coherentLines(const cv::Mat &img, const int kernel_radius = 5,
                      const int etf_iterations = 5, const double p_s = 0.99,
                      const double sigma_c = 1.0, const double sigma_m = 3.0,
                      const double thrs = 0.2, const double delta_m = 1.0,
                      const double delta_n = 1.0) {
  cv::Mat gray;
  cv::cvtColor(img, gray, CV_BGR2GRAY);
  const auto etf = ETF(gray, kernel_radius, etf_iterations);
  const auto fdog =
      FDOG(gray, etf, p_s, sigma_c, sigma_m, thrs, delta_m, delta_n);
  return cv::Mat();
}

// Sample

int main(int argc, char **argv) {
  if (argc != 2) {
    std::cout << argv[0] << " <path_to_image>" << std::endl;
    return 1;
  }

  auto img = cv::imread(argv[1]);
  cv::imshow("input", img);
  cv::waitKey(100);

  coherentLines(img);
  cv::waitKey(0);

  return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright 2021 Google LLC.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Apply a model on a dataset and export the predictions to disk.

#include <memory>
#include <vector>

#include "absl/flags/flag.h"
#include "yggdrasil_decision_forests/dataset/data_spec.pb.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h"
#include "yggdrasil_decision_forests/model/abstract_model.h"
#include "yggdrasil_decision_forests/model/model_library.h"
#include "yggdrasil_decision_forests/model/prediction.pb.h"
#include "yggdrasil_decision_forests/utils/evaluation.h"
#include "yggdrasil_decision_forests/utils/logging.h"

ABSL_FLAG(std::string, model, "", "Model directory.");

ABSL_FLAG(std::string, dataset, "",
          "Typed path to dataset i.e. [type]:[path] format.");

ABSL_FLAG(std::string, output, "",
          "Output prediction specified with [type]:[path] format. e.g. "
          "\"csv:/path/to/dataset.csv\".");

ABSL_FLAG(int, num_records_by_shard_in_output, -1,
          "Number of records per output shards. Only valid if the output "
          "path is sharded (e.g. contains @10).");

constexpr char kUsageMessage[] =
    "Apply a model on a dataset and export the predictions to disk.";

namespace yggdrasil_decision_forests {
namespace cli {

void ConvertDataset() {
  // Check required flags.
  QCHECK(!absl::GetFlag(FLAGS_dataset).empty());
  QCHECK(!absl::GetFlag(FLAGS_output).empty());
  QCHECK(!absl::GetFlag(FLAGS_model).empty());

  // Load the model
  std::unique_ptr<model::AbstractModel> model;
  QCHECK_OK(model::LoadModel(absl::GetFlag(FLAGS_model), &model));
  const auto& label_column = model->data_spec().columns(model->label_col_idx());

  // Load dataset.
  dataset::VerticalDataset dataset;
  QCHECK_OK(LoadVerticalDataset(absl::GetFlag(FLAGS_dataset),
                                model->data_spec(), &dataset));

  // Compute the predictions.
  std::vector<model::proto::Prediction> predictions;
  predictions.resize(dataset.nrow());

  auto engine_or = model->BuildFastEngine();
  if (engine_or.ok()) {
    LOG(INFO) << "Run predictions with semi-fast engine";

    // Convert dataset to efficient format.
    auto engine = std::move(engine_or.value());
    auto examples = engine->AllocateExamples(dataset.nrow());
    QCHECK_OK(serving::CopyVerticalDatasetToAbstractExampleSet(
        dataset, 0, dataset.nrow(), engine->features(), examples.get()));

    // Apply the model.
    std::vector<float> fast_predictions;
    engine->Predict(*examples, dataset.nrow(), &fast_predictions);
    examples.reset();

    // Convert the prediction to the expected format.
    const int num_prediction_dimensions = engine->NumPredictionDimension();
    for (dataset::VerticalDataset::row_t example_idx = 0;
         example_idx < dataset.nrow(); example_idx++) {
      model::FloatToProtoPrediction(fast_predictions, example_idx,
                                    model->task(), num_prediction_dimensions,
                                    &predictions[example_idx]);
    }
  } else {
    LOG(INFO) << "Run predictions with slow engine";
    for (dataset::VerticalDataset::row_t example_idx = 0;
         example_idx < dataset.nrow(); example_idx++) {
      LOG_INFO_EVERY_N_SEC(30, _ << example_idx << "/" << dataset.nrow()
                                 << " predictions computed.");
      model->Predict(dataset, example_idx, &predictions[example_idx]);
    }
  }

  // Save the predictions.
  QCHECK_OK(utils::ExportPredictions(
      predictions, model->task(), label_column, absl::GetFlag(FLAGS_output),
      absl::GetFlag(FLAGS_num_records_by_shard_in_output)));
}

}  // namespace cli
}  // namespace yggdrasil_decision_forests

int main(int argc, char** argv) {
  InitLogging(kUsageMessage, &argc, &argv, true);
  yggdrasil_decision_forests::cli::ConvertDataset();
  return 0;
}
<commit_msg>Internal change<commit_after>/*
 * Copyright 2021 Google LLC.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Apply a model on a dataset and export the predictions to disk.

#include <memory>
#include <vector>

#include "absl/flags/flag.h"
#include "yggdrasil_decision_forests/dataset/data_spec.pb.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h"
#include "yggdrasil_decision_forests/model/abstract_model.h"
#include "yggdrasil_decision_forests/model/model_library.h"
#include "yggdrasil_decision_forests/model/prediction.pb.h"
#include "yggdrasil_decision_forests/utils/evaluation.h"
#include "yggdrasil_decision_forests/utils/logging.h"

ABSL_FLAG(std::string, model, "", "Model directory.");

ABSL_FLAG(std::string, dataset, "",
          "Typed path to dataset i.e. [type]:[path] format.");

ABSL_FLAG(std::string, output, "",
          "Output prediction specified with [type]:[path] format. e.g. "
          "\"csv:/path/to/dataset.csv\".");

ABSL_FLAG(int, num_records_by_shard_in_output, -1,
          "Number of records per output shards. Only valid if the output "
          "path is sharded (e.g. contains @10).");

constexpr char kUsageMessage[] =
    "Apply a model on a dataset and export the predictions to disk.";

namespace yggdrasil_decision_forests {
namespace cli {

void ConvertDataset() {
  // Check required flags.
  QCHECK(!absl::GetFlag(FLAGS_dataset).empty());
  QCHECK(!absl::GetFlag(FLAGS_output).empty());
  QCHECK(!absl::GetFlag(FLAGS_model).empty());

  // Load the model
  std::unique_ptr<model::AbstractModel> model;
  QCHECK_OK(model::LoadModel(absl::GetFlag(FLAGS_model), &model));
  const auto& label_column = model->data_spec().columns(model->label_col_idx());

  // Load dataset.
  //
  // The columns corresponding to the model input features are required (i.e.
  // loading the dataset will fail if one of them is missing). The other columns
  // (e.g. label, weights) are optional.
  dataset::VerticalDataset dataset;
  QCHECK_OK(LoadVerticalDataset(
      absl::GetFlag(FLAGS_dataset), model->data_spec(), &dataset,
      /*ensure_non_missing=*/model->input_features()));

  // Compute the predictions.
  std::vector<model::proto::Prediction> predictions;
  predictions.resize(dataset.nrow());

  auto engine_or = model->BuildFastEngine();
  if (engine_or.ok()) {
    LOG(INFO) << "Run predictions with semi-fast engine";

    // Convert dataset to efficient format.
    auto engine = std::move(engine_or.value());
    auto examples = engine->AllocateExamples(dataset.nrow());
    QCHECK_OK(serving::CopyVerticalDatasetToAbstractExampleSet(
        dataset, 0, dataset.nrow(), engine->features(), examples.get()));

    // Apply the model.
    std::vector<float> fast_predictions;
    engine->Predict(*examples, dataset.nrow(), &fast_predictions);
    examples.reset();

    // Convert the prediction to the expected format.
    const int num_prediction_dimensions = engine->NumPredictionDimension();
    for (dataset::VerticalDataset::row_t example_idx = 0;
         example_idx < dataset.nrow(); example_idx++) {
      model::FloatToProtoPrediction(fast_predictions, example_idx,
                                    model->task(), num_prediction_dimensions,
                                    &predictions[example_idx]);
    }
  } else {
    LOG(INFO) << "Run predictions with slow engine";
    for (dataset::VerticalDataset::row_t example_idx = 0;
         example_idx < dataset.nrow(); example_idx++) {
      LOG_INFO_EVERY_N_SEC(30, _ << example_idx << "/" << dataset.nrow()
                                 << " predictions computed.");
      model->Predict(dataset, example_idx, &predictions[example_idx]);
    }
  }

  // Save the predictions.
  QCHECK_OK(utils::ExportPredictions(
      predictions, model->task(), label_column, absl::GetFlag(FLAGS_output),
      absl::GetFlag(FLAGS_num_records_by_shard_in_output)));
}

}  // namespace cli
}  // namespace yggdrasil_decision_forests

int main(int argc, char** argv) {
  InitLogging(kUsageMessage, &argc, &argv, true);
  yggdrasil_decision_forests::cli::ConvertDataset();
  return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/Benchmark.h>
#include <folly/Range.h>

#include <algorithm>

using namespace std;
using namespace folly;

string lorem_ipsum =
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam"
"lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam"
"viverra nec consectetur ante hendrerit. Donec et mollis"
"dolor. Praesent et diam eget libero egestas mattis sit amet vitae"
"augue. Nam tincidunt congue enim, ut porta lorem lacinia"
"consectetur. Donec ut libero sed arcu vehicula ultricies a non"
"tortor. Lorem ipsum dolor sit amet, consectetur adipiscing"
"elit. Aenean ut gravida lorem. Ut turpis felis, pulvinar a semper sed,"
"adipiscing id dolor. Pellentesque auctor nisi id magna consequat"
"sagittis. Curabitur dapibus enim sit amet elit pharetra tincidunt"
"feugiat nisl imperdiet. Ut convallis libero in urna ultrices"
"accumsan. Donec sed odio eros. Donec viverra mi quis quam pulvinar at"
"malesuada arcu rhoncus. Cum sociis natoque penatibus et magnis dis"
"parturient montes, nascetur ridiculus mus. In rutrum accumsan"
"ultricies. Mauris vitae nisi at sem facilisis semper ac in est."
"\n"
"Vivamus fermentum semper porta. Nunc diam velit, adipiscing ut"
"tristique vitae, sagittis vel odio. Maecenas convallis ullamcorper"
"ultricies. Curabitur ornare, ligula semper consectetur sagittis, nisi"
"diam iaculis velit, id fringilla sem nunc vel mi. Nam dictum, odio nec"
"pretium volutpat, arcu ante placerat erat, non tristique elit urna et"
"turpis. Quisque mi metus, ornare sit amet fermentum et, tincidunt et"
"orci. Fusce eget orci a orci congue vestibulum. Ut dolor diam,"
"elementum et vestibulum eu, porttitor vel elit. Curabitur venenatis"
"pulvinar tellus gravida ornare. Sed et erat faucibus nunc euismod"
"ultricies ut id justo. Nullam cursus suscipit nisi, et ultrices justo"
"sodales nec. Fusce venenatis facilisis lectus ac semper. Aliquam at"
"massa ipsum. Quisque bibendum purus convallis nulla ultrices"
"ultricies. Nullam aliquam, mi eu aliquam tincidunt, purus velit"
"laoreet tortor, viverra pretium nisi quam vitae mi. Fusce vel volutpat"
"elit. Nam sagittis nisi dui."
"\n"
"Suspendisse lectus leo, consectetur in tempor sit amet, placerat quis"
"neque. Etiam luctus porttitor lorem, sed suscipit est rutrum"
"non. Curabitur lobortis nisl a enim congue semper. Aenean commodo"
"ultrices imperdiet. Vestibulum ut justo vel sapien venenatis"
"tincidunt. Phasellus eget dolor sit amet ipsum dapibus condimentum"
"vitae quis lectus. Aliquam ut massa in turpis dapibus"
"convallis. Praesent elit lacus, vestibulum at malesuada et, ornare et"
"est. Ut augue nunc, sodales ut euismod non, adipiscing vitae"
"orci. Mauris ut placerat justo. Mauris in ultricies enim. Quisque nec"
"est eleifend nulla ultrices egestas quis ut quam. Donec sollicitudin"
"lectus a mauris pulvinar id aliquam urna cursus. Cras quis ligula sem,"
"vel elementum mi. Phasellus non ullamcorper urna."
"\n"
"Class aptent taciti sociosqu ad litora torquent per conubia nostra,"
"per inceptos himenaeos. In euismod ultrices facilisis. Vestibulum"
"porta sapien adipiscing augue congue id pretium lectus molestie. Proin"
"quis dictum nisl. Morbi id quam sapien, sed vestibulum sem. Duis"
"elementum rutrum mauris sed convallis. Proin vestibulum magna"
"mi. Aenean tristique hendrerit magna, ac facilisis nulla hendrerit"
"ut. Sed non tortor sodales quam auctor elementum. Donec hendrerit nunc"
"eget elit pharetra pulvinar. Suspendisse id tempus tortor. Aenean"
"luctus, elit commodo laoreet commodo, justo nisi consequat massa, sed"
"vulputate quam urna quis eros. Donec vel."
"\n";

string needle = "commodo";

// legacy implementation
struct AsciiCaseInsensitiveLegacy {
  bool operator()(char lhs, char rhs) const {
    return toupper(lhs) == toupper(rhs);
  }
};

template<typename Cmp>
inline void test_operator_on_search(int iters) {
  Cmp cmp;
  int dummy = 0;
  for (int i = 0; i < iters; ++i) {
    dummy += std::search(
      lorem_ipsum.begin(), lorem_ipsum.end(),
      needle.begin(), needle.end(),
      cmp
    ) - lorem_ipsum.begin();
  }
}

BENCHMARK(LegacyCaseInsensitiveCheck, iters) {
  test_operator_on_search<AsciiCaseInsensitiveLegacy>(iters);
}

BENCHMARK(CurrentCaseInsensitiveCheck, iters) {
  test_operator_on_search<AsciiCaseInsensitive>(iters);
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);
  folly::runBenchmarks();
  if (FLAGS_benchmark) {
    folly::runBenchmarks();
  }
  return 0;
}

/*
============================================================================
folly/test/AsciiCaseInsensitiveBenchmark.cpp    relative  time/iter  iters/s
============================================================================
LegacyCaseInsensitiveCheck                                   9.04us  110.60K
CurrentCaseInsensitiveCheck                                  2.96us  337.59K
============================================================================
*/
<commit_msg>Add const to the variable: needle<commit_after>/*
 * Copyright 2015 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/Benchmark.h>
#include <folly/Range.h>

#include <algorithm>

using namespace std;
using namespace folly;

string lorem_ipsum =
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam"
"lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam"
"viverra nec consectetur ante hendrerit. Donec et mollis"
"dolor. Praesent et diam eget libero egestas mattis sit amet vitae"
"augue. Nam tincidunt congue enim, ut porta lorem lacinia"
"consectetur. Donec ut libero sed arcu vehicula ultricies a non"
"tortor. Lorem ipsum dolor sit amet, consectetur adipiscing"
"elit. Aenean ut gravida lorem. Ut turpis felis, pulvinar a semper sed,"
"adipiscing id dolor. Pellentesque auctor nisi id magna consequat"
"sagittis. Curabitur dapibus enim sit amet elit pharetra tincidunt"
"feugiat nisl imperdiet. Ut convallis libero in urna ultrices"
"accumsan. Donec sed odio eros. Donec viverra mi quis quam pulvinar at"
"malesuada arcu rhoncus. Cum sociis natoque penatibus et magnis dis"
"parturient montes, nascetur ridiculus mus. In rutrum accumsan"
"ultricies. Mauris vitae nisi at sem facilisis semper ac in est."
"\n"
"Vivamus fermentum semper porta. Nunc diam velit, adipiscing ut"
"tristique vitae, sagittis vel odio. Maecenas convallis ullamcorper"
"ultricies. Curabitur ornare, ligula semper consectetur sagittis, nisi"
"diam iaculis velit, id fringilla sem nunc vel mi. Nam dictum, odio nec"
"pretium volutpat, arcu ante placerat erat, non tristique elit urna et"
"turpis. Quisque mi metus, ornare sit amet fermentum et, tincidunt et"
"orci. Fusce eget orci a orci congue vestibulum. Ut dolor diam,"
"elementum et vestibulum eu, porttitor vel elit. Curabitur venenatis"
"pulvinar tellus gravida ornare. Sed et erat faucibus nunc euismod"
"ultricies ut id justo. Nullam cursus suscipit nisi, et ultrices justo"
"sodales nec. Fusce venenatis facilisis lectus ac semper. Aliquam at"
"massa ipsum. Quisque bibendum purus convallis nulla ultrices"
"ultricies. Nullam aliquam, mi eu aliquam tincidunt, purus velit"
"laoreet tortor, viverra pretium nisi quam vitae mi. Fusce vel volutpat"
"elit. Nam sagittis nisi dui."
"\n"
"Suspendisse lectus leo, consectetur in tempor sit amet, placerat quis"
"neque. Etiam luctus porttitor lorem, sed suscipit est rutrum"
"non. Curabitur lobortis nisl a enim congue semper. Aenean commodo"
"ultrices imperdiet. Vestibulum ut justo vel sapien venenatis"
"tincidunt. Phasellus eget dolor sit amet ipsum dapibus condimentum"
"vitae quis lectus. Aliquam ut massa in turpis dapibus"
"convallis. Praesent elit lacus, vestibulum at malesuada et, ornare et"
"est. Ut augue nunc, sodales ut euismod non, adipiscing vitae"
"orci. Mauris ut placerat justo. Mauris in ultricies enim. Quisque nec"
"est eleifend nulla ultrices egestas quis ut quam. Donec sollicitudin"
"lectus a mauris pulvinar id aliquam urna cursus. Cras quis ligula sem,"
"vel elementum mi. Phasellus non ullamcorper urna."
"\n"
"Class aptent taciti sociosqu ad litora torquent per conubia nostra,"
"per inceptos himenaeos. In euismod ultrices facilisis. Vestibulum"
"porta sapien adipiscing augue congue id pretium lectus molestie. Proin"
"quis dictum nisl. Morbi id quam sapien, sed vestibulum sem. Duis"
"elementum rutrum mauris sed convallis. Proin vestibulum magna"
"mi. Aenean tristique hendrerit magna, ac facilisis nulla hendrerit"
"ut. Sed non tortor sodales quam auctor elementum. Donec hendrerit nunc"
"eget elit pharetra pulvinar. Suspendisse id tempus tortor. Aenean"
"luctus, elit commodo laoreet commodo, justo nisi consequat massa, sed"
"vulputate quam urna quis eros. Donec vel."
"\n";

const string needle = "commodo";

// legacy implementation
struct AsciiCaseInsensitiveLegacy {
  bool operator()(char lhs, char rhs) const {
    return toupper(lhs) == toupper(rhs);
  }
};

template<typename Cmp>
inline void test_operator_on_search(int iters) {
  Cmp cmp;
  int dummy = 0;
  for (int i = 0; i < iters; ++i) {
    dummy += std::search(
      lorem_ipsum.begin(), lorem_ipsum.end(),
      needle.begin(), needle.end(),
      cmp
    ) - lorem_ipsum.begin();
  }
}

BENCHMARK(LegacyCaseInsensitiveCheck, iters) {
  test_operator_on_search<AsciiCaseInsensitiveLegacy>(iters);
}

BENCHMARK(CurrentCaseInsensitiveCheck, iters) {
  test_operator_on_search<AsciiCaseInsensitive>(iters);
}

int main(int argc, char** argv) {
  gflags::ParseCommandLineFlags(&argc, &argv, true);
  folly::runBenchmarks();
  if (FLAGS_benchmark) {
    folly::runBenchmarks();
  }
  return 0;
}

/*
============================================================================
folly/test/AsciiCaseInsensitiveBenchmark.cpp    relative  time/iter  iters/s
============================================================================
LegacyCaseInsensitiveCheck                                   9.04us  110.60K
CurrentCaseInsensitiveCheck                                  2.96us  337.59K
============================================================================
*/
<|endoftext|>


<commit_before>//* This file is part of the MOOSE framework
//* https://www.mooseframework.org
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html

#include "StatisticsVectorPostprocessor.h"

// MOOSE includes
#include "MooseVariable.h"
#include "ThreadedElementLoopBase.h"
#include "ThreadedNodeLoop.h"

#include "libmesh/quadrature.h"

#include <numeric>

registerMooseObject("MooseApp", StatisticsVectorPostprocessor);

template <>
InputParameters
validParams<StatisticsVectorPostprocessor>()
{
  InputParameters params = validParams<GeneralVectorPostprocessor>();
  params.addClassDescription("Compute statistical values of a given VectorPostprocessor.  The "
                             "statistics are computed for each column.");

  params.addRequiredParam<VectorPostprocessorName>(
      "vpp", "The VectorPostprocessor to compute statistics for.");

  // These are directly numbered to ensure that any changes to this list will not cause a bug
  // Do not change the numbers here without changing the corresponding code in computeStatVector()
  MultiMooseEnum stats("min=0 max=1 sum=2 average=3 stddev=4 norm2=5");
  params.addRequiredParam<MultiMooseEnum>(
      "stats",
      stats,
      "The statistics you would like to compute for each column of the VectorPostprocessor");

  return params;
}

StatisticsVectorPostprocessor::StatisticsVectorPostprocessor(const InputParameters & parameters)
  : GeneralVectorPostprocessor(parameters),
    _vpp_name(getParam<VectorPostprocessorName>("vpp")),
    _stats(getParam<MultiMooseEnum>("stats")),
    _stat_type_vector(declareVector("stat_type"))
{
}

void
StatisticsVectorPostprocessor::initialize()
{
  const auto & vpp_vectors = _fe_problem.getVectorPostprocessorVectors(_vpp_name);

  // Add vectors for each column, the reason for the extra logic here is that the columns a VPP
  // produces can change
  for (const auto & the_pair : vpp_vectors)
  {
    const auto & name = the_pair.first;

    if (_stat_vectors.find(name) == _stat_vectors.end())
      _stat_vectors[name] = &declareVector(name);
  }
}

void
StatisticsVectorPostprocessor::execute()
{
  if (processor_id() == 0) // Only compute on processor 0
  {
    const auto & vpp_vectors = _fe_problem.getVectorPostprocessorVectors(_vpp_name);

    _stat_type_vector.clear();

    // Add the stat IDs into the first colum
    for (const auto & stat : _stats)
    {
      auto stat_id = stat.id();

      _stat_type_vector.push_back(stat_id);
    }

    // For each value vector compute the stats
    for (auto & the_pair : vpp_vectors)
    {
      const auto & name = the_pair.first;
      const auto & values = *the_pair.second.current;

      mooseAssert(_stat_vectors.count(name), "Error retrieving VPP vector");
      auto & stat_vector = *_stat_vectors.at(name);

      stat_vector.clear();

      for (const auto & stat : _stats)
      {
        auto stat_id = stat.id();

        stat_vector.push_back(computeStatValue(stat_id, values));
      }
    }
  }
}

void
StatisticsVectorPostprocessor::finalize()
{
}

Real
StatisticsVectorPostprocessor::computeStatValue(int stat_id, const std::vector<Real> & stat_vector)
{
  switch (stat_id)
  {
    case 0: // min
      return *std::min_element(stat_vector.begin(), stat_vector.end());
    case 1: // max
      return *std::max_element(stat_vector.begin(), stat_vector.end());
    case 2: // sum
      return std::accumulate(stat_vector.begin(), stat_vector.end(), 0);
    case 3: // average
      return std::accumulate(stat_vector.begin(), stat_vector.end(), 0) /
             static_cast<Real>(stat_vector.size());
    case 4: // stddev
    {
      auto mean = std::accumulate(stat_vector.begin(), stat_vector.end(), 0) /
                  static_cast<Real>(stat_vector.size());

      auto the_sum = std::accumulate(stat_vector.begin(),
                                     stat_vector.end(),
                                     0,
                                     [&mean](Real running_value, Real current_value) {
                                       return running_value + std::pow(current_value - mean, 2);
                                     });

      return std::sqrt(the_sum / (stat_vector.size() - 1.));
    }
    case 5: // norm2
      return std::sqrt(std::accumulate(
          stat_vector.begin(), stat_vector.end(), 0, [](Real running_value, Real current_value) {
            return running_value + std::pow(current_value, 2);

          }));
    default:
      mooseError("Unknown statistics type: ", stat_id);
  }
}
<commit_msg>Fixed  std::accumulate in StatisticsVectorPostprocessor<commit_after>//* This file is part of the MOOSE framework
//* https://www.mooseframework.org
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html

#include "StatisticsVectorPostprocessor.h"

// MOOSE includes
#include "MooseVariable.h"
#include "ThreadedElementLoopBase.h"
#include "ThreadedNodeLoop.h"

#include "libmesh/quadrature.h"

#include <numeric>

registerMooseObject("MooseApp", StatisticsVectorPostprocessor);

template <>
InputParameters
validParams<StatisticsVectorPostprocessor>()
{
  InputParameters params = validParams<GeneralVectorPostprocessor>();
  params.addClassDescription("Compute statistical values of a given VectorPostprocessor.  The "
                             "statistics are computed for each column.");

  params.addRequiredParam<VectorPostprocessorName>(
      "vpp", "The VectorPostprocessor to compute statistics for.");

  // These are directly numbered to ensure that any changes to this list will not cause a bug
  // Do not change the numbers here without changing the corresponding code in computeStatVector()
  MultiMooseEnum stats("min=0 max=1 sum=2 average=3 stddev=4 norm2=5");
  params.addRequiredParam<MultiMooseEnum>(
      "stats",
      stats,
      "The statistics you would like to compute for each column of the VectorPostprocessor");

  return params;
}

StatisticsVectorPostprocessor::StatisticsVectorPostprocessor(const InputParameters & parameters)
  : GeneralVectorPostprocessor(parameters),
    _vpp_name(getParam<VectorPostprocessorName>("vpp")),
    _stats(getParam<MultiMooseEnum>("stats")),
    _stat_type_vector(declareVector("stat_type"))
{
}

void
StatisticsVectorPostprocessor::initialize()
{
  const auto & vpp_vectors = _fe_problem.getVectorPostprocessorVectors(_vpp_name);

  // Add vectors for each column, the reason for the extra logic here is that the columns a VPP
  // produces can change
  for (const auto & the_pair : vpp_vectors)
  {
    const auto & name = the_pair.first;

    if (_stat_vectors.find(name) == _stat_vectors.end())
      _stat_vectors[name] = &declareVector(name);
  }
}

void
StatisticsVectorPostprocessor::execute()
{
  if (processor_id() == 0) // Only compute on processor 0
  {
    const auto & vpp_vectors = _fe_problem.getVectorPostprocessorVectors(_vpp_name);

    _stat_type_vector.clear();

    // Add the stat IDs into the first colum
    for (const auto & stat : _stats)
    {
      auto stat_id = stat.id();

      _stat_type_vector.push_back(stat_id);
    }

    // For each value vector compute the stats
    for (auto & the_pair : vpp_vectors)
    {
      const auto & name = the_pair.first;
      const auto & values = *the_pair.second.current;

      mooseAssert(_stat_vectors.count(name), "Error retrieving VPP vector");
      auto & stat_vector = *_stat_vectors.at(name);

      stat_vector.clear();

      for (const auto & stat : _stats)
      {
        auto stat_id = stat.id();

        stat_vector.push_back(computeStatValue(stat_id, values));
      }
    }
  }
}

void
StatisticsVectorPostprocessor::finalize()
{
}

Real
StatisticsVectorPostprocessor::computeStatValue(int stat_id, const std::vector<Real> & stat_vector)
{
  switch (stat_id)
  {
    case 0: // min
      return *std::min_element(stat_vector.begin(), stat_vector.end());
    case 1: // max
      return *std::max_element(stat_vector.begin(), stat_vector.end());
    case 2: // sum
      return std::accumulate(stat_vector.begin(), stat_vector.end(), 0.);
    case 3: // average
      return std::accumulate(stat_vector.begin(), stat_vector.end(), 0.) /
             static_cast<Real>(stat_vector.size());
    case 4: // stddev
    {
      auto mean = std::accumulate(stat_vector.begin(), stat_vector.end(), 0.) /
                  static_cast<Real>(stat_vector.size());

      auto the_sum = std::accumulate(stat_vector.begin(),
                                     stat_vector.end(),
                                     0.,
                                     [&mean](Real running_value, Real current_value) {
                                       return running_value + std::pow(current_value - mean, 2);
                                     });

      return std::sqrt(the_sum / (stat_vector.size() - 1.));
    }
    case 5: // norm2
      return std::sqrt(std::accumulate(
          stat_vector.begin(), stat_vector.end(), 0., [](Real running_value, Real current_value) {
            return running_value + std::pow(current_value, 2);

          }));
    default:
      mooseError("Unknown statistics type: ", stat_id);
  }
}
<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "webkit/plugins/ppapi/ppapi_webplugin_impl.h"

#include <cmath>

#include "base/message_loop.h"
#include "googleurl/src/gurl.h"
#include "ppapi/c/pp_var.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebBindings.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebPluginParams.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebPoint.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebRect.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebView.h"
#include "webkit/plugins/ppapi/message_channel.h"
#include "webkit/plugins/ppapi/plugin_module.h"
#include "webkit/plugins/ppapi/ppapi_plugin_instance.h"
#include "webkit/plugins/ppapi/ppb_url_loader_impl.h"
#include "webkit/plugins/ppapi/var.h"

using WebKit::WebCanvas;
using WebKit::WebPluginContainer;
using WebKit::WebPluginParams;
using WebKit::WebPoint;
using WebKit::WebRect;
using WebKit::WebString;
using WebKit::WebURL;
using WebKit::WebVector;
using WebKit::WebView;

namespace webkit {
namespace ppapi {

struct WebPluginImpl::InitData {
  scoped_refptr<PluginModule> module;
  base::WeakPtr<PluginDelegate> delegate;
  std::vector<std::string> arg_names;
  std::vector<std::string> arg_values;
  GURL url;
};

WebPluginImpl::WebPluginImpl(
    PluginModule* plugin_module,
    const WebPluginParams& params,
    const base::WeakPtr<PluginDelegate>& plugin_delegate)
    : init_data_(new InitData()),
      full_frame_(params.loadManually) {
  DCHECK(plugin_module);
  init_data_->module = plugin_module;
  init_data_->delegate = plugin_delegate;
  for (size_t i = 0; i < params.attributeNames.size(); ++i) {
    init_data_->arg_names.push_back(params.attributeNames[i].utf8());
    init_data_->arg_values.push_back(params.attributeValues[i].utf8());
  }
  init_data_->url = params.url;
}

WebPluginImpl::~WebPluginImpl() {
}

bool WebPluginImpl::initialize(WebPluginContainer* container) {
  // The plugin delegate may have gone away.
  if (!init_data_->delegate)
    return false;

  instance_ = init_data_->module->CreateInstance(init_data_->delegate);
  if (!instance_)
    return false;

  bool success = instance_->Initialize(container,
                                       init_data_->arg_names,
                                       init_data_->arg_values,
                                       init_data_->url,
                                       full_frame_);
  if (!success) {
    instance_->Delete();
    instance_ = NULL;
    return false;
  }

  init_data_.reset();
  return true;
}

void WebPluginImpl::destroy() {
  if (instance_) {
    instance_->Delete();
    instance_ = NULL;
  }

  MessageLoop::current()->DeleteSoon(FROM_HERE, this);
}

NPObject* WebPluginImpl::scriptableObject() {
  scoped_refptr<ObjectVar> object(
      ObjectVar::FromPPVar(instance_->GetInstanceObject()));
  // If there's an InstanceObject, tell the Instance's MessageChannel to pass
  // any non-postMessage calls to it.
  if (object) {
    instance_->message_channel().SetPassthroughObject(object->np_object());
  }
  NPObject* message_channel_np_object(instance_->message_channel().np_object());
  // The object is expected to be retained before it is returned.
  WebKit::WebBindings::retainObject(message_channel_np_object);
  return message_channel_np_object;
}

bool WebPluginImpl::getFormValue(WebString* value) {
  return false;
}

void WebPluginImpl::paint(WebCanvas* canvas, const WebRect& rect) {
  if (!instance_->IsFullscreenOrPending())
    instance_->Paint(canvas, plugin_rect_, rect);
}

void WebPluginImpl::updateGeometry(
    const WebRect& window_rect,
    const WebRect& clip_rect,
    const WebVector<WebRect>& cut_outs_rects,
    bool is_visible) {
  plugin_rect_ = window_rect;
  if (!instance_->IsFullscreenOrPending())
    instance_->ViewChanged(plugin_rect_, clip_rect);
}

void WebPluginImpl::updateFocus(bool focused) {
  instance_->SetWebKitFocus(focused);
}

void WebPluginImpl::updateVisibility(bool visible) {
}

bool WebPluginImpl::acceptsInputEvents() {
  return true;
}

bool WebPluginImpl::handleInputEvent(const WebKit::WebInputEvent& event,
                                     WebKit::WebCursorInfo& cursor_info) {
  if (instance_->IsFullscreenOrPending())
    return false;
  return instance_->HandleInputEvent(event, &cursor_info);
}

void WebPluginImpl::didReceiveResponse(
    const WebKit::WebURLResponse& response) {
  DCHECK(!document_loader_);

  document_loader_ = new PPB_URLLoader_Impl(instance_, true);
  document_loader_->didReceiveResponse(NULL, response);

  if (!instance_->HandleDocumentLoad(document_loader_))
    document_loader_ = NULL;
}

void WebPluginImpl::didReceiveData(const char* data, int data_length) {
  if (document_loader_)
    document_loader_->didReceiveData(NULL, data, data_length, data_length);
}

void WebPluginImpl::didFinishLoading() {
  if (document_loader_) {
    document_loader_->didFinishLoading(NULL, 0);
    document_loader_ = NULL;
  }
}

void WebPluginImpl::didFailLoading(const WebKit::WebURLError& error) {
  if (document_loader_) {
    document_loader_->didFail(NULL, error);
    document_loader_ = NULL;
  }
}

void WebPluginImpl::didFinishLoadingFrameRequest(const WebKit::WebURL& url,
                                                 void* notify_data) {
}

void WebPluginImpl::didFailLoadingFrameRequest(
    const WebKit::WebURL& url,
    void* notify_data,
    const WebKit::WebURLError& error) {
}

bool WebPluginImpl::hasSelection() const {
  return !selectionAsText().isEmpty();
}

WebString WebPluginImpl::selectionAsText() const {
  return instance_->GetSelectedText(false);
}

WebString WebPluginImpl::selectionAsMarkup() const {
  return instance_->GetSelectedText(true);
}

WebURL WebPluginImpl::linkAtPosition(const WebPoint& position) const {
  return GURL(instance_->GetLinkAtPosition(position));
}

void WebPluginImpl::setZoomLevel(double level, bool text_only) {
  instance_->Zoom(WebView::zoomLevelToZoomFactor(level), text_only);
}

bool WebPluginImpl::startFind(const WebKit::WebString& search_text,
                              bool case_sensitive,
                              int identifier) {
  return instance_->StartFind(search_text, case_sensitive, identifier);
}

void WebPluginImpl::selectFindResult(bool forward) {
  instance_->SelectFindResult(forward);
}

void WebPluginImpl::stopFind() {
  instance_->StopFind();
}

bool WebPluginImpl::supportsPaginatedPrint() {
  return instance_->SupportsPrintInterface();
}

int WebPluginImpl::printBegin(const WebKit::WebRect& printable_area,
                              int printer_dpi) {
  return instance_->PrintBegin(printable_area, printer_dpi);
}

bool WebPluginImpl::printPage(int page_number,
                              WebKit::WebCanvas* canvas) {
  return instance_->PrintPage(page_number, canvas);
}

void WebPluginImpl::printEnd() {
  return instance_->PrintEnd();
}

}  // namespace ppapi
}  // namespace webkit
<commit_msg>Fix re-entrancy case in WebPluginImpl::scriptableObject.<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "webkit/plugins/ppapi/ppapi_webplugin_impl.h"

#include <cmath>

#include "base/message_loop.h"
#include "googleurl/src/gurl.h"
#include "ppapi/c/pp_var.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebBindings.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebPluginParams.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebPoint.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebRect.h"
#include "third_party/WebKit/Source/WebKit/chromium/public/WebView.h"
#include "webkit/plugins/ppapi/message_channel.h"
#include "webkit/plugins/ppapi/plugin_module.h"
#include "webkit/plugins/ppapi/ppapi_plugin_instance.h"
#include "webkit/plugins/ppapi/ppb_url_loader_impl.h"
#include "webkit/plugins/ppapi/var.h"

using WebKit::WebCanvas;
using WebKit::WebPluginContainer;
using WebKit::WebPluginParams;
using WebKit::WebPoint;
using WebKit::WebRect;
using WebKit::WebString;
using WebKit::WebURL;
using WebKit::WebVector;
using WebKit::WebView;

namespace webkit {
namespace ppapi {

struct WebPluginImpl::InitData {
  scoped_refptr<PluginModule> module;
  base::WeakPtr<PluginDelegate> delegate;
  std::vector<std::string> arg_names;
  std::vector<std::string> arg_values;
  GURL url;
};

WebPluginImpl::WebPluginImpl(
    PluginModule* plugin_module,
    const WebPluginParams& params,
    const base::WeakPtr<PluginDelegate>& plugin_delegate)
    : init_data_(new InitData()),
      full_frame_(params.loadManually) {
  DCHECK(plugin_module);
  init_data_->module = plugin_module;
  init_data_->delegate = plugin_delegate;
  for (size_t i = 0; i < params.attributeNames.size(); ++i) {
    init_data_->arg_names.push_back(params.attributeNames[i].utf8());
    init_data_->arg_values.push_back(params.attributeValues[i].utf8());
  }
  init_data_->url = params.url;
}

WebPluginImpl::~WebPluginImpl() {
}

bool WebPluginImpl::initialize(WebPluginContainer* container) {
  // The plugin delegate may have gone away.
  if (!init_data_->delegate)
    return false;

  instance_ = init_data_->module->CreateInstance(init_data_->delegate);
  if (!instance_)
    return false;

  bool success = instance_->Initialize(container,
                                       init_data_->arg_names,
                                       init_data_->arg_values,
                                       init_data_->url,
                                       full_frame_);
  if (!success) {
    instance_->Delete();
    instance_ = NULL;
    return false;
  }

  init_data_.reset();
  return true;
}

void WebPluginImpl::destroy() {
  if (instance_) {
    instance_->Delete();
    instance_ = NULL;
  }

  MessageLoop::current()->DeleteSoon(FROM_HERE, this);
}

NPObject* WebPluginImpl::scriptableObject() {
  scoped_refptr<ObjectVar> object(
      ObjectVar::FromPPVar(instance_->GetInstanceObject()));
  // GetInstanceObject talked to the plugin which may have removed the instance
  // from the DOM, in which case instance_ would be NULL now.
  if (!instance_)
    return NULL;

  // If there's an InstanceObject, tell the Instance's MessageChannel to pass
  // any non-postMessage calls to it.
  if (object) {
    instance_->message_channel().SetPassthroughObject(object->np_object());
  }
  NPObject* message_channel_np_object(instance_->message_channel().np_object());
  // The object is expected to be retained before it is returned.
  WebKit::WebBindings::retainObject(message_channel_np_object);
  return message_channel_np_object;
}

bool WebPluginImpl::getFormValue(WebString* value) {
  return false;
}

void WebPluginImpl::paint(WebCanvas* canvas, const WebRect& rect) {
  if (!instance_->IsFullscreenOrPending())
    instance_->Paint(canvas, plugin_rect_, rect);
}

void WebPluginImpl::updateGeometry(
    const WebRect& window_rect,
    const WebRect& clip_rect,
    const WebVector<WebRect>& cut_outs_rects,
    bool is_visible) {
  plugin_rect_ = window_rect;
  if (!instance_->IsFullscreenOrPending())
    instance_->ViewChanged(plugin_rect_, clip_rect);
}

void WebPluginImpl::updateFocus(bool focused) {
  instance_->SetWebKitFocus(focused);
}

void WebPluginImpl::updateVisibility(bool visible) {
}

bool WebPluginImpl::acceptsInputEvents() {
  return true;
}

bool WebPluginImpl::handleInputEvent(const WebKit::WebInputEvent& event,
                                     WebKit::WebCursorInfo& cursor_info) {
  if (instance_->IsFullscreenOrPending())
    return false;
  return instance_->HandleInputEvent(event, &cursor_info);
}

void WebPluginImpl::didReceiveResponse(
    const WebKit::WebURLResponse& response) {
  DCHECK(!document_loader_);

  document_loader_ = new PPB_URLLoader_Impl(instance_, true);
  document_loader_->didReceiveResponse(NULL, response);

  if (!instance_->HandleDocumentLoad(document_loader_))
    document_loader_ = NULL;
}

void WebPluginImpl::didReceiveData(const char* data, int data_length) {
  if (document_loader_)
    document_loader_->didReceiveData(NULL, data, data_length, data_length);
}

void WebPluginImpl::didFinishLoading() {
  if (document_loader_) {
    document_loader_->didFinishLoading(NULL, 0);
    document_loader_ = NULL;
  }
}

void WebPluginImpl::didFailLoading(const WebKit::WebURLError& error) {
  if (document_loader_) {
    document_loader_->didFail(NULL, error);
    document_loader_ = NULL;
  }
}

void WebPluginImpl::didFinishLoadingFrameRequest(const WebKit::WebURL& url,
                                                 void* notify_data) {
}

void WebPluginImpl::didFailLoadingFrameRequest(
    const WebKit::WebURL& url,
    void* notify_data,
    const WebKit::WebURLError& error) {
}

bool WebPluginImpl::hasSelection() const {
  return !selectionAsText().isEmpty();
}

WebString WebPluginImpl::selectionAsText() const {
  return instance_->GetSelectedText(false);
}

WebString WebPluginImpl::selectionAsMarkup() const {
  return instance_->GetSelectedText(true);
}

WebURL WebPluginImpl::linkAtPosition(const WebPoint& position) const {
  return GURL(instance_->GetLinkAtPosition(position));
}

void WebPluginImpl::setZoomLevel(double level, bool text_only) {
  instance_->Zoom(WebView::zoomLevelToZoomFactor(level), text_only);
}

bool WebPluginImpl::startFind(const WebKit::WebString& search_text,
                              bool case_sensitive,
                              int identifier) {
  return instance_->StartFind(search_text, case_sensitive, identifier);
}

void WebPluginImpl::selectFindResult(bool forward) {
  instance_->SelectFindResult(forward);
}

void WebPluginImpl::stopFind() {
  instance_->StopFind();
}

bool WebPluginImpl::supportsPaginatedPrint() {
  return instance_->SupportsPrintInterface();
}

int WebPluginImpl::printBegin(const WebKit::WebRect& printable_area,
                              int printer_dpi) {
  return instance_->PrintBegin(printable_area, printer_dpi);
}

bool WebPluginImpl::printPage(int page_number,
                              WebKit::WebCanvas* canvas) {
  return instance_->PrintPage(page_number, canvas);
}

void WebPluginImpl::printEnd() {
  return instance_->PrintEnd();
}

}  // namespace ppapi
}  // namespace webkit
<|endoftext|>


<commit_before>#undef DEBUG
#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <cmath>

//needed for select
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>

//needed for poll
#include <poll.h>

//for connection to server
#include "../sockets/SocketW.h"
bool ready4data = false;//set to true when streaming starts
bool inited = false;
bool stopparsing = false;
timeval lastrec;

#include "parsechunks.cpp" //chunkstream parsing
#include "handshake.cpp" //handshaking
#include "../util/flv_sock.cpp" //FLV parsing with SocketW

int main(){
  unsigned int ts;
  unsigned int fts = 0;
  unsigned int ftst;
  SWUnixSocket ss;

  fd_set pollset;
  struct timeval timeout;

  pollfd cinfd[1];
  cinfd[0].fd = fileno(stdin);
  cinfd[0].events = POLLIN;

  //first timestamp set
  firsttime = getNowMS();

  #ifdef DEBUG
  fprintf(stderr, "Doing handshake...\n");
  #endif
  if (doHandshake()){
    #ifdef DEBUG
    fprintf(stderr, "Handshake succcess!\n");
    #endif
  }else{
    #ifdef DEBUG
    fprintf(stderr, "Handshake fail!\n");
    #endif
    return 0;
  }
  #ifdef DEBUG
  fprintf(stderr, "Starting processing...\n");
  #endif
  int infile = fileno(stdin);
  while (std::cin.good() && std::cout.good()){
    timeout.tv_sec = 1; timeout.tv_usec = 0;
    FD_ZERO(&pollset);//clear the polling set
    FD_SET(infile, &pollset);//add stdin to polling set
    select(1, &pollset, 0, 0, &timeout);
    //only parse input from stdin if available or not yet init'ed
    if ((!ready4data || (snd_cnt - snd_window_at >= snd_window_size)) && poll(cinfd, 1, (signed int)500)){parseChunk();fflush(stdout);}
    if (ready4data){
      if (!inited){
        //we are ready, connect the socket!
        if (!ss.connect(streamname.c_str())){
          #ifdef DEBUG
          fprintf(stderr, "Could not connect to server!\n");
          #endif
          return 0;
        }
        FLV_Readheader(ss);//read the header, we don't want it
        #ifdef DEBUG
        fprintf(stderr, "Header read, starting to send video data...\n");
        #endif
        inited = true;
      }
      //only send data if previous data has been ACK'ed...
      if (snd_cnt - snd_window_at < snd_window_size){
        if (FLV_GetPacket(ss)){//able to read a full packet?
          ts = FLVbuffer[7] * 256*256*256;
          ts += FLVbuffer[4] * 256*256;
          ts += FLVbuffer[5] * 256;
          ts += FLVbuffer[6];
          if (ts != 0){
            if (fts == 0){fts = ts;ftst = getNowMS();}
            ts -= fts;
            FLVbuffer[7] = ts / (256*256*256);
            FLVbuffer[4] = ts / (256*256);
            FLVbuffer[5] = ts / 256;
            FLVbuffer[6] = ts % 256;
            ts += ftst;
          }else{
            ftst = getNowMS();
            FLVbuffer[7] = ftst / (256*256*256);
            FLVbuffer[4] = ftst / (256*256);
            FLVbuffer[5] = ftst / 256;
            FLVbuffer[6] = ftst % 256;
          }
          SendMedia((unsigned char)FLVbuffer[0], (unsigned char *)FLVbuffer+11, FLV_len-15, ts);
          FLV_Dump();//dump packet and get ready for next
        }
        if ((SWBerr != SWBaseSocket::ok) && (SWBerr != SWBaseSocket::notReady)){
          #ifdef DEBUG
          fprintf(stderr, "No more data! :-(  (%s)\n", SWBerr.get_error().c_str());
          #endif
          return 0;//no more input possible! Fail immediately.
        }
      }
    }
    //send ACK if we received a whole window
    if (rec_cnt - rec_window_at > rec_window_size){
      rec_window_at = rec_cnt;
      SendCTL(3, rec_cnt);//send ack (msg 3)
    }
  }
  #ifdef DEBUG
  fprintf(stderr, "User disconnected.\n");
  #endif
  return 0;
}//main
<commit_msg>Polling poging 3<commit_after>#undef DEBUG
#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <cmath>

//needed for select
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>

//needed for poll
#include <poll.h>

//for connection to server
#include "../sockets/SocketW.h"
bool ready4data = false;//set to true when streaming starts
bool inited = false;
bool stopparsing = false;
timeval lastrec;

#include "parsechunks.cpp" //chunkstream parsing
#include "handshake.cpp" //handshaking
#include "../util/flv_sock.cpp" //FLV parsing with SocketW

int main(){
  unsigned int ts;
  unsigned int fts = 0;
  unsigned int ftst;
  SWUnixSocket ss;

  fd_set pollset;
  struct timeval timeout;

  pollfd cinfd[1];
  cinfd[0].fd = fileno(stdin);
  cinfd[0].events = POLLIN;

  //first timestamp set
  firsttime = getNowMS();

  #ifdef DEBUG
  fprintf(stderr, "Doing handshake...\n");
  #endif
  if (doHandshake()){
    #ifdef DEBUG
    fprintf(stderr, "Handshake succcess!\n");
    #endif
  }else{
    #ifdef DEBUG
    fprintf(stderr, "Handshake fail!\n");
    #endif
    return 0;
  }
  #ifdef DEBUG
  fprintf(stderr, "Starting processing...\n");
  #endif
  int infile = fileno(stdin);
  while (std::cin.good() && std::cout.good()){
    //only parse input from stdin if available or not yet init'ed
    if ((!ready4data || (snd_cnt - snd_window_at >= snd_window_size)) && poll(cinfd, 1, (signed int)500)){parseChunk();fflush(stdout);}
    if (ready4data){
      if (!inited){
        //we are ready, connect the socket!
        if (!ss.connect(streamname.c_str())){
          #ifdef DEBUG
          fprintf(stderr, "Could not connect to server!\n");
          #endif
          return 0;
        }
        FLV_Readheader(ss);//read the header, we don't want it
        #ifdef DEBUG
        fprintf(stderr, "Header read, starting to send video data...\n");
        #endif
        inited = true;
      }
      //only send data if previous data has been ACK'ed...
      if (snd_cnt - snd_window_at < snd_window_size){
        if (FLV_GetPacket(ss)){//able to read a full packet?
          ts = FLVbuffer[7] * 256*256*256;
          ts += FLVbuffer[4] * 256*256;
          ts += FLVbuffer[5] * 256;
          ts += FLVbuffer[6];
          if (ts != 0){
            if (fts == 0){fts = ts;ftst = getNowMS();}
            ts -= fts;
            FLVbuffer[7] = ts / (256*256*256);
            FLVbuffer[4] = ts / (256*256);
            FLVbuffer[5] = ts / 256;
            FLVbuffer[6] = ts % 256;
            ts += ftst;
          }else{
            ftst = getNowMS();
            FLVbuffer[7] = ftst / (256*256*256);
            FLVbuffer[4] = ftst / (256*256);
            FLVbuffer[5] = ftst / 256;
            FLVbuffer[6] = ftst % 256;
          }
          SendMedia((unsigned char)FLVbuffer[0], (unsigned char *)FLVbuffer+11, FLV_len-15, ts);
          FLV_Dump();//dump packet and get ready for next
        }
        if ((SWBerr != SWBaseSocket::ok) && (SWBerr != SWBaseSocket::notReady)){
          #ifdef DEBUG
          fprintf(stderr, "No more data! :-(  (%s)\n", SWBerr.get_error().c_str());
          #endif
          return 0;//no more input possible! Fail immediately.
        }
      }
    }
    //send ACK if we received a whole window
    if (rec_cnt - rec_window_at > rec_window_size){
      rec_window_at = rec_cnt;
      SendCTL(3, rec_cnt);//send ack (msg 3)
    }
  }
  #ifdef DEBUG
  fprintf(stderr, "User disconnected.\n");
  #endif
  return 0;
}//main
<|endoftext|>


<commit_before>/*
 * opencog/learning/PatternMiner/Parameters.cc
 *
 * Copyright (C) 2017 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Nil Geisweiller
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <thread>

#include <boost/range/algorithm/remove.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/case_conv.hpp>

#include <opencog/util/Config.h>
#include <opencog/util/oc_assert.h>
#include <opencog/atoms/base/ClassServer.h>

#include "Parameters.h"

namespace opencog { namespace PatternMining {

using namespace std;

void Parameters::reSetAllSettingsFromConfig()
{
	int max_gram = config().get_int("Pattern_Max_Gram");
	MAX_GRAM = (unsigned int)max_gram;

	pattern_mining_mode = config().get("Pattern_mining_mode"); // option: Breadth_First , Depth_First
    OC_ASSERT(pattern_mining_mode == "Breadth_First" || pattern_mining_mode == "Depth_First");

	enable_interesting_pattern = config().get_bool("Enable_Interesting_Pattern");
	enable_interaction_information = config().get_bool("Enable_Interaction_Information");
	enable_surprisingness = config().get_bool("Enable_surprisingness");

	THREAD_NUM = config().get_int("Max_thread_num");
	unsigned int system_thread_num = std::thread::hardware_concurrency();
	if (THREAD_NUM > system_thread_num - 1)
	{
		THREAD_NUM = std::max(1U, system_thread_num - 1);

		cout << "\nThere is only " << system_thread_num
		     << " cores in this machine, so the Max_thread_num = "
		     << config().get_int("Max_thread_num")
		     << " will not be used. " << THREAD_NUM
		     << " threads will be used instead." << std::endl;
	}

	threshold_frequency = config().get_int("Frequency_threshold");

	max_var_num_percent = config().get_double("max_var_num_percent");

	if_quote_output_pattern = config().get_bool("if_quote_output_pattern");
	string quotedTypeStr = config().get("output_pattern_quoted_linktype");
//    cout << "quotedTypeStr = " << quotedTypeStr << std::endl;
	output_pattern_quoted_linktype = classserver().getType(quotedTypeStr);
	if (output_pattern_quoted_linktype == NOTYPE)
	{
		cout << "\nError: output_pattern_quoted_linktype : "<< quotedTypeStr << " in config file doesn't exist!" << std::endl;
	}

	calculate_type_b_surprisingness = config().get_bool("calculate_type_b_surprisingness");

	use_keyword_black_list = config().get_bool("use_keyword_black_list");
	use_keyword_white_list = config().get_bool("use_keyword_white_list");

	keyword_black_logic_is_contain = config().get_bool("keyword_black_logic_is_contain");

	string keyword_black_list_str  = config().get("keyword_black_list");
	keyword_black_list = parse_comma_separated_set(keyword_black_list_str);

	string keyword_white_list_str  = config().get("keyword_white_list");
	keyword_white_list = parse_comma_separated_set(keyword_white_list_str);

	string keyword_white_list_logic_str = config().get("keyword_white_list_logic");
	keyword_white_list_logic =
		boost::algorithm::to_lower_copy(keyword_white_list_logic_str) == "and" ?
		QUERY_LOGIC::AND : QUERY_LOGIC::OR;

	use_linktype_black_list = config().get_bool("use_linktype_black_list");
	use_linktype_white_list = config().get_bool("use_linktype_white_list");

	// use_linktype_black_list and use_linktype_white_list should not both be true
	assert((! use_linktype_black_list) || (! use_linktype_white_list));

	linktype_black_list.clear();
	string linktype_black_list_str = config().get("linktype_black_list");
	addAtomTypesFromString(linktype_black_list_str, linktype_black_list);

	linktype_white_list.clear();
	string linktype_white_list_str = config().get("linktype_white_list");
	addAtomTypesFromString(linktype_white_list_str, linktype_white_list);

	enable_filter_leaves_should_not_be_vars = config().get_bool("enable_filter_leaves_should_not_be_vars");
	enable_filter_links_should_connect_by_vars = config().get_bool("enable_filter_links_should_connect_by_vars");
	enable_filter_node_types_should_not_be_vars =  config().get_bool("enable_filter_node_types_should_not_be_vars");
	enable_filter_node_types_should_be_vars =  config().get_bool("enable_filter_node_types_should_be_vars");
	enable_filter_links_of_same_type_not_share_second_outgoing = config().get_bool("enable_filter_links_of_same_type_not_share_second_outgoing");
	enable_filter_not_all_first_outgoing_const = config().get_bool("enable_filter_not_all_first_outgoing_const");
	enable_filter_not_same_var_from_same_predicate = config().get_bool("enable_filter_not_same_var_from_same_predicate");
	enable_filter_first_outgoing_evallink_should_be_var = config().get_bool("enable_filter_first_outgoing_evallink_should_be_var");

	node_types_should_not_be_vars.clear();
	string node_types_str = config().get("node_types_should_not_be_vars");
	addAtomTypesFromString(node_types_str, node_types_should_not_be_vars);

	node_types_should_be_vars.clear();
	node_types_str = config().get("node_types_should_be_vars");
	addAtomTypesFromString(node_types_str, node_types_should_be_vars);

	same_link_types_not_share_second_outgoing.clear();
	string link_types_str = config().get("same_link_types_not_share_second_outgoing");
	addAtomTypesFromString(link_types_str, same_link_types_not_share_second_outgoing);

	only_mine_patterns_start_from_white_list = config().get_bool("only_mine_patterns_start_from_white_list");
	only_mine_patterns_start_from_white_list_contain = config().get_bool("only_mine_patterns_start_from_white_list_contain");

	only_output_patterns_contains_white_keywords = config().get_bool("only_output_patterns_contains_white_keywords");

	enable_unify_unordered_links = config().get_bool("enable_unify_unordered_links");
}

bool Parameters::add_linktype_to_white_list(Type _type)
{
    return linktype_white_list.insert(_type).second;
}

bool Parameters::remove_linktype_from_white_list(Type _type)
{
    return linktype_white_list.erase(_type);
}

bool Parameters::add_ignore_link_type(Type _type)
{
    return linktype_black_list.insert(_type).second;
}

bool Parameters::remove_ignore_link_type(Type _type)
{
    return linktype_black_list.erase(_type);
}

bool Parameters::add_keyword_to_black_list(const string& _keyword)
{
    if (_keyword == "")
        return false;
    return keyword_black_list.insert(_keyword).second;
}

bool Parameters::remove_keyword_from_black_list(const string& _keyword)
{
    return keyword_black_list.erase(_keyword);
}

bool Parameters::add_keyword_to_white_list(const string& _keyword)
{
    return keyword_white_list.insert(_keyword).second;
}

bool Parameters::remove_keyword_from_white_list(const string& _keyword)
{
    return keyword_white_list.erase(_keyword);
}

bool Parameters::add_link_type_to_same_link_types_not_share_second_outgoing(Type _type)
{
    return same_link_types_not_share_second_outgoing.insert(_type).second;
}

bool Parameters::remove_link_type_from_same_link_types_not_share_second_outgoing(Type _type)
{
    return same_link_types_not_share_second_outgoing.erase(_type);
}

bool Parameters::add_node_type_to_node_types_should_not_be_vars(Type _type)
{
    return node_types_should_not_be_vars.insert(_type).second;
}

bool Parameters::remove_node_type_from_node_types_should_not_be_vars(Type _type)
{
    return node_types_should_not_be_vars.erase(_type);
}

bool Parameters::add_node_type_to_node_types_should_be_vars(Type _type)
{
    return node_types_should_be_vars.insert(_type).second;
}

bool Parameters::remove_node_type_from_node_types_should_be_vars(Type _type)
{
    return node_types_should_be_vars.erase(_type);
}

void Parameters::addAtomTypesFromString(const string& node_types_str,
                                        set<Type>& types)
{
    for (const string& typestr : parse_comma_separated_list(node_types_str))
    {
        Type atomType = classserver().getType(typestr);
        if (atomType == NOTYPE)
        {
            cout << "\nCannot find Node Type: " << typestr << " in config file.\n";
            continue;
        }
        types.insert(atomType);
    }
}

vector<string> Parameters::parse_comma_separated_list(string str)
{
    str.erase(boost::remove(str, ' '), str.end());
    vector<string> strs;
    boost::split(strs, str, boost::is_any_of(","));
    return strs;
}

set<string> Parameters::parse_comma_separated_set(string str)
{
    vector<string> l = parse_comma_separated_list(str);
    return set<string>(l.begin(), l.end());
}

}}
<commit_msg>Try to avoid a pointless crash<commit_after>/*
 * opencog/learning/PatternMiner/Parameters.cc
 *
 * Copyright (C) 2017 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Nil Geisweiller
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <thread>

#include <boost/range/algorithm/remove.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/case_conv.hpp>

#include <opencog/util/Config.h>
#include <opencog/util/oc_assert.h>
#include <opencog/atoms/base/ClassServer.h>

#include "Parameters.h"

namespace opencog { namespace PatternMining {

using namespace std;

void Parameters::reSetAllSettingsFromConfig()
{
	pattern_mining_mode = config().get("Pattern_mining_mode");

	if (nullptr == pattern_mining_mode)
	{
		config().load("opencog_patternminer.conf", true);
		pattern_mining_mode = config().get("Pattern_mining_mode");
	}

	OC_ASSERT(pattern_mining_mode == "Breadth_First" || pattern_mining_mode == "Depth_First");

	int max_gram = config().get_int("Pattern_Max_Gram");
	MAX_GRAM = (unsigned int)max_gram;

	enable_interesting_pattern = config().get_bool("Enable_Interesting_Pattern");
	enable_interaction_information = config().get_bool("Enable_Interaction_Information");
	enable_surprisingness = config().get_bool("Enable_surprisingness");

	THREAD_NUM = config().get_int("Max_thread_num");
	unsigned int system_thread_num = std::thread::hardware_concurrency();
	if (THREAD_NUM > system_thread_num - 1)
	{
		THREAD_NUM = std::max(1U, system_thread_num - 1);

		cout << "\nThere is only " << system_thread_num
		     << " cores in this machine, so the Max_thread_num = "
		     << config().get_int("Max_thread_num")
		     << " will not be used. " << THREAD_NUM
		     << " threads will be used instead." << std::endl;
	}

	threshold_frequency = config().get_int("Frequency_threshold");

	max_var_num_percent = config().get_double("max_var_num_percent");

	if_quote_output_pattern = config().get_bool("if_quote_output_pattern");
	string quotedTypeStr = config().get("output_pattern_quoted_linktype");
//    cout << "quotedTypeStr = " << quotedTypeStr << std::endl;
	output_pattern_quoted_linktype = classserver().getType(quotedTypeStr);
	if (output_pattern_quoted_linktype == NOTYPE)
	{
		cout << "\nError: output_pattern_quoted_linktype : "<< quotedTypeStr << " in config file doesn't exist!" << std::endl;
	}

	calculate_type_b_surprisingness = config().get_bool("calculate_type_b_surprisingness");

	use_keyword_black_list = config().get_bool("use_keyword_black_list");
	use_keyword_white_list = config().get_bool("use_keyword_white_list");

	keyword_black_logic_is_contain = config().get_bool("keyword_black_logic_is_contain");

	string keyword_black_list_str  = config().get("keyword_black_list");
	keyword_black_list = parse_comma_separated_set(keyword_black_list_str);

	string keyword_white_list_str  = config().get("keyword_white_list");
	keyword_white_list = parse_comma_separated_set(keyword_white_list_str);

	string keyword_white_list_logic_str = config().get("keyword_white_list_logic");
	keyword_white_list_logic =
		boost::algorithm::to_lower_copy(keyword_white_list_logic_str) == "and" ?
		QUERY_LOGIC::AND : QUERY_LOGIC::OR;

	use_linktype_black_list = config().get_bool("use_linktype_black_list");
	use_linktype_white_list = config().get_bool("use_linktype_white_list");

	// use_linktype_black_list and use_linktype_white_list should not both be true
	assert((! use_linktype_black_list) || (! use_linktype_white_list));

	linktype_black_list.clear();
	string linktype_black_list_str = config().get("linktype_black_list");
	addAtomTypesFromString(linktype_black_list_str, linktype_black_list);

	linktype_white_list.clear();
	string linktype_white_list_str = config().get("linktype_white_list");
	addAtomTypesFromString(linktype_white_list_str, linktype_white_list);

	enable_filter_leaves_should_not_be_vars = config().get_bool("enable_filter_leaves_should_not_be_vars");
	enable_filter_links_should_connect_by_vars = config().get_bool("enable_filter_links_should_connect_by_vars");
	enable_filter_node_types_should_not_be_vars =  config().get_bool("enable_filter_node_types_should_not_be_vars");
	enable_filter_node_types_should_be_vars =  config().get_bool("enable_filter_node_types_should_be_vars");
	enable_filter_links_of_same_type_not_share_second_outgoing = config().get_bool("enable_filter_links_of_same_type_not_share_second_outgoing");
	enable_filter_not_all_first_outgoing_const = config().get_bool("enable_filter_not_all_first_outgoing_const");
	enable_filter_not_same_var_from_same_predicate = config().get_bool("enable_filter_not_same_var_from_same_predicate");
	enable_filter_first_outgoing_evallink_should_be_var = config().get_bool("enable_filter_first_outgoing_evallink_should_be_var");

	node_types_should_not_be_vars.clear();
	string node_types_str = config().get("node_types_should_not_be_vars");
	addAtomTypesFromString(node_types_str, node_types_should_not_be_vars);

	node_types_should_be_vars.clear();
	node_types_str = config().get("node_types_should_be_vars");
	addAtomTypesFromString(node_types_str, node_types_should_be_vars);

	same_link_types_not_share_second_outgoing.clear();
	string link_types_str = config().get("same_link_types_not_share_second_outgoing");
	addAtomTypesFromString(link_types_str, same_link_types_not_share_second_outgoing);

	only_mine_patterns_start_from_white_list = config().get_bool("only_mine_patterns_start_from_white_list");
	only_mine_patterns_start_from_white_list_contain = config().get_bool("only_mine_patterns_start_from_white_list_contain");

	only_output_patterns_contains_white_keywords = config().get_bool("only_output_patterns_contains_white_keywords");

	enable_unify_unordered_links = config().get_bool("enable_unify_unordered_links");
}

bool Parameters::add_linktype_to_white_list(Type _type)
{
    return linktype_white_list.insert(_type).second;
}

bool Parameters::remove_linktype_from_white_list(Type _type)
{
    return linktype_white_list.erase(_type);
}

bool Parameters::add_ignore_link_type(Type _type)
{
    return linktype_black_list.insert(_type).second;
}

bool Parameters::remove_ignore_link_type(Type _type)
{
    return linktype_black_list.erase(_type);
}

bool Parameters::add_keyword_to_black_list(const string& _keyword)
{
    if (_keyword == "")
        return false;
    return keyword_black_list.insert(_keyword).second;
}

bool Parameters::remove_keyword_from_black_list(const string& _keyword)
{
    return keyword_black_list.erase(_keyword);
}

bool Parameters::add_keyword_to_white_list(const string& _keyword)
{
    return keyword_white_list.insert(_keyword).second;
}

bool Parameters::remove_keyword_from_white_list(const string& _keyword)
{
    return keyword_white_list.erase(_keyword);
}

bool Parameters::add_link_type_to_same_link_types_not_share_second_outgoing(Type _type)
{
    return same_link_types_not_share_second_outgoing.insert(_type).second;
}

bool Parameters::remove_link_type_from_same_link_types_not_share_second_outgoing(Type _type)
{
    return same_link_types_not_share_second_outgoing.erase(_type);
}

bool Parameters::add_node_type_to_node_types_should_not_be_vars(Type _type)
{
    return node_types_should_not_be_vars.insert(_type).second;
}

bool Parameters::remove_node_type_from_node_types_should_not_be_vars(Type _type)
{
    return node_types_should_not_be_vars.erase(_type);
}

bool Parameters::add_node_type_to_node_types_should_be_vars(Type _type)
{
    return node_types_should_be_vars.insert(_type).second;
}

bool Parameters::remove_node_type_from_node_types_should_be_vars(Type _type)
{
    return node_types_should_be_vars.erase(_type);
}

void Parameters::addAtomTypesFromString(const string& node_types_str,
                                        set<Type>& types)
{
    for (const string& typestr : parse_comma_separated_list(node_types_str))
    {
        Type atomType = classserver().getType(typestr);
        if (atomType == NOTYPE)
        {
            cout << "\nCannot find Node Type: " << typestr << " in config file.\n";
            continue;
        }
        types.insert(atomType);
    }
}

vector<string> Parameters::parse_comma_separated_list(string str)
{
    str.erase(boost::remove(str, ' '), str.end());
    vector<string> strs;
    boost::split(strs, str, boost::is_any_of(","));
    return strs;
}

set<string> Parameters::parse_comma_separated_set(string str)
{
    vector<string> l = parse_comma_separated_list(str);
    return set<string>(l.begin(), l.end());
}

}}
<|endoftext|>