<commit_before>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Da Zheng (zhengda1936@gmail.com)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <string.h>
#include <assert.h>

#include <vector>
#include <string>

#include "common.h"

#include "native_file.h"
#include "log.h"
#include "FG_basic_types.h"
#include "vertex.h"
#include "graph_file_header.h"
#include "vertex_index.h"
#include "in_mem_storage.h"

#include "generic_type.h"
#include "data_io.h"
#include "data_frame.h"
#include "vector.h"
#include "vector_vector.h"
#include "fm_utils.h"
#include "matrix_config.h"
#include "sparse_matrix.h"

using namespace fm;

/*
 * This class parses a line into an edge (source, destination).
 */
class edge_parser: public line_parser
{
public:
	size_t parse(const std::vector<std::string> &lines, data_frame &df) const;
	size_t get_num_cols() const {
		return 2;
	}

	std::string get_col_name(off_t idx) const {
		if (idx == 0)
			return "source";
		else
			return "dest";
	}

	const scalar_type &get_col_type(off_t idx) const {
		return get_scalar_type<fg::vertex_id_t>();
	}
};

size_t edge_parser::parse(const std::vector<std::string> &lines,
		data_frame &df) const
{
	detail::smp_vec_store::ptr froms = detail::smp_vec_store::create(lines.size(),
			get_scalar_type<fg::vertex_id_t>());
	detail::smp_vec_store::ptr tos = detail::smp_vec_store::create(lines.size(),
			get_scalar_type<fg::vertex_id_t>());
	for (size_t i = 0; i < lines.size(); i++) {
		const char *line = lines[i].c_str();
		int len = strlen(line);
		const char *first = line;
		// Skip space
		for (; isspace(*first); first++);
		if (*first == '#')
			continue;

		// Make sure we get a number.
		if (!isdigit(*first)) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("the first entry isn't a number: ") + first;
			continue;
		}
		fg::vertex_id_t from = atol(first);
		assert(from >= 0 && from < fg::MAX_VERTEX_ID);

		const char *second = first;
		// Go to the end of the first number.
		for (; isdigit(*second); second++);
		if (second - line == len) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("there isn't second entry: ") + line;
			continue;
		}
		// Skip space between two numbers.
		for (; isspace(*second); second++);
		// Make sure we get a number.
		if (!isdigit(*second)) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("the second entry isn't a number: ") + second;
			continue;
		}
		fg::vertex_id_t to = atol(second);
		assert(to >= 0 && to < fg::MAX_VERTEX_ID);

		froms->set(i, from);
		tos->set(i, to);
	}

	df.get_vec(0)->append(*froms);
	df.get_vec(1)->append(*tos);
	return froms->get_length();
}

void print_usage()
{
	fprintf(stderr, "convert an edge list to adjacency lists\n");
	fprintf(stderr, "el2al [options] conf_file edge_file graph_name\n");
	fprintf(stderr, "-u: undirected graph\n");
	fprintf(stderr, "-e: use external memory\n");
	fprintf(stderr, "-s size: sort buffer size\n");
	fprintf(stderr, "-g size: groupby buffer size\n");
}

int main(int argc, char *argv[])
{
	bool directed = true;
	bool in_mem = true;
	size_t sort_buf_size = 1UL * 1024 * 1024 * 1024;
	size_t groupby_buf_size = 1UL * 1024 * 1024 * 1024;
	int opt;
	int num_opts = 0;
	while ((opt = getopt(argc, argv, "ues:g:")) != -1) {
		num_opts++;
		switch (opt) {
			case 'u':
				directed = false;
				break;
			case 'e':
				in_mem = false;
				break;
			case 's':
				sort_buf_size = str2size(optarg);
				num_opts++;
				break;
			case 'g':
				groupby_buf_size = str2size(optarg);
				num_opts++;
				break;
			default:
				print_usage();
				exit(1);
		}
	}

	argv += 1 + num_opts;
	argc -= 1 + num_opts;
	if (argc < 2) {
		print_usage();
		exit(1);
	}

	std::string conf_file = argv[0];
	std::string file_name = argv[1];
	std::string graph_name = argv[2];
	std::string adj_file = graph_name + ".adj";
	std::string index_file = graph_name + ".index";

	std::vector<std::string> files;
	safs::native_file f(file_name);
	if (f.exist() && !f.is_dir())
		files.push_back(file_name);
	else if (f.exist() && f.is_dir()) {
		safs::native_dir d(file_name);
		d.read_all_files(files);
		for (size_t i = 0; i < files.size(); i++)
			files[i] = file_name + "/" + files[i];
	}
	else {
		fprintf(stderr, "The input file %s doesn't exist\n", file_name.c_str());
		return -1;
	}
	printf("read edges from %ld files\n", files.size());

	config_map::ptr configs = config_map::create(conf_file);
	init_flash_matrix(configs);
	matrix_conf.set_sort_buf_size(sort_buf_size);
	matrix_conf.set_groupby_buf_size(groupby_buf_size);
	printf("sort buf size: %ld, groupby buf size: %ld\n",
			matrix_conf.get_sort_buf_size(), matrix_conf.get_groupby_buf_size());

	struct timeval start, end;
	edge_parser parser;
	/*
	 * We only need to indicate here whether we use external memory or not.
	 * If the columns in the data frame are in external memory, it'll always
	 * use external data containers for the remaining of processing.
	 */
	printf("start to read and parse edge list\n");
	gettimeofday(&start, NULL);
	data_frame::ptr df = read_lines(files, parser, in_mem);
	gettimeofday(&end, NULL);
	printf("It takes %.3f seconds to parse the edge lists\n",
			time_diff(start, end));
	printf("There are %ld edges\n", df->get_num_entries());

	fg::vertex_id_t max_vid = 0;
	gettimeofday(&start, NULL);
	for (size_t i = 0; i < df->get_num_vecs(); i++) {
		vector::ptr vec = vector::create(df->get_vec(i));
		max_vid = std::max(max_vid, vec->max<fg::vertex_id_t>());
	}
	gettimeofday(&end, NULL);
	printf("It takes %.3f seconds to get the maximal vertex ID\n",
			time_diff(start, end));
	printf("max id: %d\n", max_vid);
	detail::vec_store::ptr seq_vec = detail::create_vec_store<fg::vertex_id_t>(
			0, max_vid, 1);
	detail::vec_store::ptr rep_vec = detail::create_vec_store<fg::vertex_id_t>(
			max_vid + 1, fg::INVALID_VERTEX_ID);
	assert(seq_vec->get_length() == rep_vec->get_length());

	fg::FG_graph::ptr graph;
	if (directed) {
		gettimeofday(&start, NULL);
		// I artificially add an invalid out-edge for each vertex, so it's
		// guaranteed that each vertex exists in the adjacency lists.
		data_frame::ptr new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), seq_vec);
		new_df->add_vec(df->get_vec_name(1), rep_vec);
		df->append(new_df);

		// I artificially add an invalid in-edge for each vertex.
		new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(1), seq_vec);
		new_df->add_vec(df->get_vec_name(0), rep_vec);
		df->append(new_df);
		gettimeofday(&end, NULL);
		printf("It takes %.3f seconds to prepare for constructing a graph\n",
				time_diff(start, end));

		printf("start to construct FlashGraph graph\n");
		graph = create_fg_graph(graph_name, df, true);
	}
	else {
		gettimeofday(&start, NULL);
		// I artificially add an invalid out-edge for each vertex, so it's
		// guaranteed that each vertex exists in the adjacency lists.
		data_frame::ptr new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), seq_vec);
		new_df->add_vec(df->get_vec_name(1), rep_vec);
		df->append(new_df);

		detail::vec_store::ptr vec0 = df->get_vec(0)->deep_copy();
		detail::vec_store::ptr vec1 = df->get_vec(1)->deep_copy();
		vec0->append(df->get_vec_ref(1));
		vec1->append(df->get_vec_ref(0));
		new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), vec0);
		new_df->add_vec(df->get_vec_name(1), vec1);
		df = new_df;
		gettimeofday(&end, NULL);
		printf("It takes %.3f seconds to prepare for constructing a graph\n",
				time_diff(start, end));

		printf("start to construct FlashGraph graph\n");
		graph = create_fg_graph(graph_name, df, false);
	}

	if (graph->get_index_data())
		graph->get_index_data()->dump(index_file);
	if (graph->get_graph_data())
		graph->get_graph_data()->dump(adj_file);

	df = NULL;
	destroy_flash_matrix();

	return 0;
}
<commit_msg>[Matrix]: fix a bug in parse text edges in el2al.<commit_after>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Da Zheng (zhengda1936@gmail.com)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <string.h>
#include <assert.h>

#include <vector>
#include <string>

#include "common.h"

#include "native_file.h"
#include "log.h"
#include "FG_basic_types.h"
#include "vertex.h"
#include "graph_file_header.h"
#include "vertex_index.h"
#include "in_mem_storage.h"

#include "generic_type.h"
#include "data_io.h"
#include "data_frame.h"
#include "vector.h"
#include "vector_vector.h"
#include "fm_utils.h"
#include "matrix_config.h"
#include "sparse_matrix.h"

using namespace fm;

/*
 * This class parses a line into an edge (source, destination).
 */
class edge_parser: public line_parser
{
public:
	size_t parse(const std::vector<std::string> &lines, data_frame &df) const;
	size_t get_num_cols() const {
		return 2;
	}

	std::string get_col_name(off_t idx) const {
		if (idx == 0)
			return "source";
		else
			return "dest";
	}

	const scalar_type &get_col_type(off_t idx) const {
		return get_scalar_type<fg::vertex_id_t>();
	}
};

size_t edge_parser::parse(const std::vector<std::string> &lines,
		data_frame &df) const
{
	detail::smp_vec_store::ptr froms = detail::smp_vec_store::create(lines.size(),
			get_scalar_type<fg::vertex_id_t>());
	detail::smp_vec_store::ptr tos = detail::smp_vec_store::create(lines.size(),
			get_scalar_type<fg::vertex_id_t>());
	size_t entry_idx = 0;
	for (size_t i = 0; i < lines.size(); i++) {
		const char *line = lines[i].c_str();
		int len = strlen(line);
		const char *first = line;
		// Skip space
		for (; isspace(*first); first++);
		if (*first == '#')
			continue;

		// Make sure we get a number.
		if (!isdigit(*first)) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("the first entry isn't a number: ") + first;
			continue;
		}
		fg::vertex_id_t from = atol(first);
		assert(from >= 0 && from < fg::MAX_VERTEX_ID);

		const char *second = first;
		// Go to the end of the first number.
		for (; isdigit(*second); second++);
		if (second - line == len) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("there isn't second entry: ") + line;
			continue;
		}
		// Skip space between two numbers.
		for (; isspace(*second); second++);
		// Make sure we get a number.
		if (!isdigit(*second)) {
			BOOST_LOG_TRIVIAL(error)
				<< std::string("the second entry isn't a number: ") + second;
			continue;
		}
		fg::vertex_id_t to = atol(second);
		assert(to >= 0 && to < fg::MAX_VERTEX_ID);

		froms->set(entry_idx, from);
		tos->set(entry_idx, to);
		entry_idx++;
	}
	froms->resize(entry_idx);
	tos->resize(entry_idx);

	df.get_vec(0)->append(*froms);
	df.get_vec(1)->append(*tos);
	return froms->get_length();
}

void print_usage()
{
	fprintf(stderr, "convert an edge list to adjacency lists\n");
	fprintf(stderr, "el2al [options] conf_file edge_file graph_name\n");
	fprintf(stderr, "-u: undirected graph\n");
	fprintf(stderr, "-e: use external memory\n");
	fprintf(stderr, "-s size: sort buffer size\n");
	fprintf(stderr, "-g size: groupby buffer size\n");
}

int main(int argc, char *argv[])
{
	bool directed = true;
	bool in_mem = true;
	size_t sort_buf_size = 1UL * 1024 * 1024 * 1024;
	size_t groupby_buf_size = 1UL * 1024 * 1024 * 1024;
	int opt;
	int num_opts = 0;
	while ((opt = getopt(argc, argv, "ues:g:")) != -1) {
		num_opts++;
		switch (opt) {
			case 'u':
				directed = false;
				break;
			case 'e':
				in_mem = false;
				break;
			case 's':
				sort_buf_size = str2size(optarg);
				num_opts++;
				break;
			case 'g':
				groupby_buf_size = str2size(optarg);
				num_opts++;
				break;
			default:
				print_usage();
				exit(1);
		}
	}

	argv += 1 + num_opts;
	argc -= 1 + num_opts;
	if (argc < 2) {
		print_usage();
		exit(1);
	}

	std::string conf_file = argv[0];
	std::string file_name = argv[1];
	std::string graph_name = argv[2];
	std::string adj_file = graph_name + ".adj";
	std::string index_file = graph_name + ".index";

	std::vector<std::string> files;
	safs::native_file f(file_name);
	if (f.exist() && !f.is_dir())
		files.push_back(file_name);
	else if (f.exist() && f.is_dir()) {
		safs::native_dir d(file_name);
		d.read_all_files(files);
		for (size_t i = 0; i < files.size(); i++)
			files[i] = file_name + "/" + files[i];
	}
	else {
		fprintf(stderr, "The input file %s doesn't exist\n", file_name.c_str());
		return -1;
	}
	printf("read edges from %ld files\n", files.size());

	config_map::ptr configs = config_map::create(conf_file);
	init_flash_matrix(configs);
	matrix_conf.set_sort_buf_size(sort_buf_size);
	matrix_conf.set_groupby_buf_size(groupby_buf_size);
	printf("sort buf size: %ld, groupby buf size: %ld\n",
			matrix_conf.get_sort_buf_size(), matrix_conf.get_groupby_buf_size());

	struct timeval start, end;
	edge_parser parser;
	/*
	 * We only need to indicate here whether we use external memory or not.
	 * If the columns in the data frame are in external memory, it'll always
	 * use external data containers for the remaining of processing.
	 */
	printf("start to read and parse edge list\n");
	gettimeofday(&start, NULL);
	data_frame::ptr df = read_lines(files, parser, in_mem);
	gettimeofday(&end, NULL);
	printf("It takes %.3f seconds to parse the edge lists\n",
			time_diff(start, end));
	printf("There are %ld edges\n", df->get_num_entries());

	fg::vertex_id_t max_vid = 0;
	gettimeofday(&start, NULL);
	for (size_t i = 0; i < df->get_num_vecs(); i++) {
		vector::ptr vec = vector::create(df->get_vec(i));
		max_vid = std::max(max_vid, vec->max<fg::vertex_id_t>());
	}
	gettimeofday(&end, NULL);
	printf("It takes %.3f seconds to get the maximal vertex ID\n",
			time_diff(start, end));
	printf("max id: %d\n", max_vid);
	detail::vec_store::ptr seq_vec = detail::create_vec_store<fg::vertex_id_t>(
			0, max_vid, 1);
	detail::vec_store::ptr rep_vec = detail::create_vec_store<fg::vertex_id_t>(
			max_vid + 1, fg::INVALID_VERTEX_ID);
	assert(seq_vec->get_length() == rep_vec->get_length());

	fg::FG_graph::ptr graph;
	if (directed) {
		gettimeofday(&start, NULL);
		// I artificially add an invalid out-edge for each vertex, so it's
		// guaranteed that each vertex exists in the adjacency lists.
		data_frame::ptr new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), seq_vec);
		new_df->add_vec(df->get_vec_name(1), rep_vec);
		df->append(new_df);

		// I artificially add an invalid in-edge for each vertex.
		new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(1), seq_vec);
		new_df->add_vec(df->get_vec_name(0), rep_vec);
		df->append(new_df);
		gettimeofday(&end, NULL);
		printf("It takes %.3f seconds to prepare for constructing a graph\n",
				time_diff(start, end));

		printf("start to construct FlashGraph graph\n");
		graph = create_fg_graph(graph_name, df, true);
	}
	else {
		gettimeofday(&start, NULL);
		// I artificially add an invalid out-edge for each vertex, so it's
		// guaranteed that each vertex exists in the adjacency lists.
		data_frame::ptr new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), seq_vec);
		new_df->add_vec(df->get_vec_name(1), rep_vec);
		df->append(new_df);

		detail::vec_store::ptr vec0 = df->get_vec(0)->deep_copy();
		detail::vec_store::ptr vec1 = df->get_vec(1)->deep_copy();
		vec0->append(df->get_vec_ref(1));
		vec1->append(df->get_vec_ref(0));
		new_df = data_frame::create();
		new_df->add_vec(df->get_vec_name(0), vec0);
		new_df->add_vec(df->get_vec_name(1), vec1);
		df = new_df;
		gettimeofday(&end, NULL);
		printf("It takes %.3f seconds to prepare for constructing a graph\n",
				time_diff(start, end));

		printf("start to construct FlashGraph graph\n");
		graph = create_fg_graph(graph_name, df, false);
	}

	if (graph->get_index_data())
		graph->get_index_data()->dump(index_file);
	if (graph->get_graph_data())
		graph->get_graph_data()->dump(adj_file);

	df = NULL;
	destroy_flash_matrix();

	return 0;
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "rtc_base/rate_statistics.h"

#include <algorithm>
#include <memory>

#include "rtc_base/checks.h"

namespace webrtc {

RateStatistics::RateStatistics(int64_t window_size_ms, float scale)
    : buckets_(new Bucket[window_size_ms]()),
      accumulated_count_(0),
      num_samples_(0),
      oldest_time_(-window_size_ms),
      oldest_index_(0),
      scale_(scale),
      max_window_size_ms_(window_size_ms),
      current_window_size_ms_(max_window_size_ms_) {}

RateStatistics::RateStatistics(const RateStatistics& other)
    : accumulated_count_(other.accumulated_count_),
      num_samples_(other.num_samples_),
      oldest_time_(other.oldest_time_),
      oldest_index_(other.oldest_index_),
      scale_(other.scale_),
      max_window_size_ms_(other.max_window_size_ms_),
      current_window_size_ms_(other.current_window_size_ms_) {
  buckets_ = std::make_unique<Bucket[]>(other.max_window_size_ms_);
  std::copy(other.buckets_.get(),
            other.buckets_.get() + other.max_window_size_ms_, buckets_.get());
}

RateStatistics::RateStatistics(RateStatistics&& other) = default;

RateStatistics::~RateStatistics() {}

void RateStatistics::Reset() {
  accumulated_count_ = 0;
  num_samples_ = 0;
  oldest_time_ = -max_window_size_ms_;
  oldest_index_ = 0;
  current_window_size_ms_ = max_window_size_ms_;
  for (int64_t i = 0; i < max_window_size_ms_; i++)
    buckets_[i] = Bucket();
}

void RateStatistics::Update(size_t count, int64_t now_ms) {
  if (now_ms < oldest_time_) {
    // Too old data is ignored.
    return;
  }

  EraseOld(now_ms);

  // First ever sample, reset window to start now.
  if (!IsInitialized())
    oldest_time_ = now_ms;

  uint32_t now_offset = static_cast<uint32_t>(now_ms - oldest_time_);
  RTC_DCHECK_LT(now_offset, max_window_size_ms_);
  uint32_t index = oldest_index_ + now_offset;
  if (index >= max_window_size_ms_)
    index -= max_window_size_ms_;
  buckets_[index].sum += count;
  ++buckets_[index].samples;
  accumulated_count_ += count;
  ++num_samples_;
}

absl::optional<uint32_t> RateStatistics::Rate(int64_t now_ms) const {
  // Yeah, this const_cast ain't pretty, but the alternative is to declare most
  // of the members as mutable...
  const_cast<RateStatistics*>(this)->EraseOld(now_ms);

  // If window is a single bucket or there is only one sample in a data set that
  // has not grown to the full window size, treat this as rate unavailable.
  int64_t active_window_size = now_ms - oldest_time_ + 1;
  if (num_samples_ == 0 || active_window_size <= 1 ||
      (num_samples_ <= 1 && active_window_size < current_window_size_ms_)) {
    return absl::nullopt;
  }

  float scale = scale_ / active_window_size;
  return static_cast<uint32_t>(accumulated_count_ * scale + 0.5f);
}

void RateStatistics::EraseOld(int64_t now_ms) {
  if (!IsInitialized())
    return;

  // New oldest time that is included in data set.
  int64_t new_oldest_time = now_ms - current_window_size_ms_ + 1;

  // New oldest time is older than the current one, no need to cull data.
  if (new_oldest_time <= oldest_time_)
    return;

  // Loop over buckets and remove too old data points.
  while (num_samples_ > 0 && oldest_time_ < new_oldest_time) {
    const Bucket& oldest_bucket = buckets_[oldest_index_];
    RTC_DCHECK_GE(accumulated_count_, oldest_bucket.sum);
    RTC_DCHECK_GE(num_samples_, oldest_bucket.samples);
    accumulated_count_ -= oldest_bucket.sum;
    num_samples_ -= oldest_bucket.samples;
    buckets_[oldest_index_] = Bucket();
    if (++oldest_index_ >= max_window_size_ms_)
      oldest_index_ = 0;
    ++oldest_time_;
  }
  oldest_time_ = new_oldest_time;
}

bool RateStatistics::SetWindowSize(int64_t window_size_ms, int64_t now_ms) {
  if (window_size_ms <= 0 || window_size_ms > max_window_size_ms_)
    return false;

  current_window_size_ms_ = window_size_ms;
  EraseOld(now_ms);
  return true;
}

bool RateStatistics::IsInitialized() const {
  return oldest_time_ != -max_window_size_ms_;
}

}  // namespace webrtc
<commit_msg>Return unavailable rate rather than garbage value.<commit_after>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "rtc_base/rate_statistics.h"

#include <algorithm>
#include <limits>
#include <memory>

#include "rtc_base/checks.h"

namespace webrtc {

RateStatistics::RateStatistics(int64_t window_size_ms, float scale)
    : buckets_(new Bucket[window_size_ms]()),
      accumulated_count_(0),
      num_samples_(0),
      oldest_time_(-window_size_ms),
      oldest_index_(0),
      scale_(scale),
      max_window_size_ms_(window_size_ms),
      current_window_size_ms_(max_window_size_ms_) {}

RateStatistics::RateStatistics(const RateStatistics& other)
    : accumulated_count_(other.accumulated_count_),
      num_samples_(other.num_samples_),
      oldest_time_(other.oldest_time_),
      oldest_index_(other.oldest_index_),
      scale_(other.scale_),
      max_window_size_ms_(other.max_window_size_ms_),
      current_window_size_ms_(other.current_window_size_ms_) {
  buckets_ = std::make_unique<Bucket[]>(other.max_window_size_ms_);
  std::copy(other.buckets_.get(),
            other.buckets_.get() + other.max_window_size_ms_, buckets_.get());
}

RateStatistics::RateStatistics(RateStatistics&& other) = default;

RateStatistics::~RateStatistics() {}

void RateStatistics::Reset() {
  accumulated_count_ = 0;
  num_samples_ = 0;
  oldest_time_ = -max_window_size_ms_;
  oldest_index_ = 0;
  current_window_size_ms_ = max_window_size_ms_;
  for (int64_t i = 0; i < max_window_size_ms_; i++)
    buckets_[i] = Bucket();
}

void RateStatistics::Update(size_t count, int64_t now_ms) {
  if (now_ms < oldest_time_) {
    // Too old data is ignored.
    return;
  }

  EraseOld(now_ms);

  // First ever sample, reset window to start now.
  if (!IsInitialized())
    oldest_time_ = now_ms;

  uint32_t now_offset = static_cast<uint32_t>(now_ms - oldest_time_);
  RTC_DCHECK_LT(now_offset, max_window_size_ms_);
  uint32_t index = oldest_index_ + now_offset;
  if (index >= max_window_size_ms_)
    index -= max_window_size_ms_;
  buckets_[index].sum += count;
  ++buckets_[index].samples;
  accumulated_count_ += count;
  ++num_samples_;
}

absl::optional<uint32_t> RateStatistics::Rate(int64_t now_ms) const {
  // Yeah, this const_cast ain't pretty, but the alternative is to declare most
  // of the members as mutable...
  const_cast<RateStatistics*>(this)->EraseOld(now_ms);

  // If window is a single bucket or there is only one sample in a data set that
  // has not grown to the full window size, treat this as rate unavailable.
  int64_t active_window_size = now_ms - oldest_time_ + 1;
  if (num_samples_ == 0 || active_window_size <= 1 ||
      (num_samples_ <= 1 && active_window_size < current_window_size_ms_)) {
    return absl::nullopt;
  }

  float scale = scale_ / active_window_size;
  float result = accumulated_count_ * scale + 0.5f;

  // Better return unavailable rate than garbage value (undefined behavior).
  if (result > std::numeric_limits<uint32_t>::max()) {
    return absl::nullopt;
  }
  return static_cast<uint32_t>(result);
}

void RateStatistics::EraseOld(int64_t now_ms) {
  if (!IsInitialized())
    return;

  // New oldest time that is included in data set.
  int64_t new_oldest_time = now_ms - current_window_size_ms_ + 1;

  // New oldest time is older than the current one, no need to cull data.
  if (new_oldest_time <= oldest_time_)
    return;

  // Loop over buckets and remove too old data points.
  while (num_samples_ > 0 && oldest_time_ < new_oldest_time) {
    const Bucket& oldest_bucket = buckets_[oldest_index_];
    RTC_DCHECK_GE(accumulated_count_, oldest_bucket.sum);
    RTC_DCHECK_GE(num_samples_, oldest_bucket.samples);
    accumulated_count_ -= oldest_bucket.sum;
    num_samples_ -= oldest_bucket.samples;
    buckets_[oldest_index_] = Bucket();
    if (++oldest_index_ >= max_window_size_ms_)
      oldest_index_ = 0;
    ++oldest_time_;
  }
  oldest_time_ = new_oldest_time;
}

bool RateStatistics::SetWindowSize(int64_t window_size_ms, int64_t now_ms) {
  if (window_size_ms <= 0 || window_size_ms > max_window_size_ms_)
    return false;

  current_window_size_ms_ = window_size_ms;
  EraseOld(now_ms);
  return true;
}

bool RateStatistics::IsInitialized() const {
  return oldest_time_ != -max_window_size_ms_;
}

}  // namespace webrtc
<|endoftext|>


<commit_before>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include <stdio.h>

#include "bin/file.h"
#include "bin/dartutils.h"
#include "bin/platform.h"

#include "vm/benchmark_test.h"
#include "vm/dart.h"
#include "vm/unit_test.h"

// TODO(iposva, asiva): This is a placeholder for the real unittest framework.
namespace dart {

// Defined in vm/os_thread_win.cc
extern bool private_flag_windows_run_tls_destructors;

// vm_isolate_snapshot_buffer points to a snapshot for the vm isolate if we
// link in a snapshot otherwise it is initialized to NULL.
extern const uint8_t* bin::vm_isolate_snapshot_buffer;

// Only run tests that match the filter string. The default does not match any
// tests.
static const char* const kNone = "No Test or Benchmarks";
static const char* const kList = "List all Tests and Benchmarks";
static const char* const kAllBenchmarks = "All Benchmarks";
static const char* run_filter = kNone;

static int run_matches = 0;

void TestCase::Run() {
  fprintf(stdout, "Running test: %s\n", name());
  (*run_)();
  fprintf(stdout, "Done: %s\n", name());
}

void TestCaseBase::RunTest() {
  if (strcmp(run_filter, this->name()) == 0) {
    this->Run();
    run_matches++;
  } else if (run_filter == kList) {
    fprintf(stdout, "%s\n", this->name());
    run_matches++;
  }
}

void Benchmark::RunBenchmark() {
  if ((run_filter == kAllBenchmarks) ||
      (strcmp(run_filter, this->name()) == 0)) {
    this->Run();
    OS::Print("%s(%s): %" Pd64 "\n",
              this->name(), this->score_kind(), this->score());
    run_matches++;
  } else if (run_filter == kList) {
    fprintf(stdout, "%s\n", this->name());
    run_matches++;
  }
}

static void PrintUsage() {
  fprintf(stderr, "run_vm_tests [--list | --benchmarks | "
                  "<test name> | <benchmark name>]\n");
  fprintf(stderr, "run_vm_tests [vm-flags ...] <test name>\n");
  fprintf(stderr, "run_vm_tests [vm-flags ...] <benchmark name>\n");
}

static int Main(int argc, const char** argv) {
  // Flags being passed to the Dart VM.
  int dart_argc = 0;
  const char** dart_argv = NULL;

  if (argc < 2) {
    // Bad parameter count.
    PrintUsage();
    return 1;
  } else if (argc == 2) {
    if (strcmp(argv[1], "--list") == 0) {
      run_filter = kList;
      // List all tests and benchmarks and exit without initializing the VM.
      TestCaseBase::RunAll();
      Benchmark::RunAll(argv[0]);
      return 0;
    } else if (strcmp(argv[1], "--benchmarks") == 0) {
      run_filter = kAllBenchmarks;
    } else {
      run_filter = argv[1];
    }
  } else {
    // Last argument is the test name, the rest are vm flags.
    run_filter = argv[argc - 1];
    // Remove the first value (executable) from the arguments and
    // exclude the last argument which is the test name.
    dart_argc = argc - 2;
    dart_argv = &argv[1];
  }
  bool set_vm_flags_success = Flags::ProcessCommandLineFlags(dart_argc,
                                                             dart_argv);
  ASSERT(set_vm_flags_success);
  const char* err_msg = Dart::InitOnce(dart::bin::vm_isolate_snapshot_buffer,
                                       NULL,
                                       NULL, NULL, NULL, NULL,
                                       dart::bin::DartUtils::OpenFile,
                                       dart::bin::DartUtils::ReadFile,
                                       dart::bin::DartUtils::WriteFile,
                                       dart::bin::DartUtils::CloseFile,
                                       NULL,
                                       NULL);
  ASSERT(err_msg == NULL);
  // Apply the filter to all registered tests.
  TestCaseBase::RunAll();
  // Apply the filter to all registered benchmarks.
  Benchmark::RunAll(argv[0]);

  if (Flags::IsSet("shutdown")) {
    err_msg = Dart::Cleanup();
    ASSERT(err_msg == NULL);
  }

  // TODO(zra): Remove once VM shuts down cleanly.
  private_flag_windows_run_tls_destructors = false;

  // Print a warning message if no tests or benchmarks were matched.
  if (run_matches == 0) {
    fprintf(stderr, "No tests matched: %s\n", run_filter);
    return 1;
  }
  return 0;
}

}  // namespace dart

int main(int argc, const char** argv) {
  return dart::Main(argc, argv);
}
<commit_msg>Fix non-Windows builds<commit_after>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

#include <stdio.h>

#include "bin/file.h"
#include "bin/dartutils.h"
#include "bin/platform.h"

#include "vm/benchmark_test.h"
#include "vm/dart.h"
#include "vm/unit_test.h"

// TODO(iposva, asiva): This is a placeholder for the real unittest framework.
namespace dart {

// Defined in vm/os_thread_win.cc
extern bool private_flag_windows_run_tls_destructors;

// vm_isolate_snapshot_buffer points to a snapshot for the vm isolate if we
// link in a snapshot otherwise it is initialized to NULL.
extern const uint8_t* bin::vm_isolate_snapshot_buffer;

// Only run tests that match the filter string. The default does not match any
// tests.
static const char* const kNone = "No Test or Benchmarks";
static const char* const kList = "List all Tests and Benchmarks";
static const char* const kAllBenchmarks = "All Benchmarks";
static const char* run_filter = kNone;

static int run_matches = 0;

void TestCase::Run() {
  fprintf(stdout, "Running test: %s\n", name());
  (*run_)();
  fprintf(stdout, "Done: %s\n", name());
}

void TestCaseBase::RunTest() {
  if (strcmp(run_filter, this->name()) == 0) {
    this->Run();
    run_matches++;
  } else if (run_filter == kList) {
    fprintf(stdout, "%s\n", this->name());
    run_matches++;
  }
}

void Benchmark::RunBenchmark() {
  if ((run_filter == kAllBenchmarks) ||
      (strcmp(run_filter, this->name()) == 0)) {
    this->Run();
    OS::Print("%s(%s): %" Pd64 "\n",
              this->name(), this->score_kind(), this->score());
    run_matches++;
  } else if (run_filter == kList) {
    fprintf(stdout, "%s\n", this->name());
    run_matches++;
  }
}

static void PrintUsage() {
  fprintf(stderr, "run_vm_tests [--list | --benchmarks | "
                  "<test name> | <benchmark name>]\n");
  fprintf(stderr, "run_vm_tests [vm-flags ...] <test name>\n");
  fprintf(stderr, "run_vm_tests [vm-flags ...] <benchmark name>\n");
}

static int Main(int argc, const char** argv) {
  // Flags being passed to the Dart VM.
  int dart_argc = 0;
  const char** dart_argv = NULL;

  if (argc < 2) {
    // Bad parameter count.
    PrintUsage();
    return 1;
  } else if (argc == 2) {
    if (strcmp(argv[1], "--list") == 0) {
      run_filter = kList;
      // List all tests and benchmarks and exit without initializing the VM.
      TestCaseBase::RunAll();
      Benchmark::RunAll(argv[0]);
      return 0;
    } else if (strcmp(argv[1], "--benchmarks") == 0) {
      run_filter = kAllBenchmarks;
    } else {
      run_filter = argv[1];
    }
  } else {
    // Last argument is the test name, the rest are vm flags.
    run_filter = argv[argc - 1];
    // Remove the first value (executable) from the arguments and
    // exclude the last argument which is the test name.
    dart_argc = argc - 2;
    dart_argv = &argv[1];
  }
  bool set_vm_flags_success = Flags::ProcessCommandLineFlags(dart_argc,
                                                             dart_argv);
  ASSERT(set_vm_flags_success);
  const char* err_msg = Dart::InitOnce(dart::bin::vm_isolate_snapshot_buffer,
                                       NULL,
                                       NULL, NULL, NULL, NULL,
                                       dart::bin::DartUtils::OpenFile,
                                       dart::bin::DartUtils::ReadFile,
                                       dart::bin::DartUtils::WriteFile,
                                       dart::bin::DartUtils::CloseFile,
                                       NULL,
                                       NULL);
  ASSERT(err_msg == NULL);
  // Apply the filter to all registered tests.
  TestCaseBase::RunAll();
  // Apply the filter to all registered benchmarks.
  Benchmark::RunAll(argv[0]);

  if (Flags::IsSet("shutdown")) {
    err_msg = Dart::Cleanup();
    ASSERT(err_msg == NULL);
  }

#if defined(TARGET_OS_WINDOWS)
  // TODO(zra): Remove once VM shuts down cleanly.
  private_flag_windows_run_tls_destructors = false;
#endif

  // Print a warning message if no tests or benchmarks were matched.
  if (run_matches == 0) {
    fprintf(stderr, "No tests matched: %s\n", run_filter);
    return 1;
  }
  return 0;
}

}  // namespace dart

int main(int argc, const char** argv) {
  return dart::Main(argc, argv);
}
<|endoftext|>


<commit_before>#include "numerics/davenport_q_method.hpp"

#include <random>
#include <vector>

#include "geometry/frame.hpp"
#include "geometry/grassmann.hpp"
#include "gtest/gtest.h"
#include "quantities/quantities.hpp"
#include "quantities/si.hpp"
#include "testing_utilities/almost_equals.hpp"

namespace principia {
namespace numerics {

using geometry::Frame;
using geometry::Quaternion;
using geometry::Vector;
using quantities::Length;
using quantities::si::Metre;
using testing_utilities::AlmostEquals;

constexpr int number_of_test_vectors = 100;

class DavenportQMethodTest : public ::testing::Test {
 protected:
  using World1 = Frame<enum class World1Tag>;
  using World2 = Frame<enum class World2Tag>;

  DavenportQMethodTest() {
    std::mt19937_64 random(42);
    std::uniform_int_distribution<std::int64_t> coordinate_distribution(-5, 5);
    for (int i = 0; i < number_of_test_vectors; ++i) {
      double const x = coordinate_distribution(random);
      double const y = coordinate_distribution(random);
      double const z = coordinate_distribution(random);
      Vector<double, World1> const v({x, y, z});
      vectors1_.push_back(v / v.Norm());
      weights_.push_back(1 * Metre);
    }
  }

  std::vector<Vector<double, World1>> vectors1_;
  std::vector<Length> weights_;
};

TEST_F(DavenportQMethodTest, Identity) {
  EXPECT_THAT(DavenportQMethod(vectors1_, vectors1_, weights_),
              AlmostEquals(Quaternion(1), 0));
}

}  // namespace numerics
}  // namespace principia
<commit_msg>A test that fails.<commit_after>#include "numerics/davenport_q_method.hpp"

#include <random>
#include <vector>

#include "geometry/frame.hpp"
#include "geometry/grassmann.hpp"
#include "geometry/r3_element.hpp"
#include "geometry/rotation.hpp"
#include "gtest/gtest.h"
#include "quantities/quantities.hpp"
#include "quantities/si.hpp"
#include "testing_utilities/almost_equals.hpp"

namespace principia {
namespace numerics {

using geometry::Bivector;
using geometry::Frame;
using geometry::Quaternion;
using geometry::R3Element;
using geometry::Rotation;
using geometry::Vector;
using quantities::Length;
using quantities::si::Metre;
using quantities::si::Radian;
using testing_utilities::AlmostEquals;

constexpr int number_of_test_vectors = 100;

class DavenportQMethodTest : public ::testing::Test {
 protected:
  using World1 = Frame<enum class World1Tag>;
  using World2 = Frame<enum class World2Tag>;

  DavenportQMethodTest() {
    std::mt19937_64 random(42);
    std::uniform_int_distribution<std::int64_t> coordinate_distribution(-5, 5);
    for (int i = 0; i < number_of_test_vectors; ++i) {
      double const x = coordinate_distribution(random);
      double const y = coordinate_distribution(random);
      double const z = coordinate_distribution(random);
      Vector<double, World1> const v({x, y, z});
      vectors1_.push_back(v / v.Norm());
      weights_.push_back(1 * Metre);
    }
  }

  std::vector<Vector<double, World1>> vectors1_;
  std::vector<Length> weights_;
};

TEST_F(DavenportQMethodTest, Identity) {
  EXPECT_THAT(DavenportQMethod(vectors1_, vectors1_, weights_),
              AlmostEquals(Quaternion(1), 0));
}

TEST_F(DavenportQMethodTest, Rotation) {
  Quaternion const q(2, R3Element<double>({1, -3, -2}));
  auto const normalized_q = q / q.Norm();
  Rotation<World1, World2> const rotation{normalized_q};

  std::vector<Vector<double, World2>> vectors2;
  for (auto const& vector1 : vectors1_) {
    vectors2.push_back(rotation(vector1));
  }

  EXPECT_THAT(DavenportQMethod(vectors1_, vectors2, weights_),
              AlmostEquals(normalized_q , 0));
}

}  // namespace numerics
}  // namespace principia
<|endoftext|>


<commit_before>#include <mimosa/tpl/cache.hh>
#include <mimosa/log/log.hh>

#include "config.hh"
#include "load-tpl.hh"

mimosa::tpl::Template::ConstPtr loadTpl(const std::string & path)
{
  static mimosa::tpl::Cache cache;

  std::string real_path(Config::tplDir() + path);

  // auto tpl = cache.get(real_path)->get();
  auto tpl = mimosa::tpl::Template::parseFile(real_path);
  if (tpl)
    return tpl;

  mimosa::log::error("failed to load template: %s", real_path);
  return nullptr;
}
<commit_msg>use the template cache<commit_after>#include <mimosa/tpl/cache.hh>
#include <mimosa/log/log.hh>

#include "config.hh"
#include "load-tpl.hh"

mimosa::tpl::Template::ConstPtr loadTpl(const std::string & path)
{
  static mimosa::tpl::Cache cache;

  std::string real_path(Config::tplDir() + path);

  auto tpl = cache.get(real_path)->get();
  if (tpl)
    return tpl;

  mimosa::log::error("failed to load template: %s", real_path);
  return nullptr;
}
<|endoftext|>


<commit_before>#include <cstdio>
#include <cstdlib>
#include <locale>
#include <string>
#include <vector>
#include <algorithm>
#include <iostream>  // NOLINT
#include <algorithm>
#include <tr1/array>
#include "config/config.h"

#include "stringpiece.h"
#include "ustringpiece.h"

#include "cmdline.h"
#include "ast.h"

#include "ast_serializer.h"

#include "parser.h"
#include "factory.h"

#include "interpreter.h"
#include "context.h"
#include "jsast.h"
#include "jsval.h"
#include "jsstring.h"
#include "jsscript.h"
#include "print.h"
#include "interactive.h"

#include "icu/ustream.h"
#include "icu/source.h"

#include "fpu.h"

int main(int argc, char **argv) {
  using iv::lv5::JSVal;
  iv::lv5::FixFPU();

  std::locale::global(std::locale(""));

  iv::cmdline::Parser cmd("lv5");

  cmd.Add("help",
          "help",
          'h', "print this message");
  cmd.Add("version",
          "version",
          'v', "print the version");
  cmd.Add("warning",
          "",
          'W', "set warning level 0 - 2", false, 0, iv::cmdline::range(0, 2));
  cmd.Add("ast",
          "ast",
          0, "print ast");
  cmd.Add("copyright",
          "copyright",
          0,   "print the copyright");
  cmd.set_footer("[program_file] [arguments]");

  const bool cmd_parse_success = cmd.Parse(argc, argv);
  if (!cmd_parse_success) {
    std::cerr << cmd.error() << std::endl << cmd.usage();
    return EXIT_FAILURE;
  }

  if (cmd.Exist("help")) {
    std::cout << cmd.usage();
    return EXIT_SUCCESS;
  }

  if (cmd.Exist("version")) {
    printf("lv5 %s (compiled %s %s)\n", IV_VERSION, __DATE__, __TIME__);
    return EXIT_SUCCESS;
  }

  if (cmd.Exist("copyright")) {
    printf("lv5 - Copyright (C) 2010 %s\n", IV_DEVELOPER);
    return EXIT_SUCCESS;
  }

  const std::vector<std::string>& rest = cmd.rest();
  const std::string& filename = rest.front();
  if (!rest.empty()) {
    std::string str;
    if (std::FILE* fp = std::fopen(filename.c_str(), "r")) {
      std::tr1::array<char, 1024> buf;
      while (const std::size_t len = std::fread(
              buf.data(),
              1,
              buf.size(), fp)) {
        str.append(buf.data(), len);
      }
      std::fclose(fp);
    } else {
      std::string err("lv5 can't open \"");
      err.append(filename);
      err.append("\"");
      std::perror(err.c_str());
      return EXIT_FAILURE;
    }

    iv::icu::Source src(str, filename);
    iv::lv5::Context ctx;
    iv::lv5::AstFactory factory(&ctx);
    iv::core::Parser<iv::lv5::AstFactory, iv::icu::Source>
        parser(&factory, &src);
    const iv::lv5::FunctionLiteral* const global = parser.ParseProgram();

    if (!global) {
      std::cerr << parser.error() << std::endl;
      return EXIT_FAILURE;
    }

    if (cmd.Exist("ast")) {
      iv::core::ast::AstSerializer<iv::lv5::AstFactory> ser;
      global->Accept(&ser);
      std::cout << ser.out() << std::endl;
    } else {
      ctx.DefineFunction(&iv::lv5::Print, "print", 1);
      iv::lv5::JSScript* const script = iv::lv5::JSGlobalScript::New(
          &ctx, global, &factory, &src);
      if (ctx.Run(script)) {
        const JSVal e = ctx.ErrorVal();
        ctx.error()->Clear();
        const iv::lv5::JSString* const str = e.ToString(&ctx, ctx.error());
        if (!*ctx.error()) {
          std::cerr << *str << std::endl;
          return EXIT_FAILURE;
        }
      }
    }
  } else {
    // Interactive Shell Mode
    iv::lv5::Interactive shell;
    return shell.Run();
  }
  return 0;
}
<commit_msg>some fix for MacOSX part 1<commit_after>#include <cstdio>
#include <cstdlib>
#include <locale>
#include <string>
#include <vector>
#include <algorithm>
#include <iostream>  // NOLINT
#include <algorithm>
#include <tr1/array>
#include <gc.h>
#include "config/config.h"

#include "stringpiece.h"
#include "ustringpiece.h"

#include "cmdline.h"
#include "ast.h"

#include "ast_serializer.h"

#include "parser.h"
#include "factory.h"

#include "interpreter.h"
#include "context.h"
#include "jsast.h"
#include "jsval.h"
#include "jsstring.h"
#include "jsscript.h"
#include "print.h"
#include "interactive.h"

#include "icu/ustream.h"
#include "icu/source.h"

#include "fpu.h"

int main(int argc, char **argv) {
  using iv::lv5::JSVal;
  iv::lv5::FixFPU();
  GC_init();

  iv::cmdline::Parser cmd("lv5");

  cmd.Add("help",
          "help",
          'h', "print this message");
  cmd.Add("version",
          "version",
          'v', "print the version");
  cmd.Add("warning",
          "",
          'W', "set warning level 0 - 2", false, 0, iv::cmdline::range(0, 2));
  cmd.Add("ast",
          "ast",
          0, "print ast");
  cmd.Add("copyright",
          "copyright",
          0,   "print the copyright");
  cmd.set_footer("[program_file] [arguments]");

  const bool cmd_parse_success = cmd.Parse(argc, argv);
  if (!cmd_parse_success) {
    std::cerr << cmd.error() << std::endl << cmd.usage();
    return EXIT_FAILURE;
  }

  if (cmd.Exist("help")) {
    std::cout << cmd.usage();
    return EXIT_SUCCESS;
  }

  if (cmd.Exist("version")) {
    printf("lv5 %s (compiled %s %s)\n", IV_VERSION, __DATE__, __TIME__);
    return EXIT_SUCCESS;
  }

  if (cmd.Exist("copyright")) {
    printf("lv5 - Copyright (C) 2010 %s\n", IV_DEVELOPER);
    return EXIT_SUCCESS;
  }

  const std::vector<std::string>& rest = cmd.rest();
  const std::string& filename = rest.front();
  if (!rest.empty()) {
    std::string str;
    if (std::FILE* fp = std::fopen(filename.c_str(), "r")) {
      std::tr1::array<char, 1024> buf;
      while (const std::size_t len = std::fread(
              buf.data(),
              1,
              buf.size(), fp)) {
        str.append(buf.data(), len);
      }
      std::fclose(fp);
    } else {
      std::string err("lv5 can't open \"");
      err.append(filename);
      err.append("\"");
      std::perror(err.c_str());
      return EXIT_FAILURE;
    }

    iv::icu::Source src(str, filename);
    iv::lv5::Context ctx;
    iv::lv5::AstFactory factory(&ctx);
    iv::core::Parser<iv::lv5::AstFactory, iv::icu::Source>
        parser(&factory, &src);
    const iv::lv5::FunctionLiteral* const global = parser.ParseProgram();

    if (!global) {
      std::cerr << parser.error() << std::endl;
      return EXIT_FAILURE;
    }

    if (cmd.Exist("ast")) {
      iv::core::ast::AstSerializer<iv::lv5::AstFactory> ser;
      global->Accept(&ser);
      std::cout << ser.out() << std::endl;
    } else {
      ctx.DefineFunction(&iv::lv5::Print, "print", 1);
      iv::lv5::JSScript* const script = iv::lv5::JSGlobalScript::New(
          &ctx, global, &factory, &src);
      if (ctx.Run(script)) {
        const JSVal e = ctx.ErrorVal();
        ctx.error()->Clear();
        const iv::lv5::JSString* const str = e.ToString(&ctx, ctx.error());
        if (!*ctx.error()) {
          std::cerr << *str << std::endl;
          return EXIT_FAILURE;
        }
      }
    }
  } else {
    // Interactive Shell Mode
    iv::lv5::Interactive shell;
    return shell.Run();
  }
  return 0;
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Elviss Strazdins
// This file is part of the Ouzel engine.

#include <jni.h>
#include "core/android/ApplicationAndroid.h"
#include "core/android/WindowAndroid.h"
#include "core/Engine.h"
#include "core/Window.h"
#include "files/FileSystem.h"
#include "input/Input.h"
#include "utils/Utils.h"

ouzel::ApplicationAndroid application;

extern "C"
{
    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_setAssetManager(JNIEnv* env, jclass cls, jobject assetManager)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::assetManager = AAssetManager_fromJava(env, assetManager);
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onSurfaceCreated(JNIEnv* env, jclass cls)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        application.run();
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onSurfaceChanged(JNIEnv* env, jclass cls, jint width, jint height)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        if (ouzel::sharedEngine)
        {
            ouzel::WindowAndroid* windowAndroid = static_cast<ouzel::WindowAndroid*>(ouzel::sharedEngine->getWindow());
            windowAndroid->handleResize(ouzel::Size2(static_cast<float>(width), static_cast<float>(height)));
        }
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onDrawFrame(JNIEnv* env, jclass cls)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        if (ouzel::sharedEngine)
        {
            ouzel::sharedEngine->draw();
        }
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionDown(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchBegin(static_cast<uint64_t>(pointerId),
                                                    ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionMove(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchMove(static_cast<uint64_t>(pointerId),
                                                   ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionUp(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchEnd(static_cast<uint64_t>(pointerId),
                                                  ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionCancel(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchCancel(static_cast<uint64_t>(pointerId),
                                                     ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }
}
<commit_msg>Remove unneeded includes<commit_after>// Copyright (C) 2016 Elviss Strazdins
// This file is part of the Ouzel engine.

#include <jni.h>
#include "core/android/ApplicationAndroid.h"
#include "core/android/WindowAndroid.h"
#include "core/Engine.h"
#include "input/Input.h"

ouzel::ApplicationAndroid application;

extern "C"
{
    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_setAssetManager(JNIEnv* env, jclass cls, jobject assetManager)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::assetManager = AAssetManager_fromJava(env, assetManager);
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onSurfaceCreated(JNIEnv* env, jclass cls)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        application.run();
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onSurfaceChanged(JNIEnv* env, jclass cls, jint width, jint height)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        if (ouzel::sharedEngine)
        {
            ouzel::WindowAndroid* windowAndroid = static_cast<ouzel::WindowAndroid*>(ouzel::sharedEngine->getWindow());
            windowAndroid->handleResize(ouzel::Size2(static_cast<float>(width), static_cast<float>(height)));
        }
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_onDrawFrame(JNIEnv* env, jclass cls)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        if (ouzel::sharedEngine)
        {
            ouzel::sharedEngine->draw();
        }
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionDown(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchBegin(static_cast<uint64_t>(pointerId),
                                                    ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionMove(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchMove(static_cast<uint64_t>(pointerId),
                                                   ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionUp(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchEnd(static_cast<uint64_t>(pointerId),
                                                  ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }

    JNIEXPORT void JNICALL Java_lv_elviss_ouzel_OuzelLibJNIWrapper_handleActionCancel(JNIEnv* env, jclass cls, jint pointerId, jfloat x, jfloat y)
    {
        OUZEL_UNUSED(env);
        OUZEL_UNUSED(cls);

        ouzel::sharedEngine->getInput()->touchCancel(static_cast<uint64_t>(pointerId),
                                                     ouzel::sharedEngine->getWindow()->convertWindowToNormalizedLocation(ouzel::Vector2(x, y)));
    }
}
<|endoftext|>


<commit_before>#include "gtest_utils.h"

#include "onnx/checker.h"
#include "onnx/onnxifi.h"
#include "onnx/onnxifi_loader.h"
#include "onnx/string_utils.h"

/**
 *	In order to not test the backend itself
 *	but only to test the funcionality of this test driver,
 *	include IO and integrity checkers,
 *	please set ONNXIFI_DUMMY_BACKEND to be true when compiling.
 *	By default it is false.
 */
#ifndef ONNXIFI_DUMMY_BACKEND
#define ONNXIFI_DUMMY_BACKEND false
#endif

#ifndef ONNXIFI_TESTDATA_EPS
#define ONNXIFI_TESTDATA_EPS 1e-5
#endif

namespace ONNX_NAMESPACE {
namespace testing {
const float onnxifi_testdata_eps = ONNXIFI_TESTDATA_EPS;

template <typename T>
class CompareOnnxifiData {
 public:
  bool IsEqual(void* P, void* Q) {
    T x = *((T*)P), y = *((T*)Q);
    return ((x - y) > -onnxifi_testdata_eps) &&
        ((x - y) < onnxifi_testdata_eps);
  }
};

class ONNXCppDriverTest : public ::testing::TestWithParam<
                              ONNX_NAMESPACE::testing::ResolvedTestCase> {
 public:
  struct PrintToStringParamName {
    template <class T>
    std::string operator()(const ::testing::TestParamInfo<T>& t) const {
      auto test_case =
          static_cast<ONNX_NAMESPACE::testing::ResolvedTestCase>(t.param);
      /**
       *  We pick folder name instead of model.graph().name as test case name
       * here mostly for two reasons:
       *  1. Name of proto graph can be empty or ruined.
       *  2. Compares to the complex logic of protobuf, folder name is easier to
       * parse, to modify and to use.
       */
      return test_case.test_case_name_;
    }
  };

 protected:
  std::vector<ONNX_NAMESPACE::testing::ResolvedTestData> protos_;
  ONNX_NAMESPACE::ModelProto model_;

  // A memory pool of shape information of onnxTensorDescriptorV1 used in this
  // test driver.
  std::vector<std::vector<uint64_t>> shape_pool;
  // A memory pool of raw_data of backend output in onnxTensorDescriptorV1.
  std::vector<std::vector<uint8_t>> data_pool;

  void SetUp() override {
    ONNX_NAMESPACE::testing::ResolvedTestCase t = GetParam();
    protos_ = t.proto_test_data_;
    model_ = t.model_;
  }

  uint64_t GetDescriptorSize(const onnxTensorDescriptorV1* t) {
    uint64_t d_size = 1;
    for (int i = 0; i < t->dimensions; i++) {
      d_size *= t->shape[i];
    }
    return d_size;
  }

  bool IsGtestAssertMemorySafeSuccess(
      onnxStatus x,
      onnxGraph* graph_pointer,
      onnxifi_library& lib) {
    if (x != ONNXIFI_STATUS_SUCCESS) {
      if (graph_pointer != NULL) {
        lib.onnxReleaseGraph(*graph_pointer);
      }
    }
    bool result = (x == ONNXIFI_STATUS_SUCCESS);
    return result;
  }

  bool IsUnsupported(onnxStatus s, std::string& msg) {
    if (s == ONNXIFI_STATUS_UNSUPPORTED_VERSION) {
      msg = "Unsupported version of ONNX or operator.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_OPERATOR) {
      msg = "Unsupported operator.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE) {
      msg = "Unsupported attribute.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_SHAPE) {
      msg = "Unsupported shape.";
      return true;
    }
    if (s == ONNXIFI_STATUS_UNSUPPORTED_DATATYPE) {
      msg = "Unsupported datatype";
      return true;
    }
    return false;
  }

  bool IsDescriptorEqual(
      const onnxTensorDescriptorV1& x,
      const onnxTensorDescriptorV1& y) {
    if (x.dataType != y.dataType || x.dimensions != y.dimensions) {
      return false;
    }
    const int dims = x.dimensions;
    for (int i = 0; i < dims; i++) {
      if (x.shape[i] != y.shape[i]) {
        return false;
      }
    }
    const uint64_t d_size = GetDescriptorSize(&x);
    void* p1 = (void*)x.buffer;
    void* p2 = (void*)y.buffer;
    bool is_equal = true;
    for (uint64_t i = 0; i < d_size; i++) {
      int offset = 1;
      switch (x.dataType) {
        case ONNXIFI_DATATYPE_UNDEFINED:
        case ONNXIFI_DATATYPE_INT8:
          CompareOnnxifiData<char> compare_int8;
          is_equal &= compare_int8.IsEqual(p1, p2);
          offset = sizeof(char);
          break;
        case ONNXIFI_DATATYPE_UINT8:
          CompareOnnxifiData<unsigned char> compare_uint8;
          is_equal &= compare_uint8.IsEqual(p1, p2);
          offset = sizeof(unsigned char);
          break;
        case ONNXIFI_DATATYPE_FLOAT16:
          // no support now
          break;
        case ONNXIFI_DATATYPE_INT16:
          CompareOnnxifiData<short> compare_int16;
          is_equal &= compare_int16.IsEqual(p1, p2);
          offset = sizeof(short);
          break;
        case ONNXIFI_DATATYPE_UINT16:
          CompareOnnxifiData<unsigned short> compare_uint16;
          is_equal &= compare_uint16.IsEqual(p1, p2);
          offset = sizeof(unsigned short);
          break;
        case ONNXIFI_DATATYPE_FLOAT32:
          CompareOnnxifiData<float> compare_float32;
          is_equal &= compare_float32.IsEqual(p1, p2);
          offset = sizeof(float);
          break;
        case ONNXIFI_DATATYPE_INT32:
          CompareOnnxifiData<int> compare_int32;
          is_equal &= compare_int32.IsEqual(p1, p2);
          offset = sizeof(int);
          break;
        case ONNXIFI_DATATYPE_UINT32:
          CompareOnnxifiData<unsigned int> compare_uint32;
          is_equal &= compare_uint32.IsEqual(p1, p2);
          offset = sizeof(unsigned int);
          break;
        case ONNXIFI_DATATYPE_FLOAT64:
          CompareOnnxifiData<long double> compare_float64;
          is_equal &= compare_float64.IsEqual(p1, p2);
          offset = sizeof(long double);
          break;
        case ONNXIFI_DATATYPE_INT64:
          CompareOnnxifiData<long long> compare_int64;
          is_equal &= compare_int64.IsEqual(p1, p2);
          offset = sizeof(long long);
          break;
        case ONNXIFI_DATATYPE_UINT64:
          CompareOnnxifiData<unsigned long long> compare_uint64;
          is_equal &= compare_uint64.IsEqual(p1, p2);
          offset = sizeof(unsigned long long);
          break;
        case ONNXIFI_DATATYPE_COMPLEX64:
        case ONNXIFI_DATATYPE_COMPLEX128:
          // no support now
          break;
      }
      p1 = (char*)p1 + offset;
      p2 = (char*)p2 + offset;
      if (!is_equal) {
        return false;
      }
    }
    return true;
  }

  void RunAndVerify(
      onnxifi_library& lib,
      onnxBackend& backend,
      onnxBackendID backendID) {
    // Check Model
    ONNX_NAMESPACE::checker::check_model(model_);
    // Check Input&Output Tensors
    ONNX_NAMESPACE::checker::CheckerContext ctx;
    for (auto proto_test_data : protos_) {
      for (auto input : proto_test_data.inputs_) {
        ONNX_NAMESPACE::checker::check_tensor(input, ctx);
      }
      for (auto output : proto_test_data.outputs_) {
        ONNX_NAMESPACE::checker::check_tensor(output, ctx);
      }
    }
    if (!ONNXIFI_DUMMY_BACKEND) {
      onnxGraph graph;
      uint32_t weightCount = model_.graph().initializer_size();
      onnxTensorDescriptorV1 weightDescriptors,
          *weightDescriptors_pointer = NULL;
      if (weightCount != 0) {
        weightDescriptors =
            ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                model_.graph().initializer(0), shape_pool);
        weightDescriptors_pointer = &weightDescriptors;
      }
      std::string serialized_model;
      model_.SerializeToString(&serialized_model);
      auto is_compatible = lib.onnxGetBackendCompatibility(
          backendID, serialized_model.size(), serialized_model.data());
      std::string error_msg;
      if (IsUnsupported(is_compatible, error_msg)) {
        std::cout << "Warnning: " << error_msg
                  << " This test case will be skipped." << std::endl;
        GTEST_SKIP();
        return;
      }
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(is_compatible, NULL, lib));
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
          lib.onnxInitGraph(
              backend,
              NULL,
              serialized_model.size(),
              serialized_model.data(),
              weightCount,
              weightDescriptors_pointer,
              &graph),
          NULL,
          lib));
      for (const auto& proto_test_data : protos_) {
        std::vector<onnxTensorDescriptorV1> input_descriptor, output_descriptor,
            result_descriptor;
        for (const auto& input : proto_test_data.inputs_) {
          input_descriptor.push_back(
              ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                  input, shape_pool));
        }
        int output_count = 0;
        for (auto& output : proto_test_data.outputs_) {
          output_count++;
          output_descriptor.push_back(
              ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                  output, shape_pool));
          onnxTensorDescriptorV1 result;
          result.tag = ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1;
          std::string name_string = output_descriptor[output_count - 1].name;
          result.name = name_string.c_str();
          result.dataType = output.data_type();
          result.memoryType = ONNXIFI_MEMORY_TYPE_CPU;
          std::vector<uint64_t> shape_values(
              output.dims().begin(), output.dims().end());
          shape_pool.emplace_back(std::move(shape_values));
          result.dimensions = shape_pool.back().size();
          result.shape = shape_pool.back().data();
          std::vector<uint8_t> raw_data(output.raw_data().size(), 0);
          data_pool.emplace_back(std::move(raw_data));
          result.buffer = (onnxPointer)data_pool.back().data();
          result_descriptor.emplace_back(std::move(result));
        }
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxSetGraphIO(
                graph,
                input_descriptor.size(),
                input_descriptor.data(),
                result_descriptor.size(),
                result_descriptor.data()),
            &graph,
            lib));

        onnxMemoryFenceV1 inputFence, outputFence;
        inputFence.tag = ONNXIFI_TAG_MEMORY_FENCE_V1;
        outputFence.tag = ONNXIFI_TAG_MEMORY_FENCE_V1;
        inputFence.type = ONNXIFI_SYNCHRONIZATION_EVENT;
        outputFence.type = ONNXIFI_SYNCHRONIZATION_EVENT;
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxInitEvent(backend, &inputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxInitEvent(backend, &outputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxSignalEvent(inputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxRunGraph(graph, &inputFence, &outputFence), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxWaitEvent(outputFence.event), &graph, lib));
        lib.onnxReleaseEvent(outputFence.event);
        lib.onnxReleaseEvent(inputFence.event);
        ASSERT_EQ(output_descriptor.size(), result_descriptor.size());
        for (int i = 0; i < output_descriptor.size(); i++) {
          auto output_size = GetDescriptorSize(&output_descriptor[i]);
          auto result_size = GetDescriptorSize(&result_descriptor[i]);
          ASSERT_EQ(output_size, result_size);
          for (int j = 0; j < output_size; j++) {
            EXPECT_EQ(
                IsDescriptorEqual(output_descriptor[i], result_descriptor[i]),
                true);
          }
        }
      }
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
          lib.onnxReleaseGraph(graph), NULL, lib));
    }
  }
};

/**
 *	When testing backend, we do not specify target backend here,
 *	but always use the by-default backend of onnxifi_loader.
 *	Check onnxifi_loader.c for more detail of how to indicate specific
 *  backend.
 */
TEST_P(ONNXCppDriverTest, ONNXCppDriverUnitTest){
  onnxifi_library lib;
  onnxBackendID* backendIDs = NULL;
  onnxBackend backend;
  const size_t max_info_size = 50; // The maximum backend info size
  if (!ONNXIFI_DUMMY_BACKEND) {
    ASSERT_TRUE(onnxifi_load(1, NULL, &lib));
    size_t numBackends = 0;
    lib.onnxGetBackendIDs(backendIDs, &numBackends);
    backendIDs = (void**)malloc(numBackends * sizeof(onnxBackendID));
    lib.onnxGetBackendIDs(backendIDs, &numBackends);

    for (int i = 0; i < numBackends; i++) {
      const uint64_t backendProperties[] = {ONNXIFI_BACKEND_PROPERTY_NONE};
      lib.onnxInitBackend(backendIDs[i], backendProperties, &backend);
      char infovalue[max_info_size];
      size_t infoValueSize = max_info_size;
      EXPECT_EQ(
          lib.onnxGetBackendInfo(
              backendIDs[i], ONNXIFI_BACKEND_NAME, infovalue, &infoValueSize),
          ONNXIFI_STATUS_SUCCESS);
      RunAndVerify(lib, backend, backendIDs[i]);
      lib.onnxReleaseBackend(backend);
      lib.onnxReleaseBackendID(backendIDs[i]);
    }
    free(backendIDs);
  } else {
    RunAndVerify(lib, backend, NULL);
  }
}

INSTANTIATE_TEST_CASE_P(
    ONNXCppAllTest,
    ONNXCppDriverTest,
    ::testing::ValuesIn(GetTestCases()),
    ONNXCppDriverTest::PrintToStringParamName());
} // namespace testing
} // namespace ONNX_NAMESPACE
<commit_msg>In driver test check the return status of onnxGetBackendIDs (#1597)<commit_after>#include "gtest_utils.h"

#include "onnx/checker.h"
#include "onnx/onnxifi.h"
#include "onnx/onnxifi_loader.h"
#include "onnx/string_utils.h"

/**
 *	In order to not test the backend itself
 *	but only to test the funcionality of this test driver,
 *	include IO and integrity checkers,
 *	please set ONNXIFI_DUMMY_BACKEND to be true when compiling.
 *	By default it is false.
 */
#ifndef ONNXIFI_DUMMY_BACKEND
#define ONNXIFI_DUMMY_BACKEND false
#endif

#ifndef ONNXIFI_TESTDATA_EPS
#define ONNXIFI_TESTDATA_EPS 1e-5
#endif

namespace ONNX_NAMESPACE {
namespace testing {
const float onnxifi_testdata_eps = ONNXIFI_TESTDATA_EPS;

template <typename T>
class CompareOnnxifiData {
 public:
  bool IsEqual(void* P, void* Q) {
    T x = *((T*)P), y = *((T*)Q);
    return ((x - y) > -onnxifi_testdata_eps) &&
        ((x - y) < onnxifi_testdata_eps);
  }
};

class ONNXCppDriverTest : public ::testing::TestWithParam<
                              ONNX_NAMESPACE::testing::ResolvedTestCase> {
 public:
  struct PrintToStringParamName {
    template <class T>
    std::string operator()(const ::testing::TestParamInfo<T>& t) const {
      auto test_case =
          static_cast<ONNX_NAMESPACE::testing::ResolvedTestCase>(t.param);
      /**
       *  We pick folder name instead of model.graph().name as test case name
       * here mostly for two reasons:
       *  1. Name of proto graph can be empty or ruined.
       *  2. Compares to the complex logic of protobuf, folder name is easier to
       * parse, to modify and to use.
       */
      return test_case.test_case_name_;
    }
  };

 protected:
  std::vector<ONNX_NAMESPACE::testing::ResolvedTestData> protos_;
  ONNX_NAMESPACE::ModelProto model_;

  // A memory pool of shape information of onnxTensorDescriptorV1 used in this
  // test driver.
  std::vector<std::vector<uint64_t>> shape_pool;
  // A memory pool of raw_data of backend output in onnxTensorDescriptorV1.
  std::vector<std::vector<uint8_t>> data_pool;

  void SetUp() override {
    ONNX_NAMESPACE::testing::ResolvedTestCase t = GetParam();
    protos_ = t.proto_test_data_;
    model_ = t.model_;
  }

  uint64_t GetDescriptorSize(const onnxTensorDescriptorV1* t) {
    uint64_t d_size = 1;
    for (int i = 0; i < t->dimensions; i++) {
      d_size *= t->shape[i];
    }
    return d_size;
  }

  bool IsGtestAssertMemorySafeSuccess(
      onnxStatus x,
      onnxGraph* graph_pointer,
      onnxifi_library& lib) {
    if (x != ONNXIFI_STATUS_SUCCESS) {
      if (graph_pointer != NULL) {
        lib.onnxReleaseGraph(*graph_pointer);
      }
    }
    bool result = (x == ONNXIFI_STATUS_SUCCESS);
    return result;
  }

  bool IsUnsupported(onnxStatus s, std::string& msg) {
    if (s == ONNXIFI_STATUS_UNSUPPORTED_VERSION) {
      msg = "Unsupported version of ONNX or operator.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_OPERATOR) {
      msg = "Unsupported operator.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE) {
      msg = "Unsupported attribute.";
      return true;
    }

    if (s == ONNXIFI_STATUS_UNSUPPORTED_SHAPE) {
      msg = "Unsupported shape.";
      return true;
    }
    if (s == ONNXIFI_STATUS_UNSUPPORTED_DATATYPE) {
      msg = "Unsupported datatype";
      return true;
    }
    return false;
  }

  bool IsDescriptorEqual(
      const onnxTensorDescriptorV1& x,
      const onnxTensorDescriptorV1& y) {
    if (x.dataType != y.dataType || x.dimensions != y.dimensions) {
      return false;
    }
    const int dims = x.dimensions;
    for (int i = 0; i < dims; i++) {
      if (x.shape[i] != y.shape[i]) {
        return false;
      }
    }
    const uint64_t d_size = GetDescriptorSize(&x);
    void* p1 = (void*)x.buffer;
    void* p2 = (void*)y.buffer;
    bool is_equal = true;
    for (uint64_t i = 0; i < d_size; i++) {
      int offset = 1;
      switch (x.dataType) {
        case ONNXIFI_DATATYPE_UNDEFINED:
        case ONNXIFI_DATATYPE_INT8:
          CompareOnnxifiData<char> compare_int8;
          is_equal &= compare_int8.IsEqual(p1, p2);
          offset = sizeof(char);
          break;
        case ONNXIFI_DATATYPE_UINT8:
          CompareOnnxifiData<unsigned char> compare_uint8;
          is_equal &= compare_uint8.IsEqual(p1, p2);
          offset = sizeof(unsigned char);
          break;
        case ONNXIFI_DATATYPE_FLOAT16:
          // no support now
          break;
        case ONNXIFI_DATATYPE_INT16:
          CompareOnnxifiData<short> compare_int16;
          is_equal &= compare_int16.IsEqual(p1, p2);
          offset = sizeof(short);
          break;
        case ONNXIFI_DATATYPE_UINT16:
          CompareOnnxifiData<unsigned short> compare_uint16;
          is_equal &= compare_uint16.IsEqual(p1, p2);
          offset = sizeof(unsigned short);
          break;
        case ONNXIFI_DATATYPE_FLOAT32:
          CompareOnnxifiData<float> compare_float32;
          is_equal &= compare_float32.IsEqual(p1, p2);
          offset = sizeof(float);
          break;
        case ONNXIFI_DATATYPE_INT32:
          CompareOnnxifiData<int> compare_int32;
          is_equal &= compare_int32.IsEqual(p1, p2);
          offset = sizeof(int);
          break;
        case ONNXIFI_DATATYPE_UINT32:
          CompareOnnxifiData<unsigned int> compare_uint32;
          is_equal &= compare_uint32.IsEqual(p1, p2);
          offset = sizeof(unsigned int);
          break;
        case ONNXIFI_DATATYPE_FLOAT64:
          CompareOnnxifiData<long double> compare_float64;
          is_equal &= compare_float64.IsEqual(p1, p2);
          offset = sizeof(long double);
          break;
        case ONNXIFI_DATATYPE_INT64:
          CompareOnnxifiData<long long> compare_int64;
          is_equal &= compare_int64.IsEqual(p1, p2);
          offset = sizeof(long long);
          break;
        case ONNXIFI_DATATYPE_UINT64:
          CompareOnnxifiData<unsigned long long> compare_uint64;
          is_equal &= compare_uint64.IsEqual(p1, p2);
          offset = sizeof(unsigned long long);
          break;
        case ONNXIFI_DATATYPE_COMPLEX64:
        case ONNXIFI_DATATYPE_COMPLEX128:
          // no support now
          break;
      }
      p1 = (char*)p1 + offset;
      p2 = (char*)p2 + offset;
      if (!is_equal) {
        return false;
      }
    }
    return true;
  }

  void RunAndVerify(
      onnxifi_library& lib,
      onnxBackend& backend,
      onnxBackendID backendID) {
    // Check Model
    ONNX_NAMESPACE::checker::check_model(model_);
    // Check Input&Output Tensors
    ONNX_NAMESPACE::checker::CheckerContext ctx;
    for (auto proto_test_data : protos_) {
      for (auto input : proto_test_data.inputs_) {
        ONNX_NAMESPACE::checker::check_tensor(input, ctx);
      }
      for (auto output : proto_test_data.outputs_) {
        ONNX_NAMESPACE::checker::check_tensor(output, ctx);
      }
    }
    if (!ONNXIFI_DUMMY_BACKEND) {
      onnxGraph graph;
      uint32_t weightCount = model_.graph().initializer_size();
      onnxTensorDescriptorV1 weightDescriptors,
          *weightDescriptors_pointer = NULL;
      if (weightCount != 0) {
        weightDescriptors =
            ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                model_.graph().initializer(0), shape_pool);
        weightDescriptors_pointer = &weightDescriptors;
      }
      std::string serialized_model;
      model_.SerializeToString(&serialized_model);
      auto is_compatible = lib.onnxGetBackendCompatibility(
          backendID, serialized_model.size(), serialized_model.data());
      std::string error_msg;
      if (IsUnsupported(is_compatible, error_msg)) {
        std::cout << "Warnning: " << error_msg
                  << " This test case will be skipped." << std::endl;
        GTEST_SKIP();
        return;
      }
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(is_compatible, NULL, lib));
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
          lib.onnxInitGraph(
              backend,
              NULL,
              serialized_model.size(),
              serialized_model.data(),
              weightCount,
              weightDescriptors_pointer,
              &graph),
          NULL,
          lib));
      for (const auto& proto_test_data : protos_) {
        std::vector<onnxTensorDescriptorV1> input_descriptor, output_descriptor,
            result_descriptor;
        for (const auto& input : proto_test_data.inputs_) {
          input_descriptor.push_back(
              ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                  input, shape_pool));
        }
        int output_count = 0;
        for (auto& output : proto_test_data.outputs_) {
          output_count++;
          output_descriptor.push_back(
              ONNX_NAMESPACE::testing::ProtoToOnnxTensorDescriptor(
                  output, shape_pool));
          onnxTensorDescriptorV1 result;
          result.tag = ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1;
          std::string name_string = output_descriptor[output_count - 1].name;
          result.name = name_string.c_str();
          result.dataType = output.data_type();
          result.memoryType = ONNXIFI_MEMORY_TYPE_CPU;
          std::vector<uint64_t> shape_values(
              output.dims().begin(), output.dims().end());
          shape_pool.emplace_back(std::move(shape_values));
          result.dimensions = shape_pool.back().size();
          result.shape = shape_pool.back().data();
          std::vector<uint8_t> raw_data(output.raw_data().size(), 0);
          data_pool.emplace_back(std::move(raw_data));
          result.buffer = (onnxPointer)data_pool.back().data();
          result_descriptor.emplace_back(std::move(result));
        }
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxSetGraphIO(
                graph,
                input_descriptor.size(),
                input_descriptor.data(),
                result_descriptor.size(),
                result_descriptor.data()),
            &graph,
            lib));

        onnxMemoryFenceV1 inputFence, outputFence;
        inputFence.tag = ONNXIFI_TAG_MEMORY_FENCE_V1;
        outputFence.tag = ONNXIFI_TAG_MEMORY_FENCE_V1;
        inputFence.type = ONNXIFI_SYNCHRONIZATION_EVENT;
        outputFence.type = ONNXIFI_SYNCHRONIZATION_EVENT;
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxInitEvent(backend, &inputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxInitEvent(backend, &outputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxSignalEvent(inputFence.event), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxRunGraph(graph, &inputFence, &outputFence), &graph, lib));
        ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
            lib.onnxWaitEvent(outputFence.event), &graph, lib));
        lib.onnxReleaseEvent(outputFence.event);
        lib.onnxReleaseEvent(inputFence.event);
        ASSERT_EQ(output_descriptor.size(), result_descriptor.size());
        for (int i = 0; i < output_descriptor.size(); i++) {
          auto output_size = GetDescriptorSize(&output_descriptor[i]);
          auto result_size = GetDescriptorSize(&result_descriptor[i]);
          ASSERT_EQ(output_size, result_size);
          for (int j = 0; j < output_size; j++) {
            EXPECT_EQ(
                IsDescriptorEqual(output_descriptor[i], result_descriptor[i]),
                true);
          }
        }
      }
      ASSERT_TRUE(IsGtestAssertMemorySafeSuccess(
          lib.onnxReleaseGraph(graph), NULL, lib));
    }
  }
};

/**
 *	When testing backend, we do not specify target backend here,
 *	but always use the by-default backend of onnxifi_loader.
 *	Check onnxifi_loader.c for more detail of how to indicate specific
 *  backend.
 */
TEST_P(ONNXCppDriverTest, ONNXCppDriverUnitTest){
  onnxifi_library lib;
  onnxBackendID* backendIDs = NULL;
  onnxBackend backend;
  const size_t max_info_size = 50; // The maximum backend info size
  if (!ONNXIFI_DUMMY_BACKEND) {
    ASSERT_TRUE(onnxifi_load(1, NULL, &lib));
    size_t numBackends = 0;
    ASSERT_EQ(
        lib.onnxGetBackendIDs(backendIDs, &numBackends),
        ONNXIFI_STATUS_SUCCESS);
    backendIDs = (void**)malloc(numBackends * sizeof(onnxBackendID));
    ASSERT_EQ(
        lib.onnxGetBackendIDs(backendIDs, &numBackends),
        ONNXIFI_STATUS_SUCCESS);

    for (int i = 0; i < numBackends; i++) {
      const uint64_t backendProperties[] = {ONNXIFI_BACKEND_PROPERTY_NONE};
      ASSERT_EQ(
          lib.onnxInitBackend(backendIDs[i], backendProperties, &backend),
          ONNXIFI_STATUS_SUCCESS);
      char infovalue[max_info_size];
      size_t infoValueSize = max_info_size;
      ASSERT_EQ(
          lib.onnxGetBackendInfo(
              backendIDs[i], ONNXIFI_BACKEND_NAME, infovalue, &infoValueSize),
          ONNXIFI_STATUS_SUCCESS);
      RunAndVerify(lib, backend, backendIDs[i]);
      lib.onnxReleaseBackend(backend);
      lib.onnxReleaseBackendID(backendIDs[i]);
    }
    free(backendIDs);
  } else {
    RunAndVerify(lib, backend, NULL);
  }
}

INSTANTIATE_TEST_CASE_P(
    ONNXCppAllTest,
    ONNXCppDriverTest,
    ::testing::ValuesIn(GetTestCases()),
    ONNXCppDriverTest::PrintToStringParamName());
} // namespace testing
} // namespace ONNX_NAMESPACE
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/
#include "otbWrapperListViewParameter.h"

namespace otb
{
namespace Wrapper
{

ListViewParameter::ListViewParameter()
  : m_CurrentChoice(0)
{
}

ListViewParameter::~ListViewParameter()
{
}

void
ListViewParameter::AddChoice( std::string choicekey, std::string choiceName )
{
  ListViewChoice choice;
  choice.m_Key = choicekey;
  choice.m_Name = choiceName;
  m_ChoiceList.push_back(choice);
}

std::string
ListViewParameter::GetChoiceKey( int i )
{
  return m_ChoiceList[i].m_Key;
}

std::vector<std::string>
ListViewParameter::GetChoiceKeys( )
{
  std::vector<std::string> ret;
  ChoiceList::iterator it = m_ChoiceList.begin();

  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    ret.push_back(it->m_Key);
    }

  return ret;
}

std::string
ListViewParameter::GetChoiceName( int i )
{
  return m_ChoiceList[i].m_Name;
}

std::vector<std::string>
ListViewParameter::GetChoiceNames( )
{
  std::vector<std::string> ret;
  ChoiceList::iterator it = m_ChoiceList.begin();

  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    ret.push_back(it->m_Name);
    }
  return ret;
}

unsigned int
ListViewParameter::GetNbChoices( void )
{
  return m_ChoiceList.size();
}

void
ListViewParameter::SetValue(unsigned int v)
{
  m_CurrentChoice = v;
  SetActive(true);
  // Call Modified();
  this->Modified();
}

void
ListViewParameter::SetValue(std::string choiceKey)
{
  ChoiceList::iterator it = m_ChoiceList.begin();

  unsigned int i = 0;
  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    if ( it->m_Key == choiceKey )
      {
      SetValue(i);
      return;
      }
    ++i;
    }

  itkExceptionMacro(<< "Cannot find " << choiceKey);
}

unsigned int
ListViewParameter::GetValue()
{
  return m_CurrentChoice;
}

/** Clear choices */
void
ListViewParameter::ClearChoices()
{
  m_ChoiceList.clear();
}

void
ListViewParameter::SetSelectedItemsByNames()
{
  std::vector<int> selectedItems;
  std::vector<std::string> names = this->GetChoiceNames();
  for(unsigned int i=0; i<m_SelectedNames.size(); i++)
    {
    const std::string selectedName = m_SelectedNames[i];
    unsigned int j(0);
      for( ; j<names.size(); j++)
      {
      if( names[j] == selectedName )
            {
        selectedItems.push_back(j);
        break;
        }
      }
    // If not found
    if( j==names.size() )
      {
      std::ostringstream oss;
      for( j=0; j<names.size(); j++)
        {
        oss<<names[j];
        if( j < names.size()-1 )
          {
          oss<<", ";
          }
        }
      itkExceptionMacro("Value "<<selectedName<<" not found in the choices possibilities"<<oss.str()<<".");
      }
    }
  this->SetSelectedItems(selectedItems);
}

void
ListViewParameter::SetSelectedItemsByKeys()
{
  std::vector<int> selectedItems;
  std::vector<std::string> keys = this->GetChoiceKeys();
  for(unsigned int i=0; i<m_SelectedKeys.size(); i++)
    {
    const std::string selectedKey = m_SelectedKeys[i];
    unsigned int j(0);
    std::ostringstream oss;
      for(; j<keys.size(); j++)
      {
      if( keys[j] == selectedKey )
            {
        selectedItems.push_back(j);
        break;
        }
      }
    // If not found
    if( j==keys.size() )
      {
      for( j=0; j<keys.size(); j++)
        {
        oss<<keys[j];
        if( j < keys.size()-1 )
          {
          oss<<", ";
          }
        }
      itkExceptionMacro("Value "<<selectedKey<<" not found in the choices possibilities"<<oss.str()<<".");
      }
    }
  this->SetSelectedItems(selectedItems);
}
}
}

<commit_msg>STYLE<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/
#include "otbWrapperListViewParameter.h"

namespace otb
{
namespace Wrapper
{

ListViewParameter::ListViewParameter()
  : m_CurrentChoice(0)
{
}

ListViewParameter::~ListViewParameter()
{
}

void
ListViewParameter::AddChoice( std::string choicekey, std::string choiceName )
{
  ListViewChoice choice;
  choice.m_Key = choicekey;
  choice.m_Name = choiceName;
  m_ChoiceList.push_back(choice);
}

std::string
ListViewParameter::GetChoiceKey( int i )
{
  return m_ChoiceList[i].m_Key;
}

std::vector<std::string>
ListViewParameter::GetChoiceKeys( )
{
  std::vector<std::string> ret;
  ChoiceList::iterator it = m_ChoiceList.begin();

  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    ret.push_back(it->m_Key);
    }

  return ret;
}

std::string
ListViewParameter::GetChoiceName( int i )
{
  return m_ChoiceList[i].m_Name;
}

std::vector<std::string>
ListViewParameter::GetChoiceNames( )
{
  std::vector<std::string> ret;
  ChoiceList::iterator it = m_ChoiceList.begin();

  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    ret.push_back(it->m_Name);
    }
  return ret;
}

unsigned int
ListViewParameter::GetNbChoices( void )
{
  return m_ChoiceList.size();
}

void
ListViewParameter::SetValue(unsigned int v)
{
  m_CurrentChoice = v;
  SetActive(true);
  // Call Modified();
  this->Modified();
}

void
ListViewParameter::SetValue(std::string choiceKey)
{
  ChoiceList::iterator it = m_ChoiceList.begin();

  unsigned int i = 0;
  for (it = m_ChoiceList.begin(); it != m_ChoiceList.end(); ++it)
    {
    if ( it->m_Key == choiceKey )
      {
      SetValue(i);
      return;
      }
    ++i;
    }

  itkExceptionMacro(<< "Cannot find " << choiceKey);
}

unsigned int
ListViewParameter::GetValue()
{
  return m_CurrentChoice;
}

/** Clear choices */
void
ListViewParameter::ClearChoices()
{
  m_ChoiceList.clear();
}

void
ListViewParameter::SetSelectedItemsByNames()
{
  std::vector<int> selectedItems;
  std::vector<std::string> names = this->GetChoiceNames();
  for(unsigned int i=0; i<m_SelectedNames.size(); i++)
    {
    const std::string selectedName = m_SelectedNames[i];
    unsigned int j(0);
      for(; j<names.size(); j++)
      {
      if( names[j] == selectedName )
            {
        selectedItems.push_back(j);
        break;
        }
      }
    // If not found
    if( j==names.size() )
      {
      std::ostringstream oss;
      for( j=0; j<names.size(); j++)
        {
        oss<<names[j];
        if( j < names.size()-1 )
          {
          oss<<", ";
          }
        }
      itkExceptionMacro("Value "<<selectedName<<" not found in the choices possibilities"<<oss.str()<<".");
      }
    }
  this->SetSelectedItems(selectedItems);
}

void
ListViewParameter::SetSelectedItemsByKeys()
{
  std::vector<int> selectedItems;
  std::vector<std::string> keys = this->GetChoiceKeys();
  for(unsigned int i=0; i<m_SelectedKeys.size(); i++)
    {
    const std::string selectedKey = m_SelectedKeys[i];
    unsigned int j(0);
    std::ostringstream oss;
      for(; j<keys.size(); j++)
      {
      if( keys[j] == selectedKey )
            {
        selectedItems.push_back(j);
        break;
        }
      }
    // If not found
    if( j==keys.size() )
      {
      for( j=0; j<keys.size(); j++)
        {
        oss<<keys[j];
        if( j < keys.size()-1 )
          {
          oss<<", ";
          }
        }
      itkExceptionMacro("Value "<<selectedKey<<" not found in the choices possibilities"<<oss.str()<<".");
      }
    }
  this->SetSelectedItems(selectedItems);
}
}
}
<|endoftext|>