<commit_before>#include "../src/compress.h"
#include <stdio.h>
#include <stdlib.h>
#include <vector>
#include <string>
#include "../libuv/include/uv.h"

void show_version() {
	fprintf(stderr, "maxcso v%s\n", maxcso::VERSION);
}

void show_help(const char *arg0) {
	show_version();
	fprintf(stderr, "Usage: %s [--args] input.iso [-o output.cso]\n", arg0);
	fprintf(stderr, "\n");
	fprintf(stderr, "Multiple files may be specified.  Inputs can be iso or cso files.\n");
	fprintf(stderr, "\n");
	fprintf(stderr, "   --threads=N     Specify N threads for I/O and compression\n");
	fprintf(stderr, "   --quiet         Suppress status output\n");
	fprintf(stderr, "   --fast          Use only basic zlib for fastest result\n");
	// TODO: Bring this back once it's functional.
	//fprintf(stderr, "   --smallest      Force compression of all sectors for smallest result\n");
	fprintf(stderr, "   --use-METHOD    Use a compression method: zlib, Zopfli, or 7zdeflate\n");
	fprintf(stderr, "                   By default, zlib and 7zdeflate are used\n");
	fprintf(stderr, "   --no-METHOD     Disable a compression method\n");
}

bool has_arg_value(int &i, char *argv[], const std::string &arg, const char *&val) {
	if (arg.compare(0, arg.npos, argv[i], arg.size()) == 0) {
		if (strlen(argv[i]) == arg.size()) {
			val = argv[++i];
			return true;
		} else if (argv[i][arg.size()] == '=') {
			val = argv[i] + arg.size() + 1;
			return true;
		}
	}

	return false;
}

bool has_arg(int &i, char *argv[], const std::string &arg) {
	if (arg.compare(argv[i]) == 0) {
		return true;
	}

	return false;
}

struct Arguments {
	std::vector<std::string> inputs;
	std::vector<std::string> outputs;
	int threads;
	uint32_t flags;
	bool quiet;
};

void default_args(Arguments &args) {
	args.threads = 0;
	args.flags = maxcso::TASKFLAG_NO_ZOPFLI;
	args.quiet = false;
}

int parse_args(Arguments &args, int argc, char *argv[]) {
	const char *val = nullptr;
	int i;
	for (i = 1; i < argc; ++i) {
		if (argv[i][0] == '-') {
			if (has_arg(i, argv, "--help") || has_arg(i, argv, "-h")) {
				show_help(argv[0]);
				return 1;
			} else if (has_arg(i, argv, "--version") || has_arg(i, argv, "-v")) {
				show_version();
				return 1;
			} else if (has_arg_value(i, argv, "--threads", val)) {
				args.threads = atoi(val);
			} else if (has_arg(i, argv, "--quiet")) {
				args.quiet = true;
			} else if (has_arg(i, argv, "--fast")) {
				args.flags |= maxcso::TASKFLAG_NO_ZLIB_BRUTE | maxcso::TASKFLAG_NO_ZOPFLI | maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg(i, argv, "--smallest")) {
				args.flags |= maxcso::TASKFLAG_FORCE_ALL;
			} else if (has_arg(i, argv, "--use-zopfli")) {
				args.flags &= ~maxcso::TASKFLAG_NO_ZOPFLI;
			} else if (has_arg(i, argv, "--use-zlib")) {
				args.flags &= ~maxcso::TASKFLAG_NO_ZLIB;
			} else if (has_arg(i, argv, "--use-7zdeflate")) {
				args.flags &= ~maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg(i, argv, "--no-zopfli")) {
				args.flags |= maxcso::TASKFLAG_NO_ZOPFLI;
			} else if (has_arg(i, argv, "--no-zlib")) {
				args.flags |= maxcso::TASKFLAG_NO_ZLIB;
			} else if (has_arg(i, argv, "--no-7zdeflate")) {
				args.flags |= maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg_value(i, argv, "--out", val) || has_arg_value(i, argv, "-o", val)) {
				args.outputs.push_back(val);
			} else if (has_arg(i, argv, "--")) {
				break;
			} else {
				show_help(argv[0]);
				fprintf(stderr, "\nERROR: Unknown argument: %s\n", argv[i]);
				return 1;
			}
		} else {
			args.inputs.push_back(argv[i]);
		}
	}

	for (; i < argc; ++i) {
		// If we're here, it means we hit a "--".  The rest are inputs, not args.
		args.inputs.push_back(argv[i]);
	}

	return 0;
}

int validate_args(const char *arg0, Arguments &args) {
	if (args.threads == 0) {
		uv_cpu_info_t *cpus;
		uv_cpu_info(&cpus, &args.threads);
		uv_free_cpu_info(cpus, args.threads);
	}

	if (args.inputs.size() < args.outputs.size()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: Too many output files.\n");
		return 1;
	}

	// Automatically write to .cso files if not specified.
	for (size_t i = args.outputs.size(); i < args.inputs.size(); ++i) {
		if (args.inputs[i].size() <= 4) {
			continue;
		}

		const std::string ext = args.inputs[i].substr(args.inputs[i].size() - 4);
		if (ext == ".iso" || ext == ".ISO") {
			args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".cso");
		}
	}

	if (args.inputs.size() != args.outputs.size()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: Too few output files.\n");
		return 1;
	}

	if (args.inputs.empty()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: No input files.\n");
		return 1;
	}

	return 0;
}

#ifdef _WIN32
int setenv(const char *name, const char *value, int) {
	return _putenv_s(name, value);
}
#endif

inline uv_buf_t uv_buf_init(const char *str) {
	return uv_buf_init(const_cast<char *>(str), static_cast<unsigned int>(strlen(str)));
}

inline uv_buf_t uv_buf_init(const std::string &str) {
	return uv_buf_init(const_cast<char *>(str.c_str()), static_cast<unsigned int>(str.size()));
}

const std::string ANSI_RESET_LINE = "\033[2K\033[0G";

int main(int argc, char *argv[]) {
	Arguments args;
	default_args(args);
	int result = parse_args(args, argc, argv);
	if (result != 0) {
		return result;
	}
	result = validate_args(argv[0], args);
	if (result != 0) {
		return result;
	}

	char threadpool_size[32];
	sprintf(threadpool_size, "%d", args.threads);
	setenv("UV_THREADPOOL_SIZE", threadpool_size, 1);

	uv_loop_t loop;
	uv_tty_t tty;
	uv_loop_init(&loop);
	uv_tty_init(&loop, &tty, 2, 0);
	uv_tty_set_mode(&tty, 0);
	bool formatting = uv_guess_handle(2) == UV_TTY && !args.quiet;

	int64_t next = uv_hrtime();
	int64_t lastPos = 0;
	// 50ms
	static const int64_t interval_ns = 50000000LL;
	static const double interval_to_s = 1000.0 / 50.0;

	maxcso::ProgressCallback progress = [&next, &lastPos, formatting, &tty] (const maxcso::Task *task, maxcso::TaskStatus status, int64_t pos, int64_t total, int64_t written) {
		if (!formatting) {
			return;
		}

		std::string statusInfo;
		if (status == maxcso::TASK_INPROGRESS) {
			int64_t now = uv_hrtime();
			if (now >= next) {
				double percent = (pos * 100.0) / total;
				double ratio = (written * 100.0) / pos;
				double speed = 0.0;
				int64_t diff = pos - lastPos;
				if (diff > 0) {
					speed = (diff / 1024.0 / 1024.0) * interval_to_s;
				}

				char temp[128];
				sprintf(temp, "%3.0f%%, ratio=%3.0f%%, speed=%5.2f MB/s", percent, ratio, speed);
				statusInfo = temp;

				next = now + interval_ns;
				lastPos = pos;
			}
		} else if (status == maxcso::TASK_SUCCESS) {
			statusInfo = "Complete\n";
		} else {
			// This shouldn't happen.
			statusInfo = "Something went wrong.\n";
		}

		if (statusInfo.empty()) {
			return;
		}

		uv_buf_t bufs[2];
		uv_write_t write_req;

		unsigned int nbufs = 0;
		if (formatting) {
			bufs[nbufs++] = uv_buf_init(ANSI_RESET_LINE);
		}

		if (task->input.size() > 38) {
			statusInfo = "..." + task->input.substr(task->input.size() - 35) + ": " + statusInfo;
		} else {
			statusInfo = task->input + ": " + statusInfo;
		}

		bufs[nbufs++] = uv_buf_init(statusInfo);
		uv_write(&write_req, reinterpret_cast<uv_stream_t *>(&tty), bufs, nbufs, nullptr);
	};
	maxcso::ErrorCallback error = [&args, formatting, &tty] (const maxcso::Task *task, maxcso::TaskStatus status, const char *reason) {
		std::string line;
		uv_buf_t buf;
		uv_write_t write_req;

		if (args.quiet) {
			return;
		}

		line = (formatting ? ANSI_RESET_LINE : "") + "Error while processing " + task->input + ": " + reason + "\n";
		buf = uv_buf_init(line);
		uv_write(&write_req, reinterpret_cast<uv_stream_t *>(&tty), &buf, 1, nullptr);
	};

	std::vector<maxcso::Task> tasks;
	for (size_t i = 0; i < args.inputs.size(); ++i) {
		maxcso::Task task;
		task.input = args.inputs[i];
		task.output = args.outputs[i];
		task.progress = progress;
		task.error = error;
		task.flags = args.flags;
		tasks.push_back(std::move(task));
	}

	maxcso::Compress(tasks);

	uv_tty_reset_mode();
	uv_loop_close(&loop);
	return 0;
}
<commit_msg>Avoid a possible problem from compiler optims.<commit_after>#include "../src/compress.h"
#include <stdio.h>
#include <stdlib.h>
#include <vector>
#include <string>
#include "../libuv/include/uv.h"

void show_version() {
	fprintf(stderr, "maxcso v%s\n", maxcso::VERSION);
}

void show_help(const char *arg0) {
	show_version();
	fprintf(stderr, "Usage: %s [--args] input.iso [-o output.cso]\n", arg0);
	fprintf(stderr, "\n");
	fprintf(stderr, "Multiple files may be specified.  Inputs can be iso or cso files.\n");
	fprintf(stderr, "\n");
	fprintf(stderr, "   --threads=N     Specify N threads for I/O and compression\n");
	fprintf(stderr, "   --quiet         Suppress status output\n");
	fprintf(stderr, "   --fast          Use only basic zlib for fastest result\n");
	// TODO: Bring this back once it's functional.
	//fprintf(stderr, "   --smallest      Force compression of all sectors for smallest result\n");
	fprintf(stderr, "   --use-METHOD    Use a compression method: zlib, Zopfli, or 7zdeflate\n");
	fprintf(stderr, "                   By default, zlib and 7zdeflate are used\n");
	fprintf(stderr, "   --no-METHOD     Disable a compression method\n");
}

bool has_arg_value(int &i, char *argv[], const std::string &arg, const char *&val) {
	if (arg.compare(0, arg.npos, argv[i], arg.size()) == 0) {
		if (strlen(argv[i]) == arg.size()) {
			val = argv[++i];
			return true;
		} else if (argv[i][arg.size()] == '=') {
			val = argv[i] + arg.size() + 1;
			return true;
		}
	}

	return false;
}

bool has_arg(int &i, char *argv[], const std::string &arg) {
	if (arg.compare(argv[i]) == 0) {
		return true;
	}

	return false;
}

struct Arguments {
	std::vector<std::string> inputs;
	std::vector<std::string> outputs;
	int threads;
	uint32_t flags;
	bool quiet;
};

void default_args(Arguments &args) {
	args.threads = 0;
	args.flags = maxcso::TASKFLAG_NO_ZOPFLI;
	args.quiet = false;
}

int parse_args(Arguments &args, int argc, char *argv[]) {
	const char *val = nullptr;
	int i;
	for (i = 1; i < argc; ++i) {
		if (argv[i][0] == '-') {
			if (has_arg(i, argv, "--help") || has_arg(i, argv, "-h")) {
				show_help(argv[0]);
				return 1;
			} else if (has_arg(i, argv, "--version") || has_arg(i, argv, "-v")) {
				show_version();
				return 1;
			} else if (has_arg_value(i, argv, "--threads", val)) {
				args.threads = atoi(val);
			} else if (has_arg(i, argv, "--quiet")) {
				args.quiet = true;
			} else if (has_arg(i, argv, "--fast")) {
				args.flags |= maxcso::TASKFLAG_NO_ZLIB_BRUTE | maxcso::TASKFLAG_NO_ZOPFLI | maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg(i, argv, "--smallest")) {
				args.flags |= maxcso::TASKFLAG_FORCE_ALL;
			} else if (has_arg(i, argv, "--use-zopfli")) {
				args.flags &= ~maxcso::TASKFLAG_NO_ZOPFLI;
			} else if (has_arg(i, argv, "--use-zlib")) {
				args.flags &= ~maxcso::TASKFLAG_NO_ZLIB;
			} else if (has_arg(i, argv, "--use-7zdeflate")) {
				args.flags &= ~maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg(i, argv, "--no-zopfli")) {
				args.flags |= maxcso::TASKFLAG_NO_ZOPFLI;
			} else if (has_arg(i, argv, "--no-zlib")) {
				args.flags |= maxcso::TASKFLAG_NO_ZLIB;
			} else if (has_arg(i, argv, "--no-7zdeflate")) {
				args.flags |= maxcso::TASKFLAG_NO_7ZIP;
			} else if (has_arg_value(i, argv, "--out", val) || has_arg_value(i, argv, "-o", val)) {
				args.outputs.push_back(val);
			} else if (has_arg(i, argv, "--")) {
				break;
			} else {
				show_help(argv[0]);
				fprintf(stderr, "\nERROR: Unknown argument: %s\n", argv[i]);
				return 1;
			}
		} else {
			args.inputs.push_back(argv[i]);
		}
	}

	for (; i < argc; ++i) {
		// If we're here, it means we hit a "--".  The rest are inputs, not args.
		args.inputs.push_back(argv[i]);
	}

	return 0;
}

int validate_args(const char *arg0, Arguments &args) {
	if (args.threads == 0) {
		uv_cpu_info_t *cpus;
		uv_cpu_info(&cpus, &args.threads);
		uv_free_cpu_info(cpus, args.threads);
	}

	if (args.inputs.size() < args.outputs.size()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: Too many output files.\n");
		return 1;
	}

	// Automatically write to .cso files if not specified.
	for (size_t i = args.outputs.size(); i < args.inputs.size(); ++i) {
		if (args.inputs[i].size() <= 4) {
			continue;
		}

		const std::string ext = args.inputs[i].substr(args.inputs[i].size() - 4);
		if (ext == ".iso" || ext == ".ISO") {
			args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".cso");
		}
	}

	if (args.inputs.size() != args.outputs.size()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: Too few output files.\n");
		return 1;
	}

	if (args.inputs.empty()) {
		show_help(arg0);
		fprintf(stderr, "\nERROR: No input files.\n");
		return 1;
	}

	return 0;
}

#ifdef _WIN32
int setenv(const char *name, const char *value, int) {
	return _putenv_s(name, value);
}
#endif

inline uv_buf_t uv_buf_init(const char *str) {
	return uv_buf_init(const_cast<char *>(str), static_cast<unsigned int>(strlen(str)));
}

inline uv_buf_t uv_buf_init(const std::string &str) {
	return uv_buf_init(const_cast<char *>(str.c_str()), static_cast<unsigned int>(str.size()));
}

const std::string ANSI_RESET_LINE = "\033[2K\033[0G";

int main(int argc, char *argv[]) {
	Arguments args;
	default_args(args);
	int result = parse_args(args, argc, argv);
	if (result != 0) {
		return result;
	}
	result = validate_args(argv[0], args);
	if (result != 0) {
		return result;
	}

	char threadpool_size[32];
	sprintf(threadpool_size, "%d", args.threads);
	setenv("UV_THREADPOOL_SIZE", threadpool_size, 1);

	uv_loop_t loop;
	uv_tty_t tty;
	uv_loop_init(&loop);
	uv_tty_init(&loop, &tty, 2, 0);
	uv_tty_set_mode(&tty, 0);
	bool formatting = uv_guess_handle(2) == UV_TTY && !args.quiet;

	int64_t next = uv_hrtime();
	int64_t lastPos = 0;
	// 50ms
	static const int64_t interval_ns = 50000000LL;
	static const double interval_to_s = 1000.0 / 50.0;

	maxcso::ProgressCallback progress = [&next, &lastPos, formatting, &tty] (const maxcso::Task *task, maxcso::TaskStatus status, int64_t pos, int64_t total, int64_t written) {
		if (!formatting) {
			return;
		}

		std::string statusInfo;
		if (status == maxcso::TASK_INPROGRESS) {
			int64_t now = uv_hrtime();
			if (now >= next) {
				double percent = (pos * 100.0) / total;
				double ratio = (written * 100.0) / pos;
				double speed = 0.0;
				int64_t diff = pos - lastPos;
				if (diff > 0) {
					speed = (diff / 1024.0 / 1024.0) * interval_to_s;
				}

				char temp[128];
				sprintf(temp, "%3.0f%%, ratio=%3.0f%%, speed=%5.2f MB/s", percent, ratio, speed);
				statusInfo = temp;

				next = now + interval_ns;
				lastPos = pos;
			}
		} else if (status == maxcso::TASK_SUCCESS) {
			statusInfo = "Complete\n";
		} else {
			// This shouldn't happen.
			statusInfo = "Something went wrong.\n";
		}

		if (statusInfo.empty()) {
			return;
		}

		uv_buf_t bufs[2];
		uv_write_t write_req;

		unsigned int nbufs = 0;
		if (formatting) {
			bufs[nbufs++] = uv_buf_init(ANSI_RESET_LINE);
		}

		if (task->input.size() > 38) {
			statusInfo = "..." + task->input.substr(task->input.size() - 35) + ": " + statusInfo;
		} else {
			statusInfo = task->input + ": " + statusInfo;
		}

		bufs[nbufs++] = uv_buf_init(statusInfo);
		uv_write(&write_req, reinterpret_cast<uv_stream_t *>(&tty), bufs, nbufs, nullptr);
	};
	maxcso::ErrorCallback error = [&args, formatting, &tty] (const maxcso::Task *task, maxcso::TaskStatus status, const char *reason) {
		static std::string line;
		static uv_buf_t buf;
		uv_write_t write_req;

		if (args.quiet) {
			return;
		}

		line = (formatting ? ANSI_RESET_LINE : "") + "Error while processing " + task->input + ": " + reason + "\n";
		buf = uv_buf_init(line);
		uv_write(&write_req, reinterpret_cast<uv_stream_t *>(&tty), &buf, 1, nullptr);
	};

	std::vector<maxcso::Task> tasks;
	for (size_t i = 0; i < args.inputs.size(); ++i) {
		maxcso::Task task;
		task.input = args.inputs[i];
		task.output = args.outputs[i];
		task.progress = progress;
		task.error = error;
		task.flags = args.flags;
		tasks.push_back(std::move(task));
	}

	maxcso::Compress(tasks);

	uv_tty_reset_mode();
	uv_loop_close(&loop);
	return 0;
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "orc/orc-config.hh"
#include "orc/ColumnPrinter.hh"
#include "Exceptions.hh"

#include <string>
#include <memory>
#include <iostream>
#include <map>
#include <exception>

class TestMemoryPool: public orc::MemoryPool {
private:
  std::map<char*, uint64_t> blocks;
  uint64_t totalMemory;
  uint64_t maxMemory;

public:
  char* malloc(uint64_t size) override {
    char* p = static_cast<char*>(std::malloc(size));
    blocks[p] = size ;
    totalMemory += size;
    if (maxMemory < totalMemory) {
      maxMemory = totalMemory;
    }
    return p;
  }

  void free(char* p) override {
    std::free(p);
    totalMemory -= blocks[p] ;
    blocks.erase(p);
  }

  uint64_t getMaxMemory() {
    return maxMemory ;
  }

  TestMemoryPool(): totalMemory(0), maxMemory(0) {}
  ~TestMemoryPool();
};

TestMemoryPool::~TestMemoryPool() {}

int main(int argc, char* argv[]) {
  if (argc < 2) {
    std::cout << "Usage: file-memory [--columns=column1,column2,...] "
        << "[--batch=rows_in_batch] <filename> \n";
    return 1;
  }

  const std::string COLUMNS_PREFIX = "--columns=";
  const std::string BATCH_PREFIX = "--batch=";
  char* filename = ORC_NULLPTR;

  // Default parameters
  std::list<uint64_t> cols;
  uint32_t batchSize = 1000;

  // Read command-line options
  char *param, *value;
  for (int i = 1; i < argc; i++) {
    if ( (param = std::strstr(argv[i], COLUMNS_PREFIX.c_str())) ) {
      value = std::strtok(param+COLUMNS_PREFIX.length(), "," );
      while (value) {
        cols.push_back(static_cast<uint64_t>(std::atoi(value)));
        value = std::strtok(ORC_NULLPTR, "," );
      }
    } else if ( (param=strstr(argv[i], BATCH_PREFIX.c_str())) ) {
      batchSize = static_cast<uint32_t>(std::atoi(param+BATCH_PREFIX.length()));
    } else {
      filename = argv[i];
    }
  }

  if (filename == ORC_NULLPTR) {
    std::cout << "Error: Filename not provided.\n";
    return 1;
  }

  orc::ReaderOptions opts;
  if (cols.size() > 0) {
    opts.include(cols);
  }
  std::unique_ptr<orc::MemoryPool> pool(new TestMemoryPool());
  opts.setMemoryPool(*(pool.get()));

  std::unique_ptr<orc::Reader> reader;
  try{
    reader = orc::createReader(orc::readLocalFile(std::string(filename)), opts);
  } catch (orc::ParseError e) {
    std::cout << "Error reading file " << filename << "! " << e.what() << "\n";
    return -1;
  }

  std::unique_ptr<orc::ColumnVectorBatch> batch =
      reader->createRowBatch(batchSize);
  uint64_t readerMemory = reader->getMemoryUse();
  uint64_t batchMemory = batch->getMemoryUsage();
  while (reader->next(*batch)) {}
  uint64_t actualMemory =
      static_cast<TestMemoryPool*>(pool.get())->getMaxMemory();
  std::cout << "Reader memory estimate: " << readerMemory
            << "\nBatch memory estimate:  " ;
  if (batch->hasVariableLength()) {
    std::cout << "Cannot estimate because reading ARRAY or MAP columns";
  } else {
    std::cout << batchMemory
              << "\nTotal memory estimate:  " << readerMemory + batchMemory;
  }
  std::cout << "\nActual max memory used: " << actualMemory << "\n";

  return 0;
}
<commit_msg>ORC-43. FileMemory.cc throws uncaught exception. (omalley reviewed by asandryh)<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "orc/orc-config.hh"
#include "orc/ColumnPrinter.hh"
#include "Exceptions.hh"

#include <string>
#include <memory>
#include <iostream>
#include <map>
#include <exception>

class TestMemoryPool: public orc::MemoryPool {
private:
  std::map<char*, uint64_t> blocks;
  uint64_t totalMemory;
  uint64_t maxMemory;

public:
  char* malloc(uint64_t size) override {
    char* p = static_cast<char*>(std::malloc(size));
    blocks[p] = size ;
    totalMemory += size;
    if (maxMemory < totalMemory) {
      maxMemory = totalMemory;
    }
    return p;
  }

  void free(char* p) override {
    std::free(p);
    totalMemory -= blocks[p] ;
    blocks.erase(p);
  }

  uint64_t getMaxMemory() {
    return maxMemory ;
  }

  TestMemoryPool(): totalMemory(0), maxMemory(0) {}
  ~TestMemoryPool();
};

TestMemoryPool::~TestMemoryPool() {}

void processFile(const char* filename,
                 const std::list<uint64_t>& cols,
                 uint32_t batchSize) {
  orc::ReaderOptions opts;
  if (cols.size() > 0) {
    opts.include(cols);
  }
  std::unique_ptr<orc::MemoryPool> pool(new TestMemoryPool());
  opts.setMemoryPool(*(pool.get()));

  std::unique_ptr<orc::Reader> reader =
    orc::createReader(orc::readLocalFile(std::string(filename)), opts);

  std::unique_ptr<orc::ColumnVectorBatch> batch =
      reader->createRowBatch(batchSize);
  uint64_t readerMemory = reader->getMemoryUse();
  uint64_t batchMemory = batch->getMemoryUsage();
  while (reader->next(*batch)) {}
  uint64_t actualMemory =
      static_cast<TestMemoryPool*>(pool.get())->getMaxMemory();
  std::cout << "Reader memory estimate: " << readerMemory
            << "\nBatch memory estimate:  " ;
  if (batch->hasVariableLength()) {
    std::cout << "Cannot estimate because reading ARRAY or MAP columns";
  } else {
    std::cout << batchMemory
              << "\nTotal memory estimate:  " << readerMemory + batchMemory;
  }
  std::cout << "\nActual max memory used: " << actualMemory << "\n";
}

int main(int argc, char* argv[]) {
  if (argc < 2) {
    std::cout << "Usage: file-memory [--columns=column1,column2,...] "
        << "[--batch=rows_in_batch] <filename> \n";
    return 1;
  }

  const std::string COLUMNS_PREFIX = "--columns=";
  const std::string BATCH_PREFIX = "--batch=";
  char* filename = ORC_NULLPTR;

  // Default parameters
  std::list<uint64_t> cols;
  uint32_t batchSize = 1000;

  // Read command-line options
  char *param, *value;
  for (int i = 1; i < argc; i++) {
    if ( (param = std::strstr(argv[i], COLUMNS_PREFIX.c_str())) ) {
      value = std::strtok(param+COLUMNS_PREFIX.length(), "," );
      while (value) {
        cols.push_back(static_cast<uint64_t>(std::atoi(value)));
        value = std::strtok(ORC_NULLPTR, "," );
      }
    } else if ( (param=strstr(argv[i], BATCH_PREFIX.c_str())) ) {
      batchSize =
        static_cast<uint32_t>(std::atoi(param+BATCH_PREFIX.length()));
    } else {
      filename = argv[i];
    }
  }

  if (filename == ORC_NULLPTR) {
    std::cout << "Error: Filename not provided.\n";
    return 1;
  }

  try {
    processFile(filename, cols, batchSize);
    return 0;
  } catch (std::exception& ex) {
    std::cerr << "Caught exception: " << ex.what() << "\n";
    return 1;
  }
}
<|endoftext|>


<commit_before>#pragma once
/*
**  Copyright (C) 2012 Aldebaran Robotics
**  See COPYING for the license
*/

#ifndef _QITYPE_FUTURE_HPP_
#define _QITYPE_FUTURE_HPP_

#include <vector>
#include <qi/atomic.hpp>
#include <qi/config.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/function.hpp>
#include <boost/bind.hpp>

#ifdef _MSC_VER
#  pragma warning( push )
#  pragma warning( disable: 4251 )
#endif

namespace qi {

  template<typename T> struct FutureType
  {
    typedef T type;
    typedef T typecast;
  };

  struct FutureHasNoValue{};
  // Hold a void* for Future<void>
  template<> struct FutureType<void>
  {
    typedef void* type;
    typedef FutureHasNoValue typecast;
  };

  template <typename T> class FutureInterface;
  template <typename T> class Future;
  template <typename T> class FutureSync;
  template <typename T> class Promise;

  namespace detail {
    template <typename T> class FutureState;
  }

  template <typename T>
  class Future {
  public:
    typedef typename FutureType<T>::type ValueType;
    typedef typename FutureType<T>::typecast ValueTypeCast;

    Future()
      : _p(boost::make_shared<detail::FutureState<T> >())
    {
    }

    Future(const FutureSync<T>& b)
    : _p(b._p)
    {
      b._sync = false;
    }

    Future(const Future<T>& b)
    : _p(b._p)
    {}

    bool operator==(const Future<T> &other)
    {
      return _p.get() == other._p.get();
    }

    inline Future<T>& operator = (const FutureSync<T>& b)
    {
      b._sync = false;
      _p = b._p;
      return *this;
    }

    explicit Future<T>(const ValueType& v)
    {
      Promise<T> promise;
      promise.setValue(v);
      *this = promise.future();
    }

    inline const ValueType &value() const    { return _p->value(); }

    /** Wait for future, and return a default value in case of error.
     * @param defaultVal the value to return in case of Future error
     * @return the future value, or \p defaultVal if hasError() is true.
     */
    inline const ValueType &valueWithDefault(const ValueType& defaultVal = ValueType()) const;

    inline operator const ValueTypeCast&() const { return _p->value(); }

    /** Wait for future to contain a value or an error
     @param msecs: Maximum time to wait in milliseconds, 0 means forever and -1 means return immediately.
     @return true if future contains a value or an error, false if timeout was reached
     */
    inline bool wait(int msecs = 0) const             { return _p->wait(msecs); }
    inline bool isReady() const                       { return _p->isReady(); }
    inline bool hasError(int msecs = 0) const         { return _p->hasError(msecs); }
    inline const std::string &error() const           { return _p->error(); }

    inline FutureSync<T> sync()
    {
      return FutureSync<T>(*this);
    };

    /** cancel() the asynchronous operation if possible
    * Exact effect is controlled by the cancel implementation, but it is
    * expected to set a value or an error to the Future as fast as possible.
    * Note that cancelation may be asynchronous.
    * @throw runtime_error if isCancelleable() is false.
    */
    void cancel()
    {
      _p->cancel();
    }

    bool isCanceleable() const
    {
      return _p->isCanceleable();
    }
  public: //Signals
    typedef boost::function<void (Future<T>) > Connection;
    inline void connect(const Connection& s) { _p->connect(*this, s);}
    //qi::Signal<void (qi::Future<T>)> &onResult() { return _p->_onResult; }

  protected:
    // C4251 needs to have dll-interface to be used by clients of class 'qi::Future<T>'
    boost::shared_ptr< detail::FutureState<T> > _p;
    friend class Promise<T>;
    friend class FutureSync<T>;
  };

  template<typename T> class FutureSync: public Future<T>
  {
  public:
    // This future cannot be set, so sync starts at false
    FutureSync() : _sync(false) {}

    FutureSync(const Future<T>& b)
    : Future<T>(b)
    , _sync(true)
    {
    }

    FutureSync(const FutureSync<T>& b)
    : Future<T>(b)
    , _sync(true)
    {
      b._sync = false;
    }

    explicit FutureSync<T>(const typename Future<T>::ValueType& v)
    : _sync(false)
    {
      Promise<T> promise;
      promise.setValue(v);
      *this = promise.future();
    }

    inline FutureSync<T>& operator = (const FutureSync<T>& b)
    {
      this->_p = b._p;
      _sync = true;
      b._sync = false;
      return *this;
    }

    inline FutureSync<T>& operator = (const Future<T>& b)
    {
     this->_p = b._p;
      _sync = true;
      return *this;
    }

    ~FutureSync()
    {
      if (_sync)
        this->value();
    }

    inline const typename Future<T>::ValueType &valueWithDefault(const typename Future<T>::ValueType& defaultVal = typename Future<T>::ValueType()) const
    {
      _sync = false;
      return Future<T>::valueWithDefault(defaultVal);
    }
    inline const typename Future<T>::ValueType &value() const        { _sync = false; return Future<T>::value(); }
    inline operator const typename Future<T>::ValueTypeCast&() const { _sync = false; return Future<T>::value(); }
    inline bool wait(int msecs = 0) const                   { _sync = false; return Future<T>::wait(msecs); }
    inline bool isReady() const                             { _sync = false; return Future<T>::isReady(); }
    inline bool hasError(int msecs = 0) const               { _sync = false; return Future<T>::hasError(msecs); }
    inline const std::string &error() const                 { _sync = false; return Future<T>::error(); }
    inline void cancel()                                    { _sync = false; Future<T>::cancel(); }
    bool isCanceleable() const                              { _sync = false; return Future<T>::isCanceleable(); }

    Future<T> async()
    {
      return *this;
    }

  private:
    mutable bool _sync;
    friend class Future<T>;
  };

  template <typename T>
  class Promise {
  public:
    typedef typename FutureType<T>::type ValueType;

    Promise() { }

    /** Create a cancelleable promise. If Future<T>::cancel is invoked,
     * onCancel() will be called. It is expected to call setValue() or
     * setError() as quickly as possible, but can do so in an asynchronous
     * way.
    */
    Promise(boost::function<void ()> onCancel)
    {
       _f._p->setOnCancel(onCancel);
    }

    void setValue(const ValueType &value) {
      _f._p->setValue(_f, value);
    }

    void setError(const std::string &msg) {
      _f._p->setError(_f, msg);
    }

    void reset() {
      _f._p->reset();
    }

    Future<T> future() { return _f; }

  protected:
    Future<T> _f;
  };

  template<typename T>
  class FutureBarrier {
  public:
    /// FutureBarrier constructor taking no argument.
    FutureBarrier()
      : _closed(false)
      , _count(0)
      , _futures()
      , _promise()
    {}

    /// Adds the future to the barrier.
    bool addFuture(qi::Future<T> fut) {
      // Can't add future from closed qi::FutureBarrier.
      if (this->_closed)
        return false;

      ++(this->_count);
      fut.connect(boost::bind<void>(&FutureBarrier::onFutureDone, this));
      this->_futures.push_back(fut);
      return true;
    }

    /// Gets the future result for the barrier.
    Future< std::vector< Future<T> > > future() {
      this->close();
      return this->_promise.future();
    }

  protected:
    bool _closed;
    Atomic<int> _count;
    std::vector< Future<T> > _futures;
    Promise< std::vector< Future<T> > > _promise;

  private:
    void onFutureDone() {
      if (--(this->_count) == 0 && this->_closed) {
        this->_promise.setValue(this->_futures);
      }
    }

    void close() {
      this->_closed = true;
      if (*(this->_count) == 0) {
        this->_promise.setValue(this->_futures);
      }
    }
  };

  template <typename T>
  qi::Future<T> makeFutureError(const std::string &value);

  /// Helper function to wait on a vector of futures.
  template <typename T>
  void waitForAll(std::vector< Future<T> >& vect);

  /// Helper function to wait for the first valid future.
  template <typename T>
  qi::FutureSync< qi::Future<T> > waitForFirst(std::vector< Future<T> >& vect);
}

#ifdef _MSC_VER
#  pragma warning( pop )
#endif

#include <qi/details/future.hxx>

#endif  // _QITYPE_FUTURE_HPP_
<commit_msg>FutureSync: unlock on connect too<commit_after>#pragma once
/*
**  Copyright (C) 2012 Aldebaran Robotics
**  See COPYING for the license
*/

#ifndef _QITYPE_FUTURE_HPP_
#define _QITYPE_FUTURE_HPP_

#include <vector>
#include <qi/atomic.hpp>
#include <qi/config.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/function.hpp>
#include <boost/bind.hpp>

#ifdef _MSC_VER
#  pragma warning( push )
#  pragma warning( disable: 4251 )
#endif

namespace qi {

  template<typename T> struct FutureType
  {
    typedef T type;
    typedef T typecast;
  };

  struct FutureHasNoValue{};
  // Hold a void* for Future<void>
  template<> struct FutureType<void>
  {
    typedef void* type;
    typedef FutureHasNoValue typecast;
  };

  template <typename T> class FutureInterface;
  template <typename T> class Future;
  template <typename T> class FutureSync;
  template <typename T> class Promise;

  namespace detail {
    template <typename T> class FutureState;
  }

  template <typename T>
  class Future {
  public:
    typedef typename FutureType<T>::type ValueType;
    typedef typename FutureType<T>::typecast ValueTypeCast;

    Future()
      : _p(boost::make_shared<detail::FutureState<T> >())
    {
    }

    Future(const FutureSync<T>& b)
    : _p(b._p)
    {
      b._sync = false;
    }

    Future(const Future<T>& b)
    : _p(b._p)
    {}

    bool operator==(const Future<T> &other)
    {
      return _p.get() == other._p.get();
    }

    inline Future<T>& operator = (const FutureSync<T>& b)
    {
      b._sync = false;
      _p = b._p;
      return *this;
    }

    explicit Future<T>(const ValueType& v)
    {
      Promise<T> promise;
      promise.setValue(v);
      *this = promise.future();
    }

    inline const ValueType &value() const    { return _p->value(); }

    /** Wait for future, and return a default value in case of error.
     * @param defaultVal the value to return in case of Future error
     * @return the future value, or \p defaultVal if hasError() is true.
     */
    inline const ValueType &valueWithDefault(const ValueType& defaultVal = ValueType()) const;

    inline operator const ValueTypeCast&() const { return _p->value(); }

    /** Wait for future to contain a value or an error
     @param msecs: Maximum time to wait in milliseconds, 0 means forever and -1 means return immediately.
     @return true if future contains a value or an error, false if timeout was reached
     */
    inline bool wait(int msecs = 0) const             { return _p->wait(msecs); }
    inline bool isReady() const                       { return _p->isReady(); }
    inline bool hasError(int msecs = 0) const         { return _p->hasError(msecs); }
    inline const std::string &error() const           { return _p->error(); }

    inline FutureSync<T> sync()
    {
      return FutureSync<T>(*this);
    };

    /** cancel() the asynchronous operation if possible
    * Exact effect is controlled by the cancel implementation, but it is
    * expected to set a value or an error to the Future as fast as possible.
    * Note that cancelation may be asynchronous.
    * @throw runtime_error if isCancelleable() is false.
    */
    void cancel()
    {
      _p->cancel();
    }

    bool isCanceleable() const
    {
      return _p->isCanceleable();
    }
  public: //Signals
    typedef boost::function<void (Future<T>) > Connection;
    inline void connect(const Connection& s) { _p->connect(*this, s);}
    //qi::Signal<void (qi::Future<T>)> &onResult() { return _p->_onResult; }

  protected:
    // C4251 needs to have dll-interface to be used by clients of class 'qi::Future<T>'
    boost::shared_ptr< detail::FutureState<T> > _p;
    friend class Promise<T>;
    friend class FutureSync<T>;
  };

  template<typename T> class FutureSync: public Future<T>
  {
  public:
    // This future cannot be set, so sync starts at false
    FutureSync() : _sync(false) {}

    FutureSync(const Future<T>& b)
    : Future<T>(b)
    , _sync(true)
    {
    }

    FutureSync(const FutureSync<T>& b)
    : Future<T>(b)
    , _sync(true)
    {
      b._sync = false;
    }

    explicit FutureSync<T>(const typename Future<T>::ValueType& v)
    : _sync(false)
    {
      Promise<T> promise;
      promise.setValue(v);
      *this = promise.future();
    }

    inline FutureSync<T>& operator = (const FutureSync<T>& b)
    {
      this->_p = b._p;
      _sync = true;
      b._sync = false;
      return *this;
    }

    inline FutureSync<T>& operator = (const Future<T>& b)
    {
     this->_p = b._p;
      _sync = true;
      return *this;
    }

    ~FutureSync()
    {
      if (_sync)
        this->value();
    }

    inline const typename Future<T>::ValueType &valueWithDefault(const typename Future<T>::ValueType& defaultVal = typename Future<T>::ValueType()) const
    {
      _sync = false;
      return Future<T>::valueWithDefault(defaultVal);
    }
    inline const typename Future<T>::ValueType &value() const        { _sync = false; return Future<T>::value(); }
    inline operator const typename Future<T>::ValueTypeCast&() const { _sync = false; return Future<T>::value(); }
    inline bool wait(int msecs = 0) const                   { _sync = false; return Future<T>::wait(msecs); }
    inline bool isReady() const                             { _sync = false; return Future<T>::isReady(); }
    inline bool hasError(int msecs = 0) const               { _sync = false; return Future<T>::hasError(msecs); }
    inline const std::string &error() const                 { _sync = false; return Future<T>::error(); }
    inline void cancel()                                    { _sync = false; Future<T>::cancel(); }
    bool isCanceleable() const                              { _sync = false; return Future<T>::isCanceleable(); }
    inline void connect(const typename Future<T>::Connection& s) { _sync = false; Future<T>::connect(s);}
    Future<T> async()
    {
      return *this;
    }

  private:
    mutable bool _sync;
    friend class Future<T>;
  };

  template <typename T>
  class Promise {
  public:
    typedef typename FutureType<T>::type ValueType;

    Promise() { }

    /** Create a cancelleable promise. If Future<T>::cancel is invoked,
     * onCancel() will be called. It is expected to call setValue() or
     * setError() as quickly as possible, but can do so in an asynchronous
     * way.
    */
    Promise(boost::function<void ()> onCancel)
    {
       _f._p->setOnCancel(onCancel);
    }

    void setValue(const ValueType &value) {
      _f._p->setValue(_f, value);
    }

    void setError(const std::string &msg) {
      _f._p->setError(_f, msg);
    }

    void reset() {
      _f._p->reset();
    }

    Future<T> future() { return _f; }

  protected:
    Future<T> _f;
  };

  template<typename T>
  class FutureBarrier {
  public:
    /// FutureBarrier constructor taking no argument.
    FutureBarrier()
      : _closed(false)
      , _count(0)
      , _futures()
      , _promise()
    {}

    /// Adds the future to the barrier.
    bool addFuture(qi::Future<T> fut) {
      // Can't add future from closed qi::FutureBarrier.
      if (this->_closed)
        return false;

      ++(this->_count);
      fut.connect(boost::bind<void>(&FutureBarrier::onFutureDone, this));
      this->_futures.push_back(fut);
      return true;
    }

    /// Gets the future result for the barrier.
    Future< std::vector< Future<T> > > future() {
      this->close();
      return this->_promise.future();
    }

  protected:
    bool _closed;
    Atomic<int> _count;
    std::vector< Future<T> > _futures;
    Promise< std::vector< Future<T> > > _promise;

  private:
    void onFutureDone() {
      if (--(this->_count) == 0 && this->_closed) {
        this->_promise.setValue(this->_futures);
      }
    }

    void close() {
      this->_closed = true;
      if (*(this->_count) == 0) {
        this->_promise.setValue(this->_futures);
      }
    }
  };

  template <typename T>
  qi::Future<T> makeFutureError(const std::string &value);

  /// Helper function to wait on a vector of futures.
  template <typename T>
  void waitForAll(std::vector< Future<T> >& vect);

  /// Helper function to wait for the first valid future.
  template <typename T>
  qi::FutureSync< qi::Future<T> > waitForFirst(std::vector< Future<T> >& vect);
}

#ifdef _MSC_VER
#  pragma warning( pop )
#endif

#include <qi/details/future.hxx>

#endif  // _QITYPE_FUTURE_HPP_
<|endoftext|>


<commit_before>/*
 * Copyright 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <seastar/core/future.hh>
#include <seastar/core/future-util.hh>
#include <seastar/core/gate.hh>
#include <seastar/core/shared_ptr.hh>
#include "seastarx.hh"

namespace utils {

// Synchronizer which allows to track and wait for asynchronous operations
// which were in progress at the time of wait initiation.
class phased_barrier {
public:
    using phase_type = uint64_t;
private:
    using gate = seastar::gate;
    lw_shared_ptr<gate> _gate;
    phase_type _phase;
public:
    phased_barrier()
        : _gate(make_lw_shared<gate>())
        , _phase(0)
    { }

    class operation {
        lw_shared_ptr<gate> _gate;
    public:
        operation() : _gate() {}
        operation(lw_shared_ptr<gate> g) : _gate(std::move(g)) {}
        operation(const operation&) = delete;
        operation(operation&&) = default;
        operation& operator=(operation&&) = default;
        ~operation() {
            if (_gate) {
                _gate->leave();
            }
        }
    };

    // Starts new operation. The operation ends when the "operation" object is destroyed.
    // The operation may last longer than the life time of the phased_barrier.
    operation start() {
        _gate->enter();
        return { _gate };
    }

    // Starts a new phase and waits for all operations started in any of the earlier phases.
    // It is fine to start multiple awaits in parallel.
    future<> advance_and_await() {
        ++_phase;
        auto old_gate = std::move(_gate);
        _gate = make_lw_shared<gate>();
        return old_gate->close().then([old_gate, op = start()] {});
    }

    // Returns current phase number. The smallest value returned is 0.
    phase_type phase() const {
        return _phase;
    }
};

}
<commit_msg>utils: phased_barrier: Make advance_and_await() have strong exception guarantees<commit_after>/*
 * Copyright 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <seastar/core/future.hh>
#include <seastar/core/future-util.hh>
#include <seastar/core/gate.hh>
#include <seastar/core/shared_ptr.hh>
#include "seastarx.hh"

namespace utils {

// Synchronizer which allows to track and wait for asynchronous operations
// which were in progress at the time of wait initiation.
class phased_barrier {
public:
    using phase_type = uint64_t;
private:
    using gate = seastar::gate;
    lw_shared_ptr<gate> _gate;
    phase_type _phase;
public:
    phased_barrier()
        : _gate(make_lw_shared<gate>())
        , _phase(0)
    { }

    class operation {
        lw_shared_ptr<gate> _gate;
    public:
        operation() : _gate() {}
        operation(lw_shared_ptr<gate> g) : _gate(std::move(g)) {}
        operation(const operation&) = delete;
        operation(operation&&) = default;
        operation& operator=(operation&&) = default;
        ~operation() {
            if (_gate) {
                _gate->leave();
            }
        }
    };

    // Starts new operation. The operation ends when the "operation" object is destroyed.
    // The operation may last longer than the life time of the phased_barrier.
    operation start() {
        _gate->enter();
        return { _gate };
    }

    // Starts a new phase and waits for all operations started in any of the earlier phases.
    // It is fine to start multiple awaits in parallel.
    // Strong exception guarantees.
    future<> advance_and_await() {
        auto new_gate = make_lw_shared<gate>();
        ++_phase;
        auto old_gate = std::exchange(_gate, std::move(new_gate));
        return old_gate->close().then([old_gate, op = start()] {});
    }

    // Returns current phase number. The smallest value returned is 0.
    phase_type phase() const {
        return _phase;
    }
};

}
<|endoftext|>


<commit_before>#include "covcorr.hpp"

#include <opencv2/opencv.hpp>

#include "cvutils.hpp"
#include "features.hpp"

// First, let me try the rude, non-integral-image approach.
// Iv'e got the feeling it will either be as-fast-as or not-
// much-slower-than the II-approach, but use way less memory.
//
// Of course, I might be mistaken.

static cv::Mat extract_cov(const warco::Features& feats, unsigned x, unsigned y, unsigned w, unsigned h)
{
    auto nfeats = feats.size();
    std::vector<float> means(nfeats, 0.0f);

    if(w == 1 && h == 1)
        throw std::runtime_error("Covariance of a single point O.o");

    cv::Rect r(x, y, w, h);

    for(auto i = 0 ; i < nfeats ; ++i)
        means[i] = mean(feats[i](r))[0];

    cv::Mat cov(nfeats, nfeats, CV_32FC1, 0.0f);

    // TODO: Use calcCovarMatrix?
    // Doesn't seem to work in our use-case where
    // the different dimensions of samples are spread across cvMats

    // TODO: Could use iterators?
    std::vector<const float*> lines(nfeats);
    for(auto iy = 0 ; iy < h ; ++iy) {
        for(auto i = 0 ; i < nfeats ; ++i)
            lines[i] = feats[i].ptr<float>(y + iy);

        for(auto ix = 0 ; ix < w ; ++ix) {
            for(auto i = 0 ; i < nfeats ; ++i) {
                float fi = *lines[i]-means[i];
                cov.at<float>(i,i) += fi*fi;
                for(auto j = i+1 ; j < nfeats ; ++j) {
                    cov.at<float>(i,j) += fi*(*lines[j]-means[j]);
                    cov.at<float>(j,i) += fi*(*lines[j]-means[j]);
                }
            }

            for(auto& line : lines) ++line;
        }
    }

    return cov/(w*h-1);
}

// TODO: Use a unittesting framework.
static void test_cov()
{
    std::cout << "cov... " << std::flush;

    warco::Features fts = {
        (cv::Mat_<float>(3,3) <<
            1.f, 2.f, 3.f,
            4.f, 5.f, 6.f,
            7.f, 8.f, 9.f
        ),
        (cv::Mat_<float>(3,3) <<
            .1f, .2f, .3f,
            .4f, .5f, .6f,
            .7f, .8f, .9f
        ),
    };

    // With a unittest framework, I'd check for this throwing.
    //cv::Mat cov = extract_cov(fts, 0, 0, 1, 1);

    cv::Mat cov = extract_cov(fts, 0, 1, 2, 2);
    warco::assert_mat_almost_eq(cov, (cv::Mat_<float>(2,2) <<
        10./3., 1./3.,
         1./3., .1/3.
    ));

    cov = extract_cov(fts, 0, 0, 3, 3);
    warco::assert_mat_almost_eq(cov, (cv::Mat_<float>(2,2) <<
        7.5, .75,
        .75, .075
    ));

    std::cout << "SUCCESS" << std::endl;
}

static cv::Mat cov2corr(const cv::Mat& cov)
{
    // "make invertible" ->
    //     Clamp |eigenvalues| to 1e-4
    cv::Mat nrvo = warco::eig_fn(cov, [](double l) {
        return std::copysign(std::max(1e-4, std::abs(l)), l);
    });

    // now, make a corr out of this.

    std::vector<float> stddev(nrvo.cols);
    // TODO: Doesn't work. Why?
    //sqrt(nrvo.diag(), stddev);
    auto diag = nrvo.diag();
    for(auto i = 0 ; i < stddev.size() ; ++i)
        stddev[i] = sqrt(diag.at<float>(i));

    for(auto y = 0 ; y < nrvo.rows ; ++y) {
        float* line = nrvo.ptr<float>(y);
        for(auto x = 0 ; x < nrvo.cols ; ++x)
            *line++ /= stddev[x]*stddev[y];
    }

    return nrvo;
}

static void test_cov2corr()
{
    std::cout << "cov2corr... " << std::flush;

    cv::Mat cov = (cv::Mat_<float>(2,2) <<
         4.f, 20.f,
        20.f, 25.f
    );

    warco::assert_mat_almost_eq(cov2corr(cov), (cv::Mat_<float>(2,2) <<
        1.f, 2.f,
        2.f, 1.f
    ));

    std::cout << "SUCCESS" << std::endl;
}

void warco::test_covcorr()
{
    test_cov();
    test_cov2corr();
}

cv::Mat warco::extract_corr(const Features& feats, unsigned x, unsigned y, unsigned w, unsigned h)
{
    return cov2corr(extract_cov(feats, x, y, w, h));
}

std::vector<cv::Mat> warco::extract_corrs(const Features& feats)
{
    std::vector<cv::Mat> nrvo(25);
    auto i = nrvo.begin();

    if(feats[0].cols != 50 || feats[0].rows != 50)
        throw std::runtime_error("Only works on 50x50 images. Sorry mate.");

    for(auto y = 0 ; y < 5 ; ++y)
        for(auto x = 0 ; x < 5 ; ++x)
            *i++ = cov2corr(extract_cov(feats, 1+8*x, 1+8*y, 16, 16));

    return nrvo;
}

<commit_msg>A little more info on cov/corr computation.<commit_after>#include "covcorr.hpp"

#include <opencv2/opencv.hpp>

#include "cvutils.hpp"
#include "features.hpp"

// First, let me try the rude, non-integral-image approach.
// Iv'e got the feeling it will either be as-fast-as or not-
// much-slower-than the II-approach, but use way less memory.
//
// Of course, I might be mistaken.

static cv::Mat extract_cov(const warco::Features& feats, unsigned x, unsigned y, unsigned w, unsigned h)
{
    auto nfeats = feats.size();
    std::vector<float> means(nfeats, 0.0f);

    if(w == 1 && h == 1)
        throw std::runtime_error("Covariance of a single point O.o");

    cv::Rect r(x, y, w, h);

    for(auto i = 0 ; i < nfeats ; ++i)
        means[i] = mean(feats[i](r))[0];

    cv::Mat cov(nfeats, nfeats, CV_32FC1, 0.0f);

    // TODO: Use calcCovarMatrix?
    // Doesn't seem to work in our use-case where
    // the different dimensions of samples are spread across cvMats

    // TODO: Could use iterators?
    std::vector<const float*> lines(nfeats);
    for(auto iy = 0 ; iy < h ; ++iy) {
        for(auto i = 0 ; i < nfeats ; ++i)
            lines[i] = feats[i].ptr<float>(y + iy);

        for(auto ix = 0 ; ix < w ; ++ix) {
            for(auto i = 0 ; i < nfeats ; ++i) {
                float fi = *lines[i]-means[i];
                cov.at<float>(i,i) += fi*fi;
                for(auto j = i+1 ; j < nfeats ; ++j) {
                    cov.at<float>(i,j) += fi*(*lines[j]-means[j]);
                    cov.at<float>(j,i) += fi*(*lines[j]-means[j]);
                }
            }

            for(auto& line : lines) ++line;
        }
    }

    return cov/(w*h-1);
}

// TODO: Use a unittesting framework.
static void test_cov()
{
    std::cout << "cov... " << std::flush;

    warco::Features fts = {
        (cv::Mat_<float>(3,3) <<
            1.f, 2.f, 3.f,
            4.f, 5.f, 6.f,
            7.f, 8.f, 9.f
        ),
        (cv::Mat_<float>(3,3) <<
            .1f, .2f, .3f,
            .4f, .5f, .6f,
            .7f, .8f, .9f
        ),
    };

    // With a unittest framework, I'd check for this throwing.
    //cv::Mat cov = extract_cov(fts, 0, 0, 1, 1);

    cv::Mat cov = extract_cov(fts, 0, 1, 2, 2);
    warco::assert_mat_almost_eq(cov, (cv::Mat_<float>(2,2) <<
        10./3., 1./3.,
         1./3., .1/3.
    ));

    cov = extract_cov(fts, 0, 0, 3, 3);
    warco::assert_mat_almost_eq(cov, (cv::Mat_<float>(2,2) <<
        7.5, .75,
        .75, .075
    ));

    std::cout << "SUCCESS" << std::endl;
}

static cv::Mat cov2corr(const cv::Mat& cov)
{
    // "make invertible" ->
    //     Clamp |eigenvalues| to 1e-4
    //
    //  This is to work around patches where all pixels have the same value
    //  for a certain feature, in which case there'd be a division by 0 in the
    //  correlation computation.
    cv::Mat nrvo = warco::eig_fn(cov, [](double l) {
        return std::copysign(std::max(1e-4, std::abs(l)), l);
    });

    // now, make a corr out of this.

    std::vector<float> stddev(nrvo.cols);
    // TODO: Doesn't work. Why?
    //sqrt(nrvo.diag(), stddev);
    auto diag = nrvo.diag();
    for(auto i = 0 ; i < stddev.size() ; ++i)
        stddev[i] = sqrt(diag.at<float>(i));

    for(auto y = 0 ; y < nrvo.rows ; ++y) {
        float* line = nrvo.ptr<float>(y);
        for(auto x = 0 ; x < nrvo.cols ; ++x)
            *line++ /= stddev[x]*stddev[y];
    }

    return nrvo;
}

static void test_cov2corr()
{
    std::cout << "cov2corr... " << std::flush;

    cv::Mat cov = (cv::Mat_<float>(2,2) <<
         4.f, 20.f,
        20.f, 25.f
    );

    warco::assert_mat_almost_eq(cov2corr(cov), (cv::Mat_<float>(2,2) <<
        1.f, 2.f,
        2.f, 1.f
    ));

    std::cout << "SUCCESS" << std::endl;
}

void warco::test_covcorr()
{
    test_cov();
    test_cov2corr();
}

cv::Mat warco::extract_corr(const Features& feats, unsigned x, unsigned y, unsigned w, unsigned h)
{
    return cov2corr(extract_cov(feats, x, y, w, h));
}

std::vector<cv::Mat> warco::extract_corrs(const Features& feats)
{
    std::vector<cv::Mat> nrvo(25);
    auto i = nrvo.begin();

    if(feats[0].cols != 50 || feats[0].rows != 50)
        throw std::runtime_error("Only works on 50x50 images. Sorry mate.");

    for(auto y = 0 ; y < 5 ; ++y)
        for(auto x = 0 ; x < 5 ; ++x)
            *i++ = cov2corr(extract_cov(feats, 1+8*x, 1+8*y, 16, 16));

    return nrvo;
}

<|endoftext|>


<commit_before>/*
 * fMBT, free Model Based Testing tool
 * Copyright (c) 2011, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include "adapter_dummy.hh"
#include "log.hh"
#include <cstdio>
#include <sstream>

Adapter_dummy::Adapter_dummy(Log& l, std::string params) :
  Adapter::Adapter(l)
{
}

void Adapter_dummy::execute(std::vector<int>& action)
{
  log.push("Adapter_dummy");

  log.print("<dummy_execute type=\"input\" name=\"%s\"/>\n",
	    getUActionName(action[0]));

  log.pop();
}

int   Adapter_dummy::observe(std::vector<int> &action,
				bool block)
{
  return false;
}

FACTORY_DEFAULT_CREATOR(Adapter, Adapter_dummy, "dummy")
<commit_msg>fix dummy observe...<commit_after>/*
 * fMBT, free Model Based Testing tool
 * Copyright (c) 2011, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include "adapter_dummy.hh"
#include "log.hh"
#include <cstdio>
#include <sstream>

Adapter_dummy::Adapter_dummy(Log& l, std::string params) :
  Adapter::Adapter(l)
{
}

void Adapter_dummy::execute(std::vector<int>& action)
{
  log.push("Adapter_dummy");

  log.print("<dummy_execute type=\"input\" name=\"%s\"/>\n",
	    getUActionName(action[0]));

  log.pop();
}

int   Adapter_dummy::observe(std::vector<int> &action,
				bool block)
{
  return SILENCE;
}

FACTORY_DEFAULT_CREATOR(Adapter, Adapter_dummy, "dummy")
<|endoftext|>


<commit_before>/*
 * Heap (priortity_queue / multiset)
 *
 */
class Solution {
public:
    vector<double> medianSlidingWindow(vector<int>& nums, int k) {
        vector<double> res;
        multiset<int> small, large;
        for (int i = 0; i < nums.size(); ++i) {
            if (i >= k) {
                if (small.count(nums[i - k])) small.erase(small.find(nums[i - k]));
                else if (large.count(nums[i - k])) large.erase(large.find(nums[i - k]));
            }
            if (small.size() <= large.size()) {
                if (large.empty() || nums[i] <= *large.begin()) small.insert(nums[i]);
                else {
                    small.insert(*large.begin());
                    large.erase(large.begin());
                    large.insert(nums[i]);
                }
            } else {
                if (nums[i] >= *small.rbegin()) large.insert(nums[i]);
                else {
                    large.insert(*small.rbegin());
                    small.erase(--small.end());
                    small.insert(nums[i]);
                }
            }
            if (i >= (k - 1)) {
                if (k % 2) res.push_back(*small.rbegin());
                else res.push_back(((double)*small.rbegin() + *large.begin()) / 2);
            }
        }
        return res;
    }
};

// Conlcusion:
// 把插入新值和删除元素分开。

<commit_msg>Update 480.cpp<commit_after>/*
 * Heap (priortity_queue / multiset)
 *
 */

// version 1:
class Solution {
public:
    vector<double> medianSlidingWindow(vector<int>& nums, int k) {
        vector<double> res;
        multiset<int> small, large;
        for (int i = 0; i < nums.size(); ++i) {
            if (i >= k) {
                if (small.count(nums[i - k])) small.erase(small.find(nums[i - k]));
                else if (large.count(nums[i - k])) large.erase(large.find(nums[i - k]));
            }
            if (small.size() <= large.size()) {
                if (large.empty() || nums[i] <= *large.begin()) small.insert(nums[i]);
                else {
                    small.insert(*large.begin());
                    large.erase(large.begin());
                    large.insert(nums[i]);
                }
            } else {
                if (nums[i] >= *small.rbegin()) large.insert(nums[i]);
                else {
                    large.insert(*small.rbegin());
                    small.erase(--small.end());
                    small.insert(nums[i]);
                }
            }
            if (i >= (k - 1)) {
                if (k % 2) res.push_back(*small.rbegin());
                else res.push_back(((double)*small.rbegin() + *large.begin()) / 2);
            }
        }
        return res;
    }
};

// version 2:
class Solution {
public:
    vector<double> medianSlidingWindow(vector<int>& nums, int k) {
        vector<double> res;
        multiset<double, greater<double>> small, large;

        for(int i = 0; i < nums.size(); i++) {
            if(small.size() + large.size() < k) {
                small.insert(nums[i]);
                large.insert(-*small.begin());
                small.erase(small.begin());
                if(small.size() < large.size()) {
                    small.insert(-*large.begin());
                    large.erase(large.begin());
                }
            }
            if(small.size() + large.size() == k) {
                if(k % 2 == 1) res.push_back(*small.begin());
                else res.push_back((*small.begin() - *large.begin())/2);
                if(small.count(nums[i-k+1])) small.erase(small.find(nums[i-k+1]));
                else large.erase(large.find(-nums[i-k+1]));
            }
        }

        return res;
    }
};

// Conlcusion:
// 把插入新值和删除元素分开。

<|endoftext|>


<commit_before>//= UnixAPIChecker.h - Checks preconditions for various Unix APIs --*- C++ -*-//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This defines UnixAPIChecker, which is an assortment of checks on calls
// to various, widely used UNIX/Posix functions.
//
//===----------------------------------------------------------------------===//

#include "GRExprEngineInternalChecks.h"
#include "clang/Checker/PathSensitive/CheckerVisitor.h"
#include "clang/Checker/BugReporter/BugType.h"
#include "llvm/ADT/StringSwitch.h"
#include <fcntl.h>

using namespace clang;

namespace {
class UnixAPIChecker : public CheckerVisitor<UnixAPIChecker> {
  enum SubChecks {
    OpenFn = 0,
    PthreadOnceFn = 1,
    NumChecks
  };

  BugType *BTypes[NumChecks];

public:
  UnixAPIChecker() { memset(BTypes, 0, sizeof(*BTypes) * NumChecks); }
  static void *getTag() { static unsigned tag = 0; return &tag; }

  void PreVisitCallExpr(CheckerContext &C, const CallExpr *CE);
};
} //end anonymous namespace

void clang::RegisterUnixAPIChecker(GRExprEngine &Eng) {
  Eng.registerCheck(new UnixAPIChecker());
}

//===----------------------------------------------------------------------===//
// Utility functions.
//===----------------------------------------------------------------------===//

static inline void LazyInitialize(BugType *&BT, const char *name) {
  if (BT)
    return;
  BT = new BugType(name, "Unix API");
}

//===----------------------------------------------------------------------===//
// "open" (man 2 open)
//===----------------------------------------------------------------------===//

static void CheckOpen(CheckerContext &C, const CallExpr *CE, BugType *&BT) {
  LazyInitialize(BT, "Improper use of 'open'");

  // Look at the 'oflags' argument for the O_CREAT flag.
  const GRState *state = C.getState();

  if (CE->getNumArgs() < 2) {
    // The frontend should issue a warning for this case, so this is a sanity
    // check.
    return;
  }

  // Now check if oflags has O_CREAT set.
  const Expr *oflagsEx = CE->getArg(1);
  const SVal V = state->getSVal(oflagsEx);
  if (!isa<NonLoc>(V)) {
    // The case where 'V' can be a location can only be due to a bad header,
    // so in this case bail out.
    return;
  }
  NonLoc oflags = cast<NonLoc>(V);
  NonLoc ocreateFlag =
    cast<NonLoc>(C.getValueManager().makeIntVal((uint64_t) O_CREAT,
                                                oflagsEx->getType()));
  SVal maskedFlagsUC = C.getSValuator().EvalBinOpNN(state, BinaryOperator::And,
                                                    oflags, ocreateFlag,
                                                    oflagsEx->getType());
  if (maskedFlagsUC.isUnknownOrUndef())
    return;
  DefinedSVal maskedFlags = cast<DefinedSVal>(maskedFlagsUC);

  // Check if maskedFlags is non-zero.
  const GRState *trueState, *falseState;
  llvm::tie(trueState, falseState) = state->Assume(maskedFlags);

  // Only emit an error if the value of 'maskedFlags' is properly
  // constrained;
  if (!(trueState && !falseState))
    return;

  if (CE->getNumArgs() < 3) {
    ExplodedNode *N = C.GenerateSink(trueState);
    if (!N)
      return;

    EnhancedBugReport *report =
      new EnhancedBugReport(*BT,
                            "Call to 'open' requires a third argument when "
                            "the 'O_CREAT' flag is set", N);
    report->addRange(oflagsEx->getSourceRange());
    C.EmitReport(report);
  }
}

//===----------------------------------------------------------------------===//
// pthread_once
//===----------------------------------------------------------------------===//

static void CheckPthreadOnce(CheckerContext &C, const CallExpr *CE,
                             BugType *&BT) {

  // This is similar to 'CheckDispatchOnce' in the MacOSXAPIChecker.
  // They can possibly be refactored.

  LazyInitialize(BT, "Improper use of 'pthread_once'");

  if (CE->getNumArgs() < 1)
    return;

  // Check if the first argument is stack allocated.  If so, issue a warning
  // because that's likely to be bad news.
  const GRState *state = C.getState();
  const MemRegion *R = state->getSVal(CE->getArg(0)).getAsRegion();
  if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
    return;

  ExplodedNode *N = C.GenerateSink(state);
  if (!N)
    return;

  llvm::SmallString<256> S;
  llvm::raw_svector_ostream os(S);
  os << "Call to 'pthread_once' uses";
  if (const VarRegion *VR = dyn_cast<VarRegion>(R))
    os << " the local variable '" << VR->getDecl()->getName() << '\'';
  else
    os << " stack allocated memory";
  os << " for the \"control\" value.  Using such transient memory for "
  "the control value is potentially dangerous.";
  if (isa<VarRegion>(R) && isa<StackLocalsSpaceRegion>(R->getMemorySpace()))
    os << "  Perhaps you intended to declare the variable as 'static'?";

  EnhancedBugReport *report = new EnhancedBugReport(*BT, os.str(), N);
  report->addRange(CE->getArg(0)->getSourceRange());
  C.EmitReport(report);
}

//===----------------------------------------------------------------------===//
// Central dispatch function.
//===----------------------------------------------------------------------===//

typedef void (*SubChecker)(CheckerContext &C, const CallExpr *CE, BugType *&BT);
namespace {
  class SubCheck {
    SubChecker SC;
    BugType **BT;
  public:
    SubCheck(SubChecker sc, BugType *& bt) : SC(sc), BT(&bt) {}
    SubCheck() : SC(NULL), BT(NULL) {}

    void run(CheckerContext &C, const CallExpr *CE) const {
      if (SC)
        SC(C, CE, *BT);
    }
  };
} // end anonymous namespace

void UnixAPIChecker::PreVisitCallExpr(CheckerContext &C, const CallExpr *CE) {
  // Get the callee.  All the functions we care about are C functions
  // with simple identifiers.
  const GRState *state = C.getState();
  const Expr *Callee = CE->getCallee();
  const FunctionTextRegion *Fn =
    dyn_cast_or_null<FunctionTextRegion>(state->getSVal(Callee).getAsRegion());

  if (!Fn)
    return;

  const IdentifierInfo *FI = Fn->getDecl()->getIdentifier();
  if (!FI)
    return;

  const SubCheck &SC =
    llvm::StringSwitch<SubCheck>(FI->getName())
      .Case("open", SubCheck(CheckOpen, BTypes[OpenFn]))
      .Case("pthread_once", SubCheck(CheckPthreadOnce, BTypes[PthreadOnceFn]))
      .Default(SubCheck());

  SC.run(C, CE);
}
<commit_msg>Temporarily only enable 'open' check on Mac OS X to unbreak Windows buildbot.  I'm looking into an alternate fix right now.<commit_after>//= UnixAPIChecker.h - Checks preconditions for various Unix APIs --*- C++ -*-//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This defines UnixAPIChecker, which is an assortment of checks on calls
// to various, widely used UNIX/Posix functions.
//
//===----------------------------------------------------------------------===//

#include "GRExprEngineInternalChecks.h"
#include "clang/Checker/PathSensitive/CheckerVisitor.h"
#include "clang/Checker/BugReporter/BugType.h"
#include "clang/Analysis/Support/Optional.h"
#include "llvm/ADT/StringSwitch.h"
#include <fcntl.h>

#include "clang/Basic/TargetInfo.h"

using namespace clang;

namespace {
class UnixAPIChecker : public CheckerVisitor<UnixAPIChecker> {
  enum SubChecks {
    OpenFn = 0,
    PthreadOnceFn = 1,
    NumChecks
  };

  BugType *BTypes[NumChecks];

public:
  UnixAPIChecker() { memset(BTypes, 0, sizeof(*BTypes) * NumChecks); }
  static void *getTag() { static unsigned tag = 0; return &tag; }

  void PreVisitCallExpr(CheckerContext &C, const CallExpr *CE);
};
} //end anonymous namespace

void clang::RegisterUnixAPIChecker(GRExprEngine &Eng) {
  Eng.registerCheck(new UnixAPIChecker());
}

//===----------------------------------------------------------------------===//
// Utility functions.
//===----------------------------------------------------------------------===//

static inline void LazyInitialize(BugType *&BT, const char *name) {
  if (BT)
    return;
  BT = new BugType(name, "Unix API");
}

//===----------------------------------------------------------------------===//
// "open" (man 2 open)
//===----------------------------------------------------------------------===//

static void CheckOpen(CheckerContext &C, const CallExpr *CE, BugType *&BT) {
  if (C.getASTContext().Target.getTriple().getVendor() != llvm::Triple::Apple)
    return;

  LazyInitialize(BT, "Improper use of 'open'");

  // Look at the 'oflags' argument for the O_CREAT flag.
  const GRState *state = C.getState();

  if (CE->getNumArgs() < 2) {
    // The frontend should issue a warning for this case, so this is a sanity
    // check.
    return;
  }

  // Now check if oflags has O_CREAT set.
  const Expr *oflagsEx = CE->getArg(1);
  const SVal V = state->getSVal(oflagsEx);
  if (!isa<NonLoc>(V)) {
    // The case where 'V' can be a location can only be due to a bad header,
    // so in this case bail out.
    return;
  }
  NonLoc oflags = cast<NonLoc>(V);
  NonLoc ocreateFlag =
    cast<NonLoc>(C.getValueManager().makeIntVal((uint64_t) O_CREAT,
                                                oflagsEx->getType()));
  SVal maskedFlagsUC = C.getSValuator().EvalBinOpNN(state, BinaryOperator::And,
                                                    oflags, ocreateFlag,
                                                    oflagsEx->getType());
  if (maskedFlagsUC.isUnknownOrUndef())
    return;
  DefinedSVal maskedFlags = cast<DefinedSVal>(maskedFlagsUC);

  // Check if maskedFlags is non-zero.
  const GRState *trueState, *falseState;
  llvm::tie(trueState, falseState) = state->Assume(maskedFlags);

  // Only emit an error if the value of 'maskedFlags' is properly
  // constrained;
  if (!(trueState && !falseState))
    return;

  if (CE->getNumArgs() < 3) {
    ExplodedNode *N = C.GenerateSink(trueState);
    if (!N)
      return;

    EnhancedBugReport *report =
      new EnhancedBugReport(*BT,
                            "Call to 'open' requires a third argument when "
                            "the 'O_CREAT' flag is set", N);
    report->addRange(oflagsEx->getSourceRange());
    C.EmitReport(report);
  }
}

//===----------------------------------------------------------------------===//
// pthread_once
//===----------------------------------------------------------------------===//

static void CheckPthreadOnce(CheckerContext &C, const CallExpr *CE,
                             BugType *&BT) {

  // This is similar to 'CheckDispatchOnce' in the MacOSXAPIChecker.
  // They can possibly be refactored.

  LazyInitialize(BT, "Improper use of 'pthread_once'");

  if (CE->getNumArgs() < 1)
    return;

  // Check if the first argument is stack allocated.  If so, issue a warning
  // because that's likely to be bad news.
  const GRState *state = C.getState();
  const MemRegion *R = state->getSVal(CE->getArg(0)).getAsRegion();
  if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
    return;

  ExplodedNode *N = C.GenerateSink(state);
  if (!N)
    return;

  llvm::SmallString<256> S;
  llvm::raw_svector_ostream os(S);
  os << "Call to 'pthread_once' uses";
  if (const VarRegion *VR = dyn_cast<VarRegion>(R))
    os << " the local variable '" << VR->getDecl()->getName() << '\'';
  else
    os << " stack allocated memory";
  os << " for the \"control\" value.  Using such transient memory for "
  "the control value is potentially dangerous.";
  if (isa<VarRegion>(R) && isa<StackLocalsSpaceRegion>(R->getMemorySpace()))
    os << "  Perhaps you intended to declare the variable as 'static'?";

  EnhancedBugReport *report = new EnhancedBugReport(*BT, os.str(), N);
  report->addRange(CE->getArg(0)->getSourceRange());
  C.EmitReport(report);
}

//===----------------------------------------------------------------------===//
// Central dispatch function.
//===----------------------------------------------------------------------===//

typedef void (*SubChecker)(CheckerContext &C, const CallExpr *CE, BugType *&BT);
namespace {
  class SubCheck {
    SubChecker SC;
    BugType **BT;
  public:
    SubCheck(SubChecker sc, BugType *& bt) : SC(sc), BT(&bt) {}
    SubCheck() : SC(NULL), BT(NULL) {}

    void run(CheckerContext &C, const CallExpr *CE) const {
      if (SC)
        SC(C, CE, *BT);
    }
  };
} // end anonymous namespace

void UnixAPIChecker::PreVisitCallExpr(CheckerContext &C, const CallExpr *CE) {
  // Get the callee.  All the functions we care about are C functions
  // with simple identifiers.
  const GRState *state = C.getState();
  const Expr *Callee = CE->getCallee();
  const FunctionTextRegion *Fn =
    dyn_cast_or_null<FunctionTextRegion>(state->getSVal(Callee).getAsRegion());

  if (!Fn)
    return;

  const IdentifierInfo *FI = Fn->getDecl()->getIdentifier();
  if (!FI)
    return;

  const SubCheck &SC =
    llvm::StringSwitch<SubCheck>(FI->getName())
      .Case("open", SubCheck(CheckOpen, BTypes[OpenFn]))
      .Case("pthread_once", SubCheck(CheckPthreadOnce, BTypes[PthreadOnceFn]))
      .Default(SubCheck());

  SC.run(C, CE);
}
<|endoftext|>


<commit_before>//===-- lib/CodeGen/ELFCodeEmitter.cpp ------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "elfce"

#include "ELF.h"
#include "ELFWriter.h"
#include "ELFCodeEmitter.h"
#include "llvm/Constants.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Function.h"
#include "llvm/CodeGen/BinaryObject.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRelocation.h"
#include "llvm/Target/TargetData.h"
#include "llvm/Target/TargetELFWriterInfo.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetAsmInfo.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"

//===----------------------------------------------------------------------===//
//                       ELFCodeEmitter Implementation
//===----------------------------------------------------------------------===//

namespace llvm {

/// startFunction - This callback is invoked when a new machine function is
/// about to be emitted.
void ELFCodeEmitter::startFunction(MachineFunction &MF) {
  DEBUG(errs() << "processing function: "
        << MF.getFunction()->getName() << "\n");

  // Get the ELF Section that this function belongs in.
  ES = &EW.getTextSection(MF.getFunction());

  // Set the desired binary object to be used by the code emitters
  setBinaryObject(ES);

  // Get the function alignment in bytes
  unsigned Align = (1 << MF.getAlignment());

  // The function must start on its required alignment
  ES->emitAlignment(Align);

  // Update the section alignment if needed.
  ES->Align = std::max(ES->Align, Align);

  // Record the function start offset
  FnStartOff = ES->getCurrentPCOffset();

  // Emit constant pool and jump tables to their appropriate sections.
  // They need to be emitted before the function because in some targets
  // the later may reference JT or CP entry address.
  emitConstantPool(MF.getConstantPool());
  emitJumpTables(MF.getJumpTableInfo());
}

/// finishFunction - This callback is invoked after the function is completely
/// finished.
bool ELFCodeEmitter::finishFunction(MachineFunction &MF) {
  // Add a symbol to represent the function.
  const Function *F = MF.getFunction();
  ELFSym *FnSym = ELFSym::getGV(F, EW.getGlobalELFBinding(F), ELFSym::STT_FUNC,
                                EW.getGlobalELFVisibility(F));
  FnSym->SectionIdx = ES->SectionIdx;
  FnSym->Size = ES->getCurrentPCOffset()-FnStartOff;
  EW.AddPendingGlobalSymbol(F, true);

  // Offset from start of Section
  FnSym->Value = FnStartOff;

  if (!F->hasPrivateLinkage())
    EW.SymbolList.push_back(FnSym);

  // Patch up Jump Table Section relocations to use the real MBBs offsets
  // now that the MBB label offsets inside the function are known.
  if (!MF.getJumpTableInfo()->isEmpty()) {
    ELFSection &JTSection = EW.getJumpTableSection();
    for (std::vector<MachineRelocation>::iterator MRI = JTRelocations.begin(),
         MRE = JTRelocations.end(); MRI != MRE; ++MRI) {
      MachineRelocation &MR = *MRI;
      unsigned MBBOffset = getMachineBasicBlockAddress(MR.getBasicBlock());
      MR.setResultPointer((void*)MBBOffset);
      MR.setConstantVal(ES->SectionIdx);
      JTSection.addRelocation(MR);
    }
  }

  // If we have emitted any relocations to function-specific objects such as
  // basic blocks, constant pools entries, or jump tables, record their
  // addresses now so that we can rewrite them with the correct addresses later
  for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
    MachineRelocation &MR = Relocations[i];
    intptr_t Addr;
    if (MR.isGlobalValue()) {
      EW.AddPendingGlobalSymbol(MR.getGlobalValue());
    } else if (MR.isExternalSymbol()) {
      EW.AddPendingExternalSymbol(MR.getExternalSymbol());
    } else if (MR.isBasicBlock()) {
      Addr = getMachineBasicBlockAddress(MR.getBasicBlock());
      MR.setConstantVal(ES->SectionIdx);
      MR.setResultPointer((void*)Addr);
    } else if (MR.isConstantPoolIndex()) {
      Addr = getConstantPoolEntryAddress(MR.getConstantPoolIndex());
      MR.setConstantVal(CPSections[MR.getConstantPoolIndex()]);
      MR.setResultPointer((void*)Addr);
    } else if (MR.isJumpTableIndex()) {
      ELFSection &JTSection = EW.getJumpTableSection();
      Addr = getJumpTableEntryAddress(MR.getJumpTableIndex());
      MR.setConstantVal(JTSection.SectionIdx);
      MR.setResultPointer((void*)Addr);
    } else {
      llvm_unreachable("Unhandled relocation type");
    }
    ES->addRelocation(MR);
  }

  // Clear per-function data structures.
  JTRelocations.clear();
  Relocations.clear();
  CPLocations.clear();
  CPSections.clear();
  JTLocations.clear();
  MBBLocations.clear();
  return false;
}

/// emitConstantPool - For each constant pool entry, figure out which section
/// the constant should live in and emit the constant
void ELFCodeEmitter::emitConstantPool(MachineConstantPool *MCP) {
  const std::vector<MachineConstantPoolEntry> &CP = MCP->getConstants();
  if (CP.empty()) return;

  // TODO: handle PIC codegen
  assert(TM.getRelocationModel() != Reloc::PIC_ &&
         "PIC codegen not yet handled for elf constant pools!");

  for (unsigned i = 0, e = CP.size(); i != e; ++i) {
    MachineConstantPoolEntry CPE = CP[i];

    // Record the constant pool location and the section index
    ELFSection &CstPool = EW.getConstantPoolSection(CPE);
    CPLocations.push_back(CstPool.size());
    CPSections.push_back(CstPool.SectionIdx);

    if (CPE.isMachineConstantPoolEntry())
      assert("CPE.isMachineConstantPoolEntry not supported yet");

    // Emit the constant to constant pool section
    EW.EmitGlobalConstant(CPE.Val.ConstVal, CstPool);
  }
}

/// emitJumpTables - Emit all the jump tables for a given jump table info
/// record to the appropriate section.
void ELFCodeEmitter::emitJumpTables(MachineJumpTableInfo *MJTI) {
  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
  if (JT.empty()) return;

  // FIXME: handle PIC codegen
  assert(TM.getRelocationModel() != Reloc::PIC_ &&
         "PIC codegen not yet handled for elf jump tables!");

  const TargetELFWriterInfo *TEW = TM.getELFWriterInfo();
  unsigned EntrySize = MJTI->getEntrySize();

  // Get the ELF Section to emit the jump table
  ELFSection &JTSection = EW.getJumpTableSection();

  // For each JT, record its offset from the start of the section
  for (unsigned i = 0, e = JT.size(); i != e; ++i) {
    const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;

    // Record JT 'i' offset in the JT section
    JTLocations.push_back(JTSection.size());

    // Each MBB entry in the Jump table section has a relocation entry
    // against the current text section.
    for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
      unsigned MachineRelTy = TEW->getAbsoluteLabelMachineRelTy();
      MachineRelocation MR =
        MachineRelocation::getBB(JTSection.size(), MachineRelTy, MBBs[mi]);

      // Add the relocation to the Jump Table section
      JTRelocations.push_back(MR);

      // Output placeholder for MBB in the JT section
      for (unsigned s=0; s < EntrySize; ++s)
        JTSection.emitByte(0);
    }
  }
}

} // end namespace llvm
<commit_msg><commit_after>//===-- lib/CodeGen/ELFCodeEmitter.cpp ------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "elfce"

#include "ELF.h"
#include "ELFWriter.h"
#include "ELFCodeEmitter.h"
#include "llvm/Constants.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Function.h"
#include "llvm/CodeGen/BinaryObject.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRelocation.h"
#include "llvm/Target/TargetData.h"
#include "llvm/Target/TargetELFWriterInfo.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetAsmInfo.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"

//===----------------------------------------------------------------------===//
//                       ELFCodeEmitter Implementation
//===----------------------------------------------------------------------===//

namespace llvm {

/// startFunction - This callback is invoked when a new machine function is
/// about to be emitted.
void ELFCodeEmitter::startFunction(MachineFunction &MF) {
  DEBUG(errs() << "processing function: "
        << MF.getFunction()->getName() << "\n");

  // Get the ELF Section that this function belongs in.
  ES = &EW.getTextSection(MF.getFunction());

  // Set the desired binary object to be used by the code emitters
  setBinaryObject(ES);

  // Get the function alignment in bytes
  unsigned Align = (1 << MF.getAlignment());

  // The function must start on its required alignment
  ES->emitAlignment(Align);

  // Update the section alignment if needed.
  ES->Align = std::max(ES->Align, Align);

  // Record the function start offset
  FnStartOff = ES->getCurrentPCOffset();

  // Emit constant pool and jump tables to their appropriate sections.
  // They need to be emitted before the function because in some targets
  // the later may reference JT or CP entry address.
  emitConstantPool(MF.getConstantPool());
  emitJumpTables(MF.getJumpTableInfo());
}

/// finishFunction - This callback is invoked after the function is completely
/// finished.
bool ELFCodeEmitter::finishFunction(MachineFunction &MF) {
  // Add a symbol to represent the function.
  const Function *F = MF.getFunction();
  ELFSym *FnSym = ELFSym::getGV(F, EW.getGlobalELFBinding(F), ELFSym::STT_FUNC,
                                EW.getGlobalELFVisibility(F));
  FnSym->SectionIdx = ES->SectionIdx;
  FnSym->Size = ES->getCurrentPCOffset()-FnStartOff;
  EW.AddPendingGlobalSymbol(F, true);

  // Offset from start of Section
  FnSym->Value = FnStartOff;

  if (!F->hasPrivateLinkage())
    EW.SymbolList.push_back(FnSym);

  // Patch up Jump Table Section relocations to use the real MBBs offsets
  // now that the MBB label offsets inside the function are known.
  if (!MF.getJumpTableInfo()->isEmpty()) {
    ELFSection &JTSection = EW.getJumpTableSection();
    for (std::vector<MachineRelocation>::iterator MRI = JTRelocations.begin(),
         MRE = JTRelocations.end(); MRI != MRE; ++MRI) {
      MachineRelocation &MR = *MRI;
      unsigned MBBOffset = getMachineBasicBlockAddress(MR.getBasicBlock());
      MR.setResultPointer((void*)MBBOffset);
      MR.setConstantVal(ES->SectionIdx);
      JTSection.addRelocation(MR);
    }
  }

  // If we have emitted any relocations to function-specific objects such as
  // basic blocks, constant pools entries, or jump tables, record their
  // addresses now so that we can rewrite them with the correct addresses later
  for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
    MachineRelocation &MR = Relocations[i];
    intptr_t Addr;
    if (MR.isGlobalValue()) {
      EW.AddPendingGlobalSymbol(MR.getGlobalValue());
    } else if (MR.isExternalSymbol()) {
      EW.AddPendingExternalSymbol(MR.getExternalSymbol());
    } else if (MR.isBasicBlock()) {
      Addr = getMachineBasicBlockAddress(MR.getBasicBlock());
      MR.setConstantVal(ES->SectionIdx);
      MR.setResultPointer((void*)Addr);
    } else if (MR.isConstantPoolIndex()) {
      Addr = getConstantPoolEntryAddress(MR.getConstantPoolIndex());
      MR.setConstantVal(CPSections[MR.getConstantPoolIndex()]);
      MR.setResultPointer((void*)Addr);
    } else if (MR.isJumpTableIndex()) {
      ELFSection &JTSection = EW.getJumpTableSection();
      Addr = getJumpTableEntryAddress(MR.getJumpTableIndex());
      MR.setConstantVal(JTSection.SectionIdx);
      MR.setResultPointer((void*)Addr);
    } else {
      llvm_unreachable("Unhandled relocation type");
    }
    ES->addRelocation(MR);
  }

  // Clear per-function data structures.
  JTRelocations.clear();
  Relocations.clear();
  CPLocations.clear();
  CPSections.clear();
  JTLocations.clear();
  MBBLocations.clear();
  return false;
}

/// emitConstantPool - For each constant pool entry, figure out which section
/// the constant should live in and emit the constant
void ELFCodeEmitter::emitConstantPool(MachineConstantPool *MCP) {
  const std::vector<MachineConstantPoolEntry> &CP = MCP->getConstants();
  if (CP.empty()) return;

  // TODO: handle PIC codegen
  assert(TM.getRelocationModel() != Reloc::PIC_ &&
         "PIC codegen not yet handled for elf constant pools!");

  for (unsigned i = 0, e = CP.size(); i != e; ++i) {
    MachineConstantPoolEntry CPE = CP[i];

    // Record the constant pool location and the section index
    ELFSection &CstPool = EW.getConstantPoolSection(CPE);
    CPLocations.push_back(CstPool.size());
    CPSections.push_back(CstPool.SectionIdx);

    if (CPE.isMachineConstantPoolEntry())
      assert("CPE.isMachineConstantPoolEntry not supported yet");

    // Emit the constant to constant pool section
    EW.EmitGlobalConstant(CPE.Val.ConstVal, CstPool);
  }
}

/// emitJumpTables - Emit all the jump tables for a given jump table info
/// record to the appropriate section.
void ELFCodeEmitter::emitJumpTables(MachineJumpTableInfo *MJTI) {
  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
  if (JT.empty()) return;

  // FIXME: handle PIC codegen
  assert(TM.getRelocationModel() != Reloc::PIC_ &&
         "PIC codegen not yet handled for elf jump tables!");

  const TargetELFWriterInfo *TEW = TM.getELFWriterInfo();
  unsigned EntrySize = MJTI->getEntrySize();

  // Get the ELF Section to emit the jump table
  ELFSection &JTSection = EW.getJumpTableSection();

  // For each JT, record its offset from the start of the section
  for (unsigned i = 0, e = JT.size(); i != e; ++i) {
    const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;

    // Record JT 'i' offset in the JT section
    JTLocations.push_back(JTSection.size());

    // Each MBB entry in the Jump table section has a relocation entry
    // against the current text section.
    for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
      unsigned MachineRelTy = TEW->getAbsoluteLabelMachineRelTy();
      MachineRelocation MR =
        MachineRelocation::getBB(JTSection.size(), MachineRelTy, MBBs[mi]);

      // Add the relocation to the Jump Table section
      JTRelocations.push_back(MR);

      // Output placeholder for MBB in the JT section
      for (unsigned s=0; s < EntrySize; ++s)
        JTSection.emitByte(0);
    }
  }
}

} // end namespace llvm
<|endoftext|>