<commit_before>#include <fstream>
#include <iostream>
#include <omp.h>
#include <stdio.h>
#include <time.h>

#include "../Neural_Networks.h"

void Read_MNIST(string training_set_images, string training_set_labels, string test_set_images, string test_set_labels, int number_training, int number_test, float **input, float **target_output) {
	ifstream file(training_set_images, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 4; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = 0; h < number_training; h++) {
			unsigned char pixel;

			for (int j = 0; j < 28 * 28; j++) {
				file.read((char*)(&pixel), 1);
				input[h][j] = pixel / 255.0;
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + training_set_images + " not found" << endl;
	}

	file.open(training_set_labels, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 2; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = 0; h < number_training; h++) {
			unsigned char label;

			file.read((char*)(&label), 1);

			for (int j = 0; j < 10; j++) {
				target_output[h][j] = (j == label);
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + training_set_labels + " not found" << endl;
	}

	file.open(test_set_images, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 4; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = number_training; h < number_training + number_test; h++) {
			unsigned char pixel;

			for (int j = 0; j < 28 * 28; j++) {
				file.read((char*)(&pixel), 1);
				input[h][j] = pixel / 255.0;
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + test_set_images + " not found" << endl;
	}

	file.open(test_set_labels, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 2; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = number_training; h < number_training + number_test; h++) {
			unsigned char label;

			file.read((char*)(&label), 1);

			for (int j = 0; j < 10; j++) {
				target_output[h][j] = (j == label);
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + test_set_labels + " not found" << endl;
	}
}

int main() {
	int batch_size		= 60;
	int number_iterations = 50;
	int number_threads;
	int number_training	= 60000;
	int number_test		= 10000;

	float **input = new float*[number_training + number_test];
	float **target_output = new float*[number_training + number_test];

	double epsilon		 = 0.001;
	double learning_rate = 0.005;
	double decay_rate	 = 0.977;

	string path;

	vector<int> number_nodes = { 784, 10 };

	vector<Layer*> layer;

	// train from scratch *****************
	Neural_Networks NN = Neural_Networks();

	layer.push_back(NN.Add(new Layer("MNIST",	1, 28, 28)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 24, 24, 24)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("",		24, 12, 12)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 24, 8, 8)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("BN,ReLU", 48, 8, 8)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("",		48, 4, 4)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 192)));			// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("CE,softmax", 10)));		// cross-entropy loss, softmax activation

	layer[1]->Connect(layer[0], "W");		// 5x5 convolution
	layer[2]->Connect(layer[1], "P,max");	// 2x2 max pooling
	layer[3]->Connect(layer[2], "W,DS");	// 5x5 depthwise separable convolution
	layer[4]->Connect(layer[3], "W");		// 1x1 convolution
	layer[5]->Connect(layer[4], "P,max");	// 2x2 max pooling
	layer[6]->Connect(layer[5], "W");		// fully connected
	layer[7]->Connect(layer[6], "W");		// fully connected

	srand(0); NN.Initialize(0.01);
	// ************************************

	// or load pretrained model
	// Neural_Networks NN = Neural_Networks("MNIST_CNN.txt");

	cout << "The number of threads : ";
	cin >> number_threads;
	cin.ignore();

	cout << "path where MNIST handwritten digits dataset is : ";
	getline(cin, path);

	for (int h = 0; h < number_training + number_test; h++) {
		input[h] = new float[number_nodes.front()];
		target_output[h] = new float[number_nodes.back()];
	}
	Read_MNIST(path + "train-images.idx3-ubyte", path + "train-labels.idx1-ubyte", path + "t10k-images.idx3-ubyte", path + "t10k-labels.idx1-ubyte", number_training, number_test, input, target_output);

	omp_set_num_threads(number_threads);

	for (int g = 0, time = clock(); g < number_iterations; g++) {
		int score[2] = { 0, };

		float **output = new float*[batch_size];

		double loss = NN.Train(batch_size, number_training, input, target_output, learning_rate, epsilon);

		for (int h = 0; h < batch_size; h++) {
			output[h] = new float[number_nodes.back()];
		}
		for (int i = 0, h = 0; i < number_training + number_test; i++) {
			if (++h == batch_size || i == number_training + number_test - 1) {
				NN.Test(h, &input[i - h + 1], output);

				for (int g = 0, argmax; g < h; g++) {
					double max = 0;

					for (int j = 0; j < number_nodes.back(); j++) {
						if (max < output[g][j]) {
							max = output[g][argmax = j];
						}
					}
					score[(i - h + g + 1 < number_training) ? (0) : (1)] += (int)target_output[i - h + g + 1][argmax];
				}
				h = 0;
			}
		}
		printf("."); NN.Save("NN.txt");

		printf("score: %d / %d, %d / %d  loss: %lf  step %d  %.2lf sec\n", score[0], number_training, score[1], number_test, loss, g + 1, (double)(clock() - time) / CLOCKS_PER_SEC);
		learning_rate *= decay_rate;

		for (int h = 0; h < batch_size; h++) {
			delete[] output[h];
		}
		delete[] output;
	}

	for (int h = 0; h < number_training + number_test; h++) {
		delete[] input[h];
		delete[] target_output[h];
	}
	delete[] input;
	delete[] target_output;

	return 0;
}
<commit_msg>Update MNIST_CNN.cpp<commit_after>#include <fstream>
#include <iostream>
#include <omp.h>
#include <stdio.h>
#include <time.h>

#include "../Neural_Networks.h"

void Read_MNIST(string training_set_images, string training_set_labels, string test_set_images, string test_set_labels, int number_training, int number_test, float **input, float **target_output) {
	ifstream file(training_set_images, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 4; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = 0; h < number_training; h++) {
			unsigned char pixel;

			for (int j = 0; j < 28 * 28; j++) {
				file.read((char*)(&pixel), 1);
				input[h][j] = pixel / 255.0;
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + training_set_images + " not found" << endl;
	}

	file.open(training_set_labels, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 2; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = 0; h < number_training; h++) {
			unsigned char label;

			file.read((char*)(&label), 1);

			for (int j = 0; j < 10; j++) {
				target_output[h][j] = (j == label);
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + training_set_labels + " not found" << endl;
	}

	file.open(test_set_images, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 4; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = number_training; h < number_training + number_test; h++) {
			unsigned char pixel;

			for (int j = 0; j < 28 * 28; j++) {
				file.read((char*)(&pixel), 1);
				input[h][j] = pixel / 255.0;
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + test_set_images + " not found" << endl;
	}

	file.open(test_set_labels, ifstream::binary);

	if (file.is_open()) {
		for (int h = 0, value; h < 2; h++) {
			file.read((char*)(&value), sizeof(int));
		}
		for (int h = number_training; h < number_training + number_test; h++) {
			unsigned char label;

			file.read((char*)(&label), 1);

			for (int j = 0; j < 10; j++) {
				target_output[h][j] = (j == label);
			}
		}
		file.close();
	}
	else {
		cerr << "[Read_MNIST], " + test_set_labels + " not found" << endl;
	}
}

int main() {
	int batch_size		= 60;
	int number_iterations = 50;
	int number_threads;
	int number_training	= 60000;
	int number_test		= 10000;

	float **input = new float*[number_training + number_test];
	float **target_output = new float*[number_training + number_test];

	double epsilon		 = 0.001;
	double learning_rate = 0.005;
	double decay_rate	 = 0.977;

	string path;

	vector<int> number_nodes = { 784, 10 };

	vector<Layer*> layer;

	// train from scratch *****************
	Neural_Networks NN = Neural_Networks();

	layer.push_back(NN.Add(new Layer("MNIST",	1, 28, 28)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 24, 24, 24)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("",		24, 12, 12)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 24, 8, 8)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("BN,ReLU", 48, 8, 8)));	// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("",		48, 4, 4)));
	layer.push_back(NN.Add(new Layer("BN,ReLU", 192)));			// batch normalization, ReLU activation
	layer.push_back(NN.Add(new Layer("CE,softmax", 10)));		// cross-entropy loss, softmax activation

	layer[1]->Connect(layer[0], "W");		// 5x5 convolution
	layer[2]->Connect(layer[1], "P,max");	// 2x2 max pooling
	layer[3]->Connect(layer[2], "W,DS");	// 5x5 depthwise separable convolution
	layer[4]->Connect(layer[3], "W");		// 1x1 convolution
	layer[5]->Connect(layer[4], "P,max");	// 2x2 max pooling
	layer[6]->Connect(layer[5], "W");		// fully connected
	layer[7]->Connect(layer[6], "W");		// fully connected

	srand(0); NN.Initialize(0.01);
	// ************************************

	// or load pretrained model
	// Neural_Networks NN = Neural_Networks("MNIST_CNN.txt");

	cout << "The number of threads : ";
	cin >> number_threads;
	cin.ignore();

	omp_set_num_threads(number_threads);

	cout << "path where MNIST handwritten digits dataset is : ";
	getline(cin, path);

	for (int h = 0; h < number_training + number_test; h++) {
		input[h] = new float[number_nodes.front()];
		target_output[h] = new float[number_nodes.back()];
	}
	Read_MNIST(path + "train-images.idx3-ubyte", path + "train-labels.idx1-ubyte", path + "t10k-images.idx3-ubyte", path + "t10k-labels.idx1-ubyte", number_training, number_test, input, target_output);

	for (int g = 0, time = clock(); g < number_iterations; g++) {
		int score[2] = { 0, };

		float **output = new float*[batch_size];

		double loss = NN.Train(batch_size, number_training, input, target_output, learning_rate, epsilon);

		for (int h = 0; h < batch_size; h++) {
			output[h] = new float[number_nodes.back()];
		}
		for (int i = 0, h = 0; i < number_training + number_test; i++) {
			if (++h == batch_size || i == number_training + number_test - 1) {
				NN.Test(h, &input[i - h + 1], output);

				for (int g = 0, argmax; g < h; g++) {
					double max = 0;

					for (int j = 0; j < number_nodes.back(); j++) {
						if (max < output[g][j]) {
							max = output[g][argmax = j];
						}
					}
					score[(i - h + g + 1 < number_training) ? (0) : (1)] += (int)target_output[i - h + g + 1][argmax];
				}
				h = 0;
			}
		}
		printf("."); NN.Save("NN.txt");

		printf("score: %d / %d, %d / %d  loss: %lf  step %d  %.2lf sec\n", score[0], number_training, score[1], number_test, loss, g + 1, (double)(clock() - time) / CLOCKS_PER_SEC);
		learning_rate *= decay_rate;

		for (int h = 0; h < batch_size; h++) {
			delete[] output[h];
		}
		delete[] output;
	}

	for (int h = 0; h < number_training + number_test; h++) {
		delete[] input[h];
		delete[] target_output[h];
	}
	delete[] input;
	delete[] target_output;

	return 0;
}
<|endoftext|>


<commit_before>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#include "fstream.hh"
#include "align.hh"
#include "circular_buffer.hh"
#include "semaphore.hh"
#include "reactor.hh"
#include <malloc.h>
#include <string.h>

class file_data_source_impl : public data_source_impl {
    file _file;
    file_input_stream_options _options;
    uint64_t _pos;
    circular_buffer<future<temporary_buffer<char>>> _read_buffers;
    unsigned _reads_in_progress = 0;
    std::experimental::optional<promise<>> _done;
public:
    file_data_source_impl(file f, file_input_stream_options options)
            : _file(std::move(f)), _options(options), _pos(_options.offset) {}
    virtual future<temporary_buffer<char>> get() override {
        if (_read_buffers.empty()) {
            issue_read_aheads(1);
        }
        auto ret = std::move(_read_buffers.front());
        _read_buffers.pop_front();
        return ret;
    }
    virtual future<> close() {
        _done.emplace();
        if (!_reads_in_progress) {
            _done->set_value();
        }
        return _done->get_future().then([this] {
            for (auto&& c : _read_buffers) {
                c.ignore_ready_future();
            }
        });
    }
private:
    void issue_read_aheads(unsigned min_ra = 0) {
        if (_done) {
            return;
        }
        auto ra = std::max(min_ra, _options.read_ahead);
        while (_read_buffers.size() < ra) {
            ++_reads_in_progress;
            // if _pos is not dma-aligned, we'll get a short read.  Account for that.
            auto now = _options.buffer_size - _pos % _file.disk_dma_alignment();
            _read_buffers.push_back(_file.dma_read_bulk<char>(_pos, now).then_wrapped(
                    [this] (future<temporary_buffer<char>> ret) {
                issue_read_aheads();
                --_reads_in_progress;
                if (_done && !_reads_in_progress) {
                    _done->set_value();
                }
                return ret;
            }));
            _pos += now;
        };
    }
};

class file_data_source : public data_source {
public:
    file_data_source(file f, file_input_stream_options options)
        : data_source(std::make_unique<file_data_source_impl>(
                std::move(f), options)) {}
};

input_stream<char> make_file_input_stream(
        file f, file_input_stream_options options) {
    return input_stream<char>(file_data_source(std::move(f), options));
}

input_stream<char> make_file_input_stream(
        file f, uint64_t offset, size_t buffer_size) {
    file_input_stream_options options;
    options.offset = offset;
    options.buffer_size = buffer_size;
    return make_file_input_stream(std::move(f), options);
}

class file_data_sink_impl : public data_sink_impl {
    file _file;
    file_output_stream_options _options;
    uint64_t _pos = 0;
    // At offsets < _preallocating_at, preallocation was completed.
    // At offsets >= _preallocating_at, we may be preallocating
    // disk space, so all writes must wait until _preallocation_done[i]
    // becomes ready (in _options.preallocation_size units)
    // At offsets >= _preallocating_at + _options.preallocation_size *
    //               _preallocation_done.size(), we have not yet started.
    uint64_t _preallocating_at = 0;
    // Each promise vector lists writes that are waiting for a single preallocation
    circular_buffer<std::vector<promise<>>> _preallocation_done;
    semaphore _write_behind_sem = { _options.write_behind };
    future<> _background_writes_done = make_ready_future<>();
    bool _failed = false;
public:
    file_data_sink_impl(file f, file_output_stream_options options)
            : _file(std::move(f)), _options(options) {}
    future<> put(net::packet data) { abort(); }
    virtual temporary_buffer<char> allocate_buffer(size_t size) override {
        return temporary_buffer<char>::aligned(_file.memory_dma_alignment(), size);
    }
    virtual future<> put(temporary_buffer<char> buf) override {
        uint64_t pos = _pos;
        _pos += buf.size();
        if (!_options.write_behind) {
            return do_put(pos, std::move(buf));
        }
        // Write behind strategy:
        //
        // 1. Issue N writes in parallel, using a semphore to limit to N
        // 2. Collect results in _background_writes_done, merging exception futures
        // 3. If we've already seen a failure, don't issue more writes.
        return _write_behind_sem.wait().then([this, pos, buf = std::move(buf)] () mutable {
            if (_failed) {
                _write_behind_sem.signal();
                auto ret = std::move(_background_writes_done);
                _background_writes_done = make_ready_future<>();
                return ret;
            }
            auto this_write_done = do_put(pos, std::move(buf)).finally([this] {
                _write_behind_sem.signal();
            });
            _background_writes_done = when_all(std::move(_background_writes_done), std::move(this_write_done))
                    .then([this] (std::tuple<future<>, future<>> possible_errors) {
                // merge the two errors, preferring the first
                auto& e1 = std::get<0>(possible_errors);
                auto& e2 = std::get<1>(possible_errors);
                if (e1.failed()) {
                    e2.ignore_ready_future();
                    return std::move(e1);
                } else {
                    if (e2.failed()) {
                        _failed = true;
                    }
                    return std::move(e2);
                }
            });
            return make_ready_future<>();
        });
    }
private:
    virtual future<> do_put(uint64_t pos, temporary_buffer<char> buf) {
        // put() must usually be of chunks multiple of file::dma_alignment.
        // Only the last part can have an unaligned length. If put() was
        // called again with an unaligned pos, we have a bug in the caller.
        assert(!(pos & (_file.disk_dma_alignment() - 1)));
        bool truncate = false;
        auto p = static_cast<const char*>(buf.get());
        size_t buf_size = buf.size();

        if ((buf.size() & (_file.disk_dma_alignment() - 1)) != 0) {
            // If buf size isn't aligned, copy its content into a new aligned buf.
            // This should only happen when the user calls output_stream::flush().
            auto tmp = allocate_buffer(align_up(buf.size(), _file.disk_dma_alignment()));
            ::memcpy(tmp.get_write(), buf.get(), buf.size());
            buf = std::move(tmp);
            p = buf.get();
            buf_size = buf.size();
            truncate = true;
        }
        auto prealloc = preallocate(pos, buf.size());
        return prealloc.then([this, pos, p, buf_size, truncate, buf = std::move(buf)] () mutable {
            return _file.dma_write(pos, p, buf_size).then(
                    [this, buf = std::move(buf), truncate] (size_t size) {
                if (truncate) {
                    return _file.truncate(_pos);
                }
                return make_ready_future<>();
            });
        });
    }
    future<> preallocate(uint64_t pos, uint64_t size) {
        auto ret = make_ready_future<>();
    restart:
        if (pos + size <= _preallocating_at) {
            return ret;
        }
        auto skip = std::max(_preallocating_at, pos) - pos;
        pos += skip;
        size -= skip;
        size_t idx = (pos - _preallocating_at) / _options.preallocation_size;
        while (size) {
            if (idx == _preallocation_done.size()) {
                start_new_preallocation();
                // May have caused _preallocating_at to change, so redo the loop
                goto restart;
            }
            _preallocation_done[idx].emplace_back();
            auto this_prealloc_done = _preallocation_done[idx].back().get_future();
            ret = when_all(std::move(ret), std::move(this_prealloc_done)).discard_result();
            skip = std::min<uint64_t>(size, _options.preallocation_size);
            pos += skip;
            size -= skip;
            ++idx;
        }
        return ret;
    }
    void start_new_preallocation() {
        auto pos = _preallocating_at + _preallocation_done.size() * _options.preallocation_size;
        _preallocation_done.emplace_back();
        _file.allocate(pos, _options.preallocation_size).then_wrapped([this, pos] (future<> ret) {
            complete_preallocation(pos, std::move(ret));
        });
    }
    void complete_preallocation(uint64_t pos, future<> result) {
        // Preallocation may have failed.  But it's just preallocation,
        // so we can ignore the result.
        result.ignore_ready_future();
        size_t idx = (pos - _preallocating_at) / _options.preallocation_size;
        for (auto&& pr : _preallocation_done[idx]) {
            pr.set_value();
        }
        _preallocation_done[idx].clear();
        while (!_preallocation_done.empty() && _preallocation_done.front().empty()) {
            _preallocation_done.pop_front();
            _preallocating_at += _options.preallocation_size;
        }
    }
    future<> wait() {
        return _write_behind_sem.wait(_options.write_behind).then([this] {
            return _background_writes_done.then([this] {
                // restore to pristine state; for flush() + close() sequence
                // (we allow either flush, or close, or both)
                _write_behind_sem.signal(_options.write_behind);
                _background_writes_done = make_ready_future<>();
            });
        });
    }
public:
    virtual future<> flush() override {
        return wait().then([this] {
            return _file.flush();
        });
    }
    virtual future<> close() {
        return wait().then([this] {
            return _file.close();
        });
    }
};

class file_data_sink : public data_sink {
public:
    file_data_sink(file f, file_output_stream_options options)
        : data_sink(std::make_unique<file_data_sink_impl>(
                std::move(f), options)) {}
};

output_stream<char> make_file_output_stream(file f, size_t buffer_size) {
    file_output_stream_options options;
    options.buffer_size = buffer_size;
    return make_file_output_stream(std::move(f), options);
}

output_stream<char> make_file_output_stream(file f, file_output_stream_options options) {
    return output_stream<char>(file_data_sink(std::move(f), options), options.buffer_size, true);
}

<commit_msg>fstream: remove stream-based preallocation<commit_after>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#include "fstream.hh"
#include "align.hh"
#include "circular_buffer.hh"
#include "semaphore.hh"
#include "reactor.hh"
#include <malloc.h>
#include <string.h>

class file_data_source_impl : public data_source_impl {
    file _file;
    file_input_stream_options _options;
    uint64_t _pos;
    circular_buffer<future<temporary_buffer<char>>> _read_buffers;
    unsigned _reads_in_progress = 0;
    std::experimental::optional<promise<>> _done;
public:
    file_data_source_impl(file f, file_input_stream_options options)
            : _file(std::move(f)), _options(options), _pos(_options.offset) {}
    virtual future<temporary_buffer<char>> get() override {
        if (_read_buffers.empty()) {
            issue_read_aheads(1);
        }
        auto ret = std::move(_read_buffers.front());
        _read_buffers.pop_front();
        return ret;
    }
    virtual future<> close() {
        _done.emplace();
        if (!_reads_in_progress) {
            _done->set_value();
        }
        return _done->get_future().then([this] {
            for (auto&& c : _read_buffers) {
                c.ignore_ready_future();
            }
        });
    }
private:
    void issue_read_aheads(unsigned min_ra = 0) {
        if (_done) {
            return;
        }
        auto ra = std::max(min_ra, _options.read_ahead);
        while (_read_buffers.size() < ra) {
            ++_reads_in_progress;
            // if _pos is not dma-aligned, we'll get a short read.  Account for that.
            auto now = _options.buffer_size - _pos % _file.disk_dma_alignment();
            _read_buffers.push_back(_file.dma_read_bulk<char>(_pos, now).then_wrapped(
                    [this] (future<temporary_buffer<char>> ret) {
                issue_read_aheads();
                --_reads_in_progress;
                if (_done && !_reads_in_progress) {
                    _done->set_value();
                }
                return ret;
            }));
            _pos += now;
        };
    }
};

class file_data_source : public data_source {
public:
    file_data_source(file f, file_input_stream_options options)
        : data_source(std::make_unique<file_data_source_impl>(
                std::move(f), options)) {}
};

input_stream<char> make_file_input_stream(
        file f, file_input_stream_options options) {
    return input_stream<char>(file_data_source(std::move(f), options));
}

input_stream<char> make_file_input_stream(
        file f, uint64_t offset, size_t buffer_size) {
    file_input_stream_options options;
    options.offset = offset;
    options.buffer_size = buffer_size;
    return make_file_input_stream(std::move(f), options);
}

class file_data_sink_impl : public data_sink_impl {
    file _file;
    file_output_stream_options _options;
    uint64_t _pos = 0;
    semaphore _write_behind_sem = { _options.write_behind };
    future<> _background_writes_done = make_ready_future<>();
    bool _failed = false;
public:
    file_data_sink_impl(file f, file_output_stream_options options)
            : _file(std::move(f)), _options(options) {}
    future<> put(net::packet data) { abort(); }
    virtual temporary_buffer<char> allocate_buffer(size_t size) override {
        return temporary_buffer<char>::aligned(_file.memory_dma_alignment(), size);
    }
    virtual future<> put(temporary_buffer<char> buf) override {
        uint64_t pos = _pos;
        _pos += buf.size();
        if (!_options.write_behind) {
            return do_put(pos, std::move(buf));
        }
        // Write behind strategy:
        //
        // 1. Issue N writes in parallel, using a semphore to limit to N
        // 2. Collect results in _background_writes_done, merging exception futures
        // 3. If we've already seen a failure, don't issue more writes.
        return _write_behind_sem.wait().then([this, pos, buf = std::move(buf)] () mutable {
            if (_failed) {
                _write_behind_sem.signal();
                auto ret = std::move(_background_writes_done);
                _background_writes_done = make_ready_future<>();
                return ret;
            }
            auto this_write_done = do_put(pos, std::move(buf)).finally([this] {
                _write_behind_sem.signal();
            });
            _background_writes_done = when_all(std::move(_background_writes_done), std::move(this_write_done))
                    .then([this] (std::tuple<future<>, future<>> possible_errors) {
                // merge the two errors, preferring the first
                auto& e1 = std::get<0>(possible_errors);
                auto& e2 = std::get<1>(possible_errors);
                if (e1.failed()) {
                    e2.ignore_ready_future();
                    return std::move(e1);
                } else {
                    if (e2.failed()) {
                        _failed = true;
                    }
                    return std::move(e2);
                }
            });
            return make_ready_future<>();
        });
    }
private:
    virtual future<> do_put(uint64_t pos, temporary_buffer<char> buf) {
        // put() must usually be of chunks multiple of file::dma_alignment.
        // Only the last part can have an unaligned length. If put() was
        // called again with an unaligned pos, we have a bug in the caller.
        assert(!(pos & (_file.disk_dma_alignment() - 1)));
        bool truncate = false;
        auto p = static_cast<const char*>(buf.get());
        size_t buf_size = buf.size();

        if ((buf.size() & (_file.disk_dma_alignment() - 1)) != 0) {
            // If buf size isn't aligned, copy its content into a new aligned buf.
            // This should only happen when the user calls output_stream::flush().
            auto tmp = allocate_buffer(align_up(buf.size(), _file.disk_dma_alignment()));
            ::memcpy(tmp.get_write(), buf.get(), buf.size());
            buf = std::move(tmp);
            p = buf.get();
            buf_size = buf.size();
            truncate = true;
        }

        {
            return _file.dma_write(pos, p, buf_size).then(
                    [this, buf = std::move(buf), truncate] (size_t size) {
                if (truncate) {
                    return _file.truncate(_pos);
                }
                return make_ready_future<>();
            });
        }
    }
    future<> wait() {
        return _write_behind_sem.wait(_options.write_behind).then([this] {
            return _background_writes_done.then([this] {
                // restore to pristine state; for flush() + close() sequence
                // (we allow either flush, or close, or both)
                _write_behind_sem.signal(_options.write_behind);
                _background_writes_done = make_ready_future<>();
            });
        });
    }
public:
    virtual future<> flush() override {
        return wait().then([this] {
            return _file.flush();
        });
    }
    virtual future<> close() {
        return wait().then([this] {
            return _file.close();
        });
    }
};

class file_data_sink : public data_sink {
public:
    file_data_sink(file f, file_output_stream_options options)
        : data_sink(std::make_unique<file_data_sink_impl>(
                std::move(f), options)) {}
};

output_stream<char> make_file_output_stream(file f, size_t buffer_size) {
    file_output_stream_options options;
    options.buffer_size = buffer_size;
    return make_file_output_stream(std::move(f), options);
}

output_stream<char> make_file_output_stream(file f, file_output_stream_options options) {
    return output_stream<char>(file_data_sink(std::move(f), options), options.buffer_size, true);
}

<|endoftext|>


<commit_before>/*
**  Copyright (C) - Triton
**
**  This program is under the terms of the LGPLv3 License.
*/

#include <csignal>
#include <iostream>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <vector>

#include <pin.H>

#include <AnalysisProcessor.h>
#include <IRBuilder.h>
#include <IRBuilderFactory.h>
#include <Inst.h>
#include <PINContextHandler.h>
#include <ProcessingPyConf.h>
#include <Trigger.h>
#include <boost/filesystem.hpp>

using boost::filesystem::absolute;
using boost::filesystem::path;

/* Pin options: -script */
KNOB<std::string>   KnobPythonModule(KNOB_MODE_WRITEONCE, "pintool", "script", "", "Python script");

AnalysisProcessor   ap;
Trigger             analysisTrigger = Trigger();
ProcessingPyConf    processingPyConf(&ap, &analysisTrigger);

static void callbackBefore(IRBuilder *irb, CONTEXT *ctx, BOOL hasEA, ADDRINT ea, BOOL isBranchTaken, ADDRINT branchTargetAddress, THREADID threadId) {
  /* Some configurations must be applied before processing */
  processingPyConf.applyConfBeforeProcessing(irb);

  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  if (hasEA)
    irb->setup(ea);

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Setup Information into Irb */
  irb->setThreadID(ap.getThreadID());
  irb->setBranchTaken(isBranchTaken);
  irb->setBranchTargetAddress(branchTargetAddress);

  /* Python callback before IR processing */
  processingPyConf.callbackBeforeIRProc(irb, &ap);

  Inst *inst = irb->process(ap);
  ap.addInstructionToTrace(inst);

  /* Export some information from Irb to Inst */
  inst->setNextAddress(irb->getNextAddress());
  inst->setOpcode(irb->getOpcode());
  inst->setOpcodeCategory(irb->getOpcodeCategory());
  inst->setOperands(irb->getOperands());
  inst->setBranchTaken(irb->isBranchTaken());
  inst->setBranchTargetAddress(irb->getBranchTargetAddress());

  /* Python callback before instruction processing */
  processingPyConf.callbackBefore(inst, &ap);

  /* Mutex */
  ap.unlock();
}

static void callbackAfter(CONTEXT *ctx, THREADID threadId) {
  Inst *inst;

  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Get the last instruction */
  inst = ap.getLastInstruction();

  /* Update statistics */
  #ifndef LIGHT_VERSION
  ap.incNumberOfBranchesTaken(inst->isBranch());
  #endif

  /* Python callback after instruction processing */
  processingPyConf.callbackAfter(inst, &ap);

  /* Mutex */
  ap.unlock();
}

#ifndef LIGHT_VERSION
static void callbackSnapshot(uint64 mem, uint32 writeSize) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* If the snapshot is not enable we don't save the memory */
  if (ap.isSnapshotLocked())
    return;

  /* Mutex */
  ap.lock();

  uint32 i = 0;
  for (; i < writeSize ; i++)
    ap.addSnapshotModification(mem+i, *(reinterpret_cast<uint8*>(mem+i)));

  /* Mutex */
  ap.unlock();
}
#endif /* LIGHT_VERSION */

static void TRACE_Instrumentation(TRACE trace, VOID *programName) {
  boost::filesystem::path pname(reinterpret_cast<char*>(programName));

  for (BBL bbl = TRACE_BblHead(trace); BBL_Valid(bbl); bbl = BBL_Next(bbl)) {
    for (INS ins = BBL_InsHead(bbl); INS_Valid(ins); ins = INS_Next(ins)) {

      /* ---- Speed up process ---- */
      IMG currentImgName = IMG_FindByAddress(INS_Address(ins));
      if (!IMG_Valid(currentImgName))
        continue;

      boost::filesystem::path pcurrent(IMG_Name(currentImgName));
      if (!analysisTrigger.getState() && strcmp(pname.leaf().c_str(), pcurrent.leaf().c_str()))
        continue;
      /* ---- End of speed up process ---- */

      IRBuilder *irb = createIRBuilder(ins);

      /* Callback before */
      if (INS_MemoryOperandCount(ins) > 0)
        INS_InsertCall(ins, IPOINT_BEFORE, (AFUNPTR) callbackBefore,
            IARG_PTR, irb,
            IARG_CONTEXT,
            IARG_BOOL, true,
            IARG_MEMORYOP_EA, 0,
            IARG_BRANCH_TAKEN,
            IARG_BRANCH_TARGET_ADDR,
            IARG_THREAD_ID,
            IARG_END);
      else
        INS_InsertCall(ins, IPOINT_BEFORE, (AFUNPTR) callbackBefore,
            IARG_PTR, irb,
            IARG_CONTEXT,
            IARG_BOOL, false,
            IARG_ADDRINT, 0,
            IARG_BRANCH_TAKEN,
            IARG_BRANCH_TARGET_ADDR,
            IARG_THREAD_ID,
            IARG_END);

      /* Callback after */
      /* Syscall after context must be catcher with IDREF.CALLBACK.SYSCALL_EXIT */
      if (INS_IsSyscall(ins) == false) {
        IPOINT where = IPOINT_AFTER;
        if (INS_HasFallThrough(ins) == false)
          where = IPOINT_TAKEN_BRANCH;
        INS_InsertCall(ins, where, (AFUNPTR)callbackAfter, IARG_CONTEXT, IARG_THREAD_ID, IARG_END);
      }

      #ifndef LIGHT_VERSION
      /* I/O memory monitoring for snapshot */
      if (INS_OperandCount(ins) > 1 && INS_MemoryOperandIsWritten(ins, 0)) {
        INS_InsertCall(
          ins, IPOINT_BEFORE, (AFUNPTR)callbackSnapshot,
          IARG_MEMORYOP_EA, 0,
          IARG_UINT32, INS_MemoryWriteSize(ins),
          IARG_END);
      }
      #endif /* LIGHT_VERSION */

    }
  }
}

static void toggleWrapper(bool flag) {
  ap.lock();
  analysisTrigger.update(flag);
  ap.unlock();
}

static void callbackRoutineEntry(CONTEXT *ctx, THREADID threadId, PyObject *callback) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex lock */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  processingPyConf.callbackRoutine(threadId, callback);

  /* Mutex unlock */
  ap.unlock();
}

static void callbackRoutineExit(CONTEXT *ctx, THREADID threadId, PyObject *callback) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex lock */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  processingPyConf.callbackRoutine(threadId, callback);

  /* Mutex unlock */
  ap.unlock();
}

static void IMG_Instrumentation(IMG img, VOID *) {
  /* Lock / Unlock the Analysis */
  if (PyTritonOptions::startAnalysisFromSymbol != nullptr){

    RTN targetRTN = RTN_FindByName(img, PyTritonOptions::startAnalysisFromSymbol);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);

      RTN_InsertCall(targetRTN,
          IPOINT_BEFORE,
          (AFUNPTR) toggleWrapper,
          IARG_BOOL, true,
          IARG_END);

      RTN_InsertCall(targetRTN,
          IPOINT_AFTER,
          (AFUNPTR) toggleWrapper,
          IARG_BOOL, false,
          IARG_END);

      RTN_Close(targetRTN);
    }
  }

  /* Callback on routine entry */
  std::map<const char *, PyObject *>::iterator it;
  for (it = PyTritonOptions::callbackRoutineEntry.begin(); it != PyTritonOptions::callbackRoutineEntry.end(); it++){
    RTN targetRTN = RTN_FindByName(img, it->first);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);
      RTN_InsertCall(targetRTN, IPOINT_BEFORE, (AFUNPTR)callbackRoutineEntry, IARG_CONTEXT, IARG_THREAD_ID, IARG_PTR, it->second, IARG_END);
      RTN_Close(targetRTN);
    }
  }

  /* Callback on routine exit */
  for (it = PyTritonOptions::callbackRoutineExit.begin(); it != PyTritonOptions::callbackRoutineExit.end(); it++){
    RTN targetRTN = RTN_FindByName(img, it->first);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);
      RTN_InsertCall(targetRTN, IPOINT_AFTER, (AFUNPTR)callbackRoutineExit, IARG_CONTEXT, IARG_THREAD_ID, IARG_PTR, it->second, IARG_END);
      RTN_Close(targetRTN);
    }
  }
}

/* Callback at the end of the execution */
static void Fini(INT32, VOID *) {
  /* Python callback at the end of execution */
  processingPyConf.callbackFini();

  /* End of Python */
  Py_Finalize();
}

/* Callback at the syscall entry */
static void callbackSyscallEntry(THREADID threadId, CONTEXT *ctx, SYSCALL_STANDARD std, VOID *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSyscallEntry(threadId, std);

  /* Mutex */
  ap.unlock();
}

/* Callback at the syscall exit */
static void callbackSyscallExit(THREADID threadId, CONTEXT *ctx, SYSCALL_STANDARD std, VOID *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSyscallExit(threadId, std);

  /* Mutex */
  ap.unlock();
}

/* Callback when an image is loaded */
static void callbackImageLoad(IMG img, VOID *v) {
  /* Mutex */
  ap.lock();

  if (!IMG_Valid(img))
  /* Invalid image */
    return;

  /* Collect image's informations */
  string imagePath = IMG_Name(img);
  uint64 imageBase = IMG_LowAddress(img);
  uint64 imageSize = (IMG_HighAddress(img) + 1) - imageBase;

  /* Python callback for image loading */
  processingPyConf.callbackImageLoad(imagePath, imageBase, imageSize);

  /* Mutex */
  ap.unlock();
}

/* Callback when a signals occurs */
static bool callbackSignals(THREADID threadId, sint32 sig, CONTEXT *ctx, bool hasHandler, const EXCEPTION_INFO *pExceptInfo, void *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return false;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSignals(threadId, sig);

  /* Mutex */
  ap.unlock();

  /*
   * We must exit. If you don't want to exit,
   * you must use the restoreSnapshot() function.
   */
  exit(0);

  return true;
}

/* Callback when a thread is created */
static void callbackThreadEntry(THREADID threadId, CONTEXT *ctx, sint32 flags, void *v) {
  /* Mutex */
  ap.lock();

  // TODO #30: Create a map entry of (thread -> {taintEngine, symEngine}).

  /* Mutex */
  ap.unlock();
}

/* Callback when a thread is destroyed */
static void callbackThreadExit(THREADID threadId, const CONTEXT *ctx, sint32 flags, void *v) {
  /* Mutex */
  ap.lock();

  // TODO #30: Destroy the map entry corresponding to the threadId.

  /* Mutex */
  ap.unlock();
}

/*
 * Usage function if Pin fail to start.
 * Display the help message.
 */
static sint32 Usage() {
  std::cerr << KNOB_BASE::StringKnobSummary() << std::endl;
  return -1;
}

/* Get the name of the target binary */
static char *getProgramName(char *argv[]) {
  uint64 offset;
  for (offset = 0; argv[offset]; offset++){
    if (!strcmp(argv[offset], "--") && argv[offset+1])
      return argv[offset+1];
  }
  return nullptr;
}

int main(int argc, char *argv[]) {
  PIN_InitSymbols();
  PIN_SetSyntaxIntel();
  if(PIN_Init(argc, argv))
      return Usage();

  /* Init Python Bindings */
  initBindings();

  /* Image callback */
  IMG_AddInstrumentFunction(IMG_Instrumentation, nullptr);

  /* Instruction callback */
  TRACE_AddInstrumentFunction(TRACE_Instrumentation, getProgramName(argv));

  /* End instrumentation callback */
  PIN_AddFiniFunction(Fini, nullptr);

  /* Syscall entry callback */
  PIN_AddSyscallEntryFunction(callbackSyscallEntry, nullptr);

  /* Syscall exit callback */
  PIN_AddSyscallExitFunction(callbackSyscallExit, nullptr);

  /* Image load callback */
  IMG_AddInstrumentFunction(callbackImageLoad, nullptr);

  /* Signals callback */
  PIN_InterceptSignal(SIGFPE,  callbackSignals, nullptr); /* Floating point exception */
  PIN_InterceptSignal(SIGILL,  callbackSignals, nullptr); /* Illegal Instruction */
  PIN_InterceptSignal(SIGKILL, callbackSignals, nullptr); /* Kill signal */
  PIN_InterceptSignal(SIGPIPE, callbackSignals, nullptr); /* Broken pipe: write to pipe with no readers */
  PIN_InterceptSignal(SIGSEGV, callbackSignals, nullptr); /* Invalid memory reference */

  /* Threads callbacks */
  PIN_AddThreadStartFunction(callbackThreadEntry, nullptr);
  PIN_AddThreadFiniFunction(callbackThreadExit, nullptr);

  /* Exec the python bindings file */
  if (!execBindings(KnobPythonModule.Value().c_str())) {
    std::cerr << "Error: Script file can't be found!" << std::endl;
    exit(1);
  }

  return 0;
}

<commit_msg>Fix mutex<commit_after>/*
**  Copyright (C) - Triton
**
**  This program is under the terms of the LGPLv3 License.
*/

#include <csignal>
#include <iostream>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <vector>

#include <pin.H>

#include <AnalysisProcessor.h>
#include <IRBuilder.h>
#include <IRBuilderFactory.h>
#include <Inst.h>
#include <PINContextHandler.h>
#include <ProcessingPyConf.h>
#include <Trigger.h>
#include <boost/filesystem.hpp>

using boost::filesystem::absolute;
using boost::filesystem::path;

/* Pin options: -script */
KNOB<std::string>   KnobPythonModule(KNOB_MODE_WRITEONCE, "pintool", "script", "", "Python script");

AnalysisProcessor   ap;
Trigger             analysisTrigger = Trigger();
ProcessingPyConf    processingPyConf(&ap, &analysisTrigger);

static void callbackBefore(IRBuilder *irb, CONTEXT *ctx, BOOL hasEA, ADDRINT ea, BOOL isBranchTaken, ADDRINT branchTargetAddress, THREADID threadId) {
  /* Some configurations must be applied before processing */
  processingPyConf.applyConfBeforeProcessing(irb);

  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  if (hasEA)
    irb->setup(ea);

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Setup Information into Irb */
  irb->setThreadID(ap.getThreadID());
  irb->setBranchTaken(isBranchTaken);
  irb->setBranchTargetAddress(branchTargetAddress);

  /* Python callback before IR processing */
  processingPyConf.callbackBeforeIRProc(irb, &ap);

  Inst *inst = irb->process(ap);
  ap.addInstructionToTrace(inst);

  /* Export some information from Irb to Inst */
  inst->setNextAddress(irb->getNextAddress());
  inst->setOpcode(irb->getOpcode());
  inst->setOpcodeCategory(irb->getOpcodeCategory());
  inst->setOperands(irb->getOperands());
  inst->setBranchTaken(irb->isBranchTaken());
  inst->setBranchTargetAddress(irb->getBranchTargetAddress());

  /* Python callback before instruction processing */
  processingPyConf.callbackBefore(inst, &ap);

  /* Mutex */
  ap.unlock();
}

static void callbackAfter(CONTEXT *ctx, THREADID threadId) {
  Inst *inst;

  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Get the last instruction */
  inst = ap.getLastInstruction();

  /* Update statistics */
  #ifndef LIGHT_VERSION
  ap.incNumberOfBranchesTaken(inst->isBranch());
  #endif

  /* Python callback after instruction processing */
  processingPyConf.callbackAfter(inst, &ap);

  /* Mutex */
  ap.unlock();
}

#ifndef LIGHT_VERSION
static void callbackSnapshot(uint64 mem, uint32 writeSize) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* If the snapshot is not enable we don't save the memory */
  if (ap.isSnapshotLocked())
    return;

  /* Mutex */
  ap.lock();

  uint32 i = 0;
  for (; i < writeSize ; i++)
    ap.addSnapshotModification(mem+i, *(reinterpret_cast<uint8*>(mem+i)));

  /* Mutex */
  ap.unlock();
}
#endif /* LIGHT_VERSION */

static void TRACE_Instrumentation(TRACE trace, VOID *programName) {
  boost::filesystem::path pname(reinterpret_cast<char*>(programName));

  for (BBL bbl = TRACE_BblHead(trace); BBL_Valid(bbl); bbl = BBL_Next(bbl)) {
    for (INS ins = BBL_InsHead(bbl); INS_Valid(ins); ins = INS_Next(ins)) {

      /* ---- Speed up process ---- */
      IMG currentImgName = IMG_FindByAddress(INS_Address(ins));
      if (!IMG_Valid(currentImgName))
        continue;

      boost::filesystem::path pcurrent(IMG_Name(currentImgName));
      if (!analysisTrigger.getState() && strcmp(pname.leaf().c_str(), pcurrent.leaf().c_str()))
        continue;
      /* ---- End of speed up process ---- */

      IRBuilder *irb = createIRBuilder(ins);

      /* Callback before */
      if (INS_MemoryOperandCount(ins) > 0)
        INS_InsertCall(ins, IPOINT_BEFORE, (AFUNPTR) callbackBefore,
            IARG_PTR, irb,
            IARG_CONTEXT,
            IARG_BOOL, true,
            IARG_MEMORYOP_EA, 0,
            IARG_BRANCH_TAKEN,
            IARG_BRANCH_TARGET_ADDR,
            IARG_THREAD_ID,
            IARG_END);
      else
        INS_InsertCall(ins, IPOINT_BEFORE, (AFUNPTR) callbackBefore,
            IARG_PTR, irb,
            IARG_CONTEXT,
            IARG_BOOL, false,
            IARG_ADDRINT, 0,
            IARG_BRANCH_TAKEN,
            IARG_BRANCH_TARGET_ADDR,
            IARG_THREAD_ID,
            IARG_END);

      /* Callback after */
      /* Syscall after context must be catcher with IDREF.CALLBACK.SYSCALL_EXIT */
      if (INS_IsSyscall(ins) == false) {
        IPOINT where = IPOINT_AFTER;
        if (INS_HasFallThrough(ins) == false)
          where = IPOINT_TAKEN_BRANCH;
        INS_InsertCall(ins, where, (AFUNPTR)callbackAfter, IARG_CONTEXT, IARG_THREAD_ID, IARG_END);
      }

      #ifndef LIGHT_VERSION
      /* I/O memory monitoring for snapshot */
      if (INS_OperandCount(ins) > 1 && INS_MemoryOperandIsWritten(ins, 0)) {
        INS_InsertCall(
          ins, IPOINT_BEFORE, (AFUNPTR)callbackSnapshot,
          IARG_MEMORYOP_EA, 0,
          IARG_UINT32, INS_MemoryWriteSize(ins),
          IARG_END);
      }
      #endif /* LIGHT_VERSION */

    }
  }
}

static void toggleWrapper(bool flag) {
  ap.lock();
  analysisTrigger.update(flag);
  ap.unlock();
}

static void callbackRoutineEntry(CONTEXT *ctx, THREADID threadId, PyObject *callback) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex lock */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  processingPyConf.callbackRoutine(threadId, callback);

  /* Mutex unlock */
  ap.unlock();
}

static void callbackRoutineExit(CONTEXT *ctx, THREADID threadId, PyObject *callback) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex lock */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  processingPyConf.callbackRoutine(threadId, callback);

  /* Mutex unlock */
  ap.unlock();
}

static void IMG_Instrumentation(IMG img, VOID *) {
  /* Lock / Unlock the Analysis */
  if (PyTritonOptions::startAnalysisFromSymbol != nullptr){

    RTN targetRTN = RTN_FindByName(img, PyTritonOptions::startAnalysisFromSymbol);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);

      RTN_InsertCall(targetRTN,
          IPOINT_BEFORE,
          (AFUNPTR) toggleWrapper,
          IARG_BOOL, true,
          IARG_END);

      RTN_InsertCall(targetRTN,
          IPOINT_AFTER,
          (AFUNPTR) toggleWrapper,
          IARG_BOOL, false,
          IARG_END);

      RTN_Close(targetRTN);
    }
  }

  /* Callback on routine entry */
  std::map<const char *, PyObject *>::iterator it;
  for (it = PyTritonOptions::callbackRoutineEntry.begin(); it != PyTritonOptions::callbackRoutineEntry.end(); it++){
    RTN targetRTN = RTN_FindByName(img, it->first);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);
      RTN_InsertCall(targetRTN, IPOINT_BEFORE, (AFUNPTR)callbackRoutineEntry, IARG_CONTEXT, IARG_THREAD_ID, IARG_PTR, it->second, IARG_END);
      RTN_Close(targetRTN);
    }
  }

  /* Callback on routine exit */
  for (it = PyTritonOptions::callbackRoutineExit.begin(); it != PyTritonOptions::callbackRoutineExit.end(); it++){
    RTN targetRTN = RTN_FindByName(img, it->first);
    if (RTN_Valid(targetRTN)){
      RTN_Open(targetRTN);
      RTN_InsertCall(targetRTN, IPOINT_AFTER, (AFUNPTR)callbackRoutineExit, IARG_CONTEXT, IARG_THREAD_ID, IARG_PTR, it->second, IARG_END);
      RTN_Close(targetRTN);
    }
  }
}

/* Callback at the end of the execution */
static void Fini(INT32, VOID *) {
  /* Python callback at the end of execution */
  processingPyConf.callbackFini();

  /* End of Python */
  Py_Finalize();
}

/* Callback at the syscall entry */
static void callbackSyscallEntry(THREADID threadId, CONTEXT *ctx, SYSCALL_STANDARD std, VOID *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSyscallEntry(threadId, std);

  /* Mutex */
  ap.unlock();
}

/* Callback at the syscall exit */
static void callbackSyscallExit(THREADID threadId, CONTEXT *ctx, SYSCALL_STANDARD std, VOID *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSyscallExit(threadId, std);

  /* Mutex */
  ap.unlock();
}

/* Callback when an image is loaded */
static void callbackImageLoad(IMG img, VOID *v) {
  if (!IMG_Valid(img))
  /* Invalid image */
    return;

  /* Mutex */
  ap.lock();

  /* Collect image's informations */
  string imagePath = IMG_Name(img);
  uint64 imageBase = IMG_LowAddress(img);
  uint64 imageSize = (IMG_HighAddress(img) + 1) - imageBase;

  /* Python callback for image loading */
  processingPyConf.callbackImageLoad(imagePath, imageBase, imageSize);

  /* Mutex */
  ap.unlock();
}

/* Callback when a signals occurs */
static bool callbackSignals(THREADID threadId, sint32 sig, CONTEXT *ctx, bool hasHandler, const EXCEPTION_INFO *pExceptInfo, void *v) {
  if (!analysisTrigger.getState())
  /* Analysis locked */
    return false;

  /* Mutex */
  ap.lock();

  /* Update the current context handler */
  ap.updateCurrentCtxH(new PINContextHandler(ctx, threadId));

  /* Python callback at the end of execution */
  processingPyConf.callbackSignals(threadId, sig);

  /* Mutex */
  ap.unlock();

  /*
   * We must exit. If you don't want to exit,
   * you must use the restoreSnapshot() function.
   */
  exit(0);

  return true;
}

/* Callback when a thread is created */
static void callbackThreadEntry(THREADID threadId, CONTEXT *ctx, sint32 flags, void *v) {
  /* Mutex */
  ap.lock();

  // TODO #30: Create a map entry of (thread -> {taintEngine, symEngine}).

  /* Mutex */
  ap.unlock();
}

/* Callback when a thread is destroyed */
static void callbackThreadExit(THREADID threadId, const CONTEXT *ctx, sint32 flags, void *v) {
  /* Mutex */
  ap.lock();

  // TODO #30: Destroy the map entry corresponding to the threadId.

  /* Mutex */
  ap.unlock();
}

/*
 * Usage function if Pin fail to start.
 * Display the help message.
 */
static sint32 Usage() {
  std::cerr << KNOB_BASE::StringKnobSummary() << std::endl;
  return -1;
}

/* Get the name of the target binary */
static char *getProgramName(char *argv[]) {
  uint64 offset;
  for (offset = 0; argv[offset]; offset++){
    if (!strcmp(argv[offset], "--") && argv[offset+1])
      return argv[offset+1];
  }
  return nullptr;
}

int main(int argc, char *argv[]) {
  PIN_InitSymbols();
  PIN_SetSyntaxIntel();
  if(PIN_Init(argc, argv))
      return Usage();

  /* Init Python Bindings */
  initBindings();

  /* Image callback */
  IMG_AddInstrumentFunction(IMG_Instrumentation, nullptr);

  /* Instruction callback */
  TRACE_AddInstrumentFunction(TRACE_Instrumentation, getProgramName(argv));

  /* End instrumentation callback */
  PIN_AddFiniFunction(Fini, nullptr);

  /* Syscall entry callback */
  PIN_AddSyscallEntryFunction(callbackSyscallEntry, nullptr);

  /* Syscall exit callback */
  PIN_AddSyscallExitFunction(callbackSyscallExit, nullptr);

  /* Image load callback */
  IMG_AddInstrumentFunction(callbackImageLoad, nullptr);

  /* Signals callback */
  PIN_InterceptSignal(SIGFPE,  callbackSignals, nullptr); /* Floating point exception */
  PIN_InterceptSignal(SIGILL,  callbackSignals, nullptr); /* Illegal Instruction */
  PIN_InterceptSignal(SIGKILL, callbackSignals, nullptr); /* Kill signal */
  PIN_InterceptSignal(SIGPIPE, callbackSignals, nullptr); /* Broken pipe: write to pipe with no readers */
  PIN_InterceptSignal(SIGSEGV, callbackSignals, nullptr); /* Invalid memory reference */

  /* Threads callbacks */
  PIN_AddThreadStartFunction(callbackThreadEntry, nullptr);
  PIN_AddThreadFiniFunction(callbackThreadExit, nullptr);

  /* Exec the python bindings file */
  if (!execBindings(KnobPythonModule.Value().c_str())) {
    std::cerr << "Error: Script file can't be found!" << std::endl;
    exit(1);
  }

  return 0;
}

<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file
 * \brief Contains thresholds to select implementations based
 * on the expression size
 */

#pragma once

namespace etl {

#ifdef ETL_DEBUG_THRESHOLDS

constexpr size_t gemm_std_max    = 75 * 75;   ///< The maximum number of elements to be handled by std algorithm
constexpr size_t gemm_cublas_min = 180 * 180; ///< The minimum number or elements before considering cublas

constexpr size_t gemm_rr_small_threshold = 1000; ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)
constexpr size_t gemm_cc_small_threshold = 1000; ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)

constexpr size_t gevm_rm_small_threshold = 1000; ///< The number of elements of b after which we use BLAS-like kernel
constexpr size_t gevm_cm_small_threshold = 1000; ///< The number of elements of b after which we use BLAS-like kernel

constexpr size_t gemv_rm_small_threshold = 1000; ///< The number of elements of A after which we use BLAS-like kernel
constexpr size_t gemv_cm_small_threshold = 1000; ///< The number of elements of A after which we use BLAS-like kernel

constexpr size_t parallel_threshold = 6 * 1024; ///< The minimum number of elements before considering parallel implementation

constexpr size_t sum_parallel_threshold = 1024 * 2; ///< The minimum number of elements before considering parallel acc implementation

constexpr size_t conv1_parallel_threshold_conv   = 100; ///< The mimum output size before considering parallel convolution
constexpr size_t conv1_parallel_threshold_kernel = 16;  ///< The mimum kernel size before considering parallel convolution

constexpr size_t fft1_many_threshold_transforms = 16;  ///< The mimum number of transforms to parallelize them
constexpr size_t fft1_many_threshold_n          = 768; ///< The mimum size of the transforms to parallelize them

constexpr size_t fft2_many_threshold_transforms = 16;   ///< The mimum number of transforms to parallelize them
constexpr size_t fft2_many_threshold_n          = 1024; ///< The mimum size of the transforms to parallelize them

constexpr size_t stream_threshold = 1024; ///< The threshold at which stream is used

#else

constexpr size_t gemm_std_max    = 75 * 75;   ///< The maximum number of elements to be handled by std algorithm
constexpr size_t gemm_cublas_min = 180 * 180; ///< The minimum number or elements before considering cublas

constexpr size_t gemm_rr_small_threshold = 10000;   ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)
constexpr size_t gemm_cc_small_threshold = 40000;   ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)

constexpr size_t gevm_rm_small_threshold = 62000;   ///< The number of elements of b after which we use BLAS-like kernel
constexpr size_t gevm_cm_small_threshold = 4000000; ///< The number of elements of b after which we use BLAS-like kernel

constexpr size_t gemv_rm_small_threshold = 4500000; ///< The number of elements of A after which we use BLAS-like kernel
constexpr size_t gemv_cm_small_threshold = 2400000; ///< The number of elements of A after which we use BLAS-like kernel

constexpr size_t parallel_threshold = 128 * 1024; ///< The minimum number of elements before considering parallel implementation

constexpr size_t sum_parallel_threshold = 1024 * 32; ///< The minimum number of elements before considering parallel acc implementation

constexpr size_t conv1_parallel_threshold_conv   = 100; ///< The mimum output size before considering parallel convolution
constexpr size_t conv1_parallel_threshold_kernel = 16;  ///< The mimum kernel size before considering parallel convolution

constexpr size_t fft1_many_threshold_transforms = 16;  ///< The mimum number of transforms to parallelize them
constexpr size_t fft1_many_threshold_n          = 768; ///< The mimum size of the transforms to parallelize them

constexpr size_t fft2_many_threshold_transforms = 16;   ///< The mimum number of transforms to parallelize them
constexpr size_t fft2_many_threshold_n          = 1024; ///< The mimum size of the transforms to parallelize them

constexpr size_t stream_threshold = cache_size; ///< The threshold at which stream is used

#endif

} //end of namespace etl
<commit_msg>Better threshold for gevm<commit_after>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file
 * \brief Contains thresholds to select implementations based
 * on the expression size
 */

#pragma once

namespace etl {

#ifdef ETL_DEBUG_THRESHOLDS

constexpr size_t gemm_std_max    = 75 * 75;   ///< The maximum number of elements to be handled by std algorithm
constexpr size_t gemm_cublas_min = 180 * 180; ///< The minimum number or elements before considering cublas

constexpr size_t gemm_rr_small_threshold = 1000; ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)
constexpr size_t gemm_cc_small_threshold = 1000; ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)

constexpr size_t gevm_rm_small_threshold = 1000; ///< The number of elements of b after which we use BLAS-like kernel
constexpr size_t gevm_cm_small_threshold = 1000; ///< The number of elements of b after which we use BLAS-like kernel

constexpr size_t gemv_rm_small_threshold = 1000; ///< The number of elements of A after which we use BLAS-like kernel
constexpr size_t gemv_cm_small_threshold = 1000; ///< The number of elements of A after which we use BLAS-like kernel

constexpr size_t parallel_threshold = 6 * 1024; ///< The minimum number of elements before considering parallel implementation

constexpr size_t sum_parallel_threshold = 1024 * 2; ///< The minimum number of elements before considering parallel acc implementation

constexpr size_t conv1_parallel_threshold_conv   = 100; ///< The mimum output size before considering parallel convolution
constexpr size_t conv1_parallel_threshold_kernel = 16;  ///< The mimum kernel size before considering parallel convolution

constexpr size_t fft1_many_threshold_transforms = 16;  ///< The mimum number of transforms to parallelize them
constexpr size_t fft1_many_threshold_n          = 768; ///< The mimum size of the transforms to parallelize them

constexpr size_t fft2_many_threshold_transforms = 16;   ///< The mimum number of transforms to parallelize them
constexpr size_t fft2_many_threshold_n          = 1024; ///< The mimum size of the transforms to parallelize them

constexpr size_t stream_threshold = 1024; ///< The threshold at which stream is used

#else

constexpr size_t gemm_std_max    = 75 * 75;   ///< The maximum number of elements to be handled by std algorithm
constexpr size_t gemm_cublas_min = 180 * 180; ///< The minimum number or elements before considering cublas

constexpr size_t gemm_rr_small_threshold = 10000;   ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)
constexpr size_t gemm_cc_small_threshold = 40000;   ///< The number of elements of B after which we use BLAS-like kernel (for GEMM)

constexpr size_t gevm_rm_small_threshold = 72000;   ///< The number of elements of b after which we use BLAS-like kernel
constexpr size_t gevm_cm_small_threshold = 4000000; ///< The number of elements of b after which we use BLAS-like kernel

constexpr size_t gemv_rm_small_threshold = 4500000; ///< The number of elements of A after which we use BLAS-like kernel
constexpr size_t gemv_cm_small_threshold = 2400000; ///< The number of elements of A after which we use BLAS-like kernel

constexpr size_t parallel_threshold = 128 * 1024; ///< The minimum number of elements before considering parallel implementation

constexpr size_t sum_parallel_threshold = 1024 * 32; ///< The minimum number of elements before considering parallel acc implementation

constexpr size_t conv1_parallel_threshold_conv   = 100; ///< The mimum output size before considering parallel convolution
constexpr size_t conv1_parallel_threshold_kernel = 16;  ///< The mimum kernel size before considering parallel convolution

constexpr size_t fft1_many_threshold_transforms = 16;  ///< The mimum number of transforms to parallelize them
constexpr size_t fft1_many_threshold_n          = 768; ///< The mimum size of the transforms to parallelize them

constexpr size_t fft2_many_threshold_transforms = 16;   ///< The mimum number of transforms to parallelize them
constexpr size_t fft2_many_threshold_n          = 1024; ///< The mimum size of the transforms to parallelize them

constexpr size_t stream_threshold = cache_size; ///< The threshold at which stream is used

#endif

} //end of namespace etl
<|endoftext|>


<commit_before>//
//  CRTMachine.hpp
//  Clock Signal
//
//  Created by Thomas Harte on 31/05/2016.
//  Copyright © 2016 Thomas Harte. All rights reserved.
//

#ifndef CRTMachine_hpp
#define CRTMachine_hpp

#include "../Outputs/CRT/CRT.hpp"
#include "../Outputs/Speaker.hpp"

namespace CRTMachine {

/*!
	A CRTMachine::Machine is a mostly-abstract base class for machines that connect to a CRT,
	that optionally provide a speaker, and that nominate a clock rate and can announce to a delegate
	should that clock rate change.
*/
class Machine {
	public:
		Machine() : clock_is_unlimited_(false) {}

		virtual void setup_output(float aspect_ratio) = 0;
		virtual void close_output() = 0;

		virtual std::shared_ptr<Outputs::CRT::CRT> get_crt() = 0;
		virtual std::shared_ptr<Outputs::Speaker> get_speaker() = 0;

		virtual void run_for_cycles(int number_of_cycles) = 0;

		// TODO: sever the clock-rate stuff.
		double get_clock_rate() {
			return clock_rate_;
		}
		bool get_clock_is_unlimited() {
			return clock_is_unlimited_;
		}
		class Delegate {
			public:
				virtual void machine_did_change_clock_rate(Machine *machine) = 0;
				virtual void machine_did_change_clock_is_unlimited(Machine *machine) = 0;
		};
		void set_delegate(Delegate *delegate) { this->delegate_ = delegate; }

	protected:
		void set_clock_rate(double clock_rate) {
			if(clock_rate_ != clock_rate) {
				clock_rate_ = clock_rate;
				if(delegate_) delegate_->machine_did_change_clock_rate(this);
			}
		}
		void set_clock_is_unlimited(bool clock_is_unlimited) {
			if(clock_is_unlimited != clock_is_unlimited_) {
				clock_is_unlimited_ = clock_is_unlimited;
				if(delegate_) delegate_->machine_did_change_clock_is_unlimited(this);
			}
		}

	private:
		Delegate *delegate_;
		double clock_rate_;
		bool clock_is_unlimited_;
};

}

#endif /* CRTMachine_hpp */
<commit_msg>Added documentation.<commit_after>//
//  CRTMachine.hpp
//  Clock Signal
//
//  Created by Thomas Harte on 31/05/2016.
//  Copyright © 2016 Thomas Harte. All rights reserved.
//

#ifndef CRTMachine_hpp
#define CRTMachine_hpp

#include "../Outputs/CRT/CRT.hpp"
#include "../Outputs/Speaker.hpp"

namespace CRTMachine {

/*!
	A CRTMachine::Machine is a mostly-abstract base class for machines that connect to a CRT,
	that optionally provide a speaker, and that nominate a clock rate and can announce to a delegate
	should that clock rate change.
*/
class Machine {
	public:
		Machine() : clock_is_unlimited_(false) {}

		/*!
			Causes the machine to set up its CRT and, if it has one, speaker. The caller guarantees
			that an OpenGL context is bound.
		*/
		virtual void setup_output(float aspect_ratio) = 0;

		/*!
			Gives the machine a chance to release all owned resources. The caller guarantees that the
			OpenGL context is bound.
		*/
		virtual void close_output() = 0;

		/// @returns The CRT this machine is drawing to. Should not be @c nullptr.
		virtual std::shared_ptr<Outputs::CRT::CRT> get_crt() = 0;

		/// @returns The speaker that receives this machine's output, or @c nullptr if this machine is mute.
		virtual std::shared_ptr<Outputs::Speaker> get_speaker() = 0;

		/// Runs the machine for @c number_of_cycle cycles.
		virtual void run_for_cycles(int number_of_cycles) = 0;

		// TODO: sever the clock-rate stuff.
		double get_clock_rate() {
			return clock_rate_;
		}
		bool get_clock_is_unlimited() {
			return clock_is_unlimited_;
		}
		class Delegate {
			public:
				virtual void machine_did_change_clock_rate(Machine *machine) = 0;
				virtual void machine_did_change_clock_is_unlimited(Machine *machine) = 0;
		};
		void set_delegate(Delegate *delegate) { this->delegate_ = delegate; }

	protected:
		void set_clock_rate(double clock_rate) {
			if(clock_rate_ != clock_rate) {
				clock_rate_ = clock_rate;
				if(delegate_) delegate_->machine_did_change_clock_rate(this);
			}
		}
		void set_clock_is_unlimited(bool clock_is_unlimited) {
			if(clock_is_unlimited != clock_is_unlimited_) {
				clock_is_unlimited_ = clock_is_unlimited;
				if(delegate_) delegate_->machine_did_change_clock_is_unlimited(this);
			}
		}

	private:
		Delegate *delegate_;
		double clock_rate_;
		bool clock_is_unlimited_;
};

}

#endif /* CRTMachine_hpp */
<|endoftext|>


<commit_before>#include "ofMain.h"
#include "ofAppGLFWWindow.h"
#include "pineal.h"

int main(int argc, char ** argv){
	auto app = make_shared<Pineal>(argc, argv);
	auto out = make_shared<outApp>(app);

	ofGLFWWindowSettings settings;
	auto appWindow = ofCreateWindow(settings);

	settings.shareContextWith = appWindow;
	auto outWindow = ofCreateWindow(settings);

	ofRunApp(appWindow, app);
	ofRunApp(outWindow, out);
	ofRunMainLoop();
}
<commit_msg>Explicit shared_ptr<commit_after>#include "ofMain.h"
#include "ofAppGLFWWindow.h"
#include "pineal.h"

int main(int argc, char ** argv){
	shared_ptr<Pineal> app = make_shared<Pineal>(argc, argv);
	shared_ptr<outApp> out = make_shared<outApp>(app);

	ofGLFWWindowSettings settings;
	shared_ptr<ofAppBaseWindow> appWindow = ofCreateWindow(settings);

	settings.shareContextWith = appWindow;
	shared_ptr<ofAppBaseWindow> outWindow = ofCreateWindow(settings);

	ofRunApp(appWindow, app);
	ofRunApp(outWindow, out);
	ofRunMainLoop();
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2011, Willow Garage, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Willow Garage, Inc. nor the names of its
 *       contributors may be used to endorse or promote products derived from
 *       this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#pragma once
//do not include this in ecto lib files, only in client modules

#pragma push_macro("_POSIX_C_SOURCE")
#pragma push_macro("_XOPEN_SOURCE")
#undef _POSIX_C_SOURCE
#undef _XOPEN_SOURCE
#include <Python.h>
#include <boost/python.hpp>
#include <boost/thread/thread.hpp>
#include <map>
//#pragma pop_macro("_POSIX_C_SOURCE")
//#pragma pop_macro("_XOPEN_C_SOURCE")

#include <boost/noncopyable.hpp>

namespace ecto {
  namespace py {

    struct gilstatus
    {
      const char* file;
      unsigned line;
      const char* what;
      gilstatus(const char* f, unsigned l, const char* w);
    };

    //
    //  used in the interpreter thread during blocking operations
    //  RIAA-style  Py_BEGIN_ALLOW_THREADS
    //
    class scoped_gil_release : boost::noncopyable
    {
      static std::map<boost::thread::id, PyThreadState*> thread_states;
      bool mine;
      gilstatus mystatus;
    public:
      scoped_gil_release(const char* file, unsigned line);
      ~scoped_gil_release();
    };

    //
    //  For non-python created threads
    //
    class scoped_call_back_to_python : boost::noncopyable
    {
      PyGILState_STATE gilstate;
      bool have;
      gilstatus mystatus;

    public:

      scoped_call_back_to_python(const char* file, unsigned line);
      ~scoped_call_back_to_python();
    };
  }
}

//#define ECTO_SCOPED_GILRELEASE() ecto::py::scoped_gil_release gilrelease ## __LINE__(__FILE__, __LINE__)
#define ECTO_SCOPED_GILRELEASE()

//#define ECTO_SCOPED_CALLPYTHON() ecto::py::scoped_call_back_to_python gilcall ## __LINE__(__FILE__, __LINE__)
#define ECTO_SCOPED_CALLPYTHON()
<commit_msg>activate gil release/call macros<commit_after>/*
 * Copyright (c) 2011, Willow Garage, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Willow Garage, Inc. nor the names of its
 *       contributors may be used to endorse or promote products derived from
 *       this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
#pragma once
//do not include this in ecto lib files, only in client modules

#pragma push_macro("_POSIX_C_SOURCE")
#pragma push_macro("_XOPEN_SOURCE")
#undef _POSIX_C_SOURCE
#undef _XOPEN_SOURCE
#include <Python.h>
#include <boost/python.hpp>
#include <boost/thread/thread.hpp>
#include <map>
//#pragma pop_macro("_POSIX_C_SOURCE")
//#pragma pop_macro("_XOPEN_C_SOURCE")

#include <boost/noncopyable.hpp>

namespace ecto {
  namespace py {

    struct gilstatus
    {
      const char* file;
      unsigned line;
      const char* what;
      gilstatus(const char* f, unsigned l, const char* w);
    };

    //
    //  used in the interpreter thread during blocking operations
    //  RIAA-style  Py_BEGIN_ALLOW_THREADS
    //
    class scoped_gil_release : boost::noncopyable
    {
      static std::map<boost::thread::id, PyThreadState*> thread_states;
      bool mine;
      gilstatus mystatus;
    public:
      scoped_gil_release(const char* file, unsigned line);
      ~scoped_gil_release();
    };

    //
    //  For non-python created threads
    //
    class scoped_call_back_to_python : boost::noncopyable
    {
      PyGILState_STATE gilstate;
      bool have;
      gilstatus mystatus;

    public:

      scoped_call_back_to_python(const char* file, unsigned line);
      ~scoped_call_back_to_python();
    };
  }
}

#define ECTO_SCOPED_GILRELEASE() ecto::py::scoped_gil_release gilrelease ## __LINE__(__FILE__, __LINE__)
#define ECTO_SCOPED_CALLPYTHON() ecto::py::scoped_call_back_to_python gilcall ## __LINE__(__FILE__, __LINE__)
<|endoftext|>


<commit_before>#include <opencv2/core/core.hpp>
#include <iostream>
#include <unistd.h>
#include <sys/types.h>
#include <ifaddrs.h>
#include <netinet/in.h>
#include <string.h>
#include <arpa/inet.h>

#include <ros/ros.h>

#include "libfreenect.hpp"
#include "CvKinect.hpp"
#include "CvImageProcessor.hpp"
#include "Communicator.hpp"

bool startsWith(const char *prefix, const char *str)
{
	size_t lenPrefix = strlen(prefix),
	lenStr = strlen(str);
	return lenStr < lenPrefix ? false : strncmp(prefix, str, lenPrefix) == 0;
}

// Gets the ip address of the machine
char* getIpAddress()
{
	struct ifaddrs * ifAddrStruct = NULL;
	struct ifaddrs * ifa = NULL;
	void * tmpAddrPtr = NULL;
	char* result = 0;

	getifaddrs(&ifAddrStruct);

	for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) {
		// If address type is IPv4
		if (ifa ->ifa_addr->sa_family==AF_INET) {
            tmpAddrPtr=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
			char addressBuffer[INET_ADDRSTRLEN];
			inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN);

			if (startsWith("eth", ifa->ifa_name)) {
				result = (char*) malloc(INET_ADDRSTRLEN * sizeof(char));
				strcpy(result, addressBuffer);
				break;
			}

			// std::cout << ifa->ifa_name << " IP Address is " << addressBuffer << std::endl;
		} else
			// If address type is IPv6
			if (ifa->ifa_addr->sa_family==AF_INET6) {
            tmpAddrPtr=&((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
			char addressBuffer[INET6_ADDRSTRLEN];
			inet_ntop(AF_INET6, tmpAddrPtr, addressBuffer, INET6_ADDRSTRLEN);
			// printf("%s IP Address %s\n", ifa->ifa_name, addressBuffer);
		}
	}

	if (ifAddrStruct!=NULL) {
		freeifaddrs(ifAddrStruct);
	}

	return result;
}

int main(int argc, char** argv)
{
	std::cout << "Starting Camera Application" << std::endl;

	char* ip = getIpAddress();

	if (ip == 0) {
		std::cout << "No ip address found. Terminating" << std::endl;
		exit(1);
	} else {
		// Replace . with _
		for (int i = 0; ip[i] != 0; i++) {
			if (ip[i] == '.') {
				ip[i] = '_';
			}
		}
	}

	// Create node name.
	char nodeName[80];
	strcpy(nodeName, "Camera_Application_");
	strcat(nodeName, ip);
	free(ip);

	ros::init(argc, argv, nodeName);

	if( ros::console::set_logger_level(ROSCONSOLE_DEFAULT_NAME, ros::console::levels::Debug) ) {
		ros::console::notifyLoggerLevelsChanged();
	}

	Freenect::Freenect freenect;
	CvKinect& device = freenect.createDevice<CvKinect>(0);
	// CvImageProcessor* analyzer = new CvImageProcessor(&device, 0, true, false);
	CvImageProcessor* analyzer = new CvImageProcessor(&device, 0);

	// Do ROS stuff.
	Communicator comm(&device, analyzer);
	analyzer->setDataReceiver(&comm);
	analyzer->start();

	ROS_INFO("Initialized Kinect.");
	ros::spin();
	ROS_INFO("Shutting down.");

	analyzer->stop();

	ros::shutdown();
	std::cout << "ros::shutdown() sent" << std::endl;

	// Wait for ros to shutdown.
	while (ros::ok()) {
		usleep(10000);
	}

	std::cout << "Camera Application successfully terminated" << std::endl;
}
<commit_msg>Made tracking visual<commit_after>#include <opencv2/core/core.hpp>
#include <iostream>
#include <unistd.h>
#include <sys/types.h>
#include <ifaddrs.h>
#include <netinet/in.h>
#include <string.h>
#include <arpa/inet.h>

#include <ros/ros.h>

#include "libfreenect.hpp"
#include "CvKinect.hpp"
#include "CvImageProcessor.hpp"
#include "Communicator.hpp"

bool startsWith(const char *prefix, const char *str)
{
	size_t lenPrefix = strlen(prefix),
	lenStr = strlen(str);
	return lenStr < lenPrefix ? false : strncmp(prefix, str, lenPrefix) == 0;
}

// Gets the ip address of the machine
char* getIpAddress()
{
	struct ifaddrs * ifAddrStruct = NULL;
	struct ifaddrs * ifa = NULL;
	void * tmpAddrPtr = NULL;
	char* result = 0;

	getifaddrs(&ifAddrStruct);

	for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) {
		// If address type is IPv4
		if (ifa ->ifa_addr->sa_family==AF_INET) {
            tmpAddrPtr=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
			char addressBuffer[INET_ADDRSTRLEN];
			inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN);

			if (startsWith("eth", ifa->ifa_name)) {
				result = (char*) malloc(INET_ADDRSTRLEN * sizeof(char));
				strcpy(result, addressBuffer);
				break;
			}

			// std::cout << ifa->ifa_name << " IP Address is " << addressBuffer << std::endl;
		} else
			// If address type is IPv6
			if (ifa->ifa_addr->sa_family==AF_INET6) {
            tmpAddrPtr=&((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
			char addressBuffer[INET6_ADDRSTRLEN];
			inet_ntop(AF_INET6, tmpAddrPtr, addressBuffer, INET6_ADDRSTRLEN);
			// printf("%s IP Address %s\n", ifa->ifa_name, addressBuffer);
		}
	}

	if (ifAddrStruct!=NULL) {
		freeifaddrs(ifAddrStruct);
	}

	return result;
}

int main(int argc, char** argv)
{
	std::cout << "Starting Camera Application" << std::endl;

	char* ip = getIpAddress();

	if (ip == 0) {
		std::cout << "No ip address found. Terminating" << std::endl;
		exit(1);
	} else {
		// Replace . with _
		for (int i = 0; ip[i] != 0; i++) {
			if (ip[i] == '.') {
				ip[i] = '_';
			}
		}
	}

	// Create node name.
	char nodeName[80];
	strcpy(nodeName, "Camera_Application_");
	strcat(nodeName, ip);
	free(ip);

	ros::init(argc, argv, nodeName);

	if( ros::console::set_logger_level(ROSCONSOLE_DEFAULT_NAME, ros::console::levels::Debug) ) {
		ros::console::notifyLoggerLevelsChanged();
	}

	Freenect::Freenect freenect;
	CvKinect& device = freenect.createDevice<CvKinect>(0);
	CvImageProcessor* analyzer = new CvImageProcessor(&device, 0, true, false);
// 	CvImageProcessor* analyzer = new CvImageProcessor(&device, 0);

	// Do ROS stuff.
	Communicator comm(&device, analyzer);
	analyzer->setDataReceiver(&comm);
	analyzer->start();

	ROS_INFO("Initialized Kinect.");
	ros::spin();
	ROS_INFO("Shutting down.");

	analyzer->stop();

	ros::shutdown();
	std::cout << "ros::shutdown() sent" << std::endl;

	// Wait for ros to shutdown.
	while (ros::ok()) {
		usleep(10000);
	}

	std::cout << "Camera Application successfully terminated" << std::endl;
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2014, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <proxygen/lib/http/codec/compress/HPACKCodec.h>

#include <algorithm>
#include <folly/String.h>
#include <folly/io/Cursor.h>
#include <proxygen/lib/http/codec/compress/HPACKHeader.h>

using folly::IOBuf;
using folly::io::Cursor;
using proxygen::compress::Header;
using proxygen::compress::HeaderPiece;
using proxygen::compress::HeaderPieceList;
using std::unique_ptr;
using std::vector;

namespace proxygen {

const std::string kHpackNpn = "spdy/3.1-fb-0.5";

HPACKCodec::HPACKCodec(TransportDirection direction) {
  HPACK::MessageType encoderType;
  HPACK::MessageType decoderType;
  if (direction == TransportDirection::DOWNSTREAM) {
    decoderType = HPACK::MessageType::REQ;
    encoderType = HPACK::MessageType::RESP;
  } else {
    // UPSTREAM
    decoderType = HPACK::MessageType::RESP;
    encoderType = HPACK::MessageType::REQ;
  }
  encoder_ = folly::make_unique<HPACKEncoder>(encoderType, true);
  decoder_ = folly::make_unique<HPACKDecoder>(decoderType);
}

unique_ptr<IOBuf> HPACKCodec::encode(vector<Header>& headers) noexcept {
  vector<HPACKHeader> converted;
  // convert to HPACK API format
  uint32_t uncompressed = 0;
  for (const auto& h : headers) {
    HPACKHeader header(*h.name, *h.value);
    // This is ugly but since we're not changing the size
    // of the string I'm assuming this is OK
    char* mutableName = const_cast<char*>(header.name.data());
    folly::toLowerAscii(mutableName, header.name.size());
    converted.push_back(header);
    uncompressed += header.name.size() + header.value.size() + 2;
  }
  auto buf = encoder_->encode(converted, encodeHeadroom_);
  encodedSize_.compressed = 0;
  if (buf) {
    encodedSize_.compressed = buf->computeChainDataLength();
  }
  encodedSize_.uncompressed = uncompressed;
  if (stats_) {
    stats_->recordEncode(Type::HPACK, encodedSize_);
  }
  return std::move(buf);
}

Result<HeaderDecodeResult, HeaderDecodeError>
HPACKCodec::decode(Cursor& cursor, uint32_t length) noexcept {
  outHeaders_.clear();
  decodedHeaders_.clear();
  auto consumed = decoder_->decode(cursor, length, decodedHeaders_);
  if (decoder_->hasError()) {
    if (stats_) {
      stats_->recordDecodeError(Type::HPACK);
    }
    if (decoder_->getError() == HPACKDecoder::Error::HEADERS_TOO_LARGE) {
      return HeaderDecodeError::HEADERS_TOO_LARGE;
    }
    return HeaderDecodeError::BAD_ENCODING;
  }
  // convert to HeaderPieceList
  uint32_t uncompressed = 0;
  // sort the headers to detect duplicates/multi-valued headers
  // * this is really unrelated to HPACK, just a need to mimic GzipHeaderCodec
  sort(decodedHeaders_.begin(), decodedHeaders_.end());

  for (uint32_t i = 0; i < decodedHeaders_.size(); i++) {
    const HPACKHeader& h = decodedHeaders_[i];
    bool multiValued =
      (i > 0 && decodedHeaders_[i - 1].name == h.name) ||
      (i < decodedHeaders_.size() - 1 && decodedHeaders_[i + 1].name == h.name);
    // one entry for the name and one for the value
    outHeaders_.push_back(HeaderPiece((char *)h.name.c_str(), h.name.size(),
                                     false, multiValued));
    outHeaders_.push_back(HeaderPiece((char *)h.value.c_str(), h.value.size(),
                                     false, multiValued));
    uncompressed += h.name.size() + h.value.size() + 2;
  }
  decodedSize_.compressed = consumed;
  decodedSize_.uncompressed = uncompressed;
  if (stats_) {
    stats_->recordDecode(Type::HPACK, decodedSize_);
  }
  return HeaderDecodeResult{outHeaders_, consumed};
}

}
<commit_msg>Remove extra sort()<commit_after>/*
 *  Copyright (c) 2014, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <proxygen/lib/http/codec/compress/HPACKCodec.h>

#include <algorithm>
#include <folly/String.h>
#include <folly/io/Cursor.h>
#include <proxygen/lib/http/codec/compress/HPACKHeader.h>

using folly::IOBuf;
using folly::io::Cursor;
using proxygen::compress::Header;
using proxygen::compress::HeaderPiece;
using proxygen::compress::HeaderPieceList;
using std::unique_ptr;
using std::vector;

namespace proxygen {

const std::string kHpackNpn = "spdy/3.1-fb-0.5";

HPACKCodec::HPACKCodec(TransportDirection direction) {
  HPACK::MessageType encoderType;
  HPACK::MessageType decoderType;
  if (direction == TransportDirection::DOWNSTREAM) {
    decoderType = HPACK::MessageType::REQ;
    encoderType = HPACK::MessageType::RESP;
  } else {
    // UPSTREAM
    decoderType = HPACK::MessageType::RESP;
    encoderType = HPACK::MessageType::REQ;
  }
  encoder_ = folly::make_unique<HPACKEncoder>(encoderType, true);
  decoder_ = folly::make_unique<HPACKDecoder>(decoderType);
}

unique_ptr<IOBuf> HPACKCodec::encode(vector<Header>& headers) noexcept {
  vector<HPACKHeader> converted;
  // convert to HPACK API format
  uint32_t uncompressed = 0;
  for (const auto& h : headers) {
    HPACKHeader header(*h.name, *h.value);
    // This is ugly but since we're not changing the size
    // of the string I'm assuming this is OK
    char* mutableName = const_cast<char*>(header.name.data());
    folly::toLowerAscii(mutableName, header.name.size());
    converted.push_back(header);
    uncompressed += header.name.size() + header.value.size() + 2;
  }
  auto buf = encoder_->encode(converted, encodeHeadroom_);
  encodedSize_.compressed = 0;
  if (buf) {
    encodedSize_.compressed = buf->computeChainDataLength();
  }
  encodedSize_.uncompressed = uncompressed;
  if (stats_) {
    stats_->recordEncode(Type::HPACK, encodedSize_);
  }
  return std::move(buf);
}

Result<HeaderDecodeResult, HeaderDecodeError>
HPACKCodec::decode(Cursor& cursor, uint32_t length) noexcept {
  outHeaders_.clear();
  decodedHeaders_.clear();
  auto consumed = decoder_->decode(cursor, length, decodedHeaders_);
  if (decoder_->hasError()) {
    if (stats_) {
      stats_->recordDecodeError(Type::HPACK);
    }
    if (decoder_->getError() == HPACKDecoder::Error::HEADERS_TOO_LARGE) {
      return HeaderDecodeError::HEADERS_TOO_LARGE;
    }
    return HeaderDecodeError::BAD_ENCODING;
  }
  // convert to HeaderPieceList
  uint32_t uncompressed = 0;
  for (uint32_t i = 0; i < decodedHeaders_.size(); i++) {
    const HPACKHeader& h = decodedHeaders_[i];
    // SPDYCodec uses this 'multi-valued' flag to detect illegal duplicates
    // Since HPACK does not preclude duplicates, pretend everything is
    // multi-valued
    bool multiValued = true;
    // one entry for the name and one for the value
    outHeaders_.emplace_back((char *)h.name.c_str(), h.name.size(),
                             false, multiValued);
    outHeaders_.emplace_back((char *)h.value.c_str(), h.value.size(),
                             false, multiValued);
    uncompressed += h.name.size() + h.value.size() + 2;
  }
  decodedSize_.compressed = consumed;
  decodedSize_.uncompressed = uncompressed;
  if (stats_) {
    stats_->recordDecode(Type::HPACK, decodedSize_);
  }
  return HeaderDecodeResult{outHeaders_, consumed};
}

}
<|endoftext|>