<commit_before>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    // REVIEW(an): this is useless, you never use the inheritance.  But, you
    // actually SHOULD use it! for all member fields and methods that are in
    // common. But NOT for virtual functions. Remove the virtuals. Find out what
    // functions the methods below have in common and make them functions of the
    // superclass.
    class InputLineIterator
    {
    public:
        static const bool debug = false;
        const size_t read_size = 2 * 1024 * 1024;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;
            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                c_file_ = OpenFile(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = lseek(c_file_, my_start - files_[current_file_].second, SEEK_CUR);
            buffer_.Reserve(Base::read_size);
            ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
                    for (auto it = current_; it != buffer_.end(); it++) {
                        if (THRILL_UNLIKELY(*it == '\n')) {
                            current_ = it + 1;
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
			data_.clear();
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        current_ = it + 1;
						return data_;
                    } else {
						data_.push_back(*it);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    close(c_file_);
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = OpenFile(files_[current_file_].first);
                        ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() + files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (data_.length()) {
                        return data_;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_; // REVIEW(an): use a const & to the vector
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        int c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
		//! String, which Next() references to
		std::string data_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
            while (true) {
				data_.clear();
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        current_ = it + 1;
						return data_;
                    } else {
						data_.push_back(*it);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    c_file_.close();
                    current_file_++;

                    if (current_file_ < NumFiles()) {
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                    }

                    if (data_.length()) {
                        LOG << "end - returning string of length" << data_.length();
                        return data_;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    c_file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(c_file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile c_file_;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
		//! String, which Next() references to
		std::string data_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<commit_msg>more data_ to base<commit_after>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    // REVIEW(an): this is useless, you never use the inheritance.  But, you
    // actually SHOULD use it! for all member fields and methods that are in
    // common. But NOT for virtual functions. Remove the virtuals. Find out what
    // functions the methods below have in common and make them functions of the
    // superclass.
    class InputLineIterator
    {
    public:
        static const bool debug = false;
        const size_t read_size = 2 * 1024 * 1024;

		//! String, which Next() references to
		std::string data_;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
			  my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;
            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                c_file_ = OpenFile(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = lseek(c_file_, my_start - files_[current_file_].second, SEEK_CUR);
            buffer_.Reserve(Base::read_size);
            ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
                    for (auto it = current_; it != buffer_.end(); it++) {
                        if (THRILL_UNLIKELY(*it == '\n')) {
                            current_ = it + 1;
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
			Base::data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
			Base::data_.clear();
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        current_ = it + 1;
						return Base::data_;
                    } else {
						Base::data_.push_back(*it);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    close(c_file_);
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = OpenFile(files_[current_file_].first);
                        ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() + files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (Base::data_.length()) {
                        return Base::data_;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_; // REVIEW(an): use a const & to the vector
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        int c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
			Base::data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
            while (true) {
				Base::data_.clear();
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        current_ = it + 1;
						return Base::data_;
                    } else {
						Base::data_.push_back(*it);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    c_file_.close();
                    current_file_++;

                    if (current_file_ < NumFiles()) {
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                    }

                    if (Base::data_.length()) {
                        LOG << "end - returning string of length" << Base::data_.length();
                        return Base::data_;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    c_file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(c_file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile c_file_;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_);

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_);

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    class InputLineIterator
    {
    public:
		InputLineIterator(
			const std::vector<FileSizePair>& files) : files_(files) { };

        static const bool debug = false;

		//! Block read size
        const size_t read_size = 2 * 1024 * 1024;
		//! String, which Next() references to
		std::string data_;
        //! Input files with size prefixsum.
        const std::vector<FileSizePair>& files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using IteratorBase = InputLineIterator;
		using IteratorBase::buffer_;
		using IteratorBase::current_;
		using IteratorBase::my_end_;
		using IteratorBase::current_file_;
		using IteratorBase::files_;
		using IteratorBase::data_;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            const std::vector<FileSizePair>& files,
            Context& ctx)
            : IteratorBase(files) {

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(files[NumFiles()].second, ctx.num_workers(), ctx.my_rank());

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = file_.lseek(my_start - files_[current_file_].second);
            buffer_.Reserve(IteratorBase::read_size);
            ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
					while (current_ < buffer_.end()) {
                        if (THRILL_UNLIKELY(*current_++ == '\n')) {
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
			data_.clear();
            while (true) {
				while (current_ < buffer_.end()) {
                    if (THRILL_UNLIKELY(*current_ == '\n')) {
						current_++;
						return data_;
                    } else {
						data_.push_back(*current_++);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    file_.close();
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() +
							files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (data_.length()) {
                        return data_;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Offset of current block in file_.
        size_t offset_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile file_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using IteratorBase = InputLineIterator;
		using IteratorBase::current_file_;
		using IteratorBase::files_;
		using IteratorBase::buffer_;
		using IteratorBase::current_;
		using IteratorBase::my_end_;
		using IteratorBase::data_;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            const std::vector<FileSizePair>& files,
			Context& ctx)
            : IteratorBase(files) {

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(files[NumFiles()].second, ctx.num_workers(), ctx.my_rank());

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
            while (true) {
				data_.clear();
				while (current_ < buffer_.end()) {
                    if (THRILL_UNLIKELY(*current_ == '\n')) {
						current_++;
						return data_;
                    } else {
						data_.push_back(*current_++);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    file_.close();
                    current_file_++;

                    if (current_file_ < NumFiles()) {
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                    }

                    if (data_.length()) {
                        LOG << "end - returning string of length" << data_.length();
                        return data_;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }
	private:
        //! File handle to files_[current_file_]
        core::SysFile file_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<commit_msg>bugfix -> no clearing of string before returning a reference to it<commit_after>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_);

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_);

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    class InputLineIterator
    {
    public:
		InputLineIterator(
			const std::vector<FileSizePair>& files) : files_(files) { };

        static const bool debug = false;

		//! Block read size
        const size_t read_size = 2 * 1024 * 1024;
		//! String, which Next() references to
		std::string data_;
        //! Input files with size prefixsum.
        const std::vector<FileSizePair>& files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using IteratorBase = InputLineIterator;
		using IteratorBase::buffer_;
		using IteratorBase::current_;
		using IteratorBase::my_end_;
		using IteratorBase::current_file_;
		using IteratorBase::files_;
		using IteratorBase::data_;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            const std::vector<FileSizePair>& files,
            Context& ctx)
            : IteratorBase(files) {

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(files[NumFiles()].second, ctx.num_workers(), ctx.my_rank());

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = file_.lseek(my_start - files_[current_file_].second);
            buffer_.Reserve(IteratorBase::read_size);
            ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
					while (current_ < buffer_.end()) {
                        if (THRILL_UNLIKELY(*current_++ == '\n')) {
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
			data_.clear();
            while (true) {
				while (current_ < buffer_.end()) {
                    if (THRILL_UNLIKELY(*current_ == '\n')) {
						current_++;
						return data_;
                    } else {
						data_.push_back(*current_++);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    file_.close();
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = file_.read(buffer_.data(), IteratorBase::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() +
							files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (data_.length()) {
                        return data_;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Offset of current block in file_.
        size_t offset_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile file_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using IteratorBase = InputLineIterator;
		using IteratorBase::current_file_;
		using IteratorBase::files_;
		using IteratorBase::buffer_;
		using IteratorBase::current_;
		using IteratorBase::my_end_;
		using IteratorBase::data_;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            const std::vector<FileSizePair>& files,
			Context& ctx)
            : IteratorBase(files) {

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(files[NumFiles()].second, ctx.num_workers(), ctx.my_rank());

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
			data_.reserve(4 * 1024);
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        const std::string& Next() {
			data_.clear();
            while (true) {
				while (current_ < buffer_.end()) {
                    if (THRILL_UNLIKELY(*current_ == '\n')) {
						current_++;
						return data_;
                    } else {
						data_.push_back(*current_++);
					}
                }
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    file_.close();
                    current_file_++;

                    if (current_file_ < NumFiles()) {
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                    }

                    if (data_.length()) {
                        LOG << "end - returning string of length" << data_.length();
                        return data_;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = file_.read(buffer_.data(), read_size);
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }
	private:
        //! File handle to files_[current_file_]
        core::SysFile file_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>#include "GLTextureManager.h"

#include <iostream>
#include <cassert>

using namespace std;

// ----------------------------------------------------------------------------
class GLTextureManager::GLManagedTexture
{
public:
	GLManagedTexture(const std::string &name, GLTexture *texture)
	{
		this->name = name;
		this->texture = texture;
		this->slot = -1;
	}

	string name;
	GLTexture *texture;
	int slot;
};

// ----------------------------------------------------------------------------
GLTextureManager *GLTextureManager::New()
{
	if (GLEW_VERSION_1_3)
	{
		cout << "GLTextureManager: Using OpenGL 1.3 or higher" << endl;
		return new GLTextureManager();
	}
	else if (GLEW_ARB_multitexture)
	{
		// TODO: add ARB fallback
		cerr << "GLTextureManager: Falling back to ARB (not implemented!)"
			<< endl;
		return 0;
	}
	else
	{
		cerr << "GLTextureManager: Multitexturing not supported!" << endl;
		return 0;
	}
}

// ----------------------------------------------------------------------------
GLTextureManager::GLTextureManager()
{
	cout << "GLTextureManager: Constructor" << endl;
	// Initialize inuse vector
	for (unsigned int i = 0; i < GL_MAX_TEXTURE_UNITS; ++i)
	{
		inuse.push_back(false);
		assignments.push_back(0);
	}
}

// ----------------------------------------------------------------------------
GLTextureManager::~GLTextureManager()
{
	cout << "GLTextureManager: Destructor" << endl;
	// Delete all textures
	map<string, GLManagedTexture*>::iterator i = textures.begin();
	while (i != textures.end())
	{
		const string name = i->first;
		GLManagedTexture *mtex = i->second;
		++i;
		if (mtex->texture)
		{
			DeleteTexture(name);
		}
		else
		{
			RemoveTexture(name);
		}
	}
}

// ----------------------------------------------------------------------------
bool GLTextureManager::AddTexture(const string &name, GLTexture *tex)
{
	// Create a wrapper
	GLManagedTexture *mtex = new GLManagedTexture(name, tex);
	// Add the texture to the list
	textures[name] = mtex;

	return true;
}

// ----------------------------------------------------------------------------
bool GLTextureManager::AddReservedSlot(const string &name, unsigned int slot)
{
	assert(0 <= slot && slot < GL_MAX_TEXTURE_UNITS);

	// Is this slot currently free?
	if (inuse[slot])
	{
		cerr << "GLTextureManager: Error adding reserved slot '"
			<< name << "', slot already in use." << endl;
		return false;
	}

	// Create a wrapper and reserve the slot
	GLManagedTexture *mtex = new GLManagedTexture(name, 0);
	mtex->slot = slot;
	textures[name] = mtex;
	assignments[slot] = mtex;
	inuse[slot] = true;

	return true;
}

// ----------------------------------------------------------------------------
GLTexture *GLTextureManager::GetTexture(const string &name)
{
	// Do we know this texture?
	if (textures.find(name) == textures.end())
	{
		cerr << "GLTextureManager: Error getting texture '"
			<< name << "', texture not found" << endl;
		return 0;
	}
	// Reserved slots will just return null here
	return textures[name]->texture;
}

// ----------------------------------------------------------------------------
GLTexture *GLTextureManager::RemoveTexture(const string &name)
{
	// Do we know this texture?
	if (textures.find(name) == textures.end())
	{
		cerr << "GLTextureManager: Error removing texture '"
			<< name << "', texture not found" << endl;
		return 0;
	}
	GLManagedTexture *mtex = textures[name];
	GLTexture *tex = mtex->texture;
	int slot = mtex->slot;
	// Don't unbind unassigned textures
	if (slot >= 0)
	{
		// Don't unbind reserved slots
		if (tex)
		{
			// Unbind the texture
			glActiveTexture(GL_TEXTURE0 + slot);
			tex->UnbindCurrent();
		}
		// Free the slot
		assignments[slot] = 0;
		inuse[slot] = false;
	}
	// Now forget about the thing
	textures.erase(name);
	delete mtex;
	// Release texture to caller
	return tex;
}

// ----------------------------------------------------------------------------
void GLTextureManager::DeleteTexture(const string &name)
{
	GLTexture *tex = RemoveTexture(name);
	if (tex)
	{
		//cout << "GLTextureManager: Deleting texture '"
		//	<< name << "'" << endl;
		delete tex;
	}
}

// ----------------------------------------------------------------------------
// Reset all assignments, except reserved slots
void GLTextureManager::BeginNewPass()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		GLManagedTexture *mtex = assignments[slot];
		// Ignore unused slots
		if (inuse[slot])
		{
			assert(mtex);
			// Leave the reserved slots in place until they are explicitly removed
			if (mtex->texture)
			{
				inuse[slot] = false;
				assignments[slot] = 0;
				mtex->slot = -1;
			}
		}
	}
}

// ----------------------------------------------------------------------------
// Bind all textures which have been assigned a slot
void GLTextureManager::Bind()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		// Ignore unused slots
		if (inuse[slot])
		{
			// Get info
			GLManagedTexture *mtex = assignments[slot];
			assert(mtex);
			GLTexture *tex = mtex->texture;
			// Don't touch the reserved texture units
			if (tex)
			{
				// Set up texture in OpenGL
				glActiveTexture(GL_TEXTURE0 + slot);
				tex->BindToCurrent();
			}
		}
	}
	glActiveTexture(GL_TEXTURE0);
}

// ----------------------------------------------------------------------------
void GLTextureManager::Unbind()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		// Ignore unused slots
		if (inuse[slot])
		{
			// Get info
			GLManagedTexture *mtex = assignments[slot];
			assert(mtex);
			GLTexture *tex = mtex->texture;
			// Don't touch the reserved texture units
			if (tex)
			{
				// Unbind the texture
				glActiveTexture(GL_TEXTURE0 + slot);
				tex->UnbindCurrent();
			}
		}
	}
	glActiveTexture(GL_TEXTURE0);
}

// ----------------------------------------------------------------------------
bool GLTextureManager::SetupProgram(GLProgram &prog)
{
	// Find the first free texture unit, if any
	int firstfree = 0;
	while (firstfree < GL_MAX_TEXTURE_UNITS)
	{
		if (!inuse[firstfree]) break;
		++firstfree;
	}
	if (firstfree == GL_MAX_TEXTURE_UNITS)
	{
		// No free texture units
		firstfree = -1;
	}

	// Iterate over all our textures
	for (map<string, GLManagedTexture*>::iterator i = textures.begin();
		i != textures.end(); ++i)
	{
		// Get info
		string name = i->first;
		GLManagedTexture *mtex = i->second;

		// Does this texture already have a slot?
		int slot = mtex->slot;
		if (slot < 0)
		{
			slot = firstfree;
		}

		// Does the program need the texture?
		if (prog.UseTexture(name, (slot >= 0 ? slot : 0)))
		{
			// Texture was used by program
			if (slot < 0) {
				// Oops, the slot wasn't valid
				cerr << "GLTextureManager: Error setting up program, "
					<< "no free slots for texture '" << name << "'" << endl;
				return false;
			}

			// Store slot assignment
			mtex->slot = slot;
			assignments[slot] = mtex;
			inuse[slot] = true;

			// Find next free slot, if any
			while (firstfree < GL_MAX_TEXTURE_UNITS && firstfree >= 0)
			{
				if (!inuse[firstfree]) break;
				++firstfree;
			}
			if (firstfree == GL_MAX_TEXTURE_UNITS)
			{
				// No free texture units
				firstfree = -1;
			}
		}
	}

	return true;
}
<commit_msg>Removed error message when removing non-existent texture.<commit_after>#include "GLTextureManager.h"

#include <iostream>
#include <cassert>

using namespace std;

// ----------------------------------------------------------------------------
class GLTextureManager::GLManagedTexture
{
public:
	GLManagedTexture(const std::string &name, GLTexture *texture)
	{
		this->name = name;
		this->texture = texture;
		this->slot = -1;
	}

	string name;
	GLTexture *texture;
	int slot;
};

// ----------------------------------------------------------------------------
GLTextureManager *GLTextureManager::New()
{
	if (GLEW_VERSION_1_3)
	{
		cout << "GLTextureManager: Using OpenGL 1.3 or higher" << endl;
		return new GLTextureManager();
	}
	else if (GLEW_ARB_multitexture)
	{
		// TODO: add ARB fallback
		cerr << "GLTextureManager: Falling back to ARB (not implemented!)"
			<< endl;
		return 0;
	}
	else
	{
		cerr << "GLTextureManager: Multitexturing not supported!" << endl;
		return 0;
	}
}

// ----------------------------------------------------------------------------
GLTextureManager::GLTextureManager()
{
	cout << "GLTextureManager: Constructor" << endl;
	// Initialize inuse vector
	for (unsigned int i = 0; i < GL_MAX_TEXTURE_UNITS; ++i)
	{
		inuse.push_back(false);
		assignments.push_back(0);
	}
}

// ----------------------------------------------------------------------------
GLTextureManager::~GLTextureManager()
{
	cout << "GLTextureManager: Destructor" << endl;
	// Delete all textures
	map<string, GLManagedTexture*>::iterator i = textures.begin();
	while (i != textures.end())
	{
		const string name = i->first;
		GLManagedTexture *mtex = i->second;
		++i;
		if (mtex->texture)
		{
			DeleteTexture(name);
		}
		else
		{
			RemoveTexture(name);
		}
	}
}

// ----------------------------------------------------------------------------
bool GLTextureManager::AddTexture(const string &name, GLTexture *tex)
{
	// Create a wrapper
	GLManagedTexture *mtex = new GLManagedTexture(name, tex);
	// Add the texture to the list
	textures[name] = mtex;

	return true;
}

// ----------------------------------------------------------------------------
bool GLTextureManager::AddReservedSlot(const string &name, unsigned int slot)
{
	assert(0 <= slot && slot < GL_MAX_TEXTURE_UNITS);

	// Is this slot currently free?
	if (inuse[slot])
	{
		cerr << "GLTextureManager: Error adding reserved slot '"
			<< name << "', slot already in use." << endl;
		return false;
	}

	// Create a wrapper and reserve the slot
	GLManagedTexture *mtex = new GLManagedTexture(name, 0);
	mtex->slot = slot;
	textures[name] = mtex;
	assignments[slot] = mtex;
	inuse[slot] = true;

	return true;
}

// ----------------------------------------------------------------------------
GLTexture *GLTextureManager::GetTexture(const string &name)
{
	// Do we know this texture?
	if (textures.find(name) == textures.end())
	{
		cerr << "GLTextureManager: Error getting texture '"
			<< name << "', texture not found" << endl;
		return 0;
	}
	// Reserved slots will just return null here
	return textures[name]->texture;
}

// ----------------------------------------------------------------------------
GLTexture *GLTextureManager::RemoveTexture(const string &name)
{
	// Do we know this texture?
	if (textures.find(name) == textures.end())
	{
		//cerr << "GLTextureManager: Error removing texture '"
		//	<< name << "', texture not found" << endl;
		return 0;
	}
	GLManagedTexture *mtex = textures[name];
	GLTexture *tex = mtex->texture;
	int slot = mtex->slot;
	// Don't unbind unassigned textures
	if (slot >= 0)
	{
		// Don't unbind reserved slots
		if (tex)
		{
			// Unbind the texture
			glActiveTexture(GL_TEXTURE0 + slot);
			tex->UnbindCurrent();
		}
		// Free the slot
		assignments[slot] = 0;
		inuse[slot] = false;
	}
	// Now forget about the thing
	textures.erase(name);
	delete mtex;
	// Release texture to caller
	return tex;
}

// ----------------------------------------------------------------------------
void GLTextureManager::DeleteTexture(const string &name)
{
	GLTexture *tex = RemoveTexture(name);
	if (tex)
	{
		//cout << "GLTextureManager: Deleting texture '"
		//	<< name << "'" << endl;
		delete tex;
	}
}

// ----------------------------------------------------------------------------
// Reset all assignments, except reserved slots
void GLTextureManager::BeginNewPass()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		GLManagedTexture *mtex = assignments[slot];
		// Ignore unused slots
		if (inuse[slot])
		{
			assert(mtex);
			// Leave the reserved slots in place until they are explicitly removed
			if (mtex->texture)
			{
				inuse[slot] = false;
				assignments[slot] = 0;
				mtex->slot = -1;
			}
		}
	}
}

// ----------------------------------------------------------------------------
// Bind all textures which have been assigned a slot
void GLTextureManager::Bind()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		// Ignore unused slots
		if (inuse[slot])
		{
			// Get info
			GLManagedTexture *mtex = assignments[slot];
			assert(mtex);
			GLTexture *tex = mtex->texture;
			// Don't touch the reserved texture units
			if (tex)
			{
				// Set up texture in OpenGL
				glActiveTexture(GL_TEXTURE0 + slot);
				tex->BindToCurrent();
			}
		}
	}
	glActiveTexture(GL_TEXTURE0);
}

// ----------------------------------------------------------------------------
void GLTextureManager::Unbind()
{
	for (unsigned int slot = 0; slot < GL_MAX_TEXTURE_UNITS; ++slot)
	{
		// Ignore unused slots
		if (inuse[slot])
		{
			// Get info
			GLManagedTexture *mtex = assignments[slot];
			assert(mtex);
			GLTexture *tex = mtex->texture;
			// Don't touch the reserved texture units
			if (tex)
			{
				// Unbind the texture
				glActiveTexture(GL_TEXTURE0 + slot);
				tex->UnbindCurrent();
			}
		}
	}
	glActiveTexture(GL_TEXTURE0);
}

// ----------------------------------------------------------------------------
bool GLTextureManager::SetupProgram(GLProgram &prog)
{
	// Find the first free texture unit, if any
	int firstfree = 0;
	while (firstfree < GL_MAX_TEXTURE_UNITS)
	{
		if (!inuse[firstfree]) break;
		++firstfree;
	}
	if (firstfree == GL_MAX_TEXTURE_UNITS)
	{
		// No free texture units
		firstfree = -1;
	}

	// Iterate over all our textures
	for (map<string, GLManagedTexture*>::iterator i = textures.begin();
		i != textures.end(); ++i)
	{
		// Get info
		string name = i->first;
		GLManagedTexture *mtex = i->second;

		// Does this texture already have a slot?
		int slot = mtex->slot;
		if (slot < 0)
		{
			slot = firstfree;
		}

		// Does the program need the texture?
		if (prog.UseTexture(name, (slot >= 0 ? slot : 0)))
		{
			// Texture was used by program
			if (slot < 0) {
				// Oops, the slot wasn't valid
				cerr << "GLTextureManager: Error setting up program, "
					<< "no free slots for texture '" << name << "'" << endl;
				return false;
			}

			// Store slot assignment
			mtex->slot = slot;
			assignments[slot] = mtex;
			inuse[slot] = true;

			// Find next free slot, if any
			while (firstfree < GL_MAX_TEXTURE_UNITS && firstfree >= 0)
			{
				if (!inuse[firstfree]) break;
				++firstfree;
			}
			if (firstfree == GL_MAX_TEXTURE_UNITS)
			{
				// No free texture units
				firstfree = -1;
			}
		}
	}

	return true;
}
<|endoftext|>


<commit_before>/* Copyright (C) 2013 Slawomir Cygan <slawomir.cygan@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma warning(disable : 4996)

#ifdef _WIN32
    #include <windows.h>
    #include "CompleteInject/CompleteInject.h"
    #define snprintf _snprintf
#else
    #include <sys/types.h>
    #include <sys/wait.h>
#endif

#include "process.h"

#include <cstdlib>
#include <stdexcept>
#include <sstream>
#include <cstdio>
#include <stdint.h>

#pragma warning(push)
#pragma warning(disable:4512)//'boost::program_options::options_description' : assignment operator could not be generated
#include <boost/program_options/options_description.hpp>
#pragma warning(pop)
#include <boost/program_options/positional_options.hpp>
#include <boost/program_options/variables_map.hpp>
#include <boost/program_options/parsers.hpp>

#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/shared_memory_object.hpp>
#include <boost/interprocess/sync/named_semaphore.hpp>

#include <DGLCommon/os.h>
#include <DGLCommon/wa.h>
#include <DGLCommon/ipc.h>

#ifdef __ANDROID__
#include <dlfcn.h>
#endif

#pragma warning(disable:4503)

#ifdef _WIN32
bool isProcess64Bit(HANDLE hProcess) {
    //get IsWow64Process function
    typedef BOOL (WINAPI *LPFN_ISWOW64PROCESS) (HANDLE, PBOOL);
    LPFN_ISWOW64PROCESS fnIsWow64Process =
        fnIsWow64Process = (LPFN_ISWOW64PROCESS) GetProcAddress(
        GetModuleHandle(TEXT("kernel32")),"IsWow64Process");
    BOOL isWow;

    //check if windows is 64-bit
    bool isWindows64bit = false;

#ifdef _WIN64
    //if we are a 64 bit app, windows is definitely 64 bit
    isWindows64bit = true;
#else
    //we are 32-bit application. Check if we are in Wow64 mode
    if (fnIsWow64Process && fnIsWow64Process(GetCurrentProcess(),&isWow) && isWow) {
        //if we are in Wow64 mode, windows is 64 bit
        isWindows64bit = true;
    }
#endif

    //check if process is 64 bit
    if (isWindows64bit && fnIsWow64Process && fnIsWow64Process(hProcess, &isWow) && !isWow) {
        //windows is 64 bit, process is not in Wow64 mode, so process is 64 bit. This implies usage of 64 bit wrapper
        return true;
    }
    return false;
}
#endif

struct IPCMessage {
    IPCMessage():m_ok(true) { m_message[0] = 0; };
    int8_t m_message[1000];
    int8_t m_ok;
};

std::string getWrapperPath() {
#ifdef _WIN32
#ifdef _WIN64
    std::string ret = "DGLWrapper64\\DGLWrapper.dll";
#else
    std::string ret = "DGLWrapper\\DGLWrapper.dll";
#endif

    char fileName[MAX_PATH];
    if (!GetModuleFileName(NULL, fileName, MAX_PATH)) {
        return ret;
    }
    std::string tmp = fileName;
    size_t splitPoint=tmp.find_last_of("/\\");
    return tmp.substr(0, splitPoint) + "\\" + ret;
#elif defined(__ANDROID__)
    return "/data/local/tmp/libdglwrapper.so";
#else
    return "libdglwrapper.so";
#endif
}

namespace po = boost::program_options;
using namespace std;

int main(int argc, char** argv) {

    IPCMessage message;
    IPCMessage* ipcMessage = &message;

    DGLProcess::native_process_handle_t childHandle = 0;

    try {

        po::options_description desc("Allowed options");
        desc.add_options()
            ("help,h", "produce help message")
            ("egl", "use egl mode");

        desc.add_options()
            ("port", po::value< vector<string> >(), "Debugger TCP port number.");

        po::options_description mandatory("Mandatory options");
        mandatory.add_options()
            ("execute", po::value< vector<string> >()->composing(), "command to execute. Use -- top supply additional args.");

        po::positional_options_description p;
        p.add("execute", -1);

        po::options_description all;
        all.add(desc).add(mandatory);

        po::variables_map vm;
        po::store(po::command_line_parser(argc, argv).
            options(all).positional(p).run(), vm);
        po::notify(vm);

        if (vm.count("help")) {
            std::ostringstream out;
            out << desc << "\n" << mandatory << "\n";
            throw std::runtime_error(out.str());
        }

        if (!vm.count("execute")) {
            std::ostringstream out;
            out << "Nothing to execute.\n" << desc << "\n" << mandatory << "\n";
            throw std::runtime_error(out.str());
        }

        std::string executable = vm["execute"].as< vector<string> >()[0];
        std::vector<std::string> arguments;

        for (size_t i = 0; i < vm["execute"].as< vector<string> >().size(); i++) {
            arguments.push_back(vm["execute"].as< vector<string> >()[i]);
        }

        std::string wrapperPath = getWrapperPath();
        Os::info("Executable: %s\nWrapper: %s\n\n\n", executable.c_str(), wrapperPath.c_str());

        std::shared_ptr<DGLIPC> dglIPC = DGLIPC::Create();
        Os::setEnv("dgl_uuid", dglIPC->getUUID().c_str());

        if (vm.count("egl")) {
            dglIPC->setDebuggerMode(DGLIPC::DebuggerMode::EGL);
        }

        if (vm.count("port")) {
            std::istringstream portStr(vm["port"].as< vector<string> >()[0]);
            unsigned short port;
            portStr >> port;
            dglIPC->setDebuggerPort(port);
        }

        DGLProcess process(executable, arguments);

        if (process.getHandle() == 0) {
            //we are in forked out process
#ifdef _WIN32
            assert(!"forked out on windows");
#else
            Os::setEnv("LD_PRELOAD", wrapperPath.c_str());

#ifdef __ANDROID__
            const char* baseAddr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlclose));
            {
                int addr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlopen)) - baseAddr;
                std::ostringstream str; str << addr;
                Os::setEnv("dlopen_addr", str.str().c_str());
            }
            {
                int addr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlsym)) - baseAddr;
                std::ostringstream str; str << addr;
                Os::setEnv("dlsym_addr", str.str().c_str());
            }
#endif
            process.do_execvp();

#endif
        } else {
            childHandle = process.getHandle();
        }

#ifdef _WIN32
#ifdef _WIN64
        if (!isProcess64Bit(process.getHandle())) {
            throw std::runtime_error("Incompatible loader version: used 64bit, but process is not 64bit");
        }
#else
        if (isProcess64Bit(process.getHandle())) {
            throw std::runtime_error("Incompatible loader version: used 32bit, but process is not 32bit");
        }
#endif

        //process is running, but is suspended (user thread not started, some DLLMain-s may be run by now)

        //inject thread with wrapper library loading code, run through DLLMain and InitializeThread() function

#ifdef WA_ARM_MALI_EMU_LOADERTHREAD_KEEP
        Inject(process.getHandle(), wrapperPath.c_str(), "LoaderThread");
        dglIPC->waitForRemoteThreadSemaphore();
#else
        HANDLE thread = Inject(process.getHandle(), wrapperPath.c_str(), "LoaderThread");
        //wait for loader thread to finish dll inject
        WaitForSingleObject(thread, INFINITE);
#endif

        //resume process - now user thread is running
        //whole OpenGL should be wrapped by now

        if (ResumeThread(process.getMainThread()) == -1) {
            throw std::runtime_error("Cannot resume process");
        }
#endif

    } catch (const std::exception& e) {

        static_assert(sizeof(char) == sizeof(ipcMessage->m_message[0]), "Wrong IPC message element size");

        if (int osError = Os::getLastosError()) {
            snprintf((char*)ipcMessage->m_message, 1000, "%s: %s\n", e.what(), Os::translateOsError(osError).c_str());
        } else {
            snprintf((char*)ipcMessage->m_message, 1000, "%s\n", e.what());
        }

        if (childHandle) {
            //We were able to start child process, but furhter setup failed.
            //Kill the child ASAP.
#ifdef _WIN32
            TerminateProcess(childHandle, EXIT_FAILURE);
#else
            kill(childHandle, SIGPIPE);
#endif
            childHandle = 0;
        }

        ipcMessage->m_ok = false;
    }

    Os::info("%s", ipcMessage->m_message);

    std::string shmemName = Os::getEnv("dgl_loader_shmem");
    if (shmemName.length()) {
        boost::interprocess::shared_memory_object shobj(boost::interprocess::open_only, shmemName.c_str(), boost::interprocess::read_write);
        boost::interprocess::mapped_region region(shobj, boost::interprocess::read_write);
        std::memset(region.get_address(), 0, region.get_size());
        memcpy(region.get_address(), ipcMessage, sizeof(IPCMessage));
    }

    std::string semaphore = Os::getEnv("dgl_loader_semaphore");
    if (semaphore.length()) {
        boost::interprocess::named_semaphore sem(boost::interprocess::open_only, semaphore.c_str());
        sem.post();
    }

    if (!ipcMessage->m_ok) {
        assert(!childHandle);

        //failure - we were not able to setup properly exit code.
        //The failure reason is passed via IPC. Parrent process should check IPC data to distinguish
        //loader failure vs. non-zero exit code from loader's child.
        return EXIT_FAILURE;
    } else {

        //Wait for child process to exit. Return child process exit code.
#ifdef _WIN32
        DWORD exitCode;
        WaitForSingleObject( childHandle, INFINITE );
        GetExitCodeProcess( childHandle, &exitCode);
        return exitCode;
#else
        int exitCode;
        waitpid(childHandle, &exitCode, 0);
        return WEXITSTATUS(exitCode);
#endif
    }
}
<commit_msg>dglloader: do not set dl-intercept env vars, if already set. (this way processes sprawned by dgloader can run dglloader again on Android)<commit_after>/* Copyright (C) 2013 Slawomir Cygan <slawomir.cygan@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma warning(disable : 4996)

#ifdef _WIN32
    #include <windows.h>
    #include "CompleteInject/CompleteInject.h"
    #define snprintf _snprintf
#else
    #include <sys/types.h>
    #include <sys/wait.h>
#endif

#include "process.h"

#include <cstdlib>
#include <stdexcept>
#include <sstream>
#include <cstdio>
#include <stdint.h>

#pragma warning(push)
#pragma warning(disable:4512)//'boost::program_options::options_description' : assignment operator could not be generated
#include <boost/program_options/options_description.hpp>
#pragma warning(pop)
#include <boost/program_options/positional_options.hpp>
#include <boost/program_options/variables_map.hpp>
#include <boost/program_options/parsers.hpp>

#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/shared_memory_object.hpp>
#include <boost/interprocess/sync/named_semaphore.hpp>

#include <DGLCommon/os.h>
#include <DGLCommon/wa.h>
#include <DGLCommon/ipc.h>

#ifdef __ANDROID__
#include <dlfcn.h>
#endif

#pragma warning(disable:4503)

#ifdef _WIN32
bool isProcess64Bit(HANDLE hProcess) {
    //get IsWow64Process function
    typedef BOOL (WINAPI *LPFN_ISWOW64PROCESS) (HANDLE, PBOOL);
    LPFN_ISWOW64PROCESS fnIsWow64Process =
        fnIsWow64Process = (LPFN_ISWOW64PROCESS) GetProcAddress(
        GetModuleHandle(TEXT("kernel32")),"IsWow64Process");
    BOOL isWow;

    //check if windows is 64-bit
    bool isWindows64bit = false;

#ifdef _WIN64
    //if we are a 64 bit app, windows is definitely 64 bit
    isWindows64bit = true;
#else
    //we are 32-bit application. Check if we are in Wow64 mode
    if (fnIsWow64Process && fnIsWow64Process(GetCurrentProcess(),&isWow) && isWow) {
        //if we are in Wow64 mode, windows is 64 bit
        isWindows64bit = true;
    }
#endif

    //check if process is 64 bit
    if (isWindows64bit && fnIsWow64Process && fnIsWow64Process(hProcess, &isWow) && !isWow) {
        //windows is 64 bit, process is not in Wow64 mode, so process is 64 bit. This implies usage of 64 bit wrapper
        return true;
    }
    return false;
}
#endif

struct IPCMessage {
    IPCMessage():m_ok(true) { m_message[0] = 0; };
    int8_t m_message[1000];
    int8_t m_ok;
};

std::string getWrapperPath() {
#ifdef _WIN32
#ifdef _WIN64
    std::string ret = "DGLWrapper64\\DGLWrapper.dll";
#else
    std::string ret = "DGLWrapper\\DGLWrapper.dll";
#endif

    char fileName[MAX_PATH];
    if (!GetModuleFileName(NULL, fileName, MAX_PATH)) {
        return ret;
    }
    std::string tmp = fileName;
    size_t splitPoint=tmp.find_last_of("/\\");
    return tmp.substr(0, splitPoint) + "\\" + ret;
#elif defined(__ANDROID__)
    return "/data/local/tmp/libdglwrapper.so";
#else
    return "libdglwrapper.so";
#endif
}

namespace po = boost::program_options;
using namespace std;

int main(int argc, char** argv) {

    IPCMessage message;
    IPCMessage* ipcMessage = &message;

    DGLProcess::native_process_handle_t childHandle = 0;

    try {

        po::options_description desc("Allowed options");
        desc.add_options()
            ("help,h", "produce help message")
            ("egl", "use egl mode");

        desc.add_options()
            ("port", po::value< vector<string> >(), "Debugger TCP port number.");

        po::options_description mandatory("Mandatory options");
        mandatory.add_options()
            ("execute", po::value< vector<string> >()->composing(), "command to execute. Use -- top supply additional args.");

        po::positional_options_description p;
        p.add("execute", -1);

        po::options_description all;
        all.add(desc).add(mandatory);

        po::variables_map vm;
        po::store(po::command_line_parser(argc, argv).
            options(all).positional(p).run(), vm);
        po::notify(vm);

        if (vm.count("help")) {
            std::ostringstream out;
            out << desc << "\n" << mandatory << "\n";
            throw std::runtime_error(out.str());
        }

        if (!vm.count("execute")) {
            std::ostringstream out;
            out << "Nothing to execute.\n" << desc << "\n" << mandatory << "\n";
            throw std::runtime_error(out.str());
        }

        std::string executable = vm["execute"].as< vector<string> >()[0];
        std::vector<std::string> arguments;

        for (size_t i = 0; i < vm["execute"].as< vector<string> >().size(); i++) {
            arguments.push_back(vm["execute"].as< vector<string> >()[i]);
        }

        std::string wrapperPath = getWrapperPath();
        Os::info("Executable: %s\nWrapper: %s\n\n\n", executable.c_str(), wrapperPath.c_str());

        std::shared_ptr<DGLIPC> dglIPC = DGLIPC::Create();
        Os::setEnv("dgl_uuid", dglIPC->getUUID().c_str());

        if (vm.count("egl")) {
            dglIPC->setDebuggerMode(DGLIPC::DebuggerMode::EGL);
        }

        if (vm.count("port")) {
            std::istringstream portStr(vm["port"].as< vector<string> >()[0]);
            unsigned short port;
            portStr >> port;
            dglIPC->setDebuggerPort(port);
        }

        DGLProcess process(executable, arguments);

        if (process.getHandle() == 0) {
            //we are in forked out process
#ifdef _WIN32
            assert(!"forked out on windows");
#else
            Os::setEnv("LD_PRELOAD", wrapperPath.c_str());

#ifdef __ANDROID__
            {
                const char* baseAddr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlclose));
                if (!Os::getEnv("dlopen_addr").length()) {

                    int addr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlopen)) - baseAddr;
                    std::ostringstream str; str << addr;
                    Os::setEnv("dlopen_addr", str.str().c_str());

                }
                 if (!Os::getEnv("dlsym_addr").length()) {
                     int addr = reinterpret_cast<char*>(reinterpret_cast<intptr_t>(&dlsym)) - baseAddr;
                     std::ostringstream str; str << addr;
                     Os::setEnv("dlsym_addr", str.str().c_str());
                 }
            }
#endif
            process.do_execvp();

#endif
        } else {
            childHandle = process.getHandle();
        }

#ifdef _WIN32
#ifdef _WIN64
        if (!isProcess64Bit(process.getHandle())) {
            throw std::runtime_error("Incompatible loader version: used 64bit, but process is not 64bit");
        }
#else
        if (isProcess64Bit(process.getHandle())) {
            throw std::runtime_error("Incompatible loader version: used 32bit, but process is not 32bit");
        }
#endif

        //process is running, but is suspended (user thread not started, some DLLMain-s may be run by now)

        //inject thread with wrapper library loading code, run through DLLMain and InitializeThread() function

#ifdef WA_ARM_MALI_EMU_LOADERTHREAD_KEEP
        Inject(process.getHandle(), wrapperPath.c_str(), "LoaderThread");
        dglIPC->waitForRemoteThreadSemaphore();
#else
        HANDLE thread = Inject(process.getHandle(), wrapperPath.c_str(), "LoaderThread");
        //wait for loader thread to finish dll inject
        WaitForSingleObject(thread, INFINITE);
#endif

        //resume process - now user thread is running
        //whole OpenGL should be wrapped by now

        if (ResumeThread(process.getMainThread()) == -1) {
            throw std::runtime_error("Cannot resume process");
        }
#endif

    } catch (const std::exception& e) {

        static_assert(sizeof(char) == sizeof(ipcMessage->m_message[0]), "Wrong IPC message element size");

        if (int osError = Os::getLastosError()) {
            snprintf((char*)ipcMessage->m_message, 1000, "%s: %s\n", e.what(), Os::translateOsError(osError).c_str());
        } else {
            snprintf((char*)ipcMessage->m_message, 1000, "%s\n", e.what());
        }

        if (childHandle) {
            //We were able to start child process, but furhter setup failed.
            //Kill the child ASAP.
#ifdef _WIN32
            TerminateProcess(childHandle, EXIT_FAILURE);
#else
            kill(childHandle, SIGPIPE);
#endif
            childHandle = 0;
        }

        ipcMessage->m_ok = false;
    }

    Os::info("%s", ipcMessage->m_message);

    std::string shmemName = Os::getEnv("dgl_loader_shmem");
    if (shmemName.length()) {
        boost::interprocess::shared_memory_object shobj(boost::interprocess::open_only, shmemName.c_str(), boost::interprocess::read_write);
        boost::interprocess::mapped_region region(shobj, boost::interprocess::read_write);
        std::memset(region.get_address(), 0, region.get_size());
        memcpy(region.get_address(), ipcMessage, sizeof(IPCMessage));
    }

    std::string semaphore = Os::getEnv("dgl_loader_semaphore");
    if (semaphore.length()) {
        boost::interprocess::named_semaphore sem(boost::interprocess::open_only, semaphore.c_str());
        sem.post();
    }

    if (!ipcMessage->m_ok) {
        assert(!childHandle);

        //failure - we were not able to setup properly exit code.
        //The failure reason is passed via IPC. Parrent process should check IPC data to distinguish
        //loader failure vs. non-zero exit code from loader's child.
        return EXIT_FAILURE;
    } else {

        //Wait for child process to exit. Return child process exit code.
#ifdef _WIN32
        DWORD exitCode;
        WaitForSingleObject( childHandle, INFINITE );
        GetExitCodeProcess( childHandle, &exitCode);
        return exitCode;
#else
        int exitCode;
        waitpid(childHandle, &exitCode, 0);
        return WEXITSTATUS(exitCode);
#endif
    }
}
<|endoftext|>