<commit_before>/** \brief Utility for augmenting MARC records with links to a local full-text database.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <vector>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <kchashdb.h>
#include "Compiler.h"
#include "DbConnection.h"
#include "DirectoryEntry.h"
#include "ExecUtil.h"
#include "FileLocker.h"
#include "FileUtil.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "PdfUtil.h"
#include "SmartDownloader.h"
#include "SqlUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"
#include "VuFind.h"
#include "XmlWriter.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " file_offset counter marc_input marc_output full_text_db\n\n"
              << "       file_offset  Where to start reading a MARC data set from in marc_input.\n\n";
    std::exit(EXIT_FAILURE);
}

bool GetDocumentAndMediaType(const std::string &url, const unsigned timeout,
                             std::string * const document, std::string * const media_type)
{
    if (not SmartDownload(url, timeout, document)) {
        std::cerr << "Failed to download the document for " << url << " (timeout: " << timeout << " sec)\n";
        return false;
    }

    *media_type = MediaTypeUtil::GetMediaType(*document, /* auto_simplify = */ false);
    if (media_type->empty())
        return false;

    return true;
}

static std::map<std::string, std::string> marc_to_tesseract_language_codes_map {
    { "fre", "fra" },
    { "eng", "eng" },
    { "ger", "deu" },
    { "ita", "ita" },
    { "dut", "nld" },
    { "swe", "swe" },
    { "dan", "dan" },
    { "nor", "nor" },
    { "rus", "rus" },
    { "fin", "fin" },
    { "por", "por" },
    { "pol", "pol" },
    { "slv", "slv" },
    { "hun", "hun" },
    { "cze", "ces" },
    { "bul", "bul" },
};

std::string GetTesseractLanguageCode(const MarcUtil::Record &record) {
    const auto map_iter(marc_to_tesseract_language_codes_map.find(
        record.getLanguageCode()));
    return (map_iter == marc_to_tesseract_language_codes_map.cend()) ? "" : map_iter->second;
}

bool GetTextFromImagePDF(const std::string &document, const std::string &media_type, const std::string &original_url,
			 const MarcUtil::Record &record, const std::string &pdf_images_script, std::string * const extracted_text)
{
    extracted_text->clear();

    if (not StringUtil::StartsWith(media_type, "application/pdf") or not PdfDocContainsNoText(document))
        return false;

    std::cerr << "Found a PDF w/ no text.\n";

    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &input_filename(auto_temp_file.getFilePath());
    if (not FileUtil::WriteString(input_filename, document))
        Error("failed to write the PDF to a temp file!");

    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &output_filename(auto_temp_file2.getFilePath());
    const std::string language_code(GetTesseractLanguageCode(record));
    static constexpr unsigned TIMEOUT(60); // in seconds
    if (ExecUtil::Exec(pdf_images_script, { input_filename, output_filename, language_code }, "", "", "", TIMEOUT)
	!= 0)
    {
        Warning("failed to execute conversion script \"" + pdf_images_script + "\" w/in "
                + std::to_string(TIMEOUT) + " seconds ! (original Url: " + original_url + ")");
        return false;
    }

    std::string plain_text;
    if (not ReadFile(output_filename, extracted_text))
        Error("failed to read OCR output!");

    if (extracted_text->empty())
        std::cerr << "Warning: OCR output is empty!\n";
    else
        std::cerr << "Whoohoo, got OCR'ed text.\n";

    return true;
}

// Checks subfields "3" and "z" to see if they start w/ "Rezension".
bool IsProbablyAReview(const Subfields &subfields) {
    const auto _3_begin_end(subfields.getIterators('3'));
    if (_3_begin_end.first != _3_begin_end.second) {
        if (StringUtil::StartsWith(_3_begin_end.first->second, "Rezension"))
            return true;
    } else {
        const auto z_begin_end(subfields.getIterators('z'));
        if (z_begin_end.first != z_begin_end.second
            and StringUtil::StartsWith(z_begin_end.first->second, "Rezension"))
            return true;
    }

    return false;
}

/** Writes "media_type" and "document" to "db" and returns the unique key that was generated for the write. */
std::string FileLockedWriteDocumentWithMediaType(const std::string &media_type, const std::string &document,
                                                 const std::string &db_filename)
{
    FileLocker file_locker(db_filename, FileLocker::WRITE_ONLY);

    kyotocabinet::HashDB db;
    if (not db.open(db_filename, kyotocabinet::HashDB::OWRITER))
        Error("Failed to open database \"" + db_filename + "\" for writing ("
              + std::string(db.error().message()) + ")!");

    const std::string key(std::to_string(db.count() + 1));
    if (not db.add(key, "Content-type: " + media_type + "\r\n\r\n" + document))
        Error("Failed to add key/value pair to database \"" + db_filename + "\" ("
              + std::string(db.error().message()) + ")!");

    return key;
}

bool GetExtractedTextFromDatabase(DbConnection * const db_connection, const std::string &url, const std::string &document,
				  std::string * const extracted_text)
{
    const std::string QUERY("SELECT hash,full_text FROM full_text_cache WHERE url=\"" + url + "\"");
    if (not db_connection->query(QUERY))
	throw std::runtime_error("Query \"" + QUERY + "\" failed because: " + db_connection->getLastErrorMessage());

    DbResultSet result_set(db_connection->getLastResultSet());
    if (result_set.empty())
	return false;

    assert(result_set.size() == 1);
    DbRow row(result_set.getNextRow());

    const std::string hash(StringUtil::ToHexString(StringUtil::Sha1(document)));
    if (unlikely(hash != row["hash"]))
	return false; // The document must have changed!

    *extracted_text = row["full_text"];

    // Update the timestap:
    const time_t now(std::time(nullptr));
    const std::string current_datetime(SqlUtil::TimeTToDatetime(now));
    const std::string UPDATE_STMT("UPDATE full_text_cache SET last_used=\"" + current_datetime + "\" WHERE url=\"" + url + "\"");
    if (not db_connection->query(UPDATE_STMT))
	throw std::runtime_error("Query \"" + UPDATE_STMT + "\" failed because: " + db_connection->getLastErrorMessage());

    return true;
}

// Returns true if text has been successfully extrcated, else false.
bool ProcessRecord(File * const input, const std::string &marc_output_filename,
		   const std::string &pdf_images_script, const std::string &db_filename)
{
    MarcUtil::Record record = MarcUtil::Record::XmlFactory(input);

    ssize_t _856_index(record.getFieldIndex("856"));
    if (_856_index == -1)
	Error("no 856 tag found!");

    constexpr unsigned PER_DOC_TIMEOUT(20);
    bool succeeded(false);

    const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
    const std::vector<std::string> &fields(record.getFields());
    const ssize_t dir_entry_count(static_cast<ssize_t>(dir_entries.size()));
    for (/* Empty! */; _856_index < dir_entry_count and dir_entries[_856_index].getTag() == "856"; ++_856_index) {
        Subfields subfields(fields[_856_index]);
        const auto u_begin_end(subfields.getIterators('u'));
        if (u_begin_end.first == u_begin_end.second) // No subfield 'u'.
            continue;

        if (IsProbablyAReview(subfields))
            continue;

        std::string document, media_type;
        const std::string url(u_begin_end.first->second);
        if (not GetDocumentAndMediaType(url, PER_DOC_TIMEOUT, &document, &media_type))
            continue;

	std::string mysql_url;
	VuFind::GetMysqlURL(&mysql_url);
	DbConnection db_connection(mysql_url);

        std::string extracted_text, key;
	if (GetExtractedTextFromDatabase(&db_connection, url, document, &extracted_text))
            key = FileLockedWriteDocumentWithMediaType("text/plain", extracted_text, db_filename);
	else if (GetTextFromImagePDF(document, media_type, url, record, pdf_images_script, &extracted_text)) {
            key = FileLockedWriteDocumentWithMediaType("text/plain", extracted_text, db_filename);
            const std::string hash(StringUtil::ToHexString(StringUtil::Sha1(document)));
	    const time_t now(std::time(nullptr));
	    const std::string current_datetime(SqlUtil::TimeTToDatetime(now));
	    const std::string INSERT_STMT("REPLACE INTO full_text_cache SET url=\"" + url + "\", hash=\"" + hash
					  + "\", full_text=\"" + SqlUtil::EscapeBlob(&extracted_text)
					  + "\", last_used=\"" + current_datetime + "\"");
	    if (not db_connection.query(INSERT_STMT))
		throw std::runtime_error("Query \"" + INSERT_STMT + "\" failed because: " + db_connection.getLastErrorMessage());
        } else
            key = FileLockedWriteDocumentWithMediaType(media_type, document, db_filename);

        subfields.addSubfield('e', "http://localhost/cgi-bin/full_text_lookup?id=" + key);
        const std::string new_856_field(subfields.toString());
        record.updateField(_856_index, new_856_field);

	succeeded = true;
    }

    // Safely append the modified MARC data to the MARC output file:
    FileLocker file_locker(marc_output_filename, FileLocker::WRITE_ONLY);
    File marc_output(marc_output_filename, "ab");
    if (not marc_output)
        Error("can't open \"" + marc_output_filename + "\" for appending!");
    XmlWriter xml_writer(&marc_output);
    record.write(&xml_writer);

    return succeeded;
}

const std::string BASH_HELPER("pdf_images_to_text.sh");

std::string GetPathToPdfImagesScript(const char * const argv0) {
    #pragma GCC diagnostic ignored "-Wvla"
    char path[std::strlen(argv0) + 1];
    #pragma GCC diagnostic warning "-Wvla"
    std::strcpy(path, argv0);
    const std::string pdf_images_script_path(ExecUtil::Which(BASH_HELPER));
    if (::access(pdf_images_script_path.c_str(), X_OK) != 0)
        Error("can't execute \"" + pdf_images_script_path + "\"!");
    return pdf_images_script_path;
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
	Usage();

    long offset;
    if (not StringUtil::ToNumber(argv[1], &offset))
	Error("file offset must be a number!");

    const std::string marc_input_filename(argv[2]);
    File marc_input(marc_input_filename, "rm");
    if (not marc_input)
        Error("can't open \"" + marc_input_filename + "\" for reading!");

    const std::string marc_output_filename(argv[3]);

    if (not marc_input.seek(offset, SEEK_SET))
	Error("failed to position " + marc_input_filename + " at offset " + std::to_string(offset)
	      + "! (" + std::to_string(errno) + ")");

    try {
        return ProcessRecord(&marc_input, marc_output_filename, GetPathToPdfImagesScript(argv[0]), argv[4])
	       ? EXIT_SUCCESS : EXIT_FAILURE;
    } catch (const std::exception &e) {
        Error("Caught exception: " + std::string(e.what()));
    }
}
<commit_msg>Better error output when we catch an exception.<commit_after>/** \brief Utility for augmenting MARC records with links to a local full-text database.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015 Universitätsbiblothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <memory>
#include <vector>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <kchashdb.h>
#include "Compiler.h"
#include "DbConnection.h"
#include "DirectoryEntry.h"
#include "ExecUtil.h"
#include "FileLocker.h"
#include "FileUtil.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "PdfUtil.h"
#include "SmartDownloader.h"
#include "SqlUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"
#include "VuFind.h"
#include "XmlWriter.h"

static void Usage() __attribute__((noreturn));

static void Usage() {
    std::cerr << "Usage: " << ::progname << " file_offset counter marc_input marc_output full_text_db\n\n"
              << "       file_offset  Where to start reading a MARC data set from in marc_input.\n\n";
    std::exit(EXIT_FAILURE);
}

bool GetDocumentAndMediaType(const std::string &url, const unsigned timeout,
                             std::string * const document, std::string * const media_type)
{
    if (not SmartDownload(url, timeout, document)) {
        std::cerr << "Failed to download the document for " << url << " (timeout: " << timeout << " sec)\n";
        return false;
    }

    *media_type = MediaTypeUtil::GetMediaType(*document, /* auto_simplify = */ false);
    if (media_type->empty())
        return false;

    return true;
}

static std::map<std::string, std::string> marc_to_tesseract_language_codes_map {
    { "fre", "fra" },
    { "eng", "eng" },
    { "ger", "deu" },
    { "ita", "ita" },
    { "dut", "nld" },
    { "swe", "swe" },
    { "dan", "dan" },
    { "nor", "nor" },
    { "rus", "rus" },
    { "fin", "fin" },
    { "por", "por" },
    { "pol", "pol" },
    { "slv", "slv" },
    { "hun", "hun" },
    { "cze", "ces" },
    { "bul", "bul" },
};

std::string GetTesseractLanguageCode(const MarcUtil::Record &record) {
    const auto map_iter(marc_to_tesseract_language_codes_map.find(
        record.getLanguageCode()));
    return (map_iter == marc_to_tesseract_language_codes_map.cend()) ? "" : map_iter->second;
}

bool GetTextFromImagePDF(const std::string &document, const std::string &media_type, const std::string &original_url,
			 const MarcUtil::Record &record, const std::string &pdf_images_script, std::string * const extracted_text)
{
    extracted_text->clear();

    if (not StringUtil::StartsWith(media_type, "application/pdf") or not PdfDocContainsNoText(document))
        return false;

    std::cerr << "Found a PDF w/ no text.\n";

    const FileUtil::AutoTempFile auto_temp_file;
    const std::string &input_filename(auto_temp_file.getFilePath());
    if (not FileUtil::WriteString(input_filename, document))
        Error("failed to write the PDF to a temp file!");

    const FileUtil::AutoTempFile auto_temp_file2;
    const std::string &output_filename(auto_temp_file2.getFilePath());
    const std::string language_code(GetTesseractLanguageCode(record));
    static constexpr unsigned TIMEOUT(60); // in seconds
    if (ExecUtil::Exec(pdf_images_script, { input_filename, output_filename, language_code }, "", "", "", TIMEOUT)
	!= 0)
    {
        Warning("failed to execute conversion script \"" + pdf_images_script + "\" w/in "
                + std::to_string(TIMEOUT) + " seconds ! (original Url: " + original_url + ")");
        return false;
    }

    std::string plain_text;
    if (not ReadFile(output_filename, extracted_text))
        Error("failed to read OCR output!");

    if (extracted_text->empty())
        std::cerr << "Warning: OCR output is empty!\n";
    else
        std::cerr << "Whoohoo, got OCR'ed text.\n";

    return true;
}

// Checks subfields "3" and "z" to see if they start w/ "Rezension".
bool IsProbablyAReview(const Subfields &subfields) {
    const auto _3_begin_end(subfields.getIterators('3'));
    if (_3_begin_end.first != _3_begin_end.second) {
        if (StringUtil::StartsWith(_3_begin_end.first->second, "Rezension"))
            return true;
    } else {
        const auto z_begin_end(subfields.getIterators('z'));
        if (z_begin_end.first != z_begin_end.second
            and StringUtil::StartsWith(z_begin_end.first->second, "Rezension"))
            return true;
    }

    return false;
}

/** Writes "media_type" and "document" to "db" and returns the unique key that was generated for the write. */
std::string DbLockedWriteDocumentWithMediaType(const std::string &media_type, const std::string &document,
                                                 const std::string &db_filename)
{
    FileLocker file_locker(db_filename, FileLocker::WRITE_ONLY);

    kyotocabinet::HashDB db;
    if (not db.open(db_filename, kyotocabinet::HashDB::OWRITER))
        Error("Failed to open database \"" + db_filename + "\" for writing ("
              + std::string(db.error().message()) + ")!");

    const std::string key(std::to_string(db.count() + 1));
    if (not db.add(key, "Content-type: " + media_type + "\r\n\r\n" + document))
        Error("Failed to add key/value pair to database \"" + db_filename + "\" ("
              + std::string(db.error().message()) + ")!");

    return key;
}

bool GetExtractedTextFromDatabase(DbConnection * const db_connection, const std::string &url, const std::string &document,
				  std::string * const extracted_text)
{
    const std::string QUERY("SELECT hash,full_text FROM full_text_cache WHERE url=\"" + url + "\"");
    if (not db_connection->query(QUERY))
	throw std::runtime_error("Query \"" + QUERY + "\" failed because: " + db_connection->getLastErrorMessage());

    DbResultSet result_set(db_connection->getLastResultSet());
    if (result_set.empty())
	return false;

    assert(result_set.size() == 1);
    DbRow row(result_set.getNextRow());

    const std::string hash(StringUtil::ToHexString(StringUtil::Sha1(document)));
    if (unlikely(hash != row["hash"]))
	return false; // The document must have changed!

    *extracted_text = row["full_text"];

    // Update the timestap:
    const time_t now(std::time(nullptr));
    const std::string current_datetime(SqlUtil::TimeTToDatetime(now));
    const std::string UPDATE_STMT("UPDATE full_text_cache SET last_used=\"" + current_datetime + "\" WHERE url=\"" + url + "\"");
    if (not db_connection->query(UPDATE_STMT))
	throw std::runtime_error("Query \"" + UPDATE_STMT + "\" failed because: " + db_connection->getLastErrorMessage());

    return true;
}

// Returns true if text has been successfully extrcated, else false.
bool ProcessRecord(File * const input, const std::string &marc_output_filename,
		   const std::string &pdf_images_script, const std::string &db_filename)
{
    MarcUtil::Record record = MarcUtil::Record::XmlFactory(input);

    ssize_t _856_index(record.getFieldIndex("856"));
    if (_856_index == -1)
	Error("no 856 tag found!");

    constexpr unsigned PER_DOC_TIMEOUT(20);
    bool succeeded(false);

    const std::vector<DirectoryEntry> &dir_entries(record.getDirEntries());
    const std::vector<std::string> &fields(record.getFields());
    const ssize_t dir_entry_count(static_cast<ssize_t>(dir_entries.size()));
    for (/* Empty! */; _856_index < dir_entry_count and dir_entries[_856_index].getTag() == "856"; ++_856_index) {
        Subfields subfields(fields[_856_index]);
        const auto u_begin_end(subfields.getIterators('u'));
        if (u_begin_end.first == u_begin_end.second) // No subfield 'u'.
            continue;

        if (IsProbablyAReview(subfields))
            continue;

        std::string document, media_type;
        const std::string url(u_begin_end.first->second);
        if (not GetDocumentAndMediaType(url, PER_DOC_TIMEOUT, &document, &media_type))
            continue;

	std::string mysql_url;
	VuFind::GetMysqlURL(&mysql_url);
	DbConnection db_connection(mysql_url);

        std::string extracted_text, key;
	if (GetExtractedTextFromDatabase(&db_connection, url, document, &extracted_text))
            key = DbLockedWriteDocumentWithMediaType("text/plain", extracted_text, db_filename);
	else if (GetTextFromImagePDF(document, media_type, url, record, pdf_images_script, &extracted_text)) {
            key = DbLockedWriteDocumentWithMediaType("text/plain", extracted_text, db_filename);
            const std::string hash(StringUtil::ToHexString(StringUtil::Sha1(document)));
	    const time_t now(std::time(nullptr));
	    const std::string current_datetime(SqlUtil::TimeTToDatetime(now));
	    const std::string INSERT_STMT("REPLACE INTO full_text_cache SET url=\"" + url + "\", hash=\"" + hash
					  + "\", full_text=\"" + SqlUtil::EscapeBlob(&extracted_text)
					  + "\", last_used=\"" + current_datetime + "\"");
	    if (not db_connection.query(INSERT_STMT))
		throw std::runtime_error("Query \"" + INSERT_STMT + "\" failed because: " + db_connection.getLastErrorMessage());
        } else
            key = DbLockedWriteDocumentWithMediaType(media_type, document, db_filename);

        subfields.addSubfield('e', "http://localhost/cgi-bin/full_text_lookup?id=" + key);
        const std::string new_856_field(subfields.toString());
        record.updateField(_856_index, new_856_field);

	succeeded = true;
    }

    // Safely append the modified MARC data to the MARC output file:
    FileLocker file_locker(marc_output_filename, FileLocker::WRITE_ONLY);
    File marc_output(marc_output_filename, "ab");
    if (not marc_output)
        Error("can't open \"" + marc_output_filename + "\" for appending!");
    XmlWriter xml_writer(&marc_output);
    record.write(&xml_writer);

    return succeeded;
}

const std::string BASH_HELPER("pdf_images_to_text.sh");

std::string GetPathToPdfImagesScript(const char * const argv0) {
    #pragma GCC diagnostic ignored "-Wvla"
    char path[std::strlen(argv0) + 1];
    #pragma GCC diagnostic warning "-Wvla"
    std::strcpy(path, argv0);
    const std::string pdf_images_script_path(ExecUtil::Which(BASH_HELPER));
    if (::access(pdf_images_script_path.c_str(), X_OK) != 0)
        Error("can't execute \"" + pdf_images_script_path + "\"!");
    return pdf_images_script_path;
}

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 5)
	Usage();

    long offset;
    if (not StringUtil::ToNumber(argv[1], &offset))
	Error("file offset must be a number!");

    const std::string marc_input_filename(argv[2]);
    File marc_input(marc_input_filename, "rm");
    if (not marc_input)
        Error("can't open \"" + marc_input_filename + "\" for reading!");

    const std::string marc_output_filename(argv[3]);

    if (not marc_input.seek(offset, SEEK_SET))
	Error("failed to position " + marc_input_filename + " at offset " + std::to_string(offset)
	      + "! (" + std::to_string(errno) + ")");

    try {
        return ProcessRecord(&marc_input, marc_output_filename, GetPathToPdfImagesScript(argv[0]), argv[4])
	       ? EXIT_SUCCESS : EXIT_FAILURE;
    } catch (const std::exception &e) {
        Error("While reading \"" + marc_input_filename + "\" starting at offset \"" + std::string(argv[1])
	      + "\", caught exception: " + std::string(e.what()));
    }
}
<|endoftext|>


<commit_before>//============================================================================
// MCKL/include/mckl/core/particle.hpp
//----------------------------------------------------------------------------
// MCKL: Monte Carlo Kernel Library
//----------------------------------------------------------------------------
// Copyright (c) 2013-2017, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#ifndef MCKL_CORE_PARTICLE_HPP
#define MCKL_CORE_PARTICLE_HPP

#include <mckl/internal/common.hpp>
#include <mckl/core/iterator.hpp>
#include <mckl/core/weight.hpp>
#include <mckl/random/rng_set.hpp>
#include <mckl/random/seed.hpp>

namespace mckl
{

template <typename>
class Particle;

/// \brief A thin wrapper over a complete Particle
/// \ingroup Core
template <typename T>
class ParticleIndexBase
{
    public:
    ParticleIndexBase() : pptr_(nullptr), i_(0) {}

    ParticleIndexBase(typename Particle<T>::size_type i, Particle<T> *pptr)
        : pptr_(pptr), i_(i)
    {
    }

    Particle<T> &particle() const { return *pptr_; }

    Particle<T> *particle_ptr() const { return pptr_; }

    T &state() const { return pptr_->state(); }

    typename Particle<T>::size_type i() const { return i_; }

    typename Particle<T>::rng_type &rng() const { return pptr_->rng(i_); }

    private:
    Particle<T> *pptr_;
    typename Particle<T>::size_type i_;
}; // class ParticleIndexBase

/// \brief ParticleIndex base class trait
/// \ingroup Traits
MCKL_DEFINE_TYPE_TEMPLATE_DISPATCH_TRAIT(
    ParticleIndexBaseType, particle_index_type, ParticleIndexBase)

/// \brief A thin wrapper over a complete Particle
/// \ingroup Core
///
/// \details
/// This class also serves as an random access iterator
template <typename T>
class ParticleIndex final : public ParticleIndexBaseType<T>
{
    public:
    using difference_type =
        std::make_signed_t<typename Particle<T>::size_type>;
    using value_type = ParticleIndex;
    using pointer = const ParticleIndex *;
    using reference = const ParticleIndex &;
    using iterator_category = std::random_access_iterator_tag;

    ParticleIndex() = default;

    ParticleIndex(typename Particle<T>::size_type i, Particle<T> *pptr)
        : ParticleIndexBaseType<T>(i, pptr)
    {
    }

    /// \brief Dereference operator returns a reference to the index itself
    reference operator*() const noexcept { return *this; }

    /// \brief Member selection operator returns a pointer the index itself
    pointer operator->() const noexcept { return this; }

    /// \brief Subscript operator return a new index
    template <typename IntType>
    ParticleIndex operator[](IntType n)
    {
        return ParticleIndex(static_cast<typename Particle<T>::size_type>(
                                 static_cast<difference_type>(this->i()) +
                                 static_cast<difference_type>(n)),
            this->particle_ptr());
    }

    friend bool operator==(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() == idx2.i();
    }

    friend bool operator!=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() != idx2.i();
    }

    friend bool operator<(const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() < idx2.i();
    }

    friend bool operator>(const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() > idx2.i();
    }

    friend bool operator<=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() <= idx2.i();
    }

    friend bool operator>=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() >= idx2.i();
    }

    friend ParticleIndex &operator++(ParticleIndex &idx)
    {
        return idx = ParticleIndex(idx.i() + 1, idx.particle_ptr());
    }

    friend ParticleIndex operator++(ParticleIndex &idx, int)
    {
        ParticleIndex ret(idx);
        ++idx;

        return ret;
    }

    friend ParticleIndex &operator--(ParticleIndex &idx)
    {
        return idx = ParticleIndex(idx.i() - 1, idx.particle_ptr());
    }

    friend ParticleIndex operator--(ParticleIndex &idx, int)
    {
        ParticleIndex ret(idx);
        --idx;

        return ret;
    }

    template <typename IntType>
    friend ParticleIndex operator+(const ParticleIndex &idx, IntType n)
    {
        return ParticleIndex(static_cast<typename Particle<T>::size_type>(
                                 static_cast<difference_type>(idx.i()) +
                                 static_cast<difference_type>(n)),
            idx.particle_ptr());
    }

    template <typename IntType>
    friend ParticleIndex operator+(IntType n, const ParticleIndex &idx)
    {
        return idx + n;
    }

    template <typename IntType>
    friend ParticleIndex operator-(const ParticleIndex &idx, IntType n)
    {
        return idx + (-static_cast<difference_type>(n));
    }

    template <typename IntType>
    friend ParticleIndex &operator+=(ParticleIndex &idx, IntType n)
    {
        return idx = idx + n;
    }

    template <typename IntType>
    friend ParticleIndex &operator-=(ParticleIndex &idx, IntType n)
    {
        return idx = idx - n;
    }

    friend difference_type operator-(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Substract two ParticleIndex objects from two different "
            "particle systems");

        return static_cast<difference_type>(idx1.i()) -
            static_cast<difference_type>(idx2.i());
    }
}; // class ParticleIndex

/// \brief Range of ParticleIndex
/// \ingroup Core
template <typename T>
class ParticleRange : public Range<ParticleIndex<T>>
{
    public:
    using Range<ParticleIndex<T>>::Range;

    Particle<T> &particle() const { return this->begin()->particle(); }

    Particle<T> *particle_ptr() const { return this->begin()->particle_ptr(); }

    typename Particle<T>::size_type ibegin() const
    {
        return this->begin()->i();
    }

    typename Particle<T>::size_type iend() const { return this->end()->i(); }
}; // class ParticleRange

/// \brief Particle system
/// \ingroup Core
///
/// \tparam T The sample collection type
///
/// \details
/// A collection of particles is formed by,
/// * The collection of samples
/// * The weights of samples
/// * A collection of RNG engines for parallel use
template <typename T>
class Particle
{
    public:
    using size_type = SizeType<T>;
    using state_type = T;
    using weight_type = WeightType<T>;
    using rng_set_type = RNGSetType<T>;
    using rng_type = typename rng_set_type::rng_type;

    /// \brief Default constructor
    Particle()
        : state_(0)
        , weight_(0)
        , rng_set_(0)
        , rng_(Seed<rng_type>::instance().get())
    {
    }

    /// \brief Construct a particle system
    ///
    /// \details
    /// All arguments are passed down to the constructor of `T`
    template <typename... Args>
    explicit Particle(size_type N, Args &&... args)
        : state_(N, std::forward<Args>(args)...)
        , weight_(static_cast<SizeType<weight_type>>(N))
        , rng_set_(static_cast<SizeType<rng_set_type>>(N))
        , rng_(Seed<rng_type>::instance().get())
    {
    }

    /// \brief Clone the Particle except the RNG engines
    Particle clone() const
    {
        Particle particle(*this);
        particle.rng_set_.reset();
        particle.rng_.seed(Seed<rng_type>::instance().get());

        return particle;
    }

    /// \brief The number of particles
    size_type size() const { return state_.size(); }

    /// \brief Resize by selecting according to user supplied index vector
    ///
    /// \param n The new sample size
    /// \param index N-vector of parent indices
    template <typename InputIter>
    void select(size_type n, InputIter index)
    {
        state_.select(n, index);
        weight_.resize(static_cast<SizeType<weight_type>>(n));
        weight_.set_equal();
        rng_set_.resize(static_cast<SizeType<rng_set_type>>(n));
    }

    /// \brief Read and write access to the state collection object
    state_type &state() { return state_; }

    /// \brief Read only access to the state collection object
    const state_type &state() const { return state_; }

    /// \brief Read and write access to the weight collection object
    weight_type &weight() { return weight_; }

    /// \brief Read only access to the weight collection object
    const weight_type &weight() const { return weight_; }

    /// \brief Read and write access to the RNG collection object
    rng_set_type &rng_set() { return rng_set_; }

    /// \brief Read only access to the RNG collection object
    const rng_set_type &rng_set() const { return rng_set_; }

    /// \brief Get an (parallel) RNG stream for a given particle
    rng_type &rng(size_type i)
    {
        return rng_set_[static_cast<std::size_t>(i)];
    }

    /// \brief Get an (parallel) RNG stream for a given particle
    const rng_type &rng(size_type i) const
    {
        return rng_set_[static_cast<std::size_t>(i)];
    }

    /// \brief Get the (sequential) RNG used stream for resampling
    rng_type &rng() { return rng_; }

    /// \brief Get the (sequential) RNG used stream for resampling
    const rng_type &rng() const { return rng_; }

    /// \brief Get a ParticleIndex<T> object for the i-th particle
    ParticleIndex<T> at(size_type i)
    {
        runtime_assert<std::out_of_range>(
            i <= size(), "**Particle::at** index out of range");

        return operator[](i);
    }

    /// \brief Get a ParticleIndex<T> object for the i-th particle
    ParticleIndex<T> operator[](size_type i)
    {
        return ParticleIndex<T>(i, this);
    }

    /// \brief Get a ParticleIndex<T> object for the first particle
    ParticleIndex<T> begin() { return operator[](0); }

    /// \brief Get a ParticleIndex<T> object for one pass the last particle
    ParticleIndex<T> end() { return operator[](size()); }

    /// \brief Get a Range<ParticleItrator<T>> object of all particles
    ParticleRange<T> range(size_type grainsize = 1)
    {
        return ParticleRange<T>(begin(), end(), grainsize);
    }

    /// \brief Get a ParticleRange<T> object given a range denoted by integral
    /// values
    ParticleRange<T> range(
        size_type ibegin, size_type iend, size_type grainsize = 1)
    {
        return ParticleRange<T>(operator[](ibegin), operator[](iend),
            grainsize);
    }

    private:
    state_type state_;
    weight_type weight_;
    rng_set_type rng_set_;
    rng_type rng_;
}; // class Particle

} // namespace mckl

#endif // MCKL_CORE_PARTICLE_HPP
<commit_msg>defualt particle constructor use default constructor of T<commit_after>//============================================================================
// MCKL/include/mckl/core/particle.hpp
//----------------------------------------------------------------------------
// MCKL: Monte Carlo Kernel Library
//----------------------------------------------------------------------------
// Copyright (c) 2013-2017, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#ifndef MCKL_CORE_PARTICLE_HPP
#define MCKL_CORE_PARTICLE_HPP

#include <mckl/internal/common.hpp>
#include <mckl/core/iterator.hpp>
#include <mckl/core/weight.hpp>
#include <mckl/random/rng_set.hpp>
#include <mckl/random/seed.hpp>

namespace mckl
{

template <typename>
class Particle;

/// \brief A thin wrapper over a complete Particle
/// \ingroup Core
template <typename T>
class ParticleIndexBase
{
    public:
    ParticleIndexBase() : pptr_(nullptr), i_(0) {}

    ParticleIndexBase(typename Particle<T>::size_type i, Particle<T> *pptr)
        : pptr_(pptr), i_(i)
    {
    }

    Particle<T> &particle() const { return *pptr_; }

    Particle<T> *particle_ptr() const { return pptr_; }

    T &state() const { return pptr_->state(); }

    typename Particle<T>::size_type i() const { return i_; }

    typename Particle<T>::rng_type &rng() const { return pptr_->rng(i_); }

    private:
    Particle<T> *pptr_;
    typename Particle<T>::size_type i_;
}; // class ParticleIndexBase

/// \brief ParticleIndex base class trait
/// \ingroup Traits
MCKL_DEFINE_TYPE_TEMPLATE_DISPATCH_TRAIT(
    ParticleIndexBaseType, particle_index_type, ParticleIndexBase)

/// \brief A thin wrapper over a complete Particle
/// \ingroup Core
///
/// \details
/// This class also serves as an random access iterator
template <typename T>
class ParticleIndex final : public ParticleIndexBaseType<T>
{
    public:
    using difference_type =
        std::make_signed_t<typename Particle<T>::size_type>;
    using value_type = ParticleIndex;
    using pointer = const ParticleIndex *;
    using reference = const ParticleIndex &;
    using iterator_category = std::random_access_iterator_tag;

    ParticleIndex() = default;

    ParticleIndex(typename Particle<T>::size_type i, Particle<T> *pptr)
        : ParticleIndexBaseType<T>(i, pptr)
    {
    }

    /// \brief Dereference operator returns a reference to the index itself
    reference operator*() const noexcept { return *this; }

    /// \brief Member selection operator returns a pointer the index itself
    pointer operator->() const noexcept { return this; }

    /// \brief Subscript operator return a new index
    template <typename IntType>
    ParticleIndex operator[](IntType n)
    {
        return ParticleIndex(static_cast<typename Particle<T>::size_type>(
                                 static_cast<difference_type>(this->i()) +
                                 static_cast<difference_type>(n)),
            this->particle_ptr());
    }

    friend bool operator==(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() == idx2.i();
    }

    friend bool operator!=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() != idx2.i();
    }

    friend bool operator<(const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() < idx2.i();
    }

    friend bool operator>(const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() > idx2.i();
    }

    friend bool operator<=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() <= idx2.i();
    }

    friend bool operator>=(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Compare two ParticleIndex objects from two different particle "
            "systems");

        return idx1.i() >= idx2.i();
    }

    friend ParticleIndex &operator++(ParticleIndex &idx)
    {
        return idx = ParticleIndex(idx.i() + 1, idx.particle_ptr());
    }

    friend ParticleIndex operator++(ParticleIndex &idx, int)
    {
        ParticleIndex ret(idx);
        ++idx;

        return ret;
    }

    friend ParticleIndex &operator--(ParticleIndex &idx)
    {
        return idx = ParticleIndex(idx.i() - 1, idx.particle_ptr());
    }

    friend ParticleIndex operator--(ParticleIndex &idx, int)
    {
        ParticleIndex ret(idx);
        --idx;

        return ret;
    }

    template <typename IntType>
    friend ParticleIndex operator+(const ParticleIndex &idx, IntType n)
    {
        return ParticleIndex(static_cast<typename Particle<T>::size_type>(
                                 static_cast<difference_type>(idx.i()) +
                                 static_cast<difference_type>(n)),
            idx.particle_ptr());
    }

    template <typename IntType>
    friend ParticleIndex operator+(IntType n, const ParticleIndex &idx)
    {
        return idx + n;
    }

    template <typename IntType>
    friend ParticleIndex operator-(const ParticleIndex &idx, IntType n)
    {
        return idx + (-static_cast<difference_type>(n));
    }

    template <typename IntType>
    friend ParticleIndex &operator+=(ParticleIndex &idx, IntType n)
    {
        return idx = idx + n;
    }

    template <typename IntType>
    friend ParticleIndex &operator-=(ParticleIndex &idx, IntType n)
    {
        return idx = idx - n;
    }

    friend difference_type operator-(
        const ParticleIndex &idx1, const ParticleIndex &idx2)
    {
        runtime_assert(idx1.particle_ptr() == idx2.particle_ptr(),
            "Substract two ParticleIndex objects from two different "
            "particle systems");

        return static_cast<difference_type>(idx1.i()) -
            static_cast<difference_type>(idx2.i());
    }
}; // class ParticleIndex

/// \brief Range of ParticleIndex
/// \ingroup Core
template <typename T>
class ParticleRange : public Range<ParticleIndex<T>>
{
    public:
    using Range<ParticleIndex<T>>::Range;

    Particle<T> &particle() const { return this->begin()->particle(); }

    Particle<T> *particle_ptr() const { return this->begin()->particle_ptr(); }

    typename Particle<T>::size_type ibegin() const
    {
        return this->begin()->i();
    }

    typename Particle<T>::size_type iend() const { return this->end()->i(); }
}; // class ParticleRange

/// \brief Particle system
/// \ingroup Core
///
/// \tparam T The sample collection type
///
/// \details
/// A collection of particles is formed by,
/// * The collection of samples
/// * The weights of samples
/// * A collection of RNG engines for parallel use
template <typename T>
class Particle
{
    public:
    using size_type = SizeType<T>;
    using state_type = T;
    using weight_type = WeightType<T>;
    using rng_set_type = RNGSetType<T>;
    using rng_type = typename rng_set_type::rng_type;

    /// \brief Default constructor
    Particle()
        : weight_(0), rng_set_(0), rng_(Seed<rng_type>::instance().get())
    {
    }

    /// \brief Construct a particle system
    ///
    /// \details
    /// All arguments are passed down to the constructor of `T`
    template <typename... Args>
    explicit Particle(size_type N, Args &&... args)
        : state_(N, std::forward<Args>(args)...)
        , weight_(static_cast<SizeType<weight_type>>(N))
        , rng_set_(static_cast<SizeType<rng_set_type>>(N))
        , rng_(Seed<rng_type>::instance().get())
    {
    }

    /// \brief Clone the Particle except the RNG engines
    Particle clone() const
    {
        Particle particle(*this);
        particle.rng_set_.reset();
        particle.rng_.seed(Seed<rng_type>::instance().get());

        return particle;
    }

    /// \brief The number of particles
    size_type size() const { return state_.size(); }

    /// \brief Resize by selecting according to user supplied index vector
    ///
    /// \param n The new sample size
    /// \param index N-vector of parent indices
    template <typename InputIter>
    void select(size_type n, InputIter index)
    {
        state_.select(n, index);
        weight_.resize(static_cast<SizeType<weight_type>>(n));
        weight_.set_equal();
        rng_set_.resize(static_cast<SizeType<rng_set_type>>(n));
    }

    /// \brief Read and write access to the state collection object
    state_type &state() { return state_; }

    /// \brief Read only access to the state collection object
    const state_type &state() const { return state_; }

    /// \brief Read and write access to the weight collection object
    weight_type &weight() { return weight_; }

    /// \brief Read only access to the weight collection object
    const weight_type &weight() const { return weight_; }

    /// \brief Read and write access to the RNG collection object
    rng_set_type &rng_set() { return rng_set_; }

    /// \brief Read only access to the RNG collection object
    const rng_set_type &rng_set() const { return rng_set_; }

    /// \brief Get an (parallel) RNG stream for a given particle
    rng_type &rng(size_type i)
    {
        return rng_set_[static_cast<std::size_t>(i)];
    }

    /// \brief Get an (parallel) RNG stream for a given particle
    const rng_type &rng(size_type i) const
    {
        return rng_set_[static_cast<std::size_t>(i)];
    }

    /// \brief Get the (sequential) RNG used stream for resampling
    rng_type &rng() { return rng_; }

    /// \brief Get the (sequential) RNG used stream for resampling
    const rng_type &rng() const { return rng_; }

    /// \brief Get a ParticleIndex<T> object for the i-th particle
    ParticleIndex<T> at(size_type i)
    {
        runtime_assert<std::out_of_range>(
            i <= size(), "**Particle::at** index out of range");

        return operator[](i);
    }

    /// \brief Get a ParticleIndex<T> object for the i-th particle
    ParticleIndex<T> operator[](size_type i)
    {
        return ParticleIndex<T>(i, this);
    }

    /// \brief Get a ParticleIndex<T> object for the first particle
    ParticleIndex<T> begin() { return operator[](0); }

    /// \brief Get a ParticleIndex<T> object for one pass the last particle
    ParticleIndex<T> end() { return operator[](size()); }

    /// \brief Get a Range<ParticleItrator<T>> object of all particles
    ParticleRange<T> range(size_type grainsize = 1)
    {
        return ParticleRange<T>(begin(), end(), grainsize);
    }

    /// \brief Get a ParticleRange<T> object given a range denoted by integral
    /// values
    ParticleRange<T> range(
        size_type ibegin, size_type iend, size_type grainsize = 1)
    {
        return ParticleRange<T>(operator[](ibegin), operator[](iend),
            grainsize);
    }

    private:
    state_type state_;
    weight_type weight_;
    rng_set_type rng_set_;
    rng_type rng_;
}; // class Particle

} // namespace mckl

#endif // MCKL_CORE_PARTICLE_HPP
<|endoftext|>


<commit_before>//
// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
//

#include "device/gpu/gpudefs.hpp"
#include "device/gpu/gpuprogram.hpp"
#include "device/gpu/gpukernel.hpp"
#include "acl.h"
#include "SCShadersSi.h"
#include "si_ci_vi_merged_offset.h"
#include "si_ci_vi_merged_registers.h"
#include <string>
#include <fstream>
#include <sstream>
#include <iostream>
#include <ctime>

namespace gpu {

bool
NullKernel::siCreateHwInfo(const void* shader, AMUabiAddEncoding& encoding)
{
    static const uint NumSiCsInfos = (70 + 5 + 1 + 32 + 6);
    CALProgramInfoEntry* newInfos;
    uint i = 0;
    uint infoCount = NumSiCsInfos;
    const SC_SI_HWSHADER_CS* cShader = reinterpret_cast<const SC_SI_HWSHADER_CS*>(shader);
    newInfos = new CALProgramInfoEntry[infoCount];
    encoding.progInfos = newInfos;
    if (encoding.progInfos == 0) {
        infoCount = 0;
        return false;
    }
    newInfos[i].address = AMU_ABI_USER_ELEMENT_COUNT;
    newInfos[i].value = cShader->common.userElementCount;
    i++;
    for (unsigned int j = 0; j < cShader->common.userElementCount; j++) {
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD0 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].dataClass;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD1 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].apiSlot;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD2 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].startUserReg;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD3 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].userRegCount;
        i++;
    }

    newInfos[i].address = AMU_ABI_SI_NUM_VGPRS;
    newInfos[i].value = cShader->common.numVgprs;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_SGPRS;
    newInfos[i].value = cShader->common.numSgprs;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_SGPRS_AVAIL;
    newInfos[i].value = SI_sgprs_avail; //512;//options.NumSGPRsAvailable;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_VGPRS_AVAIL;
    newInfos[i].value = SI_vgprs_avail;//options.NumVGPRsAvailable;
    i++;

    newInfos[i].address = AMU_ABI_SI_FLOAT_MODE;
    newInfos[i].value = cShader->common.floatMode;
    i++;
    newInfos[i].address = AMU_ABI_SI_IEEE_MODE;
    newInfos[i].value = cShader->common.bIeeeMode;
    i++;

    newInfos[i].address = AMU_ABI_SI_SCRATCH_SIZE;
    newInfos[i].value = cShader->common.scratchSize;;
    i++;

    newInfos[i].address = mmCOMPUTE_PGM_RSRC2;
    newInfos[i].value = cShader->computePgmRsrc2.u32All;
    i++;

    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_X;
    newInfos[i].value = cShader->numThreadX;
    i++;
    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Y;
    newInfos[i].value = cShader->numThreadY;
    i++;
    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Z;
    newInfos[i].value = cShader->numThreadZ;
    i++;

    newInfos[i].address = AMU_ABI_ORDERED_APPEND_ENABLE;
    newInfos[i].value = cShader->bOrderedAppendEnable;
    i++;

    newInfos[i].address = AMU_ABI_RAT_OP_IS_USED;
    newInfos[i].value = cShader->common.uavResourceUsage[0];
    i++;

    for (unsigned int j = 0; j < ((SC_MAX_UAV + 31) / 32); j++) {
        newInfos[i].address = AMU_ABI_UAV_RESOURCE_MASK_0 + j;
        newInfos[i].value = cShader->common.uavResourceUsage[j];
        i++;
    }

    newInfos[i].address = AMU_ABI_NUM_WAVEFRONT_PER_SIMD;      // Setting the same as for scWrapR800Info
    newInfos[i].value = 1;
    i++;

    newInfos[i].address = AMU_ABI_WAVEFRONT_SIZE;
    newInfos[i].value = nullDev().hwInfo()->simdWidth_ * 4;   //options.WavefrontSize;
    i++;

    newInfos[i].address = AMU_ABI_LDS_SIZE_AVAIL;
    newInfos[i].value = SI_ldssize_avail;    //options.LDSSize;
    i++;

    COMPUTE_PGM_RSRC2 computePgmRsrc2;
    computePgmRsrc2.u32All = cShader->computePgmRsrc2.u32All;

    newInfos[i].address = AMU_ABI_LDS_SIZE_USED;
    newInfos[i].value = 64 * 4 * computePgmRsrc2.bits.LDS_SIZE;
    i++;

    infoCount = i;
    assert((i + 4 * (16 - cShader->common.userElementCount)) == NumSiCsInfos);
    encoding.progInfosCount    = infoCount;

    CALUavMask uavMask;
    memcpy(uavMask.mask, cShader->common.uavResourceUsage, sizeof(CALUavMask));
    encoding.uavMask    = uavMask;
    encoding.textData   = HWSHADER_Get(cShader, common.hShaderMemHandle);
    encoding.textSize   = cShader->common.codeLenInByte;
    instructionCnt_     = encoding.textSize / sizeof(uint32_t);
    encoding.scratchRegisterCount  = cShader->common.scratchSize;
    encoding.UAVReturnBufferTotalSize  = 0;

    return true;
}

bool
HSAILKernel::aqlCreateHWInfo(const void* shader, size_t shaderSize)
{
    // Copy the shader_isa into a buffer
    hwMetaData_ = new char[shaderSize];
    if (hwMetaData_ == NULL) {
        return false;
    }
    memcpy(hwMetaData_, shader, shaderSize);

    SC_SI_HWSHADER_CS* siMetaData = reinterpret_cast<SC_SI_HWSHADER_CS*>(hwMetaData_);

    // Code to patch the pointers in the shader object.
    // Must be preferably done in the compiler library
    size_t offset = siMetaData->common.uSizeInBytes;
    if (siMetaData->common.u32PvtDataSizeInBytes > 0) {
        siMetaData->common.pPvtData =
            reinterpret_cast<SC_BYTE *>(
            reinterpret_cast<char *>(siMetaData) + offset);
        offset += siMetaData->common.u32PvtDataSizeInBytes;
    }
    if (siMetaData->common.codeLenInByte > 0) {
        siMetaData->common.hShaderMemHandle =
            reinterpret_cast<char *>(siMetaData) + offset;
        offset += siMetaData->common.codeLenInByte;
    }

    char* headerBaseAddress =
        reinterpret_cast<char*>(siMetaData->common.hShaderMemHandle);
    hsa_ext_code_descriptor_t* hcd =
        reinterpret_cast<hsa_ext_code_descriptor_t*>(headerBaseAddress);
    amd_kernel_code_t* akc = reinterpret_cast<amd_kernel_code_t*>(
        headerBaseAddress + hcd->code.handle);

    address codeStartAddress = reinterpret_cast<address>(akc);
    address codeEndAddress = reinterpret_cast<address>(hcd) + siMetaData->common.codeLenInByte;
    codeSize_ = codeEndAddress - codeStartAddress;
    code_ = new gpu::Memory(dev(), amd::alignUp(codeSize_, gpu::ConstBuffer::VectorSize));

    // Initialize kernel ISA code
    if ((code_ != NULL) && code_->create(Resource::Shader)) {
        address cpuCodePtr = static_cast<address>(code_->map(NULL, Resource::WriteOnly));
        // Copy only amd_kernel_code_t
        memcpy(cpuCodePtr, codeStartAddress, codeSize_);
        code_->unmap(NULL);
    }
    else {
        LogError("Failed to allocate ISA code!");
        return false;
    }
    cpuAqlCode_ = akc;

    assert((akc->workitem_private_segment_byte_size & 3) == 0 &&
        "Scratch must be DWORD aligned");
    workGroupInfo_.scratchRegs_ =
        amd::alignUp(akc->workitem_private_segment_byte_size, 16) / sizeof(uint);
    workGroupInfo_.availableSGPRs_ = dev().gslCtx()->getNumSGPRsAvailable();
    workGroupInfo_.availableVGPRs_ = dev().gslCtx()->getNumVGPRsAvailable();
    workGroupInfo_.preferredSizeMultiple_ = dev().getAttribs().wavefrontSize;
    workGroupInfo_.privateMemSize_ = akc->workitem_private_segment_byte_size;
    workGroupInfo_.localMemSize_ =
    workGroupInfo_.usedLDSSize_ = akc->workgroup_group_segment_byte_size;
    workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count;
    workGroupInfo_.usedStackSize_ = 0;
    workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count;
    workGroupInfo_.wavefrontPerSIMD_ = dev().getAttribs().wavefrontSize;

    return true;
}
} // namespace gpu
<commit_msg>P4 to Git Change 1129400 by kzhuravl@win-kzhuravl-stg-oclhsa on 2015/03/11 00:40:42<commit_after>//
// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
//

#include "device/gpu/gpudefs.hpp"
#include "device/gpu/gpuprogram.hpp"
#include "device/gpu/gpukernel.hpp"
#include "acl.h"
#include "SCShadersSi.h"
#include "si_ci_vi_merged_offset.h"
#include "si_ci_vi_merged_registers.h"
#include <string>
#include <fstream>
#include <sstream>
#include <iostream>
#include <ctime>

namespace gpu {

bool
NullKernel::siCreateHwInfo(const void* shader, AMUabiAddEncoding& encoding)
{
    static const uint NumSiCsInfos = (70 + 5 + 1 + 32 + 6);
    CALProgramInfoEntry* newInfos;
    uint i = 0;
    uint infoCount = NumSiCsInfos;
    const SC_SI_HWSHADER_CS* cShader = reinterpret_cast<const SC_SI_HWSHADER_CS*>(shader);
    newInfos = new CALProgramInfoEntry[infoCount];
    encoding.progInfos = newInfos;
    if (encoding.progInfos == 0) {
        infoCount = 0;
        return false;
    }
    newInfos[i].address = AMU_ABI_USER_ELEMENT_COUNT;
    newInfos[i].value = cShader->common.userElementCount;
    i++;
    for (unsigned int j = 0; j < cShader->common.userElementCount; j++) {
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD0 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].dataClass;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD1 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].apiSlot;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD2 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].startUserReg;
        i++;
        newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD3 + 4*j;
        newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].userRegCount;
        i++;
    }

    newInfos[i].address = AMU_ABI_SI_NUM_VGPRS;
    newInfos[i].value = cShader->common.numVgprs;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_SGPRS;
    newInfos[i].value = cShader->common.numSgprs;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_SGPRS_AVAIL;
    newInfos[i].value = SI_sgprs_avail; //512;//options.NumSGPRsAvailable;
    i++;
    newInfos[i].address = AMU_ABI_SI_NUM_VGPRS_AVAIL;
    newInfos[i].value = SI_vgprs_avail;//options.NumVGPRsAvailable;
    i++;

    newInfos[i].address = AMU_ABI_SI_FLOAT_MODE;
    newInfos[i].value = cShader->common.floatMode;
    i++;
    newInfos[i].address = AMU_ABI_SI_IEEE_MODE;
    newInfos[i].value = cShader->common.bIeeeMode;
    i++;

    newInfos[i].address = AMU_ABI_SI_SCRATCH_SIZE;
    newInfos[i].value = cShader->common.scratchSize;;
    i++;

    newInfos[i].address = mmCOMPUTE_PGM_RSRC2;
    newInfos[i].value = cShader->computePgmRsrc2.u32All;
    i++;

    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_X;
    newInfos[i].value = cShader->numThreadX;
    i++;
    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Y;
    newInfos[i].value = cShader->numThreadY;
    i++;
    newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Z;
    newInfos[i].value = cShader->numThreadZ;
    i++;

    newInfos[i].address = AMU_ABI_ORDERED_APPEND_ENABLE;
    newInfos[i].value = cShader->bOrderedAppendEnable;
    i++;

    newInfos[i].address = AMU_ABI_RAT_OP_IS_USED;
    newInfos[i].value = cShader->common.uavResourceUsage[0];
    i++;

    for (unsigned int j = 0; j < ((SC_MAX_UAV + 31) / 32); j++) {
        newInfos[i].address = AMU_ABI_UAV_RESOURCE_MASK_0 + j;
        newInfos[i].value = cShader->common.uavResourceUsage[j];
        i++;
    }

    newInfos[i].address = AMU_ABI_NUM_WAVEFRONT_PER_SIMD;      // Setting the same as for scWrapR800Info
    newInfos[i].value = 1;
    i++;

    newInfos[i].address = AMU_ABI_WAVEFRONT_SIZE;
    newInfos[i].value = nullDev().hwInfo()->simdWidth_ * 4;   //options.WavefrontSize;
    i++;

    newInfos[i].address = AMU_ABI_LDS_SIZE_AVAIL;
    newInfos[i].value = SI_ldssize_avail;    //options.LDSSize;
    i++;

    COMPUTE_PGM_RSRC2 computePgmRsrc2;
    computePgmRsrc2.u32All = cShader->computePgmRsrc2.u32All;

    newInfos[i].address = AMU_ABI_LDS_SIZE_USED;
    newInfos[i].value = 64 * 4 * computePgmRsrc2.bits.LDS_SIZE;
    i++;

    infoCount = i;
    assert((i + 4 * (16 - cShader->common.userElementCount)) == NumSiCsInfos);
    encoding.progInfosCount    = infoCount;

    CALUavMask uavMask;
    memcpy(uavMask.mask, cShader->common.uavResourceUsage, sizeof(CALUavMask));
    encoding.uavMask    = uavMask;
    encoding.textData   = HWSHADER_Get(cShader, common.hShaderMemHandle);
    encoding.textSize   = cShader->common.codeLenInByte;
    instructionCnt_     = encoding.textSize / sizeof(uint32_t);
    encoding.scratchRegisterCount  = cShader->common.scratchSize;
    encoding.UAVReturnBufferTotalSize  = 0;

    return true;
}

bool
HSAILKernel::aqlCreateHWInfo(const void* shader, size_t shaderSize)
{
    // Copy the shader_isa into a buffer
    hwMetaData_ = new char[shaderSize];
    if (hwMetaData_ == NULL) {
        return false;
    }
    memcpy(hwMetaData_, shader, shaderSize);

    SC_SI_HWSHADER_CS* siMetaData = reinterpret_cast<SC_SI_HWSHADER_CS*>(hwMetaData_);

    // Code to patch the pointers in the shader object.
    // Must be preferably done in the compiler library
    size_t offset = siMetaData->common.uSizeInBytes;
    if (siMetaData->common.u32PvtDataSizeInBytes > 0) {
        siMetaData->common.pPvtData =
            reinterpret_cast<SC_BYTE *>(
            reinterpret_cast<char *>(siMetaData) + offset);
        offset += siMetaData->common.u32PvtDataSizeInBytes;
    }
    if (siMetaData->common.codeLenInByte > 0) {
        siMetaData->common.hShaderMemHandle =
            reinterpret_cast<char *>(siMetaData) + offset;
        offset += siMetaData->common.codeLenInByte;
    }

    char* headerBaseAddress =
        reinterpret_cast<char*>(siMetaData->common.hShaderMemHandle);
    amd_kernel_code_t* akc = reinterpret_cast<amd_kernel_code_t*>(
        headerBaseAddress);

    address codeStartAddress = reinterpret_cast<address>(akc);
    address codeEndAddress = reinterpret_cast<address>(akc) + siMetaData->common.codeLenInByte;
    codeSize_ = codeEndAddress - codeStartAddress;
    code_ = new gpu::Memory(dev(), amd::alignUp(codeSize_, gpu::ConstBuffer::VectorSize));

    // Initialize kernel ISA code
    if ((code_ != NULL) && code_->create(Resource::Shader)) {
        address cpuCodePtr = static_cast<address>(code_->map(NULL, Resource::WriteOnly));
        // Copy only amd_kernel_code_t
        memcpy(cpuCodePtr, codeStartAddress, codeSize_);
        code_->unmap(NULL);
    }
    else {
        LogError("Failed to allocate ISA code!");
        return false;
    }
    cpuAqlCode_ = akc;

    assert((akc->workitem_private_segment_byte_size & 3) == 0 &&
        "Scratch must be DWORD aligned");
    workGroupInfo_.scratchRegs_ =
        amd::alignUp(akc->workitem_private_segment_byte_size, 16) / sizeof(uint);
    workGroupInfo_.availableSGPRs_ = dev().gslCtx()->getNumSGPRsAvailable();
    workGroupInfo_.availableVGPRs_ = dev().gslCtx()->getNumVGPRsAvailable();
    workGroupInfo_.preferredSizeMultiple_ = dev().getAttribs().wavefrontSize;
    workGroupInfo_.privateMemSize_ = akc->workitem_private_segment_byte_size;
    workGroupInfo_.localMemSize_ =
    workGroupInfo_.usedLDSSize_ = akc->workgroup_group_segment_byte_size;
    workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count;
    workGroupInfo_.usedStackSize_ = 0;
    workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count;
    workGroupInfo_.wavefrontPerSIMD_ = dev().getAttribs().wavefrontSize;

    return true;
}
} // namespace gpu
<|endoftext|>


<commit_before>#pragma once
/**
	@file
	@brief array iterator
	@author MITSUNARI Shigeo(@herumi)
*/
#include <assert.h>

namespace mcl { namespace fp {

/*
	get w-bit size from x[0, bitSize)
	@param x [in] data
	@param bitSize [in] data size
	@param w [in] split size < UnitBitSize
*/
template<class T>
class ArrayIterator {
	const T *x_;
	size_t bitSize_;
	const size_t w_;
	const T mask_;
	size_t pos_;
public:
	static const size_t TbitSize = sizeof(T) * 8;
	ArrayIterator(const T *x, size_t bitSize, size_t w)
		: x_(x)
		, bitSize_(bitSize)
		, w_(w)
		, mask_(makeMask(w))
		, pos_(0)
	{
		assert(w_ <= TbitSize);
	}
	static T makeMask(size_t w)
	{
		return (w == TbitSize) ? ~T(0) : (T(1) << w) - 1;
	}
	bool hasNext() const { return bitSize_ > 0; }
	T getNext(size_t w = 0)
	{
		if (w == 0) w = w_;
		assert(w <= TbitSize);
		if (w > bitSize_) {
			w = bitSize_;
		}
		if (!hasNext()) return 0;
		const T mask = w == w_ ? mask_ : makeMask(w);
		const size_t nextPos = pos_ + w;
		if (nextPos <= TbitSize) {
			T v = x_[0] >> pos_;
			if (nextPos < TbitSize) {
				pos_ = nextPos;
				v &= mask;
			} else {
				pos_ = 0;
				x_++;
			}
			bitSize_ -= w;
			return v;
		}
		T v = (x_[0] >> pos_) | (x_[1] << (TbitSize - pos_));
		v &= mask;
		pos_ = nextPos - TbitSize;
		bitSize_ -= w;
		x_++;
		return v;
	}
	// don't change iter
	bool peek1bit()
	{
		assert(hasNext());
		return (x_[0] >> pos_) & 1;
	}
	// ++iter
	void consume1bit()
	{
		assert(hasNext());
		const size_t nextPos = pos_ + 1;
		if (nextPos < TbitSize) {
			pos_ = nextPos;
		} else {
			pos_ = 0;
			x_++;
		}
		bitSize_ -= 1;
	}
};

} } // mcl::fp

<commit_msg>remove ArrayIterator<commit_after><|endoftext|>


<commit_before>/************************************ ABOUT ***********************************\
| This file is used to compile the db_schema.json file into any nessasary      |
| formats including                                                            |
| SQL Initilizer Query (For both MySQL and SQLite3)                            |
| C++ object to store a row of the database for easy access                    |
| Java object to store a row of the databasae for easy access (android)        |
| - Eventually there may be javascript support for the firefox / chrome        |
| plugins                                                                      |
\******************************************************************************/
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/regex.hpp>
// #include <boost/date.hpp>
#include <iostream>
#include <vector>
#include <sstream>
#include <map>

#include <stdlib.h>

using namespace std;

// Mapping of each SQL datatype to the corrisponding C++ datatype
map <string, string> SQLtoCPP {
    {"varchar", "std::string"},
    {"int", "int"},
    {"timestamp", "std::string"}
};

// Mapping of each sql datatype to the C++ sqlite3 function to get that type from the database
map <string, string> SQLITEtoCPP{
    {"varchar", "std::string(reinterpret_cast<const char*> (sqlite3_column_text (statement,i)))"},
    {"int", "sqlite3_column_int (statement,i)"},
    {"timestamp", "std::string(reinterpret_cast<const char*> (sqlite3_column_text (statement,i)))" } // Timestamp
};

// Mapping of each SQL datatype to the corrisponding Java datatype
map <string, string> SQLtoJava {
    {"varchar", "String"},
    {"int", "Integer"},
    {"timestamp", "String"}
};

int main() {
    cout << "Loading the JSON file" << endl;
    using boost::property_tree::ptree;
    ptree pt;

    // Load the JSON into the property tree ometrherfke
    string testfile = "db_schema.json";
    read_json(testfile, pt);

    cout << "JSON file loaded" << endl;

    cout << "Parsing File" << endl;

    stringstream CPPVariables;        // Really these should all be files
    stringstream CPPConstructor; // the construction for initilizing all of the variables from a sqlite3 statement
    stringstream SQLRows;    // to be joined and encapsulated
    stringstream SQLITERows; // special formatting for sqlite (no extra data, no newlines)
    vector<string> SQLIndexes; // to be joined as is

    // Pre loop constructors
    SQLRows << "CREATE TABLE rules (";
    SQLITERows << "CREATE TABLE rules (";

    // Allow tracking of the number of columns
    int columnCount = 0;

    // Loop Constructors
    string concatinator = "";
    for ( ptree::value_type const& v : pt.get_child("columns")) {

        string columnName = v.first;

        ptree subtree = v.second;
        string type = subtree.get<std::string>("Type");
        string size = subtree.get("Size", "");
        string description = subtree.get("Description", "");
        string indexName = subtree.get("Index", "");
        string defaultValue = subtree.get("Default", "");
        bool isNullable = subtree.get("Nullable", true);
        string extraData = subtree.get("Extra", "");
        bool isDepricated = subtree.get("Depricated", false);

        // SQL colum Value
        SQLRows << concatinator << endl << "  `" << boost::to_lower_copy(columnName) << "`";
        SQLRows << " " << boost::to_lower_copy(type); if (size.length() > 0) SQLRows << "(" << size << ")"; SQLRows << " ";
        if (!isNullable) SQLRows << "NOT NULL";
        if (defaultValue.length() > 0) SQLRows << " DEFAULT " << defaultValue;
        if (extraData.length() > 0) SQLRows << " " << extraData;
        concatinator = ",";

        //SQLITE3 Constructor (no extra data, no newlines)
        SQLITERows << concatinator << " `" << boost::to_lower_copy(columnName) << "`";
        SQLITERows << " " << boost::to_lower_copy(type); if (size.length() > 0) SQLITERows << "\\(" << size << "\\)"; SQLITERows << " ";
        if (!isNullable) SQLITERows << "NOT NULL";
        if (defaultValue.length() > 0) SQLITERows << " DEFAULT " << defaultValue;
        concatinator = ",";

        // C++ Datatype variables
        CPPVariables << "\t" << SQLtoCPP[type] << " " << columnName << ";" << endl;

        // C++ SQLITE3 CONVERSIONS
        CPPConstructor << "\t\t\tif (columnName == \"" << boost::to_lower_copy(columnName) << "\") {" << columnName << " = " << SQLITEtoCPP[type] << ";}" << endl;

        // CREATE TABLE `virtual_aliases` (
        //   `id` int(11) NOT NULL auto_increment,
        //   `domain_id` int(11) NOT NULL,
        //   `source` varchar(100) NOT NULL,
        //   `destination` varchar(100) NOT NULL,
        //   PRIMARY KEY (`id`),
        //   FOREIGN KEY (domain_id) REFERENCES virtual_domains(id) ON DELETE CASCADE
        // ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
        columnCount++;
    }

    // Post Loop Constst    ructors
    SQLRows << endl << ");" << endl; // the engine data might not be nessasary but IDK how SQL differes between implementations
    SQLITERows << ");";

    // cout << "CREATE INDEX " << indexName << "ON rules (" << columnName << ");" << endl;

    ifstream CPPFileIn;
    ofstream CPPFileOut;
    char inputFilename[] = "templates/DomainSettings.h";
    char outputFilename[] = "../CommandLineTool/DomainSettings.h";

    CPPFileIn.open(inputFilename, ios::in);

    if (!CPPFileIn) {
      cerr << "Can't open input file " << inputFilename << endl;
      exit(1);
    }

    CPPFileOut.open(outputFilename, ios::out);

    if (!CPPFileOut) {
      cerr << "Can't open output file " << outputFilename << endl;
      exit(1);
    }

    string CPPFileLine;
    while (getline(CPPFileIn, CPPFileLine)) {

        CPPFileLine = boost::regex_replace(CPPFileLine, boost::regex("\\[\\[SQLCOLUMNCOUNT\\]\\]"), std::to_string(columnCount), boost::match_default | boost::format_all);
        CPPFileLine = boost::regex_replace(CPPFileLine, boost::regex("\\[\\[SQLCREATETABLE\\]\\]"), "\""+SQLITERows.str()+"\"", boost::match_default | boost::format_all);

        if (boost::regex_match(CPPFileLine, boost::regex ("\\s*\\[\\[CPPVARIABLES\\]\\]"))) {
            CPPFileOut << CPPVariables.rdbuf();
        }
        else if (boost::regex_match(CPPFileLine, boost::regex("\\s*\\[\\[CPPCONSTRUCTOR\\]\\]"))) {
            CPPFileOut << CPPConstructor.rdbuf();
        }
        else {
            CPPFileOut << CPPFileLine << "\n"; // reappend the newline to the file
        }
    }

    cout << SQLRows.rdbuf() << endl;
    cout << CPPConstructor.rdbuf() << endl;
}<commit_msg>switched the autogenerated cpp file to use spaces instead of tabs<commit_after>/************************************ ABOUT ***********************************\
| This file is used to compile the db_schema.json file into any nessasary      |
| formats including                                                            |
| SQL Initilizer Query (For both MySQL and SQLite3)                            |
| C++ object to store a row of the database for easy access                    |
| Java object to store a row of the databasae for easy access (android)        |
| - Eventually there may be javascript support for the firefox / chrome        |
| plugins                                                                      |
\******************************************************************************/
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/regex.hpp>
// #include <boost/date.hpp>
#include <iostream>
#include <vector>
#include <string>
#include <sstream>
#include <map>

#include <stdlib.h>

using namespace std;

// Mapping of each SQL datatype to the corrisponding C++ datatype
map <string, string> SQLtoCPP {
    {"varchar", "std::string"},
    {"int", "int"},
    {"timestamp", "std::string"}
};

// Mapping of each sql datatype to the C++ sqlite3 function to get that type from the database
map <string, string> SQLITEtoCPP{
    {"varchar", "std::string(reinterpret_cast<const char*> (sqlite3_column_text (statement,i)))"},
    {"int", "sqlite3_column_int (statement,i)"},
    {"timestamp", "std::string(reinterpret_cast<const char*> (sqlite3_column_text (statement,i)))" } // Timestamp
};

// Mapping of each SQL datatype to the corrisponding Java datatype
map <string, string> SQLtoJava {
    {"varchar", "String"},
    {"int", "Integer"},
    {"timestamp", "String"}
};

int main() {
    cout << "Loading the JSON file" << endl;
    using boost::property_tree::ptree;
    ptree pt;

    // Load the JSON into the property tree ometrherfke
    string testfile = "db_schema.json";
    read_json(testfile, pt);

    cout << "JSON file loaded" << endl;

    cout << "Parsing File" << endl;

    stringstream CPPVariables;        // Really these should all be files
    stringstream CPPConstructor; // the construction for initilizing all of the variables from a sqlite3 statement
    stringstream SQLRows;    // to be joined and encapsulated
    stringstream SQLITERows; // special formatting for sqlite (no extra data, no newlines)
    vector<string> SQLIndexes; // to be joined as is

    // Pre loop constructors
    SQLRows << "CREATE TABLE rules (";
    SQLITERows << "CREATE TABLE rules (";

    // Allow tracking of the number of columns
    int columnCount = 0;

    // Loop Constructors
    string concatinator = "";
    for ( ptree::value_type const& v : pt.get_child("columns")) {

        string columnName = v.first;

        ptree subtree = v.second;
        string type = subtree.get<std::string>("Type");
        string size = subtree.get("Size", "");
        string description = subtree.get("Description", "");
        string indexName = subtree.get("Index", "");
        string defaultValue = subtree.get("Default", "");
        bool isNullable = subtree.get("Nullable", true);
        string extraData = subtree.get("Extra", "");
        bool isDepricated = subtree.get("Depricated", false);

        // SQL colum Value
        SQLRows << concatinator << endl << "  `" << boost::to_lower_copy(columnName) << "`";
        SQLRows << " " << boost::to_lower_copy(type); if (size.length() > 0) SQLRows << "(" << size << ")"; SQLRows << " ";
        if (!isNullable) SQLRows << "NOT NULL";
        if (defaultValue.length() > 0) SQLRows << " DEFAULT " << defaultValue;
        if (extraData.length() > 0) SQLRows << " " << extraData;
        concatinator = ",";

        //SQLITE3 Constructor (no extra data, no newlines)
        SQLITERows << concatinator << " `" << boost::to_lower_copy(columnName) << "`";
        SQLITERows << " " << boost::to_lower_copy(type); if (size.length() > 0) SQLITERows << "\\(" << size << "\\)"; SQLITERows << " ";
        if (!isNullable) SQLITERows << "NOT NULL";
        if (defaultValue.length() > 0) SQLITERows << " DEFAULT " << defaultValue;
        concatinator = ",";

        // C++ Datatype variables
        CPPVariables << "    " << SQLtoCPP[type] << " " << columnName << ";" << endl;

        // C++ SQLITE3 CONVERSIONS
        CPPConstructor << "            if (columnName == \"" << boost::to_lower_copy(columnName) << "\") {" << columnName << " = " << SQLITEtoCPP[type] << ";}" << endl;

        // CREATE TABLE `virtual_aliases` (
        //   `id` int(11) NOT NULL auto_increment,
        //   `domain_id` int(11) NOT NULL,
        //   `source` varchar(100) NOT NULL,
        //   `destination` varchar(100) NOT NULL,
        //   PRIMARY KEY (`id`),
        //   FOREIGN KEY (domain_id) REFERENCES virtual_domains(id) ON DELETE CASCADE
        // ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
        columnCount++;
    }

    // Post Loop Constst    ructors
    SQLRows << endl << ");" << endl; // the engine data might not be nessasary but IDK how SQL differes between implementations
    SQLITERows << ");";

    // cout << "CREATE INDEX " << indexName << "ON rules (" << columnName << ");" << endl;

    ifstream CPPFileIn;
    ofstream CPPFileOut;
    char inputFilename[] = "templates/DomainSettings.h";
    char outputFilename[] = "../CommandLineTool/DomainSettings.h";

    CPPFileIn.open(inputFilename, ios::in);

    if (!CPPFileIn) {
      cerr << "Can't open input file " << inputFilename << endl;
      exit(1);
    }

    CPPFileOut.open(outputFilename, ios::out);

    if (!CPPFileOut) {
      cerr << "Can't open output file " << outputFilename << endl;
      exit(1);
    }

    string CPPFileLine;
    while (getline(CPPFileIn, CPPFileLine)) {

        CPPFileLine = boost::regex_replace(CPPFileLine, boost::regex("\\[\\[SQLCOLUMNCOUNT\\]\\]"), std::to_string(columnCount), boost::match_default | boost::format_all);
        CPPFileLine = boost::regex_replace(CPPFileLine, boost::regex("\\[\\[SQLCREATETABLE\\]\\]"), "\""+SQLITERows.str()+"\"", boost::match_default | boost::format_all);

        if (boost::regex_match(CPPFileLine, boost::regex ("\\s*\\[\\[CPPVARIABLES\\]\\]"))) {
            CPPFileOut << CPPVariables.rdbuf();
        }
        else if (boost::regex_match(CPPFileLine, boost::regex("\\s*\\[\\[CPPCONSTRUCTOR\\]\\]"))) {
            CPPFileOut << CPPConstructor.rdbuf();
        }
        else {
            CPPFileOut << CPPFileLine << "\n"; // reappend the newline to the file
        }
    }

    cout << SQLRows.rdbuf() << endl;
    cout << CPPConstructor.rdbuf() << endl;
}<|endoftext|>