<commit_before>// A tool for adding keywords extracted from titles to MARC records.
#include <algorithm>
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <unordered_set>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "Stemmer.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "TextUtil.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << progname << " [--verbose] marc_input marc_output [stopwords_files]\n";
    std::cerr << "       The MARC-21 output will have enriched keywords based on title words that were\n";
    std::cerr << "       similar to keywords found in the MARC-21 input file.\n";
    std::cerr << "       Stopword files must be named \"stopwords.xxx\" where xxx has to be a 3-letter\n";
    std::cerr << "       language code.\n";
    std::exit(EXIT_FAILURE);
}

void LoadStopwords(const bool verbose, FILE * const input,
                   std::unordered_set<std::string> * const stopwords_set)
{
    if (verbose)
        std::cout << "Starting loading of stopwords.\n";

    unsigned count(0);
    while (not std::feof(input)) {
        char buf[1024];
        if (std::fgets(buf, sizeof buf, input) == NULL)
            break;
        if (buf[0] == '\0' or buf[0] == ';') // Empty or comment line?
            continue;

        size_t len(std::strlen(buf));
        if (buf[len - 1] == '\n')
            --len;

        stopwords_set->insert(StringUtil::ToLower(std::string(buf, len)));
        ++count;
    }

    if (std::ferror(input))
        Error("Read error while trying to read the stopwords file.");

    if (verbose)
        std::cerr << "Read " << count << " stopwords.\n";
}

void FilterOutStopwords(const std::unordered_set<std::string> &stopwords,
                        std::unordered_set<std::string> * const words)
{
    for (std::unordered_set<std::string>::iterator word(words->begin()); word != words->end(); /* Empty! */) {
        if (stopwords.find(*word) != stopwords.end())
            word = words->erase(word);
        else
            ++word;
    }
}

size_t ExtractKeywordsFromKeywordChainFields(
    const std::vector<DirectoryEntry> &dir_entries, const std::vector<std::string> &field_data,
    const Stemmer * const stemmer,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    size_t keyword_count(0);
    const auto _689_iterator(DirectoryEntry::FindField("689", dir_entries));
    if (_689_iterator != dir_entries.end()) {
	size_t field_index(_689_iterator - dir_entries.begin());
	while (field_index < field_data.size() and dir_entries[field_index].getTag() == "689") {
	    const Subfields subfields(field_data[field_index]);
	    const std::string subfield_a_value(subfields.getFirstSubfieldValue('a'));
	    if (not subfield_a_value.empty() and subfield_a_value.find(' ') == std::string::npos) {
		const std::string stemmed_keyword(stemmer == NULL ? subfield_a_value
						                  : stemmer->stem(subfield_a_value));
		(*stemmed_to_unstemmed_keywords_map)[stemmed_keyword] = subfield_a_value;
		++keyword_count;
	    }

	    ++field_index;
	}
    }

    return keyword_count;
}

size_t ExtractKeywordsFromIndividualKeywordFields(
    const std::vector<DirectoryEntry> &dir_entries,
    const std::vector<std::string> &field_data,
    const Stemmer * const stemmer,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    size_t keyword_count(0);
    std::vector<std::string> keywords;
    MarcUtil::ExtractAllSubfields("600:610:611:630:650:653:656", dir_entries, field_data, &keywords, "02");
    for (const auto &keyword : keywords) {
	if (keyword.find(' ') != std::string::npos)
	    continue;
	const std::string stemmed_keyword(stemmer == NULL ? keyword : stemmer->stem(keyword));
	(*stemmed_to_unstemmed_keywords_map)[stemmed_keyword] = keyword;
	++keyword_count;
    }

    return keyword_count;
}

/** \return A Stemmer for "language_code" if we can create or find one.  Null o/w. */
const Stemmer *GetStemmer(const std::string &language_code) {
    static std::unordered_map<std::string, const Stemmer *> code_to_stemmer_map;
    const auto code_and_stemmer_iter(code_to_stemmer_map.find(language_code));
    if (code_and_stemmer_iter != code_to_stemmer_map.end())
	return code_and_stemmer_iter->second;

    const Stemmer *new_stemmer(Stemmer::StemmerFactory(language_code));
    code_to_stemmer_map[language_code] = new_stemmer;

    return new_stemmer;
}

size_t ExtractAllKeywords(
    const std::vector<DirectoryEntry> &dir_entries, const std::vector<std::string> &field_data,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    const std::string language_code(MarcUtil::GetLanguage(dir_entries, field_data));
    const Stemmer * const stemmer(language_code.empty() ? NULL : GetStemmer(language_code));

    size_t extracted_count(ExtractKeywordsFromKeywordChainFields(dir_entries, field_data, stemmer,
								 stemmed_to_unstemmed_keywords_map));
    extracted_count += ExtractKeywordsFromIndividualKeywordFields(dir_entries, field_data, stemmer,
								  stemmed_to_unstemmed_keywords_map);
    return extracted_count;
}

void ExtractStemmedKeywords(const bool verbose, FILE * const input,
			    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    if (verbose)
        std::cerr << "Starting extraction and stemming of pre-existing keywords.\n";

    Leader *raw_leader;
    std::vector<DirectoryEntry> dir_entries;
    std::vector<std::string> field_data;
    unsigned total_count(0), records_with_keywords_count(0), keywords_count(0);
    std::string err_msg;
    while (MarcUtil::ReadNextRecord(input, &raw_leader, &dir_entries, &field_data, &err_msg)) {
        ++total_count;
        std::unique_ptr<Leader> leader(raw_leader);

	const size_t extracted_count(ExtractAllKeywords(dir_entries, field_data, stemmed_to_unstemmed_keywords_map));
	if (extracted_count > 0) {
	    ++records_with_keywords_count;
	    keywords_count += extracted_count;
	}
    }

    if (verbose) {
        std::cerr << total_count << " records processed.\n";
        std::cerr << records_with_keywords_count << " records had keywords.\n";
	std::cerr << keywords_count << " keywords were extracted of which "
		  << stemmed_to_unstemmed_keywords_map->size() << " were unique.\n";
    }
}

void AugmentRecordsWithTitleKeywords(
    const bool verbose, FILE * const input, FILE * const output,
    const std::unordered_map<std::string, std::string> &stemmed_to_unstemmed_keywords_map,
    const std::map<std::string, std::unordered_set<std::string>> &language_codes_to_stopword_sets)
{
    if (verbose)
        std::cerr << "Starting augmentation of stopwords.\n";

    Leader *raw_leader;
    std::vector<DirectoryEntry> dir_entries;
    std::vector<std::string> field_data;
    unsigned total_count(0), augmented_record_count(0);
    std::string err_msg;
    while (MarcUtil::ReadNextRecord(input, &raw_leader, &dir_entries, &field_data, &err_msg)) {
        ++total_count;
        std::unique_ptr<Leader> leader(raw_leader);

	// Look for a title...
        const auto entry_iterator(DirectoryEntry::FindField("245", dir_entries));
        if (entry_iterator == dir_entries.end()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// ...in subfield 'a':
        const size_t title_index(entry_iterator - dir_entries.begin());
        Subfields subfields(field_data[title_index]);
        if (not subfields.hasSubfield('a')) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

        const auto begin_end_a = subfields.getIterators('a');
        std::string title(begin_end_a.first->second);
        const auto begin_end_b = subfields.getIterators('b'); // optional additional title part.
        if (begin_end_b.first != begin_end_b.second)
            title += " " + begin_end_b.first->second;

        std::unordered_set<std::string> title_words;
        TextUtil::ChopIntoWords(title, &title_words, /* min_word_length = */ 3);

	// Remove language-appropriate stop words from the title words:
        const std::string language_code(MarcUtil::GetLanguage(dir_entries, field_data));
        const auto code_and_stopwords(language_codes_to_stopword_sets.find(language_code));
        if (code_and_stopwords != language_codes_to_stopword_sets.end())
            FilterOutStopwords(code_and_stopwords->second, &title_words);
        if (language_code != "eng") // Hack because people suck at cataloging!
            FilterOutStopwords(language_codes_to_stopword_sets.find("eng")->second, &title_words);

        if (title_words.empty()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// If we have an appropriate stemmer, replace the title words w/ stemmed title words:
	const Stemmer * const stemmer(language_code.empty() ? NULL : GetStemmer(language_code));
	if (stemmer != NULL) {
	    std::unordered_set<std::string> stemmed_title_words;
	    for (const auto &title_word : title_words)
		stemmed_title_words.insert(stemmer->stem(title_word));
	    std::swap(stemmed_title_words, title_words);
	}

	std::unordered_map<std::string, std::string> local_stemmed_to_unstemmed_keywords_map;
	ExtractAllKeywords(dir_entries, field_data, &local_stemmed_to_unstemmed_keywords_map);

	// Find title words that match stemmed keywords:
	std::unordered_set<std::string> new_keywords;
	for (const auto &title_word : title_words) {
	    if (local_stemmed_to_unstemmed_keywords_map.find(title_word)
		!= local_stemmed_to_unstemmed_keywords_map.end())
		continue; // We already have this word.

	    const auto stemmed_and_unstemmed_word_iter(stemmed_to_unstemmed_keywords_map.find(title_word));
	    if (stemmed_and_unstemmed_word_iter != stemmed_to_unstemmed_keywords_map.end())
		new_keywords.insert(stemmed_and_unstemmed_word_iter->second);
	}

        if (new_keywords.empty()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// Augment the record with new keywords derived from title words:
	for (const auto &new_keyword : new_keywords) {
	    const std::string field_contents("  ""\x1F""a" + new_keyword);
	    MarcUtil::InsertField(field_contents, "601", leader.get(), &dir_entries, &field_data);
	}

	MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
        ++augmented_record_count;
    }

    if (verbose)
        std::cerr << augmented_record_count << " records of " << total_count
                  << " were augmented w/ additional keywords.\n";
}

int main(int argc, char **argv) {
    progname = argv[0];

    if (argc < 3)
        Usage();

    const bool verbose(std::strcmp(argv[1], "--verbose") == 0);
    if (verbose and argc < 4)
        Usage();

    const std::string marc_input_filename(argv[verbose ? 2 : 1]);
    FILE *marc_input = std::fopen(marc_input_filename.c_str(), "rm");
    if (marc_input == NULL)
        Error("can't open \"" + marc_input_filename + "\" for reading!");

    const std::string marc_output_filename(argv[verbose ? 3 : 2]);
    FILE *marc_output = std::fopen(marc_output_filename.c_str(), "wb");
    if (marc_output == NULL)
        Error("can't open \"" + marc_output_filename + "\" for writing!");

    if (unlikely(marc_input_filename == marc_output_filename))
        Error("MARC input file name equals MARC output file name!");

    // Read optional stopword lists:
    std::map<std::string, std::unordered_set<std::string>> language_codes_to_stopword_sets;
    for (int arg_no(verbose ? 4 : 3); arg_no < argc; ++arg_no) {
	const RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("stopwords\\....$"));
        const std::string stopwords_filename(argv[arg_no]);
	std::string err_msg;
        if (not matcher->matched(stopwords_filename, &err_msg))
            Error("Invalid stopwords filename \"" + stopwords_filename + "\"!");
        const std::string language_code(stopwords_filename.substr(stopwords_filename.length() - 3));
        FILE *stopwords = std::fopen(stopwords_filename.c_str(), "rm");
        if (stopwords == NULL)
            Error("can't open \"" + stopwords_filename + "\" for reading!");
        std::unordered_set<std::string> stopwords_set;
        LoadStopwords(verbose, stopwords, &stopwords_set);
        language_codes_to_stopword_sets[language_code] = stopwords_set;
        std::fclose(stopwords);
    }

    // We always need English because librarians suck at specifying English:
    if (language_codes_to_stopword_sets.find("eng") == language_codes_to_stopword_sets.end())
        Error("You always need to provide \"stopwords.eng\"!");

    std::unordered_map<std::string, std::string> stemmed_to_unstemmed_keywords_map;
    ExtractStemmedKeywords(verbose, marc_input, &stemmed_to_unstemmed_keywords_map);

    std::rewind(marc_input);
    AugmentRecordsWithTitleKeywords(verbose, marc_input, marc_output, stemmed_to_unstemmed_keywords_map,
				    language_codes_to_stopword_sets);

    std::fclose(marc_input);
    std::fclose(marc_output);
}
<commit_msg>Minor clean-up.<commit_after>// A tool for adding keywords extracted from titles to MARC records.
#include <algorithm>
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <unordered_set>
#include <cstdlib>
#include <cstring>
#include "DirectoryEntry.h"
#include "Leader.h"
#include "MarcUtil.h"
#include "RegexMatcher.h"
#include "Stemmer.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "TextUtil.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << progname << " [--verbose] marc_input marc_output [stopwords_files]\n";
    std::cerr << "       The MARC-21 output will have enriched keywords based on title words that were\n";
    std::cerr << "       similar to keywords found in the MARC-21 input file.\n";
    std::cerr << "       Stopword files must be named \"stopwords.xxx\" where xxx has to be a 3-letter\n";
    std::cerr << "       language code.\n";
    std::exit(EXIT_FAILURE);
}

void LoadStopwords(const bool verbose, FILE * const input,
                   std::unordered_set<std::string> * const stopwords_set)
{
    if (verbose)
        std::cout << "Starting loading of stopwords.\n";

    unsigned count(0);
    while (not std::feof(input)) {
        char buf[1024];
        if (std::fgets(buf, sizeof buf, input) == NULL)
            break;
        if (buf[0] == '\0' or buf[0] == ';') // Empty or comment line?
            continue;

        size_t len(std::strlen(buf));
        if (buf[len - 1] == '\n')
            --len;

        stopwords_set->insert(StringUtil::ToLower(std::string(buf, len)));
        ++count;
    }

    if (std::ferror(input))
        Error("Read error while trying to read the stopwords file.");

    if (verbose)
        std::cerr << "Read " << count << " stopwords.\n";
}

void FilterOutStopwords(const std::unordered_set<std::string> &stopwords,
                        std::unordered_set<std::string> * const words)
{
    for (std::unordered_set<std::string>::iterator word(words->begin()); word != words->end(); /* Empty! */) {
        if (stopwords.find(*word) != stopwords.end())
            word = words->erase(word);
        else
            ++word;
    }
}

size_t ExtractKeywordsFromKeywordChainFields(
    const std::vector<DirectoryEntry> &dir_entries, const std::vector<std::string> &field_data,
    const Stemmer * const stemmer,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    size_t keyword_count(0);
    const auto _689_iterator(DirectoryEntry::FindField("689", dir_entries));
    if (_689_iterator != dir_entries.end()) {
	size_t field_index(_689_iterator - dir_entries.begin());
	while (field_index < field_data.size() and dir_entries[field_index].getTag() == "689") {
	    const Subfields subfields(field_data[field_index]);
	    const std::string subfield_a_value(subfields.getFirstSubfieldValue('a'));
	    if (not subfield_a_value.empty() and subfield_a_value.find(' ') == std::string::npos) {
		const std::string stemmed_keyword(stemmer == NULL ? subfield_a_value
						                  : stemmer->stem(subfield_a_value));
		(*stemmed_to_unstemmed_keywords_map)[stemmed_keyword] = subfield_a_value;
		++keyword_count;
	    }

	    ++field_index;
	}
    }

    return keyword_count;
}

size_t ExtractKeywordsFromIndividualKeywordFields(
    const std::vector<DirectoryEntry> &dir_entries,
    const std::vector<std::string> &field_data,
    const Stemmer * const stemmer,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    size_t keyword_count(0);
    std::vector<std::string> keywords;
    static const std::string SUBFIELD_IGNORE_LIST("02"); // Do not extract $0 and $2.
    MarcUtil::ExtractAllSubfields("600:610:611:630:650:653:656", dir_entries, field_data, &keywords,
				  SUBFIELD_IGNORE_LIST);
    for (const auto &keyword : keywords) {
	if (keyword.find(' ') != std::string::npos)
	    continue;
	const std::string stemmed_keyword(stemmer == NULL ? keyword : stemmer->stem(keyword));
	(*stemmed_to_unstemmed_keywords_map)[stemmed_keyword] = keyword;
	++keyword_count;
    }

    return keyword_count;
}

/** \return A Stemmer for "language_code" if we can create or find one.  Null o/w. */
const Stemmer *GetStemmer(const std::string &language_code) {
    static std::unordered_map<std::string, const Stemmer *> code_to_stemmer_map;
    const auto code_and_stemmer_iter(code_to_stemmer_map.find(language_code));
    if (code_and_stemmer_iter != code_to_stemmer_map.end())
	return code_and_stemmer_iter->second;

    const Stemmer *new_stemmer(Stemmer::StemmerFactory(language_code));
    code_to_stemmer_map[language_code] = new_stemmer;

    return new_stemmer;
}

size_t ExtractAllKeywords(
    const std::vector<DirectoryEntry> &dir_entries, const std::vector<std::string> &field_data,
    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    const std::string language_code(MarcUtil::GetLanguage(dir_entries, field_data));
    const Stemmer * const stemmer(language_code.empty() ? NULL : GetStemmer(language_code));

    size_t extracted_count(ExtractKeywordsFromKeywordChainFields(dir_entries, field_data, stemmer,
								 stemmed_to_unstemmed_keywords_map));
    extracted_count += ExtractKeywordsFromIndividualKeywordFields(dir_entries, field_data, stemmer,
								  stemmed_to_unstemmed_keywords_map);
    return extracted_count;
}

void ExtractStemmedKeywords(const bool verbose, FILE * const input,
			    std::unordered_map<std::string, std::string> * const stemmed_to_unstemmed_keywords_map)
{
    if (verbose)
        std::cerr << "Starting extraction and stemming of pre-existing keywords.\n";

    Leader *raw_leader;
    std::vector<DirectoryEntry> dir_entries;
    std::vector<std::string> field_data;
    unsigned total_count(0), records_with_keywords_count(0), keywords_count(0);
    std::string err_msg;
    while (MarcUtil::ReadNextRecord(input, &raw_leader, &dir_entries, &field_data, &err_msg)) {
        ++total_count;
        std::unique_ptr<Leader> leader(raw_leader);

	const size_t extracted_count(ExtractAllKeywords(dir_entries, field_data, stemmed_to_unstemmed_keywords_map));
	if (extracted_count > 0) {
	    ++records_with_keywords_count;
	    keywords_count += extracted_count;
	}
    }

    if (verbose) {
        std::cerr << total_count << " records processed.\n";
        std::cerr << records_with_keywords_count << " records had keywords.\n";
	std::cerr << keywords_count << " keywords were extracted of which "
		  << stemmed_to_unstemmed_keywords_map->size() << " were unique.\n";
    }
}

void AugmentRecordsWithTitleKeywords(
    const bool verbose, FILE * const input, FILE * const output,
    const std::unordered_map<std::string, std::string> &stemmed_to_unstemmed_keywords_map,
    const std::map<std::string, std::unordered_set<std::string>> &language_codes_to_stopword_sets)
{
    if (verbose)
        std::cerr << "Starting augmentation of stopwords.\n";

    Leader *raw_leader;
    std::vector<DirectoryEntry> dir_entries;
    std::vector<std::string> field_data;
    unsigned total_count(0), augmented_record_count(0);
    std::string err_msg;
    while (MarcUtil::ReadNextRecord(input, &raw_leader, &dir_entries, &field_data, &err_msg)) {
        ++total_count;
        std::unique_ptr<Leader> leader(raw_leader);

	// Look for a title...
        const auto entry_iterator(DirectoryEntry::FindField("245", dir_entries));
        if (entry_iterator == dir_entries.end()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// ...in subfield 'a':
        const size_t title_index(entry_iterator - dir_entries.begin());
        Subfields subfields(field_data[title_index]);
        if (not subfields.hasSubfield('a')) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

        const auto begin_end_a = subfields.getIterators('a');
        std::string title(begin_end_a.first->second);
        const auto begin_end_b = subfields.getIterators('b'); // optional additional title part.
        if (begin_end_b.first != begin_end_b.second)
            title += " " + begin_end_b.first->second;

        std::unordered_set<std::string> title_words;
        TextUtil::ChopIntoWords(title, &title_words, /* min_word_length = */ 3);

	// Remove language-appropriate stop words from the title words:
        const std::string language_code(MarcUtil::GetLanguage(dir_entries, field_data));
        const auto code_and_stopwords(language_codes_to_stopword_sets.find(language_code));
        if (code_and_stopwords != language_codes_to_stopword_sets.end())
            FilterOutStopwords(code_and_stopwords->second, &title_words);
        if (language_code != "eng") // Hack because people suck at cataloging!
            FilterOutStopwords(language_codes_to_stopword_sets.find("eng")->second, &title_words);

        if (title_words.empty()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// If we have an appropriate stemmer, replace the title words w/ stemmed title words:
	const Stemmer * const stemmer(language_code.empty() ? NULL : GetStemmer(language_code));
	if (stemmer != NULL) {
	    std::unordered_set<std::string> stemmed_title_words;
	    for (const auto &title_word : title_words)
		stemmed_title_words.insert(stemmer->stem(title_word));
	    std::swap(stemmed_title_words, title_words);
	}

	std::unordered_map<std::string, std::string> local_stemmed_to_unstemmed_keywords_map;
	ExtractAllKeywords(dir_entries, field_data, &local_stemmed_to_unstemmed_keywords_map);

	// Find title words that match stemmed keywords:
	std::unordered_set<std::string> new_keywords;
	for (const auto &title_word : title_words) {
	    if (local_stemmed_to_unstemmed_keywords_map.find(title_word)
		!= local_stemmed_to_unstemmed_keywords_map.end())
		continue; // We already have this word.

	    const auto stemmed_and_unstemmed_word_iter(stemmed_to_unstemmed_keywords_map.find(title_word));
	    if (stemmed_and_unstemmed_word_iter != stemmed_to_unstemmed_keywords_map.end())
		new_keywords.insert(stemmed_and_unstemmed_word_iter->second);
	}

        if (new_keywords.empty()) {
            MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
            continue;
        }

	// Augment the record with new keywords derived from title words:
	for (const auto &new_keyword : new_keywords) {
	    const std::string field_contents("  ""\x1F""a" + new_keyword);
	    MarcUtil::InsertField(field_contents, "601", leader.get(), &dir_entries, &field_data);
	}

	MarcUtil::ComposeAndWriteRecord(output, dir_entries, field_data, leader.get());
        ++augmented_record_count;
    }

    if (verbose)
        std::cerr << augmented_record_count << " records of " << total_count
                  << " were augmented w/ additional keywords.\n";
}

int main(int argc, char **argv) {
    progname = argv[0];

    if (argc < 3)
        Usage();

    const bool verbose(std::strcmp(argv[1], "--verbose") == 0);
    if (verbose and argc < 4)
        Usage();

    const std::string marc_input_filename(argv[verbose ? 2 : 1]);
    FILE *marc_input = std::fopen(marc_input_filename.c_str(), "rm");
    if (marc_input == NULL)
        Error("can't open \"" + marc_input_filename + "\" for reading!");

    const std::string marc_output_filename(argv[verbose ? 3 : 2]);
    FILE *marc_output = std::fopen(marc_output_filename.c_str(), "wb");
    if (marc_output == NULL)
        Error("can't open \"" + marc_output_filename + "\" for writing!");

    if (unlikely(marc_input_filename == marc_output_filename))
        Error("MARC input file name equals MARC output file name!");

    // Read optional stopword lists:
    std::map<std::string, std::unordered_set<std::string>> language_codes_to_stopword_sets;
    for (int arg_no(verbose ? 4 : 3); arg_no < argc; ++arg_no) {
	const RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("stopwords\\....$"));
        const std::string stopwords_filename(argv[arg_no]);
	std::string err_msg;
        if (not matcher->matched(stopwords_filename, &err_msg))
            Error("Invalid stopwords filename \"" + stopwords_filename + "\"!");
        const std::string language_code(stopwords_filename.substr(stopwords_filename.length() - 3));
        FILE *stopwords = std::fopen(stopwords_filename.c_str(), "rm");
        if (stopwords == NULL)
            Error("can't open \"" + stopwords_filename + "\" for reading!");
        std::unordered_set<std::string> stopwords_set;
        LoadStopwords(verbose, stopwords, &stopwords_set);
        language_codes_to_stopword_sets[language_code] = stopwords_set;
        std::fclose(stopwords);
    }

    // We always need English because librarians suck at specifying English:
    if (language_codes_to_stopword_sets.find("eng") == language_codes_to_stopword_sets.end())
        Error("You always need to provide \"stopwords.eng\"!");

    std::unordered_map<std::string, std::string> stemmed_to_unstemmed_keywords_map;
    ExtractStemmedKeywords(verbose, marc_input, &stemmed_to_unstemmed_keywords_map);

    std::rewind(marc_input);
    AugmentRecordsWithTitleKeywords(verbose, marc_input, marc_output, stemmed_to_unstemmed_keywords_map,
				    language_codes_to_stopword_sets);

    std::fclose(marc_input);
    std::fclose(marc_output);
}
<|endoftext|>


<commit_before>/* Copyright 2014 - 2015 CyberTech Labs Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. */

#include "coreDumping.h"

#include <QtCore/QDir>

#include <sys/resource.h>
#include <signal.h>
#include <unistd.h>

#include "paths.h"

void (*oldHandler)(int);

static void dumpHandler_impl(int signal, char *p)
{
	static char* path = nullptr;
	if (path == nullptr && p != nullptr) {
		path = strdup(p);
	}

	if (p == nullptr) {
		chdir(path);
		oldHandler(signal);
	}
}

static void dumpHandler(int signal)
{
	dumpHandler_impl(signal, nullptr);
}

static void initSignals()
{
	QList<int> const signalsList({SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGBUS, SIGSYS, SIGTRAP, SIGXCPU, SIGXFSZ
			, SIGIOT});

	struct sigaction oldAction;
	sigaction(SIGSEGV, nullptr, &oldAction);
	oldHandler = oldAction.sa_handler;

	struct sigaction action;
	action.sa_handler = dumpHandler;
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;

	for (int signal : signalsList) {
		sigaddset(&action.sa_mask, signal);
		sigaction(signal, &action, nullptr);
	}
}

static void setCoreLimits()
{
	rlimit core_limits;
	core_limits.rlim_cur = core_limits.rlim_max = RLIM_INFINITY;
	setrlimit(RLIMIT_CORE, &core_limits);
}

void trikKernel::coreDumping::initCoreDumping(const QString &coreDumpPath)
{
	char *path = coreDumpPath.toLocal8Bit().data();
	dumpHandler_impl(0, path);

	initSignals();
	setCoreLimits();
}
<commit_msg>[trikKernel] avoid valgind warning<commit_after>/* Copyright 2014 - 2015 CyberTech Labs Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. */

#include "coreDumping.h"

#include <QtCore/QDir>

#include <sys/resource.h>
#include <signal.h>
#include <unistd.h>

#include "paths.h"

void (*oldHandler)(int);

static void dumpHandler_impl(int signal, char *p)
{
	static char* path = nullptr;
	if (path == nullptr && p != nullptr) {
		path = p;
	}

	if (p == nullptr) {
		chdir(path);
		oldHandler(signal);
	}
}

static void dumpHandler(int signal)
{
	dumpHandler_impl(signal, nullptr);
}

static void initSignals()
{
	QList<int> const signalsList({SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGBUS, SIGSYS, SIGTRAP, SIGXCPU, SIGXFSZ
			, SIGIOT});

	struct sigaction oldAction;
	sigaction(SIGSEGV, nullptr, &oldAction);
	oldHandler = oldAction.sa_handler;

	struct sigaction action;
	action.sa_handler = dumpHandler;
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;

	for (int signal : signalsList) {
		sigaddset(&action.sa_mask, signal);
		sigaction(signal, &action, nullptr);
	}
}

static void setCoreLimits()
{
	rlimit core_limits;
	core_limits.rlim_cur = core_limits.rlim_max = RLIM_INFINITY;
	setrlimit(RLIMIT_CORE, &core_limits);
}

void trikKernel::coreDumping::initCoreDumping(const QString &coreDumpPath)
{
	auto const & p = coreDumpPath.toLocal8Bit();
	dumpHandler_impl(0, strndup(p.constData(), p.size()));
	initSignals();
	setCoreLimits();
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2013-2014 winlin

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#ifndef SRS_RTMP_PROTOCOL_AMF0_HPP
#define SRS_RTMP_PROTOCOL_AMF0_HPP

/*
#include <srs_protocol_amf0.hpp>
*/

#include <srs_core.hpp>

#include <string>
#include <vector>

class SrsStream;
class SrsAmf0Object;
class SrsAmf0EcmaArray;
class __SrsUnSortedHashtable;
class __SrsAmf0ObjectEOF;

////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
// amf0 codec
// 1. SrsAmf0Any: read any from stream
//		SrsAmf0Any* pany = NULL;
//		if ((ret = srs_amf0_read_any(stream, &pany)) != ERROR_SUCCESS) {
//			return ret;
// 		}
//		srs_assert(pany); // if success, always valid object.
// 2. SrsAmf0Any: convert to specifid type, for instance, string
//		SrsAmf0Any* pany = ...
//		if (pany->is_string()) {
//			string v = pany->to_str();
//		}
// 3. SrsAmf0Any: parse specified type to any, for instance, string
//		SrsAmf0Any* pany = SrsAmf0Any::str("winlin");
// 4. SrsAmf0Size: get amf0 instance size
//		int size = SrsAmf0Size::str("winlin");
// 5. SrsAmf0Object: create the amf0 object.
//		SrsAmf0Object* obj = SrsAmf0Any::object();
// 5. SrsAmf0EcmaArray: create the amf0 ecma array.
//		SrsAmf0EcmaArray* arr = SrsAmf0Any::ecma_array();
// for detail usage, see interfaces of each object.
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////

/**
* any amf0 value.
* 2.1 Types Overview
* value-type = number-type | boolean-type | string-type | object-type
* 		| null-marker | undefined-marker | reference-type | ecma-array-type
* 		| strict-array-type | date-type | long-string-type | xml-document-type
* 		| typed-object-type
*/
class SrsAmf0Any
{
public:
	char marker;
public:
	SrsAmf0Any();
	virtual ~SrsAmf0Any();
public:
	virtual bool is_string();
	virtual bool is_boolean();
	virtual bool is_number();
	virtual bool is_null();
	virtual bool is_undefined();
	virtual bool is_object();
	virtual bool is_object_eof();
	virtual bool is_ecma_array();
public:
	/**
	* get the string of any when is_string() indicates true.
	* user must ensure the type is a string, or assert failed.
	*/
	virtual std::string to_str();
	/**
	* get the boolean of any when is_boolean() indicates true.
	* user must ensure the type is a boolean, or assert failed.
	*/
	virtual bool to_boolean();
	/**
	* get the number of any when is_number() indicates true.
	* user must ensure the type is a number, or assert failed.
	*/
	virtual double to_number();
	/**
	* get the object of any when is_object() indicates true.
	* user must ensure the type is a object, or assert failed.
	*/
	virtual SrsAmf0Object* to_object();
	/**
	* get the ecma array of any when is_ecma_array() indicates true.
	* user must ensure the type is a ecma array, or assert failed.
	*/
	virtual SrsAmf0EcmaArray* to_ecma_array();
public:
	/**
	* get the size of amf0 any, including the marker size.
	*/
	virtual int total_size() = 0;
	/**
	* read elem from stream
	*/
	virtual int read(SrsStream* stream) = 0;
	virtual int write(SrsStream* stream) = 0;
public:
	static SrsAmf0Any* str(const char* value = NULL);
	static SrsAmf0Any* boolean(bool value = false);
	static SrsAmf0Any* number(double value = 0.0);
	static SrsAmf0Any* null();
	static SrsAmf0Any* undefined();
	static SrsAmf0Object* object();
	static SrsAmf0Any* object_eof();
	static SrsAmf0EcmaArray* ecma_array();
public:
	static int discovery(SrsStream* stream, SrsAmf0Any** ppvalue);
};

/**
* 2.5 Object Type
* anonymous-object-type = object-marker *(object-property)
* object-property = (UTF-8 value-type) | (UTF-8-empty object-end-marker)
*/
class SrsAmf0Object : public SrsAmf0Any
{
private:
	__SrsUnSortedHashtable* properties;
	__SrsAmf0ObjectEOF* eof;

private:
	// use SrsAmf0Any::object() to create it.
	friend class SrsAmf0Any;
	SrsAmf0Object();
public:
	virtual ~SrsAmf0Object();

public:
	virtual int total_size();
	virtual int read(SrsStream* stream);
	virtual int write(SrsStream* stream);

public:
	virtual int count();
	// @remark: max index is count().
	virtual std::string key_at(int index);
	// @remark: max index is count().
	virtual SrsAmf0Any* value_at(int index);

public:
	virtual void set(std::string key, SrsAmf0Any* value);
	virtual SrsAmf0Any* get_property(std::string name);
	virtual SrsAmf0Any* ensure_property_string(std::string name);
	virtual SrsAmf0Any* ensure_property_number(std::string name);
};

/**
* 2.10 ECMA Array Type
* ecma-array-type = associative-count *(object-property)
* associative-count = U32
* object-property = (UTF-8 value-type) | (UTF-8-empty object-end-marker)
*/
class SrsAmf0EcmaArray : public SrsAmf0Any
{
private:
	__SrsUnSortedHashtable* properties;
	__SrsAmf0ObjectEOF* eof;
	int32_t _count;

private:
	// use SrsAmf0Any::ecma_array() to create it.
	friend class SrsAmf0Any;
	SrsAmf0EcmaArray();
public:
	virtual ~SrsAmf0EcmaArray();

public:
	virtual int total_size();
	virtual int read(SrsStream* stream);
	virtual int write(SrsStream* stream);

public:
	virtual void clear();
	virtual int count();
	// @remark: max index is count().
	virtual std::string key_at(int index);
	// @remark: max index is count().
	virtual SrsAmf0Any* value_at(int index);

public:
	virtual void set(std::string key, SrsAmf0Any* value);
	virtual SrsAmf0Any* get_property(std::string name);
	virtual SrsAmf0Any* ensure_property_string(std::string name);
};

/**
* the class to get amf0 object size
*/
class SrsAmf0Size
{
public:
	static int utf8(std::string value);
	static int str(std::string value);
	static int number();
	static int null();
	static int undefined();
	static int boolean();
	static int object(SrsAmf0Object* obj);
	static int object_eof();
	static int ecma_array(SrsAmf0EcmaArray* arr);
	static int any(SrsAmf0Any* o);
};

/**
* read anything from stream.
* @param ppvalue, the output amf0 any elem.
* 		NULL if error; otherwise, never NULL and user must free it.
*/
extern int srs_amf0_read_any(SrsStream* stream, SrsAmf0Any** ppvalue);

/**
* read amf0 string from stream.
* 2.4 String Type
* string-type = string-marker UTF-8
*/
extern int srs_amf0_read_string(SrsStream* stream, std::string& value);
extern int srs_amf0_write_string(SrsStream* stream, std::string value);

/**
* read amf0 boolean from stream.
* 2.4 String Type
* boolean-type = boolean-marker U8
* 		0 is false, <> 0 is true
*/
extern int srs_amf0_read_boolean(SrsStream* stream, bool& value);
extern int srs_amf0_write_boolean(SrsStream* stream, bool value);

/**
* read amf0 number from stream.
* 2.2 Number Type
* number-type = number-marker DOUBLE
*/
extern int srs_amf0_read_number(SrsStream* stream, double& value);
extern int srs_amf0_write_number(SrsStream* stream, double value);

/**
* read amf0 null from stream.
* 2.7 null Type
* null-type = null-marker
*/
extern int srs_amf0_read_null(SrsStream* stream);
extern int srs_amf0_write_null(SrsStream* stream);

/**
* read amf0 undefined from stream.
* 2.8 undefined Type
* undefined-type = undefined-marker
*/
extern int srs_amf0_read_undefined(SrsStream* stream);
extern int srs_amf0_write_undefined(SrsStream* stream);

#endif<commit_msg>refine summary for amf0 hpp<commit_after>/*
The MIT License (MIT)

Copyright (c) 2013-2014 winlin

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#ifndef SRS_RTMP_PROTOCOL_AMF0_HPP
#define SRS_RTMP_PROTOCOL_AMF0_HPP

/*
#include <srs_protocol_amf0.hpp>
*/

#include <srs_core.hpp>

#include <string>
#include <vector>

class SrsStream;
class SrsAmf0Object;
class SrsAmf0EcmaArray;
class __SrsUnSortedHashtable;
class __SrsAmf0ObjectEOF;

////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
// amf0 codec
// 1. SrsAmf0Any: read any from stream
//		SrsAmf0Any* pany = NULL;
//		if ((ret = srs_amf0_read_any(stream, &pany)) != ERROR_SUCCESS) {
//			return ret;
// 		}
//		srs_assert(pany); // if success, always valid object.
// 2. SrsAmf0Any: convert to specifid type, for instance, string
//		SrsAmf0Any* pany = ...
//		if (pany->is_string()) {
//			string v = pany->to_str();
//		}
// 3. SrsAmf0Any: parse specified type to any, for instance, string
//		SrsAmf0Any* pany = SrsAmf0Any::str("winlin");
// 4. SrsAmf0Size: get amf0 instance size
//		int size = SrsAmf0Size::str("winlin");
// 5. SrsAmf0Object: create the amf0 object.
//		SrsAmf0Object* obj = SrsAmf0Any::object();
// 5. SrsAmf0EcmaArray: create the amf0 ecma array.
//		SrsAmf0EcmaArray* arr = SrsAmf0Any::ecma_array();
//
// please carefully the size and count of amf0 any:
// 1. total_size(): the total memory size the object wrote to buffer.
// 2. count(): the total element count of object, for instance, the properties
//		of amf0 object, used for key_at/value_at loop.
//
// for detail usage, see interfaces of each object.
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////

/**
* any amf0 value.
* 2.1 Types Overview
* value-type = number-type | boolean-type | string-type | object-type
* 		| null-marker | undefined-marker | reference-type | ecma-array-type
* 		| strict-array-type | date-type | long-string-type | xml-document-type
* 		| typed-object-type
*/
class SrsAmf0Any
{
public:
	char marker;
public:
	SrsAmf0Any();
	virtual ~SrsAmf0Any();
public:
	virtual bool is_string();
	virtual bool is_boolean();
	virtual bool is_number();
	virtual bool is_null();
	virtual bool is_undefined();
	virtual bool is_object();
	virtual bool is_object_eof();
	virtual bool is_ecma_array();
public:
	/**
	* get the string of any when is_string() indicates true.
	* user must ensure the type is a string, or assert failed.
	*/
	virtual std::string to_str();
	/**
	* get the boolean of any when is_boolean() indicates true.
	* user must ensure the type is a boolean, or assert failed.
	*/
	virtual bool to_boolean();
	/**
	* get the number of any when is_number() indicates true.
	* user must ensure the type is a number, or assert failed.
	*/
	virtual double to_number();
	/**
	* get the object of any when is_object() indicates true.
	* user must ensure the type is a object, or assert failed.
	*/
	virtual SrsAmf0Object* to_object();
	/**
	* get the ecma array of any when is_ecma_array() indicates true.
	* user must ensure the type is a ecma array, or assert failed.
	*/
	virtual SrsAmf0EcmaArray* to_ecma_array();
public:
	/**
	* get the size of amf0 any, including the marker size.
	*/
	virtual int total_size() = 0;
	/**
	* read elem from stream
	*/
	virtual int read(SrsStream* stream) = 0;
	virtual int write(SrsStream* stream) = 0;
public:
	static SrsAmf0Any* str(const char* value = NULL);
	static SrsAmf0Any* boolean(bool value = false);
	static SrsAmf0Any* number(double value = 0.0);
	static SrsAmf0Any* null();
	static SrsAmf0Any* undefined();
	static SrsAmf0Object* object();
	static SrsAmf0Any* object_eof();
	static SrsAmf0EcmaArray* ecma_array();
public:
	static int discovery(SrsStream* stream, SrsAmf0Any** ppvalue);
};

/**
* 2.5 Object Type
* anonymous-object-type = object-marker *(object-property)
* object-property = (UTF-8 value-type) | (UTF-8-empty object-end-marker)
*/
class SrsAmf0Object : public SrsAmf0Any
{
private:
	__SrsUnSortedHashtable* properties;
	__SrsAmf0ObjectEOF* eof;

private:
	// use SrsAmf0Any::object() to create it.
	friend class SrsAmf0Any;
	SrsAmf0Object();
public:
	virtual ~SrsAmf0Object();

public:
	virtual int total_size();
	virtual int read(SrsStream* stream);
	virtual int write(SrsStream* stream);

public:
	virtual int count();
	// @remark: max index is count().
	virtual std::string key_at(int index);
	// @remark: max index is count().
	virtual SrsAmf0Any* value_at(int index);

public:
	virtual void set(std::string key, SrsAmf0Any* value);
	virtual SrsAmf0Any* get_property(std::string name);
	virtual SrsAmf0Any* ensure_property_string(std::string name);
	virtual SrsAmf0Any* ensure_property_number(std::string name);
};

/**
* 2.10 ECMA Array Type
* ecma-array-type = associative-count *(object-property)
* associative-count = U32
* object-property = (UTF-8 value-type) | (UTF-8-empty object-end-marker)
*/
class SrsAmf0EcmaArray : public SrsAmf0Any
{
private:
	__SrsUnSortedHashtable* properties;
	__SrsAmf0ObjectEOF* eof;
	int32_t _count;

private:
	// use SrsAmf0Any::ecma_array() to create it.
	friend class SrsAmf0Any;
	SrsAmf0EcmaArray();
public:
	virtual ~SrsAmf0EcmaArray();

public:
	virtual int total_size();
	virtual int read(SrsStream* stream);
	virtual int write(SrsStream* stream);

public:
	virtual void clear();
	virtual int count();
	// @remark: max index is count().
	virtual std::string key_at(int index);
	// @remark: max index is count().
	virtual SrsAmf0Any* value_at(int index);

public:
	virtual void set(std::string key, SrsAmf0Any* value);
	virtual SrsAmf0Any* get_property(std::string name);
	virtual SrsAmf0Any* ensure_property_string(std::string name);
};

/**
* the class to get amf0 object size
*/
class SrsAmf0Size
{
public:
	static int utf8(std::string value);
	static int str(std::string value);
	static int number();
	static int null();
	static int undefined();
	static int boolean();
	static int object(SrsAmf0Object* obj);
	static int object_eof();
	static int ecma_array(SrsAmf0EcmaArray* arr);
	static int any(SrsAmf0Any* o);
};

/**
* read anything from stream.
* @param ppvalue, the output amf0 any elem.
* 		NULL if error; otherwise, never NULL and user must free it.
*/
extern int srs_amf0_read_any(SrsStream* stream, SrsAmf0Any** ppvalue);

/**
* read amf0 string from stream.
* 2.4 String Type
* string-type = string-marker UTF-8
*/
extern int srs_amf0_read_string(SrsStream* stream, std::string& value);
extern int srs_amf0_write_string(SrsStream* stream, std::string value);

/**
* read amf0 boolean from stream.
* 2.4 String Type
* boolean-type = boolean-marker U8
* 		0 is false, <> 0 is true
*/
extern int srs_amf0_read_boolean(SrsStream* stream, bool& value);
extern int srs_amf0_write_boolean(SrsStream* stream, bool value);

/**
* read amf0 number from stream.
* 2.2 Number Type
* number-type = number-marker DOUBLE
*/
extern int srs_amf0_read_number(SrsStream* stream, double& value);
extern int srs_amf0_write_number(SrsStream* stream, double value);

/**
* read amf0 null from stream.
* 2.7 null Type
* null-type = null-marker
*/
extern int srs_amf0_read_null(SrsStream* stream);
extern int srs_amf0_write_null(SrsStream* stream);

/**
* read amf0 undefined from stream.
* 2.8 undefined Type
* undefined-type = undefined-marker
*/
extern int srs_amf0_read_undefined(SrsStream* stream);
extern int srs_amf0_write_undefined(SrsStream* stream);

#endif<|endoftext|>


<commit_before>/// @file
/// @author uentity
/// @date 22.09.2017
/// @brief BS tree Python bindings
/// @copyright
/// This Source Code Form is subject to the terms of the Mozilla Public License,
/// v. 2.0. If a copy of the MPL was not distributed with this file,
/// You can obtain one at https://mozilla.org/MPL/2.0/

#include <bs/bs.h>
#include <bs/link.h>
#include <bs/node.h>
#include <boost/uuid/uuid_io.hpp>
#include <boost/uuid/string_generator.hpp>
#include <string>

NAMESPACE_BEGIN(blue_sky)
NAMESPACE_BEGIN(python)
using namespace tree;

namespace {

template<typename Link = link>
class py_link : public Link {
public:
	using Link::Link;

	sp_link clone() const override {
		PYBIND11_OVERLOAD_PURE(sp_link, Link, clone, );
	}

	sp_obj data() const override {
		PYBIND11_OVERLOAD_PURE(sp_obj, Link, data, );
	}

	std::string type_id() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, type_id, );
	}

	std::string oid() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, oid, );
	}

	std::string obj_type_id() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, obj_type_id, );
	}

	sp_node data_node() const override {
		PYBIND11_OVERLOAD_PURE(sp_node, Link, data_node, );
	}
};

static boost::uuids::string_generator uuid_from_str;

}

void py_bind_tree(py::module& m) {
	py::class_<inode>(m, "inode")
		.def(py::init<std::string, std::string>())
		.def_readonly("owner", &inode::owner)
		.def_readonly("group", &inode::group)
		.def_property_readonly("suid", [](const inode& i) { return i.suid; })
		.def_property_readonly("sgid", [](const inode& i) { return i.sgid; })
		.def_property_readonly("sticky", [](const inode& i) { return i.sticky; })
		.def_property_readonly("u", [](const inode& i) { return i.u; })
		.def_property_readonly("g", [](const inode& i) { return i.g; })
		.def_property_readonly("o", [](const inode& i) { return i.o; })
	;

	py::class_<link, py_link<>, std::shared_ptr<link>> link_pyface(m, "link");
	link_pyface
		.def(py::init<std::string>())
		.def("clone", &link::clone)
		.def("data", &link::data)
		.def("type_id", &link::type_id)
		.def("oid", &link::oid)
		.def("obj_type_id", &link::obj_type_id)
		.def("data_node", &link::data_node)
		.def_property_readonly("id", [](const link& L) {
			return boost::uuids::to_string(L.id());
		})
		.def_property_readonly("name", &link::name)
		.def_property("inode",
			py::overload_cast<>(&link::get_inode, py::const_),
			py::overload_cast<>(&link::get_inode)
		)
		.def_property_readonly("owner", &link::owner)
	;

	py::class_<hard_link, link, py_link<hard_link>, std::shared_ptr<hard_link>>(m, "hard_link")
		.def(py::init<std::string, sp_obj>())
	;
	py::class_<weak_link, link, py_link<weak_link>, std::shared_ptr<weak_link>>(m, "weak_link")
		.def(py::init<std::string, const sp_obj&>())
	;

	// forward define node
	py::class_<node, objbase, std::shared_ptr<node>> node_pyface(m, "node");

	// export node's Key enum
	py::enum_<node::Key>(node_pyface, "Key")
		.value("ID", node::Key::ID)
		.value("OID", node::Key::OID)
		.value("Name", node::Key::Name)
		.value("Type", node::Key::Type)
	;
	// export node's insert policy
	py::enum_<node::InsertPolicy>(node_pyface, "InsertPolicy")
		.value("AllowDupNames", node::InsertPolicy::AllowDupNames)
		.value("DenyDupNames", node::InsertPolicy::DenyDupNames)
		.value("RenameDup", node::InsertPolicy::RenameDup)
		.value("DenyDupOID", node::InsertPolicy::DenyDupOID)
		.value("Merge", node::InsertPolicy::Merge)
		.export_values();
	;

	// fill node with methods
	node_pyface
		BSPY_EXPORT_DEF(node)
		.def(py::init<>())
		.def("__len__", &node::size)
		.def("__iter__",
			[](const node& N) { return py::make_iterator(N.begin(), N.end()); },
			py::keep_alive<0, 1>()
		)

		// check by link name or object instance (object's ID)
		.def("__contains__", [](const node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				return N.find(py::cast<std::string>(key)) != N.end<>();
			return N.find_oid(py::cast<sp_obj>(key)->id()) != N.end<>();
		})
		// check by link ID
		.def("contains_lid", [](const node& N, const std::string& lid) {
			return N.find(uuid_from_str(lid)) != N.end<>();
		}, "LID", "Check if node contains link with given ID")
		// check by object ID
		.def("contains_oid", [](const node& N, const std::string& oid) {
			return N.find_oid(oid) != N.end<>();
		}, "OID", "Check if node contains object with given ID")
		// check by object type
		.def("contains_type", [](const node& N, const std::string& type_id) {
			return N.equal_type(type_id).first != N.end<node::Key::Type>();
		}, "obj_type_id", "Check if node contains objects with given type")

		// search by link name
		.def("__getitem__", [](const node& N, const std::string& link_name) {
			auto r = N.find(link_name);
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain link with name '" + link_name + "'");
		}, py::return_value_policy::reference_internal, "link_name"_a)
		// search by object instance
		.def("__getitem__", [](const node& N, const sp_obj& obj) {
			auto r = N.find(obj->id());
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain object with ID = " + obj->id());
		}, py::return_value_policy::reference_internal, "object"_a)
		// search by object ID
		.def("find_oid", [](const node& N, const std::string& oid) {
			auto r = N.find_oid(oid);
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain object with ID - " + oid);
		}, py::return_value_policy::reference_internal, "OID"_a, "Find item by object ID")
		// search by link ID
		.def("find_lid", [](const node& N, const std::string& lid) {
			auto r = N.find(uuid_from_str(lid));
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain link with ID - " + lid);
		}, py::return_value_policy::reference_internal, "LID"_a, "Find item by object link ID")

		// deep search by link name or object instance (OID)
		.def("deep_search", [](const node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				return N.deep_search(py::cast<std::string>(key));
			return N.deep_search_oid(py::cast<sp_obj>(key)->id());
		}, "key"_a, "Deep search for link with given name or by object instance (OID)")
		.def("deep_search_lid", [](const node& N, const std::string& lid) {
			return N.deep_search(uuid_from_str(lid));
		}, "LID"_a, "Deep search for link with given ID")
		// deep search by object ID
		.def("deep_search_oid", [](const node& N, const std::string& oid) {
			return N.deep_search_oid(oid);
		}, "OID"_a, "Deep search for object with given ID")

		.def("equal_range", [](const node& N, const std::string& link_name) {
			auto r = N.equal_range(link_name);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "link_name"_a)
		.def("equal_range_oid", [](const node& N, const std::string& oid) {
			auto r = N.equal_range_oid(oid);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "OID"_a)
		.def("equal_type", [](const node& N, const std::string& type_id) {
			auto r = N.equal_type(type_id);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "obj_type_id"_a)

		// insert hard link to given object
		.def("__setitem__", [](node& N, std::string link_name, sp_obj obj) {
			N.insert(std::move(link_name), std::move(obj));
		}, "link_name"_a, "obj"_a)
		// insert given link
		.def("insert", [](node& N, sp_link l, uint pol = node::AllowDupNames) {
			return N.insert(std::move(l)).second;
		}, "link"_a, "pol"_a = node::AllowDupNames, "Insert given link")
		// insert hard link to given object
		.def("insert", [](node& N, std::string name, sp_obj obj, uint pol = node::AllowDupNames) {
			return N.insert(std::move(name), std::move(obj), pol).second;
		}, "name"_a, "obj"_a, "pol"_a = node::AllowDupNames, "Insert hard link to given object")

		// erase by given link name or object instance (object's ID)
		.def("__delitem__", [](node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				N.erase(py::cast<std::string>(key));
			N.erase_oid(py::cast<sp_obj>(key)->id());
		})
		.def("erase", (void (node::*)(const std::string&))&node::erase,
			"link_name"_a, "Erase links with given name")
		.def("erase_oid", &node::erase_oid, "obj"_a, "Erase links to given object")
		.def("erase_lid", [](node& N, const std::string& key) {
			N.erase(uuid_from_str(key));
		}, "lid"_a, "Erase link with given ID")
		.def("erase_type", [](node& N, const std::string& type_id) {
			N.erase_type(type_id);
		}, "obj_type_id"_a, "Erase all links pointing to objects with given type")

		// misc functions
		.def_property_readonly("size", &node::size)
		.def_property_readonly("empty", &node::empty)
		.def("clear", &node::clear, "Clears all node contents")
		.def("keys", [](const node& N) {
			std::vector<std::string> res;
			res.reserve(N.size());
			for(const auto& i : N)
				res.emplace_back(i->name());
			return res;
		})
	;
}

NAMESPACE_END(python)
NAMESPACE_END(blue_sky)

<commit_msg>node.python: add `__getitem__` overload for integer index<commit_after>/// @file
/// @author uentity
/// @date 22.09.2017
/// @brief BS tree Python bindings
/// @copyright
/// This Source Code Form is subject to the terms of the Mozilla Public License,
/// v. 2.0. If a copy of the MPL was not distributed with this file,
/// You can obtain one at https://mozilla.org/MPL/2.0/

#include <bs/bs.h>
#include <bs/link.h>
#include <bs/node.h>
#include <boost/uuid/uuid_io.hpp>
#include <boost/uuid/string_generator.hpp>
#include <string>

NAMESPACE_BEGIN(blue_sky)
NAMESPACE_BEGIN(python)
using namespace tree;

namespace {

template<typename Link = link>
class py_link : public Link {
public:
	using Link::Link;

	sp_link clone() const override {
		PYBIND11_OVERLOAD_PURE(sp_link, Link, clone, );
	}

	sp_obj data() const override {
		PYBIND11_OVERLOAD_PURE(sp_obj, Link, data, );
	}

	std::string type_id() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, type_id, );
	}

	std::string oid() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, oid, );
	}

	std::string obj_type_id() const override {
		PYBIND11_OVERLOAD_PURE(std::string, Link, obj_type_id, );
	}

	sp_node data_node() const override {
		PYBIND11_OVERLOAD_PURE(sp_node, Link, data_node, );
	}
};

static boost::uuids::string_generator uuid_from_str;

}

void py_bind_tree(py::module& m) {
	py::class_<inode>(m, "inode")
		.def(py::init<std::string, std::string>())
		.def_readonly("owner", &inode::owner)
		.def_readonly("group", &inode::group)
		.def_property_readonly("suid", [](const inode& i) { return i.suid; })
		.def_property_readonly("sgid", [](const inode& i) { return i.sgid; })
		.def_property_readonly("sticky", [](const inode& i) { return i.sticky; })
		.def_property_readonly("u", [](const inode& i) { return i.u; })
		.def_property_readonly("g", [](const inode& i) { return i.g; })
		.def_property_readonly("o", [](const inode& i) { return i.o; })
	;

	py::class_<link, py_link<>, std::shared_ptr<link>> link_pyface(m, "link");
	link_pyface
		.def(py::init<std::string>())
		.def("clone", &link::clone)
		.def("data", &link::data)
		.def("type_id", &link::type_id)
		.def("oid", &link::oid)
		.def("obj_type_id", &link::obj_type_id)
		.def("data_node", &link::data_node)
		.def_property_readonly("id", [](const link& L) {
			return boost::uuids::to_string(L.id());
		})
		.def_property_readonly("name", &link::name)
		.def_property("inode",
			py::overload_cast<>(&link::get_inode, py::const_),
			py::overload_cast<>(&link::get_inode)
		)
		.def_property_readonly("owner", &link::owner)
	;

	py::class_<hard_link, link, py_link<hard_link>, std::shared_ptr<hard_link>>(m, "hard_link")
		.def(py::init<std::string, sp_obj>())
	;
	py::class_<weak_link, link, py_link<weak_link>, std::shared_ptr<weak_link>>(m, "weak_link")
		.def(py::init<std::string, const sp_obj&>())
	;

	// forward define node
	py::class_<node, objbase, std::shared_ptr<node>> node_pyface(m, "node");

	// export node's Key enum
	py::enum_<node::Key>(node_pyface, "Key")
		.value("ID", node::Key::ID)
		.value("OID", node::Key::OID)
		.value("Name", node::Key::Name)
		.value("Type", node::Key::Type)
	;
	// export node's insert policy
	py::enum_<node::InsertPolicy>(node_pyface, "InsertPolicy")
		.value("AllowDupNames", node::InsertPolicy::AllowDupNames)
		.value("DenyDupNames", node::InsertPolicy::DenyDupNames)
		.value("RenameDup", node::InsertPolicy::RenameDup)
		.value("DenyDupOID", node::InsertPolicy::DenyDupOID)
		.value("Merge", node::InsertPolicy::Merge)
		.export_values();
	;

	// fill node with methods
	node_pyface
		BSPY_EXPORT_DEF(node)
		.def(py::init<>())
		.def("__len__", &node::size)
		.def("__iter__",
			[](const node& N) { return py::make_iterator(N.begin(), N.end()); },
			py::keep_alive<0, 1>()
		)

		// check by link name or object instance (object's ID)
		.def("__contains__", [](const node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				return N.find(py::cast<std::string>(key)) != N.end<>();
			return N.find_oid(py::cast<sp_obj>(key)->id()) != N.end<>();
		})
		// check by link ID
		.def("contains_lid", [](const node& N, const std::string& lid) {
			return N.find(uuid_from_str(lid)) != N.end<>();
		}, "LID", "Check if node contains link with given ID")
		// check by object ID
		.def("contains_oid", [](const node& N, const std::string& oid) {
			return N.find_oid(oid) != N.end<>();
		}, "OID", "Check if node contains object with given ID")
		// check by object type
		.def("contains_type", [](const node& N, const std::string& type_id) {
			return N.equal_type(type_id).first != N.end<node::Key::Type>();
		}, "obj_type_id", "Check if node contains objects with given type")

		// search by link name
		.def("__getitem__", [](const node& N, const std::string& link_name) {
			auto r = N.find(link_name);
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain link with name '" + link_name + "'");
		}, py::return_value_policy::reference_internal, "link_name"_a)
		// search by object instance
		.def("__getitem__", [](const node& N, const sp_obj& obj) {
			auto r = N.find(obj->id());
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain object with ID = " + obj->id());
		}, py::return_value_policy::reference_internal, "object"_a)
		// get item by int index
		.def("__getitem__", [](const node& N, long idx) {
			if(idx < 0 || ulong(idx) >= N.size())
				throw py::key_error("Index out of bounds");
			auto r = N.begin();
			std::advance(r, idx);
			return *r;
		}, py::return_value_policy::reference_internal, "link_name"_a)
		// search by object ID
		.def("find_oid", [](const node& N, const std::string& oid) {
			auto r = N.find_oid(oid);
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain object with ID - " + oid);
		}, py::return_value_policy::reference_internal, "OID"_a, "Find item by object ID")
		// search by link ID
		.def("find_lid", [](const node& N, const std::string& lid) {
			auto r = N.find(uuid_from_str(lid));
			if(r != N.end<>()) return *r;
			throw py::key_error("Node doesn't contain link with ID - " + lid);
		}, py::return_value_policy::reference_internal, "LID"_a, "Find item by object link ID")

		// deep search by link name or object instance (OID)
		.def("deep_search", [](const node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				return N.deep_search(py::cast<std::string>(key));
			return N.deep_search_oid(py::cast<sp_obj>(key)->id());
		}, "key"_a, "Deep search for link with given name or by object instance (OID)")
		.def("deep_search_lid", [](const node& N, const std::string& lid) {
			return N.deep_search(uuid_from_str(lid));
		}, "LID"_a, "Deep search for link with given ID")
		// deep search by object ID
		.def("deep_search_oid", [](const node& N, const std::string& oid) {
			return N.deep_search_oid(oid);
		}, "OID"_a, "Deep search for object with given ID")

		.def("equal_range", [](const node& N, const std::string& link_name) {
			auto r = N.equal_range(link_name);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "link_name"_a)
		.def("equal_range_oid", [](const node& N, const std::string& oid) {
			auto r = N.equal_range_oid(oid);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "OID"_a)
		.def("equal_type", [](const node& N, const std::string& type_id) {
			auto r = N.equal_type(type_id);
			return py::make_iterator(r.first, r.second);
		}, py::keep_alive<0, 1>(), "obj_type_id"_a)

		// insert hard link to given object
		.def("__setitem__", [](node& N, std::string link_name, sp_obj obj) {
			N.insert(std::move(link_name), std::move(obj));
		}, "link_name"_a, "obj"_a)
		// insert given link
		.def("insert", [](node& N, sp_link l, uint pol = node::AllowDupNames) {
			return N.insert(std::move(l)).second;
		}, "link"_a, "pol"_a = node::AllowDupNames, "Insert given link")
		// insert hard link to given object
		.def("insert", [](node& N, std::string name, sp_obj obj, uint pol = node::AllowDupNames) {
			return N.insert(std::move(name), std::move(obj), pol).second;
		}, "name"_a, "obj"_a, "pol"_a = node::AllowDupNames, "Insert hard link to given object")

		// erase by given link name or object instance (object's ID)
		.def("__delitem__", [](node& N, py::object key) {
			if(PyString_Check(key.ptr()))
				N.erase(py::cast<std::string>(key));
			N.erase_oid(py::cast<sp_obj>(key)->id());
		})
		.def("erase", (void (node::*)(const std::string&))&node::erase,
			"link_name"_a, "Erase links with given name")
		.def("erase_oid", &node::erase_oid, "obj"_a, "Erase links to given object")
		.def("erase_lid", [](node& N, const std::string& key) {
			N.erase(uuid_from_str(key));
		}, "lid"_a, "Erase link with given ID")
		.def("erase_type", [](node& N, const std::string& type_id) {
			N.erase_type(type_id);
		}, "obj_type_id"_a, "Erase all links pointing to objects with given type")

		// misc functions
		.def_property_readonly("size", &node::size)
		.def_property_readonly("empty", &node::empty)
		.def("clear", &node::clear, "Clears all node contents")
		.def("keys", [](const node& N) {
			std::vector<std::string> res;
			res.reserve(N.size());
			for(const auto& i : N)
				res.emplace_back(i->name());
			return res;
		})
	;
}

NAMESPACE_END(python)
NAMESPACE_END(blue_sky)

<|endoftext|>


<commit_before>/// @file
/// @author uentity
/// @date 14.09.2016
/// @brief hard link implementation
/// @copyright
/// This Source Code Form is subject to the terms of the Mozilla Public License,
/// v. 2.0. If a copy of the MPL was not distributed with this file,
/// You can obtain one at https://mozilla.org/MPL/2.0/

#include <bs/tree/link.h>
#include <bs/tree/node.h>
#include <bs/tree/errors.h>
#include <bs/kernel/types_factory.h>

#include <bs/serialize/tree.h>
#include <bs/serialize/cafbind.h>

#include "link_actor.h"

OMIT_OBJ_SERIALIZATION

NAMESPACE_BEGIN(blue_sky::tree)
/*-----------------------------------------------------------------------------
 *  hard_link
 *-----------------------------------------------------------------------------*/
///////////////////////////////////////////////////////////////////////////////
//  impl
//
hard_link_impl::hard_link_impl(std::string name, sp_obj data, Flags f)
	: super(std::move(name), data, f)
{
	set_data(std::move(data));
}

hard_link_impl::hard_link_impl()
	: super()
{}

auto hard_link_impl::data() -> result_or_err<sp_obj> { return data_; }

auto hard_link_impl::set_data(sp_obj obj) -> void {
	inode_ = make_inode(obj, inode_);
	if(data_ = std::move(obj); data_) {
		rs_reset(Req::Data, ReqReset::Always, ReqStatus::OK);
		rs_reset(
			Req::DataNode, ReqReset::Always,
			data_->is_node() ? ReqStatus::OK : ReqStatus::Error
		);
		//std::cout << "hard link " << to_string(id_) << ", name " << name_ << ": impl created " <<
		//	(int)status_[0].value << (int)status_[1].value << std::endl;
	}
}

auto hard_link_impl::spawn_actor(std::shared_ptr<link_impl> limpl) const -> caf::actor {
	return spawn_lactor<fast_link_actor>(std::move(limpl));
}

///////////////////////////////////////////////////////////////////////////////
//  class
//
hard_link::hard_link(std::string name, sp_obj data, Flags f) :
	super(std::make_shared<hard_link_impl>(std::move(name), data, f))
{}

hard_link::hard_link() :
	super(std::make_shared<hard_link_impl>(), false)
{}

link::sp_link hard_link::clone(bool deep) const {
	return std::make_shared<hard_link>(
		name(),
		deep ? kernel::tfactory::clone_object(pimpl()->data_) : pimpl()->data_,
		flags()
	);
}

std::string hard_link::type_id() const {
	return "hard_link";
}

auto hard_link::pimpl() const -> hard_link_impl* {
	return static_cast<hard_link_impl*>(super::pimpl());
}

/*-----------------------------------------------------------------------------
 *  weak_link
 *-----------------------------------------------------------------------------*/
///////////////////////////////////////////////////////////////////////////////
//  impl + actor
//
weak_link_impl::weak_link_impl(std::string name, const sp_obj& obj, Flags f)
	: super(std::move(name), obj, f)
{
	set_data(obj);
}

weak_link_impl::weak_link_impl()
	: super()
{}

auto weak_link_impl::data() -> result_or_err<sp_obj> {
	using result_t = result_or_err<sp_obj>;
	return data_.expired() ?
		tl::make_unexpected(error::quiet(Error::LinkExpired)) :
		result_t{ data_.lock() };
}

auto weak_link_impl::set_data(const sp_obj& obj) -> void {
	inode_ = make_inode(obj, inode_);
	if(data_ = obj; obj) {
		// set status silently
		rs_reset(Req::Data, ReqReset::Always, ReqStatus::OK);
		rs_reset(
			Req::DataNode, ReqReset::Always,
			obj->is_node() ? ReqStatus::OK : ReqStatus::Error
		);
	}
}

auto weak_link_impl::spawn_actor(std::shared_ptr<link_impl> limpl) const -> caf::actor {
	return spawn_lactor<fast_link_actor>(std::move(limpl));
}

///////////////////////////////////////////////////////////////////////////////
//  class
//
weak_link::weak_link(std::string name, const sp_obj& data, Flags f) :
	super(std::make_shared<weak_link_impl>(std::move(name), data, f))
{}

weak_link::weak_link() :
	super(std::make_shared<weak_link_impl>(), false)
{}

link::sp_link weak_link::clone(bool deep) const {
	// cannot make deep copy of object pointee
	return deep ? nullptr : std::make_shared<weak_link>(name(), pimpl()->data_.lock(), flags());
}

std::string weak_link::type_id() const {
	return "weak_link";
}

auto weak_link::pimpl() const -> weak_link_impl* {
	return static_cast<weak_link_impl*>(super::pimpl());
}

auto weak_link::propagate_handle() -> result_or_err<sp_node> {
	// weak link cannot be a node's handle
	return data_node_ex();
}

NAMESPACE_END(blue_sky::tree)
<commit_msg>tree/hard_link: set DataNode status to Void if pointee isn't a node<commit_after>/// @file
/// @author uentity
/// @date 14.09.2016
/// @brief hard link implementation
/// @copyright
/// This Source Code Form is subject to the terms of the Mozilla Public License,
/// v. 2.0. If a copy of the MPL was not distributed with this file,
/// You can obtain one at https://mozilla.org/MPL/2.0/

#include <bs/tree/link.h>
#include <bs/tree/node.h>
#include <bs/tree/errors.h>
#include <bs/kernel/types_factory.h>

#include <bs/serialize/tree.h>
#include <bs/serialize/cafbind.h>

#include "link_actor.h"

OMIT_OBJ_SERIALIZATION

NAMESPACE_BEGIN(blue_sky::tree)
/*-----------------------------------------------------------------------------
 *  hard_link
 *-----------------------------------------------------------------------------*/
///////////////////////////////////////////////////////////////////////////////
//  impl
//
hard_link_impl::hard_link_impl(std::string name, sp_obj data, Flags f)
	: super(std::move(name), data, f)
{
	set_data(std::move(data));
}

hard_link_impl::hard_link_impl()
	: super()
{}

auto hard_link_impl::data() -> result_or_err<sp_obj> { return data_; }

auto hard_link_impl::set_data(sp_obj obj) -> void {
	inode_ = make_inode(obj, inode_);
	if(data_ = std::move(obj); data_) {
		rs_reset(Req::Data, ReqReset::Always, ReqStatus::OK);
		rs_reset(
			Req::DataNode, ReqReset::Always,
			data_->is_node() ? ReqStatus::OK : ReqStatus::Void
		);
		//std::cout << "hard link " << to_string(id_) << ", name " << name_ << ": impl created " <<
		//	(int)status_[0].value << (int)status_[1].value << std::endl;
	}
}

auto hard_link_impl::spawn_actor(std::shared_ptr<link_impl> limpl) const -> caf::actor {
	return spawn_lactor<fast_link_actor>(std::move(limpl));
}

///////////////////////////////////////////////////////////////////////////////
//  class
//
hard_link::hard_link(std::string name, sp_obj data, Flags f) :
	super(std::make_shared<hard_link_impl>(std::move(name), data, f))
{}

hard_link::hard_link() :
	super(std::make_shared<hard_link_impl>(), false)
{}

link::sp_link hard_link::clone(bool deep) const {
	return std::make_shared<hard_link>(
		name(),
		deep ? kernel::tfactory::clone_object(pimpl()->data_) : pimpl()->data_,
		flags()
	);
}

std::string hard_link::type_id() const {
	return "hard_link";
}

auto hard_link::pimpl() const -> hard_link_impl* {
	return static_cast<hard_link_impl*>(super::pimpl());
}

/*-----------------------------------------------------------------------------
 *  weak_link
 *-----------------------------------------------------------------------------*/
///////////////////////////////////////////////////////////////////////////////
//  impl + actor
//
weak_link_impl::weak_link_impl(std::string name, const sp_obj& obj, Flags f)
	: super(std::move(name), obj, f)
{
	set_data(obj);
}

weak_link_impl::weak_link_impl()
	: super()
{}

auto weak_link_impl::data() -> result_or_err<sp_obj> {
	using result_t = result_or_err<sp_obj>;
	return data_.expired() ?
		tl::make_unexpected(error::quiet(Error::LinkExpired)) :
		result_t{ data_.lock() };
}

auto weak_link_impl::set_data(const sp_obj& obj) -> void {
	inode_ = make_inode(obj, inode_);
	if(data_ = obj; obj) {
		// set status silently
		rs_reset(Req::Data, ReqReset::Always, ReqStatus::OK);
		rs_reset(
			Req::DataNode, ReqReset::Always,
			obj->is_node() ? ReqStatus::OK : ReqStatus::Void
		);
	}
}

auto weak_link_impl::spawn_actor(std::shared_ptr<link_impl> limpl) const -> caf::actor {
	return spawn_lactor<fast_link_actor>(std::move(limpl));
}

///////////////////////////////////////////////////////////////////////////////
//  class
//
weak_link::weak_link(std::string name, const sp_obj& data, Flags f) :
	super(std::make_shared<weak_link_impl>(std::move(name), data, f))
{}

weak_link::weak_link() :
	super(std::make_shared<weak_link_impl>(), false)
{}

link::sp_link weak_link::clone(bool deep) const {
	// cannot make deep copy of object pointee
	return deep ? nullptr : std::make_shared<weak_link>(name(), pimpl()->data_.lock(), flags());
}

std::string weak_link::type_id() const {
	return "weak_link";
}

auto weak_link::pimpl() const -> weak_link_impl* {
	return static_cast<weak_link_impl*>(super::pimpl());
}

auto weak_link::propagate_handle() -> result_or_err<sp_node> {
	// weak link cannot be a node's handle
	return data_node_ex();
}

NAMESPACE_END(blue_sky::tree)
<|endoftext|>


<commit_before>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once
#ifndef HW_PCI_HPP
#define HW_PCI_HPP

#include <cstdint>
#include <arch.hpp>

namespace hw {

  typedef uint16_t port_t;

  static inline uint8_t inp(port_t port)
  {
    uint8_t ret;
#if defined(ARCH_x86)
    __asm__ volatile("xorl %eax,%eax");
    __asm__ volatile("inb %%dx,%%al"
                     :"=a"(ret)
                     :"d"(port));
#else
#error "inp() not implemented for selected arch"
#endif
    return ret;
  }

  static inline uint16_t inpw(port_t port)
  {
    uint16_t ret;
#if defined(ARCH_x86)
    __asm__ volatile("xorl %eax,%eax");
    __asm__ volatile("inw %%dx,%%ax"
                     :"=a"(ret)
                     :"d"(port));
#else
#error "inpw() not implemented for selected arch"
#endif
    return ret;
  }

  static inline uint32_t inpd(port_t port)
  {
    uint32_t ret;
#if defined(ARCH_x86)
    __asm__ volatile("xorl %eax,%eax");
    __asm__ volatile("inl %%dx,%%eax"
                     :"=a"(ret)
                     :"d"(port));
#else
#error "inpd() not implemented for selected arch"
#endif

    return ret;
  }

  static inline void outp(port_t port, uint8_t data)
  {
#if defined(ARCH_x86)
    __asm__ volatile ("outb %%al,%%dx"::"a" (data), "d"(port));
#else
#error "outp() not implemented for selected arch"
#endif
  }
  static inline void outpw(port_t port, uint16_t data)
  {
#if defined(ARCH_x86)
    __asm__ volatile ("outw %%ax,%%dx"::"a" (data), "d"(port));
#else
#error "outpw() not implemented for selected arch"
#endif
  }
  static inline void outpd(port_t port, uint32_t data)
  {
#if defined(ARCH_x86)
    __asm__ volatile ("outl %%eax,%%dx"::"a" (data), "d"(port));
#else
#error "outpd() not implemented for selected arch"
#endif
  }

} //< namespace hw

#endif
<commit_msg>pci: Refactor port IO inline asm<commit_after>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once
#ifndef HW_PCI_HPP
#define HW_PCI_HPP

#include <cstdint>
#include <arch.hpp>

namespace hw {

  static inline uint8_t inp(uint16_t port)
  {
    uint8_t ret;
#if defined(ARCH_x86)
    asm volatile("inb %w1,%b0" : "=a"(ret) : "Nd"(port));
#else
#error "inp() not implemented for selected arch"
#endif
    return ret;
  }

  static inline uint16_t inpw(uint16_t port)
  {
    uint16_t ret;
#if defined(ARCH_x86)
    asm volatile("inw %w1,%w0" : "=a"(ret) : "Nd"(port));
#else
#error "inpw() not implemented for selected arch"
#endif
    return ret;
  }

  static inline uint32_t inpd(uint16_t port)
  {
    uint32_t ret;
#if defined(ARCH_x86)
    asm volatile("inl %w1,%d0" : "=a"(ret) : "Nd"(port));
#else
#error "inpd() not implemented for selected arch"
#endif
    return ret;
  }

  static inline void outp(uint16_t port, uint8_t data)
  {
#if defined(ARCH_x86)
    asm volatile ("outb %b0,%w1" :: "a"(data), "Nd"(port));
#else
#error "outp() not implemented for selected arch"
#endif
  }
  static inline void outpw(uint16_t port, uint16_t data)
  {
#if defined(ARCH_x86)
    asm volatile ("outw %w0,%w1" :: "a" (data), "Nd"(port));
#else
#error "outpw() not implemented for selected arch"
#endif
  }
  static inline void outpd(uint16_t port, uint32_t data)
  {
#if defined(ARCH_x86)
    asm volatile ("outl %d0,%w1" :: "a" (data), "Nd"(port));
#else
#error "outpd() not implemented for selected arch"
#endif
  }

} //< namespace hw

#endif
<|endoftext|>