<commit_before>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "core/iostream.hh"
#include "core/fstream.hh"
#include "types.hh"
#include "compress.hh"
#include <seastar/core/byteorder.hh>

namespace sstables {

class file_writer {
    output_stream<char> _out;
    size_t _offset = 0;
public:
    file_writer(file f, file_output_stream_options options)
        : _out(make_file_output_stream(std::move(f), std::move(options))) {}

    file_writer(output_stream<char>&& out)
        : _out(std::move(out)) {}

    virtual ~file_writer() = default;
    file_writer(file_writer&&) = default;

    future<> write(const char* buf, size_t n) {
        _offset += n;
        return _out.write(buf, n);
    }
    future<> write(const bytes& s) {
        _offset += s.size();
        return _out.write(s);
    }
    future<> flush() {
        return _out.flush();
    }
    future<> close() {
        return _out.close();
    }
    size_t offset() {
        return _offset;
    }
};

class sizing_data_sink : public data_sink_impl {
    uint64_t& _size;
public:
    explicit sizing_data_sink(uint64_t& dest) : _size(dest) {
        _size = 0;
    }
    virtual temporary_buffer<char> allocate_buffer(size_t size) {
        return temporary_buffer<char>(size);
    }
    virtual future<> put(net::packet data) override {
        _size += data.len();
        return make_ready_future<>();
    }
    virtual future<> put(std::vector<temporary_buffer<char>> data) override {
        _size += boost::accumulate(data | boost::adaptors::transformed(std::mem_fn(&temporary_buffer<char>::size)), 0);
        return make_ready_future<>();
    }
    virtual future<> put(temporary_buffer<char> buf) override {
        _size += buf.size();
        return make_ready_future<>();
    }
    virtual future<> flush() override {
        return make_ready_future<>();
    }
    virtual future<> close() override {
        return make_ready_future<>();
    }
};

inline
output_stream<char>
make_sizing_output_stream(uint64_t& dest) {
    return output_stream<char>(data_sink(std::make_unique<sizing_data_sink>(std::ref(dest))), 4096);
}

// Must be called from a thread
template <typename T>
uint64_t
serialized_size(const T& object) {
    uint64_t size = 0;
    auto writer = file_writer(make_sizing_output_stream(size));
    write(writer, object);
    writer.flush().get();
    writer.close().get();
    return size;
}

output_stream<char> make_checksummed_file_output_stream(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options);

class checksummed_file_writer : public file_writer {
    checksum _c;
    uint32_t _full_checksum;
public:
    checksummed_file_writer(file f, file_output_stream_options options, bool checksum_file = false)
            : file_writer(make_checksummed_file_output_stream(std::move(f), _c, _full_checksum, checksum_file, options))
            , _c({uint32_t(std::min(size_t(DEFAULT_CHUNK_SIZE), size_t(options.buffer_size)))})
            , _full_checksum(init_checksum_adler32()) {}

    // Since we are exposing a reference to _full_checksum, we delete the move
    // constructor.  If it is moved, the reference will refer to the old
    // location.
    checksummed_file_writer(checksummed_file_writer&&) = delete;
    checksummed_file_writer(const checksummed_file_writer&) = default;

    checksum& finalize_checksum() {
        return _c;
    }
    uint32_t full_checksum() {
        return _full_checksum;
    }
};

class checksummed_file_data_sink_impl : public data_sink_impl {
    output_stream<char> _out;
    struct checksum& _c;
    uint32_t& _full_checksum;
    bool _checksum_file;
public:
    checksummed_file_data_sink_impl(file f, struct checksum& c, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options)
            : _out(make_file_output_stream(std::move(f), std::move(options)))
            , _c(c)
            , _full_checksum(full_file_checksum)
            , _checksum_file(checksum_file)
            {}

    future<> put(net::packet data) { abort(); }
    virtual future<> put(temporary_buffer<char> buf) override {
        // bufs will usually be a multiple of chunk size, but this won't be the case for
        // the last buffer being flushed.

        if (!_checksum_file) {
            _full_checksum = checksum_adler32(_full_checksum, buf.begin(), buf.size());
        } else {
            for (size_t offset = 0; offset < buf.size(); offset += _c.chunk_size) {
                size_t size = std::min(size_t(_c.chunk_size), buf.size() - offset);
                uint32_t per_chunk_checksum = init_checksum_adler32();

                per_chunk_checksum = checksum_adler32(per_chunk_checksum, buf.begin() + offset, size);
                _full_checksum = checksum_adler32_combine(_full_checksum, per_chunk_checksum, size);
                _c.checksums.push_back(per_chunk_checksum);
            }
        }
        auto f = _out.write(buf.begin(), buf.size());
        return f.then([buf = std::move(buf)] {});
    }

    virtual future<> close() {
        // Nothing to do, because close at the file_stream level will call flush on us.
        return _out.close();
    }
};

class checksummed_file_data_sink : public data_sink {
public:
    checksummed_file_data_sink(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options)
        : data_sink(std::make_unique<checksummed_file_data_sink_impl>(std::move(f), cinfo, full_file_checksum, checksum_file, std::move(options))) {}
};

inline
output_stream<char> make_checksummed_file_output_stream(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options) {
    auto buffer_size = options.buffer_size;
    return output_stream<char>(checksummed_file_data_sink(std::move(f), cinfo, full_file_checksum, checksum_file, std::move(options)), buffer_size, true);
}

// compressed_file_data_sink_impl works as a filter for a file output stream,
// where the buffer flushed will be compressed and its checksum computed, then
// the result passed to a regular output stream.
class compressed_file_data_sink_impl : public data_sink_impl {
    output_stream<char> _out;
    sstables::compression* _compression_metadata;
    size_t _pos = 0;
public:
    compressed_file_data_sink_impl(file f, sstables::compression* cm, file_output_stream_options options)
            : _out(make_file_output_stream(std::move(f), options))
            , _compression_metadata(cm) {}

    future<> put(net::packet data) { abort(); }
    virtual future<> put(temporary_buffer<char> buf) override {
        auto output_len = _compression_metadata->compress_max_size(buf.size());
        // account space for checksum that goes after compressed data.
        temporary_buffer<char> compressed(output_len + 4);

        // compress flushed data.
        auto len = _compression_metadata->compress(buf.get(), buf.size(), compressed.get_write(), output_len);
        if (len > output_len) {
            throw std::runtime_error("possible overflow during compression");
        }

        _compression_metadata->offsets.elements.push_back(_pos);
        // account compressed data + 32-bit checksum.
        _pos += len + 4;
        _compression_metadata->set_compressed_file_length(_pos);
        // total length of the uncompressed data.
        _compression_metadata->data_len += buf.size();

        // compute 32-bit checksum for compressed data.
        uint32_t per_chunk_checksum = checksum_adler32(compressed.get(), len);
        _compression_metadata->update_full_checksum(per_chunk_checksum, len);

        // write checksum into buffer after compressed data.
        write_be<uint32_t>(compressed.get_write() + len, per_chunk_checksum);

        compressed.trim(len + 4);

        auto f = _out.write(compressed.get(), compressed.size());
        return f.then([compressed = std::move(compressed)] {});
    }
    virtual future<> close() {
        return _out.close();
    }
};

class compressed_file_data_sink : public data_sink {
public:
    compressed_file_data_sink(file f, sstables::compression* cm, file_output_stream_options options)
        : data_sink(std::make_unique<compressed_file_data_sink_impl>(
                std::move(f), cm, options)) {}
};

static inline output_stream<char> make_compressed_file_output_stream(file f, file_output_stream_options options, sstables::compression* cm) {
    // buffer of output stream is set to chunk length, because flush must
    // happen every time a chunk was filled up.
    auto outer_buffer_size = cm->uncompressed_chunk_length();
    return output_stream<char>(compressed_file_data_sink(std::move(f), cm, options), outer_buffer_size, true);
}

}
<commit_msg>sstables: make writer::offset const qualified and uint64_t<commit_after>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "core/iostream.hh"
#include "core/fstream.hh"
#include "types.hh"
#include "compress.hh"
#include <seastar/core/byteorder.hh>

namespace sstables {

class file_writer {
    output_stream<char> _out;
    uint64_t _offset = 0;
public:
    file_writer(file f, file_output_stream_options options)
        : _out(make_file_output_stream(std::move(f), std::move(options))) {}

    file_writer(output_stream<char>&& out)
        : _out(std::move(out)) {}

    virtual ~file_writer() = default;
    file_writer(file_writer&&) = default;

    future<> write(const char* buf, size_t n) {
        _offset += n;
        return _out.write(buf, n);
    }
    future<> write(const bytes& s) {
        _offset += s.size();
        return _out.write(s);
    }
    future<> flush() {
        return _out.flush();
    }
    future<> close() {
        return _out.close();
    }
    uint64_t offset() const {
        return _offset;
    }
};

class sizing_data_sink : public data_sink_impl {
    uint64_t& _size;
public:
    explicit sizing_data_sink(uint64_t& dest) : _size(dest) {
        _size = 0;
    }
    virtual temporary_buffer<char> allocate_buffer(size_t size) {
        return temporary_buffer<char>(size);
    }
    virtual future<> put(net::packet data) override {
        _size += data.len();
        return make_ready_future<>();
    }
    virtual future<> put(std::vector<temporary_buffer<char>> data) override {
        _size += boost::accumulate(data | boost::adaptors::transformed(std::mem_fn(&temporary_buffer<char>::size)), 0);
        return make_ready_future<>();
    }
    virtual future<> put(temporary_buffer<char> buf) override {
        _size += buf.size();
        return make_ready_future<>();
    }
    virtual future<> flush() override {
        return make_ready_future<>();
    }
    virtual future<> close() override {
        return make_ready_future<>();
    }
};

inline
output_stream<char>
make_sizing_output_stream(uint64_t& dest) {
    return output_stream<char>(data_sink(std::make_unique<sizing_data_sink>(std::ref(dest))), 4096);
}

// Must be called from a thread
template <typename T>
uint64_t
serialized_size(const T& object) {
    uint64_t size = 0;
    auto writer = file_writer(make_sizing_output_stream(size));
    write(writer, object);
    writer.flush().get();
    writer.close().get();
    return size;
}

output_stream<char> make_checksummed_file_output_stream(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options);

class checksummed_file_writer : public file_writer {
    checksum _c;
    uint32_t _full_checksum;
public:
    checksummed_file_writer(file f, file_output_stream_options options, bool checksum_file = false)
            : file_writer(make_checksummed_file_output_stream(std::move(f), _c, _full_checksum, checksum_file, options))
            , _c({uint32_t(std::min(size_t(DEFAULT_CHUNK_SIZE), size_t(options.buffer_size)))})
            , _full_checksum(init_checksum_adler32()) {}

    // Since we are exposing a reference to _full_checksum, we delete the move
    // constructor.  If it is moved, the reference will refer to the old
    // location.
    checksummed_file_writer(checksummed_file_writer&&) = delete;
    checksummed_file_writer(const checksummed_file_writer&) = default;

    checksum& finalize_checksum() {
        return _c;
    }
    uint32_t full_checksum() {
        return _full_checksum;
    }
};

class checksummed_file_data_sink_impl : public data_sink_impl {
    output_stream<char> _out;
    struct checksum& _c;
    uint32_t& _full_checksum;
    bool _checksum_file;
public:
    checksummed_file_data_sink_impl(file f, struct checksum& c, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options)
            : _out(make_file_output_stream(std::move(f), std::move(options)))
            , _c(c)
            , _full_checksum(full_file_checksum)
            , _checksum_file(checksum_file)
            {}

    future<> put(net::packet data) { abort(); }
    virtual future<> put(temporary_buffer<char> buf) override {
        // bufs will usually be a multiple of chunk size, but this won't be the case for
        // the last buffer being flushed.

        if (!_checksum_file) {
            _full_checksum = checksum_adler32(_full_checksum, buf.begin(), buf.size());
        } else {
            for (size_t offset = 0; offset < buf.size(); offset += _c.chunk_size) {
                size_t size = std::min(size_t(_c.chunk_size), buf.size() - offset);
                uint32_t per_chunk_checksum = init_checksum_adler32();

                per_chunk_checksum = checksum_adler32(per_chunk_checksum, buf.begin() + offset, size);
                _full_checksum = checksum_adler32_combine(_full_checksum, per_chunk_checksum, size);
                _c.checksums.push_back(per_chunk_checksum);
            }
        }
        auto f = _out.write(buf.begin(), buf.size());
        return f.then([buf = std::move(buf)] {});
    }

    virtual future<> close() {
        // Nothing to do, because close at the file_stream level will call flush on us.
        return _out.close();
    }
};

class checksummed_file_data_sink : public data_sink {
public:
    checksummed_file_data_sink(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options)
        : data_sink(std::make_unique<checksummed_file_data_sink_impl>(std::move(f), cinfo, full_file_checksum, checksum_file, std::move(options))) {}
};

inline
output_stream<char> make_checksummed_file_output_stream(file f, struct checksum& cinfo, uint32_t& full_file_checksum, bool checksum_file, file_output_stream_options options) {
    auto buffer_size = options.buffer_size;
    return output_stream<char>(checksummed_file_data_sink(std::move(f), cinfo, full_file_checksum, checksum_file, std::move(options)), buffer_size, true);
}

// compressed_file_data_sink_impl works as a filter for a file output stream,
// where the buffer flushed will be compressed and its checksum computed, then
// the result passed to a regular output stream.
class compressed_file_data_sink_impl : public data_sink_impl {
    output_stream<char> _out;
    sstables::compression* _compression_metadata;
    size_t _pos = 0;
public:
    compressed_file_data_sink_impl(file f, sstables::compression* cm, file_output_stream_options options)
            : _out(make_file_output_stream(std::move(f), options))
            , _compression_metadata(cm) {}

    future<> put(net::packet data) { abort(); }
    virtual future<> put(temporary_buffer<char> buf) override {
        auto output_len = _compression_metadata->compress_max_size(buf.size());
        // account space for checksum that goes after compressed data.
        temporary_buffer<char> compressed(output_len + 4);

        // compress flushed data.
        auto len = _compression_metadata->compress(buf.get(), buf.size(), compressed.get_write(), output_len);
        if (len > output_len) {
            throw std::runtime_error("possible overflow during compression");
        }

        _compression_metadata->offsets.elements.push_back(_pos);
        // account compressed data + 32-bit checksum.
        _pos += len + 4;
        _compression_metadata->set_compressed_file_length(_pos);
        // total length of the uncompressed data.
        _compression_metadata->data_len += buf.size();

        // compute 32-bit checksum for compressed data.
        uint32_t per_chunk_checksum = checksum_adler32(compressed.get(), len);
        _compression_metadata->update_full_checksum(per_chunk_checksum, len);

        // write checksum into buffer after compressed data.
        write_be<uint32_t>(compressed.get_write() + len, per_chunk_checksum);

        compressed.trim(len + 4);

        auto f = _out.write(compressed.get(), compressed.size());
        return f.then([compressed = std::move(compressed)] {});
    }
    virtual future<> close() {
        return _out.close();
    }
};

class compressed_file_data_sink : public data_sink {
public:
    compressed_file_data_sink(file f, sstables::compression* cm, file_output_stream_options options)
        : data_sink(std::make_unique<compressed_file_data_sink_impl>(
                std::move(f), cm, options)) {}
};

static inline output_stream<char> make_compressed_file_output_stream(file f, file_output_stream_options options, sstables::compression* cm) {
    // buffer of output stream is set to chunk length, because flush must
    // happen every time a chunk was filled up.
    auto outer_buffer_size = cm->uncompressed_chunk_length();
    return output_stream<char>(compressed_file_data_sink(std::move(f), cm, options), outer_buffer_size, true);
}

}
<|endoftext|>


<commit_before>
// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// (c) COPYRIGHT URI/MIT 1997-1999
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
// Authors:
//      jhrg,jimg       James Gallagher <jgallagher@gso.uri.edu>

// This is the source to `geturl'; a simple tool to exercise the Connect
// class. It can be used to get naked URLs as well as the DAP2 DAS and DDS
// objects.  jhrg.

#include "config.h"

static char rcsid[] not_used =
    { "$Id$" };

#include <stdio.h>
#ifdef WIN32
#include <io.h>
#include <fcntl.h>
#endif

#include <GetOpt.h>
#include <string>

#include "AISConnect.h"
#include "Response.h"
#include "StdinResponse.h"

using std::cerr;
using std::endl;

const char *version = "$Revision$";

extern int dods_keep_temps;     // defined in HTTPResponse.h

void usage(string name)
{
    cerr << "Usage: " << name << endl;
    cerr <<
        " [idDaxAVvks] [-B <db>][-c <expr>][-m <num>] <url> [<url> ...]" <<
        endl;
    cerr << " [Vvks] <file> [<file> ...]" << endl;
    cerr << endl;
    cerr << "In the first form of the command, dereference the URL and"
        << endl;
    cerr << "perform the requested operations. This includes routing" <<
        endl;
    cerr << "the returned information through the DAP processing" << endl;
    cerr << "library (parsing the returned objects, et c.). If none" <<
        endl;
    cerr << "of a, d, or D are used with a URL, then the DAP library" <<
        endl;
    cerr << "routines are NOT used and the URLs contents are dumped" <<
        endl;
    cerr << "to standard output." << endl;
    cerr << endl;
    cerr << "In the second form of the command, assume the files are" <<
        endl;
    cerr << "DataDDS objects (stored in files or read from pipes)" << endl;
    cerr << "and process them as if -D were given. In this case the" <<
        endl;
    cerr << "information *must* contain valid MIME header in order" <<
        endl;
    cerr << "to be processed." << endl;
    cerr << endl;
    cerr << "Options:" << endl;
    cerr << "        i: For each URL, get the server version." << endl;
    cerr << "        d: For each URL, get the the DDS." << endl;
    cerr << "        a: For each URL, get the the DAS." << endl;
    cerr << "        A: Use the AIS for DAS objects." << endl;
    cerr << "        D: For each URL, get the the DataDDS." << endl;
    cerr <<
        "        x: For each URL, get the DDX object. Does not get data."
        << endl;
    cerr << "        X: Build a DDX in getdap using the DDS and DAS." << endl;
    cerr << "        B: <AIS xml dataBase>. Overrides .dodsrc." << endl;
    cerr << "        v: Verbose." << endl;
    cerr << "        V: Version." << endl;
    cerr << "        c: <expr> is a contraint expression. Used with -D." <<
        endl;
    cerr << "           NB: You can use a `?' for the CE also." << endl;
    cerr << "        k: Keep temporary files created by libdap core\n" <<
        endl;
    cerr << "        m: Request the same URL <num> times." << endl;
    cerr << "        z: Ask the server to compress data." << endl;
    cerr << "        s: Print Sequences using numbered rows." << endl;
}

bool read_data(FILE * fp)
{
    if (!fp) {
        fprintf(stderr, "geturl: Whoa!!! Null stream pointer.\n");
        return false;
    }
    // Changed from a loop that used getc() to one that uses fread(). getc()
    // worked fine for transfers of text information, but *not* for binary
    // transfers. fread() will handle both.
    char c;
    while (fp && !feof(fp) && fread(&c, 1, 1, fp))
        printf("%c", c);        // stick with stdio

    return true;
}

static void print_data(DDS & dds, bool print_rows = false)
{
    fprintf(stdout, "The data:\n");

    for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++) {
        BaseType *v = *i;
        if (print_rows && (*i)->type() == dods_sequence_c)
            dynamic_cast < Sequence * >(*i)->print_val_by_rows(stdout);
        else
            v->print_val(stdout);
    }

    fprintf(stdout, "\n");
    fflush(stdout);
}

int main(int argc, char *argv[])
{
    GetOpt getopt(argc, argv, "idaDxXAVvkB:c:m:zsh?");
    int option_char;

    bool get_das = false;
    bool get_dds = false;
    bool get_data = false;
    bool get_ddx = false;
    bool build_ddx = false;
    bool get_version = false;
    bool cexpr = false;
    bool verbose = false;
    bool multi = false;
    bool accept_deflate = false;
    bool print_rows = false;
    bool use_ais = false;
    int times = 1;
    string expr = "";
    string ais_db = "";

#ifdef WIN32
    _setmode(_fileno(stdout), _O_BINARY);
#endif

    while ((option_char = getopt()) != EOF)
        switch (option_char) {
        case 'd':
            get_dds = true;
            break;
        case 'a':
            get_das = true;
            break;
        case 'D':
            get_data = true;
            break;
        case 'x':
            get_ddx = true;
            break;
        case 'X':
            build_ddx = true;
            break;
        case 'A':
            use_ais = true;
            break;
        case 'V':
            fprintf(stderr, "geturl version: %s\n", version);
            exit(0);
        case 'i':
            get_version = true;
            break;
        case 'v':
            verbose = true;
            break;
        case 'k':
            dods_keep_temps = 1;
            break;              // keep_temp is in Connect.cc
        case 'c':
            cexpr = true;
            expr = getopt.optarg;
            break;
        case 'm':
            multi = true;
            times = atoi(getopt.optarg);
            break;
        case 'B':
            use_ais = true;
            ais_db = getopt.optarg;
            break;
        case 'z':
            accept_deflate = true;
            break;
        case 's':
            print_rows = true;
            break;
        case 'h':
        case '?':
        default:
            usage(argv[0]);
            exit(1);
            break;
        }

    try {
        // If after processing all the command line options there is nothing
        // left (no URL or file) assume that we should read from stdin.
        for (int i = getopt.optind; i < argc; ++i) {
            if (verbose)
                fprintf(stderr, "Fetching: %s\n", argv[i]);

            string name = argv[i];
            Connect *url = 0;;
            if (use_ais) {
                if (!ais_db.empty())
                    url = new AISConnect(name, ais_db);
                else
                    url = new AISConnect(name);
            } else {
                url = new Connect(name);
            }

            // This overrides the value set in the .dodsrc file.
            if (accept_deflate)
                url->set_accept_deflate(accept_deflate);

            if (url->is_local()) {
                if (verbose) {
                    fprintf(stderr,
"Assuming that the argument %s is a file that contains a DAP2 data object; decoding.\n", argv[i]);
                }

                Response *r = 0;
                BaseTypeFactory factory;
                DataDDS dds(&factory);

                try {
                    if (strcmp(argv[i], "-") == 0) {
                        r = new StdinResponse(stdin);

                        if (!r->get_stream())
                            throw Error("Could not open standard input.");

                        url->read_data(dds, r);
                    } else {
                        r = new Response(fopen(argv[i], "r"));

                        if (!r->get_stream())
                            throw Error(string("The input source: ")
                                        + string(argv[i])
                                        + string(" could not be opened"));

                        url->read_data_no_mime(dds, r);
                    }
                }
                catch(Error & e) {
                    cerr << e.get_error_message() << endl;
                    delete r;
                    r = 0;
                    delete url;
                    url = 0;
                    break;
                }

                if (verbose)
                    fprintf(stderr, "Server version: %s\n",
                            url->get_version().c_str());

                print_data(dds, print_rows);

            }

            else if (get_version) {
                fprintf(stderr, "Server version: %s\n",
                        url->request_version().c_str());
            }

            else if (get_das) {
                for (int j = 0; j < times; ++j) {
                    DAS das;
                    try {
                        url->request_das(das);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;
                    }

                    if (verbose) {
                        fprintf(stderr, "Server version: %s\n",
                                url->get_version().c_str());
                        fprintf(stderr, "DAS:\n");
                    }

                    das.print(stdout);
                }
            }

            else if (get_dds) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_dds(dds);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
                        fprintf(stderr, "Server version: %s\n",
                                url->get_version().c_str());
                        fprintf(stderr, "DDS:\n");
                    }

                    dds.print(stdout);
                }
            }

            else if (get_ddx) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_ddx(dds);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
                        fprintf(stderr, "Server version: %s\n",
                                url->get_version().c_str());
                        fprintf(stderr, "DDX:\n");
                    }

                    dds.print_xml(stdout, false, "geturl; no blob yet");
                }
            }

            else if (build_ddx) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_dds(dds);
                        DAS das;
                        url->request_das(das);
                        dds.transfer_attributes(&das);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
                        fprintf(stderr, "Server version: %s\n",
                                url->get_version().c_str());
                        fprintf(stderr, "Client-built DDX:\n");
                    }

                    dds.print_xml(stdout, false, "geturl; no blob yet");
                }
            }

            else if (get_data) {
                if (expr.empty() && name.find('?') == string::npos)
                    expr = "";

                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DataDDS dds(&factory);
                    try {
                        DBG(cerr << "URL: " << url->URL(false) << endl);
                        DBG(cerr << "CE: " << expr << endl);
                        url->request_data(dds, expr);

                        if (verbose)
                            fprintf(stderr, "Server version: %s\n",
                                    url->get_version().c_str());

                        print_data(dds, print_rows);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;
                    }
                }
            }

            else {              // if (!get_das && !get_dds && !get_data)
                // This code uses HTTPConnect::fetch_url which cannot be
                // accessed using an instance of Connect. So some of the
                // options supported by other URLs won't work here (e.g., the
                // verbose option doesn't show the server version number).
                HTTPConnect http(RCReader::instance());

                // This overrides the value set in the .dodsrc file.
                if (accept_deflate)
                    http.set_accept_deflate(accept_deflate);

                string url_string = argv[i];
                for (int j = 0; j < times; ++j) {
                    try {
                        Response *r = http.fetch_url(url_string);
                        if (!read_data(r->get_stream())) {
                            continue;
                        }
                        delete r;
                        r = 0;
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;
                    }
                }
            }

            delete url;
            url = 0;
        }
    }
    catch(Error &e) {
        cerr << e.get_error_message() << endl;
        return 1;
    }
    catch(exception &e) {
        cerr << "C++ library exception: " << e.what() << endl;
        return 1;
    }

    return 0;
}
<commit_msg>getdap.cc: getdap now reports the protocol version in addition to the server version.<commit_after>
// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// (c) COPYRIGHT URI/MIT 1997-1999
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
// Authors:
//      jhrg,jimg       James Gallagher <jgallagher@gso.uri.edu>

// This is the source to `geturl'; a simple tool to exercise the Connect
// class. It can be used to get naked URLs as well as the DAP2 DAS and DDS
// objects.  jhrg.

#include "config.h"

static char rcsid[] not_used =
    { "$Id$" };

#include <stdio.h>
#ifdef WIN32
#include <io.h>
#include <fcntl.h>
#endif

#include <GetOpt.h>
#include <string>

#include "AISConnect.h"
#include "Response.h"
#include "StdinResponse.h"

using std::cerr;
using std::endl;

const char *version = "$Revision$";

extern int dods_keep_temps;     // defined in HTTPResponse.h

void usage(string name)
{
    cerr << "Usage: " << name << endl;
    cerr <<
        " [idDaxAVvks] [-B <db>][-c <expr>][-m <num>] <url> [<url> ...]" <<
        endl;
    cerr << " [Vvks] <file> [<file> ...]" << endl;
    cerr << endl;
    cerr << "In the first form of the command, dereference the URL and"
        << endl;
    cerr << "perform the requested operations. This includes routing" <<
        endl;
    cerr << "the returned information through the DAP processing" << endl;
    cerr << "library (parsing the returned objects, et c.). If none" <<
        endl;
    cerr << "of a, d, or D are used with a URL, then the DAP library" <<
        endl;
    cerr << "routines are NOT used and the URLs contents are dumped" <<
        endl;
    cerr << "to standard output." << endl;
    cerr << endl;
    cerr << "In the second form of the command, assume the files are" <<
        endl;
    cerr << "DataDDS objects (stored in files or read from pipes)" << endl;
    cerr << "and process them as if -D were given. In this case the" <<
        endl;
    cerr << "information *must* contain valid MIME header in order" <<
        endl;
    cerr << "to be processed." << endl;
    cerr << endl;
    cerr << "Options:" << endl;
    cerr << "        i: For each URL, get the server version." << endl;
    cerr << "        d: For each URL, get the the DDS." << endl;
    cerr << "        a: For each URL, get the the DAS." << endl;
    cerr << "        A: Use the AIS for DAS objects." << endl;
    cerr << "        D: For each URL, get the the DataDDS." << endl;
    cerr <<
        "        x: For each URL, get the DDX object. Does not get data."
        << endl;
    cerr << "        X: Build a DDX in getdap using the DDS and DAS." << endl;
    cerr << "        B: <AIS xml dataBase>. Overrides .dodsrc." << endl;
    cerr << "        v: Verbose." << endl;
    cerr << "        V: Version." << endl;
    cerr << "        c: <expr> is a contraint expression. Used with -D." <<
        endl;
    cerr << "           NB: You can use a `?' for the CE also." << endl;
    cerr << "        k: Keep temporary files created by libdap core\n" <<
        endl;
    cerr << "        m: Request the same URL <num> times." << endl;
    cerr << "        z: Ask the server to compress data." << endl;
    cerr << "        s: Print Sequences using numbered rows." << endl;
}

bool read_data(FILE * fp)
{
    if (!fp) {
        fprintf(stderr, "geturl: Whoa!!! Null stream pointer.\n");
        return false;
    }
    // Changed from a loop that used getc() to one that uses fread(). getc()
    // worked fine for transfers of text information, but *not* for binary
    // transfers. fread() will handle both.
    char c;
    while (fp && !feof(fp) && fread(&c, 1, 1, fp))
        printf("%c", c);        // stick with stdio

    return true;
}

static void print_data(DDS & dds, bool print_rows = false)
{
    fprintf(stdout, "The data:\n");

    for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++) {
        BaseType *v = *i;
        if (print_rows && (*i)->type() == dods_sequence_c)
            dynamic_cast < Sequence * >(*i)->print_val_by_rows(stdout);
        else
            v->print_val(stdout);
    }

    fprintf(stdout, "\n");
    fflush(stdout);
}

int main(int argc, char *argv[])
{
    GetOpt getopt(argc, argv, "idaDxXAVvkB:c:m:zsh?");
    int option_char;

    bool get_das = false;
    bool get_dds = false;
    bool get_data = false;
    bool get_ddx = false;
    bool build_ddx = false;
    bool get_version = false;
    bool cexpr = false;
    bool verbose = false;
    bool multi = false;
    bool accept_deflate = false;
    bool print_rows = false;
    bool use_ais = false;
    int times = 1;
    string expr = "";
    string ais_db = "";

#ifdef WIN32
    _setmode(_fileno(stdout), _O_BINARY);
#endif

    while ((option_char = getopt()) != EOF)
        switch (option_char) {
        case 'd':
            get_dds = true;
            break;
        case 'a':
            get_das = true;
            break;
        case 'D':
            get_data = true;
            break;
        case 'x':
            get_ddx = true;
            break;
        case 'X':
            build_ddx = true;
            break;
        case 'A':
            use_ais = true;
            break;
        case 'V':
            fprintf(stderr, "geturl version: %s\n", version);
            exit(0);
        case 'i':
            get_version = true;
            break;
        case 'v':
            verbose = true;
            break;
        case 'k':
            dods_keep_temps = 1;
            break;              // keep_temp is in Connect.cc
        case 'c':
            cexpr = true;
            expr = getopt.optarg;
            break;
        case 'm':
            multi = true;
            times = atoi(getopt.optarg);
            break;
        case 'B':
            use_ais = true;
            ais_db = getopt.optarg;
            break;
        case 'z':
            accept_deflate = true;
            break;
        case 's':
            print_rows = true;
            break;
        case 'h':
        case '?':
        default:
            usage(argv[0]);
            exit(1);
            break;
        }

    try {
        // If after processing all the command line options there is nothing
        // left (no URL or file) assume that we should read from stdin.
        for (int i = getopt.optind; i < argc; ++i) {
            if (verbose)
                fprintf(stderr, "Fetching: %s\n", argv[i]);

            string name = argv[i];
            Connect *url = 0;;
            if (use_ais) {
                if (!ais_db.empty())
                    url = new AISConnect(name, ais_db);
                else
                    url = new AISConnect(name);
            } else {
                url = new Connect(name);
            }

            // This overrides the value set in the .dodsrc file.
            if (accept_deflate)
                url->set_accept_deflate(accept_deflate);

            if (url->is_local()) {
                if (verbose) {
                    fprintf(stderr,
"Assuming that the argument %s is a file that contains a DAP2 data object; decoding.\n", argv[i]);
                }

                Response *r = 0;
                BaseTypeFactory factory;
                DataDDS dds(&factory);

                try {
                    if (strcmp(argv[i], "-") == 0) {
                        r = new StdinResponse(stdin);

                        if (!r->get_stream())
                            throw Error("Could not open standard input.");

                        url->read_data(dds, r);
                    } else {
                        r = new Response(fopen(argv[i], "r"));

                        if (!r->get_stream())
                            throw Error(string("The input source: ")
                                        + string(argv[i])
                                        + string(" could not be opened"));

                        url->read_data_no_mime(dds, r);
                    }
                }
                catch(Error & e) {
                    cerr << e.get_error_message() << endl;
                    delete r;
                    r = 0;
                    delete url;
                    url = 0;
                    break;
                }

                if (verbose)
                    fprintf(stderr, "DAP version: %s, Server version: %s\n",
			    url->get_protocol().c_str(),
                            url->get_version().c_str());

                print_data(dds, print_rows);

            }

            else if (get_version) {
                    fprintf(stderr, "DAP version: %s, Server version: %s\n",
			    url->request_protocol().c_str(),
                            url->get_version().c_str());
            }

            else if (get_das) {
                for (int j = 0; j < times; ++j) {
                    DAS das;
                    try {
                        url->request_das(das);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;
                    }

                    if (verbose) {
			fprintf(stderr, "DAP version: %s, Server version: %s\n",
				url->get_protocol().c_str(),
				url->get_version().c_str());

                        fprintf(stderr, "DAS:\n");
                    }

                    das.print(stdout);
                }
            }

            else if (get_dds) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_dds(dds);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
			fprintf(stderr, "DAP version: %s, Server version: %s\n",
				url->get_protocol().c_str(),
				url->get_version().c_str());

                        fprintf(stderr, "DDS:\n");
                    }

                    dds.print(stdout);
                }
            }

            else if (get_ddx) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_ddx(dds);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
			fprintf(stderr, "DAP version: %s, Server version: %s\n",
				url->get_protocol().c_str(),
				url->get_version().c_str());

                        fprintf(stderr, "DDX:\n");
                    }

                    dds.print_xml(stdout, false, "geturl; no blob yet");
                }
            }

            else if (build_ddx) {
                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DDS dds(&factory);
                    try {
                        url->request_dds(dds);
                        DAS das;
                        url->request_das(das);
                        dds.transfer_attributes(&das);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;       // Goto the next URL or exit the loop.
                    }

                    if (verbose) {
			fprintf(stderr, "DAP version: %s, Server version: %s\n",
				url->get_protocol().c_str(),
				url->get_version().c_str());

                        fprintf(stderr, "Client-built DDX:\n");
                    }

                    dds.print_xml(stdout, false, "geturl; no blob yet");
                }
            }

            else if (get_data) {
                if (expr.empty() && name.find('?') == string::npos)
                    expr = "";

                for (int j = 0; j < times; ++j) {
                    BaseTypeFactory factory;
                    DataDDS dds(&factory);
                    try {
                        DBG(cerr << "URL: " << url->URL(false) << endl);
                        DBG(cerr << "CE: " << expr << endl);
                        url->request_data(dds, expr);

                        if (verbose)
			    fprintf(stderr, "DAP version: %s, Server version: %s\n",
				    url->get_protocol().c_str(),
				    url->get_version().c_str());

                        print_data(dds, print_rows);
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        delete url;
                        url = 0;
                        continue;
                    }
                }
            }

            else {              // if (!get_das && !get_dds && !get_data)
                // This code uses HTTPConnect::fetch_url which cannot be
                // accessed using an instance of Connect. So some of the
                // options supported by other URLs won't work here (e.g., the
                // verbose option doesn't show the server version number).
                HTTPConnect http(RCReader::instance());

                // This overrides the value set in the .dodsrc file.
                if (accept_deflate)
                    http.set_accept_deflate(accept_deflate);

                string url_string = argv[i];
                for (int j = 0; j < times; ++j) {
                    try {
                        Response *r = http.fetch_url(url_string);
                        if (!read_data(r->get_stream())) {
                            continue;
                        }
                        delete r;
                        r = 0;
                    }
                    catch(Error & e) {
                        cerr << e.get_error_message() << endl;
                        continue;
                    }
                }
            }

            delete url;
            url = 0;
        }
    }
    catch(Error &e) {
        cerr << e.get_error_message() << endl;
        return 1;
    }
    catch(exception &e) {
        cerr << "C++ library exception: " << e.what() << endl;
        return 1;
    }

    return 0;
}
<|endoftext|>


<commit_before>/**
* MSYS2 Launcher
* Copyright (C) 2016, ForceStudio All Rights Reserved.
**/
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <Windows.h>
#include <Shlwapi.h>
#include <iostream>
#define  SECURITY_WIN32
#include <Security.h>
#include <string>
#include <vector>

#if defined(_MSC_VER)
#if defined(_WIN32_WINNT) &&_WIN32_WINNT>=_WIN32_WINNT_WIN8
#include <Processthreadsapi.h>
#endif
#endif

#include "cpptoml.h"

struct LauncherStructure{
  std::wstring env;
  std::wstring root;
  std::wstring wd;
  std::wstring mintty;
  std::wstring icon;
  std::wstring othrShell;
  std::wstring shellArgs;
  std::vector<std::wstring> appendPath;
  bool enableZshell;
  bool clearEnvironment;
};

class Characters{
private:
    char *str;
public:
    Characters(const wchar_t *wstr):str(nullptr)
    {
      if(wstr==nullptr) return ;
      int iTextLen=WideCharToMultiByte(CP_ACP, 0, wstr, -1, NULL, 0, NULL, NULL);
      str = new char[iTextLen + 1];
      if(str==nullptr) return;
      str[iTextLen]=0;
      WideCharToMultiByte(CP_ACP, 0, wstr, -1, str, iTextLen,NULL, NULL);
    }
    const char *Get()
    {
        if(!str)
            return nullptr;
        return const_cast<const char *>(str);
    }
    ~Characters()
    {
        if(str)
            delete[] str;
    }
};

class WCharacters{
private:
    wchar_t *wstr;
public:
    WCharacters(const char *str):wstr(nullptr)
    {
        if(str==nullptr)
            return ;
        int unicodeLen = ::MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
        if(unicodeLen==0)
            return ;
        wstr = new wchar_t[unicodeLen + 1];
        if(wstr==nullptr) return;
        wstr[unicodeLen]=0;
        ::MultiByteToWideChar(CP_UTF8, 0,str, -1, (LPWSTR)wstr,unicodeLen);
    }
    const wchar_t *Get()
    {
        if(!wstr)
            return nullptr;
        return const_cast<const wchar_t *>(wstr);
    }
    ~WCharacters()
    {
        if(wstr)
            delete[] wstr;
    }
};

bool GetProcessImageFileFolder(std::wstring &wstr)
{
  wchar_t szbuf[32767];
  if(!GetModuleFileNameW(nullptr,szbuf,32767))
    return false;
  PathRemoveFileSpecW(szbuf);
  wstr=szbuf;
  return true;
}

bool LauncherProfile(const wchar_t *cf,LauncherStructure &config)
{
  std::wstring pfile;
  if(cf==nullptr){
    std::wstring folder;
    if(!GetProcessImageFileFolder(folder)){
      MessageBoxW(nullptr,
        L"GetModuleFileNameW Failed!",
        L"Internal System Error",
        MB_OK|MB_ICONERROR);
      return false;
    }
    pfile=folder+L"/launcher.toml";
    if(_waccess_s(pfile.c_str(),04)!=0){
      pfile=folder+L"/launcher.exe.toml";
      if(_waccess_s(pfile.c_str(),04)!=0){
        MessageBoxW(nullptr,
          folder.c_str(),
          L"Cannot open launcher.toml or launcher.exe.toml on this path",
          MB_OK|MB_ICONERROR);
        return false;
      }
    }
  }else{
    pfile=cf;
  }
  Characters chars(pfile.c_str());
  std::shared_ptr<cpptoml::table> g;
  try {
    g = cpptoml::parse_file(chars.Get());
    std::cout << (*g) << std::endl;
  } catch (const cpptoml::parse_exception &e) {
    (void)e;
    MessageBoxW(nullptr,
      pfile.c_str(),
      L"Cannot parser toml-format profile!",
      MB_OK|MB_ICONERROR);
    return false;
  }
  auto Strings = [&](const char *key, const wchar_t *v, std::wstring &sv) {
    if (g->contains_qualified(key)) {
      std::string astr = g->get_qualified(key)->as<std::string>()->get();
      WCharacters was(astr.c_str());
      sv=was.Get();
      return;
    }
    if(v){
        sv = v;
      }
  };
  auto Boolean = [&](const char *key, bool b) {
    if (g->contains_qualified(key)) {
      return g->get_qualified(key)->as<bool>()->get();
    }
    return b;
  };
  auto Vector = [&](const char *key, std::vector<std::wstring> &v) {
    if (g->contains_qualified(key) && g->get_qualified(key)->is_array()) {
      auto av = g->get_qualified(key)->as_array();
      for (auto &e : av->get()) {
        WCharacters was(e->as<std::string>()->get().c_str());
        v.push_back(was.Get());
      }
    }
  };
  Strings("Launcher.MSYSTEM",L"MINGW64",config.env);
  Strings("Launcher.Root",L"C:/MSYS2",config.root);
  Strings("Launcher.WD",L"C:/MSYS2/usr/bin",config.wd);
  Strings("Launcher.Mintty",L"C:/MSYS2/usr/bin/mintty.exe",config.mintty);
  Strings("Launcher.ICONPath",L"/msys2.ico",config.icon);
  Strings("Launcher.Shell",L"bash",config.othrShell);
  Strings("Launcher.AppendShellArgs",nullptr,config.shellArgs);
  Vector("Launcher.AppendPath",config.appendPath);
  config.enableZshell=Boolean("Launcher.EnableZshell",false);
  config.clearEnvironment=Boolean("Launcher.UseClearEnv",false);
  return true;
}

enum KEnvStateMachine : int {
  kClearReset = 0,
  kEscapeAllow = 1,
  kMarkAllow = 2,
  kBlockBegin = 3,
  kBlockEnd = 4
};

static bool ResolveEnvironment(const std::wstring &k, std::wstring &v) {
  wchar_t buffer[32767];
  auto dwSize=GetEnvironmentVariableW(k.c_str(),buffer,32767);
  if(dwSize==0||dwSize>=32767)
    return false;
  buffer[dwSize]=0;
  v=buffer;
  return true;
}

static bool EnvironmentExpend(std::wstring &va){
  if (va.empty())
    return false;
  std::wstring ns, ks;
  auto p = va.c_str();
  auto n = va.size();
  int pre = 0;
  size_t i = 0;
  KEnvStateMachine state = kClearReset;
  for (; i < n; i++) {
    switch (p[i]) {
    case '`': {
      switch (state) {
      case kClearReset:
        state = kEscapeAllow;
        break;
      case kEscapeAllow:
        ns.push_back('`');
        state = kClearReset;
        break;
      case kMarkAllow:
        state = kEscapeAllow;
        ns.push_back('$');
        break;
      case kBlockBegin:
        continue;
      default:
        ns.push_back('`');
        continue;
      }
    } break;
    case '$': {
      switch (state) {
      case kClearReset:
        state = kMarkAllow;
        break;
      case kEscapeAllow:
        ns.push_back('$');
        state = kClearReset;
        break;
      case kMarkAllow:
        ns.push_back('$');
        state = kClearReset;
        break;
      case kBlockBegin:
      case kBlockEnd:
      default:
        ns.push_back('$');
        continue;
      }
    } break;
    case '{': {
      switch (state) {
      case kClearReset:
      case kEscapeAllow:
        ns.push_back('{');
        state = kClearReset;
        break;
      case kMarkAllow: {
        state = kBlockBegin;
        pre = i;
      } break;
      case kBlockBegin:
        ns.push_back('{');
        break;
      default:
        continue;
      }
    } break;
    case '}': {
      switch (state) {
      case kClearReset:
      case kEscapeAllow:
        ns.push_back('}');
        state = kClearReset;
        break;
      case kMarkAllow:
        state = kClearReset;
        ns.push_back('$');
        ns.push_back('}');
        break;
      case kBlockBegin: {
        ks.assign(&p[pre + 1], i - pre - 1);
        std::wstring v;
        if (ResolveEnvironment(ks, v))
          ns.append(v);
        state = kClearReset;
      } break;
      default:
        continue;
      }
    } break;
    default: {
      switch (state) {
      case kClearReset:
        ns.push_back(p[i]);
        break;
      case kEscapeAllow:
        ns.push_back('`');
        ns.push_back(p[i]);
        state = kClearReset;
        break;
      case kMarkAllow:
        ns.push_back('$');
        ns.push_back(p[i]);
        state = kClearReset;
        break;
      case kBlockBegin:
      default:
        continue;
      }
    } break;
    }
  }
  va = ns;
  return true;
}

bool ClearEnvironmentVariableW(){
  //WINDIR expend like C:\WINDOW
  std::wstring path=L"${WINDIR}\\System32;${WINDIR};${WINDIR}\\System32\\Wbem;${WINDIR}\\System32\\WindowsPowerShell\\v1.0\\";
  if(EnvironmentExpend(path)){
    SetEnvironmentVariableW(L"PATH",path.c_str());
    return true;
  }
  return false;
}

bool PutEnvironmentVariableW(const wchar_t *name,const wchar_t *va)
{
  wchar_t buffer[32767];
  auto dwSize=GetEnvironmentVariableW(name,buffer,32767);
  if(dwSize<=0){
    return SetEnvironmentVariableW(name,va)?true:false;
  }
  if(dwSize>=32766) return false;
  if(buffer[dwSize-1]!=';'){
    buffer[dwSize]=';';
    dwSize++;
    buffer[dwSize]=0;
  }
  wcscat_s(buffer,32767-dwSize,va);
  return SetEnvironmentVariableW(name,buffer)?true:false;
}

bool PutPathEnvironmentVariableW(std::vector<std::wstring> &pv){
  std::wstring path(L";");
  for(auto &s:pv){
    path.append(s);
    path.push_back(';');
  }
  return PutEnvironmentVariableW(L"PATH",path.c_str());
}

bool StartupMiniPosixEnv(LauncherStructure &config)
{
  SetCurrentDirectoryW(config.root.c_str());
  if(config.clearEnvironment){
    ClearEnvironmentVariableW();
  }
  SetEnvironmentVariableW(L"MSYSTEM",config.env.c_str());
  SetEnvironmentVariableW(L"WD",config.wd.c_str());
  SetEnvironmentVariableW(L"MSYSCON",L"mintty.exe");

  if(!config.appendPath.empty()){
    PutPathEnvironmentVariableW(config.appendPath);
  }

  PROCESS_INFORMATION pi;
  STARTUPINFO si;
  ZeroMemory(&si, sizeof(si));
  si.cb = sizeof(si);
  si.dwFlags = STARTF_USESHOWWINDOW;
  si.wShowWindow = SW_SHOW;

  int const ArraySize=8192;
  wchar_t cmdline[ArraySize]={0};
  auto l=wsprintfW(cmdline,L"%s -i%s /usr/bin/%s --login ",
  config.mintty.c_str(),
  config.icon.c_str(),
  config.enableZshell?L"zsh":config.othrShell.c_str());
  if(!config.shellArgs.empty()){
    if((size_t)(ArraySize-l)>config.shellArgs.size()){
      auto p=cmdline+l;
      wsprintfW(p,L"%s",config.shellArgs.c_str());
    }
  }
  ///CreateProcess
  BOOL result = CreateProcessW(
    NULL,
    cmdline,
    NULL,
    NULL,
    TRUE,
    CREATE_NEW_CONSOLE,
    NULL,
    NULL,
    &si,
    &pi
    );
  if(result){
    CloseHandle(pi.hThread);
    CloseHandle(pi.hProcess);
  }
  return true;
}

#ifdef __GNUC__
int WINAPI WinMain(
  HINSTANCE hInstance,
  HINSTANCE hPrevInstance,
  LPSTR lpCmdLine,
  int nCmdShow
){
  return wWinMain(hInstance,hPrevInstance,nullptr,nCmdShow);
}
#endif

int WINAPI wWinMain(
  HINSTANCE hInstance,
  HINSTANCE hPrevInstance,
  LPWSTR lpCmdLine,
  int nCmdShow)
{
  LauncherStructure config;
  const wchar_t *cf=nullptr;
  int Argc;
  LPWSTR *Argv = CommandLineToArgvW(GetCommandLineW(), &Argc);
  if(Argc>=3){
    if(wcscmp(Argv[1],L"-c")&&_waccess_s(Argv[2],04)==0)
      cf=Argv[1];
  }
  if(!LauncherProfile(cf,config)){
    LocalFree(Argv);
    return 1;
  }
  LocalFree(Argv);
  return StartupMiniPosixEnv(config)?2:0;
}
<commit_msg>Change WinMain<commit_after>/**
* MSYS2 Launcher
* Copyright (C) 2016, ForceStudio All Rights Reserved.
**/
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <Windows.h>
#include <Shlwapi.h>
#include <iostream>
#define  SECURITY_WIN32
#include <Security.h>
#include <string>
#include <vector>

#if defined(_MSC_VER)
#if defined(_WIN32_WINNT) &&_WIN32_WINNT>=_WIN32_WINNT_WIN8
#include <Processthreadsapi.h>
#endif
#endif

#include "cpptoml.h"

struct LauncherStructure{
  std::wstring env;
  std::wstring root;
  std::wstring wd;
  std::wstring mintty;
  std::wstring icon;
  std::wstring othrShell;
  std::wstring shellArgs;
  std::vector<std::wstring> appendPath;
  bool enableZshell;
  bool clearEnvironment;
};

class Characters{
private:
    char *str;
public:
    Characters(const wchar_t *wstr):str(nullptr)
    {
      if(wstr==nullptr) return ;
      int iTextLen=WideCharToMultiByte(CP_ACP, 0, wstr, -1, NULL, 0, NULL, NULL);
      str = new char[iTextLen + 1];
      if(str==nullptr) return;
      str[iTextLen]=0;
      WideCharToMultiByte(CP_ACP, 0, wstr, -1, str, iTextLen,NULL, NULL);
    }
    const char *Get()
    {
        if(!str)
            return nullptr;
        return const_cast<const char *>(str);
    }
    ~Characters()
    {
        if(str)
            delete[] str;
    }
};

class WCharacters{
private:
    wchar_t *wstr;
public:
    WCharacters(const char *str):wstr(nullptr)
    {
        if(str==nullptr)
            return ;
        int unicodeLen = ::MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
        if(unicodeLen==0)
            return ;
        wstr = new wchar_t[unicodeLen + 1];
        if(wstr==nullptr) return;
        wstr[unicodeLen]=0;
        ::MultiByteToWideChar(CP_UTF8, 0,str, -1, (LPWSTR)wstr,unicodeLen);
    }
    const wchar_t *Get()
    {
        if(!wstr)
            return nullptr;
        return const_cast<const wchar_t *>(wstr);
    }
    ~WCharacters()
    {
        if(wstr)
            delete[] wstr;
    }
};

bool GetProcessImageFileFolder(std::wstring &wstr)
{
  wchar_t szbuf[32767];
  if(!GetModuleFileNameW(nullptr,szbuf,32767))
    return false;
  PathRemoveFileSpecW(szbuf);
  wstr=szbuf;
  return true;
}

bool LauncherProfile(const wchar_t *cf,LauncherStructure &config)
{
  std::wstring pfile;
  if(cf==nullptr){
    std::wstring folder;
    if(!GetProcessImageFileFolder(folder)){
      MessageBoxW(nullptr,
        L"GetModuleFileNameW Failed!",
        L"Internal System Error",
        MB_OK|MB_ICONERROR);
      return false;
    }
    pfile=folder+L"/launcher.toml";
    if(_waccess_s(pfile.c_str(),04)!=0){
      pfile=folder+L"/launcher.exe.toml";
      if(_waccess_s(pfile.c_str(),04)!=0){
        MessageBoxW(nullptr,
          folder.c_str(),
          L"Cannot open launcher.toml or launcher.exe.toml on this path",
          MB_OK|MB_ICONERROR);
        return false;
      }
    }
  }else{
    pfile=cf;
  }
  Characters chars(pfile.c_str());
  std::shared_ptr<cpptoml::table> g;
  try {
    g = cpptoml::parse_file(chars.Get());
    std::cout << (*g) << std::endl;
  } catch (const cpptoml::parse_exception &e) {
    (void)e;
    MessageBoxW(nullptr,
      pfile.c_str(),
      L"Cannot parser toml-format profile!",
      MB_OK|MB_ICONERROR);
    return false;
  }
  auto Strings = [&](const char *key, const wchar_t *v, std::wstring &sv) {
    if (g->contains_qualified(key)) {
      std::string astr = g->get_qualified(key)->as<std::string>()->get();
      WCharacters was(astr.c_str());
      sv=was.Get();
      return;
    }
    if(v){
        sv = v;
      }
  };
  auto Boolean = [&](const char *key, bool b) {
    if (g->contains_qualified(key)) {
      return g->get_qualified(key)->as<bool>()->get();
    }
    return b;
  };
  auto Vector = [&](const char *key, std::vector<std::wstring> &v) {
    if (g->contains_qualified(key) && g->get_qualified(key)->is_array()) {
      auto av = g->get_qualified(key)->as_array();
      for (auto &e : av->get()) {
        WCharacters was(e->as<std::string>()->get().c_str());
        v.push_back(was.Get());
      }
    }
  };
  Strings("Launcher.MSYSTEM",L"MINGW64",config.env);
  Strings("Launcher.Root",L"C:/MSYS2",config.root);
  Strings("Launcher.WD",L"C:/MSYS2/usr/bin",config.wd);
  Strings("Launcher.Mintty",L"C:/MSYS2/usr/bin/mintty.exe",config.mintty);
  Strings("Launcher.ICONPath",L"/msys2.ico",config.icon);
  Strings("Launcher.Shell",L"bash",config.othrShell);
  Strings("Launcher.AppendShellArgs",nullptr,config.shellArgs);
  Vector("Launcher.AppendPath",config.appendPath);
  config.enableZshell=Boolean("Launcher.EnableZshell",false);
  config.clearEnvironment=Boolean("Launcher.UseClearEnv",false);
  return true;
}

enum KEnvStateMachine : int {
  kClearReset = 0,
  kEscapeAllow = 1,
  kMarkAllow = 2,
  kBlockBegin = 3,
  kBlockEnd = 4
};

static bool ResolveEnvironment(const std::wstring &k, std::wstring &v) {
  wchar_t buffer[32767];
  auto dwSize=GetEnvironmentVariableW(k.c_str(),buffer,32767);
  if(dwSize==0||dwSize>=32767)
    return false;
  buffer[dwSize]=0;
  v=buffer;
  return true;
}

static bool EnvironmentExpend(std::wstring &va){
  if (va.empty())
    return false;
  std::wstring ns, ks;
  auto p = va.c_str();
  auto n = va.size();
  int pre = 0;
  size_t i = 0;
  KEnvStateMachine state = kClearReset;
  for (; i < n; i++) {
    switch (p[i]) {
    case '`': {
      switch (state) {
      case kClearReset:
        state = kEscapeAllow;
        break;
      case kEscapeAllow:
        ns.push_back('`');
        state = kClearReset;
        break;
      case kMarkAllow:
        state = kEscapeAllow;
        ns.push_back('$');
        break;
      case kBlockBegin:
        continue;
      default:
        ns.push_back('`');
        continue;
      }
    } break;
    case '$': {
      switch (state) {
      case kClearReset:
        state = kMarkAllow;
        break;
      case kEscapeAllow:
        ns.push_back('$');
        state = kClearReset;
        break;
      case kMarkAllow:
        ns.push_back('$');
        state = kClearReset;
        break;
      case kBlockBegin:
      case kBlockEnd:
      default:
        ns.push_back('$');
        continue;
      }
    } break;
    case '{': {
      switch (state) {
      case kClearReset:
      case kEscapeAllow:
        ns.push_back('{');
        state = kClearReset;
        break;
      case kMarkAllow: {
        state = kBlockBegin;
        pre = i;
      } break;
      case kBlockBegin:
        ns.push_back('{');
        break;
      default:
        continue;
      }
    } break;
    case '}': {
      switch (state) {
      case kClearReset:
      case kEscapeAllow:
        ns.push_back('}');
        state = kClearReset;
        break;
      case kMarkAllow:
        state = kClearReset;
        ns.push_back('$');
        ns.push_back('}');
        break;
      case kBlockBegin: {
        ks.assign(&p[pre + 1], i - pre - 1);
        std::wstring v;
        if (ResolveEnvironment(ks, v))
          ns.append(v);
        state = kClearReset;
      } break;
      default:
        continue;
      }
    } break;
    default: {
      switch (state) {
      case kClearReset:
        ns.push_back(p[i]);
        break;
      case kEscapeAllow:
        ns.push_back('`');
        ns.push_back(p[i]);
        state = kClearReset;
        break;
      case kMarkAllow:
        ns.push_back('$');
        ns.push_back(p[i]);
        state = kClearReset;
        break;
      case kBlockBegin:
      default:
        continue;
      }
    } break;
    }
  }
  va = ns;
  return true;
}

bool ClearEnvironmentVariableW(){
  //WINDIR expend like C:\WINDOW
  std::wstring path=L"${WINDIR}\\System32;${WINDIR};${WINDIR}\\System32\\Wbem;${WINDIR}\\System32\\WindowsPowerShell\\v1.0\\";
  if(EnvironmentExpend(path)){
    SetEnvironmentVariableW(L"PATH",path.c_str());
    return true;
  }
  return false;
}

bool PutEnvironmentVariableW(const wchar_t *name,const wchar_t *va)
{
  wchar_t buffer[32767];
  auto dwSize=GetEnvironmentVariableW(name,buffer,32767);
  if(dwSize<=0){
    return SetEnvironmentVariableW(name,va)?true:false;
  }
  if(dwSize>=32766) return false;
  if(buffer[dwSize-1]!=';'){
    buffer[dwSize]=';';
    dwSize++;
    buffer[dwSize]=0;
  }
  wcscat_s(buffer,32767-dwSize,va);
  return SetEnvironmentVariableW(name,buffer)?true:false;
}

bool PutPathEnvironmentVariableW(std::vector<std::wstring> &pv){
  std::wstring path(L";");
  for(auto &s:pv){
    path.append(s);
    path.push_back(';');
  }
  return PutEnvironmentVariableW(L"PATH",path.c_str());
}

bool StartupMiniPosixEnv(LauncherStructure &config)
{
  SetCurrentDirectoryW(config.root.c_str());
  if(config.clearEnvironment){
    ClearEnvironmentVariableW();
  }
  SetEnvironmentVariableW(L"MSYSTEM",config.env.c_str());
  SetEnvironmentVariableW(L"WD",config.wd.c_str());
  SetEnvironmentVariableW(L"MSYSCON",L"mintty.exe");

  if(!config.appendPath.empty()){
    PutPathEnvironmentVariableW(config.appendPath);
  }

  PROCESS_INFORMATION pi;
  STARTUPINFO si;
  ZeroMemory(&si, sizeof(si));
  si.cb = sizeof(si);
  si.dwFlags = STARTF_USESHOWWINDOW;
  si.wShowWindow = SW_SHOW;

  int const ArraySize=8192;
  wchar_t cmdline[ArraySize]={0};
  auto l=wsprintfW(cmdline,L"%s -i%s /usr/bin/%s --login ",
  config.mintty.c_str(),
  config.icon.c_str(),
  config.enableZshell?L"zsh":config.othrShell.c_str());
  if(!config.shellArgs.empty()){
    if((size_t)(ArraySize-l)>config.shellArgs.size()){
      auto p=cmdline+l;
      wsprintfW(p,L"%s",config.shellArgs.c_str());
    }
  }
  ///CreateProcess
  BOOL result = CreateProcessW(
    NULL,
    cmdline,
    NULL,
    NULL,
    TRUE,
    CREATE_NEW_CONSOLE,
    NULL,
    NULL,
    &si,
    &pi
    );
  if(result){
    CloseHandle(pi.hThread);
    CloseHandle(pi.hProcess);
  }
  return true;
}

#ifdef __GNUC__
#define launcherStartup WinMain
#define CHARPTR LPSTR
#elif defined(_MSC_VER)
#define launcherStartup wWinMain
#define CHARPTR LPWSTR
#else
#error "Cannot support this compiler !"
#endif

int WINAPI launcherStartup(
  HINSTANCE hInstance,
  HINSTANCE hPrevInstance,
  CHARPTR lpCmdLine,
  int nCmdShow)
{
  LauncherStructure config;
  const wchar_t *cf=nullptr;
  int Argc;
  LPWSTR *Argv = CommandLineToArgvW(GetCommandLineW(), &Argc);
  if(Argc>=3){
    if(wcscmp(Argv[1],L"-c")&&_waccess_s(Argv[2],04)==0)
      cf=Argv[1];
  }
  if(!LauncherProfile(cf,config)){
    LocalFree(Argv);
    return 1;
  }
  LocalFree(Argv);
  return StartupMiniPosixEnv(config)?2:0;
}
<|endoftext|>


<commit_before>/** \brief Utility for comparing ngram-assigned languages to human-assigned languages in MARC records.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <vector>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include "MARC.h"
#include "NGram.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage( "[--verbose] [--limit-count=count] [--cross-valiatdion-chunks=N] [--topmost-use-count=count] marc_data "
             "[language_code1 language_code2 .. language_codeN]\n"
             "If \"--limit-count\" has been specified only the first \"count\" records will be considered.\n"
             "If \"--cross-valiatdion-chunks\" has been specified, N sets will be used.\n"
             "The default for --topmost-use-count is " + std::to_string(NGram::DEFAULT_TOPMOST_USE_COUNT) + ".");
}

#if 0
void GenerateModels(const bool verbose, const unsigned limit_count, const unsigned cross_validation_chunk_count,
                    const unsigned leave_out_index, MARC::Reader * const marc_reader, const NGram::DistanceType distance_type,
                    const std::set<std::string> &considered_languages)
{
    unsigned record_count(0);
    while (const MARC::Record record = marc_reader->read()) {
        ++record_count;
        if (record_count == limit_count)
            break;

        if (record_count % cross_validation_chunk_count == leave_out_index)
            continue;
    }
}
#endif

void ProcessRecords(const bool verbose, const unsigned limit_count, const unsigned /*cross_validation_chunk_count*/,
                    const unsigned topmost_use_count, MARC::Reader * const marc_reader, const std::set<std::string> &considered_languages,
                    std::unordered_map<std::string, unsigned> * const mismatched_assignments_to_counts_map)
{
    unsigned record_count(0), untagged_count(0), agreed_count(0);

    while (const MARC::Record record = marc_reader->read()) {
        ++record_count;
        if (record_count > limit_count) {
            --record_count;
            break;
        }

        std::string language_code(MARC::GetLanguageCode(record));
        if (language_code.empty()) {
            ++untagged_count;
            continue;
        }

        const std::string text(record.getCompleteTitle() + " " + record.getSummary());
        std::vector<std::string> top_languages;
        NGram::ClassifyLanguage(text, &top_languages, considered_languages, NGram::DEFAULT_NGRAM_NUMBER_THRESHOLD, topmost_use_count);
        if (top_languages.empty())
            continue;

        if (top_languages.front() == language_code)
            ++agreed_count;
        else {
            const std::string key(language_code + ":" + top_languages.front());
            if (verbose)
                std::cout << key << "  " << text << '\n';
            const auto mismatched_assignment_and_count(mismatched_assignments_to_counts_map->find(key));
            if (mismatched_assignment_and_count == mismatched_assignments_to_counts_map->end())
                mismatched_assignments_to_counts_map->emplace(key, 1u);
            else
                ++mismatched_assignment_and_count->second;
        }
    }

    std::cout << "Used " << record_count << " MARC record(s) of which " << untagged_count << " had no language and "
              << (agreed_count * 100.0) / (record_count - untagged_count) << "% of  which had matching languages.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 2)
        Usage();

    bool verbose(false);
    if (std::strcmp(argv[1], "--verbose") == 0) {
        verbose = true;
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned limit_count(UINT_MAX);
    if (StringUtil::StartsWith(argv[1], "--limit-count=")) {
        limit_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--limit-count="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned cross_validation_chunk_count(0);
    if (StringUtil::StartsWith(argv[1], "--cross-valiatdion-chunks=")) {
        cross_validation_chunk_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--cross-valiatdion-chunks="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned topmost_use_count(NGram::DEFAULT_TOPMOST_USE_COUNT);
    if (StringUtil::StartsWith(argv[1], "--topmost-use-count=")) {
        topmost_use_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--topmost-use-count="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    auto marc_reader(MARC::Reader::Factory(argv[1]));

    std::set<std::string> considered_languages;
    for (int arg_no(2); arg_no < argc; ++arg_no)
        considered_languages.emplace(argv[arg_no]);

    std::unordered_map<std::string, unsigned> mismatched_assignments_to_counts_map;
    ProcessRecords(verbose, limit_count, cross_validation_chunk_count, topmost_use_count, marc_reader.get(),
                   considered_languages, &mismatched_assignments_to_counts_map);

    std::vector<std::pair<std::string, unsigned>> mismatched_assignments_and_counts;
    mismatched_assignments_and_counts.reserve(mismatched_assignments_to_counts_map.size());
    for (const auto &mismatched_assignment_and_count : mismatched_assignments_to_counts_map)
        mismatched_assignments_and_counts.emplace_back(mismatched_assignment_and_count);

    std::sort(mismatched_assignments_and_counts.begin(), mismatched_assignments_and_counts.end(),
              [](const std::pair<std::string, unsigned> &pair1, const std::pair<std::string, unsigned> &pair2)
                  { return pair1.second > pair2.second; });

    for (const auto &mismatched_assignment_and_count : mismatched_assignments_and_counts)
        std::cout << mismatched_assignment_and_count.first << " = " << mismatched_assignment_and_count.second << '\n';

    return EXIT_SUCCESS;
}
<commit_msg>Update marc_ngram_language_stats.cc<commit_after>/** \brief Utility for comparing ngram-assigned languages to human-assigned languages in MARC records.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <vector>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include "MARC.h"
#include "NGram.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage( "[--verbose] [--limit-count=count] [--cross-valiatdion-chunks=N] [--topmost-use-count=count] marc_data "
             "[language_code1 language_code2 .. language_codeN]\n"
             "If \"--limit-count\" has been specified only the first \"count\" records will be considered.\n"
             "If \"--cross-valiatdion-chunks\" has been specified, N sets will be used.\n"
             "The default for --topmost-use-count is " + std::to_string(NGram::DEFAULT_TOPMOST_USE_COUNT) + ".");
}

#if 0
void GenerateModels(const bool verbose, const unsigned limit_count, const unsigned cross_validation_chunk_count,
                    const unsigned leave_out_index, MARC::Reader * const marc_reader, const NGram::DistanceType distance_type,
                    const std::set<std::string> &considered_languages)
{
    unsigned record_count(0);
    while (const MARC::Record record = marc_reader->read()) {
        ++record_count;
        if (record_count == limit_count)
            break;

        if (record_count % cross_validation_chunk_count == leave_out_index)
            continue;
    }
}
#endif

void ProcessRecords(const bool verbose, const unsigned limit_count, const unsigned /*cross_validation_chunk_count*/,
                    const unsigned topmost_use_count, MARC::Reader * const marc_reader, const std::set<std::string> &considered_languages,
                    std::unordered_map<std::string, unsigned> * const mismatched_assignments_to_counts_map)
{
    unsigned record_count(0), untagged_count(0), agreed_count(0);

    while (const MARC::Record record = marc_reader->read()) {
        if (record_count > limit_count)
            break;

        ++record_count;
        std::string language_code(MARC::GetLanguageCode(record));
        if (language_code.empty()) {
            ++untagged_count;
            continue;
        }

        const std::string text(record.getCompleteTitle() + " " + record.getSummary());
        std::vector<std::string> top_languages;
        NGram::ClassifyLanguage(text, &top_languages, considered_languages, NGram::DEFAULT_NGRAM_NUMBER_THRESHOLD, topmost_use_count);
        if (top_languages.empty())
            continue;

        if (top_languages.front() == language_code)
            ++agreed_count;
        else {
            const std::string key(language_code + ":" + top_languages.front());
            if (verbose)
                std::cout << key << "  " << text << '\n';
            const auto mismatched_assignment_and_count(mismatched_assignments_to_counts_map->find(key));
            if (mismatched_assignment_and_count == mismatched_assignments_to_counts_map->end())
                mismatched_assignments_to_counts_map->emplace(key, 1u);
            else
                ++mismatched_assignment_and_count->second;
        }
    }

    std::cout << "Used " << record_count << " MARC record(s) of which " << untagged_count << " had no language and "
              << (agreed_count * 100.0) / (record_count - untagged_count) << "% of  which had matching languages.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 2)
        Usage();

    bool verbose(false);
    if (std::strcmp(argv[1], "--verbose") == 0) {
        verbose = true;
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned limit_count(UINT_MAX);
    if (StringUtil::StartsWith(argv[1], "--limit-count=")) {
        limit_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--limit-count="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned cross_validation_chunk_count(0);
    if (StringUtil::StartsWith(argv[1], "--cross-valiatdion-chunks=")) {
        cross_validation_chunk_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--cross-valiatdion-chunks="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    unsigned topmost_use_count(NGram::DEFAULT_TOPMOST_USE_COUNT);
    if (StringUtil::StartsWith(argv[1], "--topmost-use-count=")) {
        topmost_use_count = StringUtil::ToUnsigned(argv[1] + __builtin_strlen("--topmost-use-count="));
        --argc, ++argv;
    }

    if (argc < 2)
        Usage();

    auto marc_reader(MARC::Reader::Factory(argv[1]));

    std::set<std::string> considered_languages;
    for (int arg_no(2); arg_no < argc; ++arg_no)
        considered_languages.emplace(argv[arg_no]);

    std::unordered_map<std::string, unsigned> mismatched_assignments_to_counts_map;
    ProcessRecords(verbose, limit_count, cross_validation_chunk_count, topmost_use_count, marc_reader.get(),
                   considered_languages, &mismatched_assignments_to_counts_map);

    std::vector<std::pair<std::string, unsigned>> mismatched_assignments_and_counts;
    mismatched_assignments_and_counts.reserve(mismatched_assignments_to_counts_map.size());
    for (const auto &mismatched_assignment_and_count : mismatched_assignments_to_counts_map)
        mismatched_assignments_and_counts.emplace_back(mismatched_assignment_and_count);

    std::sort(mismatched_assignments_and_counts.begin(), mismatched_assignments_and_counts.end(),
              [](const std::pair<std::string, unsigned> &pair1, const std::pair<std::string, unsigned> &pair2)
                  { return pair1.second > pair2.second; });

    for (const auto &mismatched_assignment_and_count : mismatched_assignments_and_counts)
        std::cout << mismatched_assignment_and_count.first << " = " << mismatched_assignment_and_count.second << '\n';

    return EXIT_SUCCESS;
}
<|endoftext|>