<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Modified by ScyllaDB
 * Copyright 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "load_broadcaster.hh"
#include "cache_hitrate_calculator.hh"
#include "db/system_keyspace.hh"
#include "gms/application_state.hh"
#include "service/storage_service.hh"
#include "service/view_update_backlog_broker.hh"
#include "database.hh"

#include <cstdlib>

namespace service {

constexpr std::chrono::milliseconds load_broadcaster::BROADCAST_INTERVAL;

logging::logger llogger("load_broadcaster");

void load_broadcaster::start_broadcasting() {
    _done = make_ready_future<>();

    // send the first broadcast "right away" (i.e., in 2 gossip heartbeats, when we should have someone to talk to);
    // after that send every BROADCAST_INTERVAL.

    _timer.set_callback([this] {
        llogger.debug("Disseminating load info ...");
        _done = _db.map_reduce0([](database& db) {
            int64_t res = 0;
            for (auto i : db.get_column_families()) {
                res += i.second->get_stats().live_disk_space_used;
            }
            return res;
        }, int64_t(0), std::plus<int64_t>()).then([this] (int64_t size) {
            gms::versioned_value::factory value_factory;
            return _gossiper.add_local_application_state(gms::application_state::LOAD,
                value_factory.load(size)).then([this] {
                _timer.arm(BROADCAST_INTERVAL);
                return make_ready_future<>();
            });
        });
    });

    _timer.arm(2 * gms::gossiper::INTERVAL);
}

future<> load_broadcaster::stop_broadcasting() {
    _timer.cancel();
    return std::move(_done);
}

// cache_hitrate_calculator implementation
cache_hitrate_calculator::cache_hitrate_calculator(seastar::sharded<database>& db, seastar::sharded<cache_hitrate_calculator>& me) : _db(db), _me(me),
        _timer(std::bind(std::mem_fn(&cache_hitrate_calculator::recalculate_timer), this))
{}

void cache_hitrate_calculator::recalculate_timer() {
    recalculate_hitrates().then_wrapped([p = shared_from_this()] (future<lowres_clock::duration> f) {
        lowres_clock::duration d;
        if (f.failed()) {
            d = std::chrono::milliseconds(2000);
        } else {
            d = f.get0();
        }
        p->run_on((engine().cpu_id() + 1) % smp::count, d);
    });
}

void cache_hitrate_calculator::run_on(size_t master, lowres_clock::duration d) {
    if (!_stopped) {
        _me.invoke_on(master, [d] (cache_hitrate_calculator& local) {
            local._timer.arm(d);
        }).handle_exception_type([] (seastar::no_sharded_instance_exception&) { /* ignore */ });
    }
}

future<lowres_clock::duration> cache_hitrate_calculator::recalculate_hitrates() {
    struct stat {
        float h = 0;
        float m = 0;
        stat& operator+=(stat& o) {
            h += o.h;
            m += o.m;
            return *this;
        }
    };

    static auto non_system_filter = [&] (const std::pair<utils::UUID, lw_shared_ptr<column_family>>& cf) {
        return _db.local().find_keyspace(cf.second->schema()->ks_name()).get_replication_strategy().get_type() != locator::replication_strategy_type::local;
    };

    auto cf_to_cache_hit_stats = [] (database& db) {
        return boost::copy_range<std::unordered_map<utils::UUID, stat>>(db.get_column_families() | boost::adaptors::filtered(non_system_filter) |
                boost::adaptors::transformed([]  (const std::pair<utils::UUID, lw_shared_ptr<column_family>>& cf) {
            auto& stats = cf.second->get_row_cache().stats();
            return std::make_pair(cf.first, stat{float(stats.reads_with_no_misses.rate().rates[0]), float(stats.reads_with_misses.rate().rates[0])});
        }));
    };

    auto sum_stats_per_cf = [] (std::unordered_map<utils::UUID, stat> a, std::unordered_map<utils::UUID, stat> b) {
        for (auto& r : b) {
            a[r.first] += r.second;
        }
        return std::move(a);
    };

    return _db.map_reduce0(cf_to_cache_hit_stats, std::unordered_map<utils::UUID, stat>(), sum_stats_per_cf).then([this] (std::unordered_map<utils::UUID, stat> rates) mutable {
        _diff = 0;
        // set calculated rates on all shards
        return _db.invoke_on_all([this, rates = std::move(rates), cpuid = engine().cpu_id()] (database& db) {
            sstring gstate;
            for (auto& cf : db.get_column_families() | boost::adaptors::filtered(non_system_filter)) {
                auto it = rates.find(cf.first);
                if (it == rates.end()) { // a table may be added before map/reduce compltes and this code runs
                    continue;
                }
                stat s = it->second;
                float rate = 0;
                if (s.h) {
                    rate = s.h / (s.h + s.m);
                }
                if (engine().cpu_id() == cpuid) {
                    // calculate max difference between old rate and new one for all cfs
                    _diff = std::max(_diff, std::abs(float(cf.second->get_global_cache_hit_rate()) - rate));
                    gstate += format("{}.{}:{:f};", cf.second->schema()->ks_name(), cf.second->schema()->cf_name(), rate);
                }
                cf.second->set_global_cache_hit_rate(cache_temperature(rate));
            }
            if (gstate.size()) {
                auto& g = gms::get_local_gossiper();
                auto& ss = get_local_storage_service();
                return g.add_local_application_state(gms::application_state::CACHE_HITRATES, ss.value_factory.cache_hitrates(std::move(gstate)));
            }
            return make_ready_future<>();
        });
    }).then([this] {
        // if max difference during this round is big schedule next recalculate earlier
        if (_diff < 0.01) {
            return std::chrono::milliseconds(2000);
        } else {
            return std::chrono::milliseconds(500);
        }
    });
}

future<> cache_hitrate_calculator::stop() {
    _timer.cancel();
    _stopped = true;
    return make_ready_future<>();
}

view_update_backlog_broker::view_update_backlog_broker(
        seastar::sharded<service::storage_proxy>& sp,
        gms::gossiper& gossiper)
        : _sp(sp)
        , _gossiper(gossiper) {
}

future<> view_update_backlog_broker::start() {
    _gossiper.register_(shared_from_this());
    if (engine().cpu_id() == 0) {
        // Gossiper runs only on shard 0, and there's no API to add multiple, per-shard application states.
        // Also, right now we aggregate all backlogs, since the coordinator doesn't keep per-replica shard backlogs.
        _started = seastar::async([this] {
            while (!_as.abort_requested()) {
                auto backlog = _sp.local().get_view_update_backlog();
                auto now = api::timestamp_type(std::chrono::duration_cast<std::chrono::milliseconds>(
                        std::chrono::system_clock::now().time_since_epoch()).count());
                _gossiper.add_local_application_state(
                        gms::application_state::VIEW_BACKLOG,
                        gms::versioned_value(seastar::format("{}:{}:{}", backlog.current, backlog.max, now)));
                sleep_abortable(gms::gossiper::INTERVAL, _as).get();
            }
        }).handle_exception_type([] (const seastar::sleep_aborted& ignored) { });
    }
    return make_ready_future<>();
}

future<> view_update_backlog_broker::stop() {
    _gossiper.unregister_(shared_from_this());
    _as.request_abort();
    return std::move(_started);
}

void view_update_backlog_broker::on_change(gms::inet_address endpoint, gms::application_state state, const gms::versioned_value& value) {
    if (state == gms::application_state::VIEW_BACKLOG) {
        size_t current;
        size_t max;
        api::timestamp_type ticks;
        const char* start_bound = value.value.data();
        char* end_bound;
        for (auto* ptr : {&current, &max}) {
            *ptr = std::strtoull(start_bound, &end_bound, 10);
            if (*ptr == ULLONG_MAX) {
                return;
            }
            start_bound = end_bound + 1;
        }
        if (max == 0) {
            return;
        }
        ticks = std::strtoll(start_bound, &end_bound, 10);
        if (ticks == 0 || ticks == LLONG_MAX || end_bound != value.value.data() + value.value.size()) {
            return;
        }
        auto backlog = view_update_backlog_timestamped{db::view::update_backlog{current, max}, ticks};
        auto[it, inserted] = _sp.local()._view_update_backlogs.try_emplace(endpoint, std::move(backlog));
        if (!inserted && it->second.ts < backlog.ts) {
            it->second = std::move(backlog);
        }
    }
}

void view_update_backlog_broker::on_remove(gms::inet_address endpoint) {
    _sp.local()._view_update_backlogs.erase(endpoint);
}

}
<commit_msg>cache_hitrate_calculator: fix use after free in non_system_filter lambda<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Modified by ScyllaDB
 * Copyright 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "load_broadcaster.hh"
#include "cache_hitrate_calculator.hh"
#include "db/system_keyspace.hh"
#include "gms/application_state.hh"
#include "service/storage_service.hh"
#include "service/view_update_backlog_broker.hh"
#include "database.hh"

#include <cstdlib>

namespace service {

constexpr std::chrono::milliseconds load_broadcaster::BROADCAST_INTERVAL;

logging::logger llogger("load_broadcaster");

void load_broadcaster::start_broadcasting() {
    _done = make_ready_future<>();

    // send the first broadcast "right away" (i.e., in 2 gossip heartbeats, when we should have someone to talk to);
    // after that send every BROADCAST_INTERVAL.

    _timer.set_callback([this] {
        llogger.debug("Disseminating load info ...");
        _done = _db.map_reduce0([](database& db) {
            int64_t res = 0;
            for (auto i : db.get_column_families()) {
                res += i.second->get_stats().live_disk_space_used;
            }
            return res;
        }, int64_t(0), std::plus<int64_t>()).then([this] (int64_t size) {
            gms::versioned_value::factory value_factory;
            return _gossiper.add_local_application_state(gms::application_state::LOAD,
                value_factory.load(size)).then([this] {
                _timer.arm(BROADCAST_INTERVAL);
                return make_ready_future<>();
            });
        });
    });

    _timer.arm(2 * gms::gossiper::INTERVAL);
}

future<> load_broadcaster::stop_broadcasting() {
    _timer.cancel();
    return std::move(_done);
}

// cache_hitrate_calculator implementation
cache_hitrate_calculator::cache_hitrate_calculator(seastar::sharded<database>& db, seastar::sharded<cache_hitrate_calculator>& me) : _db(db), _me(me),
        _timer(std::bind(std::mem_fn(&cache_hitrate_calculator::recalculate_timer), this))
{}

void cache_hitrate_calculator::recalculate_timer() {
    recalculate_hitrates().then_wrapped([p = shared_from_this()] (future<lowres_clock::duration> f) {
        lowres_clock::duration d;
        if (f.failed()) {
            d = std::chrono::milliseconds(2000);
        } else {
            d = f.get0();
        }
        p->run_on((engine().cpu_id() + 1) % smp::count, d);
    });
}

void cache_hitrate_calculator::run_on(size_t master, lowres_clock::duration d) {
    if (!_stopped) {
        _me.invoke_on(master, [d] (cache_hitrate_calculator& local) {
            local._timer.arm(d);
        }).handle_exception_type([] (seastar::no_sharded_instance_exception&) { /* ignore */ });
    }
}

future<lowres_clock::duration> cache_hitrate_calculator::recalculate_hitrates() {
    struct stat {
        float h = 0;
        float m = 0;
        stat& operator+=(stat& o) {
            h += o.h;
            m += o.m;
            return *this;
        }
    };

    auto non_system_filter = [&] (const std::pair<utils::UUID, lw_shared_ptr<column_family>>& cf) {
        return _db.local().find_keyspace(cf.second->schema()->ks_name()).get_replication_strategy().get_type() != locator::replication_strategy_type::local;
    };

    auto cf_to_cache_hit_stats = [non_system_filter] (database& db) {
        return boost::copy_range<std::unordered_map<utils::UUID, stat>>(db.get_column_families() | boost::adaptors::filtered(non_system_filter) |
                boost::adaptors::transformed([]  (const std::pair<utils::UUID, lw_shared_ptr<column_family>>& cf) {
            auto& stats = cf.second->get_row_cache().stats();
            return std::make_pair(cf.first, stat{float(stats.reads_with_no_misses.rate().rates[0]), float(stats.reads_with_misses.rate().rates[0])});
        }));
    };

    auto sum_stats_per_cf = [] (std::unordered_map<utils::UUID, stat> a, std::unordered_map<utils::UUID, stat> b) {
        for (auto& r : b) {
            a[r.first] += r.second;
        }
        return std::move(a);
    };

    return _db.map_reduce0(cf_to_cache_hit_stats, std::unordered_map<utils::UUID, stat>(), sum_stats_per_cf).then([this, non_system_filter] (std::unordered_map<utils::UUID, stat> rates) mutable {
        _diff = 0;
        // set calculated rates on all shards
        return _db.invoke_on_all([this, rates = std::move(rates), cpuid = engine().cpu_id(), non_system_filter] (database& db) {
            sstring gstate;
            for (auto& cf : db.get_column_families() | boost::adaptors::filtered(non_system_filter)) {
                auto it = rates.find(cf.first);
                if (it == rates.end()) { // a table may be added before map/reduce compltes and this code runs
                    continue;
                }
                stat s = it->second;
                float rate = 0;
                if (s.h) {
                    rate = s.h / (s.h + s.m);
                }
                if (engine().cpu_id() == cpuid) {
                    // calculate max difference between old rate and new one for all cfs
                    _diff = std::max(_diff, std::abs(float(cf.second->get_global_cache_hit_rate()) - rate));
                    gstate += format("{}.{}:{:f};", cf.second->schema()->ks_name(), cf.second->schema()->cf_name(), rate);
                }
                cf.second->set_global_cache_hit_rate(cache_temperature(rate));
            }
            if (gstate.size()) {
                auto& g = gms::get_local_gossiper();
                auto& ss = get_local_storage_service();
                return g.add_local_application_state(gms::application_state::CACHE_HITRATES, ss.value_factory.cache_hitrates(std::move(gstate)));
            }
            return make_ready_future<>();
        });
    }).then([this] {
        // if max difference during this round is big schedule next recalculate earlier
        if (_diff < 0.01) {
            return std::chrono::milliseconds(2000);
        } else {
            return std::chrono::milliseconds(500);
        }
    });
}

future<> cache_hitrate_calculator::stop() {
    _timer.cancel();
    _stopped = true;
    return make_ready_future<>();
}

view_update_backlog_broker::view_update_backlog_broker(
        seastar::sharded<service::storage_proxy>& sp,
        gms::gossiper& gossiper)
        : _sp(sp)
        , _gossiper(gossiper) {
}

future<> view_update_backlog_broker::start() {
    _gossiper.register_(shared_from_this());
    if (engine().cpu_id() == 0) {
        // Gossiper runs only on shard 0, and there's no API to add multiple, per-shard application states.
        // Also, right now we aggregate all backlogs, since the coordinator doesn't keep per-replica shard backlogs.
        _started = seastar::async([this] {
            while (!_as.abort_requested()) {
                auto backlog = _sp.local().get_view_update_backlog();
                auto now = api::timestamp_type(std::chrono::duration_cast<std::chrono::milliseconds>(
                        std::chrono::system_clock::now().time_since_epoch()).count());
                _gossiper.add_local_application_state(
                        gms::application_state::VIEW_BACKLOG,
                        gms::versioned_value(seastar::format("{}:{}:{}", backlog.current, backlog.max, now)));
                sleep_abortable(gms::gossiper::INTERVAL, _as).get();
            }
        }).handle_exception_type([] (const seastar::sleep_aborted& ignored) { });
    }
    return make_ready_future<>();
}

future<> view_update_backlog_broker::stop() {
    _gossiper.unregister_(shared_from_this());
    _as.request_abort();
    return std::move(_started);
}

void view_update_backlog_broker::on_change(gms::inet_address endpoint, gms::application_state state, const gms::versioned_value& value) {
    if (state == gms::application_state::VIEW_BACKLOG) {
        size_t current;
        size_t max;
        api::timestamp_type ticks;
        const char* start_bound = value.value.data();
        char* end_bound;
        for (auto* ptr : {&current, &max}) {
            *ptr = std::strtoull(start_bound, &end_bound, 10);
            if (*ptr == ULLONG_MAX) {
                return;
            }
            start_bound = end_bound + 1;
        }
        if (max == 0) {
            return;
        }
        ticks = std::strtoll(start_bound, &end_bound, 10);
        if (ticks == 0 || ticks == LLONG_MAX || end_bound != value.value.data() + value.value.size()) {
            return;
        }
        auto backlog = view_update_backlog_timestamped{db::view::update_backlog{current, max}, ticks};
        auto[it, inserted] = _sp.local()._view_update_backlogs.try_emplace(endpoint, std::move(backlog));
        if (!inserted && it->second.ts < backlog.ts) {
            it->second = std::move(backlog);
        }
    }
}

void view_update_backlog_broker::on_remove(gms::inet_address endpoint) {
    _sp.local()._view_update_backlogs.erase(endpoint);
}

}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2010-2012 Jeremy Lainé
 * Contact: http://code.google.com/p/qdjango/
 *
 * This file is part of the QDjango Library.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 */

#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QtTest>
#include <QUrl>

#include "QDjangoHttpController.h"
#include "QDjangoHttpRequest.h"
#include "QDjangoHttpResponse.h"
#include "QDjangoHttpServer.h"
#include "QDjangoUrlResolver.h"

#include "http.h"

void TestHttp::cleanupTestCase()
{
    delete httpServer;
}

void TestHttp::initTestCase()
{
    httpServer = new QDjangoHttpServer;
    httpServer->urls()->set(QRegExp("^$"), this, "_q_index");
    httpServer->urls()->set(QRegExp("^internal-server-error$"), this, "_q_error");
    QCOMPARE(httpServer->listen(QHostAddress::LocalHost, 8123), true);
}

void TestHttp::testGet_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<int>("err");
    QTest::addColumn<QByteArray>("body");

    const QString errorTemplate(
        ""
        "<title>Error</title>"
        "
%1
"
        "");

    QTest::newRow("root") << "/" << int(QNetworkReply::NoError) << QByteArray("method=GET|path=/");
    QTest::newRow("query-string") << "/?message=bar" << int(QNetworkReply::NoError) << QByteArray("method=GET|path=/|get=bar");
    QTest::newRow("not-found") << "/not-found" << int(QNetworkReply::ContentNotFoundError) << errorTemplate.arg("The document you requested was not found.").toUtf8();
    QTest::newRow("internal-server-error") << "/internal-server-error" << int(QNetworkReply::UnknownContentError) << errorTemplate.arg("An internal server error was encountered.").toUtf8();
}

void TestHttp::testGet()
{
    QFETCH(QString, path);
    QFETCH(int, err);
    QFETCH(QByteArray, body);

    QNetworkAccessManager network;
    QNetworkReply *reply = network.get(QNetworkRequest(QUrl("http://127.0.0.1:8123" + path)));

    QEventLoop loop;
    QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
    loop.exec();

    QVERIFY(reply);
    QCOMPARE(int(reply->error()), err);
    QCOMPARE(reply->readAll(), body);
    delete reply;
}

void TestHttp::testPost_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<QByteArray>("data");
    QTest::addColumn<int>("err");
    QTest::addColumn<QByteArray>("body");

    QTest::newRow("empty") << "/" << QByteArray() << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/");
    QTest::newRow("simple") << "/" << QByteArray("message=bar") << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/|post=bar");
    QTest::newRow("multi") << "/" << QByteArray("bob=wiz&message=bar&zoo=wow") << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/|post=bar");
}

void TestHttp::testPost()
{
    QFETCH(QString, path);
    QFETCH(QByteArray, data);
    QFETCH(int, err);
    QFETCH(QByteArray, body);

    QNetworkAccessManager network;
    QNetworkRequest req(QUrl("http://127.0.0.1:8123" + path));
    req.setRawHeader("Content-Type", "application/x-www-form-urlencoded");
    QNetworkReply *reply = network.post(req, data);

    QEventLoop loop;
    QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
    loop.exec();

    QVERIFY(reply);
    QCOMPARE(int(reply->error()), err);
    QCOMPARE(reply->readAll(), body);
    delete reply;
}

QDjangoHttpResponse *TestHttp::_q_index(const QDjangoHttpRequest &request)
{
    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader("Content-Type", "text/plain");

    QString output = "method=" + request.method();
    output += "|path=" + request.path();

    const QString getValue = request.get("message");
    if (!getValue.isEmpty())
        output += "|get=" + getValue;

    const QString postValue = request.post("message");
    if (!postValue.isEmpty())
        output += "|post=" + postValue;

    response->setBody(output.toUtf8());
    return response;
}

QDjangoHttpResponse *TestHttp::_q_error(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return QDjangoHttpController::serveInternalServerError(request);
}

QDjangoHttpResponse* tst_QDjangoUrlHelper::_q_index(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader("Content-Type", "text/plain");
    response->setBody("sub index");
    return response;
}

QDjangoHttpResponse* tst_QDjangoUrlHelper::_q_test(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader("Content-Type", "text/plain");
    response->setBody("sub test");
    return response;
}

void tst_QDjangoUrlResolver::cleanupTestCase()
{
    delete urlResolver;
}

void tst_QDjangoUrlResolver::initTestCase()
{
    urlHelper = new tst_QDjangoUrlHelper;
    urlSub = new QDjangoUrlResolver;
    QVERIFY(urlSub->set(QRegExp("^$"), urlHelper, "_q_index"));
    QVERIFY(urlSub->set(QRegExp("^test/$"), urlHelper, "_q_test"));

    urlResolver = new QDjangoUrlResolver;
    QVERIFY(urlResolver->set(QRegExp("^$"), this, "_q_index"));
    QVERIFY(urlResolver->set(QRegExp("^test/$"), this, "_q_noArgs"));
    QVERIFY(urlResolver->set(QRegExp("^test/([0-9]+)/$"), this, "_q_oneArg"));
    QVERIFY(urlResolver->set(QRegExp("^test/([0-9]+)/([a-z]+)/$"), this, "_q_twoArgs"));
    QVERIFY(urlResolver->include(QRegExp("^recurse/"), urlSub));
}

void tst_QDjangoUrlResolver::testRespond_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<int>("err");
    QTest::addColumn<QString>("body");

    QTest::newRow("root") << "/" << 200 << "";
    QTest::newRow("not-found") << "/non-existent/" << 404 << "";
    QTest::newRow("no-args") << "/test/" << 200 << "";
    QTest::newRow("one-args") << "/test/123/" << 200 << "";
    QTest::newRow("two-args") << "/test/123/delete/" << 200 << "";
    QTest::newRow("three-args") << "/test/123/delete/zoo/" << 404 << "";
    QTest::newRow("recurse-not-found") << "/recurse/non-existent/" << 404 << "";
    QTest::newRow("recurse-index") << "/recurse/" << 200 << "";
    QTest::newRow("recurse-test") << "/recurse/test/" << 200 << "";
}

void tst_QDjangoUrlResolver::testRespond()
{
    QFETCH(QString, path);
    QFETCH(int, err);
    QFETCH(QString, body);

    QDjangoHttpTestRequest request("GET", path);
    QDjangoHttpResponse *response = urlResolver->respond(request, path);
    QVERIFY(response);
    QCOMPARE(int(response->statusCode()), err);
}

void tst_QDjangoUrlResolver::testReverse_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<QObject*>("receiver");
    QTest::addColumn<QString>("member");
    QTest::addColumn<QString>("args");

    QObject *receiver = this;
    QTest::newRow("root") << "/" << receiver << "_q_index" << "";
    QTest::newRow("no-args") << "/test/" << receiver << "_q_noArgs" << "";
    QTest::newRow("one-arg") << "/test/123/" << receiver << "_q_oneArg" << "123";
    QTest::newRow("two-args") << "/test/123/delete/" << receiver << "_q_twoArgs" << "123|delete";
    QTest::newRow("too-few-args") << "" << receiver << "_q_oneArg" << "";
    QTest::newRow("too-many-args") << "" << receiver << "_q_noArgs" << "123";

    receiver = urlHelper;
    QTest::newRow("recurse-index") << "/recurse/" << receiver << "_q_index" << "";
    QTest::newRow("recurse-test") << "/recurse/test/" << receiver << "_q_test" << "";
}

void tst_QDjangoUrlResolver::testReverse()
{
    QFETCH(QString, path);
    QFETCH(QObject*, receiver);
    QFETCH(QString, member);
    QFETCH(QString, args);

    QVariantList varArgs;
    if (!args.isEmpty()) {
        foreach (const QString &bit, args.split('|'))
            varArgs << bit;
    }
    QCOMPARE(urlResolver->reverse(receiver, member.toLatin1(), varArgs), path);
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_index(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_noArgs(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_oneArg(const QDjangoHttpRequest &request, const QString &id)
{
    Q_UNUSED(request);
    Q_UNUSED(id);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_twoArgs(const QDjangoHttpRequest &request, const QString &id, const QString &action)
{
    Q_UNUSED(request);
    Q_UNUSED(id);
    Q_UNUSED(action);

    return new QDjangoHttpResponse;
}
<commit_msg>use QLatin1String where appropriate<commit_after>/*
 * Copyright (C) 2010-2012 Jeremy Lainé
 * Contact: http://code.google.com/p/qdjango/
 *
 * This file is part of the QDjango Library.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 */

#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QtTest>
#include <QUrl>

#include "QDjangoHttpController.h"
#include "QDjangoHttpRequest.h"
#include "QDjangoHttpResponse.h"
#include "QDjangoHttpServer.h"
#include "QDjangoUrlResolver.h"

#include "http.h"

void TestHttp::cleanupTestCase()
{
    delete httpServer;
}

void TestHttp::initTestCase()
{
    httpServer = new QDjangoHttpServer;
    httpServer->urls()->set(QRegExp(QLatin1String(QLatin1String("^$"))), this, "_q_index");
    httpServer->urls()->set(QRegExp(QLatin1String("^internal-server-error$")), this, "_q_error");
    QCOMPARE(httpServer->listen(QHostAddress::LocalHost, 8123), true);
}

void TestHttp::testGet_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<int>("err");
    QTest::addColumn<QByteArray>("body");

    const QString errorTemplate = QLatin1String(
        ""
        "<title>Error</title>"
        "
%1
"
        "");

    QTest::newRow("root") << "/" << int(QNetworkReply::NoError) << QByteArray("method=GET|path=/");
    QTest::newRow("query-string") << "/?message=bar" << int(QNetworkReply::NoError) << QByteArray("method=GET|path=/|get=bar");
    QTest::newRow("not-found") << "/not-found" << int(QNetworkReply::ContentNotFoundError) << errorTemplate.arg(QLatin1String("The document you requested was not found.")).toUtf8();
    QTest::newRow("internal-server-error") << "/internal-server-error" << int(QNetworkReply::UnknownContentError) << errorTemplate.arg(QLatin1String("An internal server error was encountered.")).toUtf8();
}

void TestHttp::testGet()
{
    QFETCH(QString, path);
    QFETCH(int, err);
    QFETCH(QByteArray, body);

    QNetworkAccessManager network;
    QNetworkReply *reply = network.get(QNetworkRequest(QUrl(QLatin1String("http://127.0.0.1:8123") + path)));

    QEventLoop loop;
    QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
    loop.exec();

    QVERIFY(reply);
    QCOMPARE(int(reply->error()), err);
    QCOMPARE(reply->readAll(), body);
    delete reply;
}

void TestHttp::testPost_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<QByteArray>("data");
    QTest::addColumn<int>("err");
    QTest::addColumn<QByteArray>("body");

    QTest::newRow("empty") << "/" << QByteArray() << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/");
    QTest::newRow("simple") << "/" << QByteArray("message=bar") << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/|post=bar");
    QTest::newRow("multi") << "/" << QByteArray("bob=wiz&message=bar&zoo=wow") << int(QNetworkReply::NoError) << QByteArray("method=POST|path=/|post=bar");
}

void TestHttp::testPost()
{
    QFETCH(QString, path);
    QFETCH(QByteArray, data);
    QFETCH(int, err);
    QFETCH(QByteArray, body);

    QNetworkAccessManager network;
    QNetworkRequest req(QUrl(QLatin1String("http://127.0.0.1:8123") + path));
    req.setRawHeader("Content-Type", "application/x-www-form-urlencoded");
    QNetworkReply *reply = network.post(req, data);

    QEventLoop loop;
    QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
    loop.exec();

    QVERIFY(reply);
    QCOMPARE(int(reply->error()), err);
    QCOMPARE(reply->readAll(), body);
    delete reply;
}

QDjangoHttpResponse *TestHttp::_q_index(const QDjangoHttpRequest &request)
{
    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader(QLatin1String("Content-Type"), QLatin1String("text/plain"));

    QString output = QLatin1String("method=") + request.method();
    output += QLatin1String("|path=") + request.path();

    const QString getValue = request.get(QLatin1String("message"));
    if (!getValue.isEmpty())
        output += QLatin1String("|get=") + getValue;

    const QString postValue = request.post(QLatin1String("message"));
    if (!postValue.isEmpty())
        output += QLatin1String("|post=") + postValue;

    response->setBody(output.toUtf8());
    return response;
}

QDjangoHttpResponse *TestHttp::_q_error(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return QDjangoHttpController::serveInternalServerError(request);
}

QDjangoHttpResponse* tst_QDjangoUrlHelper::_q_index(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader(QLatin1String("Content-Type"), QLatin1String("text/plain"));
    response->setBody("sub index");
    return response;
}

QDjangoHttpResponse* tst_QDjangoUrlHelper::_q_test(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    QDjangoHttpResponse *response = new QDjangoHttpResponse;
    response->setHeader(QLatin1String("Content-Type"), QLatin1String("text/plain"));
    response->setBody("sub test");
    return response;
}

void tst_QDjangoUrlResolver::cleanupTestCase()
{
    delete urlResolver;
}

void tst_QDjangoUrlResolver::initTestCase()
{
    urlHelper = new tst_QDjangoUrlHelper;
    urlSub = new QDjangoUrlResolver;
    QVERIFY(urlSub->set(QRegExp(QLatin1String("^$")), urlHelper, "_q_index"));
    QVERIFY(urlSub->set(QRegExp(QLatin1String("^test/$")), urlHelper, "_q_test"));

    urlResolver = new QDjangoUrlResolver;
    QVERIFY(urlResolver->set(QRegExp(QLatin1String("^$")), this, "_q_index"));
    QVERIFY(urlResolver->set(QRegExp(QLatin1String("^test/$")), this, "_q_noArgs"));
    QVERIFY(urlResolver->set(QRegExp(QLatin1String("^test/([0-9]+)/$")), this, "_q_oneArg"));
    QVERIFY(urlResolver->set(QRegExp(QLatin1String("^test/([0-9]+)/([a-z]+)/$")), this, "_q_twoArgs"));
    QVERIFY(urlResolver->include(QRegExp(QLatin1String("^recurse/")), urlSub));
}

void tst_QDjangoUrlResolver::testRespond_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<int>("err");
    QTest::addColumn<QString>("body");

    QTest::newRow("root") << "/" << 200 << "";
    QTest::newRow("not-found") << "/non-existent/" << 404 << "";
    QTest::newRow("no-args") << "/test/" << 200 << "";
    QTest::newRow("one-args") << "/test/123/" << 200 << "";
    QTest::newRow("two-args") << "/test/123/delete/" << 200 << "";
    QTest::newRow("three-args") << "/test/123/delete/zoo/" << 404 << "";
    QTest::newRow("recurse-not-found") << "/recurse/non-existent/" << 404 << "";
    QTest::newRow("recurse-index") << "/recurse/" << 200 << "";
    QTest::newRow("recurse-test") << "/recurse/test/" << 200 << "";
}

void tst_QDjangoUrlResolver::testRespond()
{
    QFETCH(QString, path);
    QFETCH(int, err);
    QFETCH(QString, body);

    QDjangoHttpTestRequest request(QLatin1String("GET"), path);
    QDjangoHttpResponse *response = urlResolver->respond(request, path);
    QVERIFY(response);
    QCOMPARE(int(response->statusCode()), err);
}

void tst_QDjangoUrlResolver::testReverse_data()
{
    QTest::addColumn<QString>("path");
    QTest::addColumn<QObject*>("receiver");
    QTest::addColumn<QString>("member");
    QTest::addColumn<QString>("args");

    QObject *receiver = this;
    QTest::newRow("root") << "/" << receiver << "_q_index" << "";
    QTest::newRow("no-args") << "/test/" << receiver << "_q_noArgs" << "";
    QTest::newRow("one-arg") << "/test/123/" << receiver << "_q_oneArg" << "123";
    QTest::newRow("two-args") << "/test/123/delete/" << receiver << "_q_twoArgs" << "123|delete";
    QTest::newRow("too-few-args") << "" << receiver << "_q_oneArg" << "";
    QTest::newRow("too-many-args") << "" << receiver << "_q_noArgs" << "123";

    receiver = urlHelper;
    QTest::newRow("recurse-index") << "/recurse/" << receiver << "_q_index" << "";
    QTest::newRow("recurse-test") << "/recurse/test/" << receiver << "_q_test" << "";
}

void tst_QDjangoUrlResolver::testReverse()
{
    QFETCH(QString, path);
    QFETCH(QObject*, receiver);
    QFETCH(QString, member);
    QFETCH(QString, args);

    QVariantList varArgs;
    if (!args.isEmpty()) {
        foreach (const QString &bit, args.split(QLatin1Char('|')))
            varArgs << bit;
    }
    QCOMPARE(urlResolver->reverse(receiver, member.toLatin1(), varArgs), path);
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_index(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_noArgs(const QDjangoHttpRequest &request)
{
    Q_UNUSED(request);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_oneArg(const QDjangoHttpRequest &request, const QString &id)
{
    Q_UNUSED(request);
    Q_UNUSED(id);

    return new QDjangoHttpResponse;
}

QDjangoHttpResponse* tst_QDjangoUrlResolver::_q_twoArgs(const QDjangoHttpRequest &request, const QString &id, const QString &action)
{
    Q_UNUSED(request);
    Q_UNUSED(id);
    Q_UNUSED(action);

    return new QDjangoHttpResponse;
}
<|endoftext|>


<commit_before>#include "catch.hpp"
#include <iostream>
//#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string const & file_name, size_t const index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
bool isANotMoreThanB(person const & A, person const & B) {
        char * A_name = A.getName();
        char * B_name = B.getName();
	for (size_t i = 0; i < A.name_length; i++) {
		char A_char = A_name[i];
		char B_char = (i < B.name_length) ? B_name[i] : ' ';
		if (letterI(A_name[i], i == 0) < letterI(B_name[i], i == 0)) {
			return true;
		}
		else {
			if (A_name[i] != B_name[i]) {
				return false;
			}
		}
	}
        delete []A_name;
        delete []B_name;
	return true;
}

SCENARIO("Sort", "[s]") {
	size_t n_persons = 200;
	//system("start www.cyberforum.ru/cpp-beginners/thread82643.html#post458604");
	size_t RAM_amount = 1;
	std::string names_file_name = "F:\\1\\names.txt";
	std::string surnames_file_name = "F:\\1\\surnames.txt";
	std::string database_file_name = "8.txt";
	std::string output_file_name = "F:\\1\\sorted_database.txt";

	//Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
	Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
	size_t start = clock();
	database_sorter.sortDatabase();
	size_t result = clock() - start;
	database_sorter.closeDatabase();
	std::cout << result << std::endl;

	for (size_t i = 0; i < 100; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}
}
<commit_msg>Update init.cpp<commit_after>#include "catch.hpp"
#include <iostream>
//#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string const & file_name, size_t const index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
bool isANotMoreThanB(person const & A, person const & B) {
        char * A_name = A.getName();
        char * B_name = B.getName();
	for (size_t i = 0; i < A.name_length; i++) {
		char A_char = A_name[i];
		char B_char = (i < B.name_length) ? B_name[i] : ' ';
		if (letterI(A_name[i], i == 0) < letterI(B_name[i], i == 0)) {
			return true;
		}
		else {
			if (A_name[i] != B_name[i]) {
				return false;
			}
		}
	}
        delete []A_name;
        delete []B_name;
	return true;
}

SCENARIO("Sort", "[s]") {
	size_t n_persons = 200;
	//system("start www.cyberforum.ru/cpp-beginners/thread82643.html#post458604");
	size_t RAM_amount = 1;
	std::string names_file_name = "F:\\1\\names.txt";
	std::string surnames_file_name = "F:\\1\\surnames.txt";
	std::string database_file_name = "8.txt";
	std::string output_file_name = "F:\\1\\sorted_database.txt";

	//Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
	Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
	size_t start = clock();
	database_sorter.sortDatabase();
	size_t result = clock() - start;
	database_sorter.closeDatabase();
	std::cout << "Result " << result << std::endl;

	for (size_t i = 0; i < 100; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}
}
<|endoftext|>


<commit_before>#include "catch.hpp"
#include "stack.hpp"
#include <thread>
#include <iostream>
#include <chrono>

const int N = 20;
TEST_CASE("drt", "[T]") {
	bool tests[N];
	for (auto test : tests) {
		test = false;
	}

	stack<int> st;
	std::thread th2([&](stack<int> & st, bool * tests) {
		for (size_t i = 0; i < N;) {
			if (!st.empty()) {
				std::cout << st.top() << std::endl;
				st.pop();
				i++;
				std::this_thread::sleep_for(std::chrono::milliseconds(30));
			}
		}
	}, std::ref(st), tests);
	std::thread th1([&](stack<int> & st, bool * tests) {
		for (size_t i = 0; i < N; i++) {
			st.push(i);
			std::this_thread::sleep_for(std::chrono::milliseconds(20));
		}
	}, std::ref(st), tests);
	th1.join();
	th2.join();
	for (auto test : tests) {
		REQUIRE(true);
	}
}
TEST_CASE("Cyrillic_test", "[instantiation]") {
	stack<int> st;
	for (size_t i = 0; i < 4; i++) {
		st.push(i);
	}
	for (size_t i = 0; i < 2; i++) {
		st.pop();
	}
}
TEST_CASE("Stack can be instantiated by various types", "[instantiation]") {
	REQUIRE_NOTHROW(stack<int> st1);
	REQUIRE_NOTHROW(stack<double> st2);
	REQUIRE_NOTHROW(stack<char> st3);
	REQUIRE_NOTHROW(stack<int*> st4);
	REQUIRE_NOTHROW(stack<stack<int>> st5);
}
TEST_CASE("Push, pop, top", "[push_pop_top]") {
	stack<int> st;
	st.push(1);
	st.push(2);
	st.top();
	REQUIRE(st.top() == 2);
	st.pop();
	REQUIRE(st.top() == 1);
	st.pop();
}
TEST_CASE("count", "[count]") {
	stack<int> st;
	REQUIRE(st.count() == 0);
	st.push(1);
	REQUIRE(st.count() == 1);
	st.push(2);
	REQUIRE(st.count() == 2);
	st.pop();
	REQUIRE(st.count() == 1);
	st.pop();
	REQUIRE(st.count() == 0);
}
TEST_CASE("Copy constructor", "[copy_ctr]") {
	stack<double> st1;
	st1.push(1);
	st1.push(2);
	//REQUIRE_NOTHROW(stack<double> st2 = st1);

	stack<double> st3 = st1;
	stack<double> st4 = st1;
	REQUIRE(st1.top() == 2);
	REQUIRE(st3.top() == 2);
	st3.pop();
	REQUIRE(st3.top() == 1);
	REQUIRE(st4.top() == 2);
	st4.pop();
	REQUIRE(st4.top() == 1);
}
TEST_CASE("Equal", "[equal]") {
	stack<int> st1;
	REQUIRE_NOTHROW(stack<int> st2;
	st2 = st1);
	stack<int> st2;
	st1.push(1);
	st1.push(4);
	st2 = st1;
	REQUIRE(st2.top() == 4);
	st2.pop();
	REQUIRE(st2.top() == 1);
}
TEST_CASE("Empty", "[empty]") {
	stack<int> st1;
	REQUIRE(st1.empty());
	st1.push(38);
	REQUIRE(!st1.empty());
}
TEST_CASE("Allocator doesn't (use or need) default ctors", "[allocator]") {
	class A {
	public:
		A() {
			throw "ctor is called";
		}
		A(int a) {
			;
		}
	};
	REQUIRE_NOTHROW(
		stack<A> st;
	st.push(4);
	st.push(20);
	);
	class B {
	public:
		B(int a) {
			;
		}
	};
	stack<B> st;
	st.push(40);
}
bool dtor_is_called12 = false;
TEST_CASE("Allocator calls dtor when pop", "[allocator_dtor]") {
	class A {
	public:
		A(int a) {
			//std::cout << "A(int)" << std::endl;
		}
		A(const A & a) {
			//std::cout << "copy constructor is called " << std::endl;
		}
		~A() {
			dtor_is_called12 = true;
		}
	};
	stack<A> st;
	st.push(32);
	st.pop();
	REQUIRE(dtor_is_called12);
}
<commit_msg>Update init.cpp<commit_after>#include "catch.hpp"
#include "stack.hpp"
#include <thread>
#include <iostream>
#include <chrono>

const int N = 17;
TEST_CASE("drt", "[T]") {
	bool tests[N];
	for (auto test : tests) {
		test = true;
	}

	stack<int> st;
	std::thread th2([&](stack<int> & st, bool * tests) {
		std::cout << "Popping thread\n";
		st.pop();
		/*for (size_t i = 0; i < N;) {
			std::cout << "Popping thread\n";
			std::shared_ptr<int> current_element = st.pop();
			if (current_element) {
				std::cout << *(current_element);
				i++;
				std::this_thread::sleep_for(std::chrono::milliseconds(30));
			}
		}*/
	}, std::ref(st), tests);
	std::thread th1([&](stack<int> & st, bool * tests) {
		for (size_t i = 0; i < N; i++) {
			std::cout << "Pushing thread\n";
			st.push(i);
			std::this_thread::sleep_for(std::chrono::milliseconds(20));
		}
	}, std::ref(st), tests);
	th1.join();
	th2.join();
	for (auto test : tests) {
		REQUIRE(true);
	}
}
TEST_CASE("Cyrillic_test", "[instantiation]") {
	stack<int> st;
	for (size_t i = 0; i < 4; i++) {
		st.push(i);
	}
	for (size_t i = 0; i < 2; i++) {
		st.pop();
	}
}
TEST_CASE("Stack can be instantiated by various types", "[instantiation]") {
	REQUIRE_NOTHROW(stack<int> st1);
	REQUIRE_NOTHROW(stack<double> st2);
	REQUIRE_NOTHROW(stack<char> st3);
	REQUIRE_NOTHROW(stack<int*> st4);
	REQUIRE_NOTHROW(stack<stack<int>> st5);
}
TEST_CASE("Push, pop, top", "[push_pop_top]") {
	stack<int> st;
	st.push(1);
	st.push(2);
	REQUIRE(*(st.pop()) == 2);
	REQUIRE(*(st.pop()) == 1);
}
TEST_CASE("count", "[count]") {
	stack<int> st;
	REQUIRE(st.count() == 0);
	st.push(1);
	REQUIRE(st.count() == 1);
	st.push(2);
	REQUIRE(st.count() == 2);
	st.pop();
	REQUIRE(st.count() == 1);
	st.pop();
	REQUIRE(st.count() == 0);
}
TEST_CASE("Copy constructor", "[copy_ctr]") {
	stack<double> st1;
	st1.push(1);
	st1.push(2);
	//REQUIRE_NOTHROW(stack<double> st2 = st1);

	stack<double> st3 = st1;
	stack<double> st4 = st1;
	REQUIRE(*(st1.pop()) == 2);
	REQUIRE(*(st3.pop()) == 2);
	REQUIRE(*(st3.pop()) == 1);
	REQUIRE(*(st4.pop()) == 2);
	REQUIRE(*(st4.pop()) == 1);
}
TEST_CASE("Equal", "[equal]") {
	stack<int> st1;
	REQUIRE_NOTHROW(stack<int> st2;
	st2 = st1);
	stack<int> st2;
	st1.push(1);
	st1.push(4);
	st2 = st1;
	REQUIRE(*(st2.pop()) == 4);
	REQUIRE(*(st2.pop()) == 1);
}
TEST_CASE("Empty", "[empty]") {
	stack<int> st1;
	REQUIRE(st1.empty());
	st1.push(38);
	REQUIRE(!st1.empty());
}
TEST_CASE("Allocator doesn't (use or need) default ctors", "[allocator]") {
	class A {
	public:
		A() {
			throw "ctor is called";
		}
		A(int a) {
			;
		}
	};
	REQUIRE_NOTHROW(
		stack<A> st;
	st.push(4);
	st.push(20);
	);
	class B {
	public:
		B(int a) {
			;
		}
	};
	stack<B> st;
	st.push(40);
}
bool dtor_is_called12 = false;
TEST_CASE("Allocator calls dtor when pop", "[allocator_dtor]") {
	class A {
	public:
		A(int a) {
			//std::cout << "A(int)" << std::endl;
		}
		A(const A & a) {
			//std::cout << "copy constructor is called " << std::endl;
		}
		~A() {
			dtor_is_called12 = true;
		}
	};
	stack<A> st;
	st.push(32);
	st.pop();
	REQUIRE(dtor_is_called12);
}
<|endoftext|>


<commit_before>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO ("init", "[init]")
{
  BinaryTree<int> obj;
  REQUIRE(obj.root_() == nullptr);
}

SCENARIO ("output to cout", "<<")
{
  BinaryTree<int> tree;
  tree.insert_node(3);
  REQUIRE(tree.show(tree.root_()) != nullptr, 3);
}

SCENARIO("insert", "[insert]")
{
  BinaryTree<int> obj;
  obj.insert_node(3);
  REQUIRE(obj.find_node(3, obj.root_())->data == 3);
}

SCENARIO("find_node", "[find_node]")
{
  BinaryTree<int> obj;
  obj.insert_node(2);
  REQUIRE(obj.find_node(2, obj.root_()) != nullptr);
  REQUIRE(obj.find_node(2, obj.root_())->data == 2);
}

SCENARIO("get root", "[get root]")
{
  BinaryTree<int> obj;
  obj.insert_node(4);
  REQUIRE(obj.root_() != 0);
}
<commit_msg>Update init.cpp<commit_after>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO ("init", "[init]")
{
  BinaryTree<int> obj;
  REQUIRE(obj.root_() == nullptr);
}

SCENARIO ("output to cout", "<<")
{
  BinaryTree<int> tree;
  tree.insert_node(3);
  REQUIRE(tree.show(tree.root_())->data == 3, 3);
}

SCENARIO("insert", "[insert]")
{
  BinaryTree<int> obj;
  obj.insert_node(3);
  REQUIRE(obj.find_node(3, obj.root_())->data == 3);
}

SCENARIO("find_node", "[find_node]")
{
  BinaryTree<int> obj;
  obj.insert_node(2);
  REQUIRE(obj.find_node(2, obj.root_()) != nullptr);
  REQUIRE(obj.find_node(2, obj.root_())->data == 2);
}

SCENARIO("get root", "[get root]")
{
  BinaryTree<int> obj;
  obj.insert_node(4);
  REQUIRE(obj.root_() != 0);
}
<|endoftext|>


<commit_before>#include <stack.hpp>
#include <catch.hpp>

SCENARIO("init", "[init]") {
	stack<int> A;
	REQUIRE(A.count() == 0);
}

SCENARIO("Push", "[push]") {
	stack<int> A;
	A.push(1);
	A.push(2);
	REQUIRE(A.count() == 2));
}

SCENARIO("Pop", "[pop]") {
	stack<int> A;
	A.push(1);
	A.push(2);
	A.pop();
	REQUIRE(A.count() == 1));
}

SCENARIO("oper=", "[oper=]"){
	stack<int> A;
	A.push(1);
	A.push(2);
	stack<int> B;
	B = A;
	REQUIRE(B.count() == 2));
}
<commit_msg>Update init.cpp<commit_after>#include <stack.hpp>
#include <catch.hpp>

SCENARIO("init", "[init]") {
	stack<int> A;
	REQUIRE(A.count() == 0);
}

SCENARIO("Push", "[push]") {
	stack<int> A;
	A.push(1);
	A.push(2);
	REQUIRE(A.count() == 2);
}

SCENARIO("Pop", "[pop]") {
	stack<int> A;
	A.push(1);
	A.push(2);
	A.pop();
	REQUIRE(A.count() == 1);
}

SCENARIO("oper=", "[oper=]"){
	stack<int> A;
	A.push(1);
	A.push(2);
	stack<int> B;
	B = A;
	REQUIRE(B.count() == 2);
}
<|endoftext|>


<commit_before>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO ("init", "[init]")
{
  BinaryTree<int> obj;
  REQUIRE(obj.root_() == nullptr);
}

SCENARIO("insert", "[insert]")
{
  BinaryTree<int> obj;
  obj.insert_node(3);
  REQUIRE(obj.find_node(3, obj.root_())->data == 3);
}

SCENARIO("find_node", "[find_node]")
{
  BinaryTree<int> obj;
  obj.insert_node(2);
  REQUIRE(obj.find_node(2, obj.root_()) != nullptr);
  REQUIRE(obj.find_node(2, obj.root_())->data == 2);
}

SCENARIO("get root", "[get root]")
{
  BinaryTree<int> obj;
  obj.insert_node(4);
  REQUIRE(obj.root_() != 0);
}

SCENARIO ("output to cout", "<<")
{
 BinaryTree<int> tree;
 tree.insert_node(3);
 REQUIRE( std::cout << tree );
}
<commit_msg>Update init.cpp<commit_after>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO ("init", "[init]")
{
  BinaryTree<int> obj;
  REQUIRE(obj.root_() == nullptr);
}

SCENARIO("insert", "[insert]")
{
  BinaryTree<int> obj;
  obj.insert_node(3);
  REQUIRE(obj.find_node(3, obj.root_())->data == 3);
}

SCENARIO("find_node", "[find_node]")
{
  BinaryTree<int> obj;
  obj.insert_node(2);
  REQUIRE(obj.find_node(2, obj.root_()) != nullptr);
  REQUIRE(obj.find_node(2, obj.root_())->data == 2);
}

SCENARIO("get root", "[get root]")
{
  BinaryTree<int> obj;
  obj.insert_node(4);
  REQUIRE(obj.root_() != 0);
}

SCENARIO ("output to cout", "<<")
{
 BinaryTree<int> tree;
 tree.insert_node(3);
// REQUIRE( std::cout << tree );
}
<|endoftext|>


<commit_before>
#include <tree.hpp>
#include <catch.hpp>

SCENARIO("null")
{
	Tree<int> a;
	REQUIRE(a.tree_one()==NULL);
}
SCENARIO("add")
{
	Tree <int> a;
	bool b=a.add(5);
	REQUIRE(b == 1);
}
SCENARIO("find")
{
	Tree<int> a;
	a.add(5);
	bool b = a.find(5);
	REQUIRE(b == 1);
}
SCENARIO("file")
{
	Tree<int> a, b;
	b.add(3);
	b.add(2);
	b.add(1);
	b.add(5);
	b.pr("Tr.txt");
	a.file_tree("Tr.txt");
	bool b = a.find(3);
	REQUIRE(b == 1);
	b = a.find(2);
	REQUIRE(b == 1);
	b = a.find(1);
	REQUIRE(b == 1);
	b = a.find(5);
	REQUIRE(b == 1);
}
<commit_msg>Update init.cpp<commit_after>
#include <tree.hpp>
#include <catch.hpp>

SCENARIO("null")
{
	Tree<int> a;
	REQUIRE(a.tree_one()==NULL);
}
SCENARIO("add")
{
	Tree <int> a;
	bool b=a.add(5);
	REQUIRE(b == 1);
}
SCENARIO("find")
{
	Tree<int> a;
	a.add(5);
	bool b = a.find(5);
	REQUIRE(b == 1);
}
SCENARIO("file")
{
	Tree<int> a, c;
	c.add(3);
	c.add(2);
	c.add(1);
	c.add(5);
	c.pr("Tr.txt");
	a.file_tree("Tr.txt");
	bool b = a.find(3);
	REQUIRE(b == 1);
	b = a.find(2);
	REQUIRE(b == 1);
	b = a.find(1);
	REQUIRE(b == 1);
	b = a.find(5);
	REQUIRE(b == 1);
}
<|endoftext|>


<commit_before>#include "catch.hpp"
#include <iostream>
#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string file_name, size_t index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
bool isALessThanB(char A, char B, bool is_capital) {
	std::string alphabet;
	if (is_capital) {
		alphabet = " АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	}
	else {
		alphabet = " абвгдеёжзийклмнопрстуфхцчшщъыьэюя";
	}
	size_t A_i;
	size_t B_i;
	for (size_t i = 0; i < alphabet.size(); i++) {
		if (alphabet[i] == A) {
			A_i = i;
		}
		if (alphabet[i] == B) {
			B_i = i;
		}
	}
	return A_i < B_i;
}
bool isANotMoreThanB(person A, person B) {
	for (size_t i = 0; i < A.surname.length(); i++) {
		char A_char = A.surname[i];
		char B_char = (i < B.surname.length()) ? B.surname[i] : ' ';
		if (isALessThanB(A.surname[i], B.surname[i], i == 0)) {
			return true;
		}
		else {
			if (A.surname[i] != B.surname[i]) {
				return false;
			}
		}
	}
	return true;
}

SCENARIO("Sort", "[s]") {
	//setlocale(LC_ALL, "ru_RU.utf8");
	std::ifstream ru("russian_letters.txt");
	for (size_t i = 0; i < 6; i++) {
		std::string str;
		ru >> str;
		russian_letters[i] = str[0];
	}
	ru.close();
	size_t n_persons = 2001;
	size_t RAM_amount = 10000;
	std::string names_file_name = "names.txt";
	std::string surnames_file_name = "surnames.txt";
	std::string database_file_name = "database.txt";
	std::string output_file_name = "sorted_database.txt";
	{
		Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
		Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
		size_t start = clock();
		//database_sorter.sortDatabase();
		size_t result = clock() - start;
		std::cout << result << std::endl;
		system("pause");
	}

	for (size_t i = 0; i < 0; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}

	std::string str = "АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	REQUIRE(letterI(str[0], true) == 1);
	REQUIRE(letterI(str[1], true) == 2);
	REQUIRE(letterI(str[2], true) == 3);
	REQUIRE(letterI(str[3], true) == 4);
	REQUIRE(letterI(str[4], true) == 5);
}
<commit_msg>Update init.cpp<commit_after>#include "catch.hpp"
#include <iostream>
#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string file_name, size_t index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
/*bool isALessThanB(char A, char B, bool is_capital) {
	std::string alphabet;
	if (is_capital) {
		alphabet = " АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	}
	else {
		alphabet = " абвгдеёжзийклмнопрстуфхцчшщъыьэюя";
	}
	size_t A_i;
	size_t B_i;
	for (size_t i = 0; i < alphabet.size(); i++) {
		if (alphabet[i] == A) {
			A_i = i;
		}
		if (alphabet[i] == B) {
			B_i = i;
		}
	}
	return A_i < B_i;
}*/
bool isANotMoreThanB(person A, person B) {
	for (size_t i = 0; i < A.surname.length(); i++) {
		char A_char = A.surname[i];
		char B_char = (i < B.surname.length()) ? B.surname[i] : ' ';
		if (letterI(A.surname[i], i == 0) < letterI(B.surname[i], i == 0)) {
			return true;
		}
		else {
			if (A.surname[i] != B.surname[i]) {
				return false;
			}
		}
	}
	return true;
}

SCENARIO("Sort", "[s]") {
	//setlocale(LC_ALL, "ru_RU.utf8");
	std::ifstream ru("russian_letters.txt");
	for (size_t i = 0; i < 6; i++) {
		std::string str;
		ru >> str;
		russian_letters[i] = str[0];
	}
	ru.close();
	size_t n_persons = 2001;
	size_t RAM_amount = 10000;
	std::string names_file_name = "names.txt";
	std::string surnames_file_name = "surnames.txt";
	std::string database_file_name = "database.txt";
	std::string output_file_name = "sorted_database.txt";
	{
		Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
		Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
		size_t start = clock();
		//database_sorter.sortDatabase();
		size_t result = clock() - start;
		std::cout << result << std::endl;
		system("pause");
	}

	for (size_t i = 0; i < 0; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}

	std::string str = "АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	REQUIRE(letterI(str[0], true) == 1);
	REQUIRE(letterI(str[1], true) == 2);
	REQUIRE(letterI(str[2], true) == 3);
	REQUIRE(letterI(str[3], true) == 4);
	REQUIRE(letterI(str[4], true) == 5);
}
<|endoftext|>


<commit_before>#include "nodemanager.h"
#include <string>
#include <iostream>
#include <random>
#include <ctime>

class test : public dgal::individual {
	public:
		test() : dgal::individual(10) {}
		test(const std::shared_ptr<test> a, const std::shared_ptr<test> b) : dgal::individual(a, b) {}
		//std::string serialize() const {std::cout << "A" << std::endl; return "";}
		void print(){ for(size_t i = 0; i < weights.size(); i++){ std::cout << weights[i] << " "; } std::cout << std::endl; }
};

int main(){
	std::srand(std::time(0));
	std::shared_ptr<test> A(new test);
	std::shared_ptr<test> B(new test);

	std::shared_ptr<test> C(new test(A,B));

	std::cout << "A:\n";
	A->print();
	std::cout << "B:\n";
	B->print();
	std::cout << "C:\n";
	C->print();

	return 0;
}<commit_msg>Adding beginning of test for node manager<commit_after>#include "nodemanager.h"
#include <string>
#include <iostream>
#include <random>
#include <ctime>

class test : public dgal::individual {
	public:
		test() : dgal::individual(10) {}
		test(const std::shared_ptr<test> a, const std::shared_ptr<test> b) : dgal::individual(a, b) {}
		//std::string serialize() const {std::cout << "A" << std::endl; return "";}
		void print(){ for(size_t i = 0; i < weights.size(); i++){ std::cout << weights[i] << " "; } std::cout << std::endl; }

		void run(){ fitness = rand(); }
};

int main(){

	//Testing custom individuals
	std::srand(std::time(0));
	std::shared_ptr<test> A(new test);
	std::shared_ptr<test> B(new test);

	std::shared_ptr<test> C(new test(A,B));

	std::cout << "A:\n";
	A->print();
	std::cout << "B:\n";
	B->print();
	std::cout << "C:\n";
	C->print();

	//End testing of individuals

	//Start testing of node manager

	dgal::nodeManager<test> node;

	return 0;
}
<|endoftext|>


<commit_before>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/tpu/tpu_initializer_helper.h"

#include <dirent.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>

#include <filesystem>
#include <fstream>

#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow/core/platform/logging.h"

namespace tensorflow {
namespace tpu {
namespace {

static std::string GetEnvVar(const char* name) {
  // Constructing a std::string directly from nullptr is undefined behavior.
  return absl::StrCat(getenv(name));
}

bool GetEnvBool(const char* name, bool defval) {
  const char* env = getenv(name);
  if (env == nullptr) {
    return defval;
  }
  if (std::strcmp(env, "true") == 0) {
    return true;
  }
  if (std::strcmp(env, "false") == 0) {
    return false;
  }
  int int_env;
  bool has_int = absl::SimpleAtoi(env, &int_env);
  return has_int && int_env != 0;
}

}  // namespace

// This function gets pid of a process and checks if that process is using tpu.
// It is not able to check processes that are owned by another user.
bool IsTpuUsed(int64_t pid) {
  std::string path = absl::StrCat("/proc/", pid, "/fd");
  DIR* raw_fd_dir = opendir(path.c_str());
  if (!raw_fd_dir) {
    return false;
  }
  std::unique_ptr<DIR, int (*)(DIR*)> fd_dir(raw_fd_dir, closedir);
  struct dirent* ent;
  std::string line;
  std::string tpu_dev_path = "/dev/accel0";
  line.resize(tpu_dev_path.size());
  while ((ent = readdir(raw_fd_dir))) {
    if (!isdigit(*ent->d_name)) continue;
    int64_t fd = strtol(ent->d_name, nullptr, 10);
    path = absl::StrCat("/proc/", pid, "/fd/", fd);
    if (!readlink(path.c_str(), &line[0], line.size())) continue;
    if (line != tpu_dev_path) continue;
    return true;
  }
  return false;
}

// This function iterates through all the processes in /proc and logs if any
// process it was able to check is using the TPU. It does not have permission to
// processes owned by another user.
// TODO (shahrokhi) use tensorflow/core/platform/filesystem (GetChildren) for
// this.
bool FindAndLogLibtpuProcess() {
  DIR* proc = opendir("/proc");

  if (proc == nullptr) {
    return false;
  }
  std::unique_ptr<DIR, int (*)(DIR*)> proc_dir(proc, closedir);
  struct dirent* ent;
  int64_t pid;
  while ((ent = readdir(proc))) {
    if (!isdigit(*ent->d_name)) continue;

    pid = strtol(ent->d_name, nullptr, 10);
    if (IsTpuUsed(pid)) {
      LOG(INFO) << "libtpu.so is already in use by process with pid " << pid
                << ". Not attempting to load libtpu.so in this process.";
      return true;
    }
  }
  return false;
}

bool TryAcquireTpuLock() {
  static absl::Mutex* mu = new absl::Mutex();
  absl::MutexLock l(mu);

  static bool attempted_file_open = false;
  static bool should_load_library = false;

  if (!attempted_file_open) {
    std::string load_library_override =
        absl::StrCat(getenv("TPU_LOAD_LIBRARY"));

    if (load_library_override == "1") {
      return true;
    } else if (load_library_override == "0") {
      return false;
    }
    should_load_library = true;

    // If TPU_CHIPS_PER_PROCESS_BOUNDS doesn't include all chips, we assume
    // we're using different chips in different processes and thus multiple
    // libtpu loads are ok.
    // TODO(skyewm): we could make per-chip lock files and look at
    // TPU_VISIBLE_DEVICES if we wanted to make this really precise.
    std::string chips_per_process_bounds =
        GetEnvVar("TPU_CHIPS_PER_PROCESS_BOUNDS");
    bool allow_multiple_libtpu_load =
        GetEnvBool("ALLOW_MULTIPLE_LIBTPU_LOAD", false);
    // TODO(skyewm): remove this when TPU_CHIPS_PER_HOST_BOUNDS is fully
    // deprecated
    if (chips_per_process_bounds.empty()) {
      chips_per_process_bounds = GetEnvVar("TPU_CHIPS_PER_HOST_BOUNDS");
    }
    if ((chips_per_process_bounds.empty() ||
         chips_per_process_bounds == "2,2,1") &&
        !allow_multiple_libtpu_load) {
      int fd = open("/tmp/libtpu_lockfile", O_CREAT | O_RDWR, 0644);

      // This lock is held until the process exits intentionally. The underlying
      // TPU device will be held on until it quits.
      if (lockf(fd, F_TLOCK, 0) != 0) {
        if (!FindAndLogLibtpuProcess()) {
          LOG(INFO) << "libtpu.so already in use by another process probably"
                       " owned by another user. "
                       "Run \"$ sudo lsof -w /dev/accel0\" to figure out "
                       "which process is using the TPU. Not "
                       "attempting to load libtpu.so in this process.";
        }
        should_load_library = false;
      } else {
        should_load_library = true;
      }
    } else {
      VLOG(1) << "TPU_CHIPS_PER_PROCESS_BOUNDS is not empty or "
                 "ALLOW_MULTIPLE_LIBTPU_LOAD is set to True, "
                 "therefore allowing multiple libtpu.so loads.";
      should_load_library = true;
    }
  }

  return should_load_library;
}

std::pair<std::vector<std::string>, std::vector<const char*>>
GetLibTpuInitArguments() {
  // We make copies of the arguments returned by getenv because the memory
  // returned may be altered or invalidated by further calls to getenv.
  std::vector<std::string> args;
  std::vector<const char*> arg_ptrs;

  // Retrieve arguments from environment if applicable.
  char* env = getenv("LIBTPU_INIT_ARGS");
  if (env != nullptr) {
    // TODO(frankchn): Handles quotes properly if necessary.
    args = absl::StrSplit(env, ' ');
  }

  arg_ptrs.reserve(args.size());
  for (int i = 0; i < args.size(); ++i) {
    arg_ptrs.push_back(args[i].data());
  }

  return {std::move(args), std::move(arg_ptrs)};
}

}  // namespace tpu
}  // namespace tensorflow
<commit_msg>Remove unused `<filesystem>` header include.<commit_after>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/tpu/tpu_initializer_helper.h"

#include <dirent.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>

#include <fstream>

#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow/core/platform/logging.h"

namespace tensorflow {
namespace tpu {
namespace {

static std::string GetEnvVar(const char* name) {
  // Constructing a std::string directly from nullptr is undefined behavior.
  return absl::StrCat(getenv(name));
}

bool GetEnvBool(const char* name, bool defval) {
  const char* env = getenv(name);
  if (env == nullptr) {
    return defval;
  }
  if (std::strcmp(env, "true") == 0) {
    return true;
  }
  if (std::strcmp(env, "false") == 0) {
    return false;
  }
  int int_env;
  bool has_int = absl::SimpleAtoi(env, &int_env);
  return has_int && int_env != 0;
}

}  // namespace

// This function gets pid of a process and checks if that process is using tpu.
// It is not able to check processes that are owned by another user.
bool IsTpuUsed(int64_t pid) {
  std::string path = absl::StrCat("/proc/", pid, "/fd");
  DIR* raw_fd_dir = opendir(path.c_str());
  if (!raw_fd_dir) {
    return false;
  }
  std::unique_ptr<DIR, int (*)(DIR*)> fd_dir(raw_fd_dir, closedir);
  struct dirent* ent;
  std::string line;
  std::string tpu_dev_path = "/dev/accel0";
  line.resize(tpu_dev_path.size());
  while ((ent = readdir(raw_fd_dir))) {
    if (!isdigit(*ent->d_name)) continue;
    int64_t fd = strtol(ent->d_name, nullptr, 10);
    path = absl::StrCat("/proc/", pid, "/fd/", fd);
    if (!readlink(path.c_str(), &line[0], line.size())) continue;
    if (line != tpu_dev_path) continue;
    return true;
  }
  return false;
}

// This function iterates through all the processes in /proc and logs if any
// process it was able to check is using the TPU. It does not have permission to
// processes owned by another user.
// TODO (shahrokhi) use tensorflow/core/platform/filesystem (GetChildren) for
// this.
bool FindAndLogLibtpuProcess() {
  DIR* proc = opendir("/proc");

  if (proc == nullptr) {
    return false;
  }
  std::unique_ptr<DIR, int (*)(DIR*)> proc_dir(proc, closedir);
  struct dirent* ent;
  int64_t pid;
  while ((ent = readdir(proc))) {
    if (!isdigit(*ent->d_name)) continue;

    pid = strtol(ent->d_name, nullptr, 10);
    if (IsTpuUsed(pid)) {
      LOG(INFO) << "libtpu.so is already in use by process with pid " << pid
                << ". Not attempting to load libtpu.so in this process.";
      return true;
    }
  }
  return false;
}

bool TryAcquireTpuLock() {
  static absl::Mutex* mu = new absl::Mutex();
  absl::MutexLock l(mu);

  static bool attempted_file_open = false;
  static bool should_load_library = false;

  if (!attempted_file_open) {
    std::string load_library_override =
        absl::StrCat(getenv("TPU_LOAD_LIBRARY"));

    if (load_library_override == "1") {
      return true;
    } else if (load_library_override == "0") {
      return false;
    }
    should_load_library = true;

    // If TPU_CHIPS_PER_PROCESS_BOUNDS doesn't include all chips, we assume
    // we're using different chips in different processes and thus multiple
    // libtpu loads are ok.
    // TODO(skyewm): we could make per-chip lock files and look at
    // TPU_VISIBLE_DEVICES if we wanted to make this really precise.
    std::string chips_per_process_bounds =
        GetEnvVar("TPU_CHIPS_PER_PROCESS_BOUNDS");
    bool allow_multiple_libtpu_load =
        GetEnvBool("ALLOW_MULTIPLE_LIBTPU_LOAD", false);
    // TODO(skyewm): remove this when TPU_CHIPS_PER_HOST_BOUNDS is fully
    // deprecated
    if (chips_per_process_bounds.empty()) {
      chips_per_process_bounds = GetEnvVar("TPU_CHIPS_PER_HOST_BOUNDS");
    }
    if ((chips_per_process_bounds.empty() ||
         chips_per_process_bounds == "2,2,1") &&
        !allow_multiple_libtpu_load) {
      int fd = open("/tmp/libtpu_lockfile", O_CREAT | O_RDWR, 0644);

      // This lock is held until the process exits intentionally. The underlying
      // TPU device will be held on until it quits.
      if (lockf(fd, F_TLOCK, 0) != 0) {
        if (!FindAndLogLibtpuProcess()) {
          LOG(INFO) << "libtpu.so already in use by another process probably"
                       " owned by another user. "
                       "Run \"$ sudo lsof -w /dev/accel0\" to figure out "
                       "which process is using the TPU. Not "
                       "attempting to load libtpu.so in this process.";
        }
        should_load_library = false;
      } else {
        should_load_library = true;
      }
    } else {
      VLOG(1) << "TPU_CHIPS_PER_PROCESS_BOUNDS is not empty or "
                 "ALLOW_MULTIPLE_LIBTPU_LOAD is set to True, "
                 "therefore allowing multiple libtpu.so loads.";
      should_load_library = true;
    }
  }

  return should_load_library;
}

std::pair<std::vector<std::string>, std::vector<const char*>>
GetLibTpuInitArguments() {
  // We make copies of the arguments returned by getenv because the memory
  // returned may be altered or invalidated by further calls to getenv.
  std::vector<std::string> args;
  std::vector<const char*> arg_ptrs;

  // Retrieve arguments from environment if applicable.
  char* env = getenv("LIBTPU_INIT_ARGS");
  if (env != nullptr) {
    // TODO(frankchn): Handles quotes properly if necessary.
    args = absl::StrSplit(env, ' ');
  }

  arg_ptrs.reserve(args.size());
  for (int i = 0; i < args.size(); ++i) {
    arg_ptrs.push_back(args[i].data());
  }

  return {std::move(args), std::move(arg_ptrs)};
}

}  // namespace tpu
}  // namespace tensorflow
<|endoftext|>


<commit_before>#ifndef MJOLNIR_READ_SPATIAL_PARTITION
#define MJOLNIR_READ_SPATIAL_PARTITION
#include <mjolnir/core/UnlimitedGridCellList.hpp>
#include <mjolnir/core/PeriodicGridCellList.hpp>
#include <mjolnir/core/NaivePairCalculation.hpp>
#include <mjolnir/core/VerletList.hpp>
#include <mjolnir/util/make_unique.hpp>

namespace mjolnir
{

template<typename boundaryT, typename traitsT>
struct celllist_dispatcher;

template<typename realT, typename coordT, typename traitsT>
struct celllist_dispatcher<UnlimitedBoundary<realT, coordT>, traitsT>
{
    typedef UnlimitedGridCellList<traitsT> type;
    typedef realT real_type;
    static UnlimitedGridCellList<traitsT>
    invoke(const real_type cutoff, const real_type mergin)
    {
        return UnlimitedGridCellList<traitsT>{cutoff, mergin};
    }
};

template<typename realT, typename coordT, typename traitsT>
struct celllist_dispatcher<CubicPeriodicBoundary<realT, coordT>, traitsT>
{
    typedef PeriodicGridCellList<traitsT> type;
    typedef realT real_type;
    static PeriodicGridCellList<traitsT>
    invoke(const real_type cutoff, const real_type mergin)
    {
        return PeriodicGridCellList<traitsT>{cutoff, mergin};
    }
};

template<typename partitionT>
partitionT
read_exception_information(const toml::Table& global, partitionT&& sp)
{
    const auto& params = global.at("parameters").cast<toml::value_t::Array>();
    for(const auto& tab : params)
    {
        const auto& info = tab.cast<toml::value_t::Table>();
        const auto  idx = toml::get<std::size_t>(info.at("index"));
        sp.chain_index(idx) = toml::get<std::size_t>(info.at("chain"));
        for(auto exc : toml::get<std::vector<std::size_t>>(
                    info.at("except_chains")))
        {
            sp.except_chains(idx).push_back(exc);
        }
        for(auto exb : toml::get<std::vector<std::size_t>>(
                    info.at("except_beads")))
        {
            sp.except_indices(idx).push_back(exb);
        }
    }
    return sp;
}

template<typename traitsT, typename potentialT>
std::unique_ptr<GlobalInteractionBase<traitsT>>
read_spatial_partition_for_distance(const toml::Table& global, potentialT&& pot)
{
    typedef typename traitsT::real_type real_type;

    const auto& sp = global.at("spatial_partition").cast<toml::value_t::Table>();
    const auto  type = toml::get<std::string>(sp.at("type"));
    if(type == "CellList")
    {
        typedef typename traitsT::boundary_type boundary_type;
        typedef typename celllist_dispatcher<boundary_type, traitsT>::type
                celllist_type;

        const auto co = pot.max_cutoff_length();
        const auto mg = toml::get<real_type>(sp.at("mergin"));
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, celllist_type>>(std::move(pot),
                celllist_dispatcher<boundary_type, traitsT>::invoke(co, mg));
    }
    else if(type == "VerletList")
    {
        const auto cutoff = pot.max_cutoff_length();
        const auto mergin = toml::get<real_type>(sp.at("mergin"));
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, VerletList<traitsT>>
            >(std::move(pot), VerletList<traitsT>{cutoff, mergin});
    }
    else if(type == "Naive")
    {
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, NaivePairCalculation<traitsT>>
            >(std::move(pot), NaivePairCalculation<traitsT>{});
    }
    else
    {
        throw std::runtime_error("invalid spatial partition type: " + type);
    }
}

}
#endif// MJOLNIR_READ_SPATIAL_PARTITION
<commit_msg>add read spatial partition for externalMU<commit_after>#ifndef MJOLNIR_READ_SPATIAL_PARTITION
#define MJOLNIR_READ_SPATIAL_PARTITION
#include <mjolnir/core/UnlimitedGridCellList.hpp>
#include <mjolnir/core/PeriodicGridCellList.hpp>
#include <mjolnir/core/NaivePairCalculation.hpp>
#include <mjolnir/core/VerletList.hpp>
#include <mjolnir/util/make_unique.hpp>

namespace mjolnir
{

template<typename boundaryT, typename traitsT>
struct celllist_dispatcher;

template<typename realT, typename coordT, typename traitsT>
struct celllist_dispatcher<UnlimitedBoundary<realT, coordT>, traitsT>
{
    typedef UnlimitedGridCellList<traitsT> type;
    typedef realT real_type;
    static UnlimitedGridCellList<traitsT>
    invoke(const real_type cutoff, const real_type mergin)
    {
        return UnlimitedGridCellList<traitsT>{cutoff, mergin};
    }
};

template<typename realT, typename coordT, typename traitsT>
struct celllist_dispatcher<CubicPeriodicBoundary<realT, coordT>, traitsT>
{
    typedef PeriodicGridCellList<traitsT> type;
    typedef realT real_type;
    static PeriodicGridCellList<traitsT>
    invoke(const real_type cutoff, const real_type mergin)
    {
        return PeriodicGridCellList<traitsT>{cutoff, mergin};
    }
};

template<typename partitionT>
partitionT
read_exception_information(const toml::Table& global, partitionT&& sp)
{
    const auto& params = global.at("parameters").cast<toml::value_t::Array>();
    for(const auto& tab : params)
    {
        const auto& info = tab.cast<toml::value_t::Table>();
        const auto  idx = toml::get<std::size_t>(info.at("index"));
        sp.chain_index(idx) = toml::get<std::size_t>(info.at("chain"));
        for(auto exc : toml::get<std::vector<std::size_t>>(
                    info.at("except_chains")))
        {
            sp.except_chains(idx).push_back(exc);
        }
        for(auto exb : toml::get<std::vector<std::size_t>>(
                    info.at("except_beads")))
        {
            sp.except_indices(idx).push_back(exb);
        }
    }
    return sp;
}

template<typename traitsT, typename potentialT>
std::unique_ptr<GlobalInteractionBase<traitsT>>
read_spatial_partition_for_distance(const toml::Table& global, potentialT&& pot)
{
    typedef typename traitsT::real_type real_type;

    const auto& sp = global.at("spatial_partition").cast<toml::value_t::Table>();
    const auto  type = toml::get<std::string>(sp.at("type"));
    if(type == "CellList")
    {
        typedef typename traitsT::boundary_type boundary_type;
        typedef typename celllist_dispatcher<boundary_type, traitsT>::type
                celllist_type;

        const auto co = pot.max_cutoff_length();
        const auto mg = toml::get<real_type>(sp.at("mergin"));
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, celllist_type>>(std::move(pot),
                celllist_dispatcher<boundary_type, traitsT>::invoke(co, mg));
    }
    else if(type == "VerletList")
    {
        const auto cutoff = pot.max_cutoff_length();
        const auto mergin = toml::get<real_type>(sp.at("mergin"));
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, VerletList<traitsT>>
            >(std::move(pot), VerletList<traitsT>{cutoff, mergin});
    }
    else if(type == "Naive")
    {
        return make_unique<GlobalDistanceInteraction<
            traitsT, potentialT, NaivePairCalculation<traitsT>>
            >(std::move(pot), NaivePairCalculation<traitsT>{});
    }
    else
    {
        throw std::runtime_error("invalid spatial partition type: " + type);
    }
}

template<typename traitsT, typename potentialT>
std::unique_ptr<GlobalInteractionBase<traitT>>
read_spatial_partition_for_externalMU(const toml::Table& global, potentialT&& pot)
{
    return make_unique<GlobalExternalInteraction<>>
}
}
#endif// MJOLNIR_READ_SPATIAL_PARTITION
<|endoftext|>


<commit_before>#ifndef __CCCALLCC_HPP__
#define __CCCALLCC_HPP__

#include <functional>
#include <unistd.h>

template <typename T>
class cont {
public:
    static T call_cc(std::function<T (cont<T>)> f) {
        int fd[2];
        pipe(fd);
        int read_fd  = fd[0];
        int write_fd = fd[1];

        pid_t pid = fork();
        if (pid) {
            // parent
            close(read_fd);
            return f( cont<T>(write_fd) );
        } else {
            // child
            close(write_fd);
            char buf[sizeof(T)];
            if (read(read_fd, buf, sizeof(T)) < sizeof(T))
                exit(0);
            close(read_fd);
            return *reinterpret_cast<T*>(buf);
        }
    }

    void operator()(const T &x) {
        write(m_pipe, &x, sizeof(T));
        exit(0);
    }

private:
    cont<T>(int pipe)
        : m_pipe(pipe) { }

    int m_pipe;
};

template <typename T>
static inline T call_cc(std::function<T (cont<T>)> f) {
    return cont<T>::call_cc(f);
}

#endif // !defined(__CCALLCC_HPP__)
<commit_msg>move cont<T>::call_cc definition out of class body<commit_after>#ifndef __CCCALLCC_HPP__
#define __CCCALLCC_HPP__

#include <functional>
#include <unistd.h>

template <typename T>
class cont {
public:
    static T call_cc(std::function<T (cont<T>)> f);

    void operator()(const T &x) {
        write(m_pipe, &x, sizeof(T));
        exit(0);
    }

private:
    cont<T>(int pipe)
        : m_pipe(pipe) { }

    int m_pipe;
};

template <typename T>
T cont<T>::call_cc(std::function<T (cont<T>)> f) {
    int fd[2];
    pipe(fd);
    int read_fd  = fd[0];
    int write_fd = fd[1];

    pid_t pid = fork();
    if (pid) {
        // parent
        close(read_fd);
        return f( cont<T>(write_fd) );
    } else {
        // child
        close(write_fd);
        char buf[sizeof(T)];
        if (read(read_fd, buf, sizeof(T)) < sizeof(T))
            exit(0);
        close(read_fd);
        return *reinterpret_cast<T*>(buf);
    }
}

template <typename T>
static inline T call_cc(std::function<T (cont<T>)> f) {
    return cont<T>::call_cc(f);
}

#endif // !defined(__CCALLCC_HPP__)
<|endoftext|>