<commit_before>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file stochastic_gradient_descent.hpp
 * \brief Stochastic Gradient Descent (SGD) Implementation for neural networks
 *
 * This implementations supports fully-connected layers, convolutional layers,
 * RBM layers, CRBM layers, transform layers and pooling layers.
 */

#pragma once

#include "cpp_utils/static_if.hpp"
#include "cpp_utils/tuple_utils.hpp"

#include "dll/util/checks.hpp"         // For NaN checks
#include "dll/util/timers.hpp"         // For auto_timer

namespace dll {

template<template<typename, typename, size_t> class Context, typename DBN, size_t... I>
auto build_context(DBN& dbn, std::index_sequence<I...> /*seq*/){
    return std::make_tuple
        (
            (std::make_pair(
                std::ref(dbn.template layer_get<I>()),  // Reference to the layer
                std::make_shared<Context<DBN, typename DBN::template layer_type<I>, I>>(dbn.template layer_get<I>()))
            )...
        );
}

template<template<typename, typename, size_t> class Context, typename DBN>
auto build_context(DBN& dbn){
    return build_context<Context>(dbn, std::make_index_sequence<DBN::layers>());
}

template <typename DBN>
struct sgd_trainer {
    using dbn_t     = DBN;
    using weight    = typename dbn_t::weight;
    using this_type = sgd_trainer<dbn_t>;

    static constexpr auto layers     = dbn_t::layers;
    static constexpr auto batch_size = dbn_t::batch_size;

    dbn_t& dbn;
    decltype(build_context<sgd_context>(dbn)) full_context;

    // Transform layers need to inherit dimensions from back

    template<typename L1, typename L2, cpp_enable_if(decay_layer_traits<typename L2::first_type>::is_transform_layer())>
    static void inherit_from_front(L1& l1, L2& l2){
        auto& ctx1 = *l1.second;
        auto& ctx2 = *l2.second;

        if (ctx2.errors.size() == 0) {
            ctx2.output = ctx1.output;
            ctx2.errors = ctx1.output;
            ctx2.input  = ctx1.output;
        }
    }

    template<typename L1, typename L2, cpp_disable_if(decay_layer_traits<typename L2::first_type>::is_transform_layer())>
    static void inherit_from_front(L1& /*l1*/, L2& /*l2*/){ }

    explicit sgd_trainer(dbn_t& dbn) : dbn(dbn), full_context(build_context<sgd_context>(dbn)) {
        // Inherit dimensions from front to end (for transform layers)

        cpp::for_each_pair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
            constexpr bool l2_transform = decay_layer_traits<decltype(layer_ctx_2.first)>::is_transform_layer();

            if (l2_transform) {
                this_type::inherit_from_front(layer_ctx_1, layer_ctx_2);
            }
        });
    }

    void init_training(size_t) {}

    template <typename D, typename It>
    void copy_inputs(D& dest, It first, It last) {
        size_t i = 0;

        while (first != last) {
            dest(i++) = *first++;
        }
    }

    template <typename D, typename It, cpp_enable_if(etl::decay_traits<D>::dimensions() == 2)>
    void copy_labels(D& dest, It first, It last) {
        //TODO How does that work in auto encoder mode ?

        size_t i = 0;

        while (first != last) {
            for (size_t l = 0; l < etl::dim<1>(dest); ++l) {
                dest(i, l) = (*first)[l];
            }
            ++i;
            ++first;
        }
    }

    template <typename D, typename It, cpp_enable_if(etl::decay_traits<D>::dimensions() == 4)>
    void copy_labels(D& dest, It first, It last) {
        //TODO How does that work in auto encoder mode ?

        size_t i = 0;

        while (first != last) {
            dest(i++) = *first++;
        }
    }

    // TODO: There are way too many copies going in this function

    template <typename Inputs, typename Labels, typename InputTransformer>
    std::pair<double, double> train_batch(size_t epoch, const Inputs& inputs, const Labels& labels, InputTransformer input_transformer) {
        //Copy inputs into suitable data structure

        auto tilde_inputs = force_temporary(inputs);
        for(size_t i = 0; i < etl::dim<0>(tilde_inputs); ++i){
            input_transformer(tilde_inputs(i));
        }

        return train_batch(epoch, tilde_inputs, labels);
    }

    template <typename Inputs, typename Labels>
    std::pair<double, double> train_batch(size_t /*epoch*/, const Inputs& inputs, const Labels& labels) {
        dll::auto_timer timer("sgd::train_batch");

        // Ensure that the data batch and the label batch are of the same size
        cpp_assert(etl::dim<0>(inputs) == etl::dim<0>(labels), "Invalid sizes");

        const auto n = etl::dim<0>(inputs);

        auto& first_layer = std::get<0>(full_context).first;
        auto& first_ctx   = *std::get<0>(full_context).second;

        auto& last_ctx   = *std::get<layers - 1>(full_context).second;

        const bool full_batch = etl::dim<0>(inputs) == etl::dim<0>(first_ctx.input);

        //Feedforward pass

        {
            dll::auto_timer timer("sgd::forward");

            if(cpp_unlikely(!full_batch)){
                first_ctx.input  = 0;
                first_ctx.output = 0;

                for (size_t i = 0; i < etl::dim<0>(inputs); ++i) {
                    first_ctx.input(i) = inputs(i);
                }
            } else {
                first_ctx.input = inputs;
            }

            first_layer.batch_activate_hidden(first_ctx.output, first_ctx.input);

            cpp::for_each_pair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
                auto& layer_2 = layer_ctx_2.first;

                auto& ctx1 = *layer_ctx_1.second;
                auto& ctx2 = *layer_ctx_2.second;

                ctx2.input = ctx1.output;
                layer_2.batch_activate_hidden(ctx2.output, ctx2.input);
            });
        }

        {
            dll::auto_timer timer("sgd::backward");

            //Compute the errors of the last layer

            // Note: This only compute the derivative of the error function
            // The derivative of the activation function for the last layer
            // will be computed in the next step in adapt errors

            // TODO: The computation is not totally since it should in
            // fact depends on the activation function of the last layer

            // TODO: The MSE and CCE gradients should not be the same

            if /*constexpr*/ (dbn_t::loss == loss_function::CATEGORICAL_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = labels(i) - last_ctx.output(i);
                    }
                } else {
                    last_ctx.errors = labels - last_ctx.output;
                }
            } else if (dbn_t::loss == loss_function::MEAN_SQUARED_ERROR){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = labels(i) - last_ctx.output(i);
                    }
                } else {
                    last_ctx.errors = labels - last_ctx.output;
                }
            } else if (dbn_t::loss == loss_function::BINARY_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    auto& out = last_ctx.output;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = (labels(i) - out(i)) / ((1.0 - out(i)) >> out(i));
                    }
                } else {
                    auto& out = last_ctx.output;

                    last_ctx.errors = (labels - out) / ((1.0 - out) >> out);
                }
            } else {
                cpp_unreachable("Unsupported loss function");
            }

            // Backpropagate the error

            cpp::for_each_rpair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
                auto& r2 = layer_ctx_2.first;

                auto& ctx1 = *layer_ctx_1.second;
                auto& ctx2 = *layer_ctx_2.second;

                r2.adapt_errors(ctx2);
                r2.backward_batch(ctx1.errors, ctx2);
            });

            first_layer.adapt_errors(first_ctx);
        }

        // Compute and apply the gradients

        {
            dll::auto_timer timer("sgd::grad");

            cpp::for_each(full_context, [this, n](auto& layer_ctx) {
                // Compute the gradients
                layer_ctx.first.compute_gradients(*layer_ctx.second);

                // Apply the gradients
                this->apply_gradients(layer_ctx.first, *layer_ctx.second, n);
            });
        }

        // Compute error and loss

        double error = 0.0;
        double loss = 0.0;

        {
            dll::auto_timer timer("sgd::error");

            auto& out = last_ctx.output;

            if /*constexpr*/ (dbn_t::loss == loss_function::CATEGORICAL_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = (-1.0 / n) * sum(log(sout) >> labels);
                } else {
                    loss = (-1.0 / n) * sum(log(out) >> labels);
                }
            } else if (dbn_t::loss == loss_function::MEAN_SQUARED_ERROR){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = (1.0 / 2.0 * n) * sum((sout - labels) >> (sout - labels));
                } else {
                    loss = (1.0 / 2.0 * n) * sum((out - labels) >> (out - labels));
                }
            } else if (dbn_t::loss == loss_function::BINARY_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = (-1.0 / n) * sum((labels >> log(sout)) + ((1.0 - labels) >> log(1.0 - sout)));
                } else {
                    loss = (-1.0 / n) * sum((labels >> log(out)) + ((1.0 - labels) >> log(1.0 - out)));
                }
            } else {
                cpp_unreachable("Unsupported loss function");
            }

            // Compute the error

            if (cpp_unlikely(!full_batch)) {
                error = amean(labels - slice(out, 0, n));
            } else {
                error = amean(labels - out);
            }
        }

        return std::make_pair(error, loss);
    }

    template <typename L, typename C, cpp_enable_if(decay_layer_traits<L>::is_neural_layer())>
    void apply_gradients(L& layer, C& context, size_t n) {
        dll::auto_timer timer("sgd::apply_grad");

        //Update the gradients
        this->update_grad(layer.w, context.w_grad, w_decay(dbn_traits<dbn_t>::decay()), 0.0);
        this->update_grad(layer.b, context.b_grad, b_decay(dbn_traits<dbn_t>::decay()), 0.0);

        //Update with momentum and learning rate
        if (dbn_traits<dbn_t>::has_momentum()) {
            auto momentum = dbn.momentum;
            auto eps      = dbn.learning_rate;

            // Note(perf): Some performance could be gained by doing the pair of
            // operations on w in a loop to improve data locality

            context.w_inc = momentum * context.w_inc + (eps / n) * context.w_grad;
            layer.w += context.w_inc;

            context.b_inc = momentum * context.b_inc + (eps / n) * context.b_grad;
            layer.b += context.b_inc;
        } else {
            auto eps = dbn.learning_rate;

            layer.w += (eps / n) * context.w_grad;
            layer.b += (eps / n) * context.b_grad;
        }

        nan_check_deep(layer.w);
        nan_check_deep(layer.b);
    }

    template <typename L, typename C, cpp_disable_if(decay_layer_traits<L>::is_neural_layer())>
    void apply_gradients(L& /*layer*/, C& /*context*/, size_t /*n*/) {
        //Pooling and transform layers have no weights, therefore no
        //gradients
    }

    template <typename V, typename G>
    void update_grad(const V& value, G& grad, decay_type decay, double penalty) {
        if (decay == decay_type::L1) {
            grad = grad - dbn.l1_weight_cost * abs(value) - penalty;
        } else if (decay == decay_type::L2) {
            grad = grad - dbn.l2_weight_cost * value - penalty;
        } else if (decay == decay_type::L1L2) {
            grad = grad - dbn.l1_weight_cost * abs(value) - dbn.l2_weight_cost * value - penalty;
        } else {
            if(penalty != 0.0){
                grad = grad - penalty;
            }
        }
    }

    static std::string name() {
        return "Stochastic Gradient Descent";
    }
};

} //end of dll namespace
<commit_msg>Average over the number of output units<commit_after>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file stochastic_gradient_descent.hpp
 * \brief Stochastic Gradient Descent (SGD) Implementation for neural networks
 *
 * This implementations supports fully-connected layers, convolutional layers,
 * RBM layers, CRBM layers, transform layers and pooling layers.
 */

#pragma once

#include "cpp_utils/static_if.hpp"
#include "cpp_utils/tuple_utils.hpp"

#include "dll/util/checks.hpp"         // For NaN checks
#include "dll/util/timers.hpp"         // For auto_timer

namespace dll {

template<template<typename, typename, size_t> class Context, typename DBN, size_t... I>
auto build_context(DBN& dbn, std::index_sequence<I...> /*seq*/){
    return std::make_tuple
        (
            (std::make_pair(
                std::ref(dbn.template layer_get<I>()),  // Reference to the layer
                std::make_shared<Context<DBN, typename DBN::template layer_type<I>, I>>(dbn.template layer_get<I>()))
            )...
        );
}

template<template<typename, typename, size_t> class Context, typename DBN>
auto build_context(DBN& dbn){
    return build_context<Context>(dbn, std::make_index_sequence<DBN::layers>());
}

template <typename DBN>
struct sgd_trainer {
    using dbn_t     = DBN;
    using weight    = typename dbn_t::weight;
    using this_type = sgd_trainer<dbn_t>;

    static constexpr auto layers     = dbn_t::layers;
    static constexpr auto batch_size = dbn_t::batch_size;

    dbn_t& dbn;
    decltype(build_context<sgd_context>(dbn)) full_context;

    // Transform layers need to inherit dimensions from back

    template<typename L1, typename L2, cpp_enable_if(decay_layer_traits<typename L2::first_type>::is_transform_layer())>
    static void inherit_from_front(L1& l1, L2& l2){
        auto& ctx1 = *l1.second;
        auto& ctx2 = *l2.second;

        if (ctx2.errors.size() == 0) {
            ctx2.output = ctx1.output;
            ctx2.errors = ctx1.output;
            ctx2.input  = ctx1.output;
        }
    }

    template<typename L1, typename L2, cpp_disable_if(decay_layer_traits<typename L2::first_type>::is_transform_layer())>
    static void inherit_from_front(L1& /*l1*/, L2& /*l2*/){ }

    explicit sgd_trainer(dbn_t& dbn) : dbn(dbn), full_context(build_context<sgd_context>(dbn)) {
        // Inherit dimensions from front to end (for transform layers)

        cpp::for_each_pair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
            constexpr bool l2_transform = decay_layer_traits<decltype(layer_ctx_2.first)>::is_transform_layer();

            if (l2_transform) {
                this_type::inherit_from_front(layer_ctx_1, layer_ctx_2);
            }
        });
    }

    void init_training(size_t) {}

    template <typename D, typename It>
    void copy_inputs(D& dest, It first, It last) {
        size_t i = 0;

        while (first != last) {
            dest(i++) = *first++;
        }
    }

    template <typename D, typename It, cpp_enable_if(etl::decay_traits<D>::dimensions() == 2)>
    void copy_labels(D& dest, It first, It last) {
        //TODO How does that work in auto encoder mode ?

        size_t i = 0;

        while (first != last) {
            for (size_t l = 0; l < etl::dim<1>(dest); ++l) {
                dest(i, l) = (*first)[l];
            }
            ++i;
            ++first;
        }
    }

    template <typename D, typename It, cpp_enable_if(etl::decay_traits<D>::dimensions() == 4)>
    void copy_labels(D& dest, It first, It last) {
        //TODO How does that work in auto encoder mode ?

        size_t i = 0;

        while (first != last) {
            dest(i++) = *first++;
        }
    }

    // TODO: There are way too many copies going in this function

    template <typename Inputs, typename Labels, typename InputTransformer>
    std::pair<double, double> train_batch(size_t epoch, const Inputs& inputs, const Labels& labels, InputTransformer input_transformer) {
        //Copy inputs into suitable data structure

        auto tilde_inputs = force_temporary(inputs);
        for(size_t i = 0; i < etl::dim<0>(tilde_inputs); ++i){
            input_transformer(tilde_inputs(i));
        }

        return train_batch(epoch, tilde_inputs, labels);
    }

    template <typename Inputs, typename Labels>
    std::pair<double, double> train_batch(size_t /*epoch*/, const Inputs& inputs, const Labels& labels) {
        dll::auto_timer timer("sgd::train_batch");

        // Ensure that the data batch and the label batch are of the same size
        cpp_assert(etl::dim<0>(inputs) == etl::dim<0>(labels), "Invalid sizes");

        const auto n = etl::dim<0>(inputs);

        auto& first_layer = std::get<0>(full_context).first;
        auto& first_ctx   = *std::get<0>(full_context).second;

        auto& last_ctx   = *std::get<layers - 1>(full_context).second;

        const bool full_batch = etl::dim<0>(inputs) == etl::dim<0>(first_ctx.input);

        //Feedforward pass

        {
            dll::auto_timer timer("sgd::forward");

            if(cpp_unlikely(!full_batch)){
                first_ctx.input  = 0;
                first_ctx.output = 0;

                for (size_t i = 0; i < etl::dim<0>(inputs); ++i) {
                    first_ctx.input(i) = inputs(i);
                }
            } else {
                first_ctx.input = inputs;
            }

            first_layer.batch_activate_hidden(first_ctx.output, first_ctx.input);

            cpp::for_each_pair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
                auto& layer_2 = layer_ctx_2.first;

                auto& ctx1 = *layer_ctx_1.second;
                auto& ctx2 = *layer_ctx_2.second;

                ctx2.input = ctx1.output;
                layer_2.batch_activate_hidden(ctx2.output, ctx2.input);
            });
        }

        {
            dll::auto_timer timer("sgd::backward");

            //Compute the errors of the last layer

            // Note: This only compute the derivative of the error function
            // The derivative of the activation function for the last layer
            // will be computed in the next step in adapt errors

            // TODO: The computation is not totally since it should in
            // fact depends on the activation function of the last layer

            // TODO: The MSE and CCE gradients should not be the same

            if /*constexpr*/ (dbn_t::loss == loss_function::CATEGORICAL_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = labels(i) - last_ctx.output(i);
                    }
                } else {
                    last_ctx.errors = labels - last_ctx.output;
                }
            } else if (dbn_t::loss == loss_function::MEAN_SQUARED_ERROR){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = labels(i) - last_ctx.output(i);
                    }
                } else {
                    last_ctx.errors = labels - last_ctx.output;
                }
            } else if (dbn_t::loss == loss_function::BINARY_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    last_ctx.errors = 0;

                    auto& out = last_ctx.output;

                    for (size_t i = 0; i < n; ++i) {
                        last_ctx.errors(i) = (labels(i) - out(i)) / ((1.0 - out(i)) >> out(i));
                    }
                } else {
                    auto& out = last_ctx.output;

                    last_ctx.errors = (labels - out) / ((1.0 - out) >> out);
                }
            } else {
                cpp_unreachable("Unsupported loss function");
            }

            // Backpropagate the error

            cpp::for_each_rpair(full_context, [](auto& layer_ctx_1, auto& layer_ctx_2) {
                auto& r2 = layer_ctx_2.first;

                auto& ctx1 = *layer_ctx_1.second;
                auto& ctx2 = *layer_ctx_2.second;

                r2.adapt_errors(ctx2);
                r2.backward_batch(ctx1.errors, ctx2);
            });

            first_layer.adapt_errors(first_ctx);
        }

        // Compute and apply the gradients

        {
            dll::auto_timer timer("sgd::grad");

            cpp::for_each(full_context, [this, n](auto& layer_ctx) {
                // Compute the gradients
                layer_ctx.first.compute_gradients(*layer_ctx.second);

                // Apply the gradients
                this->apply_gradients(layer_ctx.first, *layer_ctx.second, n);
            });
        }

        // Compute error and loss

        double error = 0.0;
        double loss = 0.0;

        {
            dll::auto_timer timer("sgd::error");

            auto& out = last_ctx.output;

            if /*constexpr*/ (dbn_t::loss == loss_function::CATEGORICAL_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = (-1.0 / n) * sum(log(sout) >> labels);
                } else {
                    loss = (-1.0 / n) * sum(log(out) >> labels);
                }
            } else if (dbn_t::loss == loss_function::MEAN_SQUARED_ERROR){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = (1.0 / 2.0 * n) * sum((sout - labels) >> (sout - labels));
                } else {
                    loss = (1.0 / 2.0 * n) * sum((out - labels) >> (out - labels));
                }
            } else if (dbn_t::loss == loss_function::BINARY_CROSS_ENTROPY){
                if (cpp_unlikely(!full_batch)) {
                    auto sout = slice(out, 0, n);

                    loss = -1.0 * mean((labels >> log(sout)) + ((1.0 - labels) >> log(1.0 - sout)));
                } else {
                    loss = -1.0 * mean((labels >> log(out)) + ((1.0 - labels) >> log(1.0 - out)));
                }
            } else {
                cpp_unreachable("Unsupported loss function");
            }

            // Compute the error

            if (cpp_unlikely(!full_batch)) {
                error = amean(labels - slice(out, 0, n));
            } else {
                error = amean(labels - out);
            }
        }

        return std::make_pair(error, loss);
    }

    template <typename L, typename C, cpp_enable_if(decay_layer_traits<L>::is_neural_layer())>
    void apply_gradients(L& layer, C& context, size_t n) {
        dll::auto_timer timer("sgd::apply_grad");

        //Update the gradients
        this->update_grad(layer.w, context.w_grad, w_decay(dbn_traits<dbn_t>::decay()), 0.0);
        this->update_grad(layer.b, context.b_grad, b_decay(dbn_traits<dbn_t>::decay()), 0.0);

        //Update with momentum and learning rate
        if (dbn_traits<dbn_t>::has_momentum()) {
            auto momentum = dbn.momentum;
            auto eps      = dbn.learning_rate;

            // Note(perf): Some performance could be gained by doing the pair of
            // operations on w in a loop to improve data locality

            context.w_inc = momentum * context.w_inc + (eps / n) * context.w_grad;
            layer.w += context.w_inc;

            context.b_inc = momentum * context.b_inc + (eps / n) * context.b_grad;
            layer.b += context.b_inc;
        } else {
            auto eps = dbn.learning_rate;

            layer.w += (eps / n) * context.w_grad;
            layer.b += (eps / n) * context.b_grad;
        }

        nan_check_deep(layer.w);
        nan_check_deep(layer.b);
    }

    template <typename L, typename C, cpp_disable_if(decay_layer_traits<L>::is_neural_layer())>
    void apply_gradients(L& /*layer*/, C& /*context*/, size_t /*n*/) {
        //Pooling and transform layers have no weights, therefore no
        //gradients
    }

    template <typename V, typename G>
    void update_grad(const V& value, G& grad, decay_type decay, double penalty) {
        if (decay == decay_type::L1) {
            grad = grad - dbn.l1_weight_cost * abs(value) - penalty;
        } else if (decay == decay_type::L2) {
            grad = grad - dbn.l2_weight_cost * value - penalty;
        } else if (decay == decay_type::L1L2) {
            grad = grad - dbn.l1_weight_cost * abs(value) - dbn.l2_weight_cost * value - penalty;
        } else {
            if(penalty != 0.0){
                grad = grad - penalty;
            }
        }
    }

    static std::string name() {
        return "Stochastic Gradient Descent";
    }
};

} //end of dll namespace
<|endoftext|>


<commit_before>#include <aios.h>
#include <dbfe.h>
#include <parseopt.h>
#include <group.h>

ptr<dbfe> group_db, header_db;

/* Much of the structure and code here is taken from sfskey.C which is GPL2'd.
 * See http://www.fs.net/
 * See also lsd/lsdctl.C */

void udbctl_help (int argc, char *argv[]);
void udbctl_addgroup (int argc, char *argv[]);
void udbctl_listgroups (int argc, char *argv[]);
void udbctl_listheaders (int argc, char *argv[]);

struct modevec {
  const char *name;
  void (*fn) (int argc, char **argv);
  const char *usage;
};
const modevec modes[] = {
  { "help", udbctl_help, "help" },
  { "addgroup", udbctl_addgroup, "addgroup groupname" },
  { "listgroups", udbctl_listgroups, "listgroups" },
  { "listheaders", udbctl_listheaders, "listheaders" },
  { NULL, NULL, NULL }
};

static const modevec *udbctl_mode;

void
usage (void)
{
  warnx << "usage: " << progname << " [-d directory] ";
  if (udbctl_mode && udbctl_mode->usage)
    warnx << udbctl_mode->usage << "\n";
  else
    warnx << "command [args]\n";
  exit (1);
}

/**************************************/
/* The commands that do the real work */
/**************************************/

void
udbctl_help (int argc, char *argv[])
{
  strbuf msg;
  msg << "usage: " << progname << " [-d directory] command [args]\n";
  for (const modevec *mp = modes; mp->name; mp++)
    if (mp->usage)
      msg << "	 " << progname << " " << mp->usage << "\n";
  make_sync (1);
  msg.tosuio ()->output (1);
  exit (0);
}

void
udbctl_addgroup (int argc, char *argv[])
{
  if (optind + 1 != argc)
    usage ();
  char *group = argv[optind];
  bool ok = create_group (group);
  group_db->checkpoint ();
  exit (!ok); // shell is backwards
}

void
udbctl_listgroups (int argc, char *argv[])
{
  bool verbose = false;
  int ch;
  while ((ch = getopt (argc, argv, "v")) != -1)
    switch (ch) {
    case 'v':
      verbose = true;
      break;
    default:
      usage ();
      break;
    }

  /* Duplicates the function of the group list by access db directly */
  ptr<dbEnumeration> it = group_db->enumerate ();
  ptr<dbPair> d = it->nextElement ();
  while (d != NULL) {
    str groupname (d->key->value, d->key->len);
    unsigned int groupcount = 0;
    ptr<dbrec> rec = group_db->lookup (d->key);
    xdrmem x (rec->value, rec->len, XDR_DECODE);
    group_entry *group = New group_entry;
    bzero (group, sizeof (*group));
    if (xdr_group_entry (x.xdrp (), group)) {
      groupcount = group->articles.size ();
      aout << groupname << "\t" << groupcount << "\n";
      if (verbose) {
	// Dump the complete list of articles for this group
	for (size_t i = 0; i < group->articles.size (); i++)
	  aout << i << " "
	       << group->articles[i].artno << " "
	       << group->articles[i].msgid << " "
	       << group->articles[i].blkid << "\n";
      }
      aout->flush ();
      xdr_delete (reinterpret_cast<xdrproc_t> (xdr_group_entry), group);
    }
    d = it->nextElement();
  }
}

void
udbctl_listheaders (int argc, char *argv[])
{
  ptr<dbEnumeration> it = header_db->enumerate ();
  ptr<dbPair> d = it->nextElement ();
  while (d != NULL) {
    str id (d->key->value, d->key->len);
    aout << id << "\n";
    aout->flush ();
    d = it->nextElement();
  }
}

int
main (int argc, char *argv[])
{
  char *dirbase (NULL);
  setprogname (argv[0]);
  putenv ("POSIXLY_CORRECT=1"); // Prevents Linux from reordering options

  int ch;
  while ((ch = getopt (argc, argv, "d:")) != -1)
    switch (ch) {
    case 'd':
      dirbase = optarg;
      break;
    default:
      usage ();
      break;
    }
  if (optind >= argc)
    usage ();

  if (dirbase && chdir (dirbase) < 0)
    fatal << "chdir(" << dirbase << "): " << strerror (errno) << "\n";

  // Prepare to dispatch on command name
  const modevec *mp;
  for (mp = modes; mp->name; mp++)
    if (!strcmp (argv[optind], mp->name))
      break;
  if (!mp->name)
    usage ();
  udbctl_mode = mp;

  // Skip over command name...
  optind++;

  // XXX should only open the database that's needed
  dbOptions opts;
  group_db = New refcounted<dbfe> ();
  if (int err = group_db->opendb ("groups", opts)) {
    warn << "open returned: " << strerror (err) << "\n";
    exit (-1);
  }
  header_db = New refcounted<dbfe> ();
  if (int err = header_db->opendb ("headers", opts)) {
    warn << "open returned: " << strerror (err) << "\n";
    exit (-1);
  }

  mp->fn (argc, argv);

  return 0;
}
<commit_msg>Create multiple groups with a single invocation.<commit_after>#include <aios.h>
#include <dbfe.h>
#include <parseopt.h>
#include <group.h>

ptr<dbfe> group_db, header_db;

/* Much of the structure and code here is taken from sfskey.C which is GPL2'd.
 * See http://www.fs.net/
 * See also lsd/lsdctl.C */

void udbctl_help (int argc, char *argv[]);
void udbctl_addgroup (int argc, char *argv[]);
void udbctl_listgroups (int argc, char *argv[]);
void udbctl_listheaders (int argc, char *argv[]);

struct modevec {
  const char *name;
  void (*fn) (int argc, char **argv);
  const char *usage;
};
const modevec modes[] = {
  { "help", udbctl_help, "help" },
  { "addgroup", udbctl_addgroup, "addgroup groupname" },
  { "listgroups", udbctl_listgroups, "listgroups" },
  { "listheaders", udbctl_listheaders, "listheaders" },
  { NULL, NULL, NULL }
};

static const modevec *udbctl_mode;

void
usage (void)
{
  warnx << "usage: " << progname << " [-d directory] ";
  if (udbctl_mode && udbctl_mode->usage)
    warnx << udbctl_mode->usage << "\n";
  else
    warnx << "command [args]\n";
  exit (1);
}

/**************************************/
/* The commands that do the real work */
/**************************************/

void
udbctl_help (int argc, char *argv[])
{
  strbuf msg;
  msg << "usage: " << progname << " [-d directory] command [args]\n";
  for (const modevec *mp = modes; mp->name; mp++)
    if (mp->usage)
      msg << "	 " << progname << " " << mp->usage << "\n";
  make_sync (1);
  msg.tosuio ()->output (1);
  exit (0);
}

void
udbctl_addgroup (int argc, char *argv[])
{
  if (optind >= argc)
    usage ();
  char *group (NULL);
  bool ok (true);
  for (int i = optind; i < argc; i++) {
    group = argv[i];
    bool thisok = create_group (group);
    aout << group << (thisok ? " " : " not ") << "ok\n";
    ok = ok && thisok;
  }
  group_db->checkpoint ();
  exit (!ok); // shell is backwards
}

void
udbctl_listgroups (int argc, char *argv[])
{
  bool verbose = false;
  int ch;
  while ((ch = getopt (argc, argv, "v")) != -1)
    switch (ch) {
    case 'v':
      verbose = true;
      break;
    default:
      usage ();
      break;
    }

  /* Duplicates the function of the group list by access db directly */
  ptr<dbEnumeration> it = group_db->enumerate ();
  ptr<dbPair> d = it->nextElement ();
  while (d != NULL) {
    str groupname (d->key->value, d->key->len);
    unsigned int groupcount = 0;
    ptr<dbrec> rec = group_db->lookup (d->key);
    xdrmem x (rec->value, rec->len, XDR_DECODE);
    group_entry *group = New group_entry;
    bzero (group, sizeof (*group));
    if (xdr_group_entry (x.xdrp (), group)) {
      groupcount = group->articles.size ();
      aout << groupname << "\t" << groupcount << "\n";
      if (verbose) {
	// Dump the complete list of articles for this group
	for (size_t i = 0; i < group->articles.size (); i++)
	  aout << i << " "
	       << group->articles[i].artno << " "
	       << group->articles[i].msgid << " "
	       << group->articles[i].blkid << "\n";
      }
      aout->flush ();
      xdr_delete (reinterpret_cast<xdrproc_t> (xdr_group_entry), group);
    }
    d = it->nextElement();
  }
}

void
udbctl_listheaders (int argc, char *argv[])
{
  ptr<dbEnumeration> it = header_db->enumerate ();
  ptr<dbPair> d = it->nextElement ();
  while (d != NULL) {
    str id (d->key->value, d->key->len);
    aout << id << "\n";
    aout->flush ();
    d = it->nextElement();
  }
}

int
main (int argc, char *argv[])
{
  char *dirbase (NULL);
  setprogname (argv[0]);
  putenv ("POSIXLY_CORRECT=1"); // Prevents Linux from reordering options

  int ch;
  while ((ch = getopt (argc, argv, "d:")) != -1)
    switch (ch) {
    case 'd':
      dirbase = optarg;
      break;
    default:
      usage ();
      break;
    }
  if (optind >= argc)
    usage ();

  if (dirbase && chdir (dirbase) < 0)
    fatal << "chdir(" << dirbase << "): " << strerror (errno) << "\n";

  // Prepare to dispatch on command name
  const modevec *mp;
  for (mp = modes; mp->name; mp++)
    if (!strcmp (argv[optind], mp->name))
      break;
  if (!mp->name)
    usage ();
  udbctl_mode = mp;

  // Skip over command name...
  optind++;

  // XXX should only open the database that's needed
  dbOptions opts;
  group_db = New refcounted<dbfe> ();
  if (int err = group_db->opendb ("groups", opts)) {
    warn << "open returned: " << strerror (err) << "\n";
    exit (-1);
  }
  header_db = New refcounted<dbfe> ();
  if (int err = header_db->opendb ("headers", opts)) {
    warn << "open returned: " << strerror (err) << "\n";
    exit (-1);
  }

  mp->fn (argc, argv);

  return 0;
}
<|endoftext|>


<commit_before>/*

Copyright (c) 2006, Arvid Norberg & Daniel Wallin
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef TRAVERSAL_ALGORITHM_050324_HPP
#define TRAVERSAL_ALGORITHM_050324_HPP

#include <vector>
#include <set>

#include <libtorrent/kademlia/node_id.hpp>
#include <libtorrent/kademlia/routing_table.hpp>
#include <libtorrent/kademlia/logging.hpp>

#include <boost/noncopyable.hpp>
#include <boost/intrusive_ptr.hpp>
#include <boost/bind.hpp>
#include <boost/pool/pool.hpp>

namespace libtorrent { struct dht_lookup; }
namespace libtorrent { namespace dht
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_DECLARE_LOG(traversal);
#endif

class rpc_manager;
class node_impl;

// this class may not be instantiated as a stack object
class traversal_algorithm : boost::noncopyable
{
public:
	void traverse(node_id const& id, udp::endpoint addr);
	void finished(udp::endpoint const& ep);

	enum flags_t { prevent_request = 1, short_timeout = 2 };
	void failed(udp::endpoint const& ep, int flags = 0);
	virtual ~traversal_algorithm();
	boost::pool<>& allocator() const;
	void status(dht_lookup& l);

	virtual char const* name() const { return "traversal_algorithm"; }
	virtual void start();

	node_id const& target() const { return m_target; }

	void add_entry(node_id const& id, udp::endpoint addr, unsigned char flags);

	struct result
	{
		result(node_id const& id, udp::endpoint ep, unsigned char f = 0)
			: id(id), flags(f)
		{
			if (ep.address().is_v6())
			{
				flags |= ipv6_address;
				addr.v6 = ep.address().to_v6().to_bytes();
			}
			else
			{
				flags &= ~ipv6_address;
				addr.v4 = ep.address().to_v4().to_bytes();
			}
			port = ep.port();
		}

		udp::endpoint endpoint() const
		{
			if (flags & ipv6_address)
				return udp::endpoint(address_v6(addr.v6), port);
			else
				return udp::endpoint(address_v4(addr.v4), port);
		}

		node_id id;

		union addr_t
		{
			address_v4::bytes_type v4;
			address_v6::bytes_type v6;
		} addr;

		boost::uint16_t port;

		enum {
			queried = 1,
			initial = 2,
			no_id = 4,
			short_timeout = 8,
			failed = 16,
			ipv6_address = 32,
			alive = 64
		};
		unsigned char flags;
	};

	traversal_algorithm(
		node_impl& node
		, node_id target)
		: m_ref_count(0)
		, m_node(node)
		, m_target(target)
		, m_invoke_count(0)
		, m_branch_factor(3)
		, m_responses(0)
		, m_timeouts(0)
	{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(traversal) << " [" << this << "] new traversal process";
#endif
	}

protected:

	void add_requests();
	void add_router_entries();
	void init();

	virtual void done() {}
	virtual bool invoke(udp::endpoint addr) { return false; }

	friend void intrusive_ptr_add_ref(traversal_algorithm* p)
	{
		p->m_ref_count++;
	}

	friend void intrusive_ptr_release(traversal_algorithm* p)
	{
		if (--p->m_ref_count == 0)
			delete p;
	}

	int m_ref_count;

	node_impl& m_node;
	node_id m_target;
	std::vector<result> m_results;
	int m_invoke_count;
	int m_branch_factor;
	int m_responses;
	int m_timeouts;
};

} } // namespace libtorrent::dht

#endif // TRAVERSAL_ALGORITHM_050324_HPP

<commit_msg>fixed msvc warning<commit_after>/*

Copyright (c) 2006, Arvid Norberg & Daniel Wallin
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef TRAVERSAL_ALGORITHM_050324_HPP
#define TRAVERSAL_ALGORITHM_050324_HPP

#include <vector>
#include <set>

#include <libtorrent/kademlia/node_id.hpp>
#include <libtorrent/kademlia/routing_table.hpp>
#include <libtorrent/kademlia/logging.hpp>

#include <boost/noncopyable.hpp>
#include <boost/intrusive_ptr.hpp>
#include <boost/bind.hpp>
#include <boost/pool/pool.hpp>

namespace libtorrent { struct dht_lookup; }
namespace libtorrent { namespace dht
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_DECLARE_LOG(traversal);
#endif

class rpc_manager;
class node_impl;

// this class may not be instantiated as a stack object
struct traversal_algorithm : boost::noncopyable
{
	void traverse(node_id const& id, udp::endpoint addr);
	void finished(udp::endpoint const& ep);

	enum flags_t { prevent_request = 1, short_timeout = 2 };
	void failed(udp::endpoint const& ep, int flags = 0);
	virtual ~traversal_algorithm();
	boost::pool<>& allocator() const;
	void status(dht_lookup& l);

	virtual char const* name() const { return "traversal_algorithm"; }
	virtual void start();

	node_id const& target() const { return m_target; }

	void add_entry(node_id const& id, udp::endpoint addr, unsigned char flags);

	struct result
	{
		result(node_id const& id, udp::endpoint ep, unsigned char f = 0)
			: id(id), flags(f)
		{
			if (ep.address().is_v6())
			{
				flags |= ipv6_address;
				addr.v6 = ep.address().to_v6().to_bytes();
			}
			else
			{
				flags &= ~ipv6_address;
				addr.v4 = ep.address().to_v4().to_bytes();
			}
			port = ep.port();
		}

		udp::endpoint endpoint() const
		{
			if (flags & ipv6_address)
				return udp::endpoint(address_v6(addr.v6), port);
			else
				return udp::endpoint(address_v4(addr.v4), port);
		}

		node_id id;

		union addr_t
		{
			address_v4::bytes_type v4;
			address_v6::bytes_type v6;
		} addr;

		boost::uint16_t port;

		enum {
			queried = 1,
			initial = 2,
			no_id = 4,
			short_timeout = 8,
			failed = 16,
			ipv6_address = 32,
			alive = 64
		};
		unsigned char flags;
	};

	traversal_algorithm(
		node_impl& node
		, node_id target)
		: m_ref_count(0)
		, m_node(node)
		, m_target(target)
		, m_invoke_count(0)
		, m_branch_factor(3)
		, m_responses(0)
		, m_timeouts(0)
	{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(traversal) << " [" << this << "] new traversal process";
#endif
	}

protected:

	void add_requests();
	void add_router_entries();
	void init();

	virtual void done() {}
	virtual bool invoke(udp::endpoint addr) { return false; }

	friend void intrusive_ptr_add_ref(traversal_algorithm* p)
	{
		p->m_ref_count++;
	}

	friend void intrusive_ptr_release(traversal_algorithm* p)
	{
		if (--p->m_ref_count == 0)
			delete p;
	}

	int m_ref_count;

	node_impl& m_node;
	node_id m_target;
	std::vector<result> m_results;
	int m_invoke_count;
	int m_branch_factor;
	int m_responses;
	int m_timeouts;
};

} } // namespace libtorrent::dht

#endif // TRAVERSAL_ALGORITHM_050324_HPP

<|endoftext|>


<commit_before>#include "PickUpController.h"

PickUpController::PickUpController() {
    lockTarget = false;
    timeOut = false;
    nTargetsSeen = 0;
    blockYawError = 0;
    blockDist = 0;
    td = 0;

    result.pickedUp = false;
    result.cmdVel = 0;
    result.angleError = 0;
    result.fingerAngle = -1;
    result.wristAngle = -1;
    result.giveUp = false;

}

PickUpResult PickUpController::pickUpSelectedTarget(bool blockBlock) {
    //threshold distance to be from the target block before attempting pickup
    float targetDist = 0.25; //meters

    /*PickUpResult result;
  result.pickedUp = false;
  result.cmdVel = 0;
  result.angleError = 0;
  result.fingerAngle = -1;
  result.wristAngle = -1;
  result.pickedUp = false;
  result.giveUp = false;*/

    // millisecond time = current time if not in a counting state
    if (!timeOut) millTimer = ros::Time::now();

    //diffrence between current time and millisecond time
    ros::Duration Tdiff = ros::Time::now() - millTimer;
    float Td = Tdiff.sec + Tdiff.nsec/1000000000.0;
    td = Td;

    if (nTargetsSeen == 0 && !lockTarget) //if not targets detected and a target has not been locked in
    {
        if(!timeOut) //if not in a counting state
        {
            result.cmdVel = 0.0;
            result.angleError = 0.0;

            timeOut = true;
        }
        //if in a counting state and has been counting for 1 second
        else if (Td > 1 && Td < 2.5)
        {
            result.cmdVel = -0.2;
            result.angleError = 0.0;
        }
    }
    else if (blockDist > targetDist && !lockTarget) //if a target is detected but not locked, and not too close.
    {
        float vel = blockDist * 0.20;
        if (vel < 0.1) vel = 0.1;
        if (vel > 0.2) vel = 0.2;
        result.cmdVel = vel;
        result.angleError = -blockYawError/2;
        timeOut = false;
        return result;
    }
    else if (!lockTarget) //if a target hasn't been locked lock it and enter a counting state while slowly driving forward.
    {
        lockTarget = true;
        result.cmdVel = 0.18;
        result.angleError = 0.0;
        timeOut = true;
    }
    else if (Td > 2.4) //raise the wrist
    {
        result.cmdVel = -0.25;
        result.angleError = 0.0;
        result.wristAngle = 0;
    }
    else if (Td > 1.7) //close the fingers and stop driving
    {
        result.cmdVel = -0.1;
        result.angleError = 0.0;
        result.fingerAngle = 0;
        return result;
    }

    if (Td > 3.8 && timeOut) //if enough time has passed enter a recovery state to re-attempt a pickup
    {
        if (blockBlock) //if the ultrasound is blocked at less than .12 meters a block has been picked up no new pickup required
        {
            result.pickedUp = true;
        }
        else //recover begin looking for targets again
        {
            lockTarget = false;
            result.cmdVel = -0.15;
            result.angleError = 0.0;
            //set gripper
            result.fingerAngle = M_PI_2;
            result.wristAngle = 0;
        }
    }

    if (Td > 5 && timeOut) //if no targets are found after too long a period go back to search pattern
    {
        result.giveUp = true;
        lockTarget = false;
        timeOut = false;
        result.cmdVel = 0.0;
        result.angleError = 0.0;
    }

    return result;
}

PickUpResult PickUpController::selectTarget(const apriltags_ros::AprilTagDetectionArray::ConstPtr& message) {

    /*PickUpResult result;
  result.pickedUp = false;
  result.cmdVel = 0;
  result.angleError = 0;
  result.fingerAngle = -1;
  result.wristAngle = -1;
  result.giveUp = false;*/

    nTargetsSeen = 0;
    nTargetsSeen = message->detections.size();

    double closest = std::numeric_limits<double>::max();
    int target  = 0;
    for (int i = 0; i < message->detections.size(); i++) //this loop selects the closest visible block to makes goals for it
    {
        geometry_msgs::PoseStamped tagPose = message->detections[i].pose;
        double test = hypot(hypot(tagPose.pose.position.x, tagPose.pose.position.y), tagPose.pose.position.z); //absolute distance to block from camera lense

        if (closest > test)
        {
            target = i;
            closest = test;
            blockDist = hypot(tagPose.pose.position.z, tagPose.pose.position.y); //distance from bottom center of chassis ignoring height.
            blockDist = sqrt(blockDist*blockDist - 0.195*0.195);
            blockYawError = atan((tagPose.pose.position.x + 0.020)/blockDist)*1.05; //angle to block from bottom center of chassis on the horizontal.
        }
    }
    if ( blockYawError > 10) blockYawError = 10; //limits block angle error to prevent overspeed from PID.
    if ( blockYawError < - 10) blockYawError = -10; //due to detetionropping out when moveing quickly

    geometry_msgs::PoseStamped tagPose = message->detections[target].pose;

    //if target is close enough
    //diffrence between current time and millisecond time
    ros::Duration Tdiff = ros::Time::now() - millTimer;
    float Td = Tdiff.sec + Tdiff.nsec/1000000000;

    if (hypot(hypot(tagPose.pose.position.x, tagPose.pose.position.y), tagPose.pose.position.z) < 0.13 && Td < 3.8) {
        result.pickedUp = true;
    }

    //Lower wrist and open fingures if no locked targt
    else if (!lockTarget)
    {

        //set gripper;
        result.fingerAngle = M_PI_2;
        result.wristAngle = 1.25;
    }

    return result;
}

void PickUpController::reset() {
    result.pickedUp = false;
    lockTarget = false;
    timeOut = false;
    nTargetsSeen = 0;
    blockYawError = 0;
    blockDist = 0;
    td = 0;

    result.pickedUp = false;
    result.cmdVel = 0;
    result.angleError = 0;
    result.fingerAngle = -1;
    result.wristAngle = -1;
    result.giveUp = false;
}

PickUpController::~PickUpController() {
}
<commit_msg>hotfix-1.2.7 fixed ifinite turn bug When the robot lost tracking of a cube before the lockout phase it would turn in a circle untill it saw an april tag of any kind. This was caused by the target count never being zero beacuse target handler wasn't called for 0 targets. This caused the timeout function to not get triggred so the robot turned forever. This was fixed by resseting the target count back to 0 every loop through pickupcontroller.<commit_after>#include "PickUpController.h"

PickUpController::PickUpController() {
    lockTarget = false;
    timeOut = false;
    nTargetsSeen = 0;
    blockYawError = 0;
    blockDist = 0;
    td = 0;

    result.pickedUp = false;
    result.cmdVel = 0;
    result.angleError = 0;
    result.fingerAngle = -1;
    result.wristAngle = -1;
    result.giveUp = false;

}

PickUpResult PickUpController::pickUpSelectedTarget(bool blockBlock) {
    //threshold distance to be from the target block before attempting pickup
    float targetDist = 0.25; //meters

    /*PickUpResult result;
  result.pickedUp = false;
  result.cmdVel = 0;
  result.angleError = 0;
  result.fingerAngle = -1;
  result.wristAngle = -1;
  result.pickedUp = false;
  result.giveUp = false;*/

    // millisecond time = current time if not in a counting state
    if (!timeOut) millTimer = ros::Time::now();

    //diffrence between current time and millisecond time
    ros::Duration Tdiff = ros::Time::now() - millTimer;
    float Td = Tdiff.sec + Tdiff.nsec/1000000000.0;
    td = Td;

    if (nTargetsSeen == 0 && !lockTarget) //if not targets detected and a target has not been locked in
    {
        if(!timeOut) //if not in a counting state
        {
            result.cmdVel = 0.0;
            result.angleError = 0.0;

            timeOut = true;
        }
        //if in a counting state and has been counting for 1 second
        else if (Td > 1 && Td < 2.5)
        {
            result.cmdVel = -0.2;
            result.angleError = 0.0;
        }
    }
    else if (blockDist > targetDist && !lockTarget) //if a target is detected but not locked, and not too close.
    {
        float vel = blockDist * 0.20;
        if (vel < 0.1) vel = 0.1;
        if (vel > 0.2) vel = 0.2;
        result.cmdVel = vel;
        result.angleError = -blockYawError/2;
        timeOut = false;
        nTargetsSeen = 0;
        return result;
    }
    else if (!lockTarget) //if a target hasn't been locked lock it and enter a counting state while slowly driving forward.
    {
        lockTarget = true;
        result.cmdVel = 0.18;
        result.angleError = 0.0;
        timeOut = true;
    }
    else if (Td > 2.4) //raise the wrist
    {
        result.cmdVel = -0.25;
        result.angleError = 0.0;
        result.wristAngle = 0;
    }
    else if (Td > 1.7) //close the fingers and stop driving
    {
        result.cmdVel = -0.1;
        result.angleError = 0.0;
        result.fingerAngle = 0;
        return result;
    }

    if (Td > 3.8 && timeOut) //if enough time has passed enter a recovery state to re-attempt a pickup
    {
        if (blockBlock) //if the ultrasound is blocked at less than .12 meters a block has been picked up no new pickup required
        {
            result.pickedUp = true;
        }
        else //recover begin looking for targets again
        {
            lockTarget = false;
            result.cmdVel = -0.15;
            result.angleError = 0.0;
            //set gripper
            result.fingerAngle = M_PI_2;
            result.wristAngle = 0;
        }
    }

    if (Td > 5 && timeOut) //if no targets are found after too long a period go back to search pattern
    {
        result.giveUp = true;
        lockTarget = false;
        timeOut = false;
        result.cmdVel = 0.0;
        result.angleError = 0.0;
    }

    return result;
}

PickUpResult PickUpController::selectTarget(const apriltags_ros::AprilTagDetectionArray::ConstPtr& message) {

    /*PickUpResult result;
  result.pickedUp = false;
  result.cmdVel = 0;
  result.angleError = 0;
  result.fingerAngle = -1;
  result.wristAngle = -1;
  result.giveUp = false;*/

    nTargetsSeen = 0;
    nTargetsSeen = message->detections.size();

    double closest = std::numeric_limits<double>::max();
    int target  = 0;
    for (int i = 0; i < message->detections.size(); i++) //this loop selects the closest visible block to makes goals for it
    {
        geometry_msgs::PoseStamped tagPose = message->detections[i].pose;
        double test = hypot(hypot(tagPose.pose.position.x, tagPose.pose.position.y), tagPose.pose.position.z); //absolute distance to block from camera lense

        if (closest > test)
        {
            target = i;
            closest = test;
            blockDist = hypot(tagPose.pose.position.z, tagPose.pose.position.y); //distance from bottom center of chassis ignoring height.
            blockDist = sqrt(blockDist*blockDist - 0.195*0.195);
            blockYawError = atan((tagPose.pose.position.x + 0.020)/blockDist)*1.05; //angle to block from bottom center of chassis on the horizontal.
        }
    }
    if ( blockYawError > 10) blockYawError = 10; //limits block angle error to prevent overspeed from PID.
    if ( blockYawError < - 10) blockYawError = -10; //due to detetionropping out when moveing quickly

    geometry_msgs::PoseStamped tagPose = message->detections[target].pose;

    //if target is close enough
    //diffrence between current time and millisecond time
    ros::Duration Tdiff = ros::Time::now() - millTimer;
    float Td = Tdiff.sec + Tdiff.nsec/1000000000;

    if (hypot(hypot(tagPose.pose.position.x, tagPose.pose.position.y), tagPose.pose.position.z) < 0.13 && Td < 3.8) {
        result.pickedUp = true;
    }

    //Lower wrist and open fingures if no locked targt
    else if (!lockTarget)
    {

        //set gripper;
        result.fingerAngle = M_PI_2;
        result.wristAngle = 1.25;
    }

    return result;
}

void PickUpController::reset() {
    result.pickedUp = false;
    lockTarget = false;
    timeOut = false;
    nTargetsSeen = 0;
    blockYawError = 0;
    blockDist = 0;
    td = 0;

    result.pickedUp = false;
    result.cmdVel = 0;
    result.angleError = 0;
    result.fingerAngle = -1;
    result.wristAngle = -1;
    result.giveUp = false;
}

PickUpController::~PickUpController() {
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) Sjors Gielen, 2010
 * See LICENSE for license.
 */

#include "database.h"
#include "config.h"
#include <QtCore/QVariant>
#include <QtCore/QDebug>
#include <QtSql/QSqlQuery>
#include <QtSql/QSqlError>
#include <QtSql/QSqlRecord>

// #define DEBUG

/**
 * @brief Constructor.
 */
Database::Database( const QString &network, const QString &dbType,
                    const QString &databaseName, const QString &username,
                    const QString &password, const QString &hostname,
                    int port, const QString &options )
: network_(network.toLower())
, db_(QSqlDatabase::addDatabase(typeToQtPlugin(dbType)))
{
  db_.setDatabaseName( databaseName );
  db_.setUserName( username );
  db_.setPassword( password );
  db_.setHostName( hostname );
  db_.setPort( port );
  db_.setConnectOptions( options );
}

/**
 * @brief Destructor.
 */
Database::~Database()
{
}

/**
 * @brief Create a Database from database configuration.
 */
Database *Database::fromConfig(const DatabaseConfig *dbc)
{
#warning network is empty?
  return new Database( QString(), dbc->type, dbc->database, dbc->username,
                           dbc->password, dbc->hostname, dbc->port,
                           dbc->options );
}

/**
 * @brief Returns the last error in the QSqlDatabase object.
 */
QSqlError Database::lastError() const
{
  return db_.lastError();
}

/**
 * @brief Returns the network for which this Database was created.
 */
const QString &Database::network() const
{
  return network_;
}

bool Database::open()
{
  return db_.open();
}

/**
 * @brief Retrieve property from database.
 *
 * The most specific scope will be selected first, i.e. user-specific scope.
 * If that yields no results, channel-specific scope will be returned, then
 * network-specific scope, then global scope.
 *
 * Variable should be formatted using reverse-domain format, i.e. for the
 * plugin MyFirstPlugin created by John Doe, to save a 'foo' variable one
 * could use:
 * net.jondoe.myfirstplugin.foo
 */
QVariant Database::property( const QString &variable,
 const QString &networkScope, const QString &receiverScope,
 const QString &senderScope ) const
{
  // retrieve from database
  QSqlQuery data(db_);
  data.prepare(QLatin1String("SELECT value,network,receiver,sender"
     " FROM properties WHERE variable=?"));
  data.addBindValue(variable);
  if( !data.exec() )
  {
    qWarning() << "Property retrieve query failed: " << data.lastError();
    return QVariant();
  }

  // index the results
  QVariant global, network, receiver, sender;
  while( data.next() )
  {
    if( data.value(3).isNull() )
    {
      if( data.value(2).isNull() )
      {
        if( data.value(1).isNull() )
          global = data.value(0);
        else if( data.value(1).toString() == networkScope )
          network = data.value(0);
      }
      else if( data.value(1).toString() == networkScope
            && data.value(2).toString() == receiverScope )
        receiver = data.value(0);
    }
    else if( data.value(1).toString() == networkScope
          && data.value(2).toString() == receiverScope
          && data.value(3).toString() == senderScope )
      sender = data.value(0);
  }

#ifdef DEBUG
  qDebug() << "For variable " << variable;
  qDebug() << "Global: " << global;
  qDebug() << "Network: " << network;
  qDebug() << "Receiver: " << receiver;
  qDebug() << "Sender: " << sender;
#endif

  if( !sender.isNull() )
    return sender;
  if( !receiver.isNull() )
    return receiver;
  if( !network.isNull() )
    return network;
  return global;
}

void Database::setProperty( const QString &variable,
 const QVariant &value, const QString &networkScope,
 const QString &receiverScope, const QString &senderScope )
{
#ifdef DEBUG
  qDebug() << "Setting property " << variable << "to" << value;
#endif

  // first, select it from the database, to see whether to update or insert
  // (not all database engines support "on duplicate key update".)
  QSqlQuery finder(db_);
  // because size() always returns -1 on an SQLite backend, we use COUNT.
  finder.prepare(
      QString(QLatin1String("SELECT id,COUNT(id) FROM properties WHERE "
                "variable=? AND network %1 AND receiver %2 AND sender %3"))
      .arg( QLatin1String(networkScope.isEmpty()  ? "IS NULL" : "=?") )
      .arg( QLatin1String(receiverScope.isEmpty() ? "IS NULL" : "=?") )
      .arg( QLatin1String(senderScope.isEmpty()   ? "IS NULL" : "=?") )
  );
  // This ugly hack seems necessary as MySQL does not allow =NULL, requires IS NULL.
  finder.addBindValue(variable);
  if( !networkScope.isEmpty() )
    finder.addBindValue(networkScope);
  if( !receiverScope.isEmpty() )
    finder.addBindValue(receiverScope);
  if( !senderScope.isEmpty() )
    finder.addBindValue(senderScope);

  if( !finder.exec() )
  {
    qWarning() << "Select property before setting failed: " << finder.lastError();
    return;
  }

  if( !finder.next() )
  {
    qWarning() << "Could not select first value row before setting: " << finder.lastError();
    return;
  }

  int resultSize = finder.value(1).isValid() ? finder.value(1).toInt() : 0;
  int returnedId = finder.value(0).isValid() ? finder.value(0).toInt() : -1;

  if( resultSize > 1 )
    qWarning() << "*** WARNING: More than one result to variable retrieve! This is a bug. ***";

  QSqlQuery data(db_);

  if( !resultSize )
  {
    // insert
    if( !value.isValid() )
    {
      // Don't insert null values.
      return;
    }
#ifdef DEBUG
    qDebug() << "Seeing property " << variable << "for the first time, inserting.";
    qDebug() << finder.executedQuery();
    QList<QVariant> list = finder.boundValues().values();
    for (int i = 0; i < list.size(); ++i)
      qDebug() << i << ": " << list.at(i).toString().toAscii().data() << endl;
#endif
    data.prepare(QLatin1String("INSERT INTO properties (variable,value,"
                "network,receiver,sender) VALUES (?, ?, ?, ?, ?)"));
    data.addBindValue(variable);
    data.addBindValue(value);
    data.addBindValue(networkScope.isEmpty()  ? QVariant() : networkScope);
    data.addBindValue(receiverScope.isEmpty() ? QVariant() : receiverScope);
    data.addBindValue(senderScope.isEmpty()   ? QVariant() : senderScope);
  }
  else
  {
    // update
    if( !value.isValid() )
    {
      // Don't insert null values - delete the old one
      data.prepare(QLatin1String("DELETE FROM properties WHERE id=?"));
    }
    else
    {
      // update the old one
      data.prepare(QLatin1String("UPDATE properties SET value=? WHERE id=?"));
      data.addBindValue(value);
    }
    data.addBindValue(returnedId);
  }

#ifdef DEBUG
  qDebug() << "Executing data query: " << data.lastQuery();
  qDebug() << "Bound data query values: " << data.boundValues();
#endif

  if( !data.exec() )
  {
    qWarning() << "Set property failed: " << data.lastError();
  }
}

bool Database::createTable()
{
  if( tableExists() )
    return true; // database already exists

  QSqlQuery data(db_);
  QLatin1String idrow( "INT(10) PRIMARY KEY AUTO_INCREMENT" );
  if( db_.driverName().toLower().startsWith(QLatin1String("qsqlite")) )
  {
    idrow = QLatin1String("INTEGER PRIMARY KEY AUTOINCREMENT");
  }
  bool ok = data.exec(QString(QLatin1String("CREATE TABLE properties ("
                   "  id      %1,"
                   "  variable VARCHAR(150),"
                   "  network VARCHAR(50)," // enforced limit in config.cpp
                   "  receiver VARCHAR(50)," // usually, max = 9 or 30
                   "  sender VARCHAR(50)," //  usually, max = 9 or 30
                   "  value TEXT"
                   ");")).arg(idrow));
  if( !ok )
  {
    qWarning() << data.lastError();
  }
  return ok;
}
bool Database::tableExists() const
{
  return db_.record(QLatin1String("properties")).count() > 0;
}

QLatin1String Database::typeToQtPlugin(const QString &type)
{
  if( type.toLower() == QLatin1String("sqlite") )
    return "QSQLITE";
  if( type.toLower() == QLatin1String("mysql") )
    return "QMYSQL";

  qWarning() << "typeToQtPlugin: Unknown type: " << type;
  return "";
}
<commit_msg>Make Database ascii-safe, forgot this change.<commit_after>/**
 * Copyright (c) Sjors Gielen, 2010
 * See LICENSE for license.
 */

#include "database.h"
#include "config.h"
#include <QtCore/QVariant>
#include <QtCore/QDebug>
#include <QtSql/QSqlQuery>
#include <QtSql/QSqlError>
#include <QtSql/QSqlRecord>

// #define DEBUG

/**
 * @brief Constructor.
 */
Database::Database( const QString &network, const QString &dbType,
                    const QString &databaseName, const QString &username,
                    const QString &password, const QString &hostname,
                    int port, const QString &options )
: network_(network.toLower())
, db_(QSqlDatabase::addDatabase(typeToQtPlugin(dbType)))
{
  db_.setDatabaseName( databaseName );
  db_.setUserName( username );
  db_.setPassword( password );
  db_.setHostName( hostname );
  db_.setPort( port );
  db_.setConnectOptions( options );
}

/**
 * @brief Destructor.
 */
Database::~Database()
{
}

/**
 * @brief Create a Database from database configuration.
 */
Database *Database::fromConfig(const DatabaseConfig *dbc)
{
#warning network is empty?
  return new Database( QString(), dbc->type, dbc->database, dbc->username,
                           dbc->password, dbc->hostname, dbc->port,
                           dbc->options );
}

/**
 * @brief Returns the last error in the QSqlDatabase object.
 */
QSqlError Database::lastError() const
{
  return db_.lastError();
}

/**
 * @brief Returns the network for which this Database was created.
 */
const QString &Database::network() const
{
  return network_;
}

bool Database::open()
{
  return db_.open();
}

/**
 * @brief Retrieve property from database.
 *
 * The most specific scope will be selected first, i.e. user-specific scope.
 * If that yields no results, channel-specific scope will be returned, then
 * network-specific scope, then global scope.
 *
 * Variable should be formatted using reverse-domain format, i.e. for the
 * plugin MyFirstPlugin created by John Doe, to save a 'foo' variable one
 * could use:
 * net.jondoe.myfirstplugin.foo
 */
QVariant Database::property( const QString &variable,
 const QString &networkScope, const QString &receiverScope,
 const QString &senderScope ) const
{
  // retrieve from database
  QSqlQuery data(db_);
  data.prepare(QLatin1String("SELECT value,network,receiver,sender"
     " FROM properties WHERE variable=?"));
  data.addBindValue(variable);
  if( !data.exec() )
  {
    qWarning() << "Property retrieve query failed: " << data.lastError();
    return QVariant();
  }

  // index the results
  QVariant global, network, receiver, sender;
  while( data.next() )
  {
    if( data.value(3).isNull() )
    {
      if( data.value(2).isNull() )
      {
        if( data.value(1).isNull() )
          global = data.value(0);
        else if( data.value(1).toString() == networkScope )
          network = data.value(0);
      }
      else if( data.value(1).toString() == networkScope
            && data.value(2).toString() == receiverScope )
        receiver = data.value(0);
    }
    else if( data.value(1).toString() == networkScope
          && data.value(2).toString() == receiverScope
          && data.value(3).toString() == senderScope )
      sender = data.value(0);
  }

#ifdef DEBUG
  qDebug() << "For variable " << variable;
  qDebug() << "Global: " << global;
  qDebug() << "Network: " << network;
  qDebug() << "Receiver: " << receiver;
  qDebug() << "Sender: " << sender;
#endif

  if( !sender.isNull() )
    return sender;
  if( !receiver.isNull() )
    return receiver;
  if( !network.isNull() )
    return network;
  return global;
}

void Database::setProperty( const QString &variable,
 const QVariant &value, const QString &networkScope,
 const QString &receiverScope, const QString &senderScope )
{
#ifdef DEBUG
  qDebug() << "Setting property " << variable << "to" << value;
#endif

  // first, select it from the database, to see whether to update or insert
  // (not all database engines support "on duplicate key update".)
  QSqlQuery finder(db_);
  // because size() always returns -1 on an SQLite backend, we use COUNT.
  finder.prepare(
      QString(QLatin1String("SELECT id,COUNT(id) FROM properties WHERE "
                "variable=? AND network %1 AND receiver %2 AND sender %3"))
      .arg( QLatin1String(networkScope.isEmpty()  ? "IS NULL" : "=?") )
      .arg( QLatin1String(receiverScope.isEmpty() ? "IS NULL" : "=?") )
      .arg( QLatin1String(senderScope.isEmpty()   ? "IS NULL" : "=?") )
  );
  // This ugly hack seems necessary as MySQL does not allow =NULL, requires IS NULL.
  finder.addBindValue(variable);
  if( !networkScope.isEmpty() )
    finder.addBindValue(networkScope);
  if( !receiverScope.isEmpty() )
    finder.addBindValue(receiverScope);
  if( !senderScope.isEmpty() )
    finder.addBindValue(senderScope);

  if( !finder.exec() )
  {
    qWarning() << "Select property before setting failed: " << finder.lastError();
    return;
  }

  if( !finder.next() )
  {
    qWarning() << "Could not select first value row before setting: " << finder.lastError();
    return;
  }

  int resultSize = finder.value(1).isValid() ? finder.value(1).toInt() : 0;
  int returnedId = finder.value(0).isValid() ? finder.value(0).toInt() : -1;

  if( resultSize > 1 )
    qWarning() << "*** WARNING: More than one result to variable retrieve! This is a bug. ***";

  QSqlQuery data(db_);

  if( !resultSize )
  {
    // insert
    if( !value.isValid() )
    {
      // Don't insert null values.
      return;
    }
#ifdef DEBUG
    qDebug() << "Seeing property " << variable << "for the first time, inserting.";
    qDebug() << finder.executedQuery();
    QList<QVariant> list = finder.boundValues().values();
    for (int i = 0; i < list.size(); ++i)
      qDebug() << i << ": " << list.at(i).toString().toAscii().data() << endl;
#endif
    data.prepare(QLatin1String("INSERT INTO properties (variable,value,"
                "network,receiver,sender) VALUES (?, ?, ?, ?, ?)"));
    data.addBindValue(variable);
    data.addBindValue(value);
    data.addBindValue(networkScope.isEmpty()  ? QVariant() : networkScope);
    data.addBindValue(receiverScope.isEmpty() ? QVariant() : receiverScope);
    data.addBindValue(senderScope.isEmpty()   ? QVariant() : senderScope);
  }
  else
  {
    // update
    if( !value.isValid() )
    {
      // Don't insert null values - delete the old one
      data.prepare(QLatin1String("DELETE FROM properties WHERE id=?"));
    }
    else
    {
      // update the old one
      data.prepare(QLatin1String("UPDATE properties SET value=? WHERE id=?"));
      data.addBindValue(value);
    }
    data.addBindValue(returnedId);
  }

#ifdef DEBUG
  qDebug() << "Executing data query: " << data.lastQuery();
  qDebug() << "Bound data query values: " << data.boundValues();
#endif

  if( !data.exec() )
  {
    qWarning() << "Set property failed: " << data.lastError();
  }
}

bool Database::createTable()
{
  if( tableExists() )
    return true; // database already exists

  QSqlQuery data(db_);
  QLatin1String idrow( "INT(10) PRIMARY KEY AUTO_INCREMENT" );
  if( db_.driverName().toLower().startsWith(QLatin1String("qsqlite")) )
  {
    idrow = QLatin1String("INTEGER PRIMARY KEY AUTOINCREMENT");
  }
  bool ok = data.exec(QString(QLatin1String("CREATE TABLE properties ("
                   "  id      %1,"
                   "  variable VARCHAR(150),"
                   "  network VARCHAR(50)," // enforced limit in config.cpp
                   "  receiver VARCHAR(50)," // usually, max = 9 or 30
                   "  sender VARCHAR(50)," //  usually, max = 9 or 30
                   "  value TEXT"
                   ");")).arg(idrow));
  if( !ok )
  {
    qWarning() << data.lastError();
  }
  return ok;
}
bool Database::tableExists() const
{
  return db_.record(QLatin1String("properties")).count() > 0;
}

QLatin1String Database::typeToQtPlugin(const QString &type)
{
  if( type.toLower() == QLatin1String("sqlite") )
    return QLatin1String("QSQLITE");
  if( type.toLower() == QLatin1String("mysql") )
    return QLatin1String("QMYSQL");

  qWarning() << "typeToQtPlugin: Unknown type: " << type;
  return QLatin1String("");
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "stx/stdtypes.h"
#include "stx/application.h"
#include "stx/logging.h"
#include "stx/cli/flagparser.h"
#include "stx/io/fileutil.h"
#include "stx/http/httprouter.h"
#include "stx/http/httpserver.h"
#include "stx/thread/eventloop.h"
#include "stx/thread/threadpool.h"
#include "master/CustomerDirectoryMaster.h"
#include "master/CustomerDirectoryServlet.h"

using namespace stx;
using namespace cm;

stx::thread::EventLoop ev;

int main(int argc, const char** argv) {
  stx::Application::init();
  stx::Application::logToStderr();

  stx::cli::FlagParser flags;

  flags.defineFlag(
      "http_port",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "7003",
      "Start the public http server on this port",
      "<port>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "datadir path",
      "<path>");

  flags.defineFlag(
      "loglevel",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* thread pools */
  stx::thread::ThreadPool tpool;

  /* http */
  stx::http::HTTPRouter http_router;
  stx::http::HTTPServer http_server(&http_router, &ev);
  http_server.listen(flags.getInt("http_port"));

  /* customer directory */
  auto cdb_dir = FileUtil::joinPaths(
      flags.getString("datadir"),
      "master/customers");

  if (!FileUtil::exists(cdb_dir)) {
    FileUtil::mkdir(cdb_dir);
  }

  CustomerDirectoryMaster customer_dir(cdb_dir);
  CustomerDirectoryServlet customer_dir_servlet(&customer_dir);
  http_router.addRouteByPrefixMatch("/cdb", &customer_dir_servlet, &tpool);

  stx::logInfo("dxa-master", "Exiting...");

  exit(0);
}

<commit_msg>start http server<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "stx/stdtypes.h"
#include "stx/application.h"
#include "stx/logging.h"
#include "stx/cli/flagparser.h"
#include "stx/io/fileutil.h"
#include "stx/http/httprouter.h"
#include "stx/http/httpserver.h"
#include "stx/thread/eventloop.h"
#include "stx/thread/threadpool.h"
#include "master/CustomerDirectoryMaster.h"
#include "master/CustomerDirectoryServlet.h"

using namespace stx;
using namespace cm;

stx::thread::EventLoop ev;

int main(int argc, const char** argv) {
  stx::Application::init();
  stx::Application::logToStderr();

  stx::cli::FlagParser flags;

  flags.defineFlag(
      "http_port",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "7003",
      "Start the public http server on this port",
      "<port>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "datadir path",
      "<path>");

  flags.defineFlag(
      "loglevel",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* thread pools */
  stx::thread::ThreadPool tpool;

  /* http */
  stx::http::HTTPRouter http_router;
  stx::http::HTTPServer http_server(&http_router, &ev);
  http_server.listen(flags.getInt("http_port"));

  /* customer directory */
  auto cdb_dir = FileUtil::joinPaths(
      flags.getString("datadir"),
      "master/customers");

  if (!FileUtil::exists(cdb_dir)) {
    FileUtil::mkdir_p(cdb_dir);
  }

  CustomerDirectoryMaster customer_dir(cdb_dir);
  CustomerDirectoryServlet customer_dir_servlet(&customer_dir);
  http_router.addRouteByPrefixMatch("/cdb", &customer_dir_servlet, &tpool);

  ev.run();
  stx::logInfo("dxa-master", "Exiting...");

  exit(0);
}

<|endoftext|>


<commit_before>/****************************************************************************

This file is part of the QtMediaHub project on http://www.gitorious.org.

Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).*
All rights reserved.

Contact:  Nokia Corporation (qt-info@nokia.com)**

You may use this file under the terms of the BSD license as follows:

"Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Nokia Corporation and its Subsidiary(-ies) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."

****************************************************************************/

#include "mediamodel.h"
#include "mediascanner.h"
#include "dbreader.h"
#include "backend.h"

#define DEBUG if (0) qDebug() << __PRETTY_FUNCTION__

MediaModel::MediaModel(QObject *parent)
    : QAbstractItemModel(parent), m_loading(false), m_loaded(false), m_reader(0), m_readerThread(0)
{
}

MediaModel::~MediaModel()
{
    if (m_reader) {
        m_reader->stop();
        m_reader->deleteLater();
        m_readerThread->quit();
        m_readerThread->wait();
    }
}

QString MediaModel::part() const
{
    return m_structure.split("|").value(m_cursor.length());
}

QString MediaModel::mediaType() const
{
    return m_mediaType;
}

void MediaModel::setMediaType(const QString &type)
{
    if (type == m_mediaType)
        return;

    DEBUG << type;

    m_mediaType = type;
    emit mediaTypeChanged();

    // Add the fields of the table as role names
    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();
    QSqlRecord record = driver->record(m_mediaType);
    if (record.isEmpty())
        qWarning() << "Table " << type << " is not valid it seems";

    QHash<int, QByteArray> hash = roleNames();
    hash.insert(DotDotRole, "dotdot");
    hash.insert(IsLeafRole, "isLeaf");
    hash.insert(ModelIndexRole,"modelIndex");
    hash.insert(PreviewUrlRole, "previewUrl");

    for (int i = 0; i < record.count(); i++) {
        hash.insert(FieldRolesBegin + i, record.fieldName(i).toUtf8());
    }

    setRoleNames(hash);

    initialize();
}

void MediaModel::addSearchPath(const QString &path, const QString &name)
{
    if (m_mediaType.isEmpty()) {
        qWarning() << m_mediaType << " is not set yet";
        return;
    }

    QMetaObject::invokeMethod(MediaScanner::instance(), "addSearchPath", Qt::QueuedConnection,
                              Q_ARG(QString, m_mediaType), Q_ARG(QString, path), Q_ARG(QString, name));
}

void MediaModel::removeSearchPath(int index)
{
    Q_UNUSED(index);
}

QString MediaModel::structure() const
{
    return m_structure;
}

void MediaModel::setStructure(const QString &str)
{
    if (str == m_structure)
        return;
    DEBUG << str;
    m_structure = str;
    m_layoutInfo.clear();
    foreach(const QString &part, m_structure.split("|"))
        m_layoutInfo.append(part.split(","));

    initialize();

    emit structureChanged();
}

void MediaModel::enter(int index)
{
    if (m_cursor.count() + 1 == m_layoutInfo.count() && index != 0 /* up on leaf node is OK */) {
        DEBUG << "Refusing to enter leaf node";
        return;
    }

    if (index == 0 && !m_cursor.isEmpty()) {
        back();
        return;
    }

    DEBUG << "Entering " << index;
    m_cursor.append(m_data[index]);
    initialize();
    emit partChanged();
}

void MediaModel::back()
{
    m_cursor.removeLast();
    initialize();
    emit partChanged();
}

QVariant MediaModel::data(const QModelIndex &index, int role) const
{
    if (!index.isValid())
        return QVariant();

    if (role == ModelIndexRole)
        return qVariantFromValue(index);

    return m_data.value(index.row()).value(role);
}

QModelIndex MediaModel::index(int row, int col, const QModelIndex &parent) const
{
    if (parent.isValid())
        return QModelIndex();

    return createIndex(row, col);
}

QModelIndex MediaModel::parent(const QModelIndex &idx) const
{
    Q_UNUSED(idx);
    return QModelIndex();
}

int MediaModel::rowCount(const QModelIndex &parent) const
{
    if (parent.isValid())
        return 0;
    return m_data.count();
}

int MediaModel::columnCount(const QModelIndex &idx) const
{
    Q_UNUSED(idx);
    return 1;
}

bool MediaModel::hasChildren(const QModelIndex &parent) const
{
    return !parent.isValid();
}

bool MediaModel::canFetchMore(const QModelIndex &parent) const
{
    if (parent.isValid() || m_mediaType.isEmpty() || m_layoutInfo.isEmpty()) {
        DEBUG << "false " << parent.isValid() << m_mediaType.isEmpty() << m_layoutInfo.isEmpty();
        return false;
    }
    DEBUG << (!m_loading && !m_loaded);
    return !m_loading && !m_loaded;
}

void MediaModel::fetchMore(const QModelIndex &parent)
{
    if (!canFetchMore(parent))
        return;

    DEBUG << "";

    m_loading = true;

    QSqlQuery q = buildQuery();
    DEBUG << q.lastQuery();
    QMetaObject::invokeMethod(m_reader, "execute", Qt::QueuedConnection, Q_ARG(QSqlQuery, q));
}

void MediaModel::initialize()
{
    DEBUG << "";

    DbReader *newReader = new DbReader;
    if (m_reader) {
        disconnect(m_reader, 0, this, 0);
        m_reader->stop();
        m_reader->deleteLater();
    }
    m_reader = newReader;

    if (!m_readerThread) {
        m_readerThread = new QThread(this);
        m_readerThread->start();
    }
    m_reader->moveToThread(m_readerThread);
    QMetaObject::invokeMethod(m_reader, "initialize", Q_ARG(QSqlDatabase, Backend::instance()->mediaDatabase()));
    connect(m_reader, SIGNAL(dataReady(DbReader *, QList<QSqlRecord>, void *)),
            this, SLOT(handleDataReady(DbReader *, QList<QSqlRecord>, void *)));

    beginResetModel();
    m_loading = m_loaded = false;
    m_data.clear();
    endResetModel();
}

void MediaModel::handleDataReady(DbReader *reader, const QList<QSqlRecord> &records, void *node)
{
    Q_ASSERT(reader == m_reader);
    Q_UNUSED(reader);
    Q_UNUSED(node);

    DEBUG << "Received response from db of size " << records.size();

    if (records.isEmpty())
        return;

    if (!m_cursor.isEmpty()) {
        beginInsertRows(QModelIndex(), 0, records.count());
        QHash<int, QVariant> data;
        data.insert(Qt::DisplayRole, tr(".."));
        data.insert(DotDotRole, true);
        data.insert(IsLeafRole, false);
        m_data.append(data);
    } else {
        beginInsertRows(QModelIndex(), 0, records.count() - 1);
    }

    const bool isLeaf = m_cursor.count() + 1 == m_layoutInfo.count();

    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();
    const QSqlRecord tableRecord = driver->record(m_mediaType);

    for (int i = 0; i < records.count(); i++) {
        QHash<int, QVariant> data;
        for (int j = 0; j < records[i].count(); j++) {
            int idx = tableRecord.indexOf(records[i].fieldName(j));
            data.insert(FieldRolesBegin + idx, records[i].value(j));
        }

        // Provide 'display' role as , separated values
        QStringList cols = m_layoutInfo.value(m_cursor.count());
        QStringList displayString;
        for (int j = 0; j < cols.count(); j++) {
            displayString << records[i].value(cols[j]).toString();
        }
        data.insert(Qt::DisplayRole, displayString.join(", "));
        data.insert(DotDotRole, false);
        data.insert(IsLeafRole, isLeaf);

        data.insert(PreviewUrlRole, records[i].value("thumbnail").toString());

        m_data.append(data);
    }

    m_loading = false;
    m_loaded = true;

    endInsertRows();
}

void MediaModel::handleDatabaseUpdated(const QList<QSqlRecord> &records)
{
    Q_UNUSED(records);
    // not implemented yet
}

QSqlQuery MediaModel::buildQuery() const
{
    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();

    QStringList escapedCurParts;
    QStringList curParts = m_layoutInfo[m_cursor.count()];
    for (int i = 0; i < curParts.count(); i++)
        escapedCurParts.append(driver->escapeIdentifier(curParts[i], QSqlDriver::FieldName));
    QString escapedCurPart = escapedCurParts.join(",");

    QSqlQuery query(Backend::instance()->mediaDatabase());
    query.setForwardOnly(true);

    QStringList placeHolders;
    const bool lastPart = m_cursor.count() == m_layoutInfo.count()-1;
    QString queryString;
    // SQLite allows us to select columns that are not present in the GROUP BY. We use this feature
    // to select thumbnails for non-leaf nodes
    queryString.append("SELECT *");
    queryString.append(" FROM " + driver->escapeIdentifier(m_mediaType, QSqlDriver::TableName));

    if (!m_cursor.isEmpty()) {
        QStringList where;
        const QSqlRecord tableRecord = driver->record(m_mediaType);
        for (int i = 0; i < m_cursor.count(); i++) {
            QStringList subParts = m_layoutInfo[i];
            for (int j = 0; j < subParts.count(); j++) {
                where.append(subParts[j] + " = ?");
                const int role = FieldRolesBegin + tableRecord.indexOf(subParts[j]);
                placeHolders << m_cursor[i].value(role).toString();
            }
        }

        queryString.append(" WHERE " + where.join(" AND "));
    }

    if (!lastPart)
        queryString.append(" GROUP BY " + escapedCurPart);

    queryString.append(" ORDER BY " + escapedCurPart);

    query.prepare(queryString);

    foreach(const QString &placeHolder, placeHolders)
        query.addBindValue(placeHolder);

    return query;
}

<commit_msg>PreviewUrlRole should be QUrl<commit_after>/****************************************************************************

This file is part of the QtMediaHub project on http://www.gitorious.org.

Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).*
All rights reserved.

Contact:  Nokia Corporation (qt-info@nokia.com)**

You may use this file under the terms of the BSD license as follows:

"Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Nokia Corporation and its Subsidiary(-ies) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."

****************************************************************************/

#include "mediamodel.h"
#include "mediascanner.h"
#include "dbreader.h"
#include "backend.h"

#define DEBUG if (0) qDebug() << __PRETTY_FUNCTION__

MediaModel::MediaModel(QObject *parent)
    : QAbstractItemModel(parent), m_loading(false), m_loaded(false), m_reader(0), m_readerThread(0)
{
}

MediaModel::~MediaModel()
{
    if (m_reader) {
        m_reader->stop();
        m_reader->deleteLater();
        m_readerThread->quit();
        m_readerThread->wait();
    }
}

QString MediaModel::part() const
{
    return m_structure.split("|").value(m_cursor.length());
}

QString MediaModel::mediaType() const
{
    return m_mediaType;
}

void MediaModel::setMediaType(const QString &type)
{
    if (type == m_mediaType)
        return;

    DEBUG << type;

    m_mediaType = type;
    emit mediaTypeChanged();

    // Add the fields of the table as role names
    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();
    QSqlRecord record = driver->record(m_mediaType);
    if (record.isEmpty())
        qWarning() << "Table " << type << " is not valid it seems";

    QHash<int, QByteArray> hash = roleNames();
    hash.insert(DotDotRole, "dotdot");
    hash.insert(IsLeafRole, "isLeaf");
    hash.insert(ModelIndexRole,"modelIndex");
    hash.insert(PreviewUrlRole, "previewUrl");

    for (int i = 0; i < record.count(); i++) {
        hash.insert(FieldRolesBegin + i, record.fieldName(i).toUtf8());
    }

    setRoleNames(hash);

    initialize();
}

void MediaModel::addSearchPath(const QString &path, const QString &name)
{
    if (m_mediaType.isEmpty()) {
        qWarning() << m_mediaType << " is not set yet";
        return;
    }

    QMetaObject::invokeMethod(MediaScanner::instance(), "addSearchPath", Qt::QueuedConnection,
                              Q_ARG(QString, m_mediaType), Q_ARG(QString, path), Q_ARG(QString, name));
}

void MediaModel::removeSearchPath(int index)
{
    Q_UNUSED(index);
}

QString MediaModel::structure() const
{
    return m_structure;
}

void MediaModel::setStructure(const QString &str)
{
    if (str == m_structure)
        return;
    DEBUG << str;
    m_structure = str;
    m_layoutInfo.clear();
    foreach(const QString &part, m_structure.split("|"))
        m_layoutInfo.append(part.split(","));

    initialize();

    emit structureChanged();
}

void MediaModel::enter(int index)
{
    if (m_cursor.count() + 1 == m_layoutInfo.count() && index != 0 /* up on leaf node is OK */) {
        DEBUG << "Refusing to enter leaf node";
        return;
    }

    if (index == 0 && !m_cursor.isEmpty()) {
        back();
        return;
    }

    DEBUG << "Entering " << index;
    m_cursor.append(m_data[index]);
    initialize();
    emit partChanged();
}

void MediaModel::back()
{
    m_cursor.removeLast();
    initialize();
    emit partChanged();
}

QVariant MediaModel::data(const QModelIndex &index, int role) const
{
    if (!index.isValid())
        return QVariant();

    if (role == ModelIndexRole)
        return qVariantFromValue(index);

    return m_data.value(index.row()).value(role);
}

QModelIndex MediaModel::index(int row, int col, const QModelIndex &parent) const
{
    if (parent.isValid())
        return QModelIndex();

    return createIndex(row, col);
}

QModelIndex MediaModel::parent(const QModelIndex &idx) const
{
    Q_UNUSED(idx);
    return QModelIndex();
}

int MediaModel::rowCount(const QModelIndex &parent) const
{
    if (parent.isValid())
        return 0;
    return m_data.count();
}

int MediaModel::columnCount(const QModelIndex &idx) const
{
    Q_UNUSED(idx);
    return 1;
}

bool MediaModel::hasChildren(const QModelIndex &parent) const
{
    return !parent.isValid();
}

bool MediaModel::canFetchMore(const QModelIndex &parent) const
{
    if (parent.isValid() || m_mediaType.isEmpty() || m_layoutInfo.isEmpty()) {
        DEBUG << "false " << parent.isValid() << m_mediaType.isEmpty() << m_layoutInfo.isEmpty();
        return false;
    }
    DEBUG << (!m_loading && !m_loaded);
    return !m_loading && !m_loaded;
}

void MediaModel::fetchMore(const QModelIndex &parent)
{
    if (!canFetchMore(parent))
        return;

    DEBUG << "";

    m_loading = true;

    QSqlQuery q = buildQuery();
    DEBUG << q.lastQuery();
    QMetaObject::invokeMethod(m_reader, "execute", Qt::QueuedConnection, Q_ARG(QSqlQuery, q));
}

void MediaModel::initialize()
{
    DEBUG << "";

    DbReader *newReader = new DbReader;
    if (m_reader) {
        disconnect(m_reader, 0, this, 0);
        m_reader->stop();
        m_reader->deleteLater();
    }
    m_reader = newReader;

    if (!m_readerThread) {
        m_readerThread = new QThread(this);
        m_readerThread->start();
    }
    m_reader->moveToThread(m_readerThread);
    QMetaObject::invokeMethod(m_reader, "initialize", Q_ARG(QSqlDatabase, Backend::instance()->mediaDatabase()));
    connect(m_reader, SIGNAL(dataReady(DbReader *, QList<QSqlRecord>, void *)),
            this, SLOT(handleDataReady(DbReader *, QList<QSqlRecord>, void *)));

    beginResetModel();
    m_loading = m_loaded = false;
    m_data.clear();
    endResetModel();
}

void MediaModel::handleDataReady(DbReader *reader, const QList<QSqlRecord> &records, void *node)
{
    Q_ASSERT(reader == m_reader);
    Q_UNUSED(reader);
    Q_UNUSED(node);

    DEBUG << "Received response from db of size " << records.size();

    if (records.isEmpty())
        return;

    if (!m_cursor.isEmpty()) {
        beginInsertRows(QModelIndex(), 0, records.count());
        QHash<int, QVariant> data;
        data.insert(Qt::DisplayRole, tr(".."));
        data.insert(DotDotRole, true);
        data.insert(IsLeafRole, false);
        m_data.append(data);
    } else {
        beginInsertRows(QModelIndex(), 0, records.count() - 1);
    }

    const bool isLeaf = m_cursor.count() + 1 == m_layoutInfo.count();

    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();
    const QSqlRecord tableRecord = driver->record(m_mediaType);

    for (int i = 0; i < records.count(); i++) {
        QHash<int, QVariant> data;
        for (int j = 0; j < records[i].count(); j++) {
            int idx = tableRecord.indexOf(records[i].fieldName(j));
            data.insert(FieldRolesBegin + idx, records[i].value(j));
        }

        // Provide 'display' role as , separated values
        QStringList cols = m_layoutInfo.value(m_cursor.count());
        QStringList displayString;
        for (int j = 0; j < cols.count(); j++) {
            displayString << records[i].value(cols[j]).toString();
        }
        data.insert(Qt::DisplayRole, displayString.join(", "));
        data.insert(DotDotRole, false);
        data.insert(IsLeafRole, isLeaf);

        data.insert(PreviewUrlRole, QUrl::fromEncoded(records[i].value("thumbnail").toByteArray()));

        m_data.append(data);
    }

    m_loading = false;
    m_loaded = true;

    endInsertRows();
}

void MediaModel::handleDatabaseUpdated(const QList<QSqlRecord> &records)
{
    Q_UNUSED(records);
    // not implemented yet
}

QSqlQuery MediaModel::buildQuery() const
{
    QSqlDriver *driver = Backend::instance()->mediaDatabase().driver();

    QStringList escapedCurParts;
    QStringList curParts = m_layoutInfo[m_cursor.count()];
    for (int i = 0; i < curParts.count(); i++)
        escapedCurParts.append(driver->escapeIdentifier(curParts[i], QSqlDriver::FieldName));
    QString escapedCurPart = escapedCurParts.join(",");

    QSqlQuery query(Backend::instance()->mediaDatabase());
    query.setForwardOnly(true);

    QStringList placeHolders;
    const bool lastPart = m_cursor.count() == m_layoutInfo.count()-1;
    QString queryString;
    // SQLite allows us to select columns that are not present in the GROUP BY. We use this feature
    // to select thumbnails for non-leaf nodes
    queryString.append("SELECT *");
    queryString.append(" FROM " + driver->escapeIdentifier(m_mediaType, QSqlDriver::TableName));

    if (!m_cursor.isEmpty()) {
        QStringList where;
        const QSqlRecord tableRecord = driver->record(m_mediaType);
        for (int i = 0; i < m_cursor.count(); i++) {
            QStringList subParts = m_layoutInfo[i];
            for (int j = 0; j < subParts.count(); j++) {
                where.append(subParts[j] + " = ?");
                const int role = FieldRolesBegin + tableRecord.indexOf(subParts[j]);
                placeHolders << m_cursor[i].value(role).toString();
            }
        }

        queryString.append(" WHERE " + where.join(" AND "));
    }

    if (!lastPart)
        queryString.append(" GROUP BY " + escapedCurPart);

    queryString.append(" ORDER BY " + escapedCurPart);

    query.prepare(queryString);

    foreach(const QString &placeHolder, placeHolders)
        query.addBindValue(placeHolder);

    return query;
}

<|endoftext|>