<commit_before>#include "xchainer/check_backward.h"

#include <algorithm>
#include <functional>
#include <memory>
#include <string>
#include <vector>

#include "xchainer/array.h"
#include "xchainer/array_node.h"
#include "xchainer/backward.h"
#include "xchainer/error.h"
#include "xchainer/numeric.h"
#include "xchainer/numerical_gradient.h"

namespace xchainer {
namespace {

std::vector<nonstd::optional<Array>> BackwardGradients(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        std::vector<Array>& inputs,
        const nonstd::optional<std::vector<Array>>& grad_outputs,
        const GraphId& graph_id,
        DoubleBackpropOption double_backprop = DoubleBackpropOption::kEnable) {
    for (const auto& input : inputs) {
        if (internal::HasArrayNode(input, graph_id) && internal::GetArrayNode(input, graph_id)->next_node()) {
            throw GradientCheckError{"BackwardGradients: All inputs must be leaf nodes of computational graph"};
        }
    }

    std::vector<Array> outputs = func(inputs);

    for (size_t i = 0; i < inputs.size(); ++i) {
        for (size_t j = 0; j < outputs.size(); ++j) {
            if (inputs[i].body() == outputs[j].body() && inputs[i].IsGradRequired(graph_id)) {
                throw GradientCheckError{"BackwardGradients: Input ", i, " and output ", j, " of the forward function are identical."};
            }
        }
    }

    if (grad_outputs) {
        const std::size_t nout = outputs.size();
        if (nout != grad_outputs->size()) {
            throw GradientCheckError{"BackwardGradients: Size of function outputs: ",
                                     nout,
                                     " and size of grad outputs: ",
                                     grad_outputs->size(),
                                     " must be same"};
        }

        for (std::size_t i = 0; i < nout; ++i) {
            if (outputs[i].IsGradRequired(graph_id)) {
                outputs[i].SetGrad((*grad_outputs)[i], graph_id);
            }
        }
    }

    // Clear gradients which may exist if func calls backward inside of itself.
    for (Array& input : inputs) {
        if (input.IsGradRequired(graph_id)) {
            input.ClearGrad(graph_id);
        }
    }

    std::vector<ConstArrayRef> outputs_ref{outputs.begin(), outputs.end()};
    std::vector<ConstArrayRef> outputs_requiring_grad;
    std::copy_if(outputs_ref.begin(), outputs_ref.end(), std::back_inserter(outputs_requiring_grad), [&graph_id](const Array& a) {
        return a.IsGradRequired(graph_id);
    });
    Backward(outputs_requiring_grad, graph_id, double_backprop);

    std::vector<nonstd::optional<Array>> backward_grads;
    std::transform(
            inputs.begin(), inputs.end(), std::back_inserter(backward_grads), [&graph_id](const Array& input) -> nonstd::optional<Array> {
                if (!input.IsGradRequired(graph_id)) {
                    return nonstd::nullopt;
                }
                return input.GetGrad(graph_id);
            });

    return backward_grads;
}

void CheckDoubleBackpropOption(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const GraphId& graph_id) {
    std::ostringstream failure_os;

    // make it nonlinear to be double differentiable so that this utility can be used even for non double differentiable functions
    auto nonlinear_func = [&func](const std::vector<Array>& inputs) {
        std::vector<Array> nonlinear_outputs;
        for (const auto& output : func(inputs)) {
            nonlinear_outputs.emplace_back(output * output);
        }
        return nonlinear_outputs;
    };

    // Disable double backprop
    {
        std::vector<Array> inputs_copy{inputs};
        std::vector<nonstd::optional<Array>> grads =
                BackwardGradients(nonlinear_func, inputs_copy, nonstd::nullopt, graph_id, DoubleBackpropOption::kDisable);

        for (size_t i = 0; i < grads.size(); ++i) {
            if (grads[i]) {
                if (grads[i]->IsGradRequired(graph_id)) {
                    failure_os << "Gradient " << i << " / " << grads.size() << " is connected to the graph '" << graph_id
                               << "' even when double-backprop is disabled.";
                }
            }
        }
    }

    // Enable double backprop
    {
        std::vector<Array> inputs_copy{inputs};
        std::vector<nonstd::optional<Array>> grads =
                BackwardGradients(nonlinear_func, inputs_copy, nonstd::nullopt, graph_id, DoubleBackpropOption::kEnable);

        for (size_t i = 0; i < grads.size(); ++i) {
            if (grads[i]) {
                if (!grads[i]->IsGradRequired(graph_id)) {
                    failure_os << "Gradient " << i << " / " << grads.size() << " is not connected to the graph '" << graph_id
                               << "' even when double-backprop is enabled.";
                }
            }
        }
    }

    // Do nothing unless failure
    std::string failure_message = failure_os.str();
    if (!failure_message.empty()) {
        throw GradientCheckError{failure_message};
    }
}

void CheckBackwardComputation(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    std::vector<Array> inputs_copy{inputs};

    // Compute backward gradients
    const std::vector<nonstd::optional<Array>> backward_grads = BackwardGradients(func, inputs_copy, grad_outputs, graph_id);
    if (backward_grads.size() != inputs.size()) {
        throw GradientCheckError{"Number of input gradients does not match the input arrays."};
    }
    for (size_t i = 0; i < inputs.size(); ++i) {
        if (!backward_grads[i].has_value()) {
            continue;
        }
        const Array& backward_grad = *backward_grads[i];
        if (backward_grad.shape() != inputs[i].shape()) {
            throw GradientCheckError{"Shape of input gradient ",
                                     i,
                                     " of ",
                                     inputs.size(),
                                     " ",
                                     backward_grad.shape(),
                                     " does not match the corresponding input shape ",
                                     inputs[i].shape(),
                                     "."};
        }
        if (backward_grad.dtype() != inputs[i].dtype()) {
            throw GradientCheckError{"Dtype of input gradient ",
                                     i,
                                     " of ",
                                     inputs.size(),
                                     " ",
                                     GetDtypeName(backward_grad.dtype()),
                                     " does not match the corresponding input dtype ",
                                     GetDtypeName(inputs[i].dtype()),
                                     "."};
        }
    }

    // Compute numerical gradients
    const std::vector<Array> numerical_grads = CalculateNumericalGradient(func, inputs, grad_outputs, eps, graph_id);

    // If you're trapped in any of these asserts, numerical gradiends must be implemented incorrectly.
    assert(numerical_grads.size() == inputs.size());
    for (size_t i = 0; i < inputs.size(); ++i) {
        assert(numerical_grads[i].shape() == inputs[i].shape());
        assert(numerical_grads[i].dtype() == inputs[i].dtype());
    }

    // Check consistency between backward gradients and numeric gradients.
    std::ostringstream failure_os;
    for (size_t i = 0; i < inputs.size(); ++i) {
        if (!backward_grads[i].has_value()) {
            continue;
        }

        const Array& backward_grad = *backward_grads[i];
        const Array& numerical_grad = numerical_grads[i];
        if (!AllClose(backward_grad, numerical_grad, atol, rtol)) {
            failure_os << "Backward check failure on input " << i << " (Total inputs: " << inputs.size() << ")\n"
                       << "Graph name: " << graph_id << "\n"
                       << "Atol: " << atol << "\n"
                       << "Rtol: " << rtol << "\n"
                       << "Error:\n"
                       << backward_grad - numerical_grad << "\n"  // TODO(niboshi): Use abs
                       << "Backward gradients:\n"
                       << backward_grad << "\n"
                       << "Numerical gradients:\n"
                       << numerical_grad << "\n"
                       << "Eps (perturbation in numerical gradients):\n"
                       << eps[i];
        }
    }

    // Do nothing if all backward-numerical gradient pairs were close, else generate a nonfatal failure
    std::string failure_message = failure_os.str();
    if (!failure_message.empty()) {
        throw GradientCheckError{failure_message};
    }
}

}  // namespace

void CheckBackward(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    CheckDoubleBackpropOption(func, inputs, graph_id);
    CheckBackwardComputation(func, inputs, grad_outputs, eps, atol, rtol, graph_id);
}

void CheckDoubleBackwardComputation(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& grad_grad_inputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    const std::size_t nin = inputs.size();
    const std::size_t nout = grad_outputs.size();

    if (grad_grad_inputs.size() != nin) {
        throw XchainerError{"Number of input arrays and grad_grad_input arrays do not match."};
    }

    // LIMITATION: All inputs must require gradients unlike CheckBackwardComputation

    // Check all the input arrays require gradients
    for (size_t i = 0; i < nin; ++i) {
        if (!inputs[i].IsGradRequired(graph_id)) {
            throw XchainerError{"Input array ", i, " / ", nin, " is constant w.r.t. the graph '", graph_id, "'."};
        }
    }

    // Check all the output gradient arrays require gradients
    for (size_t i = 0; i < nout; ++i) {
        if (!grad_outputs[i].IsGradRequired(graph_id)) {
            throw XchainerError{"Output gradient array ", i, " / ", nout, " is constant w.r.t. the graph '", graph_id, "'."};
        }
    }

    // The "forward" function to return the first order gradients

    auto first_order_grad_func = [&func, nin, nout, &graph_id](const std::vector<Array>& inputs_and_grad_outputs) {
        // Just revert (split) inputs_and_grad_outputs into inputs and grad_outputs
        std::vector<Array> inputs{inputs_and_grad_outputs.begin(), inputs_and_grad_outputs.begin() + nin};
        std::vector<Array> grad_outputs{inputs_and_grad_outputs.begin() + nin, inputs_and_grad_outputs.end()};

        // Compute first order gradients
        std::vector<nonstd::optional<Array>> optional_backward_grads = BackwardGradients(func, inputs, grad_outputs, graph_id);

        // Check all the first order gradients are computed
        if (optional_backward_grads.size() != nin) {
            throw GradientCheckError{"Number of first-order input gradients arrays ",
                                     optional_backward_grads.size(),
                                     " do not match the number of input arrays ",
                                     nin,
                                     "."};
        }

        for (size_t i = 0; i < nin; ++i) {
            const nonstd::optional<Array>& opt_backward_grad = optional_backward_grads[i];
            if (!opt_backward_grad.has_value()) {
                throw GradientCheckError{"First-order input gradient ", i, " / ", nin, " do not exist."};
            }
        }

        // Check all the first order gradients are differentiable
        for (size_t i = 0; i < nin; ++i) {
            if (!optional_backward_grads[i]->IsGradRequired(graph_id)) {
                throw GradientCheckError{"Input gradient ", i, " is not differentiable w.r.t. the graph '", graph_id, "'."};
            }
        }

        // Convert to std::vector<Array>
        std::vector<Array> backward_grads;
        std::transform(
                optional_backward_grads.begin(),
                optional_backward_grads.end(),
                std::back_inserter(backward_grads),
                [](const nonstd::optional<Array>& optional_backward_grad) { return *optional_backward_grad; });

        assert(backward_grads.size() == nin);
        return backward_grads;
    };

    // Prepare for computing numerical and backward gradients.
    // Merge inputs and grad_outputs into inputs_and_grad_outputs.
    std::vector<Array> inputs_and_grad_outputs;
    inputs_and_grad_outputs.reserve(nin + nout);
    std::copy(inputs.begin(), inputs.end(), std::back_inserter(inputs_and_grad_outputs));
    std::copy(grad_outputs.begin(), grad_outputs.end(), std::back_inserter(inputs_and_grad_outputs));

    // Compute second order numerial gradients w.r.t. the first-order gradients.
    const std::vector<Array> numerical_grads =
            CalculateNumericalGradient(first_order_grad_func, inputs_and_grad_outputs, grad_grad_inputs, eps, graph_id);
    assert(numerical_grads.size() == nin + nout);

    // Compute second order backward gradients w.r.t. the first-order gradients.
    const std::vector<nonstd::optional<Array>> backward_grads =
            BackwardGradients(first_order_grad_func, inputs_and_grad_outputs, grad_grad_inputs, graph_id);
    assert(backward_grads.size() == nin + nout);

    std::ostringstream failure_os;
    bool has_error = false;

    // Check if all the second order gradients exist
    for (size_t i = 0; i < backward_grads.size(); ++i) {
        if (!backward_grads[i].has_value()) {
            failure_os << "Second order gradient w.r.t. the input gradient " << i << " (Total inputs: " << inputs.size()
                       << ", outputs: " << nout << ") is missing on the graph '" << graph_id << "'. "
                       << "Maybe you need additional nonlinearity in the target function.";
            has_error = true;
        }
    }
    if (has_error) {
        throw GradientCheckError{failure_os.str()};
    }

    // Check numerical consistency between numerical and backward gradients.
    for (size_t i = 0; i < backward_grads.size(); ++i) {
        if (!AllClose(*backward_grads[i], numerical_grads[i], atol, rtol)) {
            failure_os << "Backward check failure on input " << i << " (Total inputs: " << inputs.size() << ")\n"
                       << "Graph name: " << graph_id << "\n"
                       << "Atol: " << atol << "\n"
                       << "Rtol: " << rtol << "\n"
                       << "Error:\n"
                       << *backward_grads[i] - numerical_grads[i] << "\n"  // TODO(niboshi): Use abs
                       << "Backward gradients:\n"
                       << *backward_grads[i] << "\n"
                       << "Numerical gradients:\n"
                       << numerical_grads[i] << "\n"
                       << "Eps (perturbation in numerical gradients):\n"
                       << eps[i];
            has_error = true;
        }
    }
    if (has_error) {
        throw GradientCheckError{failure_os.str()};
    }
}

}  // namespace xchainer
<commit_msg>Improve check backward error<commit_after>#include "xchainer/check_backward.h"

#include <algorithm>
#include <functional>
#include <memory>
#include <string>
#include <vector>

#include "xchainer/array.h"
#include "xchainer/array_node.h"
#include "xchainer/backward.h"
#include "xchainer/error.h"
#include "xchainer/numeric.h"
#include "xchainer/numerical_gradient.h"

namespace xchainer {
namespace {

std::vector<nonstd::optional<Array>> BackwardGradients(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        std::vector<Array>& inputs,
        const nonstd::optional<std::vector<Array>>& grad_outputs,
        const GraphId& graph_id,
        DoubleBackpropOption double_backprop = DoubleBackpropOption::kEnable) {
    for (const auto& input : inputs) {
        if (internal::HasArrayNode(input, graph_id) && internal::GetArrayNode(input, graph_id)->next_node()) {
            throw GradientCheckError{"BackwardGradients: All inputs must be leaf nodes of computational graph"};
        }
    }

    std::vector<Array> outputs = func(inputs);

    for (size_t i = 0; i < inputs.size(); ++i) {
        for (size_t j = 0; j < outputs.size(); ++j) {
            if (inputs[i].body() == outputs[j].body() && inputs[i].IsGradRequired(graph_id)) {
                throw GradientCheckError{"BackwardGradients: Input ", i, " and output ", j, " of the forward function are identical."};
            }
        }
    }

    if (grad_outputs) {
        const std::size_t nout = outputs.size();
        if (nout != grad_outputs->size()) {
            throw GradientCheckError{"BackwardGradients: Size of function outputs: ",
                                     nout,
                                     " and size of grad outputs: ",
                                     grad_outputs->size(),
                                     " must be same"};
        }

        for (std::size_t i = 0; i < nout; ++i) {
            if (outputs[i].IsGradRequired(graph_id)) {
                outputs[i].SetGrad((*grad_outputs)[i], graph_id);
            }
        }
    }

    // Clear gradients which may exist if func calls backward inside of itself.
    for (Array& input : inputs) {
        if (input.IsGradRequired(graph_id)) {
            input.ClearGrad(graph_id);
        }
    }

    std::vector<ConstArrayRef> outputs_ref{outputs.begin(), outputs.end()};
    std::vector<ConstArrayRef> outputs_requiring_grad;
    std::copy_if(outputs_ref.begin(), outputs_ref.end(), std::back_inserter(outputs_requiring_grad), [&graph_id](const Array& a) {
        return a.IsGradRequired(graph_id);
    });
    Backward(outputs_requiring_grad, graph_id, double_backprop);

    std::vector<nonstd::optional<Array>> backward_grads;
    std::transform(
            inputs.begin(), inputs.end(), std::back_inserter(backward_grads), [&graph_id](const Array& input) -> nonstd::optional<Array> {
                if (!input.IsGradRequired(graph_id)) {
                    return nonstd::nullopt;
                }
                return input.GetGrad(graph_id);
            });

    return backward_grads;
}

void CheckDoubleBackpropOption(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const GraphId& graph_id) {
    std::ostringstream failure_os;

    // make it nonlinear to be double differentiable so that this utility can be used even for non double differentiable functions
    auto nonlinear_func = [&func](const std::vector<Array>& inputs) {
        std::vector<Array> nonlinear_outputs;
        for (const auto& output : func(inputs)) {
            nonlinear_outputs.emplace_back(output * output);
        }
        return nonlinear_outputs;
    };

    // Disable double backprop
    {
        std::vector<Array> inputs_copy{inputs};
        std::vector<nonstd::optional<Array>> grads =
                BackwardGradients(nonlinear_func, inputs_copy, nonstd::nullopt, graph_id, DoubleBackpropOption::kDisable);

        for (size_t i = 0; i < grads.size(); ++i) {
            if (grads[i]) {
                if (grads[i]->IsGradRequired(graph_id)) {
                    failure_os << "Gradient " << i << " / " << grads.size() << " is connected to the graph '" << graph_id
                               << "' even when double-backprop is disabled.";
                }
            }
        }
    }

    // Enable double backprop
    {
        std::vector<Array> inputs_copy{inputs};
        std::vector<nonstd::optional<Array>> grads =
                BackwardGradients(nonlinear_func, inputs_copy, nonstd::nullopt, graph_id, DoubleBackpropOption::kEnable);

        for (size_t i = 0; i < grads.size(); ++i) {
            if (grads[i]) {
                if (!grads[i]->IsGradRequired(graph_id)) {
                    failure_os << "Gradient " << i << " / " << grads.size() << " is not connected to the graph '" << graph_id
                               << "' even when double-backprop is enabled.";
                }
            }
        }
    }

    // Do nothing unless failure
    std::string failure_message = failure_os.str();
    if (!failure_message.empty()) {
        throw GradientCheckError{failure_message};
    }
}

void CheckBackwardComputation(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    std::vector<Array> inputs_copy{inputs};

    // Compute backward gradients
    const std::vector<nonstd::optional<Array>> backward_grads = BackwardGradients(func, inputs_copy, grad_outputs, graph_id);
    if (backward_grads.size() != inputs.size()) {
        throw GradientCheckError{"Number of input gradients does not match the input arrays."};
    }
    for (size_t i = 0; i < inputs.size(); ++i) {
        if (!backward_grads[i].has_value()) {
            continue;
        }
        const Array& backward_grad = *backward_grads[i];
        if (backward_grad.shape() != inputs[i].shape()) {
            throw GradientCheckError{"Shape of input gradient ",
                                     i,
                                     " of ",
                                     inputs.size(),
                                     " ",
                                     backward_grad.shape(),
                                     " does not match the corresponding input shape ",
                                     inputs[i].shape(),
                                     "."};
        }
        if (backward_grad.dtype() != inputs[i].dtype()) {
            throw GradientCheckError{"Dtype of input gradient ",
                                     i,
                                     " of ",
                                     inputs.size(),
                                     " ",
                                     GetDtypeName(backward_grad.dtype()),
                                     " does not match the corresponding input dtype ",
                                     GetDtypeName(inputs[i].dtype()),
                                     "."};
        }
    }

    // Compute numerical gradients
    const std::vector<Array> numerical_grads = CalculateNumericalGradient(func, inputs, grad_outputs, eps, graph_id);

    // If you're trapped in any of these asserts, numerical gradiends must be implemented incorrectly.
    assert(numerical_grads.size() == inputs.size());
    for (size_t i = 0; i < inputs.size(); ++i) {
        assert(numerical_grads[i].shape() == inputs[i].shape());
        assert(numerical_grads[i].dtype() == inputs[i].dtype());
    }

    // Check numerical consistency between numerical and backward gradients.
    std::vector<size_t> failed_input_indices;
    for (size_t i = 0; i < backward_grads.size(); ++i) {
        if (!backward_grads[i].has_value()) {
            continue;
        }
        if (!AllClose(*backward_grads[i], numerical_grads[i], atol, rtol)) {
            failed_input_indices.emplace_back(i);
        }
    }
    if (!failed_input_indices.empty()) {
        std::ostringstream os;
        os << "Numerical error in backward on inputs (out of " << inputs.size() << "): ";
        for (size_t j = 0; j < failed_input_indices.size(); ++j) {
            size_t i = failed_input_indices[j];
            if (i != 0) {
                os << ", ";
            }
            os << i;
        }
        os << std::endl;
        os << "Graph: " << graph_id << std::endl;
        os << "Atol: " << atol << "  Rtol: " << rtol << std::endl;
        for (size_t i : failed_input_indices) {
            os << "Error[" << i << "]:" << std::endl
               << *backward_grads[i] - numerical_grads[i] << std::endl  // TODO(niboshi): Use abs
               << "Backward gradients[" << i << "]:" << std::endl
               << *backward_grads[i] << std::endl
               << "Numerical gradients[" << i << "]:" << std ::endl
               << numerical_grads[i] << std::endl
               << "Eps[" << i << "] (perturbation in numerical gradients):" << std::endl
               << eps[i] << std::endl;
        }
        throw GradientCheckError{os.str()};
    }
}

}  // namespace

void CheckBackward(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    CheckDoubleBackpropOption(func, inputs, graph_id);
    CheckBackwardComputation(func, inputs, grad_outputs, eps, atol, rtol, graph_id);
}

void CheckDoubleBackwardComputation(
        const std::function<std::vector<Array>(const std::vector<Array>&)>& func,
        const std::vector<Array>& inputs,
        const std::vector<Array>& grad_outputs,
        const std::vector<Array>& grad_grad_inputs,
        const std::vector<Array>& eps,
        double atol,
        double rtol,
        const GraphId& graph_id) {
    const std::size_t nin = inputs.size();
    const std::size_t nout = grad_outputs.size();

    if (grad_grad_inputs.size() != nin) {
        throw XchainerError{"Number of input arrays and grad_grad_input arrays do not match."};
    }

    // LIMITATION: All inputs must require gradients unlike CheckBackwardComputation

    // Check all the input arrays require gradients
    for (size_t i = 0; i < nin; ++i) {
        if (!inputs[i].IsGradRequired(graph_id)) {
            throw XchainerError{"Input array ", i, " / ", nin, " is constant w.r.t. the graph '", graph_id, "'."};
        }
    }

    // Check all the output gradient arrays require gradients
    for (size_t i = 0; i < nout; ++i) {
        if (!grad_outputs[i].IsGradRequired(graph_id)) {
            throw XchainerError{"Output gradient array ", i, " / ", nout, " is constant w.r.t. the graph '", graph_id, "'."};
        }
    }

    // The "forward" function to return the first order gradients

    auto first_order_grad_func = [&func, nin, nout, &graph_id](const std::vector<Array>& inputs_and_grad_outputs) {
        // Just revert (split) inputs_and_grad_outputs into inputs and grad_outputs
        std::vector<Array> inputs{inputs_and_grad_outputs.begin(), inputs_and_grad_outputs.begin() + nin};
        std::vector<Array> grad_outputs{inputs_and_grad_outputs.begin() + nin, inputs_and_grad_outputs.end()};

        // Compute first order gradients
        std::vector<nonstd::optional<Array>> optional_backward_grads = BackwardGradients(func, inputs, grad_outputs, graph_id);

        // Check all the first order gradients are computed
        if (optional_backward_grads.size() != nin) {
            throw GradientCheckError{"Number of first-order input gradients arrays ",
                                     optional_backward_grads.size(),
                                     " do not match the number of input arrays ",
                                     nin,
                                     "."};
        }

        for (size_t i = 0; i < nin; ++i) {
            const nonstd::optional<Array>& opt_backward_grad = optional_backward_grads[i];
            if (!opt_backward_grad.has_value()) {
                throw GradientCheckError{"First-order input gradient ", i, " / ", nin, " do not exist."};
            }
        }

        // Check all the first order gradients are differentiable
        for (size_t i = 0; i < nin; ++i) {
            if (!optional_backward_grads[i]->IsGradRequired(graph_id)) {
                throw GradientCheckError{"Input gradient ", i, " is not differentiable w.r.t. the graph '", graph_id, "'."};
            }
        }

        // Convert to std::vector<Array>
        std::vector<Array> backward_grads;
        std::transform(
                optional_backward_grads.begin(),
                optional_backward_grads.end(),
                std::back_inserter(backward_grads),
                [](const nonstd::optional<Array>& optional_backward_grad) { return *optional_backward_grad; });

        assert(backward_grads.size() == nin);
        return backward_grads;
    };

    // Prepare for computing numerical and backward gradients.
    // Merge inputs and grad_outputs into inputs_and_grad_outputs.
    std::vector<Array> inputs_and_grad_outputs;
    inputs_and_grad_outputs.reserve(nin + nout);
    std::copy(inputs.begin(), inputs.end(), std::back_inserter(inputs_and_grad_outputs));
    std::copy(grad_outputs.begin(), grad_outputs.end(), std::back_inserter(inputs_and_grad_outputs));

    // Compute second order numerial gradients w.r.t. the first-order gradients.
    const std::vector<Array> numerical_grads =
            CalculateNumericalGradient(first_order_grad_func, inputs_and_grad_outputs, grad_grad_inputs, eps, graph_id);
    assert(numerical_grads.size() == nin + nout);

    // Compute second order backward gradients w.r.t. the first-order gradients.
    const std::vector<nonstd::optional<Array>> backward_grads =
            BackwardGradients(first_order_grad_func, inputs_and_grad_outputs, grad_grad_inputs, graph_id);
    assert(backward_grads.size() == nin + nout);

    // Check if all the second order gradients exist
    {
        std::ostringstream os;
        bool has_error = false;
        for (size_t i = 0; i < backward_grads.size(); ++i) {
            if (!backward_grads[i].has_value()) {
                os << "Second order gradient w.r.t. the input gradient " << i << " (Total inputs: " << inputs.size()
                   << ", outputs: " << nout << ") is missing on the graph '" << graph_id << "'. "
                   << "Maybe you need additional nonlinearity in the target function.";
                has_error = true;
            }
        }
        if (has_error) {
            throw GradientCheckError{os.str()};
        }
    }

    // Check numerical consistency between numerical and backward gradients.
    std::vector<size_t> failed_input_indices;
    for (size_t i = 0; i < backward_grads.size(); ++i) {
        if (!AllClose(*backward_grads[i], numerical_grads[i], atol, rtol)) {
            failed_input_indices.emplace_back(i);
        }
    }

    if (!failed_input_indices.empty()) {
        std::ostringstream os;
        os << "Numerical error in double backward on inputs (out of " << inputs.size() << "): ";
        for (size_t j = 0; j < failed_input_indices.size(); ++j) {
            size_t i = failed_input_indices[j];
            if (i != 0) {
                os << ", ";
            }
            os << i;
        }
        os << std::endl;
        os << "Graph: " << graph_id << std::endl;
        os << "Atol: " << atol << "  Rtol: " << rtol << std::endl;
        for (size_t i : failed_input_indices) {
            os << "Error[" << i << "]:" << std::endl
               << *backward_grads[i] - numerical_grads[i] << std::endl  // TODO(niboshi): Use abs
               << "Backward gradients[" << i << "]:" << std::endl
               << *backward_grads[i] << std::endl
               << "Numerical gradients[" << i << "]:" << std ::endl
               << numerical_grads[i] << std::endl
               << "Eps[" << i << "] (perturbation in numerical gradients):" << std::endl
               << eps[i] << std::endl;
        }
        throw GradientCheckError{os.str()};
    }
}

}  // namespace xchainer
<|endoftext|>


<commit_before>
#include "Animation.h"
#include "led_sequence.h"
#include "app_error.h"
#include "app_timer.h"
#include "common_defines.h"
#include "haptics_driver.h"
#include "jewelbot_gpio.h"
#include "led_driver.h"
#include "nordic_common.h"
#include "nrf.h"
#include "nrf_delay.h"
#include "nrf_gpio.h"
#include "nrf_log.h"
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include "Arduino.h"

#ifdef __cplusplus
extern "C"{
#endif // __cplusplus

  Animation::Animation()
  {
    //Init or enable code here causes the Jewelbot to lock up on reset
  }

  Animation::~Animation()
  {}

  Timer animation_timer;

  void Animation::breathe_single_color(ColorLabel color){
    enable_led();
    clear_led();
    // enable_led();
    led_cmd_t led1 = {3, 0x00, 0x00, 0x00, 0};
    led_cmd_t led2 = {1, 0x00, 0x00, 0x00, 0};
    led_cmd_t led3 = {2, 0x00, 0x00, 0x00, 0};
    led_cmd_t led4 = {0, 0x00, 0x00, 0x00, 0};
    for (uint8_t i = 0; i < 31; i++) {
      if (color == RED) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
      }
      if (color == GREEN) {
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
      }
      if (color == BLUE) {
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == YELLOW) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
      }
      if (color == MAGENTA) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == CYAN) {
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == WHITE) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      set_led_state_handler(&led1);
      set_led_state_handler(&led2);
      set_led_state_handler(&led3);
      set_led_state_handler(&led4);
      animation_timer.pause(30);
    }
    for (uint8_t i = 31; i > 0; i--) {
      if (color == RED) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
      }
      if (color == GREEN) {
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
      }
      if (color == BLUE) {
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == YELLOW) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
      }
      if (color == MAGENTA) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == CYAN) {
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == WHITE) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      set_led_state_handler(&led1);
      set_led_state_handler(&led2);
      set_led_state_handler(&led3);
      set_led_state_handler(&led4);
      animation_timer.pause(30);
    }
    animation_timer.pause(10);
    clear_led();
    disable_led();
  }

  void Animation::jewelbots_logo(void){

  enable_led();
  clear_led();
  // Fade in the logo colors quickly
  led_cmd_t led1 = {3, 0x00, 0x00, 0x00, 0};
  led_cmd_t led2 = {1, 0x00, 0x00, 0x00, 0};
  led_cmd_t led3 = {2, 0x00, 0x00, 0x00, 0};
  led_cmd_t led4 = {0, 0x00, 0x00, 0x00, 0};
  for (uint8_t i = 0; i < 12; i++) {
    led1.r = led1.r + 5;
    led2.r = led2.r + 0;
    led3.r = led3.r + 0;
    led4.r = led4.r + 0;
    led1.g = led1.g + 0;
    led2.g = led2.g + 2;
    led3.g = led3.g + 2;
    led4.g = led4.g + 2;
    led1.b = led1.b + 0;
    led2.b = led2.b + 1;
    led3.b = led3.b + 1;
    led4.b = led4.b + 1;

    set_led_state_handler(&led1);
    set_led_state_handler(&led2);
    set_led_state_handler(&led3);
    set_led_state_handler(&led4);
    animation_timer.pause(40);
  }

  //  Rotate the logo colors around the face
  led_cmd_t red_hold = {3, 0x3F, 0x00, 0x00, 0};
  led_cmd_t teal_hold1 = {1, 0x00, 0x18, 0x0C, 0};
  led_cmd_t teal_hold2 = {2, 0x00, 0x18, 0x0C, 0};
  led_cmd_t teal_hold3 = {0, 0x00, 0x18, 0x0C, 0};
  set_led_state_handler(&red_hold);
  set_led_state_handler(&teal_hold1);
  set_led_state_handler(&teal_hold2);
  set_led_state_handler(&teal_hold3);

  uint8_t red_led[] = {3, 0, 1, 2};
  uint8_t teal_led[] = {2, 3, 0, 1};
  for (uint8_t i = 0; i < 6; i++) {
    for (uint8_t j = 0; j < 4; j++) {
      led_cmd_t red_ch = {red_led[j], 0x3F, 0x00, 0x00, 0};
      led_cmd_t teal_ch = {teal_led[j], 0x00, 0x24, 0x18, 0};
      set_led_state_handler(&red_ch);
      set_led_state_handler(&teal_ch);
      animation_timer.pause(80);
    }
  }

  //  quickly fade out the colors
  for (uint8_t i = 12; i > 0; i--) {
    led1.r = led1.r - 5;
    led2.r = led2.r - 0;
    led3.r = led3.r - 0;
    led4.r = led4.r - 0;
    led1.g = led1.g - 0;
    led2.g = led2.g - 2;
    led3.g = led3.g - 2;
    led4.g = led4.g - 2;
    led1.b = led1.b - 0;
    led2.b = led2.b - 1;
    led3.b = led3.b - 1;
    led4.b = led4.b - 1;

    set_led_state_handler(&led1);
    set_led_state_handler(&led2);
    set_led_state_handler(&led3);
    set_led_state_handler(&led4);
    animation_timer.pause(40);
  }
  animation_timer.pause(10);
  clear_led();
  disable_led();
}

void Animation::rainbows(void){

    enable_led();
    clear_led();
    led_cmd_t red = {0, 0x3F, 0x00, 0x00, 1};
    led_cmd_t orange = {1, 0x3F, 0x1C, 0x00, 1};
    led_cmd_t yellow = {2, 0x3F, 0x3F, 0x00, 1};
    led_cmd_t green = {3, 0x00, 0x3F, 0x00, 1};
    led_cmd_t blue = {0, 0x00, 0x00, 0x3F, 1};
    led_cmd_t violet = {1, 0x1F, 0x00, 0x3F, 1};
    led_cmd_t pink = {2, 0x3F, 0x1A, 0x3A, 1};
    led_cmd_t magenta = {3, 0x3F, 0x0C, 0x3F, 1};

    int delay = 100;
    for (int i = 0; i < 4; i++) {
      set_led_state_handler(&red);
      animation_timer.pause(delay);
      set_led_state_handler(&orange);
      animation_timer.pause(delay);
      set_led_state_handler(&yellow);
      animation_timer.pause(delay);
      set_led_state_handler(&green);
      animation_timer.pause(delay);
      set_led_state_handler(&blue);
      animation_timer.pause(delay);
      set_led_state_handler(&violet);
      animation_timer.pause(delay);
      set_led_state_handler(&pink);
      animation_timer.pause(delay);
      set_led_state_handler(&magenta);
      animation_timer.pause(delay);
    }
    clear_led();
    disable_led();
}
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
<commit_msg>Update Animation.cpp<commit_after>
#include "Animation.h"
#include "led_sequence.h"
#include "app_error.h"
#include "app_timer.h"
#include "common_defines.h"
#include "haptics_driver.h"
#include "jewelbot_gpio.h"
#include "led_driver.h"
#include "nordic_common.h"
#include "nrf.h"
#include "nrf_delay.h"
#include "nrf_gpio.h"
#include "nrf_log.h"
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include "Arduino.h"
#include "JWB_Utils.h"

#ifdef __cplusplus
extern "C"{
#endif // __cplusplus

  Animation::Animation()
  {
    //Init or enable code here causes the Jewelbot to lock up on reset
  }

  Animation::~Animation()
  {}

  Timer animation_timer;

  void Animation::breathe_single_color(ColorLabel color){
    enable_led();
    clear_led();
    // enable_led();
    led_cmd_t led1 = {3, 0x00, 0x00, 0x00, 0};
    led_cmd_t led2 = {1, 0x00, 0x00, 0x00, 0};
    led_cmd_t led3 = {2, 0x00, 0x00, 0x00, 0};
    led_cmd_t led4 = {0, 0x00, 0x00, 0x00, 0};
    for (uint8_t i = 0; i < 31; i++) {
      if (color == RED) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
      }
      if (color == GREEN) {
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
      }
      if (color == BLUE) {
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == YELLOW) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
      }
      if (color == MAGENTA) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == CYAN) {
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      if (color == WHITE) {
        led1.r = led1.r + 1;
        led2.r = led2.r + 1;
        led3.r = led3.r + 1;
        led4.r = led4.r + 1;
        led1.g = led1.g + 1;
        led2.g = led2.g + 1;
        led3.g = led3.g + 1;
        led4.g = led4.g + 1;
        led1.b = led1.b + 1;
        led2.b = led2.b + 1;
        led3.b = led3.b + 1;
        led4.b = led4.b + 1;
      }
      set_led_state_handler(&led1);
      set_led_state_handler(&led2);
      set_led_state_handler(&led3);
      set_led_state_handler(&led4);
      animation_timer.pause(30);
    }
    for (uint8_t i = 31; i > 0; i--) {
      if (color == RED) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
      }
      if (color == GREEN) {
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
      }
      if (color == BLUE) {
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == YELLOW) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
      }
      if (color == MAGENTA) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == CYAN) {
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      if (color == WHITE) {
        led1.r = led1.r - 1;
        led2.r = led2.r - 1;
        led3.r = led3.r - 1;
        led4.r = led4.r - 1;
        led1.g = led1.g - 1;
        led2.g = led2.g - 1;
        led3.g = led3.g - 1;
        led4.g = led4.g - 1;
        led1.b = led1.b - 1;
        led2.b = led2.b - 1;
        led3.b = led3.b - 1;
        led4.b = led4.b - 1;
      }
      set_led_state_handler(&led1);
      set_led_state_handler(&led2);
      set_led_state_handler(&led3);
      set_led_state_handler(&led4);
      animation_timer.pause(30);
    }
    animation_timer.pause(10);
    clear_led();
    disable_led();
  }

  void Animation::jewelbots_logo(void){

  enable_led();
  clear_led();
  // Fade in the logo colors quickly
  led_cmd_t led1 = {3, 0x00, 0x00, 0x00, 0};
  led_cmd_t led2 = {1, 0x00, 0x00, 0x00, 0};
  led_cmd_t led3 = {2, 0x00, 0x00, 0x00, 0};
  led_cmd_t led4 = {0, 0x00, 0x00, 0x00, 0};
  for (uint8_t i = 0; i < 12; i++) {
    led1.r = led1.r + 5;
    led2.r = led2.r + 0;
    led3.r = led3.r + 0;
    led4.r = led4.r + 0;
    led1.g = led1.g + 0;
    led2.g = led2.g + 2;
    led3.g = led3.g + 2;
    led4.g = led4.g + 2;
    led1.b = led1.b + 0;
    led2.b = led2.b + 1;
    led3.b = led3.b + 1;
    led4.b = led4.b + 1;

    set_led_state_handler(&led1);
    set_led_state_handler(&led2);
    set_led_state_handler(&led3);
    set_led_state_handler(&led4);
    animation_timer.pause(40);
  }

  //  Rotate the logo colors around the face
  led_cmd_t red_hold = {3, 0x3F, 0x00, 0x00, 0};
  led_cmd_t teal_hold1 = {1, 0x00, 0x18, 0x0C, 0};
  led_cmd_t teal_hold2 = {2, 0x00, 0x18, 0x0C, 0};
  led_cmd_t teal_hold3 = {0, 0x00, 0x18, 0x0C, 0};
  set_led_state_handler(&red_hold);
  set_led_state_handler(&teal_hold1);
  set_led_state_handler(&teal_hold2);
  set_led_state_handler(&teal_hold3);

  uint8_t red_led[] = {3, 0, 1, 2};
  uint8_t teal_led[] = {2, 3, 0, 1};
  for (uint8_t i = 0; i < 6; i++) {
    for (uint8_t j = 0; j < 4; j++) {
      led_cmd_t red_ch = {red_led[j], 0x3F, 0x00, 0x00, 0};
      led_cmd_t teal_ch = {teal_led[j], 0x00, 0x24, 0x18, 0};
      set_led_state_handler(&red_ch);
      set_led_state_handler(&teal_ch);
      animation_timer.pause(80);
    }
  }

  //  quickly fade out the colors
  for (uint8_t i = 12; i > 0; i--) {
    led1.r = led1.r - 5;
    led2.r = led2.r - 0;
    led3.r = led3.r - 0;
    led4.r = led4.r - 0;
    led1.g = led1.g - 0;
    led2.g = led2.g - 2;
    led3.g = led3.g - 2;
    led4.g = led4.g - 2;
    led1.b = led1.b - 0;
    led2.b = led2.b - 1;
    led3.b = led3.b - 1;
    led4.b = led4.b - 1;

    set_led_state_handler(&led1);
    set_led_state_handler(&led2);
    set_led_state_handler(&led3);
    set_led_state_handler(&led4);
    animation_timer.pause(40);
  }
  animation_timer.pause(10);
  clear_led();
  disable_led();
}

void Animation::rainbows(void){

    enable_led();
    clear_led();
    led_cmd_t red = {0, 0x3F, 0x00, 0x00, 1};
    led_cmd_t orange = {1, 0x3F, 0x1C, 0x00, 1};
    led_cmd_t yellow = {2, 0x3F, 0x3F, 0x00, 1};
    led_cmd_t green = {3, 0x00, 0x3F, 0x00, 1};
    led_cmd_t blue = {0, 0x00, 0x00, 0x3F, 1};
    led_cmd_t violet = {1, 0x1F, 0x00, 0x3F, 1};
    led_cmd_t pink = {2, 0x3F, 0x1A, 0x3A, 1};
    led_cmd_t magenta = {3, 0x3F, 0x0C, 0x3F, 1};

    int delay = 100;
    for (int i = 0; i < 4; i++) {
      set_led_state_handler(&red);
      animation_timer.pause(delay);
      set_led_state_handler(&orange);
      animation_timer.pause(delay);
      set_led_state_handler(&yellow);
      animation_timer.pause(delay);
      set_led_state_handler(&green);
      animation_timer.pause(delay);
      set_led_state_handler(&blue);
      animation_timer.pause(delay);
      set_led_state_handler(&violet);
      animation_timer.pause(delay);
      set_led_state_handler(&pink);
      animation_timer.pause(delay);
      set_led_state_handler(&magenta);
      animation_timer.pause(delay);
    }
    clear_led();
    disable_led();
}

void Animation::bouncing_ball(void){

  enable_led();
  clear_led();
  int rand = get_random_int(1,4);
  led_cmd_t led1;
  led_cmd_t led2;
  led_cmd_t led3;
  led_cmd_t led4;
  int delay = 300;

  if(rand == 1){
  led1 = {0, 0x3F, 0x0C, 0x3F, 1};
  led2 = {1, 0x20, 0xB2, 0x50, 1};
  led3 = {2, 0x20, 0xB2, 0x50, 1};
  led4 = {3, 0x20, 0xB2, 0x50, 1};
  }
  if(rand == 2){
  led1 = {1, 0x3F, 0x0C, 0x3F, 1};
  led2 = {0, 0x20, 0xB2, 0x50, 1};
  led3 = {2, 0x20, 0xB2, 0x50, 1};
  led4 = {3, 0x20, 0xB2, 0x50, 1};
  }
  if(rand == 3){
  led1 = {2, 0x3F, 0x0C, 0x3F, 1};
  led2 = {1, 0x20, 0xB2, 0x50, 1};
  led3 = {0, 0x20, 0xB2, 0x50, 1};
  led4 = {3, 0x20, 0xB2, 0x50, 1};
  }
  if(rand == 4){
  led1 = {3, 0x3F, 0x0C, 0x3F, 1};
  led2 = {1, 0x20, 0xB2, 0x50, 1};
  led3 = {2, 0x20, 0xB2, 0x50, 1};
  led4 = {0, 0x20, 0xB2, 0x50, 1};
  }

  set_led_state_handler(&led1);
  set_led_state_handler(&led2);
  set_led_state_handler(&led3);
  set_led_state_handler(&led4);
  animation_timer.pause(delay);

  clear_led();
  disable_led();
}
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
<|endoftext|>


<commit_before>#include <connection.h>
#include <sql_buffer.h>
#include <iostream>
#include <libpq-fe.h>
#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"
#include <string>

using namespace std;
using namespace rapidjson;

/**
 * Create a database connection
 */
Connection::Connection()
{
	const char *conninfo = "dbname = foglamp";

	/* Make a connection to the database */
	dbConnection = PQconnectdb(conninfo);

	/* Check to see that the backend connection was successfully made */
	if (PQstatus(dbConnection) != CONNECTION_OK)
	{
		cerr << "Failed to connect" << endl;
	}
}

/**
 * Destructor for the database connection. Close the connection
 * to Postgres
 */
Connection::~Connection()
{
	PQfinish(dbConnection);
}

/**
 * Perform a query against a common table
 *
 * TODO Improve error handling
 */
bool Connection::retrieve(const string& table, const string& condition, string& resultSet)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;

	sql.append("SELECT * from ");
	sql.append(table);
	if (! condition.empty())
	{
		sql.append(" WHERE ");
		if (document.Parse(condition.c_str()).HasParseError())
		{
			printf("Failed to parse JSON: %s\n", condition.c_str());
		}
		else
		{
			assert(document.IsObject());

			if (document.HasMember("where"))
			{
				jsonWhereClause(document["where"], sql);
			}
			else
			{
				printf("JSON does not contain where clause: %s\n", condition.c_str());
			}
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_TUPLES_OK)
	{
		mapResultSet(res, resultSet);
		return true;
	}
 	resultSet = PQerrorMessage(dbConnection);
	return false;
}

/**
 * Insert data into a table
 */
bool Connection::insert(const std::string& table, const std::string& data)
{
SQLBuffer	sql;
Document	document;
SQLBuffer	values;
int		col;

	if (document.Parse(data.c_str()).HasParseError())
	{
		return false;
	}
 	sql.append("INSERT INTO ");
	sql.append(table);
	sql.append(" (");
	for (Value::ConstMemberIterator itr = document.MemberBegin();
		itr != document.MemberEnd(); ++itr)
	{
		if (col)
			sql.append(", ");
		sql.append(itr->name.GetString());

		if (col)
			values.append(", ");
		if (itr->value.IsString())
		{
			values.append('\'');
			values.append(itr->value.GetString());
			values.append('\'');
		}
		else if (itr->value.IsDouble())
			values.append(itr->value.GetDouble());
		else if (itr->value.IsNumber())
			values.append(itr->value.GetInt());
		col++;
	}
	sql.append(") values (");
	const char *vals = values.coalesce();
	sql.append(vals);
	delete vals;
	sql.append(");");

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
	return false;
}

/**
 * Perform an update against a common table
 *
 * TODO Improve error handling
 * TODO Handle JSON
 */
bool Connection::update(const string& table, const string& payload)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;
int		col = 0;

	if (document.Parse(payload.c_str()).HasParseError())
	{
		printf("Failed to parse JSON: %s\n", payload.c_str());
		return false;
	}
	else
	{
		sql.append("UPDATE ");
		sql.append(table);
		sql.append(" SET ");

		Value& values = document["values"];
		for (Value::ConstMemberIterator itr = values.MemberBegin();
				itr != values.MemberEnd(); ++itr)
		{
			if (col != 0)
			{
				sql.append( ", ");
			}
			sql.append(itr->name.GetString());
			sql.append(" = ");

			if (itr->value.IsString())
			{
				sql.append('"');
				sql.append(itr->value.GetString());
				sql.append('"');
			}
			else if (itr->value.IsDouble())
				sql.append(itr->value.GetDouble());
			else if (itr->value.IsNumber())
				sql.append(itr->value.GetInt());
		}

		if (document.HasMember("condition"))
		{
			sql.append(" WHERE ");
			assert(document.IsObject());

			if (document.HasMember("where"))
			{
				jsonWhereClause(document["where"], sql);
			}
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
	return false;
}

/**
 * Perform a delete against a common table
 *
 * TODO Improve error handling
 */
bool Connection::deleteRows(const string& table, const string& condition)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;

	sql.append("DELETE from ");
	sql.append(table);
	if (! condition.empty())
	{
		sql.append(" WHERE ");
		if (document.Parse(condition.c_str()).HasParseError())
		{
			printf("Failed to parse JSON: %s\n", condition.c_str());
		}
		else
		{
			assert(document.IsObject());

			if (document.HasMember("where"))
			{
				jsonWhereClause(document["where"], sql);
			}
			else
			{
				printf("JSON does not contain where clause: %s\n", condition.c_str());
			}
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
	return false;
}

void Connection::mapResultSet(PGresult *res, string& resultSet)
{
int nFields, i, j;
Document doc;

	doc.SetObject();    // Create the JSON document
	Document::AllocatorType& allocator = doc.GetAllocator();
	nFields = PQnfields(res); // No. of columns in resultset
	Value rows(kArrayType);   // Setup a rows array
	Value count;
	count.SetInt(PQntuples(res)); // Create the count
	doc.AddMember("count", count, allocator);

	// Iterate over the rows
	for (i = 0; i < PQntuples(res); i++)
	{
		Value row(kObjectType); // Create a row
		for (j = 0; j < nFields; j++)
		{
			/* TODO Improve handling of Oid's */
			Oid oid = PQftype(res, j);
			switch (oid) {

			case 3802: // JSON type hard coded in this example
			{
				Document d;
				if (d.Parse(PQgetvalue(res, i, j)).HasParseError())
					printf("Failed to parse: %s\n", PQgetvalue(res, i, j));
				Value value(d, allocator);
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, value, allocator);
				break;
			}
			case 20:
			{
				long intVal = atol(PQgetvalue(res, i, j));
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, intVal, allocator);
				break;
			}
			case 710:
			{
				double dblVal = atof(PQgetvalue(res, i, j));
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, dblVal, allocator);
				break;
			}
			default:
			{
				char *str = PQgetvalue(res, i, j);
				if (oid == 1042) // char(x) rather than varchar so trim white space
					str = trim(str);
				Value value(str, allocator);
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, value, allocator);
				break;
			}
			}
		}
		rows.PushBack(row, allocator);  // Add the row
	}
	doc.AddMember("rows", rows, allocator); // Add the rows to the JSON
	/* Write out the JSON document we created */
	StringBuffer buffer;
	Writer<StringBuffer> writer(buffer);
	doc.Accept(writer);
	resultSet = buffer.GetString();
}

/**
 * Convert a JSON where clause into a PostresSQL where clause
 *
 * TODO Improve error handling
 * TODO Add aggregates
 */
bool Connection::jsonWhereClause(const Value& whereClause, SQLBuffer& sql)
{
	assert(whereClause.IsObject());
	assert(whereClause.HasMember("column"));
	assert(whereClause.HasMember("condition"));
	assert(whereClause.HasMember("value"));

	sql.append(whereClause["column"].GetString());
	sql.append(' ');
	sql.append(whereClause["condition"].GetString());
	sql.append(' ');
	if (whereClause["value"].IsInt())
	{
		sql.append(whereClause["value"].GetInt());
	} else if (whereClause["value"].IsString())
	{
		sql.append('\'');
		sql.append(whereClause["value"].GetString());
		sql.append('\'');
	}

	if (whereClause.HasMember("and"))
	{
		sql.append(" AND ");
		jsonWhereClause(whereClause["and"], sql);
	}
	if (whereClause.HasMember("or"))
	{
		sql.append(" OR ");
		jsonWhereClause(whereClause["or"], sql);
	}
	return true;
}

/**
 * Remove whitespace at both ends of a string
 */
char *Connection::trim(char *str)
{
char *ptr;

	while (*str && *str == ' ')
		str++;

	ptr = str + strlen(str) - 1;
	while (ptr > str && *ptr == ' ')
	{
		*ptr = 0;
		ptr--;
	}
	return str;
}

<commit_msg>FOGL-197 Add support for JSON insert and update. Add support for functions, e.g. now()<commit_after>#include <connection.h>
#include <sql_buffer.h>
#include <iostream>
#include <libpq-fe.h>
#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"
#include <string>
#include <regex>

using namespace std;
using namespace rapidjson;

/**
 * Create a database connection
 */
Connection::Connection()
{
	const char *conninfo = "dbname = foglamp";

	/* Make a connection to the database */
	dbConnection = PQconnectdb(conninfo);

	/* Check to see that the backend connection was successfully made */
	if (PQstatus(dbConnection) != CONNECTION_OK)
	{
		cerr << "Failed to connect" << endl;
	}
}

/**
 * Destructor for the database connection. Close the connection
 * to Postgres
 */
Connection::~Connection()
{
	PQfinish(dbConnection);
}

/**
 * Perform a query against a common table
 *
 * TODO Improve error handling
 */
bool Connection::retrieve(const string& table, const string& condition, string& resultSet)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;

	sql.append("SELECT * from ");
	sql.append(table);
	if (! condition.empty())
	{
		sql.append(" WHERE ");
		if (document.Parse(condition.c_str()).HasParseError())
		{
			printf("Failed to parse JSON: %s\n", condition.c_str());
		}
		else
		{
			assert(document.IsObject());

			if (document.HasMember("where"))
			{
				jsonWhereClause(document["where"], sql);
			}
			else
			{
				printf("JSON does not contain where clause: %s\n", condition.c_str());
			}
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_TUPLES_OK)
	{
		mapResultSet(res, resultSet);
		return true;
	}
 	resultSet = PQerrorMessage(dbConnection);
	return false;
}

/**
 * Insert data into a table
 */
bool Connection::insert(const std::string& table, const std::string& data)
{
SQLBuffer	sql;
Document	document;
SQLBuffer	values;
int		col = 0;

	if (document.Parse(data.c_str()).HasParseError())
	{
		printf("Failed to parse JSON\n");
		return false;
	}
 	sql.append("INSERT INTO ");
	sql.append(table);
	sql.append(" (");
	for (Value::ConstMemberIterator itr = document.MemberBegin();
		itr != document.MemberEnd(); ++itr)
	{
		if (col)
			sql.append(", ");
		sql.append(itr->name.GetString());

		if (col)
			values.append(", ");
		if (itr->value.IsString())
		{
			const char *str = itr->value.GetString();
			// Check if the string is a function
			string s (str);
  			regex e ("[a-zA-Z][a-zA-Z0-9_]*\\(.*\\)");
  			if (regex_match (s,e))
			{
				values.append(str);
			}
			else
			{
				values.append('\'');
				values.append(str);
				values.append('\'');
			}
		}
		else if (itr->value.IsDouble())
			values.append(itr->value.GetDouble());
		else if (itr->value.IsNumber())
			values.append(itr->value.GetInt());
		else if (itr->value.IsObject())
		{
			StringBuffer buffer;
			Writer<StringBuffer> writer(buffer);
			itr->value.Accept(writer);
			values.append('\'');
			values.append(buffer.GetString());
			values.append('\'');
		}
		col++;
	}
	sql.append(") values (");
	const char *vals = values.coalesce();
	sql.append(vals);
	delete vals;
	sql.append(");");

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
 	printf("%s\n", PQerrorMessage(dbConnection));
	return false;
}

/**
 * Perform an update against a common table
 *
 * TODO Improve error handling
 */
bool Connection::update(const string& table, const string& payload)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;
int		col = 0;

	if (document.Parse(payload.c_str()).HasParseError())
	{
		printf("Failed to parse JSON: %s\n", payload.c_str());
		return false;
	}
	else
	{
		sql.append("UPDATE ");
		sql.append(table);
		sql.append(" SET ");

		Value& values = document["values"];
		for (Value::ConstMemberIterator itr = values.MemberBegin();
				itr != values.MemberEnd(); ++itr)
		{
			if (col != 0)
			{
				sql.append( ", ");
			}
			sql.append(itr->name.GetString());
			sql.append(" = ");

			if (itr->value.IsString())
			{
				sql.append('"');
				sql.append(itr->value.GetString());
				sql.append('"');
			}
			else if (itr->value.IsDouble())
				sql.append(itr->value.GetDouble());
			else if (itr->value.IsNumber())
				sql.append(itr->value.GetInt());
			else if (itr->value.IsObject())
			{
				StringBuffer buffer;
				Writer<StringBuffer> writer(buffer);
				itr->value.Accept(writer);
				sql.append('\'');
				sql.append(buffer.GetString());
				sql.append('\'');
			}
		}

		if (document.HasMember("condition"))
		{
			sql.append(" WHERE ");
			jsonWhereClause(document["condition"], sql);
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
 	printf("%s\n", PQerrorMessage(dbConnection));
	return false;
}

/**
 * Perform a delete against a common table
 *
 * TODO Improve error handling
 */
bool Connection::deleteRows(const string& table, const string& condition)
{
Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
SQLBuffer	sql;

	sql.append("DELETE from ");
	sql.append(table);
	if (! condition.empty())
	{
		sql.append(" WHERE ");
		if (document.Parse(condition.c_str()).HasParseError())
		{
			printf("Failed to parse JSON: %s\n", condition.c_str());
		}
		else
		{
			assert(document.IsObject());

			if (document.HasMember("where"))
			{
				jsonWhereClause(document["where"], sql);
			}
			else
			{
				printf("JSON does not contain where clause: %s\n", condition.c_str());
			}
		}
	}
	sql.append(';');

	const char *query = sql.coalesce();
	PGresult *res = PQexec(dbConnection, query);
	delete query;
	if (PQresultStatus(res) == PGRES_COMMAND_OK)
	{
		return true;
	}
	return false;
}

void Connection::mapResultSet(PGresult *res, string& resultSet)
{
int nFields, i, j;
Document doc;

	doc.SetObject();    // Create the JSON document
	Document::AllocatorType& allocator = doc.GetAllocator();
	nFields = PQnfields(res); // No. of columns in resultset
	Value rows(kArrayType);   // Setup a rows array
	Value count;
	count.SetInt(PQntuples(res)); // Create the count
	doc.AddMember("count", count, allocator);

	// Iterate over the rows
	for (i = 0; i < PQntuples(res); i++)
	{
		Value row(kObjectType); // Create a row
		for (j = 0; j < nFields; j++)
		{
			/* TODO Improve handling of Oid's */
			Oid oid = PQftype(res, j);
			switch (oid) {

			case 3802: // JSON type hard coded in this example
			{
				Document d;
				if (d.Parse(PQgetvalue(res, i, j)).HasParseError())
					printf("Failed to parse: %s\n", PQgetvalue(res, i, j));
				Value value(d, allocator);
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, value, allocator);
				break;
			}
			case 20:
			{
				long intVal = atol(PQgetvalue(res, i, j));
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, intVal, allocator);
				break;
			}
			case 710:
			{
				double dblVal = atof(PQgetvalue(res, i, j));
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, dblVal, allocator);
				break;
			}
			default:
			{
				char *str = PQgetvalue(res, i, j);
				if (oid == 1042) // char(x) rather than varchar so trim white space
					str = trim(str);
				Value value(str, allocator);
				Value name(PQfname(res, j), allocator);
				row.AddMember(name, value, allocator);
				break;
			}
			}
		}
		rows.PushBack(row, allocator);  // Add the row
	}
	doc.AddMember("rows", rows, allocator); // Add the rows to the JSON
	/* Write out the JSON document we created */
	StringBuffer buffer;
	Writer<StringBuffer> writer(buffer);
	doc.Accept(writer);
	resultSet = buffer.GetString();
}

/**
 * Convert a JSON where clause into a PostresSQL where clause
 *
 * TODO Improve error handling
 * TODO Add aggregates
 */
bool Connection::jsonWhereClause(const Value& whereClause, SQLBuffer& sql)
{
	assert(whereClause.IsObject());
	assert(whereClause.HasMember("column"));
	assert(whereClause.HasMember("condition"));
	assert(whereClause.HasMember("value"));

	sql.append(whereClause["column"].GetString());
	sql.append(' ');
	sql.append(whereClause["condition"].GetString());
	sql.append(' ');
	if (whereClause["value"].IsInt())
	{
		sql.append(whereClause["value"].GetInt());
	} else if (whereClause["value"].IsString())
	{
		sql.append('\'');
		sql.append(whereClause["value"].GetString());
		sql.append('\'');
	}

	if (whereClause.HasMember("and"))
	{
		sql.append(" AND ");
		jsonWhereClause(whereClause["and"], sql);
	}
	if (whereClause.HasMember("or"))
	{
		sql.append(" OR ");
		jsonWhereClause(whereClause["or"], sql);
	}
	return true;
}

/**
 * Remove whitespace at both ends of a string
 */
char *Connection::trim(char *str)
{
char *ptr;

	while (*str && *str == ' ')
		str++;

	ptr = str + strlen(str) - 1;
	while (ptr > str && *ptr == ' ')
	{
		*ptr = 0;
		ptr--;
	}
	return str;
}

<|endoftext|>


<commit_before>#include "chlog.hpp"
#include "ch_frb_io_internals.hpp"

using namespace std;

namespace ch_frb_io {
#if 0
};  // pacify emacs c-mode!
#endif

// Static factory function
shared_ptr<output_device> output_device::make(const output_device::initializer &ini_params)
{
    output_device *p = new output_device(ini_params);
    shared_ptr<output_device> ret(p);

    ret->output_thread = std::thread(std::bind(&output_device::io_thread_main, ret));
    return ret;
}

output_device::output_device(const output_device::initializer &ini_params_) :
    ini_params(ini_params_)
{
    // Note: no sanity-checking of ini_params needed here!
    // XXX 32M is overkill!  How much space should I use here?
    this->_buffer = unique_ptr<uint8_t[]> (new uint8_t[32 * 1024 * 1024]);
}

void output_device::io_thread_main()
{
    // Note: if 'io_thread_allowed_cores' is an empty list, no core-pinning will be done.
    pin_thread_to_cores(ini_params.io_thread_allowed_cores);

    // If io_thread_name is unspecified, assign a default thread name.
    string thread_name = ini_params.io_thread_name;
    if (thread_name.size() == 0)
	thread_name = "io_thread[" + ini_params.device_name + "]";

    chime_log_set_thread_name(thread_name);

    if (ini_params.verbosity >= 2)
	chlog("thread starting up");

    for (;;) {
	// Pull a write_chunk_request off the queue!
	shared_ptr<write_chunk_request> w = this->pop_write_request();
	if (!w)
	    break;

	shared_ptr<assembled_chunk> chunk = w->chunk;
	string error_message;
	string link_src;

	if (ini_params.verbosity >= 3) {
	    chlog("dequeued write request: filename " + w->filename
		  + ", beam " + to_string(chunk->beam_id)
		  + ", chunk " + to_string(chunk->ichunk)
		  + ", FPGA counts " + to_string(chunk->fpga_begin));
	}

	unique_lock<mutex> ulock(chunk->filename_mutex);

	// If this write request is a duplicate of a previous write request, return immediately.
	if (chunk->filename_set.count(w->filename) > 0) {
	    ulock.unlock();

	    if (ini_params.verbosity >= 3)
		chlog("write request '" + w->filename + "' is a duplicate, skipping...");

	    w->write_callback("");
            continue;
	}

	// If this write request is a "pseudo-duplicate", i.e. the same chunk has been
	// previously written to a different filename on the same output_device, then
	// we'll make a hard link instead of a copy.

	auto p = chunk->filename_map.find(ini_params.device_name);
	if (p != chunk->filename_map.end())
	    link_src = p->second;

	ulock.unlock();

	try {
	    if (file_exists(w->filename))
		throw runtime_error("file already exists");
	    else if (link_src.size() == 0)
		chunk->write_msgpack_file(w->filename, false, this->_buffer.get());  // compress=false
	    else {
		if (ini_params.verbosity >= 3)
		    chlog("write request '" + w->filename + "' is a pseudo-duplicate, will hard-link from '" + link_src + "' instead of writing new copy");
		hard_link(link_src, w->filename);
	    }
	} catch (exception &e) {
	    // Note: we now include the exception text in the error_message.
	    error_message = "Write msgpack file '" + w->filename + "' failed: " + e.what();
	}

	// Success!
	ulock.lock();
	chunk->filename_set.insert(w->filename);
	chunk->filename_map[ini_params.device_name] = w->filename;
	ulock.unlock();

	if (error_message.size() > 0 && ini_params.verbosity >= 1)
	    chlog(error_message);
	else if (ini_params.verbosity >= 3)
	    chlog("wrote " + w->filename);
    }

    if (ini_params.verbosity >= 2)
	chlog("received NULL write_chunk_request; exiting.");
}

// Called by an "external" thread (assembler thread or RPC thread).
bool output_device::enqueue_write_request(const shared_ptr<write_chunk_request> &req)
{
    if (!req)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req is null");
    if (!req->chunk)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->chunk is null");
    if (req->filename.size() == 0)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->filename is an empty string");
    if (!is_prefix(this->ini_params.device_name, req->filename))
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->filename, device_name mismatch");

    unique_lock<std::mutex> ulock(_lock);

    if (end_stream_called)
	return false;

    _write_reqs.push(req);
    _cond.notify_all();

    ulock.unlock();

    if (ini_params.verbosity >= 3) {
	chlog("enqueued write request: filename " + req->filename
	      + ", beam " + to_string(req->chunk->beam_id)
	      + ", chunk " + to_string(req->chunk->ichunk)
	      + ", FPGA counts " + to_string(req->chunk->fpga_begin));
    }

    return true;
}

// Called internally by I/O thread.
shared_ptr<write_chunk_request> output_device::pop_write_request()
{
    unique_lock<std::mutex> ulock(_lock);

    for (;;) {
	if (!_write_reqs.empty())
	    break;
	if (end_stream_called)
	    return shared_ptr<write_chunk_request> ();
	_cond.wait(ulock);
    }

    shared_ptr<write_chunk_request> ret = _write_reqs.top();
    _write_reqs.pop();
    _cond.notify_all();
    return ret;
}

void output_device::end_stream(bool wait)
{
    unique_lock<std::mutex> ulock(_lock);

    // Note: multiple calls to end_stream() are OK.
    this->end_stream_called = true;

    // If 'wait' is false, cancel all pending writes and return.
    if (!wait) {
	while (!_write_reqs.empty())
	    _write_reqs.pop();
	_cond.notify_all();
	return;
    }

    // If 'wait' is true, wait for pending writes to complete before returning.
    while (!_write_reqs.empty())
	_cond.wait(ulock);
}

void output_device::join_thread()
{
    unique_lock<std::mutex> ulock(_lock);

    if (!end_stream_called)
	throw runtime_error("output_device::join_thread() called, with no preceding call to end_stream()");

    // Multiple calls to join_thread() are OK, but if this is not the first
    // time join_thread() has been called, we still need to check that the
    // thread has actually joined, and wait for it if not.

    if (join_thread_called) {
	while (!thread_joined)
	    _cond.wait(ulock);
	return;
    }

    // If we get here, this is the first time join_thread() has been called.
    // Set the 'join_thread_called' flag and drop the lock...
    join_thread_called = true;
    _cond.notify_all();
    ulock.unlock();

    // Now join the thread.  The sychronization logic above guarantees that no
    // other threads will call output_thread.join().  (Instead, they will wait
    // for this thread to set the 'thread_joined' flag.)
    output_thread.join();

    // Now re-acquire the lock and set the 'thread_joined' flag.
    ulock.lock();
    thread_joined = true;
    _cond.notify_all();
}

}  // namespace ch_frb_io
<commit_msg>output_device: call callback after file is written (or failed)<commit_after>#include "chlog.hpp"
#include "ch_frb_io_internals.hpp"

using namespace std;

namespace ch_frb_io {
#if 0
};  // pacify emacs c-mode!
#endif

// Static factory function
shared_ptr<output_device> output_device::make(const output_device::initializer &ini_params)
{
    output_device *p = new output_device(ini_params);
    shared_ptr<output_device> ret(p);

    ret->output_thread = std::thread(std::bind(&output_device::io_thread_main, ret));
    return ret;
}

output_device::output_device(const output_device::initializer &ini_params_) :
    ini_params(ini_params_)
{
    // Note: no sanity-checking of ini_params needed here!
    // XXX 32M is overkill!  How much space should I use here?
    this->_buffer = unique_ptr<uint8_t[]> (new uint8_t[32 * 1024 * 1024]);
}

void output_device::io_thread_main()
{
    // Note: if 'io_thread_allowed_cores' is an empty list, no core-pinning will be done.
    pin_thread_to_cores(ini_params.io_thread_allowed_cores);

    // If io_thread_name is unspecified, assign a default thread name.
    string thread_name = ini_params.io_thread_name;
    if (thread_name.size() == 0)
	thread_name = "io_thread[" + ini_params.device_name + "]";

    chime_log_set_thread_name(thread_name);

    if (ini_params.verbosity >= 2)
	chlog("thread starting up");

    for (;;) {
	// Pull a write_chunk_request off the queue!
	shared_ptr<write_chunk_request> w = this->pop_write_request();
	if (!w)
	    break;

	shared_ptr<assembled_chunk> chunk = w->chunk;
	string error_message;
	string link_src;

	if (ini_params.verbosity >= 3) {
	    chlog("dequeued write request: filename " + w->filename
		  + ", beam " + to_string(chunk->beam_id)
		  + ", chunk " + to_string(chunk->ichunk)
		  + ", FPGA counts " + to_string(chunk->fpga_begin));
	}

	unique_lock<mutex> ulock(chunk->filename_mutex);

	// If this write request is a duplicate of a previous write request, return immediately.
	if (chunk->filename_set.count(w->filename) > 0) {
	    ulock.unlock();

	    if (ini_params.verbosity >= 3)
		chlog("write request '" + w->filename + "' is a duplicate, skipping...");

	    w->write_callback("");
            continue;
	}

	// If this write request is a "pseudo-duplicate", i.e. the same chunk has been
	// previously written to a different filename on the same output_device, then
	// we'll make a hard link instead of a copy.

	auto p = chunk->filename_map.find(ini_params.device_name);
	if (p != chunk->filename_map.end())
	    link_src = p->second;

	ulock.unlock();

	try {
	    if (file_exists(w->filename))
		throw runtime_error("file already exists");
	    else if (link_src.size() == 0)
		chunk->write_msgpack_file(w->filename, false, this->_buffer.get());  // compress=false
	    else {
		if (ini_params.verbosity >= 3)
		    chlog("write request '" + w->filename + "' is a pseudo-duplicate, will hard-link from '" + link_src + "' instead of writing new copy");
		hard_link(link_src, w->filename);
	    }
	} catch (exception &e) {
	    // Note: we now include the exception text in the error_message.
	    error_message = "Write msgpack file '" + w->filename + "' failed: " + e.what();
	}

	// Success!
	ulock.lock();
	chunk->filename_set.insert(w->filename);
	chunk->filename_map[ini_params.device_name] = w->filename;
	ulock.unlock();

	if (error_message.size() > 0 && ini_params.verbosity >= 1)
	    chlog(error_message);
	else if (ini_params.verbosity >= 3)
	    chlog("wrote " + w->filename);

        w->write_callback(error_message);
    }

    if (ini_params.verbosity >= 2)
	chlog("received NULL write_chunk_request; exiting.");
}

// Called by an "external" thread (assembler thread or RPC thread).
bool output_device::enqueue_write_request(const shared_ptr<write_chunk_request> &req)
{
    if (!req)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req is null");
    if (!req->chunk)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->chunk is null");
    if (req->filename.size() == 0)
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->filename is an empty string");
    if (!is_prefix(this->ini_params.device_name, req->filename))
	throw runtime_error("ch_frb_io::output_device::enqueue_write_request(): req->filename, device_name mismatch");

    unique_lock<std::mutex> ulock(_lock);

    if (end_stream_called)
	return false;

    _write_reqs.push(req);
    _cond.notify_all();

    ulock.unlock();

    if (ini_params.verbosity >= 3) {
	chlog("enqueued write request: filename " + req->filename
	      + ", beam " + to_string(req->chunk->beam_id)
	      + ", chunk " + to_string(req->chunk->ichunk)
	      + ", FPGA counts " + to_string(req->chunk->fpga_begin));
    }

    return true;
}

// Called internally by I/O thread.
shared_ptr<write_chunk_request> output_device::pop_write_request()
{
    unique_lock<std::mutex> ulock(_lock);

    for (;;) {
	if (!_write_reqs.empty())
	    break;
	if (end_stream_called)
	    return shared_ptr<write_chunk_request> ();
	_cond.wait(ulock);
    }

    shared_ptr<write_chunk_request> ret = _write_reqs.top();
    _write_reqs.pop();
    _cond.notify_all();
    return ret;
}

void output_device::end_stream(bool wait)
{
    unique_lock<std::mutex> ulock(_lock);

    // Note: multiple calls to end_stream() are OK.
    this->end_stream_called = true;

    // If 'wait' is false, cancel all pending writes and return.
    if (!wait) {
	while (!_write_reqs.empty())
	    _write_reqs.pop();
	_cond.notify_all();
	return;
    }

    // If 'wait' is true, wait for pending writes to complete before returning.
    while (!_write_reqs.empty())
	_cond.wait(ulock);
}

void output_device::join_thread()
{
    unique_lock<std::mutex> ulock(_lock);

    if (!end_stream_called)
	throw runtime_error("output_device::join_thread() called, with no preceding call to end_stream()");

    // Multiple calls to join_thread() are OK, but if this is not the first
    // time join_thread() has been called, we still need to check that the
    // thread has actually joined, and wait for it if not.

    if (join_thread_called) {
	while (!thread_joined)
	    _cond.wait(ulock);
	return;
    }

    // If we get here, this is the first time join_thread() has been called.
    // Set the 'join_thread_called' flag and drop the lock...
    join_thread_called = true;
    _cond.notify_all();
    ulock.unlock();

    // Now join the thread.  The sychronization logic above guarantees that no
    // other threads will call output_thread.join().  (Instead, they will wait
    // for this thread to set the 'thread_joined' flag.)
    output_thread.join();

    // Now re-acquire the lock and set the 'thread_joined' flag.
    ulock.lock();
    thread_joined = true;
    _cond.notify_all();
}

}  // namespace ch_frb_io
<|endoftext|>


<commit_before>#include "test_json.h"
#include "json/json_serialize.h"

class MarshallConfigTest : public testing::JsonDioTestWrapper
{
    void SetUp()
    {
        DisirLogCurrentTestEnter();

        status = disir_mold_read (instance, "test", "json_test_mold", &mold);
        ASSERT_TRUE (mold != NULL);

        status = dc_config_begin (mold, &context_config);
        EXPECT_STATUS (DISIR_STATUS_OK, status);

        writer = new dio::ConfigWriter (instance);

        DisirLogTestBodyEnter();
    }

    void TearDown()
    {
        DisirLogTestBodyExit();

        if (writer)
        {
            delete writer;
        }
        if (context_keyval)
        {
            status = dc_destroy (&context_keyval);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_section)
        {
            status = dc_destroy (&context_section);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_section_nested)
        {
            status = dc_destroy (&context_section_nested);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_config)
        {
            status = dc_destroy (&context_config);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (config)
        {
            status = disir_config_finished (&config);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (mold)
        {
            status = disir_mold_finished (&mold);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }

        DisirLogCurrentTestExit();
    }

    public:
        struct disir_mold *mold = NULL;
        struct disir_context *context_keyval = NULL;
        struct disir_context *context_config = NULL;
        struct disir_context *context_section = NULL;
        struct disir_context *context_section_nested = NULL;
        struct disir_config *config = NULL;
        dio::ConfigWriter *writer = NULL;
};

TEST_F(MarshallConfigTest, no_keyval_context)
{
    std::string json, expected, result;
    bool success;

    success = formatJson (expected,"{\n\"version\" : \"1.0.0\", \
                                     \"config\" : {\"test1\":\"k1val\"}}");
    ASSERT_TRUE (success);

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_name (context_keyval, "test1", strlen ("test1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_value_string (context_keyval, "k1val", strlen ("k1val"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    auto status = writer->serialize (config, json);
    EXPECT_STATUS (status, DISIR_STATUS_OK);

    // Checking if we got a valid json string
    success = formatJson (result, json);
    std::cerr << disir_error (instance) << std::endl;
    ASSERT_TRUE (success);

    ASSERT_STREQ (expected.c_str(), result.c_str());
}

/* Config layout in json
         * {
         *      "section_name" : {
         *        "k1" : "k1value",
         *        "k2" : "k2value"
         *        "section2" : {
         *          "k3" : "k3value"
         *        }
         *      }
         *}
         */

TEST_F(MarshallConfigTest, marshall_section_shall_pass)
{
    std::string json, expected;
    bool success;

    success = formatJson (expected,
            "{\"version\" : \"1.0.0\",  \
            \"config\" : {              \
            \"section_name\": {         \
            \"k1\" : \"k1value\",       \
            \"k2\" : \"k2value\",       \
            \"section2\" : {            \
             \"k3\" : \"k3value\"       \
             } \
            }}}");
    ASSERT_TRUE (success);

    status = dc_begin (context_config,
            DISIR_CONTEXT_SECTION, &context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_name (context_section, "section_name", strlen ("section_name"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_begin (context_section, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_name (context_keyval, "k1", strlen ("k1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_value_string (context_keyval, "k1value", strlen ("k1value"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_begin (context_section, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_name (context_keyval, "k2", strlen ("k2"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_value_string (context_keyval, "k2value", strlen ("k2value"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_begin (context_section,
            DISIR_CONTEXT_SECTION, &context_section_nested);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_begin (context_section_nested, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS  (DISIR_STATUS_OK, status);
    status = dc_set_name (context_section_nested, "section2", strlen ("section2"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_name (context_keyval, "k3", strlen ("k3"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_set_value_string (context_keyval, "k3value", strlen ("k3value"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_section_nested);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_finalize (&context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    auto status = writer->serialize (config, json);
    EXPECT_STATUS (status,DISIR_STATUS_OK);
    ASSERT_STREQ (expected.c_str(), json.c_str());
}

TEST_F(MarshallConfigTest, empty_config_shall_fail)
{
    std::string empty;

    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_OK, status);
    ASSERT_FALSE (writer->serialize (config, empty) != DISIR_STATUS_OK);
}
/*
{
    "string" : "value",
    "integer" : 123,
    "float"   : 12.123,
    "boolean" : true
}
*/
TEST_F(MarshallConfigTest, marshall_all_types)
{
   std::string json;
   std::string expected;
   bool success;
   int64_t integer = 123;

    success = formatJson (expected,
            "{\"version\" : \"1.0.0\",  \
            \"config\" : {              \
            \"test\": \"1\",            \
            \"integer\" : 123,          \
            \"float\" : 12.123,         \
            \"boolean\" : true          \
            }}");
    ASSERT_TRUE (success);

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "test1", strlen ("test"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_string (context_keyval, "1", strlen ("1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "integer", strlen ("integer"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_integer (context_keyval, integer);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "float", strlen ("float"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_float (context_keyval, (double)12.123);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_begin (context_config,
            DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "boolean", strlen ("boolean"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_boolean (context_keyval, (uint8_t)1);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    ASSERT_TRUE (writer->serialize (config, json) == DISIR_STATUS_OK);
    ASSERT_STREQ (expected.c_str(), json.c_str());
}

TEST_F (MarshallConfigTest, duplicate_keyvals_shall_succeed)
{
    std::string json;
    std::string expected;

    formatJson (expected,
            "{\"version\" : \"1.0.0\", \
                \"config\" : {          \
                \"test1\" :  \"value1\", \
                \"test1@2\" : \"value2\" }}");

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "test1", strlen ("test1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_string (context_keyval, "value1", strlen ("value1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "test1", strlen ("test1"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_value_string (context_keyval, "value2", strlen ("value2"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_INVALID_CONTEXT, status);

    ASSERT_TRUE (writer->serialize (config, json) == DISIR_STATUS_OK);

    ASSERT_JSON_STREQ (expected.c_str (), json.c_str ());
}

TEST_F (MarshallConfigTest, duplicate_sections_shall_succeed)
{
    std::string json;
    std::string expected;

    formatJson (expected,
            "{\"version\" : \"1.0.0\", \
                \"config\" : {          \
                \"section_name\" :  { }, \
                \"section_name@2\" : {}}}");

    status = dc_begin (context_config, DISIR_CONTEXT_SECTION, &context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_section, "section_name", strlen ("section_name"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_begin (context_config, DISIR_CONTEXT_SECTION, &context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_section, "section_name", strlen ("section_name"));
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_finalize (&context_section);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_INVALID_CONTEXT, status);

    ASSERT_TRUE (writer->serialize (config, json) == DISIR_STATUS_OK);

    ASSERT_JSON_STREQ (expected.c_str (), json.c_str ());
}

TEST_F (MarshallConfigTest, context_without_value_shall_succeed)
{
    std::string json;
    std::string expected;

    ASSERT_TRUE ( formatJson (expected,
            "{\"version\" : \"1.0.0\", \
                \"config\" : {          \
                \"fake_name\" : \"UNKNOWN\"}}"));

    status = dc_begin (context_config, DISIR_CONTEXT_KEYVAL, &context_keyval);
    EXPECT_STATUS (DISIR_STATUS_OK, status);

    status = dc_set_name (context_keyval, "fake_name", strlen ("fake_name"));
    EXPECT_STATUS (DISIR_STATUS_NOT_EXIST, status);

    status = dc_finalize (&context_keyval);
    EXPECT_STATUS (DISIR_STATUS_INVALID_CONTEXT, status);

    status = dc_config_finalize (&context_config, &config);
    EXPECT_STATUS (DISIR_STATUS_INVALID_CONTEXT, status);

    auto status = writer->serialize (config, json);
    ASSERT_EQ (DISIR_STATUS_OK, status);

    dc_putcontext (&context_keyval);

    ASSERT_JSON_STREQ (expected.c_str (), json.c_str ());
}

<commit_msg>fix test: rm deprecated config serialize tests<commit_after>#include "test_json.h"
#include "json/json_serialize.h"

class MarshallConfigTest : public testing::JsonDioTestWrapper
{
    void SetUp()
    {
        DisirLogCurrentTestEnter();

        status = disir_mold_read (instance, "test", "json_test_mold", &mold);
        ASSERT_TRUE (mold != NULL);

        status = dc_config_begin (mold, &context_config);
        EXPECT_STATUS (DISIR_STATUS_OK, status);

        writer = new dio::ConfigWriter (instance);

        DisirLogTestBodyEnter();
    }

    void TearDown()
    {
        DisirLogTestBodyExit();

        if (writer)
        {
            delete writer;
        }
        if (context_keyval)
        {
            status = dc_destroy (&context_keyval);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_section)
        {
            status = dc_destroy (&context_section);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_section_nested)
        {
            status = dc_destroy (&context_section_nested);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (context_config)
        {
            status = dc_destroy (&context_config);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (config)
        {
            status = disir_config_finished (&config);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }
        if (mold)
        {
            status = disir_mold_finished (&mold);
            EXPECT_STATUS (DISIR_STATUS_OK, status);
        }

        DisirLogCurrentTestExit();
    }

    public:
        struct disir_mold *mold = NULL;
        struct disir_context *context_keyval = NULL;
        struct disir_context *context_config = NULL;
        struct disir_context *context_section = NULL;
        struct disir_context *context_section_nested = NULL;
        struct disir_config *config = NULL;
        dio::ConfigWriter *writer = NULL;
};

<|endoftext|>