<commit_before>#include <stack>

#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtx/compatibility.hpp>

#include <zombye/core/game.hpp>
#include <zombye/rendering/animation_component.hpp>
#include <zombye/rendering/animation_system.hpp>
#include <zombye/rendering/rendering_system.hpp>
#include <zombye/rendering/staticmesh_component.hpp>
#include <zombye/rendering/skinned_mesh.hpp>
#include <zombye/rendering/skeleton.hpp>
#include <zombye/rendering/texture.hpp>
#include <zombye/scripting/scripting_system.hpp>
#include <zombye/utils/logger.hpp>

#include <glm/gtx/string_cast.hpp>

namespace zombye {
    animation_component::animation_component(game& game, entity& owner, const std::string& mesh, const std::string& skeleton)
    : reflective{game, owner}, current_state_{""}, elapsed_time_{0.f}, blend_{false}, blend_next_{false} {
        game_.rendering_system().register_component(this);
        game_.animation_system().register_component(this);
        load(mesh);
        load_skeleton(skeleton);
    }

    animation_component::~animation_component() noexcept {
        game_.rendering_system().unregister_component(this);
        game_.animation_system().unregister_component(this);
    }

    void animation_component::update(float delta_time) {
        static std::vector<glm::mat4> pose(skeleton_->bones().size(), glm::mat4{1.f});
        static const float fps = 1.f / 24.f;
        pose.assign(skeleton_->bones().size(), glm::mat4{1.f});

        if (current_state_ != "") {
            auto& bones = skeleton_->bones();
            auto& bone_hierachy = skeleton_->bone_hierachy_;

            elapsed_time_ += delta_time;

            if (blend_ && next_state_ != "") {
                auto& animation = skeleton_->animation(current_state_);
                auto length = animation.length * fps;
                auto& next_animation = skeleton_->animation(next_state_);
                auto next_animation_length = next_animation.length * fps;
                auto delta_factor = next_animation_length <= length ? 1.f / next_animation_length : 1.f / length;

                if (elapsed_time_ <= next_animation_length) {
                    if (elapsed_time_ >= length) {
                        current_state_ = next_state_;
                        next_state_ = "";
                        blend_ = false;
                        current_keyframes_ = next_keyframes_;
                        return;
                    } else {
                        std::stack<int> traversal;
                        traversal.push(0);
                        while(!traversal.empty()) {
                            auto i = traversal.top();
                            traversal.pop();

                            auto& cs_track = animation.tracks.at(i);

                            auto& cs_current_frame = current_keyframes_.at(i);
                            auto cs_next_frame = cs_current_frame;
                            ++cs_next_frame.tkey;
                            ++cs_next_frame.qkey;
                            ++cs_next_frame.skey;

                            auto iv_current = interpolate_translation_keyframe(cs_track, cs_current_frame, cs_next_frame, fps);
                            auto iq_current = interpolate_quaternion_keyframe(cs_track, cs_current_frame, cs_next_frame, fps);

                            auto& ns_track = next_animation.tracks.at(i);

                            auto& ns_current_frame = next_keyframes_.at(i);
                            auto ns_next_frame = ns_current_frame;
                            ++ns_next_frame.tkey;
                            ++ns_next_frame.qkey;
                            ++ns_next_frame.skey;

                            auto iv_next = interpolate_translation_keyframe(ns_track, ns_current_frame, ns_next_frame, fps);
                            auto iq_next = interpolate_quaternion_keyframe(ns_track, ns_current_frame, ns_next_frame, fps);

                            auto delta = elapsed_time_ * delta_factor;
                            delta = delta > 1.f ? 1.f : delta;
                            auto iv = glm::lerp(iv_current, iv_next, delta);
                            auto iq = glm::lerp(iq_current, iq_next, delta);

                            auto p = glm::toMat4(iq);
                            p[3].x = iv.x;
                            p[3].y = iv.y;
                            p[3].z = iv.z;

                            pose[i] *= bones.at(i).relative_transform * p;
                            if (bone_hierachy.find(i) != bone_hierachy.end()) {
                                for (auto node = bone_hierachy.begin(i); node != bone_hierachy.end(i); ++node) {
                                    auto& child = node->second;
                                    pose[child] *= pose[i];
                                    traversal.push(child);
                                }
                            }
                        }
                    }
                } else {
                    elapsed_time_ = 0.f;
                    current_keyframes_.assign(current_keyframes_.size(), frame{});
                    next_keyframes_.assign(next_keyframes_.size(), frame{});
                    current_state_ = next_state_;
                    next_state_ = "";
                    blend_ = false;
                    return;
                }
            } else {
                auto& animation = skeleton_->animation(current_state_);
                auto length = animation.length * fps;
                if (elapsed_time_ <= length) {
                    std::stack<int> traversal;
                    traversal.push(0);
                    while(!traversal.empty()) {
                        auto i = traversal.top();
                        traversal.pop();

                        auto& track = animation.tracks.at(i);

                        auto& current_frame = current_keyframes_.at(i);
                        auto next_frame = current_frame;
                        ++next_frame.tkey;
                        ++next_frame.qkey;
                        ++next_frame.skey;

                        auto iv = interpolate_translation_keyframe(track, current_frame, next_frame, fps);
                        auto iq = interpolate_quaternion_keyframe(track, current_frame, next_frame, fps);

                        auto p = glm::toMat4(iq);
                        p[3].x = iv.x;
                        p[3].y = iv.y;
                        p[3].z = iv.z;

                        pose[i] *= bones.at(i).relative_transform * p;
                        if (bone_hierachy.find(i) != bone_hierachy.end()) {
                            for (auto node = bone_hierachy.begin(i); node != bone_hierachy.end(i); ++node) {
                                auto& child = node->second;
                                pose[child] *= pose[i];
                                traversal.push(child);
                            }
                        }
                    }
                } else {
                    elapsed_time_ = 0.f;
                    for (auto& k : current_keyframes_) {
                        k.tkey = 0;
                        k.qkey = 0;
                        k.skey = 0;
                    }
                    if (blend_next_) {
                        blend_ = true;
                        blend_next_ = false;
                    }
                    return;
                }
            }
        } else {
            auto& bones = skeleton_->bones();
            for (auto i = 0u; i < bones.size(); ++i) {
                pose[i] = glm::inverse(bones.at(i).absolute_transform);
            }
        }

        auto& bones = skeleton_->bones();
        for (auto i = 0; i < pose_.size(); ++i) {
            pose_[i] = pose[i] * bones.at(i).absolute_transform;
        }
    }

    void animation_component::draw() const noexcept {
        mesh_->draw();
    }

    void animation_component::load(const std::string& mesh) {
        auto& rendering_system = game_.rendering_system();

        mesh_ = rendering_system.skinned_mesh_manager().load(mesh);
        if (!mesh_) {
            log(LOG_FATAL, "could not load mesh from file " + mesh);
        }
    }

    void animation_component::load_skeleton(const std::string& skeleton) {
        skeleton_ = game_.rendering_system().skeleton_manager().load(skeleton);
        if (!skeleton_) {
            log(LOG_FATAL, "could not load skeleton " + skeleton);
        }

        auto& bones = skeleton_->bones();
        for (auto i = 0u; i < bones.size(); ++i) {
            pose_.emplace_back(bones[i].absolute_transform);
        }

        current_keyframes_.resize(bones.size());
        next_keyframes_.resize(bones.size());
    }

    glm::vec3 animation_component::interpolate_translation_keyframe(const track& track, frame& current_frame, frame& next_frame, float fps) {
        if (track.tkeys.size() != 1) {
            auto t1 = track.tkeys[current_frame.tkey].time * fps;
            auto t2 = track.tkeys[next_frame.tkey].time * fps;
            if (elapsed_time_ >= t2 && next_frame.tkey < track.tkeys.size() - 1) {
                ++current_frame.tkey;
                ++next_frame.tkey;
                t1 = track.tkeys[current_frame.tkey].time * fps;
                t2 = track.tkeys[next_frame.tkey].time * fps;
            }

            auto v1 = track.tkeys[current_frame.tkey].translate;
            auto v2 = track.tkeys[next_frame.tkey].translate;

            auto delta = (elapsed_time_ - t1) / (t2 - t1);
            delta = delta > 1.f ? 1.f : delta;

            return glm::lerp(v1, v2, delta);
        } else {
            return track.tkeys[current_frame.tkey].translate;
        }
    }

    glm::quat animation_component::interpolate_quaternion_keyframe(const track& track, frame& current_frame, frame& next_frame, float fps) {
        if (track.qkeys.size() != 1) {
            auto t1 = track.qkeys[current_frame.qkey].time * fps;
            auto t2 = track.qkeys[next_frame.qkey].time * fps;
            if (elapsed_time_ >= t2 && next_frame.qkey < track.qkeys.size() - 1) {
                ++current_frame.qkey;
                ++next_frame.qkey;
                t1 = track.qkeys[current_frame.qkey].time * fps;
                t2 = track.qkeys[next_frame.qkey].time * fps;
            }

            auto q1 = track.qkeys[current_frame.qkey].rotate;
            q1 = glm::normalize(q1);
            auto q2 = track.qkeys[next_frame.qkey].rotate;
            q2 = glm::normalize(q2);

            auto delta = (elapsed_time_ - t1) / (t2 - t1);
            delta = delta > 1.f ? 1.f : delta;

            return glm::normalize(glm::lerp(q1, q2, delta));
        } else {
            return track.qkeys[current_frame.qkey].rotate;
        }
    }

    void animation_component::register_at_script_engine(game& game) {
        auto& scripting_system = game.scripting_system();

        scripting_system.register_type<animation_component>("animation_component");

        scripting_system.register_member_function("animation_component", "void play_ani(const string& in)",
            +[](animation_component& component, const std::string& state) { component.change_state(state); });
        scripting_system.register_member_function("animation_component", "void blend_ani(const string& in)",
            +[](animation_component& component, const std::string& state) { component.change_state_blend(state); });
        scripting_system.register_member_function("animation_component", "void is_playing(const string& in)",
            +[](animation_component& component, const std::string& state) { return component.is_playing(state); });
        scripting_system.register_member_function("animation_component", "void is_blending()",
            +[](animation_component& component) { component.is_blending(); });

        scripting_system.register_member_function("entity_impl",
            "animation_component& add_animation_component(const string& in mesh, const string& in skeleton)",
            +[](entity& owner, const std::string& mesh, const std::string& skeleton) -> animation_component& {
                return owner.emplace<animation_component>(mesh, skeleton);
            });
        scripting_system.register_member_function("entity_impl", "animation_component@ get_animation_component()",
            +[](entity& owner) { return owner.component<animation_component>(); });
    }

    animation_component::animation_component(game& game, entity& owner)
    : reflective(game, owner), current_state_{""}, next_state_{""}, elapsed_time_{0.f}, blend_{false}, blend_next_{false} {
        game_.rendering_system().register_component(this);
        game_.animation_system().register_component(this);
    }

    void animation_component::register_reflection() {
        register_property<std::string>("mesh", nullptr, &animation_component::load);
        register_property<std::string>("skeleton", nullptr, &animation_component::load_skeleton);
    }
}
<commit_msg>fix wrong return type of angelscript functions in animation_component<commit_after>#include <stack>

#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtx/compatibility.hpp>

#include <zombye/core/game.hpp>
#include <zombye/rendering/animation_component.hpp>
#include <zombye/rendering/animation_system.hpp>
#include <zombye/rendering/rendering_system.hpp>
#include <zombye/rendering/staticmesh_component.hpp>
#include <zombye/rendering/skinned_mesh.hpp>
#include <zombye/rendering/skeleton.hpp>
#include <zombye/rendering/texture.hpp>
#include <zombye/scripting/scripting_system.hpp>
#include <zombye/utils/logger.hpp>

#include <glm/gtx/string_cast.hpp>

namespace zombye {
    animation_component::animation_component(game& game, entity& owner, const std::string& mesh, const std::string& skeleton)
    : reflective{game, owner}, current_state_{""}, elapsed_time_{0.f}, blend_{false}, blend_next_{false} {
        game_.rendering_system().register_component(this);
        game_.animation_system().register_component(this);
        load(mesh);
        load_skeleton(skeleton);
    }

    animation_component::~animation_component() noexcept {
        game_.rendering_system().unregister_component(this);
        game_.animation_system().unregister_component(this);
    }

    void animation_component::update(float delta_time) {
        static std::vector<glm::mat4> pose(skeleton_->bones().size(), glm::mat4{1.f});
        static const float fps = 1.f / 24.f;
        pose.assign(skeleton_->bones().size(), glm::mat4{1.f});

        if (current_state_ != "") {
            auto& bones = skeleton_->bones();
            auto& bone_hierachy = skeleton_->bone_hierachy_;

            elapsed_time_ += delta_time;

            if (blend_ && next_state_ != "") {
                auto& animation = skeleton_->animation(current_state_);
                auto length = animation.length * fps;
                auto& next_animation = skeleton_->animation(next_state_);
                auto next_animation_length = next_animation.length * fps;
                auto delta_factor = next_animation_length <= length ? 1.f / next_animation_length : 1.f / length;

                if (elapsed_time_ <= next_animation_length) {
                    if (elapsed_time_ >= length) {
                        current_state_ = next_state_;
                        next_state_ = "";
                        blend_ = false;
                        current_keyframes_ = next_keyframes_;
                        return;
                    } else {
                        std::stack<int> traversal;
                        traversal.push(0);
                        while(!traversal.empty()) {
                            auto i = traversal.top();
                            traversal.pop();

                            auto& cs_track = animation.tracks.at(i);

                            auto& cs_current_frame = current_keyframes_.at(i);
                            auto cs_next_frame = cs_current_frame;
                            ++cs_next_frame.tkey;
                            ++cs_next_frame.qkey;
                            ++cs_next_frame.skey;

                            auto iv_current = interpolate_translation_keyframe(cs_track, cs_current_frame, cs_next_frame, fps);
                            auto iq_current = interpolate_quaternion_keyframe(cs_track, cs_current_frame, cs_next_frame, fps);

                            auto& ns_track = next_animation.tracks.at(i);

                            auto& ns_current_frame = next_keyframes_.at(i);
                            auto ns_next_frame = ns_current_frame;
                            ++ns_next_frame.tkey;
                            ++ns_next_frame.qkey;
                            ++ns_next_frame.skey;

                            auto iv_next = interpolate_translation_keyframe(ns_track, ns_current_frame, ns_next_frame, fps);
                            auto iq_next = interpolate_quaternion_keyframe(ns_track, ns_current_frame, ns_next_frame, fps);

                            auto delta = elapsed_time_ * delta_factor;
                            delta = delta > 1.f ? 1.f : delta;
                            auto iv = glm::lerp(iv_current, iv_next, delta);
                            auto iq = glm::lerp(iq_current, iq_next, delta);

                            auto p = glm::toMat4(iq);
                            p[3].x = iv.x;
                            p[3].y = iv.y;
                            p[3].z = iv.z;

                            pose[i] *= bones.at(i).relative_transform * p;
                            if (bone_hierachy.find(i) != bone_hierachy.end()) {
                                for (auto node = bone_hierachy.begin(i); node != bone_hierachy.end(i); ++node) {
                                    auto& child = node->second;
                                    pose[child] *= pose[i];
                                    traversal.push(child);
                                }
                            }
                        }
                    }
                } else {
                    elapsed_time_ = 0.f;
                    current_keyframes_.assign(current_keyframes_.size(), frame{});
                    next_keyframes_.assign(next_keyframes_.size(), frame{});
                    current_state_ = next_state_;
                    next_state_ = "";
                    blend_ = false;
                    return;
                }
            } else {
                auto& animation = skeleton_->animation(current_state_);
                auto length = animation.length * fps;
                if (elapsed_time_ <= length) {
                    std::stack<int> traversal;
                    traversal.push(0);
                    while(!traversal.empty()) {
                        auto i = traversal.top();
                        traversal.pop();

                        auto& track = animation.tracks.at(i);

                        auto& current_frame = current_keyframes_.at(i);
                        auto next_frame = current_frame;
                        ++next_frame.tkey;
                        ++next_frame.qkey;
                        ++next_frame.skey;

                        auto iv = interpolate_translation_keyframe(track, current_frame, next_frame, fps);
                        auto iq = interpolate_quaternion_keyframe(track, current_frame, next_frame, fps);

                        auto p = glm::toMat4(iq);
                        p[3].x = iv.x;
                        p[3].y = iv.y;
                        p[3].z = iv.z;

                        pose[i] *= bones.at(i).relative_transform * p;
                        if (bone_hierachy.find(i) != bone_hierachy.end()) {
                            for (auto node = bone_hierachy.begin(i); node != bone_hierachy.end(i); ++node) {
                                auto& child = node->second;
                                pose[child] *= pose[i];
                                traversal.push(child);
                            }
                        }
                    }
                } else {
                    elapsed_time_ = 0.f;
                    for (auto& k : current_keyframes_) {
                        k.tkey = 0;
                        k.qkey = 0;
                        k.skey = 0;
                    }
                    if (blend_next_) {
                        blend_ = true;
                        blend_next_ = false;
                    }
                    return;
                }
            }
        } else {
            auto& bones = skeleton_->bones();
            for (auto i = 0u; i < bones.size(); ++i) {
                pose[i] = glm::inverse(bones.at(i).absolute_transform);
            }
        }

        auto& bones = skeleton_->bones();
        for (auto i = 0; i < pose_.size(); ++i) {
            pose_[i] = pose[i] * bones.at(i).absolute_transform;
        }
    }

    void animation_component::draw() const noexcept {
        mesh_->draw();
    }

    void animation_component::load(const std::string& mesh) {
        auto& rendering_system = game_.rendering_system();

        mesh_ = rendering_system.skinned_mesh_manager().load(mesh);
        if (!mesh_) {
            log(LOG_FATAL, "could not load mesh from file " + mesh);
        }
    }

    void animation_component::load_skeleton(const std::string& skeleton) {
        skeleton_ = game_.rendering_system().skeleton_manager().load(skeleton);
        if (!skeleton_) {
            log(LOG_FATAL, "could not load skeleton " + skeleton);
        }

        auto& bones = skeleton_->bones();
        for (auto i = 0u; i < bones.size(); ++i) {
            pose_.emplace_back(bones[i].absolute_transform);
        }

        current_keyframes_.resize(bones.size());
        next_keyframes_.resize(bones.size());
    }

    glm::vec3 animation_component::interpolate_translation_keyframe(const track& track, frame& current_frame, frame& next_frame, float fps) {
        if (track.tkeys.size() != 1) {
            auto t1 = track.tkeys[current_frame.tkey].time * fps;
            auto t2 = track.tkeys[next_frame.tkey].time * fps;
            if (elapsed_time_ >= t2 && next_frame.tkey < track.tkeys.size() - 1) {
                ++current_frame.tkey;
                ++next_frame.tkey;
                t1 = track.tkeys[current_frame.tkey].time * fps;
                t2 = track.tkeys[next_frame.tkey].time * fps;
            }

            auto v1 = track.tkeys[current_frame.tkey].translate;
            auto v2 = track.tkeys[next_frame.tkey].translate;

            auto delta = (elapsed_time_ - t1) / (t2 - t1);
            delta = delta > 1.f ? 1.f : delta;

            return glm::lerp(v1, v2, delta);
        } else {
            return track.tkeys[current_frame.tkey].translate;
        }
    }

    glm::quat animation_component::interpolate_quaternion_keyframe(const track& track, frame& current_frame, frame& next_frame, float fps) {
        if (track.qkeys.size() != 1) {
            auto t1 = track.qkeys[current_frame.qkey].time * fps;
            auto t2 = track.qkeys[next_frame.qkey].time * fps;
            if (elapsed_time_ >= t2 && next_frame.qkey < track.qkeys.size() - 1) {
                ++current_frame.qkey;
                ++next_frame.qkey;
                t1 = track.qkeys[current_frame.qkey].time * fps;
                t2 = track.qkeys[next_frame.qkey].time * fps;
            }

            auto q1 = track.qkeys[current_frame.qkey].rotate;
            q1 = glm::normalize(q1);
            auto q2 = track.qkeys[next_frame.qkey].rotate;
            q2 = glm::normalize(q2);

            auto delta = (elapsed_time_ - t1) / (t2 - t1);
            delta = delta > 1.f ? 1.f : delta;

            return glm::normalize(glm::lerp(q1, q2, delta));
        } else {
            return track.qkeys[current_frame.qkey].rotate;
        }
    }

    void animation_component::register_at_script_engine(game& game) {
        auto& scripting_system = game.scripting_system();

        scripting_system.register_type<animation_component>("animation_component");

        scripting_system.register_member_function("animation_component", "void play_ani(const string& in)",
            +[](animation_component& component, const std::string& state) { component.change_state(state); });
        scripting_system.register_member_function("animation_component", "void blend_ani(const string& in)",
            +[](animation_component& component, const std::string& state) { component.change_state_blend(state); });
        scripting_system.register_member_function("animation_component", "bool is_playing(const string& in)",
            +[](animation_component& component, const std::string& state) { return component.is_playing(state); });
        scripting_system.register_member_function("animation_component", "bool is_blending()",
            +[](animation_component& component) { return component.is_blending(); });

        scripting_system.register_member_function("entity_impl",
            "animation_component& add_animation_component(const string& in mesh, const string& in skeleton)",
            +[](entity& owner, const std::string& mesh, const std::string& skeleton) -> animation_component& {
                return owner.emplace<animation_component>(mesh, skeleton);
            });
        scripting_system.register_member_function("entity_impl", "animation_component@ get_animation_component()",
            +[](entity& owner) { return owner.component<animation_component>(); });
    }

    animation_component::animation_component(game& game, entity& owner)
    : reflective(game, owner), current_state_{""}, next_state_{""}, elapsed_time_{0.f}, blend_{false}, blend_next_{false} {
        game_.rendering_system().register_component(this);
        game_.animation_system().register_component(this);
    }

    void animation_component::register_reflection() {
        register_property<std::string>("mesh", nullptr, &animation_component::load);
        register_property<std::string>("skeleton", nullptr, &animation_component::load_skeleton);
    }
}
<|endoftext|>


<commit_before>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Id$
 * $Log$
 * Revision 1.6  2001/10/09 20:50:27  peiyongz
 * init(): take 1 arg
 *
 * Revision 1.5  2001/09/27 13:51:25  peiyongz
 * DTV Reorganization: ctor/init created to be used by derived class
 *
 * Revision 1.4  2001/09/24 21:39:29  peiyongz
 * DTV Reorganization: checkValueSpace()
 *
 * Revision 1.3  2001/08/14 22:11:56  peiyongz
 * new exception message added
 *
 * Revision 1.2  2001/07/24 21:23:39  tng
 * Schema: Use DatatypeValidator for ID/IDREF/ENTITY/ENTITIES/NOTATION.
 *
 * Revision 1.1  2001/07/06 20:19:52  peiyongz
 * ENTITYDTV
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <validators/datatype/ENTITYDatatypeValidator.hpp>
#include <validators/datatype/InvalidDatatypeValueException.hpp>
#include <framework/XMLEntityDecl.hpp>

// ---------------------------------------------------------------------------
//  Constructors and Destructor
// ---------------------------------------------------------------------------
ENTITYDatatypeValidator::ENTITYDatatypeValidator()
:StringDatatypeValidator(0, 0, 0, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{}

ENTITYDatatypeValidator::ENTITYDatatypeValidator(
                          DatatypeValidator*            const baseValidator
                        , RefHashTableOf<KVStringPair>* const facets
                        , RefVectorOf<XMLCh>*           const enums
                        , const int                           finalSet)
:StringDatatypeValidator(baseValidator, facets, finalSet, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{
    init(enums);
}

ENTITYDatatypeValidator::~ENTITYDatatypeValidator()
{}

DatatypeValidator* ENTITYDatatypeValidator::newInstance(
                                      RefHashTableOf<KVStringPair>* const facets
                                    , RefVectorOf<XMLCh>*           const enums
                                    , const int                           finalSet)
{
    return (DatatypeValidator*) new ENTITYDatatypeValidator(this, facets, enums, finalSet);
}

// -----------------------------------------------------------------------
// Compare methods
// -----------------------------------------------------------------------
int ENTITYDatatypeValidator::compare(const XMLCh* const lValue
                                   , const XMLCh* const rValue)
{
    return ( XMLString::compareString(lValue, rValue)==0 ? 0 : -1);
}

void ENTITYDatatypeValidator::validate(const XMLCh* const content)
{
    // use StringDatatypeValidator (which in turn, invoke
    // the baseValidator) to validate content against
    // facets if any.
    //
    StringDatatypeValidator::validate(content);

    //
    // parse the entity iff an EntityDeclPool is provided
    //
    if (fEntityDeclPool)
    {
        DTDEntityDecl* decl = fEntityDeclPool->getByKey(content);

        if (!decl                ||
            (decl->isUnparsed())  )
        {
            ThrowXML1(InvalidDatatypeValueException
                    , XMLExcepts::VALUE_ENTITY_Invalid
                    , content);
        }

    }

    return;
}

void ENTITYDatatypeValidator::checkValueSpace(const XMLCh* const content)
{
    //
    // 3.3.11 check must: "NCName"
    //
    if ( !XMLString::isValidNCName(content))
    {
        ThrowXML1(InvalidDatatypeValueException
                , XMLExcepts::VALUE_Invalid_NCName
                , content);
    }

}

 /**
  * End of file ENTITYDatatypeValidator.cpp
  */
<commit_msg>Schema fix: Entities should be unparsed.<commit_after>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Id$
 * $Log$
 * Revision 1.7  2001/11/13 13:21:59  tng
 * Schema fix: Entities should be unparsed.
 *
 * Revision 1.6  2001/10/09 20:50:27  peiyongz
 * init(): take 1 arg
 *
 * Revision 1.5  2001/09/27 13:51:25  peiyongz
 * DTV Reorganization: ctor/init created to be used by derived class
 *
 * Revision 1.4  2001/09/24 21:39:29  peiyongz
 * DTV Reorganization: checkValueSpace()
 *
 * Revision 1.3  2001/08/14 22:11:56  peiyongz
 * new exception message added
 *
 * Revision 1.2  2001/07/24 21:23:39  tng
 * Schema: Use DatatypeValidator for ID/IDREF/ENTITY/ENTITIES/NOTATION.
 *
 * Revision 1.1  2001/07/06 20:19:52  peiyongz
 * ENTITYDTV
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <validators/datatype/ENTITYDatatypeValidator.hpp>
#include <validators/datatype/InvalidDatatypeValueException.hpp>
#include <framework/XMLEntityDecl.hpp>

// ---------------------------------------------------------------------------
//  Constructors and Destructor
// ---------------------------------------------------------------------------
ENTITYDatatypeValidator::ENTITYDatatypeValidator()
:StringDatatypeValidator(0, 0, 0, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{}

ENTITYDatatypeValidator::ENTITYDatatypeValidator(
                          DatatypeValidator*            const baseValidator
                        , RefHashTableOf<KVStringPair>* const facets
                        , RefVectorOf<XMLCh>*           const enums
                        , const int                           finalSet)
:StringDatatypeValidator(baseValidator, facets, finalSet, DatatypeValidator::ENTITY)
,fEntityDeclPool(0)
{
    init(enums);
}

ENTITYDatatypeValidator::~ENTITYDatatypeValidator()
{}

DatatypeValidator* ENTITYDatatypeValidator::newInstance(
                                      RefHashTableOf<KVStringPair>* const facets
                                    , RefVectorOf<XMLCh>*           const enums
                                    , const int                           finalSet)
{
    return (DatatypeValidator*) new ENTITYDatatypeValidator(this, facets, enums, finalSet);
}

// -----------------------------------------------------------------------
// Compare methods
// -----------------------------------------------------------------------
int ENTITYDatatypeValidator::compare(const XMLCh* const lValue
                                   , const XMLCh* const rValue)
{
    return ( XMLString::compareString(lValue, rValue)==0 ? 0 : -1);
}

void ENTITYDatatypeValidator::validate(const XMLCh* const content)
{
    // use StringDatatypeValidator (which in turn, invoke
    // the baseValidator) to validate content against
    // facets if any.
    //
    StringDatatypeValidator::validate(content);

    //
    // parse the entity iff an EntityDeclPool is provided
    //
    if (fEntityDeclPool)
    {
        DTDEntityDecl* decl = fEntityDeclPool->getByKey(content);

        if (!decl                ||
            (!decl->isUnparsed())  )
        {
            ThrowXML1(InvalidDatatypeValueException
                    , XMLExcepts::VALUE_ENTITY_Invalid
                    , content);
        }

    }

    return;
}

void ENTITYDatatypeValidator::checkValueSpace(const XMLCh* const content)
{
    //
    // 3.3.11 check must: "NCName"
    //
    if ( !XMLString::isValidNCName(content))
    {
        ThrowXML1(InvalidDatatypeValueException
                , XMLExcepts::VALUE_Invalid_NCName
                , content);
    }

}

 /**
  * End of file ENTITYDatatypeValidator.cpp
  */
<|endoftext|>


<commit_before>#include "arch/linux/disk/pool.hpp"

#define BLOCKER_POOL_QUEUE_DEPTH (MAX_CONCURRENT_IO_REQUESTS * 2)

pool_diskmgr_t::pool_diskmgr_t(
        linux_event_queue_t *queue, passive_producer_t<action_t *> *source) :
    source(source),
    blocker_pool(MAX_CONCURRENT_IO_REQUESTS, queue),
    n_pending(0)
{
    if (source->available->get()) pump();
    source->available->add_watcher(this);
}

pool_diskmgr_t::~pool_diskmgr_t() {
    source->available->remove_watcher(this);
}

void pool_diskmgr_t::action_t::run() {
    int res;
    if (is_read) res = pread(fd, buf, count, offset);
    else res = pwrite(fd, buf, count, offset);
    guarantee_err((size_t)res == count, "pread()/pwrite() failed.");
}

void pool_diskmgr_t::action_t::done() {
    parent->n_pending--;
    parent->pump();
    parent->done_fun(this);
}

void pool_diskmgr_t::on_watchable_changed() {
    /* This is called when the queue used to be empty but now has requests on
    it, and also when the queue's last request is consumed. */
    if (source->available->get()) pump();
}

void pool_diskmgr_t::pump() {
    while (source->available->get() && n_pending < BLOCKER_POOL_QUEUE_DEPTH) {
        action_t *a = source->pop();
        a->parent = this;
        n_pending++;
        blocker_pool.do_job(a);
    }
}

<commit_msg>Added include line for config/args.hpp in pool.cc.<commit_after>#include "arch/linux/disk/pool.hpp"

#include "config/args.hpp"

#define BLOCKER_POOL_QUEUE_DEPTH (MAX_CONCURRENT_IO_REQUESTS * 2)

pool_diskmgr_t::pool_diskmgr_t(
        linux_event_queue_t *queue, passive_producer_t<action_t *> *source) :
    source(source),
    blocker_pool(MAX_CONCURRENT_IO_REQUESTS, queue),
    n_pending(0)
{
    if (source->available->get()) pump();
    source->available->add_watcher(this);
}

pool_diskmgr_t::~pool_diskmgr_t() {
    source->available->remove_watcher(this);
}

void pool_diskmgr_t::action_t::run() {
    int res;
    if (is_read) res = pread(fd, buf, count, offset);
    else res = pwrite(fd, buf, count, offset);
    guarantee_err((size_t)res == count, "pread()/pwrite() failed.");
}

void pool_diskmgr_t::action_t::done() {
    parent->n_pending--;
    parent->pump();
    parent->done_fun(this);
}

void pool_diskmgr_t::on_watchable_changed() {
    /* This is called when the queue used to be empty but now has requests on
    it, and also when the queue's last request is consumed. */
    if (source->available->get()) pump();
}

void pool_diskmgr_t::pump() {
    while (source->available->get() && n_pending < BLOCKER_POOL_QUEUE_DEPTH) {
        action_t *a = source->pop();
        a->parent = this;
        n_pending++;
        blocker_pool.do_job(a);
    }
}

<|endoftext|>


<commit_before>// Copyright 2017 Adam Smith
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cctype>
#include <sstream>
#include "as/serial_value/xml.hpp"

#define ASMITH_SERIAL_PTR "as::serial_value::pointer_t="
#define ASMITH_SERIAL_XML "__as_serial_xml_element"

bool write_attribute(std::ostream& aStream, const std::string& aName, const as::serial_value& aValue) {
	//! \todo Remove chars illegal in XML
	switch (aValue.get_type()) {
	case as::serial_value::NULL_T:
		aStream << ' ' << aName << '=' << '"' << "null" << '"';
		return true;
	case as::serial_value::CHAR_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_char() << '"';
		return true;
	case as::serial_value::BOOL_T:
		aStream << ' ' << aName << '=' << '"' << (aValue.get_bool() ? "true" : "false") << '"';
		return true;
	case as::serial_value::UNSIGNED_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_unsigned() << '"';
		return true;
	case as::serial_value::SIGNED_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_signed() << '"';
		return true;
	case as::serial_value::FLOAT_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_float() << '"';
		return true;
	case as::serial_value::POINTER_T:
		aStream << ' ' << aName << '=' << '"' << ASMITH_SERIAL_PTR  << aValue.get_pointer() << '"';
		return true;
	case as::serial_value::STRING_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_string() << '"';
		return true;
	default:
		return false;
	}
}

void write_element(std::ostream& aStream, const std::string& aName, const as::serial_value& aValue) {
	//! \todo Remove chars illegal in XML
	switch (aValue.get_type()) {
	case as::serial_value::NULL_T:
		aStream << '<' << aName << '/' << '>';
		break;
	case as::serial_value::CHAR_T:
		aStream << '<' << aName << '>' << aValue.get_char() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::BOOL_T:
		aStream << '<' << aName << '>' << (aValue.get_bool() ? "true" : "false") << '<' << '/' << aName << '>';
		break;
	case as::serial_value::UNSIGNED_T:
		aStream << '<' << aName << '>' << aValue.get_unsigned() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::SIGNED_T:
		aStream << '<' << aName << '>' << aValue.get_signed() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::FLOAT_T:
		aStream << '<' << aName << '>' << aValue.get_float() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::POINTER_T:
		aStream << '<' << aName << '>' << ASMITH_SERIAL_PTR << aValue.get_pointer() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::STRING_T:
		aStream << '<' << aName << '>' << aValue.get_string() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::ARRAY_T:
		{
			aStream << '<' << aName << '>';
			const as::serial_value::array_t& array_ = aValue.get_array();
			const size_t s = array_.size();
			for(size_t i = 0; i < s; ++i) write_element(aStream, std::to_string(i), array_[i]);
			aStream << '<' << '/' << aName << '>';
		}
		break;
	case as::serial_value::OBJECT_T:
		{
			const as::serial_value::object_t& object = aValue.get_object();
			aStream << '<' << aName;
			size_t attributes = 0;
			for(const auto& i : object) {
				const as::serial_value::type t = i.second.get_type();
				if(t != as::serial_value::ARRAY_T && t != as::serial_value::OBJECT_T) {
					write_attribute(aStream, i.first, i.second);
					++attributes;
				}
			}

			if(attributes == object.size()) {
				aStream << '/' << '>';
			}else {
				aStream << '>';
				for(const auto& i : object) {
					const as::serial_value::type t = i.second.get_type();
					if(t == as::serial_value::ARRAY_T || t == as::serial_value::OBJECT_T) {
						write_element(aStream, i.first, i.second);
						++attributes;
					}
				}
				aStream << '<' << '/' << aName << '>';
			}
		}
		break;
	}
}

struct element {
	std::string name;
	std::string body;
	std::map<std::string, std::string> attributes;
	std::vector<element> elements;
};

void ignore_whitespace_x(std::istream& aStream) {
	char c = aStream.peek();
	while(std::isspace(c)) {
		aStream.read(&c, 1);
		c = aStream.peek();
	}
}

void close_closing_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;
	aStream.read(&c, 1);
	if(c != '>') throw std::runtime_error("as::deserialise_xml : Expected closing tag to end with '>'");
}

void read_closing_name(std::istream& aStream, element& e) {
	char c;
	ignore_whitespace_x(aStream);
	c = aStream.peek();
	std::string name;
	while(c != '>' && ! std::isspace(c)) {
		name += c;
		aStream.read(&c, 1);
		c = aStream.peek();
	}
	if(name != e.name) throw std::runtime_error("as::deserialise_xml : Expected closing tag to have same name as opening tag");
	// Next state
	close_closing_tag(aStream, e);
}

void open_closing_tag(std::istream& aStream, element& e) {
	char c;
	aStream.read(&c, 1);
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected closing tag to begin with '</'");
	aStream.read(&c, 1);
	if(c != '/') throw std::runtime_error("as::deserialise_xml : Expected closing tag to begin with '</'");
	// Next state
	read_closing_name(aStream, e);
}

void open_opening_tag(std::istream&, element&);

void read_elements(std::istream& aStream, element& e) {
	const auto pos = aStream.tellg();
	char c;
	c = aStream.peek();
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected element to begin with '<'");
	c = aStream.peek();
	aStream.read(&c, 1);
	c = aStream.peek();
	aStream.seekg(pos);
	if(c != '/') {
		element e2;
		open_opening_tag(aStream, e2);
		e.elements.push_back(e2);
		// Next state
		read_elements(aStream, e);
	}
}

void read_body(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;

	c = aStream.peek();
	if(c == '<') {
		read_elements(aStream, e);
	}else {
		 while (c != '<') {
			 e.body += c;
			 aStream.read(&c, 1);
			 c = aStream.peek();
		 }
	}

	// Next state
	open_closing_tag(aStream, e);
}

void close_opening_tag(std::istream&, element&);

void read_attribute(std::istream& aStream, element& e) {
	std::string name;
	std::string value;
	char c;

	// Read name
	ignore_whitespace_x(aStream);
	aStream.read(&c, 1);
	while(c != '='){
		name += c;
		aStream.read(&c, 1);
	}

	// Read value
	ignore_whitespace_x(aStream);
	aStream.read(&c, 1);
	if(c != '"') throw std::runtime_error("as::deserialise_xml : Expected attribute value to begin with '\"'");
	aStream.read(&c, 1);
	while(c != '"') {
		value += c;
		aStream.read(&c, 1);
	}

	// Next state
	e.attributes.emplace(name, value);
	close_opening_tag(aStream, e);
}

void close_opening_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c = aStream.peek();
	if(c == '/') {
		// End element
		aStream.read(&c, 1);
		aStream.read(&c, 1);
		if(c != '>') throw std::runtime_error("as::deserialise_xml : Expected opening tag to end with '>'");
	}else if(c == '>') {
		aStream.read(&c, 1);
		// Next state
		read_body(aStream, e);
	}else {
		read_attribute(aStream, e);
	}
}

void read_opening_tag_name(std::istream& aStream, element& e) {
	char c;
	ignore_whitespace_x(aStream);
	c = aStream.peek();
	while(c != '>' && c != '/' && ! std::isspace(c)) {
		e.name += c;
		aStream.read(&c, 1);
		c = aStream.peek();
	}
	// Next state
	close_opening_tag(aStream, e);
}

void open_opening_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;
	aStream.read(&c, 1);
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected opening tag to begin with '<'");
	// Next state
	read_opening_tag_name(aStream, e);
}

as::serial_value convert_element(const element& aElement) {
	// Check if array
	size_t contiguousNames = 0;
	const size_t eCount = aElement.elements.size();
	for(size_t i = 0; i < eCount; ++i) if(aElement.elements[i].name == std::to_string(i)) ++contiguousNames;
	if(eCount > 0 && contiguousNames == eCount) {
		// Element is an array
		as::serial_value tmp;
		as::serial_value::array_t& array_ = tmp.set_array();
		for(size_t i = 0; i < eCount; ++i) array_.push_back(convert_element(aElement.elements[i]));
		return tmp;
	}

	// Check if value
	if(aElement.elements.empty() && aElement.attributes.empty()) {
		// Element is a value
		if(aElement.body.empty()) return as::serial_value();
		else return as::serial_value(aElement.body);
	}

	// Element is an object
	as::serial_value tmp;
	as::serial_value::object_t& object = tmp.set_object();
	if(! aElement.body.empty()) {
		//! \todo Handle body
	}
	for(const auto& i : aElement.attributes) {
		object.emplace(i.first, as::serial_value(i.second));
	}
	for(const auto& i : aElement.elements) {
		//! \todo Handle for name conflicts
		object.emplace(i.name, convert_element(i));
	}
	return tmp;
}

namespace as {
	void serialise_xml(std::ostream& aStream, const serial_value& aValue) {
		write_element(aStream, ASMITH_SERIAL_XML, aValue);
	}

	serial_value deserialise_xml(std::istream& aStream) {
		//! \todo Handle XML header
		//! \todo Handle XML comments
		element e;
		open_opening_tag(aStream, e);
		return convert_element(e);
	}

}<commit_msg>Located bug of unknown origin<commit_after>// Copyright 2017 Adam Smith
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cctype>
#include <sstream>
#include "as/serial_value/xml.hpp"

#define ASMITH_SERIAL_PTR "as::serial_value::pointer_t="
#define ASMITH_SERIAL_XML "__as_serial_xml_element"

bool write_attribute(std::ostream& aStream, const std::string& aName, const as::serial_value& aValue) {
	//! \todo Remove chars illegal in XML
	switch (aValue.get_type()) {
	case as::serial_value::NULL_T:
		aStream << ' ' << aName << '=' << '"' << "null" << '"';
		return true;
	case as::serial_value::CHAR_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_char() << '"';
		return true;
	case as::serial_value::BOOL_T:
		aStream << ' ' << aName << '=' << '"' << (aValue.get_bool() ? "true" : "false") << '"';
		return true;
	case as::serial_value::UNSIGNED_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_unsigned() << '"';
		return true;
	case as::serial_value::SIGNED_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_signed() << '"';
		return true;
	case as::serial_value::FLOAT_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_float() << '"';
		return true;
	case as::serial_value::POINTER_T:
		aStream << ' ' << aName << '=' << '"' << ASMITH_SERIAL_PTR  << aValue.get_pointer() << '"';
		return true;
	case as::serial_value::STRING_T:
		aStream << ' ' << aName << '=' << '"' << aValue.get_string() << '"';
		return true;
	default:
		return false;
	}
}

void write_element(std::ostream& aStream, const std::string& aName, const as::serial_value& aValue) {
	//! \todo Remove chars illegal in XML
	switch (aValue.get_type()) {
	case as::serial_value::NULL_T:
		aStream << '<' << aName << '/' << '>';
		break;
	case as::serial_value::CHAR_T:
		aStream << '<' << aName << '>' << aValue.get_char() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::BOOL_T:
		aStream << '<' << aName << '>' << (aValue.get_bool() ? "true" : "false") << '<' << '/' << aName << '>';
		break;
	case as::serial_value::UNSIGNED_T:
		aStream << '<' << aName << '>' << aValue.get_unsigned() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::SIGNED_T:
		aStream << '<' << aName << '>' << aValue.get_signed() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::FLOAT_T:
		aStream << '<' << aName << '>' << aValue.get_float() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::POINTER_T:
		aStream << '<' << aName << '>' << ASMITH_SERIAL_PTR << aValue.get_pointer() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::STRING_T:
		aStream << '<' << aName << '>' << aValue.get_string() << '<' << '/' << aName << '>';
		break;
	case as::serial_value::ARRAY_T:
		{
			aStream << '<' << aName << '>';
			const as::serial_value::array_t& array_ = aValue.get_array();
			const size_t s = array_.size();
			for(size_t i = 0; i < s; ++i) write_element(aStream, std::to_string(i), array_[i]);
			aStream << '<' << '/' << aName << '>';
		}
		break;
	case as::serial_value::OBJECT_T:
		{
			const as::serial_value::object_t& object = aValue.get_object();
			aStream << '<' << aName;
			size_t attributes = 0;
			for(const auto& i : object) {
				const as::serial_value::type t = i.second.get_type();
				if(t != as::serial_value::ARRAY_T && t != as::serial_value::OBJECT_T) {
					write_attribute(aStream, i.first, i.second);
					++attributes;
				}
			}

			if(attributes == object.size()) {
				aStream << '/' << '>';
			}else {
				aStream << '>';
				for(const auto& i : object) {
					const as::serial_value::type t = i.second.get_type();
					if(t == as::serial_value::ARRAY_T || t == as::serial_value::OBJECT_T) {
						write_element(aStream, i.first, i.second);
						++attributes;
					}
				}
				aStream << '<' << '/' << aName << '>';
			}
		}
		break;
	}
}

struct element {
	std::string name;
	std::string body;
	std::map<std::string, std::string> attributes;
	std::vector<element> elements;
};

void ignore_whitespace_x(std::istream& aStream) {
	char c = aStream.peek();
	while(std::isspace(c)) {
		aStream.read(&c, 1);
		c = aStream.peek();
	}
}

void close_closing_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;
	aStream.read(&c, 1);
	if(c != '>') throw std::runtime_error("as::deserialise_xml : Expected closing tag to end with '>'");
}

void read_closing_name(std::istream& aStream, element& e) {
	char c;
	ignore_whitespace_x(aStream);
	c = aStream.peek();
	std::string name;
	while(c != '>' && ! std::isspace(c)) {
		name += c;
		aStream.read(&c, 1);
		c = aStream.peek();
	}
	if(name != e.name) throw std::runtime_error("as::deserialise_xml : Expected closing tag to have same name as opening tag");
	// Next state
	close_closing_tag(aStream, e);
}

void open_closing_tag(std::istream& aStream, element& e) {
	char c;
	aStream.read(&c, 1);
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected closing tag to begin with '</'");
	aStream.read(&c, 1);
	if(c != '/') throw std::runtime_error("as::deserialise_xml : Expected closing tag to begin with '</'");
	// Next state
	read_closing_name(aStream, e);
}

void open_opening_tag(std::istream&, element&);

void read_elements(std::istream& aStream, element& e) {
	const auto pos = aStream.tellg();
	char c;
	c = aStream.peek();
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected element to begin with '<'");
	c = aStream.peek();
	aStream.read(&c, 1);
	c = aStream.peek();
	aStream.seekg(pos);
	if(c != '/') {
		element e2;
		open_opening_tag(aStream, e2);
		e.elements.push_back(e2);
		// Next state
		read_elements(aStream, e);
	}
}

void read_body(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;

	c = aStream.peek();
	if(c == '<') {
		read_elements(aStream, e);
	}else {
		 while (c != '<') {
			 e.body += c;
			 aStream.read(&c, 1);
			 c = aStream.peek();
		 }
	}

	// Next state
	open_closing_tag(aStream, e);
}

void close_opening_tag(std::istream&, element&);

void read_attribute(std::istream& aStream, element& e) {
	std::string name;
	std::string value;
	char c;

	// Read name
	ignore_whitespace_x(aStream);
	aStream.read(&c, 1);
	while(c != '='){
		name += c;
		aStream.read(&c, 1);
	}

	// Read value
	ignore_whitespace_x(aStream);
	aStream.read(&c, 1);
	if(c != '"') throw std::runtime_error("as::deserialise_xml : Expected attribute value to begin with '\"'");
	aStream.read(&c, 1);
	while(c != '"') {
		value += c;
		aStream.read(&c, 1);
	}

	// Next state
	e.attributes.emplace(name, value);
	close_opening_tag(aStream, e);
}

void close_opening_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c = aStream.peek();
	if(c == '/') {
		// End element
		aStream.read(&c, 1);
		aStream.read(&c, 1);
		if(c != '>') throw std::runtime_error("as::deserialise_xml : Expected opening tag to end with '>'");
	}else if(c == '>') {
		aStream.read(&c, 1);
		// Next state
		read_body(aStream, e);
	}else {
		read_attribute(aStream, e);
	}
}

void read_opening_tag_name(std::istream& aStream, element& e) {
	char c;
	ignore_whitespace_x(aStream);
	c = aStream.peek();
	while(c != '>' && c != '/' && ! std::isspace(c)) {
		e.name += c;
		aStream.read(&c, 1);
		c = aStream.peek();
	}
	// Next state
	close_opening_tag(aStream, e);
}

void open_opening_tag(std::istream& aStream, element& e) {
	ignore_whitespace_x(aStream);
	char c;
	aStream.read(&c, 1);
	if(c != '<') throw std::runtime_error("as::deserialise_xml : Expected opening tag to begin with '<'");
	// Next state
	read_opening_tag_name(aStream, e);
}

as::serial_value convert_element(const element& aElement) {
	// Check if array
	size_t contiguousNames = 0;
	const size_t eCount = aElement.elements.size();
	for(size_t i = 0; i < eCount; ++i) if(aElement.elements[i].name == std::to_string(i)) ++contiguousNames;
	if(eCount > 0 && contiguousNames == eCount) {
		// Element is an array
		as::serial_value tmp;
		as::serial_value::array_t& array_ = tmp.set_array();
		for(size_t i = 0; i < eCount; ++i) array_.push_back(convert_element(aElement.elements[i]));
		return tmp;
	}

	// Check if value
	if(aElement.elements.empty() && aElement.attributes.empty()) {
		// Element is a value
		if(aElement.body.empty()) return as::serial_value();
		else return as::serial_value(aElement.body);
	}

	// Element is an object
	as::serial_value tmp;
	as::serial_value::object_t& object = tmp.set_object();
	if(! aElement.body.empty()) {
		//! \todo Handle body
	}
	for(const auto& i : aElement.attributes) {
		//! \bug String pointers in child value become null
		object.emplace(i.first, as::serial_value(i.second));
	}
	for(const auto& i : aElement.elements) {
		//! \todo Handle for name conflicts
		object.emplace(i.name, convert_element(i));
	}
	return tmp;
}

namespace as {
	void serialise_xml(std::ostream& aStream, const serial_value& aValue) {
		write_element(aStream, ASMITH_SERIAL_XML, aValue);
	}

	serial_value deserialise_xml(std::istream& aStream) {
		//! \todo Handle XML header
		//! \todo Handle XML comments
		element e;
		open_opening_tag(aStream, e);
		return convert_element(e);
	}

}<|endoftext|>


<commit_before>/*
 * Copyright (C) 2021-present ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "sstables/consumer.hh"
#include "sstables/types.hh"
#include "sstables/sstables.hh"
#include "sstables/processing_result_generator.hh"

// Expose internal types for tests

namespace sstables {
namespace kl {

// sstables::data_consume_row feeds the contents of a single row into a
// row_consumer object:
//
// * First, consume_row_start() is called, with some information about the
//   whole row: The row's key, timestamp, etc.
// * Next, consume_cell() is called once for every column.
// * Finally, consume_row_end() is called. A consumer written for a single
//   column will likely not want to do anything here.
//
// Important note: the row key, column name and column value, passed to the
// consume_* functions, are passed as a "bytes_view" object, which points to
// internal data held by the feeder. This internal data is only valid for the
// duration of the single consume function it was passed to. If the object
// wants to hold these strings longer, it must make a copy of the bytes_view's
// contents. [Note, in reality, because our implementation reads the whole
// row into one buffer, the byte_views remain valid until consume_row_end()
// is called.]
class row_consumer {
    reader_permit _permit;
    tracing::trace_state_ptr _trace_state;
    const io_priority_class& _pc;

public:
    using proceed = data_consumer::proceed;

    /*
     * In k/l formats, RTs are represented as cohesive entries so
     * setting/resetting RT start is not supported.
     */
    constexpr static bool is_setting_range_tombstone_start_supported = false;

    row_consumer(reader_permit permit, tracing::trace_state_ptr trace_state, const io_priority_class& pc)
        : _permit(std::move(permit))
        , _trace_state(std::move(trace_state))
        , _pc(pc) {
    }

    virtual ~row_consumer() = default;

    // Consume the row's key and deletion_time. The latter determines if the
    // row is a tombstone, and if so, when it has been deleted.
    // Note that the key is in serialized form, and should be deserialized
    // (according to the schema) before use.
    // As explained above, the key object is only valid during this call, and
    // if the implementation wishes to save it, it must copy the *contents*.
    virtual proceed consume_row_start(sstables::key_view key, sstables::deletion_time deltime) = 0;

    // Consume one cell (column name and value). Both are serialized, and need
    // to be deserialized according to the schema.
    // When a cell is set with an expiration time, "ttl" is the time to live
    // (in seconds) originally set for this cell, and "expiration" is the
    // absolute time (in seconds since the UNIX epoch) when this cell will
    // expire. Typical cells, not set to expire, will get expiration = 0.
    virtual proceed consume_cell(bytes_view col_name, fragmented_temporary_buffer::view value,
            int64_t timestamp,
            int64_t ttl, int64_t expiration) = 0;

    // Consume one counter cell. Column name and value are serialized, and need
    // to be deserialized according to the schema.
    virtual proceed consume_counter_cell(bytes_view col_name, fragmented_temporary_buffer::view value,
            int64_t timestamp) = 0;

    // Consume a deleted cell (i.e., a cell tombstone).
    virtual proceed consume_deleted_cell(bytes_view col_name, sstables::deletion_time deltime) = 0;

    // Consume one row tombstone.
    virtual proceed consume_shadowable_row_tombstone(bytes_view col_name, sstables::deletion_time deltime) = 0;

    // Consume one range tombstone.
    virtual proceed consume_range_tombstone(
            bytes_view start_col, bytes_view end_col,
            sstables::deletion_time deltime) = 0;

    // Called at the end of the row, after all cells.
    // Returns a flag saying whether the sstable consumer should stop now, or
    // proceed consuming more data.
    virtual proceed consume_row_end() = 0;

    // Called when the reader is fast forwarded to given element.
    virtual void reset(sstables::indexable_element) = 0;

    virtual position_in_partition_view position() = 0;

    // Under which priority class to place I/O coming from this consumer
    const io_priority_class& io_priority() const {
        return _pc;
    }

    // The permit for this read
    reader_permit& permit() {
        return _permit;
    }

    tracing::trace_state_ptr trace_state() const {
        return _trace_state;
    }
};

// data_consume_rows_context remembers the context that an ongoing
// data_consume_rows() future is in.
class data_consume_rows_context : public data_consumer::continuous_data_consumer<data_consume_rows_context> {
private:
    enum class state {
        ROW_START,
        DELETION_TIME_2,
        DELETION_TIME_3,
        ATOM_START,
        ATOM_START_2,
        ATOM_MASK_2,
        COUNTER_CELL_2,
        EXPIRING_CELL_3,
        CELL_VALUE_BYTES_2,
        RANGE_TOMBSTONE_4,
    } _state = state::ROW_START;

    row_consumer& _consumer;
    shared_sstable _sst;

    temporary_buffer<char> _key;
    temporary_buffer<char> _val;
    fragmented_temporary_buffer _val_fragmented;

    // state for reading a cell
    bool _deleted;
    bool _counter;
    uint32_t _ttl, _expiration;

    bool _shadowable;

    processing_result_generator _gen;
    temporary_buffer<char>* _processing_data;
public:
    using consumer = row_consumer;
    bool non_consuming() const {
        return (((_state == state::DELETION_TIME_3)
                || (_state == state::CELL_VALUE_BYTES_2)
                || (_state == state::ATOM_START_2)
                || (_state == state::ATOM_MASK_2)
                || (_state == state::COUNTER_CELL_2)
                || (_state == state::RANGE_TOMBSTONE_4)
                || (_state == state::EXPIRING_CELL_3)));
    }

    // process() feeds the given data into the state machine.
    // The consumer may request at any point (e.g., after reading a whole
    // row) to stop the processing, in which case we trim the buffer to
    // leave only the unprocessed part. The caller must handle calling
    // process() again, and/or refilling the buffer, as needed.
    data_consumer::processing_result process_state(temporary_buffer<char>& data) {
#if 0
        // Testing hack: call process() for tiny chunks separately, to verify
        // that primitive types crossing input buffer are handled correctly.
        constexpr size_t tiny_chunk = 1; // try various tiny sizes
        if (data.size() > tiny_chunk) {
            for (unsigned i = 0; i < data.size(); i += tiny_chunk) {
                auto chunk_size = std::min(tiny_chunk, data.size() - i);
                auto chunk = data.share(i, chunk_size);
                if (process(chunk) == row_consumer::proceed::no) {
                    data.trim_front(i + chunk_size - chunk.size());
                    return row_consumer::proceed::no;
                }
            }
            data.trim(0);
            return row_consumer::proceed::yes;
        }
#endif
        sstlog.trace("data_consume_row_context {}: state={}, size={}", fmt::ptr(this), static_cast<int>(_state), data.size());
        _processing_data = &data;
        return _gen.generate();
    }
private:
    processing_result_generator do_process_state() {
        while (true) {
            if (_state == state::ROW_START) {
                if (read_short_length_bytes(*_processing_data, _key) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_32(*_processing_data) != read_status::ready) {
                    _state = state::DELETION_TIME_2;
                    co_yield row_consumer::proceed::yes;
                }
                if (read_64(*_processing_data) != read_status::ready) {
                    _state = state::DELETION_TIME_3;
                    co_yield row_consumer::proceed::yes;
                }
                deletion_time del;
                del.local_deletion_time = _u32;
                del.marked_for_delete_at = _u64;
                _sst->get_stats().on_row_read();
                auto ret = _consumer.consume_row_start(key_view(to_bytes_view(_key)), del);
                // after calling the consume function, we can release the
                // buffers we held for it.
                _key.release();
                _state = state::ATOM_START;
                if (ret == row_consumer::proceed::no) {
                    co_yield row_consumer::proceed::no;
                }
            }
            while (true) {
                if (read_short_length_bytes(*_processing_data, _key) != read_status::ready) {
                    _state = state::ATOM_START_2;
                    co_yield row_consumer::proceed::yes;
                }
                if (_u16 == 0) {
                    // end of row marker
                    _state = state::ROW_START;
                    co_yield _consumer.consume_row_end();
                    break;
                }
                if (read_8(*_processing_data) != read_status::ready) {
                    _state = state::ATOM_MASK_2;
                    co_yield row_consumer::proceed::yes;
                }
                auto const mask = column_mask(_u8);

                if ((mask & (column_mask::range_tombstone | column_mask::shadowable)) != column_mask::none) {
                    _shadowable = (mask & column_mask::shadowable) != column_mask::none;
                    if (read_short_length_bytes(*_processing_data, _val) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    if (read_32(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    if (read_64(*_processing_data) != read_status::ready) {
                        _state = state::RANGE_TOMBSTONE_4;
                        co_yield row_consumer::proceed::yes;
                    }
                    _sst->get_stats().on_range_tombstone_read();
                    deletion_time del;
                    del.local_deletion_time = _u32;
                    del.marked_for_delete_at = _u64;
                    auto ret = _shadowable
                            ? _consumer.consume_shadowable_row_tombstone(to_bytes_view(_key), del)
                            : _consumer.consume_range_tombstone(to_bytes_view(_key), to_bytes_view(_val), del);
                    _key.release();
                    _val.release();
                    _state = state::ATOM_START;
                    co_yield ret;
                    continue;
                } else if ((mask & column_mask::counter) != column_mask::none) {
                    _deleted = false;
                    _counter = true;
                    if (read_64(*_processing_data) != read_status::ready) {
                        _state = state::COUNTER_CELL_2;
                        co_yield row_consumer::proceed::yes;
                    }
                    // _timestamp_of_last_deletion = _u64;
                } else if ((mask & column_mask::expiration) != column_mask::none) {
                    _deleted = false;
                    _counter = false;
                    if (read_32(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    _ttl = _u32;
                    if (read_32(*_processing_data) != read_status::ready) {
                        _state = state::EXPIRING_CELL_3;
                        co_yield row_consumer::proceed::yes;
                    }
                    _expiration = _u32;
                } else {
                    // FIXME: see ColumnSerializer.java:deserializeColumnBody
                    if ((mask & column_mask::counter_update) != column_mask::none) {
                        throw malformed_sstable_exception("FIXME COUNTER_UPDATE_MASK");
                    }
                    _ttl = _expiration = 0;
                    _deleted = (mask & column_mask::deletion) != column_mask::none;
                    _counter = false;
                }
                if (read_64(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_32(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_bytes(*_processing_data, _u32, _val_fragmented) != read_status::ready) {
                    _state = state::CELL_VALUE_BYTES_2;
                    co_yield row_consumer::proceed::yes;
                }
                row_consumer::proceed ret;
                if (_deleted) {
                    if (_val_fragmented.size_bytes() != 4) {
                        throw malformed_sstable_exception("deleted cell expects local_deletion_time value");
                    }
                    _val = temporary_buffer<char>(4);
                    auto v = fragmented_temporary_buffer::view(_val_fragmented);
                    read_fragmented(v, 4, reinterpret_cast<bytes::value_type*>(_val.get_write()));
                    deletion_time del;
                    del.local_deletion_time = consume_be<uint32_t>(_val);
                    del.marked_for_delete_at = _u64;
                    ret = _consumer.consume_deleted_cell(to_bytes_view(_key), del);
                    _val.release();
                } else if (_counter) {
                    ret = _consumer.consume_counter_cell(to_bytes_view(_key),
                            fragmented_temporary_buffer::view(_val_fragmented), _u64);
                } else {
                    ret = _consumer.consume_cell(to_bytes_view(_key),
                            fragmented_temporary_buffer::view(_val_fragmented), _u64, _ttl, _expiration);
                }
                // after calling the consume function, we can release the
                // buffers we held for it.
                _key.release();
                _val_fragmented.remove_prefix(_val_fragmented.size_bytes());
                _state = state::ATOM_START;
                co_yield ret;
            }
        }
    }
public:

    data_consume_rows_context(const schema&,
                              const shared_sstable sst,
                              row_consumer& consumer,
                              input_stream<char>&& input, uint64_t start, uint64_t maxlen)
                : continuous_data_consumer(consumer.permit(), std::move(input), start, maxlen)
                , _consumer(consumer)
                , _sst(std::move(sst))
                , _gen(do_process_state())
    {}

    void verify_end_state() {
        // If reading a partial row (i.e., when we have a clustering row
        // filter and using a promoted index), we may be in ATOM_START or ATOM_START_2
        // state instead of ROW_START. In that case we did not read the
        // end-of-row marker and consume_row_end() was never called.
        if (_state == state::ATOM_START || _state == state::ATOM_START_2) {
            _consumer.consume_row_end();
            return;
        }
        if (_state != state::ROW_START || primitive_consumer::active()) {
            throw malformed_sstable_exception("end of input, but not end of row");
        }
    }

    void reset(indexable_element el) {
        switch (el) {
        case indexable_element::partition:
            _state = state::ROW_START;
            break;
        case indexable_element::cell:
            _state = state::ATOM_START;
            break;
        default:
            assert(0);
        }
        _consumer.reset(el);
        _gen = do_process_state();
    }

    reader_permit& permit() {
        return _consumer.permit();
    }
};

} // namespace kl
} // namespace sstables
<commit_msg>kl sstable reader: replace states for ending with one state, simplify non_consuming<commit_after>/*
 * Copyright (C) 2021-present ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "sstables/consumer.hh"
#include "sstables/types.hh"
#include "sstables/sstables.hh"
#include "sstables/processing_result_generator.hh"

// Expose internal types for tests

namespace sstables {
namespace kl {

// sstables::data_consume_row feeds the contents of a single row into a
// row_consumer object:
//
// * First, consume_row_start() is called, with some information about the
//   whole row: The row's key, timestamp, etc.
// * Next, consume_cell() is called once for every column.
// * Finally, consume_row_end() is called. A consumer written for a single
//   column will likely not want to do anything here.
//
// Important note: the row key, column name and column value, passed to the
// consume_* functions, are passed as a "bytes_view" object, which points to
// internal data held by the feeder. This internal data is only valid for the
// duration of the single consume function it was passed to. If the object
// wants to hold these strings longer, it must make a copy of the bytes_view's
// contents. [Note, in reality, because our implementation reads the whole
// row into one buffer, the byte_views remain valid until consume_row_end()
// is called.]
class row_consumer {
    reader_permit _permit;
    tracing::trace_state_ptr _trace_state;
    const io_priority_class& _pc;

public:
    using proceed = data_consumer::proceed;

    /*
     * In k/l formats, RTs are represented as cohesive entries so
     * setting/resetting RT start is not supported.
     */
    constexpr static bool is_setting_range_tombstone_start_supported = false;

    row_consumer(reader_permit permit, tracing::trace_state_ptr trace_state, const io_priority_class& pc)
        : _permit(std::move(permit))
        , _trace_state(std::move(trace_state))
        , _pc(pc) {
    }

    virtual ~row_consumer() = default;

    // Consume the row's key and deletion_time. The latter determines if the
    // row is a tombstone, and if so, when it has been deleted.
    // Note that the key is in serialized form, and should be deserialized
    // (according to the schema) before use.
    // As explained above, the key object is only valid during this call, and
    // if the implementation wishes to save it, it must copy the *contents*.
    virtual proceed consume_row_start(sstables::key_view key, sstables::deletion_time deltime) = 0;

    // Consume one cell (column name and value). Both are serialized, and need
    // to be deserialized according to the schema.
    // When a cell is set with an expiration time, "ttl" is the time to live
    // (in seconds) originally set for this cell, and "expiration" is the
    // absolute time (in seconds since the UNIX epoch) when this cell will
    // expire. Typical cells, not set to expire, will get expiration = 0.
    virtual proceed consume_cell(bytes_view col_name, fragmented_temporary_buffer::view value,
            int64_t timestamp,
            int64_t ttl, int64_t expiration) = 0;

    // Consume one counter cell. Column name and value are serialized, and need
    // to be deserialized according to the schema.
    virtual proceed consume_counter_cell(bytes_view col_name, fragmented_temporary_buffer::view value,
            int64_t timestamp) = 0;

    // Consume a deleted cell (i.e., a cell tombstone).
    virtual proceed consume_deleted_cell(bytes_view col_name, sstables::deletion_time deltime) = 0;

    // Consume one row tombstone.
    virtual proceed consume_shadowable_row_tombstone(bytes_view col_name, sstables::deletion_time deltime) = 0;

    // Consume one range tombstone.
    virtual proceed consume_range_tombstone(
            bytes_view start_col, bytes_view end_col,
            sstables::deletion_time deltime) = 0;

    // Called at the end of the row, after all cells.
    // Returns a flag saying whether the sstable consumer should stop now, or
    // proceed consuming more data.
    virtual proceed consume_row_end() = 0;

    // Called when the reader is fast forwarded to given element.
    virtual void reset(sstables::indexable_element) = 0;

    virtual position_in_partition_view position() = 0;

    // Under which priority class to place I/O coming from this consumer
    const io_priority_class& io_priority() const {
        return _pc;
    }

    // The permit for this read
    reader_permit& permit() {
        return _permit;
    }

    tracing::trace_state_ptr trace_state() const {
        return _trace_state;
    }
};

// data_consume_rows_context remembers the context that an ongoing
// data_consume_rows() future is in.
class data_consume_rows_context : public data_consumer::continuous_data_consumer<data_consume_rows_context> {
private:
    enum class state {
        ROW_START,
        ATOM_START,
        NOT_CLOSING,
    } _state = state::ROW_START;

    row_consumer& _consumer;
    shared_sstable _sst;

    temporary_buffer<char> _key;
    temporary_buffer<char> _val;
    fragmented_temporary_buffer _val_fragmented;

    // state for reading a cell
    bool _deleted;
    bool _counter;
    uint32_t _ttl, _expiration;

    bool _shadowable;

    processing_result_generator _gen;
    temporary_buffer<char>* _processing_data;
public:
    using consumer = row_consumer;
     // assumes !primitive_consumer::active()
    bool non_consuming() const {
        return false;
    }

    // process() feeds the given data into the state machine.
    // The consumer may request at any point (e.g., after reading a whole
    // row) to stop the processing, in which case we trim the buffer to
    // leave only the unprocessed part. The caller must handle calling
    // process() again, and/or refilling the buffer, as needed.
    data_consumer::processing_result process_state(temporary_buffer<char>& data) {
#if 0
        // Testing hack: call process() for tiny chunks separately, to verify
        // that primitive types crossing input buffer are handled correctly.
        constexpr size_t tiny_chunk = 1; // try various tiny sizes
        if (data.size() > tiny_chunk) {
            for (unsigned i = 0; i < data.size(); i += tiny_chunk) {
                auto chunk_size = std::min(tiny_chunk, data.size() - i);
                auto chunk = data.share(i, chunk_size);
                if (process(chunk) == row_consumer::proceed::no) {
                    data.trim_front(i + chunk_size - chunk.size());
                    return row_consumer::proceed::no;
                }
            }
            data.trim(0);
            return row_consumer::proceed::yes;
        }
#endif
        sstlog.trace("data_consume_row_context {}: state={}, size={}", fmt::ptr(this), static_cast<int>(_state), data.size());
        _processing_data = &data;
        return _gen.generate();
    }
private:
    processing_result_generator do_process_state() {
        while (true) {
            if (_state == state::ROW_START) {
                _state = state::NOT_CLOSING;
                if (read_short_length_bytes(*_processing_data, _key) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_32(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_64(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                deletion_time del;
                del.local_deletion_time = _u32;
                del.marked_for_delete_at = _u64;
                _sst->get_stats().on_row_read();
                auto ret = _consumer.consume_row_start(key_view(to_bytes_view(_key)), del);
                // after calling the consume function, we can release the
                // buffers we held for it.
                _key.release();
                _state = state::ATOM_START;
                if (ret == row_consumer::proceed::no) {
                    co_yield row_consumer::proceed::no;
                }
            }
            while (true) {
                if (read_short_length_bytes(*_processing_data, _key) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (_u16 == 0) {
                    // end of row marker
                    _state = state::ROW_START;
                    co_yield _consumer.consume_row_end();
                    break;
                }
                _state = state::NOT_CLOSING;
                if (read_8(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                auto const mask = column_mask(_u8);

                if ((mask & (column_mask::range_tombstone | column_mask::shadowable)) != column_mask::none) {
                    _shadowable = (mask & column_mask::shadowable) != column_mask::none;
                    if (read_short_length_bytes(*_processing_data, _val) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    if (read_32(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    if (read_64(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    _sst->get_stats().on_range_tombstone_read();
                    deletion_time del;
                    del.local_deletion_time = _u32;
                    del.marked_for_delete_at = _u64;
                    auto ret = _shadowable
                            ? _consumer.consume_shadowable_row_tombstone(to_bytes_view(_key), del)
                            : _consumer.consume_range_tombstone(to_bytes_view(_key), to_bytes_view(_val), del);
                    _key.release();
                    _val.release();
                    _state = state::ATOM_START;
                    co_yield ret;
                    continue;
                } else if ((mask & column_mask::counter) != column_mask::none) {
                    _deleted = false;
                    _counter = true;
                    if (read_64(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    // _timestamp_of_last_deletion = _u64;
                } else if ((mask & column_mask::expiration) != column_mask::none) {
                    _deleted = false;
                    _counter = false;
                    if (read_32(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    _ttl = _u32;
                    if (read_32(*_processing_data) != read_status::ready) {
                        co_yield row_consumer::proceed::yes;
                    }
                    _expiration = _u32;
                } else {
                    // FIXME: see ColumnSerializer.java:deserializeColumnBody
                    if ((mask & column_mask::counter_update) != column_mask::none) {
                        throw malformed_sstable_exception("FIXME COUNTER_UPDATE_MASK");
                    }
                    _ttl = _expiration = 0;
                    _deleted = (mask & column_mask::deletion) != column_mask::none;
                    _counter = false;
                }
                if (read_64(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_32(*_processing_data) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                if (read_bytes(*_processing_data, _u32, _val_fragmented) != read_status::ready) {
                    co_yield row_consumer::proceed::yes;
                }
                row_consumer::proceed ret;
                if (_deleted) {
                    if (_val_fragmented.size_bytes() != 4) {
                        throw malformed_sstable_exception("deleted cell expects local_deletion_time value");
                    }
                    _val = temporary_buffer<char>(4);
                    auto v = fragmented_temporary_buffer::view(_val_fragmented);
                    read_fragmented(v, 4, reinterpret_cast<bytes::value_type*>(_val.get_write()));
                    deletion_time del;
                    del.local_deletion_time = consume_be<uint32_t>(_val);
                    del.marked_for_delete_at = _u64;
                    ret = _consumer.consume_deleted_cell(to_bytes_view(_key), del);
                    _val.release();
                } else if (_counter) {
                    ret = _consumer.consume_counter_cell(to_bytes_view(_key),
                            fragmented_temporary_buffer::view(_val_fragmented), _u64);
                } else {
                    ret = _consumer.consume_cell(to_bytes_view(_key),
                            fragmented_temporary_buffer::view(_val_fragmented), _u64, _ttl, _expiration);
                }
                // after calling the consume function, we can release the
                // buffers we held for it.
                _key.release();
                _val_fragmented.remove_prefix(_val_fragmented.size_bytes());
                _state = state::ATOM_START;
                co_yield ret;
            }
        }
    }
public:

    data_consume_rows_context(const schema&,
                              const shared_sstable sst,
                              row_consumer& consumer,
                              input_stream<char>&& input, uint64_t start, uint64_t maxlen)
                : continuous_data_consumer(consumer.permit(), std::move(input), start, maxlen)
                , _consumer(consumer)
                , _sst(std::move(sst))
                , _gen(do_process_state())
    {}

    void verify_end_state() {
        // If reading a partial row (i.e., when we have a clustering row
        // filter and using a promoted index), we may be in ATOM_START
        // state instead of ROW_START. In that case we did not read the
        // end-of-row marker and consume_row_end() was never called.
        if (_state == state::ATOM_START) {
            _consumer.consume_row_end();
            return;
        }
        if (_state != state::ROW_START || primitive_consumer::active()) {
            throw malformed_sstable_exception("end of input, but not end of row");
        }
    }

    void reset(indexable_element el) {
        switch (el) {
        case indexable_element::partition:
            _state = state::ROW_START;
            break;
        case indexable_element::cell:
            _state = state::ATOM_START;
            break;
        default:
            assert(0);
        }
        _consumer.reset(el);
        _gen = do_process_state();
    }

    reader_permit& permit() {
        return _consumer.permit();
    }
};

} // namespace kl
} // namespace sstables
<|endoftext|>