<commit_before>#include "BoundingBox.hpp"
#include "clipper/clipper.hpp"
#include "builders/BuilderContext.hpp"
#include "builders/terrain/TerraBuilder.hpp"
#include "builders/terrain/SurfaceGenerator.hpp"
#include "builders/terrain/ExteriorGenerator.hpp"
#include "entities/Node.hpp"
#include "entities/Way.hpp"
#include "entities/Area.hpp"
#include "entities/Relation.hpp"

#include "utils/GeometryUtils.hpp"

#include <numeric>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::entities;
using namespace utymap::index;
using namespace utymap::heightmap;
using namespace utymap::mapcss;
using namespace utymap::utils;

namespace {
    const double Scale = 1E7;

    /// Converts coordinate to clipper's IntPoint.
    IntPoint toIntPoint(double x, double y)
    {
        return IntPoint(static_cast<cInt>(x * Scale), static_cast<cInt>(y * Scale));
    }

    /// Compares regions based on their area.
    struct LessThanByArea
    {
        bool operator()(const std::shared_ptr<const Region>& lhs, const std::shared_ptr<const Region>& rhs) const
        {
            return lhs->area < rhs->area;
        }
    };

    /// Compares layers based on their sort order.
    struct MoreThanSortOrder
    {
        bool operator()(const Layer& lhs, const Layer& rhs) const
        {
            return lhs.sortOrder > rhs.sortOrder;
        }
    };

    /// Visits relation and fills region.
    struct RelationVisitor : public ElementVisitor
    {
        const Relation& relation;
        ElementVisitor& builder;
        Region& region;

        RelationVisitor(ElementVisitor& b, const Relation& r, Region& reg) :
                relation(r), builder(b), region(reg) {}

        void visitNode(const utymap::entities::Node& n) override { n.accept(builder); }

        void visitWay(const utymap::entities::Way& w) override { w.accept(builder); }

        void visitArea(const utymap::entities::Area& a) override
        {
            Path path;
            path.reserve(a.coordinates.size());
            for (const utymap::GeoCoordinate& c : a.coordinates)
                path.push_back(toIntPoint(c.longitude, c.latitude));

            region.geometry.push_back(path);
        }

        void visitRelation(const utymap::entities::Relation& r) override { r.accept(builder); }
    };
}

class TerraBuilder::TerraBuilderImpl final : public ElementBuilder
{
public:

    explicit TerraBuilderImpl(const BuilderContext& context) :
        ElementBuilder(context),
        style_(context.styleProvider.forCanvas(context.quadKey.levelOfDetail)),
        generators_(), dimenstionKey_(context.stringTable.getId(StyleConsts::DimensionKey()))
    {
        tileRect_.push_back(toIntPoint(context.boundingBox.minPoint.longitude, context.boundingBox.minPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.maxPoint.longitude, context.boundingBox.minPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.maxPoint.longitude, context.boundingBox.maxPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.minPoint.longitude, context.boundingBox.maxPoint.latitude));

        clipper_.AddPath(tileRect_, ptClip, true);

        // NOTE order is important due to propagation of region changes from top to bottom.
        generators_.push_back(utymap::utils::make_unique<ExteriorGenerator>(context, style_, tileRect_));
        generators_.push_back(utymap::utils::make_unique<SurfaceGenerator>(context, style_, tileRect_));
    }

    void visitNode(const utymap::entities::Node& node) override
    {
    }

    void visitWay(const utymap::entities::Way& way) override
    {
        Style style = context_.styleProvider.forElement(way, context_.quadKey.levelOfDetail);
        auto region = createRegion(style, way.coordinates);
        double width = getWidth(style)* Scale;

        Paths solution;
        // make polygon from line by offsetting it using width specified
        // NOTE: we should limit round shape precision due to performance reasons.
        offset_.ArcTolerance = width * 0.05;
        offset_.AddPaths(region->geometry, jtRound, etOpenRound);
        offset_.Execute(solution, width);
        offset_.Clear();

        clipper_.AddPaths(solution, ptSubject, true);
        clipper_.Execute(ctIntersection, solution);
        clipper_.removeSubject();

        region->geometry = solution;
        std::string type = region->isLayer()
            ? style.getString(StyleConsts::TerrainLayerKey())
            : "";

        addRegion(type, way, style, region);
    }

    void visitArea(const utymap::entities::Area& area) override
    {
        Style style = context_.styleProvider.forElement(area, context_.quadKey.levelOfDetail);
        auto region = createRegion(style, area.coordinates);
        std::string type = region->isLayer()
            ? style.getString(StyleConsts::TerrainLayerKey())
            : "";

        addRegion(type, area, style, region);
    }

    void visitRelation(const utymap::entities::Relation& rel) override
    {
        auto region = std::make_shared<Region>();
        RelationVisitor visitor(*this, rel, *region);

        for (const auto& element : rel.elements) {
            // if there are no tags, then this element is result of clipping
            if (element->tags.empty())
                element->tags = rel.tags;

            if (context_.styleProvider.hasStyle(*element, context_.quadKey.levelOfDetail))
                element->accept(visitor);
        }

        if (!region->geometry.empty()) {
            Style style = context_.styleProvider.forElement(rel, context_.quadKey.levelOfDetail);
            if (!style.has(context_.stringTable.getId(StyleConsts::TerrainLayerKey())))
                region->context = utymap::utils::make_unique<RegionContext>(RegionContext::create(context_, style, ""));

            std::string type = region->isLayer()
                ? style.getString(StyleConsts::TerrainLayerKey())
                : "";

            addRegion(type, rel, style, region);
        }
    }

    /// Builds tile mesh using created layers and registered terra generators.
    void complete() override
    {
        std::vector<Layer> layers;
        layers.reserve(layers_.size());

        for (auto& layerPair : layers_) {
            // named layer has to merge all regions inside
            if (!layerPair.first.empty()) {
                auto stylePrefix = layerPair.first + "-";
                layerPair.second.sortOrder = style_.getValue(stylePrefix + StyleConsts::SortOrderKey());
                mergeRegions(layerPair.second, std::make_shared<RegionContext>(
                    RegionContext::create(context_, style_, stylePrefix)));
            }
            // sort regions inside one layer based on their area
            std::sort(layerPair.second.regions.begin(), layerPair.second.regions.end(), LessThanByArea());
            layers.push_back(std::move(layerPair.second));
        }

        layers_.clear();

        // sort all layers based on their sort order
        std::sort(layers.begin(), layers.end(), MoreThanSortOrder());

        for (const auto& generator : generators_)
            generator->generateFrom(layers);
    }

private:

    /// Gets width for line offsetting taking care about dimension.
    double getWidth(const Style& style) const
    {
        // NOTE current mapcss does not support double value evaluation with dimension
        // so, special trick with mapcss key is used.
        double value = style.getValue(StyleConsts::WidthKey(), context_.boundingBox);
        return style.has(dimenstionKey_)
            ? value * style.getValue(StyleConsts::DimensionKey(), context_.boundingBox)
            : value;
    }

    void addRegion(const std::string& type, const utymap::entities::Element& element, const Style& style, std::shared_ptr<Region>& region)
    {
        for (const auto& generator : generators_) {
            layers_[type].regions.push_back(region);
            generator->onNewRegion(type, element, style, region);
        }
    }

    /// Creates region from given geometry and style.
    std::shared_ptr<Region> createRegion(const Style& style, const std::vector<GeoCoordinate>& coordinates) const
    {
        auto region = std::make_shared<Region>();
        Path path;
        path.reserve(coordinates.size());
        for (const GeoCoordinate& c : coordinates)
            path.push_back(toIntPoint(c.longitude, c.latitude));

        region->geometry.push_back(path);

        if (!style.has(context_.stringTable.getId(StyleConsts::TerrainLayerKey())))
            region->context = std::make_shared<RegionContext>(RegionContext::create(context_, style, ""));

        region->level = static_cast<int>(style.getValue(StyleConsts::LevelKey()));
        return region;
    }

    /// Merges regions into one taking into account level difference.
    void mergeRegions(Layer& layer, const std::shared_ptr<const RegionContext>& regionContext) const
    {
        std::unordered_map<int, std::pair<std::shared_ptr<Region>, std::shared_ptr<Clipper>>> regionMap;
        for (const auto& current : layer.regions) {
            auto mapPair = regionMap.find(current->level);
            if (mapPair == regionMap.end()) {
                auto clipperPair = std::make_pair(std::make_shared<Region>(), std::make_shared<Clipper>());
                mapPair = regionMap.insert(std::make_pair(current->level, clipperPair)).first;
                mapPair->second.first->level = current->level;
                mapPair->second.first->context = regionContext;
            }
            mapPair->second.second->AddPaths(current->geometry, ptSubject, true);
        }

        layer.regions.clear();
        for (auto& pair : regionMap) {
            Paths result;
            pair.second.second->Execute(ctUnion, result, pftNonZero, pftNonZero);
            pair.second.first->area = std::accumulate(result.begin(), result.end(), 0.,
                [](double acc, const Path& path) {
                    return acc + std::abs(ClipperLib::Area(path));
            }) / (Scale * Scale);

            pair.second.first->geometry = std::move(result);
            layer.regions.push_back(pair.second.first);
        }
    }

    const Style style_;
    ClipperEx clipper_;
    ClipperOffset offset_;
    std::vector<std::unique_ptr<TerraGenerator>> generators_;
    std::unordered_map<std::string, Layer> layers_;
    Path tileRect_;
    std::uint32_t dimenstionKey_;
};

void TerraBuilder::visitNode(const utymap::entities::Node& node) { pimpl_->visitNode(node); }

void TerraBuilder::visitWay(const utymap::entities::Way& way) { pimpl_->visitWay(way); }

void TerraBuilder::visitArea(const utymap::entities::Area& area) { pimpl_->visitArea(area); }

void TerraBuilder::visitRelation(const utymap::entities::Relation& relation) { pimpl_->visitRelation(relation); }

void TerraBuilder::complete() { pimpl_->complete(); }

TerraBuilder::~TerraBuilder() { }

TerraBuilder::TerraBuilder(const BuilderContext& context) :
    ElementBuilder(context), pimpl_(utymap::utils::make_unique<TerraBuilderImpl>(context))
{
}
<commit_msg>core: fix issues in terra builder<commit_after>#include "BoundingBox.hpp"
#include "clipper/clipper.hpp"
#include "builders/BuilderContext.hpp"
#include "builders/terrain/TerraBuilder.hpp"
#include "builders/terrain/SurfaceGenerator.hpp"
#include "builders/terrain/ExteriorGenerator.hpp"
#include "entities/Node.hpp"
#include "entities/Way.hpp"
#include "entities/Area.hpp"
#include "entities/Relation.hpp"

#include "utils/GeometryUtils.hpp"

#include <numeric>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::entities;
using namespace utymap::index;
using namespace utymap::heightmap;
using namespace utymap::mapcss;
using namespace utymap::utils;

namespace {
    const double Scale = 1E7;

    /// Converts coordinate to clipper's IntPoint.
    IntPoint toIntPoint(double x, double y)
    {
        return IntPoint(static_cast<cInt>(x * Scale), static_cast<cInt>(y * Scale));
    }

    /// Compares regions based on their area.
    struct LessThanByArea
    {
        bool operator()(const std::shared_ptr<const Region>& lhs, const std::shared_ptr<const Region>& rhs) const
        {
            return lhs->area < rhs->area;
        }
    };

    /// Compares layers based on their sort order.
    struct MoreThanSortOrder
    {
        bool operator()(const Layer& lhs, const Layer& rhs) const
        {
            return lhs.sortOrder > rhs.sortOrder;
        }
    };

    /// Visits relation and fills region.
    struct RelationVisitor : public ElementVisitor
    {
        const Relation& relation;
        ElementVisitor& builder;
        Region& region;

        RelationVisitor(ElementVisitor& b, const Relation& r, Region& reg) :
                relation(r), builder(b), region(reg) {}

        void visitNode(const utymap::entities::Node& n) override { n.accept(builder); }

        void visitWay(const utymap::entities::Way& w) override { w.accept(builder); }

        void visitArea(const utymap::entities::Area& a) override
        {
            Path path;
            path.reserve(a.coordinates.size());
            for (const utymap::GeoCoordinate& c : a.coordinates)
                path.push_back(toIntPoint(c.longitude, c.latitude));

            region.geometry.push_back(path);
        }

        void visitRelation(const utymap::entities::Relation& r) override { r.accept(builder); }
    };
}

class TerraBuilder::TerraBuilderImpl final : public ElementBuilder
{
public:

    explicit TerraBuilderImpl(const BuilderContext& context) :
        ElementBuilder(context),
        style_(context.styleProvider.forCanvas(context.quadKey.levelOfDetail)),
        generators_(), dimenstionKey_(context.stringTable.getId(StyleConsts::DimensionKey()))
    {
        tileRect_.push_back(toIntPoint(context.boundingBox.minPoint.longitude, context.boundingBox.minPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.maxPoint.longitude, context.boundingBox.minPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.maxPoint.longitude, context.boundingBox.maxPoint.latitude));
        tileRect_.push_back(toIntPoint(context.boundingBox.minPoint.longitude, context.boundingBox.maxPoint.latitude));

        clipper_.AddPath(tileRect_, ptClip, true);

        // NOTE order is important due to propagation of region changes from top to bottom.
        generators_.push_back(utymap::utils::make_unique<ExteriorGenerator>(context, style_, tileRect_));
        generators_.push_back(utymap::utils::make_unique<SurfaceGenerator>(context, style_, tileRect_));
    }

    void visitNode(const utymap::entities::Node& node) override
    {
    }

    void visitWay(const utymap::entities::Way& way) override
    {
        Style style = context_.styleProvider.forElement(way, context_.quadKey.levelOfDetail);
        auto region = createRegion(style, way.coordinates);
        double width = getWidth(style)* Scale;

        Paths solution;
        // make polygon from line by offsetting it using width specified
        // NOTE: we should limit round shape precision due to performance reasons.
        offset_.ArcTolerance = width * 0.05;
        offset_.AddPaths(region->geometry, jtRound, etOpenRound);
        offset_.Execute(solution, width);
        offset_.Clear();

        clipper_.AddPaths(solution, ptSubject, true);
        clipper_.Execute(ctIntersection, solution);
        clipper_.removeSubject();

        region->geometry = solution;
        std::string type = region->isLayer()
            ? style.getString(StyleConsts::TerrainLayerKey())
            : "";

        addRegion(type, way, style, region);
    }

    void visitArea(const utymap::entities::Area& area) override
    {
        Style style = context_.styleProvider.forElement(area, context_.quadKey.levelOfDetail);
        auto region = createRegion(style, area.coordinates);
        std::string type = region->isLayer()
            ? style.getString(StyleConsts::TerrainLayerKey())
            : "";

        addRegion(type, area, style, region);
    }

    void visitRelation(const utymap::entities::Relation& rel) override
    {
        auto region = std::make_shared<Region>();
        RelationVisitor visitor(*this, rel, *region);

        for (const auto& element : rel.elements) {
            // if there are no tags, then this element is result of clipping
            if (element->tags.empty())
                element->tags = rel.tags;

            if (context_.styleProvider.hasStyle(*element, context_.quadKey.levelOfDetail))
                element->accept(visitor);
        }

        if (!region->geometry.empty()) {
            Style style = context_.styleProvider.forElement(rel, context_.quadKey.levelOfDetail);
            if (!style.has(context_.stringTable.getId(StyleConsts::TerrainLayerKey())))
                region->context = utymap::utils::make_unique<RegionContext>(RegionContext::create(context_, style, ""));

            std::string type = region->isLayer()
                ? style.getString(StyleConsts::TerrainLayerKey())
                : "";

            addRegion(type, rel, style, region);
        }
    }

    /// Builds tile mesh using created layers and registered terra generators.
    void complete() override
    {
        std::vector<Layer> layers;
        layers.reserve(layers_.size());

        for (auto& layerPair : layers_) {
            // named layer has to merge all regions inside
            if (!layerPair.first.empty()) {
                auto stylePrefix = layerPair.first + "-";
                layerPair.second.sortOrder = style_.getValue(stylePrefix + StyleConsts::SortOrderKey());
                mergeRegions(layerPair.second, std::make_shared<RegionContext>(
                    RegionContext::create(context_, style_, stylePrefix)));
            }
            else
                layerPair.second.sortOrder = 0;

            // sort regions inside one layer based on their area
            std::sort(layerPair.second.regions.begin(), layerPair.second.regions.end(), LessThanByArea());
            layers.push_back(std::move(layerPair.second));
        }

        layers_.clear();

        // sort all layers based on their sort order
        std::sort(layers.begin(), layers.end(), MoreThanSortOrder());

        for (const auto& generator : generators_)
            generator->generateFrom(layers);
    }

private:

    /// Gets width for line offsetting taking care about dimension.
    double getWidth(const Style& style) const
    {
        // NOTE current mapcss does not support double value evaluation with dimension
        // so, special trick with mapcss key is used.
        double value = style.getValue(StyleConsts::WidthKey(), context_.boundingBox);
        return style.has(dimenstionKey_)
            ? value * style.getValue(StyleConsts::DimensionKey(), context_.boundingBox)
            : value;
    }

    void addRegion(const std::string& type, const utymap::entities::Element& element, const Style& style, std::shared_ptr<Region>& region)
    {
        layers_[type].regions.push_back(region);
        for (const auto& generator : generators_) {
            generator->onNewRegion(type, element, style, region);
        }
    }

    /// Creates region from given geometry and style.
    std::shared_ptr<Region> createRegion(const Style& style, const std::vector<GeoCoordinate>& coordinates) const
    {
        auto region = std::make_shared<Region>();
        Path path;
        path.reserve(coordinates.size());
        for (const GeoCoordinate& c : coordinates)
            path.push_back(toIntPoint(c.longitude, c.latitude));

        region->geometry.push_back(path);

        if (!style.has(context_.stringTable.getId(StyleConsts::TerrainLayerKey())))
            region->context = std::make_shared<RegionContext>(RegionContext::create(context_, style, ""));

        region->level = static_cast<int>(style.getValue(StyleConsts::LevelKey()));
        return region;
    }

    /// Merges regions into one taking into account level difference.
    void mergeRegions(Layer& layer, const std::shared_ptr<const RegionContext>& regionContext) const
    {
        std::unordered_map<int, std::pair<std::shared_ptr<Region>, std::shared_ptr<Clipper>>> regionMap;
        for (const auto& current : layer.regions) {
            auto mapPair = regionMap.find(current->level);
            if (mapPair == regionMap.end()) {
                auto clipperPair = std::make_pair(std::make_shared<Region>(), std::make_shared<Clipper>());
                mapPair = regionMap.insert(std::make_pair(current->level, clipperPair)).first;
                mapPair->second.first->level = current->level;
                mapPair->second.first->context = regionContext;
            }
            mapPair->second.second->AddPaths(current->geometry, ptSubject, true);
        }

        layer.regions.clear();
        for (auto& pair : regionMap) {
            Paths result;
            pair.second.second->Execute(ctUnion, result, pftNonZero, pftNonZero);
            pair.second.first->area = std::accumulate(result.begin(), result.end(), 0.,
                [](double acc, const Path& path) {
                    return acc + std::abs(ClipperLib::Area(path));
            }) / (Scale * Scale);

            pair.second.first->geometry = std::move(result);
            layer.regions.push_back(pair.second.first);
        }
    }

    const Style style_;
    ClipperEx clipper_;
    ClipperOffset offset_;
    std::vector<std::unique_ptr<TerraGenerator>> generators_;
    std::unordered_map<std::string, Layer> layers_;
    Path tileRect_;
    std::uint32_t dimenstionKey_;
};

void TerraBuilder::visitNode(const utymap::entities::Node& node) { pimpl_->visitNode(node); }

void TerraBuilder::visitWay(const utymap::entities::Way& way) { pimpl_->visitWay(way); }

void TerraBuilder::visitArea(const utymap::entities::Area& area) { pimpl_->visitArea(area); }

void TerraBuilder::visitRelation(const utymap::entities::Relation& relation) { pimpl_->visitRelation(relation); }

void TerraBuilder::complete() { pimpl_->complete(); }

TerraBuilder::~TerraBuilder() { }

TerraBuilder::TerraBuilder(const BuilderContext& context) :
    ElementBuilder(context), pimpl_(utymap::utils::make_unique<TerraBuilderImpl>(context))
{
}
<|endoftext|>


<commit_before>#include "BoundingBox.hpp"
#include "Exceptions.hpp"
#include "clipper/clipper.hpp"
#include "builders/BuilderContext.hpp"
#include "builders/terrain/LineGridSplitter.hpp"
#include "builders/terrain/TerraBuilder.hpp"
#include "entities/Node.hpp"
#include "entities/Way.hpp"
#include "entities/Area.hpp"
#include "entities/Relation.hpp"
#include "entities/ElementVisitor.hpp"
#include "meshing/Polygon.hpp"
#include "meshing/MeshBuilder.hpp"
#include "utils/CompatibilityUtils.hpp"
#include "utils/GeoUtils.hpp"
#include "utils/MapCssUtils.hpp"
#include "utils/NoiseUtils.hpp"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <functional>
#include <iterator>
#include <sstream>
#include <map>
#include <unordered_map>
#include <vector>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::entities;
using namespace utymap::index;
using namespace utymap::heightmap;
using namespace utymap::mapcss;
using namespace utymap::meshing;
using namespace utymap::utils;

const double Scale = 1E7;          // max precision for Lat/Lon: seven decimal positions
const double AreaTolerance = 1000; // Tolerance for meshing

// Represents terrain region points.
struct Region
{
    bool isLayer;
    std::shared_ptr<MeshBuilder::Options> options; // optional: might be empty if polygon is layer
    Paths points;
};

typedef std::vector<Point> Points;
typedef std::vector<Region> Regions;
typedef std::unordered_map<std::string, Regions> Layers;

// mapcss specific keys
const static std::string TerrainLayerKey = "terrain-layer";
const static std::string ColorNoiseFreqKey = "color-noise-freq";
const static std::string EleNoiseFreqKey = "ele-noise-freq";
const static std::string GradientKey= "color";
const static std::string MaxAreaKey = "max-area";
const static std::string WidthKey = "width";
const static std::string HeightKey = "height";
const static std::string LayerPriorityKey = "layer-priority";
const static std::string MeshNameKey = "mesh-name";
const static std::string GridCellSize = "grid-cell-size";

class TerraBuilder::TerraBuilderImpl : public ElementBuilder
{
public:

    TerraBuilderImpl(const BuilderContext& context) :
        ElementBuilder(context),
        style_(context.styleProvider.forCanvas(context.quadKey.levelOfDetail)),
        splitter_(),
        bbox_(GeoUtils::quadKeyToBoundingBox(context_.quadKey)),
        rect_(bbox_.minPoint.longitude, bbox_.minPoint.latitude, bbox_.maxPoint.longitude, bbox_.maxPoint.latitude),
        mesh_("terrain")
    {
        tileRect_.push_back(IntPoint(bbox_.minPoint.longitude*Scale, bbox_.minPoint.latitude *Scale));
        tileRect_.push_back(IntPoint(bbox_.maxPoint.longitude *Scale, bbox_.minPoint.latitude *Scale));
        tileRect_.push_back(IntPoint(bbox_.maxPoint.longitude *Scale, bbox_.maxPoint.latitude*Scale));
        tileRect_.push_back(IntPoint(bbox_.minPoint.longitude*Scale, bbox_.maxPoint.latitude*Scale));

        clipper_.AddPath(tileRect_, ptClip, true);

        double size = utymap::utils::getDimension(GridCellSize, context_.stringTable, style_,
            bbox_.maxPoint.latitude - bbox_.minPoint.latitude, bbox_.center());
        splitter_.setParams(Scale, size);
    }

    void visitNode(const utymap::entities::Node& node)
    {
    }

    void visitWay(const utymap::entities::Way& way)
    {
        Style style = context_.styleProvider.forElement(way, context_.quadKey.levelOfDetail);
        Region region = createRegion(style, way.coordinates);

        // make polygon from line by offsetting it using width specified
        double width = utymap::utils::getDimension(WidthKey, context_.stringTable, style,
            bbox_.maxPoint.latitude - bbox_.minPoint.latitude, bbox_.center());

        Paths solution;
        offset_.AddPaths(region.points, jtMiter, etOpenSquare);
        offset_.Execute(solution, width * Scale);
        offset_.Clear();

        clipper_.AddPaths(solution, ptSubject, true);
        clipper_.Execute(ctIntersection, solution);
        clipper_.removeSubject();

        region.points = solution;
        std::string type = region.isLayer
            ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
            : "";
        layers_[type].push_back(region);
    }

    void visitArea(const utymap::entities::Area& area)
    {
        Style style = context_.styleProvider.forElement(area, context_.quadKey.levelOfDetail);
        Region region = createRegion(style, area.coordinates);
        std::string type = region.isLayer
            ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
            : "";
        layers_[type].push_back(region);
    }

    void visitRelation(const utymap::entities::Relation& relation)
    {
        Region region;
        struct RelationVisitor : public ElementVisitor
        {
            const Relation& relation;
            TerraBuilder::TerraBuilderImpl& builder;
            Region& region;

            RelationVisitor(TerraBuilder::TerraBuilderImpl& builder, const Relation& relation, Region& region) :
                builder(builder), relation(relation), region(region) {}

            void visitNode(const utymap::entities::Node& n) { n.accept(builder); }

            void visitWay(const utymap::entities::Way& w) { w.accept(builder); }

            void visitArea(const utymap::entities::Area& a)
            {
                Path path;
                path.reserve(a.coordinates.size());
                for (const GeoCoordinate& c : a.coordinates)
                    path.push_back(IntPoint(c.longitude * Scale, c.latitude * Scale));

                region.points.push_back(path);
            }

            void visitRelation(const utymap::entities::Relation& r)  { r.accept(builder); }

        } visitor(*this, relation, region);

        for (const auto& element : relation.elements) {
            // if there are no tags, then this element is result of clipping
            if (element->tags.empty())
                element->tags = relation.tags;
            element->accept(visitor);
        }

        if (!region.points.empty()) {
            Style style = context_.styleProvider.forElement(relation, context_.quadKey.levelOfDetail);
            region.isLayer = style.has(context_.stringTable.getId(TerrainLayerKey));
            if (!region.isLayer)
                region.options = createMeshOptions(style, "");

            std::string type = region.isLayer
                ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
                : "";
            layers_[type].push_back(region);
        }
    }

    // builds tile mesh using data provided.
    void complete()
    {
        clipper_.Clear();

        buildLayers();
        buildBackground();

        context_.meshCallback(mesh_);
    }

private:

    // process all found layers.
    void buildLayers()
    {
        // 1. process layers: regions with shared properties.
        std::stringstream ss(*utymap::utils::getString(LayerPriorityKey, context_.stringTable, style_));
        while (ss.good()) {
            std::string name;
            getline(ss, name, ',');
            auto layer = layers_.find(name);
            if (layer != layers_.end()) {
                buildFromRegions(layer->second, createMeshOptions(style_, name + "-"));
                layers_.erase(layer);
            }
        }

        // 2. Process the rest: each region has aready its own properties.
        for (auto& layer : layers_)
            for (auto& region : layer.second) {
                buildFromPaths(region.points, region.options);
            }
    }

    // process the rest area.
    void buildBackground()
    {
        clipper_.AddPath(tileRect_, ptSubject, true);
        Paths background;
        clipper_.Execute(ctDifference, background, pftPositive, pftPositive);
        clipper_.Clear();

        if (!background.empty())
            populateMesh(createMeshOptions(style_, ""), background);
    }

    Region createRegion(const Style& style, const std::vector<GeoCoordinate>& coordinates)
    {
        Region region;
        Path path;
        path.reserve(coordinates.size());
        for (const GeoCoordinate& c : coordinates)
            path.push_back(IntPoint(c.longitude * Scale, c.latitude * Scale));

        region.points.push_back(path);

        region.isLayer = style.has(context_.stringTable.getId(TerrainLayerKey));
        if (!region.isLayer)
            region.options = createMeshOptions(style, "");

        return std::move(region);
    }

    std::shared_ptr<MeshBuilder::Options> createMeshOptions(const Style& style, const std::string& prefix)
    {
        double quadKeyWidth = bbox_.maxPoint.latitude - bbox_.minPoint.latitude;
        auto gradientKey = utymap::utils::getString(prefix + GradientKey, context_.stringTable, style);
        return std::shared_ptr<MeshBuilder::Options>(new MeshBuilder::Options(
            utymap::utils::getDimension(prefix + MaxAreaKey, context_.stringTable, style, quadKeyWidth * quadKeyWidth),
            utymap::utils::getDouble(prefix + EleNoiseFreqKey, context_.stringTable, style),
            utymap::utils::getDouble(prefix + ColorNoiseFreqKey, context_.stringTable, style),
            utymap::utils::getDimension(prefix + HeightKey, context_.stringTable, style, quadKeyWidth, 0),
            context_.styleProvider.getGradient(*gradientKey),
            std::numeric_limits<double>::lowest(),
            *utymap::utils::getString(prefix + MeshNameKey, context_.stringTable, style, ""),
             /* no new vertices on boundaries */ 1));
    }

    void buildFromRegions(const Regions& regions, const std::shared_ptr<MeshBuilder::Options>& options)
    {
        // merge all regions together
        Clipper clipper;
        for (const Region& region : regions)
            clipper.AddPaths(region.points, ptSubject, true);

        Paths result;
        clipper.Execute(ctUnion, result, pftPositive, pftPositive);

        buildFromPaths(result, options);
    }

    void buildFromPaths(Paths& paths, const std::shared_ptr<MeshBuilder::Options>& options)
    {
        clipper_.AddPaths(paths, ptSubject, true);
        paths.clear();
        clipper_.Execute(ctDifference, paths, pftPositive, pftPositive);
        clipper_.moveSubjectToClip();

        populateMesh(options, paths);
    }

    void populateMesh(const std::shared_ptr<MeshBuilder::Options>& options, Paths& paths)
    {
        ClipperLib::SimplifyPolygons(paths);
        ClipperLib::CleanPolygons(paths);

        bool hasHeightOffset = std::abs(options->heightOffset) > 1E-8;
        // calculate approximate size of overall points
        auto size = 0;
        for (auto i = 0; i < paths.size(); ++i)
            size += paths[i].size() * 1.5;

        Polygon polygon(size);
        for (const Path& path : paths) {
            double area = ClipperLib::Area(path);
            bool isHole = area < 0;
            if (std::abs(area) < AreaTolerance)
                continue;

            Points points = restorePoints(path);
            if (isHole)
                polygon.addHole(points);
            else
                polygon.addContour(points);

            if (hasHeightOffset)
                processHeightOffset(options, points, isHole);
        }

        if (!polygon.points.empty())
            fillMesh(options, polygon);
    }

    // restores mesh points from clipper points and injects new ones according to grid.
    Points restorePoints(const Path& path)
    {
        int lastItemIndex = path.size() - 1;
        Points points;
        points.reserve(path.size());
        for (int i = 0; i <= lastItemIndex; i++)
            splitter_.split(path[i], path[i == lastItemIndex ? 0 : i + 1], points);

        return std::move(points);
    }

    void fillMesh(const std::shared_ptr<MeshBuilder::Options>& options, Polygon& polygon)
    {
        if (options->meshName != "") {
            Mesh polygonMesh(options->meshName);
            context_.meshBuilder.addPolygon(polygonMesh, polygon, *options);
            context_.meshCallback(polygonMesh);
        }
        else {
            context_.meshBuilder.addPolygon(mesh_, polygon, *options);
        }
    }

    void processHeightOffset(const std::shared_ptr<MeshBuilder::Options>& options, const Points& points, bool isHole)
    {
        auto index = mesh_.vertices.size() / 3;
        for (auto i = 0; i < points.size(); ++i) {
            Point p1 = points[i];
            Point p2 = points[i == (points.size() - 1) ? 0 : i + 1];

            // check whether two points are on cell rect
            if (rect_.isOnBorder(p1) && rect_.isOnBorder(p2)) continue;

            context_.meshBuilder.addPlane(mesh_, p1, p2, *options);
        }
    }

const Style style_;
ClipperEx clipper_;
ClipperOffset offset_;
LineGridSplitter splitter_;
Rectangle rect_;
BoundingBox bbox_;
Path tileRect_;
Layers layers_;
Mesh mesh_;
};

void TerraBuilder::visitNode(const utymap::entities::Node& node) { pimpl_->visitNode(node); }

void TerraBuilder::visitWay(const utymap::entities::Way& way) { pimpl_->visitWay(way); }

void TerraBuilder::visitArea(const utymap::entities::Area& area) { pimpl_->visitArea(area); }

void TerraBuilder::visitRelation(const utymap::entities::Relation& relation) { pimpl_->visitRelation(relation); }

void TerraBuilder::complete() { pimpl_->complete(); }

TerraBuilder::~TerraBuilder() { }

TerraBuilder::TerraBuilder(const BuilderContext& context) :
    utymap::builders::ElementBuilder(context),
    pimpl_(new TerraBuilder::TerraBuilderImpl(context))
{
}
<commit_msg>core: fix offsetting issue<commit_after>#include "BoundingBox.hpp"
#include "Exceptions.hpp"
#include "clipper/clipper.hpp"
#include "builders/BuilderContext.hpp"
#include "builders/terrain/LineGridSplitter.hpp"
#include "builders/terrain/TerraBuilder.hpp"
#include "entities/Node.hpp"
#include "entities/Way.hpp"
#include "entities/Area.hpp"
#include "entities/Relation.hpp"
#include "entities/ElementVisitor.hpp"
#include "meshing/Polygon.hpp"
#include "meshing/MeshBuilder.hpp"
#include "utils/CompatibilityUtils.hpp"
#include "utils/GeoUtils.hpp"
#include "utils/MapCssUtils.hpp"
#include "utils/NoiseUtils.hpp"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <functional>
#include <iterator>
#include <sstream>
#include <map>
#include <unordered_map>
#include <vector>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::entities;
using namespace utymap::index;
using namespace utymap::heightmap;
using namespace utymap::mapcss;
using namespace utymap::meshing;
using namespace utymap::utils;

const double Scale = 1E7;          // max precision for Lat/Lon: seven decimal positions
const double AreaTolerance = 1000; // Tolerance for meshing

// Represents terrain region points.
struct Region
{
    bool isLayer;
    std::shared_ptr<MeshBuilder::Options> options; // optional: might be empty if polygon is layer
    Paths points;
};

typedef std::vector<Point> Points;
typedef std::vector<Region> Regions;
typedef std::unordered_map<std::string, Regions> Layers;

// mapcss specific keys
const static std::string TerrainLayerKey = "terrain-layer";
const static std::string ColorNoiseFreqKey = "color-noise-freq";
const static std::string EleNoiseFreqKey = "ele-noise-freq";
const static std::string GradientKey= "color";
const static std::string MaxAreaKey = "max-area";
const static std::string WidthKey = "width";
const static std::string HeightKey = "height";
const static std::string LayerPriorityKey = "layer-priority";
const static std::string MeshNameKey = "mesh-name";
const static std::string GridCellSize = "grid-cell-size";

class TerraBuilder::TerraBuilderImpl : public ElementBuilder
{
public:

    TerraBuilderImpl(const BuilderContext& context) :
        ElementBuilder(context),
        style_(context.styleProvider.forCanvas(context.quadKey.levelOfDetail)),
        splitter_(),
        bbox_(GeoUtils::quadKeyToBoundingBox(context_.quadKey)),
        rect_(bbox_.minPoint.longitude, bbox_.minPoint.latitude, bbox_.maxPoint.longitude, bbox_.maxPoint.latitude),
        mesh_("terrain")
    {
        tileRect_.push_back(IntPoint(bbox_.minPoint.longitude*Scale, bbox_.minPoint.latitude *Scale));
        tileRect_.push_back(IntPoint(bbox_.maxPoint.longitude *Scale, bbox_.minPoint.latitude *Scale));
        tileRect_.push_back(IntPoint(bbox_.maxPoint.longitude *Scale, bbox_.maxPoint.latitude*Scale));
        tileRect_.push_back(IntPoint(bbox_.minPoint.longitude*Scale, bbox_.maxPoint.latitude*Scale));

        clipper_.AddPath(tileRect_, ptClip, true);

        double size = utymap::utils::getDimension(GridCellSize, context_.stringTable, style_,
            bbox_.maxPoint.latitude - bbox_.minPoint.latitude, bbox_.center());
        splitter_.setParams(Scale, size);
    }

    void visitNode(const utymap::entities::Node& node)
    {
    }

    void visitWay(const utymap::entities::Way& way)
    {
        Style style = context_.styleProvider.forElement(way, context_.quadKey.levelOfDetail);
        Region region = createRegion(style, way.coordinates);

        // make polygon from line by offsetting it using width specified
        double width = utymap::utils::getDimension(WidthKey, context_.stringTable, style,
            bbox_.maxPoint.latitude - bbox_.minPoint.latitude, bbox_.center());

        Paths solution;
        offset_.AddPaths(region.points, jtMiter, etOpenSquare);
        offset_.Execute(solution, width * Scale);
        offset_.Clear();

        clipper_.AddPaths(solution, ptSubject, true);
        clipper_.Execute(ctIntersection, solution);
        clipper_.removeSubject();

        region.points = solution;
        std::string type = region.isLayer
            ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
            : "";
        layers_[type].push_back(region);
    }

    void visitArea(const utymap::entities::Area& area)
    {
        Style style = context_.styleProvider.forElement(area, context_.quadKey.levelOfDetail);
        Region region = createRegion(style, area.coordinates);
        std::string type = region.isLayer
            ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
            : "";
        layers_[type].push_back(region);
    }

    void visitRelation(const utymap::entities::Relation& relation)
    {
        Region region;
        struct RelationVisitor : public ElementVisitor
        {
            const Relation& relation;
            TerraBuilder::TerraBuilderImpl& builder;
            Region& region;

            RelationVisitor(TerraBuilder::TerraBuilderImpl& builder, const Relation& relation, Region& region) :
                builder(builder), relation(relation), region(region) {}

            void visitNode(const utymap::entities::Node& n) { n.accept(builder); }

            void visitWay(const utymap::entities::Way& w) { w.accept(builder); }

            void visitArea(const utymap::entities::Area& a)
            {
                Path path;
                path.reserve(a.coordinates.size());
                for (const GeoCoordinate& c : a.coordinates)
                    path.push_back(IntPoint(c.longitude * Scale, c.latitude * Scale));

                region.points.push_back(path);
            }

            void visitRelation(const utymap::entities::Relation& r)  { r.accept(builder); }

        } visitor(*this, relation, region);

        for (const auto& element : relation.elements) {
            // if there are no tags, then this element is result of clipping
            if (element->tags.empty())
                element->tags = relation.tags;
            element->accept(visitor);
        }

        if (!region.points.empty()) {
            Style style = context_.styleProvider.forElement(relation, context_.quadKey.levelOfDetail);
            region.isLayer = style.has(context_.stringTable.getId(TerrainLayerKey));
            if (!region.isLayer)
                region.options = createMeshOptions(style, "");

            std::string type = region.isLayer
                ? *utymap::utils::getString(TerrainLayerKey, context_.stringTable, style)
                : "";
            layers_[type].push_back(region);
        }
    }

    // builds tile mesh using data provided.
    void complete()
    {
        clipper_.Clear();

        buildLayers();
        buildBackground();

        context_.meshCallback(mesh_);
    }

private:

    // process all found layers.
    void buildLayers()
    {
        // 1. process layers: regions with shared properties.
        std::stringstream ss(*utymap::utils::getString(LayerPriorityKey, context_.stringTable, style_));
        while (ss.good()) {
            std::string name;
            getline(ss, name, ',');
            auto layer = layers_.find(name);
            if (layer != layers_.end()) {
                buildFromRegions(layer->second, createMeshOptions(style_, name + "-"));
                layers_.erase(layer);
            }
        }

        // 2. Process the rest: each region has aready its own properties.
        for (auto& layer : layers_)
            for (auto& region : layer.second) {
                buildFromPaths(region.points, region.options);
            }
    }

    // process the rest area.
    void buildBackground()
    {
        clipper_.AddPath(tileRect_, ptSubject, true);
        Paths background;
        clipper_.Execute(ctDifference, background, pftPositive, pftPositive);
        clipper_.Clear();

        if (!background.empty())
            populateMesh(createMeshOptions(style_, ""), background);
    }

    Region createRegion(const Style& style, const std::vector<GeoCoordinate>& coordinates)
    {
        Region region;
        Path path;
        path.reserve(coordinates.size());
        for (const GeoCoordinate& c : coordinates)
            path.push_back(IntPoint(c.longitude * Scale, c.latitude * Scale));

        region.points.push_back(path);

        region.isLayer = style.has(context_.stringTable.getId(TerrainLayerKey));
        if (!region.isLayer)
            region.options = createMeshOptions(style, "");

        return std::move(region);
    }

    std::shared_ptr<MeshBuilder::Options> createMeshOptions(const Style& style, const std::string& prefix)
    {
        double quadKeyWidth = bbox_.maxPoint.latitude - bbox_.minPoint.latitude;
        auto gradientKey = utymap::utils::getString(prefix + GradientKey, context_.stringTable, style);
        return std::shared_ptr<MeshBuilder::Options>(new MeshBuilder::Options(
            utymap::utils::getDimension(prefix + MaxAreaKey, context_.stringTable, style, quadKeyWidth * quadKeyWidth),
            utymap::utils::getDouble(prefix + EleNoiseFreqKey, context_.stringTable, style),
            utymap::utils::getDouble(prefix + ColorNoiseFreqKey, context_.stringTable, style),
            utymap::utils::getDimension(prefix + HeightKey, context_.stringTable, style, quadKeyWidth, 0),
            context_.styleProvider.getGradient(*gradientKey),
            std::numeric_limits<double>::lowest(),
            *utymap::utils::getString(prefix + MeshNameKey, context_.stringTable, style, ""),
             /* no new vertices on boundaries */ 1));
    }

    void buildFromRegions(const Regions& regions, const std::shared_ptr<MeshBuilder::Options>& options)
    {
        // merge all regions together
        Clipper clipper;
        for (const Region& region : regions)
            clipper.AddPaths(region.points, ptSubject, true);

        Paths result;
        clipper.Execute(ctUnion, result, pftPositive, pftPositive);

        buildFromPaths(result, options);
    }

    void buildFromPaths(Paths& paths, const std::shared_ptr<MeshBuilder::Options>& options)
    {
        clipper_.AddPaths(paths, ptSubject, true);
        paths.clear();
        clipper_.Execute(ctDifference, paths, pftPositive, pftPositive);
        clipper_.moveSubjectToClip();

        populateMesh(options, paths);
    }

    void populateMesh(const std::shared_ptr<MeshBuilder::Options>& options, Paths& paths)
    {
        ClipperLib::SimplifyPolygons(paths);
        ClipperLib::CleanPolygons(paths);

        bool hasHeightOffset = std::abs(options->heightOffset) > 1E-8;
        // calculate approximate size of overall points
        auto size = 0;
        for (auto i = 0; i < paths.size(); ++i)
            size += paths[i].size() * 1.5;

        Polygon polygon(size);
        for (const Path& path : paths) {
            double area = ClipperLib::Area(path);
            bool isHole = area < 0;
            if (std::abs(area) < AreaTolerance)
                continue;

            Points points = restorePoints(path);
            if (isHole)
                polygon.addHole(points);
            else
                polygon.addContour(points);

            if (hasHeightOffset)
                processHeightOffset(options, points, isHole);
        }

        if (!polygon.points.empty())
            fillMesh(options, polygon);
    }

    // restores mesh points from clipper points and injects new ones according to grid.
    Points restorePoints(const Path& path)
    {
        int lastItemIndex = path.size() - 1;
        Points points;
        points.reserve(path.size());
        for (int i = 0; i <= lastItemIndex; i++)
            splitter_.split(path[i], path[i == lastItemIndex ? 0 : i + 1], points);

        return std::move(points);
    }

    void fillMesh(const std::shared_ptr<MeshBuilder::Options>& options, Polygon& polygon)
    {
        if (options->meshName != "") {
            Mesh polygonMesh(options->meshName);
            context_.meshBuilder.addPolygon(polygonMesh, polygon, *options);
            context_.meshCallback(polygonMesh);
        }
        else {
            context_.meshBuilder.addPolygon(mesh_, polygon, *options);
        }
    }

    void processHeightOffset(const std::shared_ptr<MeshBuilder::Options>& options, const Points& points, bool isHole)
    {
        auto index = mesh_.vertices.size() / 3;
        for (auto i = 0; i < points.size(); ++i) {
            Point p1 = points[i];
            Point p2 = points[i == (points.size() - 1) ? 0 : i + 1];

            // check whether two points are on cell rect
            if (rect_.isOnBorder(p1) && rect_.isOnBorder(p2)) continue;

            context_.meshBuilder.addPlane(mesh_, p1, p2, *options);
        }
    }

const Style style_;
ClipperEx clipper_;
ClipperOffset offset_;
LineGridSplitter splitter_;
BoundingBox bbox_;
Rectangle rect_;
Path tileRect_;
Layers layers_;
Mesh mesh_;
};

void TerraBuilder::visitNode(const utymap::entities::Node& node) { pimpl_->visitNode(node); }

void TerraBuilder::visitWay(const utymap::entities::Way& way) { pimpl_->visitWay(way); }

void TerraBuilder::visitArea(const utymap::entities::Area& area) { pimpl_->visitArea(area); }

void TerraBuilder::visitRelation(const utymap::entities::Relation& relation) { pimpl_->visitRelation(relation); }

void TerraBuilder::complete() { pimpl_->complete(); }

TerraBuilder::~TerraBuilder() { }

TerraBuilder::TerraBuilder(const BuilderContext& context) :
    utymap::builders::ElementBuilder(context),
    pimpl_(new TerraBuilder::TerraBuilderImpl(context))
{
}
<|endoftext|>


<commit_before>/* Copyright 2007 MySQL AB. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#include "mysql_priv.h"
#include "rpl_rli.h"
#include "rpl_record.h"
#include "slave.h"                  // Need to pull in slave_print_msg
#include "rpl_utility.h"
#include "rpl_rli.h"

/**
   Pack a record of data for a table into a format suitable for
   transfer via the binary log.

   The format for a row in transfer with N fields is the following:

   ceil(N/8) null bytes:
       One null bit for every column *regardless of whether it can be
       null or not*. This simplifies the decoding. Observe that the
       number of null bits is equal to the number of set bits in the
       @c cols bitmap. The number of null bytes is the smallest number
       of bytes necessary to store the null bits.

       Padding bits are 1.

   N packets:
       Each field is stored in packed format.

   @param table    Table describing the format of the record

   @param cols     Bitmap with a set bit for each column that should
                   be stored in the row

   @param row_data Pointer to memory where row will be written

   @param record   Pointer to record that should be packed. It is
                   assumed that the pointer refers to either @c
                   record[0] or @c record[1], but no such check is
                   made since the code does not rely on that.

   @return The number of bytes written at @c row_data.
 */
#if !defined(MYSQL_CLIENT)
size_t
pack_row(TABLE *table, MY_BITMAP const* cols,
         uchar *row_data, const uchar *record)
{
  Field **p_field= table->field, *field;
  int const null_byte_count= (bitmap_bits_set(cols) + 7) / 8;
  uchar *pack_ptr = row_data + null_byte_count;
  uchar *null_ptr = row_data;
  my_ptrdiff_t const rec_offset= record - table->record[0];
  my_ptrdiff_t const def_offset= table->s->default_values - table->record[0];

  DBUG_ENTER("pack_row");

  /*
    We write the null bits and the packed records using one pass
    through all the fields. The null bytes are written little-endian,
    i.e., the first fields are in the first byte.
   */
  unsigned int null_bits= (1U << 8) - 1;
  // Mask to mask out the correct but among the null bits
  unsigned int null_mask= 1U;
  for ( ; (field= *p_field) ; p_field++)
  {
    DBUG_PRINT("debug", ("null_mask=%d; null_ptr=%p; row_data=%p; null_byte_count=%d",
                         null_mask, null_ptr, row_data, null_byte_count));
    if (bitmap_is_set(cols, p_field - table->field))
    {
      my_ptrdiff_t offset;
      if (field->is_null(rec_offset))
      {
        offset= def_offset;
        null_bits |= null_mask;
      }
      else
      {
        offset= rec_offset;
        null_bits &= ~null_mask;

        /*
          We only store the data of the field if it is non-null

          For big-endian machines, we have to make sure that the
          length is stored in little-endian format, since this is the
          format used for the binlog.
        */
        const uchar *old_pack_ptr= pack_ptr;
        pack_ptr= field->pack(pack_ptr, field->ptr + offset,
                              field->max_data_length(), TRUE);
        DBUG_PRINT("debug", ("field: %s; pack_ptr: 0x%lx;"
                             " pack_ptr':0x%lx; bytes: %d",
                             field->field_name, (ulong) old_pack_ptr,
                             (ulong) pack_ptr,
                             (int) (pack_ptr - old_pack_ptr)));
      }

      null_mask <<= 1;
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + null_byte_count);
        null_mask = 1U;
        *null_ptr++ = null_bits;
        null_bits= (1U << 8) - 1;
      }
    }
  }

  /*
    Write the last (partial) byte, if there is one
  */
  if ((null_mask & 0xFF) > 1)
  {
    DBUG_ASSERT(null_ptr < row_data + null_byte_count);
    *null_ptr++ = null_bits;
  }

  /*
    The null pointer should now point to the first byte of the
    packed data. If it doesn't, something is very wrong.
  */
  DBUG_ASSERT(null_ptr == row_data + null_byte_count);
  DBUG_DUMP("row_data", row_data, pack_ptr - row_data);
  DBUG_RETURN(static_cast<size_t>(pack_ptr - row_data));
}
#endif

/**
   Unpack a row into @c table->record[0].

   The function will always unpack into the @c table->record[0]
   record.  This is because there are too many dependencies on where
   the various member functions of Field and subclasses expect to
   write.

   The row is assumed to only consist of the fields for which the corresponding
   bit in bitset @c cols is set; the other parts of the record are left alone.

   At most @c colcnt columns are read: if the table is larger than
   that, the remaining fields are not filled in.

   @param rli     Relay log info
   @param table   Table to unpack into
   @param colcnt  Number of columns to read from record
   @param row_data
                  Packed row data
   @param cols    Pointer to bitset describing columns to fill in
   @param row_end Pointer to variable that will hold the value of the
                  one-after-end position for the row
   @param master_reclength
                  Pointer to variable that will be set to the length of the
                  record on the master side

   @retval 0 No error

   @retval ER_NO_DEFAULT_FOR_FIELD
   Returned if one of the fields existing on the slave but not on the
   master does not have a default value (and isn't nullable)

 */
#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION)
int
unpack_row(Relay_log_info const *rli,
           TABLE *table, uint const colcnt,
           uchar const *const row_data, MY_BITMAP const *cols,
           uchar const **const row_end, ulong *const master_reclength)
{
  DBUG_ENTER("unpack_row");
  DBUG_ASSERT(row_data);
  size_t const master_null_byte_count= (bitmap_bits_set(cols) + 7) / 8;
  int error= 0;

  uchar const *null_ptr= row_data;
  uchar const *pack_ptr= row_data + master_null_byte_count;

  Field **const begin_ptr = table->field;
  Field **field_ptr;
  Field **const end_ptr= begin_ptr + colcnt;

  DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);

  // Mask to mask out the correct bit among the null bits
  unsigned int null_mask= 1U;
  // The "current" null bits
  unsigned int null_bits= *null_ptr++;
  uint i= 0;
  table_def *tabledef= ((Relay_log_info*)rli)->get_tabledef(table);
  for (field_ptr= begin_ptr ; field_ptr < end_ptr && *field_ptr ; ++field_ptr)
  {
    Field *const f= *field_ptr;

    /*
      No need to bother about columns that does not exist: they have
      gotten default values when being emptied above.
     */
    if (bitmap_is_set(cols, field_ptr -  begin_ptr))
    {
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);
        null_mask= 1U;
        null_bits= *null_ptr++;
      }

      DBUG_ASSERT(null_mask & 0xFF); // One of the 8 LSB should be set

      /* Field...::unpack() cannot return 0 */
      DBUG_ASSERT(pack_ptr != NULL);

      if ((null_bits & null_mask) && f->maybe_null())
        f->set_null();
      else
      {
        f->set_notnull();

        /*
          We only unpack the field if it was non-null.
          Use the master's size information if available else call
          normal unpack operation.
        */
        uint16 const metadata= tabledef->field_metadata(i);
        uchar const *const old_pack_ptr= pack_ptr;
        pack_ptr= f->unpack(f->ptr, pack_ptr, metadata, TRUE);
	DBUG_PRINT("debug", ("field: %s; metadata: 0x%x;"
                             " pack_ptr: 0x%lx; pack_ptr': 0x%lx; bytes: %d",
                             f->field_name, metadata,
                             (ulong) old_pack_ptr, (ulong) pack_ptr,
                             (int) (pack_ptr - old_pack_ptr)));
      }

      null_mask <<= 1;
    }
    i++;
  }

  /*
    throw away master's extra fields
  */
  uint max_cols= min(tabledef->size(), cols->n_bits);
  for (; i < max_cols; i++)
  {
    if (bitmap_is_set(cols, i))
    {
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);
        null_mask= 1U;
        null_bits= *null_ptr++;
      }
      DBUG_ASSERT(null_mask & 0xFF); // One of the 8 LSB should be set

      if (!((null_bits & null_mask) && tabledef->maybe_null(i)))
        pack_ptr+= tabledef->calc_field_size(i, (uchar *) pack_ptr);
      null_mask <<= 1;
    }
  }

  /*
    We should now have read all the null bytes, otherwise something is
    really wrong.
   */
  DBUG_ASSERT(null_ptr == row_data + master_null_byte_count);

  DBUG_DUMP("row_data", row_data, pack_ptr - row_data);

  *row_end = pack_ptr;
  if (master_reclength)
  {
    if (*field_ptr)
      *master_reclength = (*field_ptr)->ptr - table->record[0];
    else
      *master_reclength = table->s->reclength;
  }

  DBUG_RETURN(error);
}

/**
  Fills @c table->record[0] with default values.

  First @c empty_record() is called and then, additionally, fields are
  initialized explicitly with a call to @c set_default().

  For optimization reasons, the explicit initialization can be skipped for
  first @c skip fields. This is useful if later we are going to fill these
  fields from other source (e.g. from a Rows replication event).

  If @c check is true, fields are explicitly initialized only if they have
  default value or can be NULL. Otherwise error is reported.

  @param log    Used to report errors.
  @param table  Table whose record[0] buffer is prepared.
  @param skip   Number of columns for which default value initialization
                should be skipped.
  @param check  Indicates if errors should be checked when setting default
                values.

  @returns 0 on success.
 */
int prepare_record(const Slave_reporting_capability *const log,
                   TABLE *const table,
                   const uint skip, const bool check)
{
  DBUG_ENTER("prepare_record");

  int error= 0;
  empty_record(table);

  if (skip >= table->s->fields)  // nothing to do
    DBUG_RETURN(0);

  /* Explicit initialization of fields */

  for (Field **field_ptr= table->field+skip ; *field_ptr ; ++field_ptr)
  {
    uint32 const mask= NOT_NULL_FLAG | NO_DEFAULT_VALUE_FLAG;
    Field *const f= *field_ptr;

    if (check && ((f->flags & mask) == mask))
    {
      DBUG_ASSERT(log);
      log->report(ERROR_LEVEL, ER_NO_DEFAULT_FOR_FIELD,
                  "Field `%s` of table `%s`.`%s` "
                  "has no default value and cannot be NULL",
                  f->field_name, table->s->db.str,
                  table->s->table_name.str);
      error = ER_NO_DEFAULT_FOR_FIELD;
    }
    else
      f->set_default();
  }

  DBUG_RETURN(error);
}

#endif // HAVE_REPLICATION
<commit_msg>Removing debug variables from non-debug builds.<commit_after>/* Copyright 2007 MySQL AB. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#include "mysql_priv.h"
#include "rpl_rli.h"
#include "rpl_record.h"
#include "slave.h"                  // Need to pull in slave_print_msg
#include "rpl_utility.h"
#include "rpl_rli.h"

/**
   Pack a record of data for a table into a format suitable for
   transfer via the binary log.

   The format for a row in transfer with N fields is the following:

   ceil(N/8) null bytes:
       One null bit for every column *regardless of whether it can be
       null or not*. This simplifies the decoding. Observe that the
       number of null bits is equal to the number of set bits in the
       @c cols bitmap. The number of null bytes is the smallest number
       of bytes necessary to store the null bits.

       Padding bits are 1.

   N packets:
       Each field is stored in packed format.

   @param table    Table describing the format of the record

   @param cols     Bitmap with a set bit for each column that should
                   be stored in the row

   @param row_data Pointer to memory where row will be written

   @param record   Pointer to record that should be packed. It is
                   assumed that the pointer refers to either @c
                   record[0] or @c record[1], but no such check is
                   made since the code does not rely on that.

   @return The number of bytes written at @c row_data.
 */
#if !defined(MYSQL_CLIENT)
size_t
pack_row(TABLE *table, MY_BITMAP const* cols,
         uchar *row_data, const uchar *record)
{
  Field **p_field= table->field, *field;
  int const null_byte_count= (bitmap_bits_set(cols) + 7) / 8;
  uchar *pack_ptr = row_data + null_byte_count;
  uchar *null_ptr = row_data;
  my_ptrdiff_t const rec_offset= record - table->record[0];
  my_ptrdiff_t const def_offset= table->s->default_values - table->record[0];

  DBUG_ENTER("pack_row");

  /*
    We write the null bits and the packed records using one pass
    through all the fields. The null bytes are written little-endian,
    i.e., the first fields are in the first byte.
   */
  unsigned int null_bits= (1U << 8) - 1;
  // Mask to mask out the correct but among the null bits
  unsigned int null_mask= 1U;
  for ( ; (field= *p_field) ; p_field++)
  {
    DBUG_PRINT("debug", ("null_mask=%d; null_ptr=%p; row_data=%p; null_byte_count=%d",
                         null_mask, null_ptr, row_data, null_byte_count));
    if (bitmap_is_set(cols, p_field - table->field))
    {
      my_ptrdiff_t offset;
      if (field->is_null(rec_offset))
      {
        offset= def_offset;
        null_bits |= null_mask;
      }
      else
      {
        offset= rec_offset;
        null_bits &= ~null_mask;

        /*
          We only store the data of the field if it is non-null

          For big-endian machines, we have to make sure that the
          length is stored in little-endian format, since this is the
          format used for the binlog.
        */
#ifndef DBUG_OFF
        const uchar *old_pack_ptr= pack_ptr;
#endif
        pack_ptr= field->pack(pack_ptr, field->ptr + offset,
                              field->max_data_length(), TRUE);
        DBUG_PRINT("debug", ("field: %s; pack_ptr: 0x%lx;"
                             " pack_ptr':0x%lx; bytes: %d",
                             field->field_name, (ulong) old_pack_ptr,
                             (ulong) pack_ptr,
                             (int) (pack_ptr - old_pack_ptr)));
      }

      null_mask <<= 1;
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + null_byte_count);
        null_mask = 1U;
        *null_ptr++ = null_bits;
        null_bits= (1U << 8) - 1;
      }
    }
  }

  /*
    Write the last (partial) byte, if there is one
  */
  if ((null_mask & 0xFF) > 1)
  {
    DBUG_ASSERT(null_ptr < row_data + null_byte_count);
    *null_ptr++ = null_bits;
  }

  /*
    The null pointer should now point to the first byte of the
    packed data. If it doesn't, something is very wrong.
  */
  DBUG_ASSERT(null_ptr == row_data + null_byte_count);
  DBUG_DUMP("row_data", row_data, pack_ptr - row_data);
  DBUG_RETURN(static_cast<size_t>(pack_ptr - row_data));
}
#endif

/**
   Unpack a row into @c table->record[0].

   The function will always unpack into the @c table->record[0]
   record.  This is because there are too many dependencies on where
   the various member functions of Field and subclasses expect to
   write.

   The row is assumed to only consist of the fields for which the corresponding
   bit in bitset @c cols is set; the other parts of the record are left alone.

   At most @c colcnt columns are read: if the table is larger than
   that, the remaining fields are not filled in.

   @param rli     Relay log info
   @param table   Table to unpack into
   @param colcnt  Number of columns to read from record
   @param row_data
                  Packed row data
   @param cols    Pointer to bitset describing columns to fill in
   @param row_end Pointer to variable that will hold the value of the
                  one-after-end position for the row
   @param master_reclength
                  Pointer to variable that will be set to the length of the
                  record on the master side

   @retval 0 No error

   @retval ER_NO_DEFAULT_FOR_FIELD
   Returned if one of the fields existing on the slave but not on the
   master does not have a default value (and isn't nullable)

 */
#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION)
int
unpack_row(Relay_log_info const *rli,
           TABLE *table, uint const colcnt,
           uchar const *const row_data, MY_BITMAP const *cols,
           uchar const **const row_end, ulong *const master_reclength)
{
  DBUG_ENTER("unpack_row");
  DBUG_ASSERT(row_data);
  size_t const master_null_byte_count= (bitmap_bits_set(cols) + 7) / 8;
  int error= 0;

  uchar const *null_ptr= row_data;
  uchar const *pack_ptr= row_data + master_null_byte_count;

  Field **const begin_ptr = table->field;
  Field **field_ptr;
  Field **const end_ptr= begin_ptr + colcnt;

  DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);

  // Mask to mask out the correct bit among the null bits
  unsigned int null_mask= 1U;
  // The "current" null bits
  unsigned int null_bits= *null_ptr++;
  uint i= 0;
  table_def *tabledef= ((Relay_log_info*)rli)->get_tabledef(table);
  for (field_ptr= begin_ptr ; field_ptr < end_ptr && *field_ptr ; ++field_ptr)
  {
    Field *const f= *field_ptr;

    /*
      No need to bother about columns that does not exist: they have
      gotten default values when being emptied above.
     */
    if (bitmap_is_set(cols, field_ptr -  begin_ptr))
    {
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);
        null_mask= 1U;
        null_bits= *null_ptr++;
      }

      DBUG_ASSERT(null_mask & 0xFF); // One of the 8 LSB should be set

      /* Field...::unpack() cannot return 0 */
      DBUG_ASSERT(pack_ptr != NULL);

      if ((null_bits & null_mask) && f->maybe_null())
        f->set_null();
      else
      {
        f->set_notnull();

        /*
          We only unpack the field if it was non-null.
          Use the master's size information if available else call
          normal unpack operation.
        */
        uint16 const metadata= tabledef->field_metadata(i);
#ifndef DBUG_OFF
        uchar const *const old_pack_ptr= pack_ptr;
#endif
        pack_ptr= f->unpack(f->ptr, pack_ptr, metadata, TRUE);
	DBUG_PRINT("debug", ("field: %s; metadata: 0x%x;"
                             " pack_ptr: 0x%lx; pack_ptr': 0x%lx; bytes: %d",
                             f->field_name, metadata,
                             (ulong) old_pack_ptr, (ulong) pack_ptr,
                             (int) (pack_ptr - old_pack_ptr)));
      }

      null_mask <<= 1;
    }
    i++;
  }

  /*
    throw away master's extra fields
  */
  uint max_cols= min(tabledef->size(), cols->n_bits);
  for (; i < max_cols; i++)
  {
    if (bitmap_is_set(cols, i))
    {
      if ((null_mask & 0xFF) == 0)
      {
        DBUG_ASSERT(null_ptr < row_data + master_null_byte_count);
        null_mask= 1U;
        null_bits= *null_ptr++;
      }
      DBUG_ASSERT(null_mask & 0xFF); // One of the 8 LSB should be set

      if (!((null_bits & null_mask) && tabledef->maybe_null(i)))
        pack_ptr+= tabledef->calc_field_size(i, (uchar *) pack_ptr);
      null_mask <<= 1;
    }
  }

  /*
    We should now have read all the null bytes, otherwise something is
    really wrong.
   */
  DBUG_ASSERT(null_ptr == row_data + master_null_byte_count);

  DBUG_DUMP("row_data", row_data, pack_ptr - row_data);

  *row_end = pack_ptr;
  if (master_reclength)
  {
    if (*field_ptr)
      *master_reclength = (*field_ptr)->ptr - table->record[0];
    else
      *master_reclength = table->s->reclength;
  }

  DBUG_RETURN(error);
}

/**
  Fills @c table->record[0] with default values.

  First @c empty_record() is called and then, additionally, fields are
  initialized explicitly with a call to @c set_default().

  For optimization reasons, the explicit initialization can be skipped for
  first @c skip fields. This is useful if later we are going to fill these
  fields from other source (e.g. from a Rows replication event).

  If @c check is true, fields are explicitly initialized only if they have
  default value or can be NULL. Otherwise error is reported.

  @param log    Used to report errors.
  @param table  Table whose record[0] buffer is prepared.
  @param skip   Number of columns for which default value initialization
                should be skipped.
  @param check  Indicates if errors should be checked when setting default
                values.

  @returns 0 on success.
 */
int prepare_record(const Slave_reporting_capability *const log,
                   TABLE *const table,
                   const uint skip, const bool check)
{
  DBUG_ENTER("prepare_record");

  int error= 0;
  empty_record(table);

  if (skip >= table->s->fields)  // nothing to do
    DBUG_RETURN(0);

  /* Explicit initialization of fields */

  for (Field **field_ptr= table->field+skip ; *field_ptr ; ++field_ptr)
  {
    uint32 const mask= NOT_NULL_FLAG | NO_DEFAULT_VALUE_FLAG;
    Field *const f= *field_ptr;

    if (check && ((f->flags & mask) == mask))
    {
      DBUG_ASSERT(log);
      log->report(ERROR_LEVEL, ER_NO_DEFAULT_FOR_FIELD,
                  "Field `%s` of table `%s`.`%s` "
                  "has no default value and cannot be NULL",
                  f->field_name, table->s->db.str,
                  table->s->table_name.str);
      error = ER_NO_DEFAULT_FOR_FIELD;
    }
    else
      f->set_default();
  }

  DBUG_RETURN(error);
}

#endif // HAVE_REPLICATION
<|endoftext|>


<commit_before>/** \file    TextUtil.h
 *  \brief   Declarations of text related utility functions.
 *  \author  Dr. Johannes Ruscheinski
 *  \author  Jiangtao Hu
 */

/*
 *  Copyright 2003-2009 Project iVia.
 *  Copyright 2003-2009 The Regents of The University of California.
 *  Copyright 2015 Universitätsbibliothek Tübingen.
 *
 *  This file is part of the libiViaCore package.
 *
 *  The libiViaCore package is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License as published
 *  by the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libiViaCore is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with libiViaCore; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "TextUtil.h"
#include <algorithm>
#include <exception>
#include <cwctype>
#include "Compiler.h"
#include "Locale.h"
#include "HtmlParser.h"
#include "RegexMatcher.h"
#include "StringUtil.h"

namespace {

class TextExtractor: public HtmlParser {
    std::string &extracted_text_;
public:
    TextExtractor(const std::string &html, std::string * const extracted_text)
        : HtmlParser(html, HtmlParser::TEXT), extracted_text_(*extracted_text) { }
    virtual void notify(const HtmlParser::Chunk &chunk);
};

void TextExtractor::notify(const HtmlParser::Chunk &chunk) {
    if (chunk.type_ == HtmlParser::TEXT)
        extracted_text_ += chunk.text_;
}

} // unnamned namespace

namespace TextUtil {

std::string ExtractText(const std::string &html) {
    std::string extracted_text;
    TextExtractor extractor(html, &extracted_text);
    extractor.parse();

    return extracted_text;
}

bool IsRomanNumeral(const std::string &s) {
    if (s.empty())
        return false;

    std::string err_msg;
    static RegexMatcher *matcher(nullptr);
    if (unlikely(matcher == nullptr)) {
        const std::string pattern("^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$");
        matcher = RegexMatcher::RegexMatcherFactory(pattern, &err_msg);
        if (unlikely(matcher == nullptr))
            throw std::runtime_error("Failed to construct a RegexMatcher for \"" + pattern
                                     + "\" in TextUtil::IsRomanNumeral: " + err_msg);
    }

    const bool retcode(matcher->matched(s, &err_msg));
    if (unlikely(not err_msg.empty()))
        throw std::runtime_error("Failed to match \"" + s + "\" against pattern \"" + matcher->getPattern()
                                 + "\" in TextUtil::IsRomanNumeral: " + err_msg);

    return retcode;
}

bool IsUnsignedInteger(const std::string &s) {
    std::string err_msg;
    static RegexMatcher *matcher(nullptr);
    if (unlikely(matcher == nullptr)) {
        const std::string pattern("^[0-9]+$");
        matcher = RegexMatcher::RegexMatcherFactory(pattern, &err_msg);
        if (unlikely(matcher == nullptr))
            throw std::runtime_error("Failed to construct a RegexMatcher for \"" + pattern
                                     + "\" in TextUtil::IsUnsignedInteger: " + err_msg);
    }

    const bool retcode(matcher->matched(s, &err_msg));
    if (unlikely(not err_msg.empty()))
        throw std::runtime_error("Failed to match \"" + s + "\" against pattern \"" + matcher->getPattern()
                                 + "\" in TextUtil::IsUnsignedInteger: " + err_msg);

    return retcode;
}

bool UTF8toWCharString(const std::string &utf8_string, std::wstring * wchar_string) {
    wchar_string->clear();

    const char *cp(utf8_string.c_str());
    size_t remainder(utf8_string.size());
    std::mbstate_t state = std::mbstate_t();
    for (;;) {
        wchar_t wch;
        const size_t retcode(std::mbrtowc(&wch, cp, remainder, &state));
        if (retcode == static_cast<size_t>(-1) or retcode == static_cast<size_t>(-2))
            return false;
        if (retcode == 0)
            return true;
        *wchar_string += wch;
        cp += retcode;
        remainder -= retcode;
    }
}

bool WCharToUTF8String(const std::wstring &wchar_string, std::string * utf8_string) {
    utf8_string->clear();

    char buf[6];
    std::mbstate_t state = std::mbstate_t();
    for (const auto wch : wchar_string) {
        const size_t retcode(std::wcrtomb(buf, wch, &state));
        if (retcode == static_cast<size_t>(-1))
            return false;
        utf8_string->append(buf, retcode);
    }

    return true;
}

bool WCharToUTF8String(const wchar_t wchar, std::string * utf8_string) {
    utf8_string->clear();

    char buf[6];
    std::mbstate_t state = std::mbstate_t();
    const size_t retcode(std::wcrtomb(buf, wchar, &state));
    if (retcode == static_cast<size_t>(-1))
        return false;
    utf8_string->append(buf, retcode);
    return true;
}

bool UTF8ToLower(const std::string &utf8_string, std::string * const lowercase_utf8_string) {
    std::wstring wchar_string;
    if (not UTF8toWCharString(utf8_string, &wchar_string))
        return false;

    // Lowercase the wide character string:
    std::wstring lowercase_wide_string;
    for (const auto wide_ch : wchar_string)
        lowercase_wide_string += std::towlower(static_cast<wint_t>(wide_ch));

    return WCharToUTF8String(lowercase_wide_string, lowercase_utf8_string);
}

namespace {

/** \return True if "number_candidate" is non-empty and consists only of characters belonging
 *          to the wide-character class "digit"
 */
bool IsNumber(const std::wstring &number_candidate) {
    if (number_candidate.empty())
        return false;

    for (const wchar_t ch : number_candidate) {
        if (not std::iswdigit(ch))
            return false;
    }

    return true;
}

template<typename ContainerType> bool ChopIntoWords(const std::string &text, ContainerType * const words,
                                                    const unsigned min_word_length)
{
    words->clear();

    std::wstring wide_text;
    if (unlikely(not UTF8toWCharString(text, &wide_text)))
        return false;

    std::wstring word;
    std::string utf8_word;
    bool leading(true);
    for (const wchar_t ch : wide_text) {
        if (leading and (ch == L'-' or ch == L'\''))
            ; // Do nothing!
        else if (iswalnum(ch) or ch == L'-' or ch == L'\'') {
            word += ch;
            leading = false;
        } else if (ch == L'.' and IsNumber(word)) {
            word += ch;
            if (word.length() >= min_word_length) {
                if (unlikely(not WCharToUTF8String(word, &utf8_word)))
                    return false;
                words->insert(words->end(), utf8_word);
            }
            word.clear();
            leading = true;
        } else {
            // Remove trailing and leading hyphens and quotes:
            while (word.length() > 0 and (word[word.length() - 1] == L'-' or word[word.length() - 1] == L'\''))
                word.resize(word.length() - 1);
            if (word.length() >= min_word_length) {
                if (unlikely(not WCharToUTF8String(word, &utf8_word)))
                    return false;
                words->insert(words->end(), utf8_word);
            }
            word.clear();
            leading = true;
        }
    }

    // Remove trailing and leading hyphens and quotes:
    while (word.length() > 0 and word[word.length() - 1] == '-')
        word.resize(word.length() - 1);
    if (word.length() >= min_word_length) {
        if (unlikely(not WCharToUTF8String(word, &utf8_word)))
            return false;
        words->insert(words->end(), utf8_word);
    }

    return true;
}

} // unnamed namespace

bool ChopIntoWords(const std::string &text, std::unordered_set<std::string> * const words,
                   const unsigned min_word_length)
{
    return ChopIntoWords<std::unordered_set<std::string>> (text, words, min_word_length);
}

bool ChopIntoWords(const std::string &text, std::vector<std::string> * const words,
                   const unsigned min_word_length)
{
    return ChopIntoWords<std::vector<std::string>> (text, words, min_word_length);
}

std::vector<std::string>::const_iterator FindSubstring(const std::vector<std::string> &haystack,
                                                       const std::vector<std::string> &needle)
{
    if (needle.empty())
        return haystack.cbegin();

    std::vector<std::string>::const_iterator search_start(haystack.cbegin());
    while (search_start != haystack.cend()) {
        const std::vector<std::string>::const_iterator haystack_start(
            std::find(search_start, haystack.cend(), needle[0]));
        if ((haystack.cend() - haystack_start) < static_cast<ssize_t>(needle.size()))
            return haystack.cend();

        std::vector<std::string>::const_iterator needle_word(needle.cbegin());
        std::vector<std::string>::const_iterator haystack_word(haystack_start);
        for (;;) {
            ++needle_word;
            if (needle_word == needle.cend())
                return haystack_start;
            ++haystack_word;
            if (haystack_word == haystack.cend())
                return haystack.cend();
            else if (*haystack_word != *needle_word) {
                search_start = haystack_start + 1;
                break;
            }
        }
    }

    return haystack.cend();
}

std::string Base64Encode(const std::string &s, const char symbol63, const char symbol64) {
    static char symbols[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\0\0";
    symbols[62] = symbol63;
    symbols[63] = symbol64;

    std::string encoded_chars;
    std::string::const_iterator ch(s.begin());
    while (ch != s.end()) {
        // Collect groups of 3 characters:
        unsigned buf(static_cast<unsigned char>(*ch));
        buf <<= 8u;
        ++ch;
        unsigned ignore_count(0);
        if (ch != s.end()) {
            buf |= static_cast<unsigned char>(*ch);
            ++ch;
        } else
            ++ignore_count;
        buf <<= 8u;
        if (ch != s.end()) {
            buf |= static_cast<unsigned char>(*ch);
            ++ch;
        }
        else
            ++ignore_count;

        // Now grab 6 bits at a time and encode them starting with the 4th character:
        char next4[4];
        for (unsigned char_no(0); char_no < 4; ++char_no) {
            next4[4 - 1 - char_no] = symbols[buf & 0x3Fu];
            buf >>= 6u;
        }

        for (unsigned char_no(0); char_no < 4 - ignore_count; ++char_no)
            encoded_chars += next4[char_no];
    }

    return encoded_chars;
}

inline bool IsWhiteSpace(const char ch) {
    return ch == ' ' or ch == '\t' or ch == '\n' or ch == '\v' or ch == '\xA0';
}

inline std::string OctalEscape(const char ch) {
    return "\\" + StringUtil::ToString(static_cast<unsigned>(ch), 8, 3);
}

std::string EscapeString(const std::string &original_string, const bool also_escape_whitespace) {
    std::string escaped_string;
    escaped_string.reserve(original_string.size() * 2);

    for (char ch : original_string) {
        if (std::iscntrl(ch) and (not also_escape_whitespace or IsWhiteSpace(ch)))
            escaped_string += OctalEscape(ch);
        else
            escaped_string += ch;
    }

    return escaped_string;
}

} // namespace TextUtil
<commit_msg>Fixed a simple logic error.<commit_after>/** \file    TextUtil.h
 *  \brief   Declarations of text related utility functions.
 *  \author  Dr. Johannes Ruscheinski
 *  \author  Jiangtao Hu
 */

/*
 *  Copyright 2003-2009 Project iVia.
 *  Copyright 2003-2009 The Regents of The University of California.
 *  Copyright 2015 Universitätsbibliothek Tübingen.
 *
 *  This file is part of the libiViaCore package.
 *
 *  The libiViaCore package is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License as published
 *  by the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libiViaCore is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with libiViaCore; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "TextUtil.h"
#include <algorithm>
#include <exception>
#include <cwctype>
#include "Compiler.h"
#include "Locale.h"
#include "HtmlParser.h"
#include "RegexMatcher.h"
#include "StringUtil.h"

namespace {

class TextExtractor: public HtmlParser {
    std::string &extracted_text_;
public:
    TextExtractor(const std::string &html, std::string * const extracted_text)
        : HtmlParser(html, HtmlParser::TEXT), extracted_text_(*extracted_text) { }
    virtual void notify(const HtmlParser::Chunk &chunk);
};

void TextExtractor::notify(const HtmlParser::Chunk &chunk) {
    if (chunk.type_ == HtmlParser::TEXT)
        extracted_text_ += chunk.text_;
}

} // unnamned namespace

namespace TextUtil {

std::string ExtractText(const std::string &html) {
    std::string extracted_text;
    TextExtractor extractor(html, &extracted_text);
    extractor.parse();

    return extracted_text;
}

bool IsRomanNumeral(const std::string &s) {
    if (s.empty())
        return false;

    std::string err_msg;
    static RegexMatcher *matcher(nullptr);
    if (unlikely(matcher == nullptr)) {
        const std::string pattern("^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$");
        matcher = RegexMatcher::RegexMatcherFactory(pattern, &err_msg);
        if (unlikely(matcher == nullptr))
            throw std::runtime_error("Failed to construct a RegexMatcher for \"" + pattern
                                     + "\" in TextUtil::IsRomanNumeral: " + err_msg);
    }

    const bool retcode(matcher->matched(s, &err_msg));
    if (unlikely(not err_msg.empty()))
        throw std::runtime_error("Failed to match \"" + s + "\" against pattern \"" + matcher->getPattern()
                                 + "\" in TextUtil::IsRomanNumeral: " + err_msg);

    return retcode;
}

bool IsUnsignedInteger(const std::string &s) {
    std::string err_msg;
    static RegexMatcher *matcher(nullptr);
    if (unlikely(matcher == nullptr)) {
        const std::string pattern("^[0-9]+$");
        matcher = RegexMatcher::RegexMatcherFactory(pattern, &err_msg);
        if (unlikely(matcher == nullptr))
            throw std::runtime_error("Failed to construct a RegexMatcher for \"" + pattern
                                     + "\" in TextUtil::IsUnsignedInteger: " + err_msg);
    }

    const bool retcode(matcher->matched(s, &err_msg));
    if (unlikely(not err_msg.empty()))
        throw std::runtime_error("Failed to match \"" + s + "\" against pattern \"" + matcher->getPattern()
                                 + "\" in TextUtil::IsUnsignedInteger: " + err_msg);

    return retcode;
}

bool UTF8toWCharString(const std::string &utf8_string, std::wstring * wchar_string) {
    wchar_string->clear();

    const char *cp(utf8_string.c_str());
    size_t remainder(utf8_string.size());
    std::mbstate_t state = std::mbstate_t();
    for (;;) {
        wchar_t wch;
        const size_t retcode(std::mbrtowc(&wch, cp, remainder, &state));
        if (retcode == static_cast<size_t>(-1) or retcode == static_cast<size_t>(-2))
            return false;
        if (retcode == 0)
            return true;
        *wchar_string += wch;
        cp += retcode;
        remainder -= retcode;
    }
}

bool WCharToUTF8String(const std::wstring &wchar_string, std::string * utf8_string) {
    utf8_string->clear();

    char buf[6];
    std::mbstate_t state = std::mbstate_t();
    for (const auto wch : wchar_string) {
        const size_t retcode(std::wcrtomb(buf, wch, &state));
        if (retcode == static_cast<size_t>(-1))
            return false;
        utf8_string->append(buf, retcode);
    }

    return true;
}

bool WCharToUTF8String(const wchar_t wchar, std::string * utf8_string) {
    utf8_string->clear();

    char buf[6];
    std::mbstate_t state = std::mbstate_t();
    const size_t retcode(std::wcrtomb(buf, wchar, &state));
    if (retcode == static_cast<size_t>(-1))
        return false;
    utf8_string->append(buf, retcode);
    return true;
}

bool UTF8ToLower(const std::string &utf8_string, std::string * const lowercase_utf8_string) {
    std::wstring wchar_string;
    if (not UTF8toWCharString(utf8_string, &wchar_string))
        return false;

    // Lowercase the wide character string:
    std::wstring lowercase_wide_string;
    for (const auto wide_ch : wchar_string)
        lowercase_wide_string += std::towlower(static_cast<wint_t>(wide_ch));

    return WCharToUTF8String(lowercase_wide_string, lowercase_utf8_string);
}

namespace {

/** \return True if "number_candidate" is non-empty and consists only of characters belonging
 *          to the wide-character class "digit"
 */
bool IsNumber(const std::wstring &number_candidate) {
    if (number_candidate.empty())
        return false;

    for (const wchar_t ch : number_candidate) {
        if (not std::iswdigit(ch))
            return false;
    }

    return true;
}

template<typename ContainerType> bool ChopIntoWords(const std::string &text, ContainerType * const words,
                                                    const unsigned min_word_length)
{
    words->clear();

    std::wstring wide_text;
    if (unlikely(not UTF8toWCharString(text, &wide_text)))
        return false;

    std::wstring word;
    std::string utf8_word;
    bool leading(true);
    for (const wchar_t ch : wide_text) {
        if (leading and (ch == L'-' or ch == L'\''))
            ; // Do nothing!
        else if (iswalnum(ch) or ch == L'-' or ch == L'\'') {
            word += ch;
            leading = false;
        } else if (ch == L'.' and IsNumber(word)) {
            word += ch;
            if (word.length() >= min_word_length) {
                if (unlikely(not WCharToUTF8String(word, &utf8_word)))
                    return false;
                words->insert(words->end(), utf8_word);
            }
            word.clear();
            leading = true;
        } else {
            // Remove trailing and leading hyphens and quotes:
            while (word.length() > 0 and (word[word.length() - 1] == L'-' or word[word.length() - 1] == L'\''))
                word.resize(word.length() - 1);
            if (word.length() >= min_word_length) {
                if (unlikely(not WCharToUTF8String(word, &utf8_word)))
                    return false;
                words->insert(words->end(), utf8_word);
            }
            word.clear();
            leading = true;
        }
    }

    // Remove trailing and leading hyphens and quotes:
    while (word.length() > 0 and word[word.length() - 1] == '-')
        word.resize(word.length() - 1);
    if (word.length() >= min_word_length) {
        if (unlikely(not WCharToUTF8String(word, &utf8_word)))
            return false;
        words->insert(words->end(), utf8_word);
    }

    return true;
}

} // unnamed namespace

bool ChopIntoWords(const std::string &text, std::unordered_set<std::string> * const words,
                   const unsigned min_word_length)
{
    return ChopIntoWords<std::unordered_set<std::string>> (text, words, min_word_length);
}

bool ChopIntoWords(const std::string &text, std::vector<std::string> * const words,
                   const unsigned min_word_length)
{
    return ChopIntoWords<std::vector<std::string>> (text, words, min_word_length);
}

std::vector<std::string>::const_iterator FindSubstring(const std::vector<std::string> &haystack,
                                                       const std::vector<std::string> &needle)
{
    if (needle.empty())
        return haystack.cbegin();

    std::vector<std::string>::const_iterator search_start(haystack.cbegin());
    while (search_start != haystack.cend()) {
        const std::vector<std::string>::const_iterator haystack_start(
            std::find(search_start, haystack.cend(), needle[0]));
        if ((haystack.cend() - haystack_start) < static_cast<ssize_t>(needle.size()))
            return haystack.cend();

        std::vector<std::string>::const_iterator needle_word(needle.cbegin());
        std::vector<std::string>::const_iterator haystack_word(haystack_start);
        for (;;) {
            ++needle_word;
            if (needle_word == needle.cend())
                return haystack_start;
            ++haystack_word;
            if (haystack_word == haystack.cend())
                return haystack.cend();
            else if (*haystack_word != *needle_word) {
                search_start = haystack_start + 1;
                break;
            }
        }
    }

    return haystack.cend();
}

std::string Base64Encode(const std::string &s, const char symbol63, const char symbol64) {
    static char symbols[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\0\0";
    symbols[62] = symbol63;
    symbols[63] = symbol64;

    std::string encoded_chars;
    std::string::const_iterator ch(s.begin());
    while (ch != s.end()) {
        // Collect groups of 3 characters:
        unsigned buf(static_cast<unsigned char>(*ch));
        buf <<= 8u;
        ++ch;
        unsigned ignore_count(0);
        if (ch != s.end()) {
            buf |= static_cast<unsigned char>(*ch);
            ++ch;
        } else
            ++ignore_count;
        buf <<= 8u;
        if (ch != s.end()) {
            buf |= static_cast<unsigned char>(*ch);
            ++ch;
        }
        else
            ++ignore_count;

        // Now grab 6 bits at a time and encode them starting with the 4th character:
        char next4[4];
        for (unsigned char_no(0); char_no < 4; ++char_no) {
            next4[4 - 1 - char_no] = symbols[buf & 0x3Fu];
            buf >>= 6u;
        }

        for (unsigned char_no(0); char_no < 4 - ignore_count; ++char_no)
            encoded_chars += next4[char_no];
    }

    return encoded_chars;
}

inline bool IsWhiteSpace(const char ch) {
    return ch == ' ' or ch == '\t' or ch == '\n' or ch == '\v' or ch == '\xA0';
}

inline std::string OctalEscape(const char ch) {
    return "\\" + StringUtil::ToString(static_cast<unsigned>(ch), 8, 3);
}

std::string EscapeString(const std::string &original_string, const bool also_escape_whitespace) {
    std::string escaped_string;
    escaped_string.reserve(original_string.size() * 2);

    for (char ch : original_string) {
        if (std::iscntrl(ch) or (not also_escape_whitespace or IsWhiteSpace(ch)))
            escaped_string += OctalEscape(ch);
        else
            escaped_string += ch;
    }

    return escaped_string;
}

} // namespace TextUtil
<|endoftext|>