<commit_before>//===- llvm/unittest/DebugInfo/PDB/HashTableTest.cpp ----------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "llvm/DebugInfo/PDB/Native/HashTable.h"
#include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h"
#include "llvm/Support/BinaryByteStream.h"
#include "llvm/Support/BinaryStreamReader.h"
#include "llvm/Support/BinaryStreamWriter.h"
#include "llvm/Testing/Support/Error.h"

#include "gtest/gtest.h"

#include <vector>

using namespace llvm;
using namespace llvm::pdb;
using namespace llvm::support;

namespace {
class HashTableInternals : public HashTable {
public:
  using HashTable::Buckets;
  using HashTable::Present;
  using HashTable::Deleted;
};
}

TEST(HashTableTest, TestSimple) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  Table.set(3, 7);
  EXPECT_EQ(1u, Table.size());
  ASSERT_NE(Table.end(), Table.find(3));
  EXPECT_EQ(7u, Table.get(3));
}

TEST(HashTableTest, TestCollision) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  // We use knowledge of the hash table's implementation details to make sure
  // to add another value that is the equivalent to the first value modulo the
  // hash table's capacity.
  uint32_t N1 = Table.capacity() + 1;
  uint32_t N2 = 2 * N1;

  Table.set(N1, 7);
  Table.set(N2, 12);
  EXPECT_EQ(2u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N2));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(12u, Table.get(N2));
}

TEST(HashTableTest, TestRemove) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  Table.set(1, 2);
  Table.set(3, 4);
  EXPECT_EQ(2u, Table.size());
  ASSERT_NE(Table.end(), Table.find(1));
  ASSERT_NE(Table.end(), Table.find(3));

  EXPECT_EQ(2u, Table.get(1));
  EXPECT_EQ(4u, Table.get(3));

  Table.remove(1u);
  EXPECT_EQ(1u, Table.size());
  EXPECT_EQ(Table.end(), Table.find(1));
  ASSERT_NE(Table.end(), Table.find(3));
  EXPECT_EQ(4u, Table.get(3));
}

TEST(HashTableTest, TestCollisionAfterMultipleProbes) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  // Probing looks for the first available slot.  A slot may already be filled
  // as a result of an item with a *different* hash value already being there.
  // Test that when this happens, the probe still finds the value.
  uint32_t N1 = Table.capacity() + 1;
  uint32_t N2 = N1 + 1;
  uint32_t N3 = 2 * N1;

  Table.set(N1, 7);
  Table.set(N2, 11);
  Table.set(N3, 13);
  EXPECT_EQ(3u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N2));
  ASSERT_NE(Table.end(), Table.find(N3));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(11u, Table.get(N2));
  EXPECT_EQ(13u, Table.get(N3));

  // Remove the one that had been filled in the middle, then insert another one
  // with a collision.  It should fill the newly emptied slot.
  Table.remove(N2);
  uint32_t N4 = N1 * 3;
  Table.set(N4, 17);
  EXPECT_EQ(3u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N3));
  ASSERT_NE(Table.end(), Table.find(N4));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(13u, Table.get(N3));
  EXPECT_EQ(17u, Table.get(N4));
}

TEST(HashTableTest, Grow) {
  // So that we are independent of the load factor, `capacity` items, which is
  // guaranteed to trigger a grow.  Then verify that the size is the same, the
  // capacity is larger, and all the original items are still in the table.

  HashTable Table;
  uint32_t OldCapacity = Table.capacity();
  for (uint32_t I = 0; I < OldCapacity; ++I) {
    Table.set(OldCapacity + I * 2 + 1, I * 2 + 3);
  }
  EXPECT_EQ(OldCapacity, Table.size());
  EXPECT_GT(Table.capacity(), OldCapacity);
  for (uint32_t I = 0; I < OldCapacity; ++I) {
    ASSERT_NE(Table.end(), Table.find(OldCapacity + I * 2 + 1));
    EXPECT_EQ(I * 2 + 3, Table.get(OldCapacity + I * 2 + 1));
  }
}

TEST(HashTableTest, Serialization) {
  HashTableInternals Table;
  uint32_t Cap = Table.capacity();
  for (uint32_t I = 0; I < Cap; ++I) {
    Table.set(Cap + I * 2 + 1, I * 2 + 3);
  }

  std::vector<uint8_t> Buffer(Table.calculateSerializedLength());
  MutableBinaryByteStream Stream(Buffer, little);
  BinaryStreamWriter Writer(Stream);
  EXPECT_THAT_ERROR(Table.commit(Writer), Succeeded());
  // We should have written precisely the number of bytes we calculated earlier.
  EXPECT_EQ(Buffer.size(), Writer.getOffset());

  HashTableInternals Table2;
  BinaryStreamReader Reader(Stream);
  EXPECT_THAT_ERROR(Table2.load(Reader), Succeeded());
  // We should have read precisely the number of bytes we calculated earlier.
  EXPECT_EQ(Buffer.size(), Reader.getOffset());

  EXPECT_EQ(Table.size(), Table2.size());
  EXPECT_EQ(Table.capacity(), Table2.capacity());
  EXPECT_EQ(Table.Buckets, Table2.Buckets);
  EXPECT_EQ(Table.Present, Table2.Present);
  EXPECT_EQ(Table.Deleted, Table2.Deleted);
}

TEST(HashTableTest, NamedStreamMap) {
  std::vector<StringRef> Streams = {"One",  "Two", "Three", "Four",
                                    "Five", "Six", "Seven"};
  StringMap<uint32_t> ExpectedIndices;
  for (uint32_t I = 0; I < Streams.size(); ++I)
    ExpectedIndices[Streams[I]] = I + 1;

  // To verify the hash table actually works, we want to verify that insertion
  // order doesn't matter.  So try inserting in every possible order of 7 items.
  do {
    NamedStreamMap NSM;
    for (StringRef S : Streams)
      NSM.set(S, ExpectedIndices[S]);

    EXPECT_EQ(Streams.size(), NSM.size());

    uint32_t N;
    EXPECT_TRUE(NSM.get("One", N));
    EXPECT_EQ(1, N);

    EXPECT_TRUE(NSM.get("Two", N));
    EXPECT_EQ(2, N);

    EXPECT_TRUE(NSM.get("Three", N));
    EXPECT_EQ(3, N);

    EXPECT_TRUE(NSM.get("Four", N));
    EXPECT_EQ(4, N);

    EXPECT_TRUE(NSM.get("Five", N));
    EXPECT_EQ(5, N);

    EXPECT_TRUE(NSM.get("Six", N));
    EXPECT_EQ(6, N);

    EXPECT_TRUE(NSM.get("Seven", N));
    EXPECT_EQ(7, N);
  } while (std::next_permutation(Streams.begin(), Streams.end()));
}
<commit_msg>Silence an unsigned vs signed compare warning.<commit_after>//===- llvm/unittest/DebugInfo/PDB/HashTableTest.cpp ----------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "llvm/DebugInfo/PDB/Native/HashTable.h"
#include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h"
#include "llvm/Support/BinaryByteStream.h"
#include "llvm/Support/BinaryStreamReader.h"
#include "llvm/Support/BinaryStreamWriter.h"
#include "llvm/Testing/Support/Error.h"

#include "gtest/gtest.h"

#include <vector>

using namespace llvm;
using namespace llvm::pdb;
using namespace llvm::support;

namespace {
class HashTableInternals : public HashTable {
public:
  using HashTable::Buckets;
  using HashTable::Present;
  using HashTable::Deleted;
};
}

TEST(HashTableTest, TestSimple) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  Table.set(3, 7);
  EXPECT_EQ(1u, Table.size());
  ASSERT_NE(Table.end(), Table.find(3));
  EXPECT_EQ(7u, Table.get(3));
}

TEST(HashTableTest, TestCollision) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  // We use knowledge of the hash table's implementation details to make sure
  // to add another value that is the equivalent to the first value modulo the
  // hash table's capacity.
  uint32_t N1 = Table.capacity() + 1;
  uint32_t N2 = 2 * N1;

  Table.set(N1, 7);
  Table.set(N2, 12);
  EXPECT_EQ(2u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N2));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(12u, Table.get(N2));
}

TEST(HashTableTest, TestRemove) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  Table.set(1, 2);
  Table.set(3, 4);
  EXPECT_EQ(2u, Table.size());
  ASSERT_NE(Table.end(), Table.find(1));
  ASSERT_NE(Table.end(), Table.find(3));

  EXPECT_EQ(2u, Table.get(1));
  EXPECT_EQ(4u, Table.get(3));

  Table.remove(1u);
  EXPECT_EQ(1u, Table.size());
  EXPECT_EQ(Table.end(), Table.find(1));
  ASSERT_NE(Table.end(), Table.find(3));
  EXPECT_EQ(4u, Table.get(3));
}

TEST(HashTableTest, TestCollisionAfterMultipleProbes) {
  HashTable Table;
  EXPECT_EQ(0u, Table.size());
  EXPECT_GT(Table.capacity(), 0u);

  // Probing looks for the first available slot.  A slot may already be filled
  // as a result of an item with a *different* hash value already being there.
  // Test that when this happens, the probe still finds the value.
  uint32_t N1 = Table.capacity() + 1;
  uint32_t N2 = N1 + 1;
  uint32_t N3 = 2 * N1;

  Table.set(N1, 7);
  Table.set(N2, 11);
  Table.set(N3, 13);
  EXPECT_EQ(3u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N2));
  ASSERT_NE(Table.end(), Table.find(N3));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(11u, Table.get(N2));
  EXPECT_EQ(13u, Table.get(N3));

  // Remove the one that had been filled in the middle, then insert another one
  // with a collision.  It should fill the newly emptied slot.
  Table.remove(N2);
  uint32_t N4 = N1 * 3;
  Table.set(N4, 17);
  EXPECT_EQ(3u, Table.size());
  ASSERT_NE(Table.end(), Table.find(N1));
  ASSERT_NE(Table.end(), Table.find(N3));
  ASSERT_NE(Table.end(), Table.find(N4));

  EXPECT_EQ(7u, Table.get(N1));
  EXPECT_EQ(13u, Table.get(N3));
  EXPECT_EQ(17u, Table.get(N4));
}

TEST(HashTableTest, Grow) {
  // So that we are independent of the load factor, `capacity` items, which is
  // guaranteed to trigger a grow.  Then verify that the size is the same, the
  // capacity is larger, and all the original items are still in the table.

  HashTable Table;
  uint32_t OldCapacity = Table.capacity();
  for (uint32_t I = 0; I < OldCapacity; ++I) {
    Table.set(OldCapacity + I * 2 + 1, I * 2 + 3);
  }
  EXPECT_EQ(OldCapacity, Table.size());
  EXPECT_GT(Table.capacity(), OldCapacity);
  for (uint32_t I = 0; I < OldCapacity; ++I) {
    ASSERT_NE(Table.end(), Table.find(OldCapacity + I * 2 + 1));
    EXPECT_EQ(I * 2 + 3, Table.get(OldCapacity + I * 2 + 1));
  }
}

TEST(HashTableTest, Serialization) {
  HashTableInternals Table;
  uint32_t Cap = Table.capacity();
  for (uint32_t I = 0; I < Cap; ++I) {
    Table.set(Cap + I * 2 + 1, I * 2 + 3);
  }

  std::vector<uint8_t> Buffer(Table.calculateSerializedLength());
  MutableBinaryByteStream Stream(Buffer, little);
  BinaryStreamWriter Writer(Stream);
  EXPECT_THAT_ERROR(Table.commit(Writer), Succeeded());
  // We should have written precisely the number of bytes we calculated earlier.
  EXPECT_EQ(Buffer.size(), Writer.getOffset());

  HashTableInternals Table2;
  BinaryStreamReader Reader(Stream);
  EXPECT_THAT_ERROR(Table2.load(Reader), Succeeded());
  // We should have read precisely the number of bytes we calculated earlier.
  EXPECT_EQ(Buffer.size(), Reader.getOffset());

  EXPECT_EQ(Table.size(), Table2.size());
  EXPECT_EQ(Table.capacity(), Table2.capacity());
  EXPECT_EQ(Table.Buckets, Table2.Buckets);
  EXPECT_EQ(Table.Present, Table2.Present);
  EXPECT_EQ(Table.Deleted, Table2.Deleted);
}

TEST(HashTableTest, NamedStreamMap) {
  std::vector<StringRef> Streams = {"One",  "Two", "Three", "Four",
                                    "Five", "Six", "Seven"};
  StringMap<uint32_t> ExpectedIndices;
  for (uint32_t I = 0; I < Streams.size(); ++I)
    ExpectedIndices[Streams[I]] = I + 1;

  // To verify the hash table actually works, we want to verify that insertion
  // order doesn't matter.  So try inserting in every possible order of 7 items.
  do {
    NamedStreamMap NSM;
    for (StringRef S : Streams)
      NSM.set(S, ExpectedIndices[S]);

    EXPECT_EQ(Streams.size(), NSM.size());

    uint32_t N;
    EXPECT_TRUE(NSM.get("One", N));
    EXPECT_EQ(1U, N);

    EXPECT_TRUE(NSM.get("Two", N));
    EXPECT_EQ(2U, N);

    EXPECT_TRUE(NSM.get("Three", N));
    EXPECT_EQ(3U, N);

    EXPECT_TRUE(NSM.get("Four", N));
    EXPECT_EQ(4U, N);

    EXPECT_TRUE(NSM.get("Five", N));
    EXPECT_EQ(5U, N);

    EXPECT_TRUE(NSM.get("Six", N));
    EXPECT_EQ(6U, N);

    EXPECT_TRUE(NSM.get("Seven", N));
    EXPECT_EQ(7U, N);
  } while (std::next_permutation(Streams.begin(), Streams.end()));
}
<|endoftext|>


<commit_before>// {{{ MIT License

// Copyright 2017 Roland Kaminski

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// }}}

#ifndef _GRINGO_REPORT_HH
#define _GRINGO_REPORT_HH

#include <bitset>
#include <cstdio>
#include <sstream>
#include <stdexcept>
#include <memory>
#include <functional>

#include <iostream>

namespace Gringo {

// {{{1 declaration of GringoError

class GringoError : public std::runtime_error {
public:
    GringoError(char const *msg) : std::runtime_error(msg) { }
};

class MessageLimitError : public std::runtime_error {
public:
    MessageLimitError(char const *msg) : std::runtime_error(msg) { }
};

// {{{1 declaration of Logger

enum class Errors : int {
    Success   = 0,
    Runtime   = 1,
    Logic     = 2,
    Bad_alloc = 3,
    Unknown   = 4
};

enum class Warnings : int {
    OperationUndefined = 0,
    RuntimeError       = 1,
    AtomUndefined      = 2,
    FileIncluded       = 3,
    VariableUnbounded  = 4,
    GlobalVariable     = 5,
    Other              = 6,
};

class Logger {
public:
    using Printer = std::function<void (Warnings, char const *)>;
    Logger(Printer p = nullptr, unsigned limit = 20)
    : p_(p)
    , limit_(limit) { }
    bool check(Errors id);
    bool check(Warnings id);
    bool hasError() const;
    void enable(Warnings id, bool enable);
    void print(Warnings code, char const *msg);
    ~Logger();
private:
    Printer p_;
    unsigned limit_;
    std::bitset<static_cast<int>(Warnings::Other)+1> disabled_;
    bool error_ = false;
};

// }}}1

// {{{1 definition of Logger

inline bool Logger::check(Warnings id) {
    if (id == Warnings::RuntimeError) {
        if (!limit_ && error_) { throw MessageLimitError("too many messages."); }
        if (limit_) { --limit_; }
        error_ = true;
        return true;
    }
    else {
        if (!limit_ && error_) { throw MessageLimitError("too many messages."); }
        return !disabled_[static_cast<int>(id)] && limit_ && (--limit_, true);
    }
}

inline bool Logger::hasError() const {
    return error_;
}

inline void Logger::enable(Warnings id, bool enabled) {
    disabled_[static_cast<int>(id)] = !enabled;
}

inline void Logger::print(Warnings code, char const *msg) {
    if (p_) { p_(code, msg); }
    else {
        fprintf(stderr, "%s\n", msg);
        fflush(stderr);
    }
}

inline Logger::~Logger() { }

// {{{1 definition of Report

class Report {
public:
    Report(Logger &p, Warnings code) : p_(p), code_(code) { }
    ~Report() { p_.print(code_, out.str().c_str()); }
    std::ostringstream out;
private:
    Logger &p_;
    Warnings code_;
};

// }}}1

} // namespace GRINGO

#define GRINGO_REPORT(p, id) \
if (!(p).check(id)) { } \
else Gringo::Report(p, id).out

#endif // _GRINGO_REPORT_HH

<commit_msg>refactor logger<commit_after>// {{{ MIT License

// Copyright 2017 Roland Kaminski

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// }}}

#ifndef GRINGO_LOGGER_HH
#define GRINGO_LOGGER_HH

#include <bitset>
#include <cstdio>
#include <sstream>
#include <stdexcept>
#include <memory>
#include <functional>

#include <iostream>

namespace Gringo {

// {{{1 declaration of GringoError

class GringoError : public std::runtime_error {
public:
    GringoError(char const *msg)
    : std::runtime_error(msg) { }
};

class MessageLimitError : public std::runtime_error {
public:
    MessageLimitError(char const *msg)
    : std::runtime_error(msg) { }
};

// {{{1 declaration of Logger

enum class Errors : int {
    Success   = 0,
    Runtime   = 1,
    Logic     = 2,
    Bad_alloc = 3,
    Unknown   = 4
};

enum class Warnings : int {
    OperationUndefined = 0,
    RuntimeError       = 1,
    AtomUndefined      = 2,
    FileIncluded       = 3,
    VariableUnbounded  = 4,
    GlobalVariable     = 5,
    Other              = 6,
};

class Logger {
public:
    using Printer = std::function<void (Warnings, char const *)>;
    Logger(Printer p = nullptr, unsigned limit = 20)
    : p_(std::move(p))
    , limit_(limit) { }

    bool check(Errors id);
    bool check(Warnings id);
    bool hasError() const;
    void enable(Warnings id, bool enable);
    void print(Warnings code, char const *msg);

private:
    Printer p_;
    unsigned limit_;
    std::bitset<static_cast<int>(Warnings::Other)+1> disabled_;
    bool error_ = false;
};

// }}}1

// {{{1 definition of Logger

inline bool Logger::check(Warnings id) {
    if (id == Warnings::RuntimeError) {
        if (limit_ == 0 && error_) {
            throw MessageLimitError("too many messages.");
        }
        if (limit_ > 0) {
            --limit_;
        }
        error_ = true;
        return true;
    }
    if (limit_  == 0 && error_) {
        throw MessageLimitError("too many messages.");
    }
    return !disabled_[static_cast<int>(id)] && limit_ > 0 && (--limit_, true);
}

inline bool Logger::hasError() const {
    return error_;
}

inline void Logger::enable(Warnings id, bool enabled) {
    disabled_[static_cast<int>(id)] = !enabled;
}

inline void Logger::print(Warnings code, char const *msg) {
    if (p_) {
        p_(code, msg);
    }
    else {
        // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
        fprintf(stderr, "%s\n", msg);
        fflush(stderr);
    }
}

// {{{1 definition of Report

class Report {
public:
    Report(Logger &p, Warnings code)
    : p_(p), code_(code) { }
    Report(Report const &other) = delete;
    Report(Report &&other) = delete;
    Report &operator=(Report const &other) = delete;
    Report &operator=(Report &&other) = delete;
    ~Report() {
        p_.print(code_, out_.str().c_str());
    }
    std::ostringstream &out() {
        return out_;
    }

private:
    std::ostringstream out_;
    Logger &p_;
    Warnings code_;
};

// }}}1

} // namespace GRINGO

// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define GRINGO_REPORT(p, id) \
if (!(p).check(id)) { } \
else Gringo::Report(p, id).out()

#endif // GRINGO_LOGGER_HH
<|endoftext|>


<commit_before>#include "spriteStyle.h"

#include "platform.h"
#include "tile/tile.h"
#include "gl/shaderProgram.h"
#include "gl/texture.h"
#include "gl/vertexLayout.h"
#include "util/builders.h"
#include "view/view.h"
#include "labels/spriteLabel.h"

#include "glm/gtc/type_ptr.hpp"

namespace Tangram {

SpriteStyle::SpriteStyle(std::string _name, GLenum _drawMode) : Style(_name, _drawMode) {
}

SpriteStyle::~SpriteStyle() {}

void SpriteStyle::constructVertexLayout() {

    // 32 bytes, good for memory aligments
    m_vertexLayout = std::shared_ptr<VertexLayout>(new VertexLayout({
        {"a_position", 2, GL_FLOAT, false, 0},
        {"a_uv", 2, GL_FLOAT, false, 0},
        {"a_color", 4, GL_UNSIGNED_BYTE, true, 0},
        {"a_screenPosition", 2, GL_FLOAT, false, 0},
        {"a_alpha", 1, GL_FLOAT, false, 0},
        {"a_rotation", 1, GL_FLOAT, false, 0},
    }));
}

void SpriteStyle::constructShaderProgram() {

    std::string fragShaderSrcStr = stringFromResource("sprite.fs");
    std::string vertShaderSrcStr = stringFromResource("point.vs");

    m_shaderProgram->setSourceStrings(fragShaderSrcStr, vertShaderSrcStr);

    // TODO : load this from stylesheet
    m_spriteAtlas = std::unique_ptr<SpriteAtlas>(new SpriteAtlas("poi_icons_32.png"));

    m_spriteAtlas->addSpriteNode("plane", {0, 0}, {32, 32});
    m_spriteAtlas->addSpriteNode("tree", {0, 185}, {32, 32});
    m_spriteAtlas->addSpriteNode("sunburst", {0, 629}, {32, 32});
    m_spriteAtlas->addSpriteNode("restaurant", {0, 777}, {32, 32});
    m_spriteAtlas->addSpriteNode("cafe", {0, 814}, {32, 32});
    m_spriteAtlas->addSpriteNode("museum", {0, 518}, {32, 32});
    m_spriteAtlas->addSpriteNode("bar", {0, 887}, {32, 32});
    m_spriteAtlas->addSpriteNode("train", {0, 74}, {32, 32});
    m_spriteAtlas->addSpriteNode("bus", {0, 148}, {32, 32});
    m_spriteAtlas->addSpriteNode("hospital", {0, 444}, {32, 32});
    m_spriteAtlas->addSpriteNode("parking", {0, 1073}, {32, 32});
    m_spriteAtlas->addSpriteNode("info", {0, 1110}, {32, 32});
    m_spriteAtlas->addSpriteNode("hotel", {0, 259}, {32, 32});
    m_spriteAtlas->addSpriteNode("bookstore", {0, 333}, {32, 32});
}

void SpriteStyle::buildPoint(const Point& _point, const DrawRule& _rule, const Properties& _props, VboMesh& _mesh, Tile& _tile) const {
    // TODO : make this configurable
    std::vector<Label::Vertex> vertices;

    // TODO : configure this
    float spriteScale = .5f;
    glm::vec2 offset = {0.f, 10.f};

    const static std::string key("kind");

    const std::string& kind = _props.getString(key);
    if(kind.length() == 0) { return; }

    if (!m_spriteAtlas->hasSpriteNode(kind)) {
        return;
    }

    SpriteNode spriteNode = m_spriteAtlas->getSpriteNode(kind);
    Label::Transform t = { glm::vec2(_point), glm::vec2(_point) };

    auto& mesh = static_cast<LabelMesh&>(_mesh);

    Label::Options options;
    options.color = 0xff000000;
    options.offset = offset;

    options.color |= (rand() % 255) << 0;
    options.color |= (rand() % 255) << 8;
    options.color |= (rand() % 255) << 16;

    std::unique_ptr<SpriteLabel> label(new SpriteLabel(t, spriteNode.m_size * spriteScale, mesh, _mesh.numVertices(), options));

    auto size = spriteNode.m_size * spriteScale;
    float halfWidth = size.x * .5f;
    float halfHeight = size.y * .5f;
    const glm::vec2& uvBL = spriteNode.m_uvBL;
    const glm::vec2& uvTR = spriteNode.m_uvTR;

    vertices.reserve(4);
    vertices.push_back({{-halfWidth, -halfHeight}, {uvBL.x, uvBL.y}, options.color});
    vertices.push_back({{-halfWidth, halfHeight}, {uvBL.x, uvTR.y}, options.color});
    vertices.push_back({{halfWidth, -halfHeight}, {uvTR.x, uvBL.y}, options.color});
    vertices.push_back({{halfWidth, halfHeight}, {uvTR.x, uvTR.y}, options.color});

    mesh.addVertices(std::move(vertices), {});
    mesh.addLabel(std::move(label));
}

void SpriteStyle::onBeginDrawFrame(const View& _view, const Scene& _scene) {
    bool contextLost = Style::glContextLost();

    m_spriteAtlas->bind();

    static bool initUniformSampler = true;

    if (initUniformSampler || contextLost) {
        m_shaderProgram->setUniformi("u_tex", 0);
        initUniformSampler = false;
    }

    if (m_dirtyViewport || contextLost) {
        m_shaderProgram->setUniformMatrix4f("u_proj", glm::value_ptr(_view.getOrthoViewportMatrix()));
        m_dirtyViewport = false;
    }

    RenderState::blending(GL_TRUE);
    RenderState::blendingFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    RenderState::depthTest(GL_FALSE);
}

}
<commit_msg>Remove random sprite color<commit_after>#include "spriteStyle.h"

#include "platform.h"
#include "tile/tile.h"
#include "gl/shaderProgram.h"
#include "gl/texture.h"
#include "gl/vertexLayout.h"
#include "util/builders.h"
#include "view/view.h"
#include "labels/spriteLabel.h"

#include "glm/gtc/type_ptr.hpp"

namespace Tangram {

SpriteStyle::SpriteStyle(std::string _name, GLenum _drawMode) : Style(_name, _drawMode) {
}

SpriteStyle::~SpriteStyle() {}

void SpriteStyle::constructVertexLayout() {

    // 32 bytes, good for memory aligments
    m_vertexLayout = std::shared_ptr<VertexLayout>(new VertexLayout({
        {"a_position", 2, GL_FLOAT, false, 0},
        {"a_uv", 2, GL_FLOAT, false, 0},
        {"a_color", 4, GL_UNSIGNED_BYTE, true, 0},
        {"a_screenPosition", 2, GL_FLOAT, false, 0},
        {"a_alpha", 1, GL_FLOAT, false, 0},
        {"a_rotation", 1, GL_FLOAT, false, 0},
    }));
}

void SpriteStyle::constructShaderProgram() {

    std::string fragShaderSrcStr = stringFromResource("sprite.fs");
    std::string vertShaderSrcStr = stringFromResource("point.vs");

    m_shaderProgram->setSourceStrings(fragShaderSrcStr, vertShaderSrcStr);

    // TODO : load this from stylesheet
    m_spriteAtlas = std::unique_ptr<SpriteAtlas>(new SpriteAtlas("poi_icons_32.png"));

    m_spriteAtlas->addSpriteNode("plane", {0, 0}, {32, 32});
    m_spriteAtlas->addSpriteNode("tree", {0, 185}, {32, 32});
    m_spriteAtlas->addSpriteNode("sunburst", {0, 629}, {32, 32});
    m_spriteAtlas->addSpriteNode("restaurant", {0, 777}, {32, 32});
    m_spriteAtlas->addSpriteNode("cafe", {0, 814}, {32, 32});
    m_spriteAtlas->addSpriteNode("museum", {0, 518}, {32, 32});
    m_spriteAtlas->addSpriteNode("bar", {0, 887}, {32, 32});
    m_spriteAtlas->addSpriteNode("train", {0, 74}, {32, 32});
    m_spriteAtlas->addSpriteNode("bus", {0, 148}, {32, 32});
    m_spriteAtlas->addSpriteNode("hospital", {0, 444}, {32, 32});
    m_spriteAtlas->addSpriteNode("parking", {0, 1073}, {32, 32});
    m_spriteAtlas->addSpriteNode("info", {0, 1110}, {32, 32});
    m_spriteAtlas->addSpriteNode("hotel", {0, 259}, {32, 32});
    m_spriteAtlas->addSpriteNode("bookstore", {0, 333}, {32, 32});
}

void SpriteStyle::buildPoint(const Point& _point, const DrawRule& _rule, const Properties& _props, VboMesh& _mesh, Tile& _tile) const {
    // TODO : make this configurable
    std::vector<Label::Vertex> vertices;

    // TODO : configure this
    float spriteScale = .5f;
    glm::vec2 offset = {0.f, 10.f};

    const static std::string key("kind");

    const std::string& kind = _props.getString(key);
    if(kind.length() == 0) { return; }

    if (!m_spriteAtlas->hasSpriteNode(kind)) {
        return;
    }

    SpriteNode spriteNode = m_spriteAtlas->getSpriteNode(kind);
    Label::Transform t = { glm::vec2(_point), glm::vec2(_point) };

    auto& mesh = static_cast<LabelMesh&>(_mesh);

    Label::Options options;
    options.offset = offset;

    std::unique_ptr<SpriteLabel> label(new SpriteLabel(t, spriteNode.m_size * spriteScale, mesh, _mesh.numVertices(), options));

    auto size = spriteNode.m_size * spriteScale;
    float halfWidth = size.x * .5f;
    float halfHeight = size.y * .5f;
    const glm::vec2& uvBL = spriteNode.m_uvBL;
    const glm::vec2& uvTR = spriteNode.m_uvTR;

    vertices.reserve(4);
    vertices.push_back({{-halfWidth, -halfHeight}, {uvBL.x, uvBL.y}, options.color});
    vertices.push_back({{-halfWidth, halfHeight}, {uvBL.x, uvTR.y}, options.color});
    vertices.push_back({{halfWidth, -halfHeight}, {uvTR.x, uvBL.y}, options.color});
    vertices.push_back({{halfWidth, halfHeight}, {uvTR.x, uvTR.y}, options.color});

    mesh.addVertices(std::move(vertices), {});
    mesh.addLabel(std::move(label));
}

void SpriteStyle::onBeginDrawFrame(const View& _view, const Scene& _scene) {
    bool contextLost = Style::glContextLost();

    m_spriteAtlas->bind();

    static bool initUniformSampler = true;

    if (initUniformSampler || contextLost) {
        m_shaderProgram->setUniformi("u_tex", 0);
        initUniformSampler = false;
    }

    if (m_dirtyViewport || contextLost) {
        m_shaderProgram->setUniformMatrix4f("u_proj", glm::value_ptr(_view.getOrthoViewportMatrix()));
        m_dirtyViewport = false;
    }

    RenderState::blending(GL_TRUE);
    RenderState::blendingFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    RenderState::depthTest(GL_FALSE);
}

}
<|endoftext|>


<commit_before>#include "util/inputHandler.h"

#include "glm/gtx/rotate_vector.hpp"
#include "glm/vec2.hpp"
#include <cmath>

// Damping factor for translation; reciprocal of the decay period in seconds
#define DAMPING_PAN 4.0f

// Damping factor for zoom; reciprocal of the decay period in seconds
#define DAMPING_ZOOM 6.0f

// Minimum translation at which momentum should start (pixels per second)
#define THRESHOLD_START_PAN 350.f

// Minimum translation at which momentum should stop (pixels per second)
#define THRESHOLD_STOP_PAN 24.f

// Minimum zoom at which momentum should start (zoom levels per second)
#define THRESHOLD_START_ZOOM 1.f

// Minimum zoom at which momentum should stop (zoom levels per second)
#define THRESHOLD_STOP_ZOOM 0.3f

namespace Tangram {

InputHandler::InputHandler(View& _view) : m_view(_view) {}

bool InputHandler::update(float _dt) {

    auto velocityPanPixels = m_view.pixelsPerMeter() / m_view.pixelScale() * m_velocityPan;

    bool isFlinging = glm::length(velocityPanPixels) > THRESHOLD_STOP_PAN ||
                      std::abs(m_velocityZoom) > THRESHOLD_STOP_ZOOM;

    if (isFlinging) {

        m_velocityPan -= _dt * DAMPING_PAN * m_velocityPan;
        m_view.translate(_dt * m_velocityPan.x, _dt * m_velocityPan.y);

        m_velocityZoom -= _dt * DAMPING_ZOOM * m_velocityZoom;
        m_view.zoom(m_velocityZoom * _dt);
    }

    return isFlinging;
}

void InputHandler::handleTapGesture(float _posX, float _posY) {
    cancelFling();

    float viewCenterX = 0.5f * m_view.getWidth();
    float viewCenterY = 0.5f * m_view.getHeight();

    m_view.screenToGroundPlane(viewCenterX, viewCenterY);
    m_view.screenToGroundPlane(_posX, _posY);

    m_view.translate((_posX - viewCenterX), (_posY - viewCenterY));
}

void InputHandler::handleDoubleTapGesture(float _posX, float _posY) {
    handlePinchGesture(_posX, _posY, 2.f, 0.f);
}

void InputHandler::handlePanGesture(float _startX, float _startY, float _endX, float _endY) {
    cancelFling();

    m_view.screenToGroundPlane(_startX, _startY);
    m_view.screenToGroundPlane(_endX, _endY);

    float dx = _startX - _endX;
    float dy = _startY - _endY;

    m_view.translate(dx, dy);
}

void InputHandler::handleFlingGesture(float _posX, float _posY, float _velocityX, float _velocityY) {

    if (glm::length(glm::vec2(_velocityX, _velocityY)) / m_view.pixelScale() <= THRESHOLD_START_PAN) {
        return;
    }

    const static float epsilon = 0.0167f;

    cancelFling();

    float startX = _posX;
    float startY = _posY;
    float endX = _posX + epsilon * _velocityX;
    float endY = _posY + epsilon * _velocityY;

    m_view.screenToGroundPlane(startX, startY);
    m_view.screenToGroundPlane(endX, endY);

    float dx = (startX - endX) / epsilon;
    float dy = (startY - endY) / epsilon;

    setVelocity(0.f, glm::vec2(dx, dy));
}

void InputHandler::handlePinchGesture(float _posX, float _posY, float _scale, float _velocity) {
    cancelFling();

    float z = m_view.getZoom();
    static float invLog2 = 1 / log(2);
    m_view.zoom(log(_scale) * invLog2);

    m_view.screenToGroundPlane(_posX, _posY);
    float s = pow(2, m_view.getZoom() - z) - 1;
    m_view.translate(s * _posX, s * _posY);

    // Take the derivative of zoom as a function of scale:
    // z(s) = log2(s) + C
    // z'(s) = s' / s / log(2)
    float vz = _velocity / _scale * invLog2;
    if (std::abs(vz) >= THRESHOLD_START_ZOOM) {
        setVelocity(vz, glm::vec2(0.f));
    }
}

void InputHandler::handleRotateGesture(float _posX, float _posY, float _radians) {
    cancelFling();

    // Get vector from center of rotation to view center
    m_view.screenToGroundPlane(_posX, _posY);
    glm::vec2 offset(_posX, _posY);

    // Rotate vector by gesture rotation and apply difference as translation
    glm::vec2 translation = offset - glm::rotate(offset, _radians);
    m_view.translate(translation.x, translation.y);

    m_view.roll(_radians);
}

void InputHandler::handleShoveGesture(float _distance) {
    cancelFling();

    float angle = -M_PI * _distance / m_view.getHeight();
    m_view.pitch(angle);

}

void InputHandler::cancelFling() {
    setVelocity(0.f, { 0.f, 0.f});
}

void InputHandler::setVelocity(float _zoom, glm::vec2 _translate) {
    // setup deltas for momentum on gesture
    m_velocityPan = _translate;
    m_velocityZoom = _zoom;
}

}
<commit_msg>Prevent overshoot on fling damping (#2208)<commit_after>#include "util/inputHandler.h"

#include "glm/gtx/rotate_vector.hpp"
#include "glm/vec2.hpp"
#include <cmath>

// Damping factor for translation; reciprocal of the decay period in seconds
#define DAMPING_PAN 4.0f

// Damping factor for zoom; reciprocal of the decay period in seconds
#define DAMPING_ZOOM 6.0f

// Minimum translation at which momentum should start (pixels per second)
#define THRESHOLD_START_PAN 350.f

// Minimum translation at which momentum should stop (pixels per second)
#define THRESHOLD_STOP_PAN 24.f

// Minimum zoom at which momentum should start (zoom levels per second)
#define THRESHOLD_START_ZOOM 1.f

// Minimum zoom at which momentum should stop (zoom levels per second)
#define THRESHOLD_STOP_ZOOM 0.3f

namespace Tangram {

InputHandler::InputHandler(View& _view) : m_view(_view) {}

bool InputHandler::update(float _dt) {

    auto velocityPanPixels = m_view.pixelsPerMeter() / m_view.pixelScale() * m_velocityPan;

    bool isFlinging = glm::length(velocityPanPixels) > THRESHOLD_STOP_PAN ||
                      std::abs(m_velocityZoom) > THRESHOLD_STOP_ZOOM;

    if (isFlinging) {

        m_velocityPan -= min(_dt * DAMPING_PAN, 1.f) * m_velocityPan;
        m_view.translate(_dt * m_velocityPan.x, _dt * m_velocityPan.y);

        m_velocityZoom -= min(_dt * DAMPING_ZOOM, 1.f) * m_velocityZoom;
        m_view.zoom(m_velocityZoom * _dt);
    }

    return isFlinging;
}

void InputHandler::handleTapGesture(float _posX, float _posY) {
    cancelFling();

    float viewCenterX = 0.5f * m_view.getWidth();
    float viewCenterY = 0.5f * m_view.getHeight();

    m_view.screenToGroundPlane(viewCenterX, viewCenterY);
    m_view.screenToGroundPlane(_posX, _posY);

    m_view.translate((_posX - viewCenterX), (_posY - viewCenterY));
}

void InputHandler::handleDoubleTapGesture(float _posX, float _posY) {
    handlePinchGesture(_posX, _posY, 2.f, 0.f);
}

void InputHandler::handlePanGesture(float _startX, float _startY, float _endX, float _endY) {
    cancelFling();

    m_view.screenToGroundPlane(_startX, _startY);
    m_view.screenToGroundPlane(_endX, _endY);

    float dx = _startX - _endX;
    float dy = _startY - _endY;

    m_view.translate(dx, dy);
}

void InputHandler::handleFlingGesture(float _posX, float _posY, float _velocityX, float _velocityY) {

    if (glm::length(glm::vec2(_velocityX, _velocityY)) / m_view.pixelScale() <= THRESHOLD_START_PAN) {
        return;
    }

    const static float epsilon = 0.0167f;

    cancelFling();

    float startX = _posX;
    float startY = _posY;
    float endX = _posX + epsilon * _velocityX;
    float endY = _posY + epsilon * _velocityY;

    m_view.screenToGroundPlane(startX, startY);
    m_view.screenToGroundPlane(endX, endY);

    float dx = (startX - endX) / epsilon;
    float dy = (startY - endY) / epsilon;

    setVelocity(0.f, glm::vec2(dx, dy));
}

void InputHandler::handlePinchGesture(float _posX, float _posY, float _scale, float _velocity) {
    cancelFling();

    float z = m_view.getZoom();
    static float invLog2 = 1 / log(2);
    m_view.zoom(log(_scale) * invLog2);

    m_view.screenToGroundPlane(_posX, _posY);
    float s = pow(2, m_view.getZoom() - z) - 1;
    m_view.translate(s * _posX, s * _posY);

    // Take the derivative of zoom as a function of scale:
    // z(s) = log2(s) + C
    // z'(s) = s' / s / log(2)
    float vz = _velocity / _scale * invLog2;
    if (std::abs(vz) >= THRESHOLD_START_ZOOM) {
        setVelocity(vz, glm::vec2(0.f));
    }
}

void InputHandler::handleRotateGesture(float _posX, float _posY, float _radians) {
    cancelFling();

    // Get vector from center of rotation to view center
    m_view.screenToGroundPlane(_posX, _posY);
    glm::vec2 offset(_posX, _posY);

    // Rotate vector by gesture rotation and apply difference as translation
    glm::vec2 translation = offset - glm::rotate(offset, _radians);
    m_view.translate(translation.x, translation.y);

    m_view.roll(_radians);
}

void InputHandler::handleShoveGesture(float _distance) {
    cancelFling();

    float angle = -M_PI * _distance / m_view.getHeight();
    m_view.pitch(angle);

}

void InputHandler::cancelFling() {
    setVelocity(0.f, { 0.f, 0.f});
}

void InputHandler::setVelocity(float _zoom, glm::vec2 _translate) {
    // setup deltas for momentum on gesture
    m_velocityPan = _translate;
    m_velocityZoom = _zoom;
}

}
<|endoftext|>


<commit_before>/** \brief Tool to export all datasets from VuFind redirect table to CSV file.
 *  \author Mario Trojan
 *
 *  \copyright 2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <cstdlib>
#include "File.h"
#include "FileUtil.h"
#include "TextUtil.h"
#include "VuFind.h"
#include "util.h"

int Main(int argc, char **argv) {
    if (argc != 2)
        ::Usage("export_file");

    const auto db_connection(VuFind::GetDbConnection());
    db_connection->queryOrDie("SELECT * FROM tuefind_redirect");
    auto result_set(db_connection->getLastResultSet());

    File csv_file(argv[1], "w", File::ThrowOnOpenBehaviour::THROW_ON_ERROR);
    csv_file.write("url;group;timestamp\n");
    while (const auto &db_row = result_set.getNextRow()) {
        std::string csv_row;
        csv_row += TextUtil::CSVEscape(db_row["url"]) + ";";
        csv_row += TextUtil::CSVEscape(db_row["group_name"]) + ";";
        csv_row += TextUtil::CSVEscape(db_row["timestamp"]) + "\n";
        csv_file.write(csv_row);
    }
    csv_file.close();

    return EXIT_SUCCESS;
}
<commit_msg>export_vufind_redirects.cc: Optimized CSV writing<commit_after>/** \brief Tool to export all datasets from VuFind redirect table to CSV file.
 *  \author Mario Trojan
 *
 *  \copyright 2020 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <cstdlib>
#include "File.h"
#include "FileUtil.h"
#include "TextUtil.h"
#include "VuFind.h"
#include "util.h"

int Main(int argc, char **argv) {
    if (argc != 2)
        ::Usage("export_file");

    const auto db_connection(VuFind::GetDbConnection());
    db_connection->queryOrDie("SELECT * FROM tuefind_redirect");
    auto result_set(db_connection->getLastResultSet());

    File csv_file(argv[1], "w", File::ThrowOnOpenBehaviour::THROW_ON_ERROR);
    csv_file.write("url;group;timestamp\n");
    while (const auto &db_row = result_set.getNextRow()) {
        csv_file.write(TextUtil::CSVEscape(db_row["url"]) + ";");
        csv_file.write(TextUtil::CSVEscape(db_row["group_name"]) + ";");
        csv_file.write(TextUtil::CSVEscape(db_row["timestamp"]) + "\n");
    }

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/** \file   SmartDownloader.cc
 *  \brief  Implementation of descedants of the SmartDownloader class.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015,2017,2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "SmartDownloader.h"
#include <iostream>
#include "Downloader.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

bool DownloadHelper(const std::string &url, const TimeLimit &time_limit,
                    std::string * const document, std::string * const http_header_charset,
                    std::string * const error_message)
{
    Downloader downloader(url, Downloader::Params(), time_limit);
    if (downloader.anErrorOccurred()) {
        *error_message = downloader.getLastErrorMessage();
        return false;
    }

    *document = downloader.getMessageBody();
    *http_header_charset = downloader.getCharset();
    return true;
}

} // unnamed namespace

SmartDownloader::SmartDownloader(const std::string &regex, const bool trace): trace_(trace) {
    std::string err_msg;
    matcher_.reset(RegexMatcher::RegexMatcherFactory(regex, &err_msg));
    if (not matcher_)
        LOG_ERROR("pattern failed to compile \"" + regex + "\"!");
}

bool SmartDownloader::canHandleThis(const std::string &url) const {
    std::string err_msg;
    if (matcher_->matched(url, &err_msg)) {
        if (not err_msg.empty())
            LOG_ERROR("an error occurred while trying to match \"" + url + "\" with \""
                  + matcher_->getPattern() + "\"! (" + err_msg + ")");

        return true;
    }

    return false;
}

bool SmartDownloader::downloadDoc(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                                  std::string * const http_header_charset, std::string * const error_message)
{
    if (downloadDocImpl(url, time_limit, document, http_header_charset, error_message)) {
        ++success_count_;
        return true;
    } else
        return false;
}

bool DSpaceDownloader::canHandleThis(const std::string &url) const {
    return url.find("dspace") != std::string::npos;
}

bool DSpaceDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                       std::string * const document, std::string * const http_header_charset,
                                       std::string * const error_message)
{
    document->clear();

    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;

    static RegexMatcher *matcher;
    if (matcher == nullptr) {
        std::string err_msg;
        matcher = RegexMatcher::RegexMatcherFactory("meta content=\"http(.*)pdf\"", &err_msg);
        if (matcher == nullptr)
            LOG_ERROR("failed to compile regex! (" + err_msg + ")");
    }

    if (not matcher->matched(*document)) {
        *error_message = "no matching DSpace structure found!";
        return false;
    }

    const std::string pdf_link("http" + (*matcher)[1] + "pdf");
    if (not DownloadHelper(pdf_link, time_limit, document, http_header_charset, error_message))
        return false;

    return true;
}

bool SimpleSuffixDownloader::canHandleThis(const std::string &url) const {
    for (const auto &suffix : suffixes_) {
        if (StringUtil::IsProperSuffixOfIgnoreCase(suffix, url))
            return true;
    }

    return false;
}

bool SimpleSuffixDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                             std::string * const document, std::string * const http_header_charset,
                                             std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    return (DownloadHelper(url, time_limit, document, http_header_charset, error_message));
}

bool SimplePrefixDownloader::canHandleThis(const std::string &url) const {
    for (const auto &prefix : prefixes_) {
        if (StringUtil::StartsWith(url, prefix, /* ignore_case = */ true))
            return true;
    }

    return false;
}

bool SimplePrefixDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                             std::string * const document, std::string * const http_header_charset,
                                             std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    return (DownloadHelper(url, time_limit, document, http_header_charset, error_message));
}

bool DigiToolSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                              std::string * const document, std::string * const http_header_charset,
                                              std::string * const error_message)
{
    static RegexMatcher * const matcher(
        RegexMatcher::RegexMatcherFactory("http://digitool.hbz-nrw.de:1801/webclient/DeliveryManager\\?pid=\\d+"));

    std::string err_msg;
    size_t start_pos, end_pos;
    if (not matcher->matched(url, &err_msg, &start_pos, &end_pos))
        LOG_ERROR("match failed: " + err_msg);

    const std::string normalised_url(url.substr(start_pos, end_pos - start_pos));

    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(normalised_url, time_limit, document, http_header_charset, error_message))
        return false;

    static const std::string ocr_text("ocr-text:\n");
    if (MediaTypeUtil::GetMediaType(*document) == "text/plain"
        and StringUtil::StartsWith(*document, ocr_text))
        *document = StringUtil::ISO8859_15ToUTF8(document->substr(ocr_text.length()));

    return true;
}

bool DiglitSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                            std::string * const document, std::string * const http_header_charset,
                                            std::string * const error_message)
{
    std::string improved_url(url);
    if (RegexMatcher::Matched("/diglit/", url))
        improved_url = StringUtil::ReplaceString("/diglit/", "/opendigi/", &improved_url);
    if (RegexMatcher::Matched("/opendigi/", improved_url))
        improved_url += "/ocr";

    if (trace_ and url != improved_url)
        LOG_INFO("converted url \"" + url + "\" to \"" + improved_url + "\"");

    if (trace_)
        LOG_INFO("about to download \"" + improved_url + "\".");
    if (not DownloadHelper(improved_url, time_limit, document, http_header_charset, error_message)) {
        if (trace_)
            LOG_WARNING("original download failed!");
        return false;
    }

    return not document->empty();
}

bool BszSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                         std::string * const document, std::string * const http_header_charset,
                                         std::string * const error_message)
{
    const std::string doc_url(url.substr(0, url.size() - 3) + "pdf");
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool BvbrSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                          std::string * const document, std::string * const http_header_charset,
                                          std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;
    const std::string start_string("<body onload=window.location=\"");
    size_t start_pos(document->find(start_string));
    if (start_pos == std::string::npos) {
        *error_message = "no matching Bvbr structure found!";
        return false;
    }
    start_pos += start_string.size();
    const size_t end_pos(document->find('"', start_pos + 1));
    if (end_pos == std::string::npos) {
        *error_message = "no matching Bvbr structure found! (part 2)";
        return false;
    }
    const std::string doc_url("http://bvbr.bib-bvb.de:8991" + document->substr(start_pos, end_pos - start_pos));
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(doc_url, time_limit, document, http_header_charset, error_message);
}

bool Bsz21SmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                           std::string * const document, std::string * const http_header_charset,
                                           std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;

    if (MediaTypeUtil::GetMediaType(*document) == "application/pdf")
        return true;

    std::string start_string("Persistente URL: <a id=\"pers_url\" href=\"");
    size_t start_pos(document->find(start_string));
    std::string doc_url;
    if (start_pos != std::string::npos) {
        start_pos += start_string.size();
        const size_t end_pos(document->find('"', start_pos + 1));
        if (end_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 1)";
            return false;
        }
        const std::string pers_url(document->substr(start_pos, end_pos - start_pos));
        const size_t last_slash_pos(pers_url.rfind('/'));
        if (last_slash_pos == std::string::npos or last_slash_pos == pers_url.size() - 1) {
            *error_message = "no matching Bsz2l structure found! (part 2)";
            return false;
        }
        doc_url = "http://idb.ub.uni-tuebingen.de/cgi-bin/digi-downloadPdf.fcgi?projectname="
                  + pers_url.substr(last_slash_pos + 1);
    } else {
        start_pos = document->find("name=\"citation_pdf_url\"");
        if (start_pos == std::string::npos)
            return true;
        start_string = "meta content=\"";
        start_pos = document->rfind(start_string, start_pos);
        if (start_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 3)";
            return false;
        }
        start_pos += start_string.size();
        const size_t end_pos(document->find('"', start_pos + 1));
        if (end_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 4)";
            return false;
        }
        doc_url = document->substr(start_pos, end_pos - start_pos);
    }

    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool LocGovSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                            std::string * const document, std::string * const http_header_charset,
                                            std::string * const error_message)
{
    if (url.length() < 11) {
        *error_message = "LocGov URL too short!";
        return false;
    }
    const std::string doc_url("http://catdir" + url.substr(10));
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");

    return DownloadHelper(doc_url, time_limit, document, http_header_charset, error_message);
}

bool DefaultDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                        std::string * const document, std::string * const http_header_charset,
                                        std::string * const error_message)
{
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool GetRedirectedUrl(const std::string &url, const TimeLimit &time_limit, std::string * const redirected_url) {
    Downloader::Params params;
    params.follow_redirects_ = false;
    Downloader downloader(url, params, time_limit);
    return downloader.getHttpRedirectedUrl(redirected_url);
}

bool SmartDownload(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                   std::string * const http_header_charset, std::string * const error_message,
                   const bool trace)
{
    document->clear();
    static std::vector<SmartDownloader *> smart_downloaders{
        new DSpaceDownloader(trace),
        new SimpleSuffixDownloader({ ".pdf", ".jpg", ".jpeg", ".txt" }, trace),
        new SimplePrefixDownloader({ "http://www.bsz-bw.de/cgi-bin/ekz.cgi?" }, trace),
        new SimplePrefixDownloader({ "http://deposit.d-nb.de/cgi-bin/dokserv?" }, trace),
        new SimplePrefixDownloader({ "http://media.obvsg.at/" }, trace),
        new SimplePrefixDownloader({ "http://d-nb.info/" }, trace),
        new DigiToolSmartDownloader(trace),
        new DiglitSmartDownloader(trace),
        new BszSmartDownloader(trace),
        new BvbrSmartDownloader(trace),
        new Bsz21SmartDownloader(trace),
        new LocGovSmartDownloader(trace),
        new PublicationsTueSmartDownloader(trace),
        new DefaultDownloader(trace)
    };

    for (auto &smart_downloader : smart_downloaders) {
        if (smart_downloader->canHandleThis(url)) {
            LOG_DEBUG("Downloading url " + url + " using " + smart_downloader->getName());
            return smart_downloader->downloadDoc(url, time_limit, document, http_header_charset, error_message);
        }
    }

    *error_message = "No downloader available for URL: " + url;
    return false;
}

bool SmartDownloadResolveFirstRedirectHop(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                   std::string * const http_header_charset, std::string * const error_message,
                   const bool trace)
{
    std::string redirected_url(url);
    if (not GetRedirectedUrl(url, time_limit, &redirected_url))
        redirected_url = url; // Make absolutely redirected_url was not changed internally
    return SmartDownload(redirected_url, time_limit, document, http_header_charset, error_message, trace);
}
<commit_msg>Omit OpenDigi documents altogether<commit_after>/** \file   SmartDownloader.cc
 *  \brief  Implementation of descedants of the SmartDownloader class.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2015,2017,2018 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "SmartDownloader.h"
#include <iostream>
#include "Downloader.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

bool DownloadHelper(const std::string &url, const TimeLimit &time_limit,
                    std::string * const document, std::string * const http_header_charset,
                    std::string * const error_message)
{
    Downloader downloader(url, Downloader::Params(), time_limit);
    if (downloader.anErrorOccurred()) {
        *error_message = downloader.getLastErrorMessage();
        return false;
    }

    *document = downloader.getMessageBody();
    *http_header_charset = downloader.getCharset();
    return true;
}

} // unnamed namespace

SmartDownloader::SmartDownloader(const std::string &regex, const bool trace): trace_(trace) {
    std::string err_msg;
    matcher_.reset(RegexMatcher::RegexMatcherFactory(regex, &err_msg));
    if (not matcher_)
        LOG_ERROR("pattern failed to compile \"" + regex + "\"!");
}

bool SmartDownloader::canHandleThis(const std::string &url) const {
    std::string err_msg;
    if (matcher_->matched(url, &err_msg)) {
        if (not err_msg.empty())
            LOG_ERROR("an error occurred while trying to match \"" + url + "\" with \""
                  + matcher_->getPattern() + "\"! (" + err_msg + ")");

        return true;
    }

    return false;
}

bool SmartDownloader::downloadDoc(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                                  std::string * const http_header_charset, std::string * const error_message)
{
    if (downloadDocImpl(url, time_limit, document, http_header_charset, error_message)) {
        ++success_count_;
        return true;
    } else
        return false;
}

bool DSpaceDownloader::canHandleThis(const std::string &url) const {
    return url.find("dspace") != std::string::npos;
}

bool DSpaceDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                       std::string * const document, std::string * const http_header_charset,
                                       std::string * const error_message)
{
    document->clear();

    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;

    static RegexMatcher *matcher;
    if (matcher == nullptr) {
        std::string err_msg;
        matcher = RegexMatcher::RegexMatcherFactory("meta content=\"http(.*)pdf\"", &err_msg);
        if (matcher == nullptr)
            LOG_ERROR("failed to compile regex! (" + err_msg + ")");
    }

    if (not matcher->matched(*document)) {
        *error_message = "no matching DSpace structure found!";
        return false;
    }

    const std::string pdf_link("http" + (*matcher)[1] + "pdf");
    if (not DownloadHelper(pdf_link, time_limit, document, http_header_charset, error_message))
        return false;

    return true;
}

bool SimpleSuffixDownloader::canHandleThis(const std::string &url) const {
    for (const auto &suffix : suffixes_) {
        if (StringUtil::IsProperSuffixOfIgnoreCase(suffix, url))
            return true;
    }

    return false;
}

bool SimpleSuffixDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                             std::string * const document, std::string * const http_header_charset,
                                             std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    return (DownloadHelper(url, time_limit, document, http_header_charset, error_message));
}

bool SimplePrefixDownloader::canHandleThis(const std::string &url) const {
    for (const auto &prefix : prefixes_) {
        if (StringUtil::StartsWith(url, prefix, /* ignore_case = */ true))
            return true;
    }

    return false;
}

bool SimplePrefixDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                             std::string * const document, std::string * const http_header_charset,
                                             std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    return (DownloadHelper(url, time_limit, document, http_header_charset, error_message));
}

bool DigiToolSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                              std::string * const document, std::string * const http_header_charset,
                                              std::string * const error_message)
{
    static RegexMatcher * const matcher(
        RegexMatcher::RegexMatcherFactory("http://digitool.hbz-nrw.de:1801/webclient/DeliveryManager\\?pid=\\d+"));

    std::string err_msg;
    size_t start_pos, end_pos;
    if (not matcher->matched(url, &err_msg, &start_pos, &end_pos))
        LOG_ERROR("match failed: " + err_msg);

    const std::string normalised_url(url.substr(start_pos, end_pos - start_pos));

    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(normalised_url, time_limit, document, http_header_charset, error_message))
        return false;

    static const std::string ocr_text("ocr-text:\n");
    if (MediaTypeUtil::GetMediaType(*document) == "text/plain"
        and StringUtil::StartsWith(*document, ocr_text))
        *document = StringUtil::ISO8859_15ToUTF8(document->substr(ocr_text.length()));

    return true;
}

bool DiglitSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                            std::string * const document, std::string * const http_header_charset,
                                            std::string * const error_message)
{
    std::string improved_url(url);
    if (RegexMatcher::Matched("/diglit/", url))
        improved_url = StringUtil::ReplaceString("/diglit/", "/opendigi/", &improved_url);
    if (RegexMatcher::Matched("/opendigi/", improved_url)) {
        /* Since we currently cannot appropriately associate the text on an article level
        / we completely omit OpenDigi documents */
        //improved_url += "/ocr";
        document->clear();
        return true;
    }

    if (trace_ and url != improved_url)
        LOG_INFO("converted url \"" + url + "\" to \"" + improved_url + "\"");

    if (trace_)
        LOG_INFO("about to download \"" + improved_url + "\".");
    if (not DownloadHelper(improved_url, time_limit, document, http_header_charset, error_message)) {
        if (trace_)
            LOG_WARNING("original download failed!");
        return false;
    }

    return not document->empty();
}

bool BszSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                         std::string * const document, std::string * const http_header_charset,
                                         std::string * const error_message)
{
    const std::string doc_url(url.substr(0, url.size() - 3) + "pdf");
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool BvbrSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                          std::string * const document, std::string * const http_header_charset,
                                          std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;
    const std::string start_string("<body onload=window.location=\"");
    size_t start_pos(document->find(start_string));
    if (start_pos == std::string::npos) {
        *error_message = "no matching Bvbr structure found!";
        return false;
    }
    start_pos += start_string.size();
    const size_t end_pos(document->find('"', start_pos + 1));
    if (end_pos == std::string::npos) {
        *error_message = "no matching Bvbr structure found! (part 2)";
        return false;
    }
    const std::string doc_url("http://bvbr.bib-bvb.de:8991" + document->substr(start_pos, end_pos - start_pos));
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(doc_url, time_limit, document, http_header_charset, error_message);
}

bool Bsz21SmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                           std::string * const document, std::string * const http_header_charset,
                                           std::string * const error_message)
{
    if (trace_)
        LOG_INFO("about to download \"" + url + "\".");
    if (not DownloadHelper(url, time_limit, document, http_header_charset, error_message))
        return false;

    if (MediaTypeUtil::GetMediaType(*document) == "application/pdf")
        return true;

    std::string start_string("Persistente URL: <a id=\"pers_url\" href=\"");
    size_t start_pos(document->find(start_string));
    std::string doc_url;
    if (start_pos != std::string::npos) {
        start_pos += start_string.size();
        const size_t end_pos(document->find('"', start_pos + 1));
        if (end_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 1)";
            return false;
        }
        const std::string pers_url(document->substr(start_pos, end_pos - start_pos));
        const size_t last_slash_pos(pers_url.rfind('/'));
        if (last_slash_pos == std::string::npos or last_slash_pos == pers_url.size() - 1) {
            *error_message = "no matching Bsz2l structure found! (part 2)";
            return false;
        }
        doc_url = "http://idb.ub.uni-tuebingen.de/cgi-bin/digi-downloadPdf.fcgi?projectname="
                  + pers_url.substr(last_slash_pos + 1);
    } else {
        start_pos = document->find("name=\"citation_pdf_url\"");
        if (start_pos == std::string::npos)
            return true;
        start_string = "meta content=\"";
        start_pos = document->rfind(start_string, start_pos);
        if (start_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 3)";
            return false;
        }
        start_pos += start_string.size();
        const size_t end_pos(document->find('"', start_pos + 1));
        if (end_pos == std::string::npos) {
            *error_message = "no matching Bsz2l structure found! (part 4)";
            return false;
        }
        doc_url = document->substr(start_pos, end_pos - start_pos);
    }

    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool LocGovSmartDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                            std::string * const document, std::string * const http_header_charset,
                                            std::string * const error_message)
{
    if (url.length() < 11) {
        *error_message = "LocGov URL too short!";
        return false;
    }
    const std::string doc_url("http://catdir" + url.substr(10));
    if (trace_)
        LOG_INFO("about to download \"" + doc_url + "\".");

    return DownloadHelper(doc_url, time_limit, document, http_header_charset, error_message);
}

bool DefaultDownloader::downloadDocImpl(const std::string &url, const TimeLimit &time_limit,
                                        std::string * const document, std::string * const http_header_charset,
                                        std::string * const error_message)
{
    return DownloadHelper(url, time_limit, document, http_header_charset, error_message);
}

bool GetRedirectedUrl(const std::string &url, const TimeLimit &time_limit, std::string * const redirected_url) {
    Downloader::Params params;
    params.follow_redirects_ = false;
    Downloader downloader(url, params, time_limit);
    return downloader.getHttpRedirectedUrl(redirected_url);
}

bool SmartDownload(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                   std::string * const http_header_charset, std::string * const error_message,
                   const bool trace)
{
    document->clear();
    static std::vector<SmartDownloader *> smart_downloaders{
        new DSpaceDownloader(trace),
        new SimpleSuffixDownloader({ ".pdf", ".jpg", ".jpeg", ".txt" }, trace),
        new SimplePrefixDownloader({ "http://www.bsz-bw.de/cgi-bin/ekz.cgi?" }, trace),
        new SimplePrefixDownloader({ "http://deposit.d-nb.de/cgi-bin/dokserv?" }, trace),
        new SimplePrefixDownloader({ "http://media.obvsg.at/" }, trace),
        new SimplePrefixDownloader({ "http://d-nb.info/" }, trace),
        new DigiToolSmartDownloader(trace),
        new DiglitSmartDownloader(trace),
        new BszSmartDownloader(trace),
        new BvbrSmartDownloader(trace),
        new Bsz21SmartDownloader(trace),
        new LocGovSmartDownloader(trace),
        new PublicationsTueSmartDownloader(trace),
        new DefaultDownloader(trace)
    };

    for (auto &smart_downloader : smart_downloaders) {
        if (smart_downloader->canHandleThis(url)) {
            LOG_DEBUG("Downloading url " + url + " using " + smart_downloader->getName());
            return smart_downloader->downloadDoc(url, time_limit, document, http_header_charset, error_message);
        }
    }

    *error_message = "No downloader available for URL: " + url;
    return false;
}

bool SmartDownloadResolveFirstRedirectHop(const std::string &url, const TimeLimit &time_limit, std::string * const document,
                   std::string * const http_header_charset, std::string * const error_message,
                   const bool trace)
{
    std::string redirected_url(url);
    if (not GetRedirectedUrl(url, time_limit, &redirected_url))
        redirected_url = url; // Make absolutely redirected_url was not changed internally
    return SmartDownload(redirected_url, time_limit, document, http_header_charset, error_message, trace);
}
<|endoftext|>


<commit_before>/// HEADER
#include "svm_ensemble.h"

/// PROJECT
#include <csapex/msg/io.h>
#include <csapex/utility/register_apex_plugin.h>
#include <csapex/param/parameter_factory.h>
#include <csapex/model/node_modifier.h>
#include <csapex/msg/generic_vector_message.hpp>
#include <csapex_opencv/cv_mat_message.h>

CSAPEX_REGISTER_CLASS(csapex::SVMEnsemble, csapex::Node)

using namespace csapex;
using namespace csapex::connection_types;

template<typename T>
inline std::string toString(const T value)
{
    std::stringstream ss;
    ss << value;
    return ss.str();
}

SVMEnsemble::SVMEnsemble() :
    loaded_(false)
{
}

void SVMEnsemble::setup(NodeModifier& node_modifier)
{
    in_ = node_modifier.addInput<GenericVectorMessage, FeaturesMessage>("features");
    out_ = node_modifier.addOutput<GenericVectorMessage, CvMatMessage::ConstPtr>("responses");

    reload_ = node_modifier.addSlot("Reload", std::bind(&SVMEnsemble::load, this));

}

void SVMEnsemble::setupParameters(Parameterizable& parameters)
{
    addParameter(csapex::param::ParameterFactory::declarePath("svm array path",
                                                      csapex::param::ParameterDescription("Path to a saved svm."),
                                                      true,
                                                      "",
                                                      "*.yaml *.tar.gz"),
                 std::bind(&SVMEnsemble::load, this));

    csapex::param::ParameterPtr param_label = csapex::param::ParameterFactory::declareBool("compute labels",
                                                                                csapex::param::ParameterDescription("Directly compute labels. 'false' allows manual threshold setting for binary classification"),
                                                                                true);
    addParameter(param_label);

    std::map<std::string, int> svm_thresh_types = {
        {">", GREATER},
        {"<" , LESS},
        {">=", GREATER_EQUAL},
        {"<=", LESS_EQUAL}
    };
    auto threshold_condition = [param_label]() { return param_label->as<bool>() == false; };
    addConditionalParameter(param::ParameterFactory::declareParameterSet("threshold/type",
                                                                         csapex::param::ParameterDescription("SVM threshold comparison type"),
                                                                         svm_thresh_types,
                                                                         (int) GREATER),
                            threshold_condition);

    addConditionalParameter(csapex::param::ParameterFactory::declareRange("threshold/value",
                                                                          csapex::param::ParameterDescription("SVM threshold for binary classification"),
                                                                          -1000.0,
                                                                          1000.0,
                                                                          0.0,
                                                                          0.01),
                            threshold_condition);
}

void SVMEnsemble::process()
{
    std::shared_ptr<std::vector<FeaturesMessage> const> input = msg::getMessage<GenericVectorMessage, FeaturesMessage>(in_);
    std::shared_ptr<std::vector<CvMatMessage::ConstPtr>> output(new std::vector<CvMatMessage::ConstPtr>);

    if(!loaded_) {
        throw std::runtime_error("No SVM is loaded!");
    }

    const bool compute_label = readParameter<bool>("compute labels");
    std::function<bool(float)> comparator;

    {
        const float threshold = readParameter<double>("threshold/value");
        const int comparison_type = readParameter<int>("threshold/type");

        switch (comparison_type)
        {
        default:
        case GREATER:
            comparator = [threshold](float value) { return value > threshold; };
            break;
        case GREATER_EQUAL:
            comparator = [threshold](float value) { return value >= threshold; };
            break;
        case LESS:
            comparator = [threshold](float value) { return value < threshold; };
            break;
        case LESS_EQUAL:
            comparator = [threshold](float value) { return value <= threshold; };
            break;
        }
    }

    /// TODO : OMP?

    std::size_t size = input->size();
    for(std::size_t i = 0 ; i < size ; ++i)
    {
        cv::Mat sample(input->at(i).value);
        CvMatMessage::Ptr result_msg(new CvMatMessage(enc::unknown, 0));
        cv::Mat &result_value = result_msg->value;
        result_value = svm_responses_.clone();

        for(std::size_t j = 0 ; j < svms_size_ ; ++j) {
            SVMPtr svm = svms_.at(j);
            if (compute_label) {
                result_value.at<float>(j,1) = svm->predict(sample);
            } else {
                const float response = svm->predict(sample, true);
                result_value.at<float>(j,1) = comparator(response) ? POSITIVE : NEGATIVE;
            }
            std::cout << svm_responses_ << std::endl;
        }
        output->push_back(result_msg);
    }

    msg::publish<GenericVectorMessage, CvMatMessage::ConstPtr>(out_, output);
}

void SVMEnsemble::load()
{
    std::string path = readParameter<std::string>("svm array path");
    if(path == "")
        return;

    const static std::string prefix = "svm_";
    cv::FileStorage fs(path, cv::FileStorage::READ);
    std::vector<int> labels;
    fs["labels"] >> labels;

    if(labels.empty())
        throw std::runtime_error("File entry 'labels' may not be empty!");

    /// get svms for labels
    svm_responses_ = cv::Mat(labels.size(), 2, CV_32FC1, cv::Scalar());
    for(std::size_t i = 0 ; i < labels.size() ; ++i) {
        std::string label = prefix + toString(labels.at(i));
        SVMPtr svm(new cv::SVM);
        svm->read(fs.fs, (CvFileNode*) fs[label].node);
        svms_.push_back(svm);
        svm_responses_.at<float>(i, 0) = labels.at(i);
    }
    svms_size_ = svms_.size();
    loaded_ = true;
}
<commit_msg>reload svm ensemble fix<commit_after>/// HEADER
#include "svm_ensemble.h"

/// PROJECT
#include <csapex/msg/io.h>
#include <csapex/utility/register_apex_plugin.h>
#include <csapex/param/parameter_factory.h>
#include <csapex/model/node_modifier.h>
#include <csapex/msg/generic_vector_message.hpp>
#include <csapex_opencv/cv_mat_message.h>

CSAPEX_REGISTER_CLASS(csapex::SVMEnsemble, csapex::Node)

using namespace csapex;
using namespace csapex::connection_types;

template<typename T>
inline std::string toString(const T value)
{
    std::stringstream ss;
    ss << value;
    return ss.str();
}

SVMEnsemble::SVMEnsemble() :
    loaded_(false)
{
}

void SVMEnsemble::setup(NodeModifier& node_modifier)
{
    in_ = node_modifier.addInput<GenericVectorMessage, FeaturesMessage>("features");
    out_ = node_modifier.addOutput<GenericVectorMessage, CvMatMessage::ConstPtr>("responses");

    reload_ = node_modifier.addSlot("Reload", std::bind(&SVMEnsemble::load, this));

}

void SVMEnsemble::setupParameters(Parameterizable& parameters)
{
    addParameter(csapex::param::ParameterFactory::declarePath("svm array path",
                                                      csapex::param::ParameterDescription("Path to a saved svm."),
                                                      true,
                                                      "",
                                                      "*.yaml *.tar.gz"),
                 std::bind(&SVMEnsemble::load, this));

    csapex::param::ParameterPtr param_label = csapex::param::ParameterFactory::declareBool("compute labels",
                                                                                csapex::param::ParameterDescription("Directly compute labels. 'false' allows manual threshold setting for binary classification"),
                                                                                true);
    addParameter(param_label);

    std::map<std::string, int> svm_thresh_types = {
        {">", GREATER},
        {"<" , LESS},
        {">=", GREATER_EQUAL},
        {"<=", LESS_EQUAL}
    };
    auto threshold_condition = [param_label]() { return param_label->as<bool>() == false; };
    addConditionalParameter(param::ParameterFactory::declareParameterSet("threshold/type",
                                                                         csapex::param::ParameterDescription("SVM threshold comparison type"),
                                                                         svm_thresh_types,
                                                                         (int) GREATER),
                            threshold_condition);

    addConditionalParameter(csapex::param::ParameterFactory::declareRange("threshold/value",
                                                                          csapex::param::ParameterDescription("SVM threshold for binary classification"),
                                                                          -1000.0,
                                                                          1000.0,
                                                                          0.0,
                                                                          0.0001),
                            threshold_condition);
}

void SVMEnsemble::process()
{
    std::shared_ptr<std::vector<FeaturesMessage> const> input = msg::getMessage<GenericVectorMessage, FeaturesMessage>(in_);
    std::shared_ptr<std::vector<CvMatMessage::ConstPtr>> output(new std::vector<CvMatMessage::ConstPtr>);

    if(!loaded_) {
        throw std::runtime_error("No SVM is loaded!");
    }

    const bool compute_label = readParameter<bool>("compute labels");
    std::function<bool(float)> comparator;

    {
        const float threshold = readParameter<double>("threshold/value");
        const int comparison_type = readParameter<int>("threshold/type");

        switch (comparison_type)
        {
        default:
        case GREATER:
            comparator = [threshold](float value) { return value > threshold; };
            break;
        case GREATER_EQUAL:
            comparator = [threshold](float value) { return value >= threshold; };
            break;
        case LESS:
            comparator = [threshold](float value) { return value < threshold; };
            break;
        case LESS_EQUAL:
            comparator = [threshold](float value) { return value <= threshold; };
            break;
        }
    }

    /// TODO : OMP?

    std::size_t size = input->size();
    for(std::size_t i = 0 ; i < size ; ++i)
    {
        cv::Mat sample(input->at(i).value);
        CvMatMessage::Ptr result_msg(new CvMatMessage(enc::unknown, 0));
        cv::Mat &result_value = result_msg->value;
        result_value = svm_responses_.clone();

        for(std::size_t j = 0 ; j < svms_size_ ; ++j) {
            SVMPtr svm = svms_.at(j);
            if (compute_label) {
                result_value.at<float>(j,1) = svm->predict(sample);
            } else {
                const float response = svm->predict(sample, true);
                result_value.at<float>(j,1) = comparator(response) ? POSITIVE : NEGATIVE;
            }
        }
        output->push_back(result_msg);
    }

    msg::publish<GenericVectorMessage, CvMatMessage::ConstPtr>(out_, output);
}

void SVMEnsemble::load()
{
    std::string path = readParameter<std::string>("svm array path");
    if(path == "")
        return;

    svms_.clear();
    const static std::string prefix = "svm_";
    cv::FileStorage fs(path, cv::FileStorage::READ);
    std::vector<int> labels;
    fs["labels"] >> labels;

    if(labels.empty())
        throw std::runtime_error("File entry 'labels' may not be empty!");

    /// get svms for labels
    svm_responses_ = cv::Mat(labels.size(), 2, CV_32FC1, cv::Scalar());
    for(std::size_t i = 0 ; i < labels.size() ; ++i) {
        std::string label = prefix + toString(labels.at(i));
        SVMPtr svm(new cv::SVM);
        svm->read(fs.fs, (CvFileNode*) fs[label].node);
        svms_.push_back(svm);
        svm_responses_.at<float>(i, 0) = labels.at(i);
        std::cout << label << std::endl;
        std::cout << svm->get_var_count() << std::endl;
    }
    svms_size_ = svms_.size();
    loaded_ = true;
}
<|endoftext|>


<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include "chunk.h"

#include "io_dispatcher.h"
#include "file.h"
#include "../file_chunk.h"

using namespace upload;

CharBuffer* Chunk::GetDeflateBuffer(const size_t bytes) {
  if (current_deflate_buffer_              == NULL ||
      current_deflate_buffer_->free_bytes() <  64) {
    if (current_deflate_buffer_ != NULL) {
      ScheduleWrite(current_deflate_buffer_);
    }
    current_deflate_buffer_ = new CharBuffer(bytes);
  }

  return current_deflate_buffer_;
}

void Chunk::ScheduleWrite(CharBuffer *buffer) {
  buffer->SetBaseOffset(compressed_size_);
  compressed_size_ += buffer->used_bytes();

  if (deferred_write_) {
    assert (! HasUploadStreamHandle());
    deferred_buffers_.push_back(buffer);
  } else {
    file_->io_dispatcher()->ScheduleWrite(this, buffer);
  }
}

void Chunk::FlushDeferredWrites(const bool delete_buffers) {
  assert (deferred_write_);

  std::vector<CharBuffer*>::const_iterator i    = deferred_buffers_.begin();
  std::vector<CharBuffer*>::const_iterator iend = deferred_buffers_.end();
  for (; i != iend; ++i) {
    file_->io_dispatcher()->ScheduleWrite(this, *i, delete_buffers);
  }
  deferred_buffers_.clear();
  deferred_write_ = false;
}

void Chunk::Initialize() {
  done_            = false;
  compressed_size_ = 0;

  content_hash_context_.buffer = malloc(content_hash_context_.size);
  shash::Init(content_hash_context_);

  zlib_context_.zalloc   = Z_NULL;
  zlib_context_.zfree    = Z_NULL;
  zlib_context_.opaque   = Z_NULL;
  zlib_context_.next_in  = Z_NULL;
  zlib_context_.avail_in = 0;
  const int zlib_retval = deflateInit(&zlib_context_, Z_DEFAULT_COMPRESSION);
  assert (zlib_retval == 0);

  zlib_initialized_         = true;
  content_hash_initialized_ = true;
}

void Chunk::Finalize() {
  assert (! done_);

  shash::Final(content_hash_context_, &content_hash_);
  free(content_hash_context_.buffer);
  content_hash_context_.buffer = NULL;

  assert (zlib_context_.avail_in == 0);
  const int retcode = deflateEnd(&zlib_context_);
  assert (retcode == Z_OK);

  if (current_deflate_buffer_ != NULL) {
    ScheduleWrite(current_deflate_buffer_);
    current_deflate_buffer_ = NULL;
  }

  if (deferred_write_) {
    FlushDeferredWrites();
  }

  done_ = true;
}

void Chunk::ScheduleCommit() {
  file_->io_dispatcher()->ScheduleCommit(this);
}

Chunk::Chunk(const Chunk &other) :
  file_(other.file_),
  file_offset_(other.file_offset_),
  chunk_size_(other.chunk_size_),
  done_(other.done_),
  is_bulk_chunk_(other.is_bulk_chunk_),
  is_fully_defined_(other.is_fully_defined_),
  deferred_write_(other.deferred_write_),
  deferred_buffers_(other.deferred_buffers_),
  zlib_initialized_(false),
  content_hash_context_(other.content_hash_context_),
  content_hash_(other.content_hash_),
  content_hash_initialized_(other.content_hash_initialized_),
  upload_stream_handle_(NULL),
  bytes_written_(other.bytes_written_),
  compressed_size_(other.compressed_size_)
{
  assert (! other.done_);
  assert (! other.HasUploadStreamHandle());
  assert (other.bytes_written_ == 0);
  assert (other.zlib_context_.avail_in == 0);

  current_deflate_buffer_ = other.current_deflate_buffer_->Clone();

  content_hash_context_.buffer = malloc(content_hash_context_.size);
  memcpy(      content_hash_context_.buffer,
         other.content_hash_context_.buffer,
               content_hash_context_.size);

  const int retval = deflateCopy(&zlib_context_,
                                 const_cast<z_streamp>(&other.zlib_context_));
  assert (retval == Z_OK);
  zlib_initialized_ = true;
}

Chunk* Chunk::CopyAsBulkChunk(const size_t file_size) {
  assert (! done_);
  assert (deferred_write_);
  assert (! HasUploadStreamHandle());
  assert (file_offset_ == 0);

  // create a new bulk chunk and upload the (copied) data buffers that have been
  // hold back (deferred write)
  // Note: The buffer data is _not_ copied when copying the chunk
  //       we therefore do not delete the uploaded data
  Chunk *new_bulk_chunk = new Chunk(*this);
  new_bulk_chunk->set_size(file_size);
  const bool should_delete_buffers = false;
  new_bulk_chunk->FlushDeferredWrites(should_delete_buffers);
  new_bulk_chunk->SetAsBulkChunk();

  // upload the deferred buffers and delete them after the upload is complete
  FlushDeferredWrites();
  assert (! deferred_write_);

  return new_bulk_chunk;
}

std::string Chunk::hash_suffix() const {
  return (IsBulkChunk()) ? file_->hash_suffix() : FileChunk::kCasSuffix;
}
<commit_msg>cosmetics: replace malloc by smalloc wrapper<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include "chunk.h"

#include "io_dispatcher.h"
#include "file.h"
#include "../file_chunk.h"
#include "../smalloc.h"

using namespace upload;

CharBuffer* Chunk::GetDeflateBuffer(const size_t bytes) {
  if (current_deflate_buffer_              == NULL ||
      current_deflate_buffer_->free_bytes() <  64) {
    if (current_deflate_buffer_ != NULL) {
      ScheduleWrite(current_deflate_buffer_);
    }
    current_deflate_buffer_ = new CharBuffer(bytes);
  }

  return current_deflate_buffer_;
}

void Chunk::ScheduleWrite(CharBuffer *buffer) {
  buffer->SetBaseOffset(compressed_size_);
  compressed_size_ += buffer->used_bytes();

  if (deferred_write_) {
    assert (! HasUploadStreamHandle());
    deferred_buffers_.push_back(buffer);
  } else {
    file_->io_dispatcher()->ScheduleWrite(this, buffer);
  }
}

void Chunk::FlushDeferredWrites(const bool delete_buffers) {
  assert (deferred_write_);

  std::vector<CharBuffer*>::const_iterator i    = deferred_buffers_.begin();
  std::vector<CharBuffer*>::const_iterator iend = deferred_buffers_.end();
  for (; i != iend; ++i) {
    file_->io_dispatcher()->ScheduleWrite(this, *i, delete_buffers);
  }
  deferred_buffers_.clear();
  deferred_write_ = false;
}

void Chunk::Initialize() {
  done_            = false;
  compressed_size_ = 0;

  content_hash_context_.buffer = smalloc(content_hash_context_.size);
  shash::Init(content_hash_context_);

  zlib_context_.zalloc   = Z_NULL;
  zlib_context_.zfree    = Z_NULL;
  zlib_context_.opaque   = Z_NULL;
  zlib_context_.next_in  = Z_NULL;
  zlib_context_.avail_in = 0;
  const int zlib_retval = deflateInit(&zlib_context_, Z_DEFAULT_COMPRESSION);
  assert (zlib_retval == 0);

  zlib_initialized_         = true;
  content_hash_initialized_ = true;
}

void Chunk::Finalize() {
  assert (! done_);

  shash::Final(content_hash_context_, &content_hash_);
  free(content_hash_context_.buffer);
  content_hash_context_.buffer = NULL;

  assert (zlib_context_.avail_in == 0);
  const int retcode = deflateEnd(&zlib_context_);
  assert (retcode == Z_OK);

  if (current_deflate_buffer_ != NULL) {
    ScheduleWrite(current_deflate_buffer_);
    current_deflate_buffer_ = NULL;
  }

  if (deferred_write_) {
    FlushDeferredWrites();
  }

  done_ = true;
}

void Chunk::ScheduleCommit() {
  file_->io_dispatcher()->ScheduleCommit(this);
}

Chunk::Chunk(const Chunk &other) :
  file_(other.file_),
  file_offset_(other.file_offset_),
  chunk_size_(other.chunk_size_),
  done_(other.done_),
  is_bulk_chunk_(other.is_bulk_chunk_),
  is_fully_defined_(other.is_fully_defined_),
  deferred_write_(other.deferred_write_),
  deferred_buffers_(other.deferred_buffers_),
  zlib_initialized_(false),
  content_hash_context_(other.content_hash_context_),
  content_hash_(other.content_hash_),
  content_hash_initialized_(other.content_hash_initialized_),
  upload_stream_handle_(NULL),
  bytes_written_(other.bytes_written_),
  compressed_size_(other.compressed_size_)
{
  assert (! other.done_);
  assert (! other.HasUploadStreamHandle());
  assert (other.bytes_written_ == 0);
  assert (other.zlib_context_.avail_in == 0);

  current_deflate_buffer_ = other.current_deflate_buffer_->Clone();

  content_hash_context_.buffer = smalloc(content_hash_context_.size);
  memcpy(      content_hash_context_.buffer,
         other.content_hash_context_.buffer,
               content_hash_context_.size);

  const int retval = deflateCopy(&zlib_context_,
                                 const_cast<z_streamp>(&other.zlib_context_));
  assert (retval == Z_OK);
  zlib_initialized_ = true;
}

Chunk* Chunk::CopyAsBulkChunk(const size_t file_size) {
  assert (! done_);
  assert (deferred_write_);
  assert (! HasUploadStreamHandle());
  assert (file_offset_ == 0);

  // create a new bulk chunk and upload the (copied) data buffers that have been
  // hold back (deferred write)
  // Note: The buffer data is _not_ copied when copying the chunk
  //       we therefore do not delete the uploaded data
  Chunk *new_bulk_chunk = new Chunk(*this);
  new_bulk_chunk->set_size(file_size);
  const bool should_delete_buffers = false;
  new_bulk_chunk->FlushDeferredWrites(should_delete_buffers);
  new_bulk_chunk->SetAsBulkChunk();

  // upload the deferred buffers and delete them after the upload is complete
  FlushDeferredWrites();
  assert (! deferred_write_);

  return new_bulk_chunk;
}

std::string Chunk::hash_suffix() const {
  return (IsBulkChunk()) ? file_->hash_suffix() : FileChunk::kCasSuffix;
}
<|endoftext|>