<commit_before>#include "MCWin32.h" // Should be included first.

#include "MCZip.h"

#include "zip.h"
#include "unzip.h"

#include <sys/types.h>
#ifndef _MSC_VER
#include <dirent.h>
#include <unistd.h>
#endif
#include <sys/stat.h>
#include <time.h>

#ifdef _MSC_VER
#define USEWIN32IOAPI
#include "ioapi.h"
#include "iowin32.h"
#endif

using namespace mailcore;

static ErrorCode addFile(zipFile file, String * path);

ErrorCode mailcore::CreateZipFileFromFolder(String * zipFilename, String * path)
{
#ifdef _MSC_VER
    zlib_filefunc64_def ffunc;
    fill_win32_filefunc64A(&ffunc);
    zipFile file = zipOpen2_64(zipFilename->fileSystemRepresentation(), APPEND_STATUS_CREATE, NULL, &ffunc);
#else
    zipFile file = zipOpen(zipFilename->fileSystemRepresentation(), APPEND_STATUS_CREATE);
#endif
    if (file == NULL) {
        return ErrorFile;
    }

    addFile(file, path);

    int err = zipClose(file, NULL);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    return ErrorNone;
}

static ErrorCode addFile(zipFile file, String * path)
{
    const char * cPath = path->fileSystemRepresentation();
    struct stat statinfo;
    int r;
    int err;

    r = stat(cPath, &statinfo);
    if (r < 0)
        return ErrorFile;

    if (S_ISDIR(statinfo.st_mode)) {
#ifndef _MSC_VER
        DIR * dir = opendir(cPath);
        if (dir == NULL) {
            return ErrorFile;
        }

        struct dirent * ent;
        while ((ent = readdir(dir)) != NULL) {
            if ((strcmp(ent->d_name, ".") == 0) || (strcmp(ent->d_name, "..") == 0)) {
                continue;
            }

            String * subpath = path->stringByAppendingPathComponent(String::stringWithFileSystemRepresentation(ent->d_name));
            addFile(file, subpath);
        }
        closedir(dir);
#else
        String * wildcard = path->stringByAppendingPathComponent(MCSTR("*"));

        HANDLE hFind = INVALID_HANDLE_VALUE;
        WIN32_FIND_DATA ffd;

        hFind = FindFirstFile(wildcard->fileSystemRepresentation(), &ffd);
        if (hFind == INVALID_HANDLE_VALUE)  {
            return ErrorFile;
        }

        do {
            if ((strcmp(ffd.cFileName, ".") == 0) || (strcmp(ffd.cFileName, "..") == 0)) {
                continue;
            }
            String * subpath = path->stringByAppendingPathComponent(String::stringWithFileSystemRepresentation(ffd.cFileName));
            addFile(file, subpath);
        }
        while (FindNextFile(hFind, &ffd) != 0);

        if (GetLastError() != ERROR_NO_MORE_FILES) {
            FindClose(hFind);
            return ErrorFile;
        }

        FindClose(hFind);
#endif

        return ErrorNone;
    }

    time_t clock;
    time(&clock);

    struct tm timevalue;
    zip_fileinfo zi;
    gmtime_r(&clock, &timevalue);
    zi.tmz_date.tm_sec = timevalue.tm_sec;
    zi.tmz_date.tm_min = timevalue.tm_min;
    zi.tmz_date.tm_hour = timevalue.tm_hour;
    zi.tmz_date.tm_mday = timevalue.tm_mday;
    zi.tmz_date.tm_mon = timevalue.tm_mon;
    zi.tmz_date.tm_year = timevalue.tm_year;
    zi.internal_fa = 0;
    zi.external_fa = 0;
    zi.dosDate = 0;

    err = zipOpenNewFileInZip3(file, path->lastPathComponent()->fileSystemRepresentation(),
        &zi, NULL, 0, NULL, 0, NULL,
        Z_DEFLATED,
        -1, 0,
        -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
        NULL, 0);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    FILE * f = fopen(cPath, "rb");
    if (f == NULL) {
        return ErrorFile;
    }

    char * bytes = (char *) malloc(4096);

    size_t count;
    do {
        count = fread(bytes, 1, 4096, f);

        err = zipWriteInFileInZip(file, bytes, (unsigned) count);
        if (err != ZIP_OK) {
            free(bytes);
            zipCloseFileInZip(file);
            return ErrorFile;
        }
    } while (count > 0);

    free(bytes);

    if (ferror(f)) {
        return ErrorFile;
    }

    fclose(f);

    err = zipCloseFileInZip(file);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    return ErrorNone;
}

String * mailcore::CreateTemporaryZipFileFromFolder(String * folder)
{
    char tempdir[] = "/tmp/mailcore2-XXXXXX";
    char * result = mkdtemp(tempdir);
    if (result == NULL) {
        return NULL;
    }

    String * tempDirectoryString = String::stringWithFileSystemRepresentation(tempdir);
    String * path = tempDirectoryString->stringByAppendingPathComponent(folder->lastPathComponent())->stringByAppendingUTF8Format(".zip");

    ErrorCode err = CreateZipFileFromFolder(path, folder);
    if (err != ErrorNone) {
        unlink(path->fileSystemRepresentation());
        unlink(tempdir);
        return NULL;
    }

    return path;
}

void mailcore::RemoveTemporaryZipFile(String * zipFilename)
{
    String * tempDir = zipFilename->stringByDeletingLastPathComponent();
    unlink(zipFilename->fileSystemRepresentation());
    unlink(tempDir->fileSystemRepresentation());
}
<commit_msg>Fixed CreateZipFileFromFolder() on win32.<commit_after>#include "MCWin32.h" // Should be included first.

#include "MCZip.h"

#include "zip.h"
#include "unzip.h"

#include <sys/types.h>
#ifndef _MSC_VER
#include <dirent.h>
#include <unistd.h>
#endif
#include <sys/stat.h>
#include <time.h>

#ifdef _MSC_VER
#define USEWIN32IOAPI
#include "ioapi.h"
#include "iowin32.h"
#endif

using namespace mailcore;

static ErrorCode addFile(zipFile file, String * path);

ErrorCode mailcore::CreateZipFileFromFolder(String * zipFilename, String * path)
{
#ifdef _MSC_VER
    zlib_filefunc64_def ffunc;
    fill_win32_filefunc64A(&ffunc);
    zipFile file = zipOpen2_64(zipFilename->fileSystemRepresentation(), APPEND_STATUS_CREATE, NULL, &ffunc);
#else
    zipFile file = zipOpen(zipFilename->fileSystemRepresentation(), APPEND_STATUS_CREATE);
#endif
    if (file == NULL) {
        return ErrorFile;
    }

    addFile(file, path);

    int err = zipClose(file, NULL);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    return ErrorNone;
}

static ErrorCode addFile(zipFile file, String * path)
{
    const char * cPath = path->fileSystemRepresentation();
    struct stat statinfo;
    int r;
    int err;

    r = stat(cPath, &statinfo);
    if (r < 0)
        return ErrorFile;

    if (S_ISDIR(statinfo.st_mode)) {
#ifndef _MSC_VER
        DIR * dir = opendir(cPath);
        if (dir == NULL) {
            return ErrorFile;
        }

        struct dirent * ent;
        while ((ent = readdir(dir)) != NULL) {
            if ((strcmp(ent->d_name, ".") == 0) || (strcmp(ent->d_name, "..") == 0)) {
                continue;
            }

            String * subpath = path->stringByAppendingPathComponent(String::stringWithFileSystemRepresentation(ent->d_name));
            addFile(file, subpath);
        }
        closedir(dir);
#else
        String * wildcard = path->stringByAppendingPathComponent(MCSTR("*"));

        HANDLE hFind = INVALID_HANDLE_VALUE;
        WIN32_FIND_DATA ffd;

        hFind = FindFirstFile(wildcard->unicodeCharacters(), &ffd);
        if (hFind == INVALID_HANDLE_VALUE)  {
            return ErrorFile;
        }

        do {
            if ((wcscmp(ffd.cFileName, L".") == 0) || (wcscmp(ffd.cFileName, L"..") == 0)) {
                continue;
            }
            String * subpath = path->stringByAppendingPathComponent(String::stringWithCharacters(ffd.cFileName));
            addFile(file, subpath);
        }
        while (FindNextFile(hFind, &ffd) != 0);

        if (GetLastError() != ERROR_NO_MORE_FILES) {
            FindClose(hFind);
            return ErrorFile;
        }

        FindClose(hFind);
#endif

        return ErrorNone;
    }

    time_t clock;
    time(&clock);

    struct tm timevalue;
    zip_fileinfo zi;
    gmtime_r(&clock, &timevalue);
    zi.tmz_date.tm_sec = timevalue.tm_sec;
    zi.tmz_date.tm_min = timevalue.tm_min;
    zi.tmz_date.tm_hour = timevalue.tm_hour;
    zi.tmz_date.tm_mday = timevalue.tm_mday;
    zi.tmz_date.tm_mon = timevalue.tm_mon;
    zi.tmz_date.tm_year = timevalue.tm_year;
    zi.internal_fa = 0;
    zi.external_fa = 0;
    zi.dosDate = 0;

    err = zipOpenNewFileInZip3(file, path->lastPathComponent()->fileSystemRepresentation(),
        &zi, NULL, 0, NULL, 0, NULL,
        Z_DEFLATED,
        -1, 0,
        -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
        NULL, 0);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    FILE * f = fopen(cPath, "rb");
    if (f == NULL) {
        return ErrorFile;
    }

    char * bytes = (char *) malloc(4096);

    size_t count;
    do {
        count = fread(bytes, 1, 4096, f);

        err = zipWriteInFileInZip(file, bytes, (unsigned) count);
        if (err != ZIP_OK) {
            free(bytes);
            zipCloseFileInZip(file);
            return ErrorFile;
        }
    } while (count > 0);

    free(bytes);

    if (ferror(f)) {
        return ErrorFile;
    }

    fclose(f);

    err = zipCloseFileInZip(file);
    if (err != ZIP_OK) {
        return ErrorFile;
    }

    return ErrorNone;
}

String * mailcore::CreateTemporaryZipFileFromFolder(String * folder)
{
    char tempdir[] = "/tmp/mailcore2-XXXXXX";
    char * result = mkdtemp(tempdir);
    if (result == NULL) {
        return NULL;
    }

    String * tempDirectoryString = String::stringWithFileSystemRepresentation(tempdir);
    String * path = tempDirectoryString->stringByAppendingPathComponent(folder->lastPathComponent())->stringByAppendingUTF8Format(".zip");

    ErrorCode err = CreateZipFileFromFolder(path, folder);
    if (err != ErrorNone) {
        unlink(path->fileSystemRepresentation());
        unlink(tempdir);
        return NULL;
    }

    return path;
}

void mailcore::RemoveTemporaryZipFile(String * zipFilename)
{
    String * tempDir = zipFilename->stringByDeletingLastPathComponent();
    unlink(zipFilename->fileSystemRepresentation());
    unlink(tempDir->fileSystemRepresentation());
}
<|endoftext|>


<commit_before>#include <iostream>
#include <cmath>
#include <locale>
#include <unistd.h>
#include "ctext.h"

int8_t my_event(ctext *context, ctext_event event)
{
  /*
  switch(event)
  {
    case CTEXT_SCROLL:
      printf("< Scroll >");
      break;

    case CTEXT_CLEAR:
      printf("< Clear >");
      break;

    case CTEXT_DATA:
      printf("< Data >");
      break;
  }
  */
  return 0;
}

int main(int argc, char **argv ){
  int speed = 150000;
  int8_t x = 0;
  locale::global(locale("en_US.utf8"));

  initscr();
  cbreak();
  curs_set(0);
  WINDOW *local_win;
  ctext_config config;

  local_win = newwin(7, 9, 5, 5);
  start_color();

  init_pair(1,COLOR_WHITE, COLOR_BLUE);
  wbkgd(local_win,COLOR_PAIR(1));

  ctext ct(local_win);

  // get the default config
  ct.get_config(&config);

  // add my handler
  config.m_on_event = my_event;
  config.m_bounding_box = false;
  config.m_scroll_on_append = true;
  //config.m_do_wrap = true;
  config.m_append_top = true;

  // set the config back
  ct.set_config(&config);

  for(x = 0; x < 15; x++) {
    ct.printf("hello %d world", x);
    usleep(speed);
  }

  for(x = 0; x < 5; x++) {
    usleep(speed);
    ct.right();
    usleep(speed);
    ct.down();
  }

  for(x = 0; x < 15; x++) {
    usleep(speed);
    ct.left();
    usleep(speed);
    ct.up();
  }

  for(x = 0; x < 18; x++) {
    ct.clear(1);
    usleep(speed);
  }

  endwin();
  return 0;
}
<commit_msg>cleanup<commit_after>#include <iostream>
#include <locale>
#include <unistd.h>
#include "ctext.h"

int8_t my_event(ctext *context, ctext_event event)
{
  /*
  switch(event)
  {
    case CTEXT_SCROLL:
      printf("< Scroll >");
      break;

    case CTEXT_CLEAR:
      printf("< Clear >");
      break;

    case CTEXT_DATA:
      printf("< Data >");
      break;
  }
  */
  return 0;
}

int main(int argc, char **argv ){
  int speed = 150000;
  int8_t x = 0;
  locale::global(locale("en_US.utf8"));

  initscr();
  cbreak();
  curs_set(0);
  WINDOW *local_win;
  ctext_config config;

  local_win = newwin(7, 9, 5, 5);
  start_color();

  init_pair(1,COLOR_WHITE, COLOR_BLUE);
  wbkgd(local_win,COLOR_PAIR(1));

  ctext ct(local_win);

  // get the default config
  ct.get_config(&config);

  // add my handler
  config.m_on_event = my_event;
  config.m_bounding_box = false;
  config.m_scroll_on_append = true;
  //config.m_do_wrap = true;
  config.m_append_top = true;

  // set the config back
  ct.set_config(&config);

  for(x = 0; x < 15; x++) {
    ct.printf("hello %d world", x);
    usleep(speed);
  }

  for(x = 0; x < 5; x++) {
    usleep(speed);
    ct.right();
    usleep(speed);
    ct.down();
  }

  for(x = 0; x < 15; x++) {
    usleep(speed);
    ct.left();
    usleep(speed);
    ct.up();
  }

  for(x = 0; x < 18; x++) {
    ct.clear(1);
    usleep(speed);
  }

  endwin();
  return 0;
}
<|endoftext|>


<commit_before>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once

#include <type_traits>
#include <ostream>
#include <bh_static_vector.hpp>
#include <bhxx/BhBase.hpp>
#include <bhxx/type_traits_util.hpp>
#include <bhxx/array_operations.hpp>

namespace bhxx {

/** Class which enqueues an BhBase object for deletion
 *  with the Bohrium Runtime, but does not actually delete
 *  it straight away.
 *
 *  \note This is needed to ensure that all BhBase objects
 *  are still around until the list of instructions has
 *  been emptied.
 */
struct RuntimeDeleter {
    void operator()(BhBase *ptr) const;
};

/** Helper function to make shared pointers to BhBase,
 *  which use the RuntimeDeleter as their deleter */
template<typename... Args>
std::shared_ptr<BhBase> make_base_ptr(Args... args) {
    return std::shared_ptr<BhBase>(new BhBase(std::forward<Args>(args)...), RuntimeDeleter{});
}

/** Static allocated shapes and strides that is interchangeable with standard C++ vector as long
 *  as the vector is smaller than `BH_MAXDIM`.
 */
typedef BhStaticVector<uint64_t> Shape;
using Stride = BhIntVec;

/** Return a contiguous stride (row-major) based on `shape` */
extern inline Stride contiguous_stride(const Shape &shape) {
    Stride ret(shape.size());
    int64_t stride = 1;
    for (int64_t i = shape.size() - 1; i >= 0; --i) {
        ret[i] = stride;
        stride *= shape[i];
    }
    return ret;
}

/** Core class that represent the core attributes of a view that isn't typed by its dtype */
class BhArrayUnTypedCore {
protected:
    /// The array offset (from the start of the base in number of elements)
    uint64_t _offset = 0;
    /// The array shape (size of each dimension in number of elements)
    Shape _shape;
    /// The array stride (the absolute stride of each dimension in number of elements)
    Stride _stride;
    /// Pointer to the base of this array
    std::shared_ptr<BhBase> _base;
    /// Metadata to support sliding views
    bh_slide _slides;
public:

    /** Default constructor that leave the instance completely uninitialized */
    BhArrayUnTypedCore() = default;

    /** Constructor to initiate all but the `_slides` attribute */
    BhArrayUnTypedCore(uint64_t offset, Shape shape, Stride stride, std::shared_ptr<BhBase> base) :
            _offset(offset), _shape(std::move(shape)), _stride(std::move(stride)), _base(std::move(base)) {
        if (_shape.size() != _stride.size()) {
            throw std::runtime_error("The shape and stride must have same length");
        }
        if (shape.prod() <= 0) {
            throw std::runtime_error("The total size must be greater than zero");
        }
    }

    /** Return a `bh_view` of the array */
    bh_view getBhView() const {
        bh_view view;
        assert(_base);
        view.base = _base.get();
        view.start = static_cast<int64_t>(offset());
        if (shape().empty()) { // Scalar views (i.e. 0-dim views) are represented as 1-dim arrays with size one.
            view.ndim = 1;
            view.shape = BhIntVec({1});
            view.stride = BhIntVec({1});
        } else {
            view.ndim = static_cast<int64_t>(shape().size());
            view.shape = BhIntVec(shape().begin(), shape().end());
            view.stride = BhIntVec(_stride.begin(), _stride.end());;
        }
        view.slides = _slides;
        return view;
    }

    /** Swapping `a` and `b` */
    friend void swap(BhArrayUnTypedCore &a, BhArrayUnTypedCore &b) noexcept {
        using std::swap; // enable ADL
        swap(a._offset, b._offset);
        swap(a._shape, b._shape);
        swap(a._stride, b._stride);
        swap(a._base, b._base);
        swap(a._slides, b._slides);
    }

    /** Return the offset of the array */
    uint64_t offset() const {
        return _offset;
    }

    /** Return the shape of the array */
    const Shape &shape() const {
        return _shape;
    }

    /** Return the stride of the array */
    const Stride &stride() const {
        return _stride;
    }

    /** Return the base of the array */
    const std::shared_ptr<BhBase> &base() const {
        return _base;
    }

    /** Return the base of the array */
    std::shared_ptr<BhBase> base() {
        return _base;
    }

    /** Set the shape and stride of the array (both must have the same lenth) */
    void setShapeAndStride(Shape shape, Stride stride) {
        if (shape.size() != stride.size()) {
            throw std::runtime_error("The shape and stride must have same length");
        }
        _shape = std::move(shape);
        _stride = std::move(stride);
    }

    /** Return the slides object of the array */
    const bh_slide &slides() const {
        return _slides;
    }

    /** Return the slides object of the array */
    bh_slide &slides() {
        return _slides;
    }
};

/** Representation of a multidimensional array that point to a `BhBase` array.
 *
 * @tparam T  The data type of the array and the underlying base array
 */
template<typename T>
class BhArray : public BhArrayUnTypedCore {
public:
    /// The data type of each array element
    typedef T scalar_type;

    /** Default constructor that leave the instance completely uninitialized. */
    BhArray() = default;

    /** Create a new array. `Shape` and `Stride` must have the same length.
     *
     * @param shape   Shape of the new array
     * @param stride  Stride of the new array
     */
    explicit BhArray(Shape shape, Stride stride) : BhArrayUnTypedCore{0, std::move(shape), std::move(stride),
                                                                      make_base_ptr(T(0), shape.prod())} {}

    /** Create a new array (contiguous stride, row-major) */
    explicit BhArray(Shape shape) : BhArray(std::move(shape), contiguous_stride(shape)) {}

    /** Create a array that points to the given base
     *
     *  \note The caller should make sure that the shared pointer
     *        uses the RuntimeDeleter as its deleter, since this is
     *        implicitly assumed throughout, i.e. if one wants to
     *        construct a BhBase object, use the make_base_ptr
     *        helper function.
     */
    explicit BhArray(std::shared_ptr<BhBase> base, Shape shape, Stride stride, uint64_t offset = 0) :
            BhArrayUnTypedCore{offset, std::move(shape), std::move(stride), std::move(base)} {
        assert(shape.size() == stride.size());
        assert(shape.prod() > 0);
    }

    /** Create a view that points to the given base (contiguous stride, row-major)
     *
     *  \note The caller should make sure that the shared pointer
     *        uses the RuntimeDeleter as its deleter, since this is
     *        implicitly assumed throughout, i.e. if one wants to
     *        construct a BhBase object, use the make_base_ptr
     *        helper function.
     */
    explicit BhArray(std::shared_ptr<BhBase> base, Shape shape) : BhArray(std::move(base), std::move(shape),
                                                                          contiguous_stride(shape), 0) {
        assert(static_cast<uint64_t>(base->nelem()) == shape.prod());
    }

    /** Create a copy of `ary` using a Bohrium `identity` operation, which copies the underlying array data.
     *
     *  \note This function implements implicit type conversion for all widening type casts
     */
    template<typename InType,
            typename std::enable_if<type_traits::is_safe_numeric_cast<scalar_type, InType>::value, int>::type = 0>
    BhArray(const BhArray<InType> &ary) : BhArray(ary.shape()) {
        bhxx::identity(*this, ary);
    }

    /** Copy constructor that only copies meta data. The underlying array data is untouched */
    BhArray(const BhArray &) = default;

    /** Move constructor that only moves meta data. The underlying array data is untouched */
    BhArray(BhArray &&) noexcept = default;

    /** Copy the data of `other` into the array using a Bohrium `identity` operation */
    BhArray<T> &operator=(const BhArray<T> &other) {
        bhxx::identity(*this, other);
        return *this;
    }

    /** Copy the data of `other` into the array using a Bohrium `identity` operation
     *
     *  \note A move assignment is the same as a copy assignment.
     */
    BhArray<T> &operator=(BhArray<T> &&other) {
        bhxx::identity(*this, other);
        other.reset();
        return *this;
    }

    /**  Copy the scalar of `scalar_value` into the array using a Bohrium `identity` operation */
    template<typename InType,
            typename std::enable_if<type_traits::is_arithmetic<InType>::value, int>::type = 0>
    BhArray<T> &operator=(const InType &scalar_value) {
        bhxx::identity(*this, scalar_value);
        return *this;
    }

    /** Reset the array to `ary` */
    void reset(BhArray<T> ary) noexcept {
        swap(*this, ary);
    }

    /** Reset the array by cleaning all meta data and leave the array uninitialized. */
    void reset() noexcept {
        reset(BhArray());
    }

    /** Return the rank of the BhArray */
    size_t rank() const {
        assert(shape().size() == _stride.size());
        return shape().size();
    }

    /** Return the number of elements */
    uint64_t size() const {
        return shape().prod();
    }

    /** Return whether the view is contiguous and row-major */
    bool isContiguous() const;

    /** Is the data referenced by this view's base array already
     *  allocated, i.e. initialised */
    bool isDataInitialised() const { return _base->getDataPtr() != nullptr; }

    /** Obtain the data pointer of the array, not taking ownership of any kind.
     *
     *  \note This pointer might be a nullptr if the data in
     *        the base data is not initialised.
     *
     *  \note No flush is done automatically. The data might be
     *        out of sync with Bohrium.
     */
    T *data() {
        T *ret = static_cast<T *>(_base->getDataPtr());
        if (ret == nullptr) {
            return nullptr;
        } else {
            return _offset + ret;
        }
    }

    /// The const version of `data()`
    const T *data() const {
        const T *ret = static_cast<T *>(_base->getDataPtr());
        if (ret == nullptr) {
            return nullptr;
        } else {
            return _offset + ret;
        }
    }

    /// Pretty printing the content of the array
    void pprint(std::ostream &os) const;

    /// Returns a new view of the `idx` dimension. Negative index counts from the back
    BhArray<T> operator[](int64_t idx) const;

    /// Return a new transposed view
    BhArray<T> transpose() const;

    /// Return a new reshaped view (the array must be contiguous)
    BhArray<T> reshape(Shape shape) const;
};

template<typename T>
std::ostream &operator<<(std::ostream &os, const BhArray<T> &ary) {
    ary.pprint(os);
    return os;
}

}  // namespace bhxx
<commit_msg>bhxx: base() now returns a reference<commit_after>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once

#include <type_traits>
#include <ostream>
#include <bh_static_vector.hpp>
#include <bhxx/BhBase.hpp>
#include <bhxx/type_traits_util.hpp>
#include <bhxx/array_operations.hpp>

namespace bhxx {

/** Class which enqueues an BhBase object for deletion
 *  with the Bohrium Runtime, but does not actually delete
 *  it straight away.
 *
 *  \note This is needed to ensure that all BhBase objects
 *  are still around until the list of instructions has
 *  been emptied.
 */
struct RuntimeDeleter {
    void operator()(BhBase *ptr) const;
};

/** Helper function to make shared pointers to BhBase,
 *  which use the RuntimeDeleter as their deleter */
template<typename... Args>
std::shared_ptr<BhBase> make_base_ptr(Args... args) {
    return std::shared_ptr<BhBase>(new BhBase(std::forward<Args>(args)...), RuntimeDeleter{});
}

/** Static allocated shapes and strides that is interchangeable with standard C++ vector as long
 *  as the vector is smaller than `BH_MAXDIM`.
 */
typedef BhStaticVector<uint64_t> Shape;
using Stride = BhIntVec;

/** Return a contiguous stride (row-major) based on `shape` */
extern inline Stride contiguous_stride(const Shape &shape) {
    Stride ret(shape.size());
    int64_t stride = 1;
    for (int64_t i = shape.size() - 1; i >= 0; --i) {
        ret[i] = stride;
        stride *= shape[i];
    }
    return ret;
}

/** Core class that represent the core attributes of a view that isn't typed by its dtype */
class BhArrayUnTypedCore {
protected:
    /// The array offset (from the start of the base in number of elements)
    uint64_t _offset = 0;
    /// The array shape (size of each dimension in number of elements)
    Shape _shape;
    /// The array stride (the absolute stride of each dimension in number of elements)
    Stride _stride;
    /// Pointer to the base of this array
    std::shared_ptr<BhBase> _base;
    /// Metadata to support sliding views
    bh_slide _slides;
public:

    /** Default constructor that leave the instance completely uninitialized */
    BhArrayUnTypedCore() = default;

    /** Constructor to initiate all but the `_slides` attribute */
    BhArrayUnTypedCore(uint64_t offset, Shape shape, Stride stride, std::shared_ptr<BhBase> base) :
            _offset(offset), _shape(std::move(shape)), _stride(std::move(stride)), _base(std::move(base)) {
        if (_shape.size() != _stride.size()) {
            throw std::runtime_error("The shape and stride must have same length");
        }
        if (shape.prod() <= 0) {
            throw std::runtime_error("The total size must be greater than zero");
        }
    }

    /** Return a `bh_view` of the array */
    bh_view getBhView() const {
        bh_view view;
        assert(_base);
        view.base = _base.get();
        view.start = static_cast<int64_t>(offset());
        if (shape().empty()) { // Scalar views (i.e. 0-dim views) are represented as 1-dim arrays with size one.
            view.ndim = 1;
            view.shape = BhIntVec({1});
            view.stride = BhIntVec({1});
        } else {
            view.ndim = static_cast<int64_t>(shape().size());
            view.shape = BhIntVec(shape().begin(), shape().end());
            view.stride = BhIntVec(_stride.begin(), _stride.end());;
        }
        view.slides = _slides;
        return view;
    }

    /** Swapping `a` and `b` */
    friend void swap(BhArrayUnTypedCore &a, BhArrayUnTypedCore &b) noexcept {
        using std::swap; // enable ADL
        swap(a._offset, b._offset);
        swap(a._shape, b._shape);
        swap(a._stride, b._stride);
        swap(a._base, b._base);
        swap(a._slides, b._slides);
    }

    /** Return the offset of the array */
    uint64_t offset() const {
        return _offset;
    }

    /** Return the shape of the array */
    const Shape &shape() const {
        return _shape;
    }

    /** Return the stride of the array */
    const Stride &stride() const {
        return _stride;
    }

    /** Return the base of the array */
    const std::shared_ptr<BhBase> &base() const {
        return _base;
    }

    /** Return the base of the array */
    std::shared_ptr<BhBase> &base() {
        return _base;
    }

    /** Set the shape and stride of the array (both must have the same lenth) */
    void setShapeAndStride(Shape shape, Stride stride) {
        if (shape.size() != stride.size()) {
            throw std::runtime_error("The shape and stride must have same length");
        }
        _shape = std::move(shape);
        _stride = std::move(stride);
    }

    /** Return the slides object of the array */
    const bh_slide &slides() const {
        return _slides;
    }

    /** Return the slides object of the array */
    bh_slide &slides() {
        return _slides;
    }
};

/** Representation of a multidimensional array that point to a `BhBase` array.
 *
 * @tparam T  The data type of the array and the underlying base array
 */
template<typename T>
class BhArray : public BhArrayUnTypedCore {
public:
    /// The data type of each array element
    typedef T scalar_type;

    /** Default constructor that leave the instance completely uninitialized. */
    BhArray() = default;

    /** Create a new array. `Shape` and `Stride` must have the same length.
     *
     * @param shape   Shape of the new array
     * @param stride  Stride of the new array
     */
    explicit BhArray(Shape shape, Stride stride) : BhArrayUnTypedCore{0, std::move(shape), std::move(stride),
                                                                      make_base_ptr(T(0), shape.prod())} {}

    /** Create a new array (contiguous stride, row-major) */
    explicit BhArray(Shape shape) : BhArray(std::move(shape), contiguous_stride(shape)) {}

    /** Create a array that points to the given base
     *
     *  \note The caller should make sure that the shared pointer
     *        uses the RuntimeDeleter as its deleter, since this is
     *        implicitly assumed throughout, i.e. if one wants to
     *        construct a BhBase object, use the make_base_ptr
     *        helper function.
     */
    explicit BhArray(std::shared_ptr<BhBase> base, Shape shape, Stride stride, uint64_t offset = 0) :
            BhArrayUnTypedCore{offset, std::move(shape), std::move(stride), std::move(base)} {
        assert(shape.size() == stride.size());
        assert(shape.prod() > 0);
    }

    /** Create a view that points to the given base (contiguous stride, row-major)
     *
     *  \note The caller should make sure that the shared pointer
     *        uses the RuntimeDeleter as its deleter, since this is
     *        implicitly assumed throughout, i.e. if one wants to
     *        construct a BhBase object, use the make_base_ptr
     *        helper function.
     */
    explicit BhArray(std::shared_ptr<BhBase> base, Shape shape) : BhArray(std::move(base), std::move(shape),
                                                                          contiguous_stride(shape), 0) {
        assert(static_cast<uint64_t>(base->nelem()) == shape.prod());
    }

    /** Create a copy of `ary` using a Bohrium `identity` operation, which copies the underlying array data.
     *
     *  \note This function implements implicit type conversion for all widening type casts
     */
    template<typename InType,
            typename std::enable_if<type_traits::is_safe_numeric_cast<scalar_type, InType>::value, int>::type = 0>
    BhArray(const BhArray<InType> &ary) : BhArray(ary.shape()) {
        bhxx::identity(*this, ary);
    }

    /** Copy constructor that only copies meta data. The underlying array data is untouched */
    BhArray(const BhArray &) = default;

    /** Move constructor that only moves meta data. The underlying array data is untouched */
    BhArray(BhArray &&) noexcept = default;

    /** Copy the data of `other` into the array using a Bohrium `identity` operation */
    BhArray<T> &operator=(const BhArray<T> &other) {
        bhxx::identity(*this, other);
        return *this;
    }

    /** Copy the data of `other` into the array using a Bohrium `identity` operation
     *
     *  \note A move assignment is the same as a copy assignment.
     */
    BhArray<T> &operator=(BhArray<T> &&other) {
        bhxx::identity(*this, other);
        other.reset();
        return *this;
    }

    /**  Copy the scalar of `scalar_value` into the array using a Bohrium `identity` operation */
    template<typename InType,
            typename std::enable_if<type_traits::is_arithmetic<InType>::value, int>::type = 0>
    BhArray<T> &operator=(const InType &scalar_value) {
        bhxx::identity(*this, scalar_value);
        return *this;
    }

    /** Reset the array to `ary` */
    void reset(BhArray<T> ary) noexcept {
        swap(*this, ary);
    }

    /** Reset the array by cleaning all meta data and leave the array uninitialized. */
    void reset() noexcept {
        reset(BhArray());
    }

    /** Return the rank of the BhArray */
    size_t rank() const {
        assert(shape().size() == _stride.size());
        return shape().size();
    }

    /** Return the number of elements */
    uint64_t size() const {
        return shape().prod();
    }

    /** Return whether the view is contiguous and row-major */
    bool isContiguous() const;

    /** Is the data referenced by this view's base array already
     *  allocated, i.e. initialised */
    bool isDataInitialised() const { return _base->getDataPtr() != nullptr; }

    /** Obtain the data pointer of the array, not taking ownership of any kind.
     *
     *  \note This pointer might be a nullptr if the data in
     *        the base data is not initialised.
     *
     *  \note No flush is done automatically. The data might be
     *        out of sync with Bohrium.
     */
    T *data() {
        T *ret = static_cast<T *>(_base->getDataPtr());
        if (ret == nullptr) {
            return nullptr;
        } else {
            return _offset + ret;
        }
    }

    /// The const version of `data()`
    const T *data() const {
        const T *ret = static_cast<T *>(_base->getDataPtr());
        if (ret == nullptr) {
            return nullptr;
        } else {
            return _offset + ret;
        }
    }

    /// Pretty printing the content of the array
    void pprint(std::ostream &os) const;

    /// Returns a new view of the `idx` dimension. Negative index counts from the back
    BhArray<T> operator[](int64_t idx) const;

    /// Return a new transposed view
    BhArray<T> transpose() const;

    /// Return a new reshaped view (the array must be contiguous)
    BhArray<T> reshape(Shape shape) const;
};

template<typename T>
std::ostream &operator<<(std::ostream &os, const BhArray<T> &ary) {
    ary.pprint(os);
    return os;
}

}  // namespace bhxx
<|endoftext|>


<commit_before><commit_msg>More careful mps split<commit_after><|endoftext|>


<commit_before>/*!
	\file CacheSimulator.cpp

	\author Sudnya Padalikar <sudnya.padalikar@gatech.edu>
	\date Sunday November 29, 2009
	\brief implementation of a CacheSimulator class to be the child of
	TraceGenerator class for evaluating performance of various STL Containers.
*/

#ifndef TRACE_CACHESIMULATOR_CPP_INCLUDED
#define TRACE_CACHESIMULATOR_CPP_INCLUDED

#include <ocelot/trace/interface/CacheSimulator.h>
#include <hydrazine/implementation/debug.h>
#include <cmath>

#ifdef REPORT_BASE
#undef REPORT_BASE
#endif

#define REPORT_BASE 0

namespace trace
{

	CacheSimulator::CacheSimulator()
	{
		_time=0;

		_missCount=0;
		_hitCount=0;
		_missLatency=0;
		_hitLatency=0;

		_memoryAccess=0;

		writebackTime=50;
		addressWidth=64;
		cacheSize=8192;
		lineSize=64;
		hitTime=1;
		missTime=200;
		ways=1;
	}

	CacheSimulator::~CacheSimulator()
	{

	}

	/*!
		called when a traced kernel is launched to retrieve some parameters
		from the kernel
	*/
	void CacheSimulator::initialize(
		const executive::EmulatedKernel *kernel)
	{
		_cache.resize(cacheSize / (lineSize * ways) );
		for(CacheContainer::iterator i = _cache.begin(); i!= _cache.end(); ++i)
		{
			i->valid = false;
			i->dirty = false;
		}
		//report( "Initializing trace generator for kernel " << kernel->name );
	}

	void CacheSimulator::lookupEntry(int setNumber, int tag, bool accessType)
	{
		if( ! _cache[setNumber].valid) //miss
		{
			++_missCount;

			_missLatency += missTime;
			_time +=missTime;

			_cache[setNumber].valid = true;
			_cache[setNumber].tag = tag;
			if(accessType)
			{
				_cache[setNumber].dirty = true;
			}
		}

		else if (_cache[setNumber].valid) // go ahead and check the tag
		{
			if(_cache[setNumber].tag == tag) //hit
			{
				++_hitCount;
				_hitLatency+=hitTime;
				_time +=hitTime;

				if(accessType)
				{
					_cache[setNumber].dirty = true;//maybe redundant here?
				}

			}

			else
			{
				++_missCount;
				_missLatency += missTime;
				_time +=missTime;
				_cache[setNumber].tag = tag;

				if(accessType) //write miss
				{
					if(_cache[setNumber].dirty)
					{
						_time+=writebackTime;//do we need this if we're over writing?
					}
					_cache[setNumber].dirty = true;
				}

				else //read miss
				{
					//check dirty bit, if set writeback and then load
					if(_cache[setNumber].dirty)
					{
						_time+=writebackTime; //???????????????
					}
				}

			}
		}
	}

	long long unsigned int CacheSimulator::getTag(ir::PTXU64 addressAccessed)
	{
		int tagBits;
		tagBits = ( addressWidth - std::log2( cacheSize / lineSize )
			- std::log2(lineSize) );
		addressAccessed >>= tagBits;
		return addressAccessed;
	}

	int CacheSimulator::findSet(ir::PTXU64 addressAccessed)
	{
		addressAccessed >>= (int)std::log2(lineSize);

		ir::PTXU64 mask = (cacheSize / lineSize) - 1;
		return addressAccessed & mask;
	}

	int CacheSimulator::getOffset(ir::PTXU64 addressAccessed)
	{
		ir::PTXU64 mask = lineSize - 1;
		return addressAccessed & mask;
	}

	bool CacheSimulator::cachelineSplit(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		unsigned int offset;
		offset = getOffset(addressAccessed);
		if(offset+bytesAccessed > lineSize)
		{
			return false;
		}

		return true;
	}

	void CacheSimulator::read(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		long long unsigned int tag;
		int setNumber;
		assertM(cachelineSplit(addressAccessed, bytesAccessed),
			" Access accross split cache lines is not supported at this time.");
		//get tags, which set to look etc
		tag = getTag(addressAccessed);
		setNumber = findSet(addressAccessed);

		lookupEntry(setNumber, tag, false);

	}

	void CacheSimulator::write(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		long long unsigned int tag;
		int setNumber;
		assertM(cachelineSplit(addressAccessed, bytesAccessed),
			" Access accross split cache lines is not supported at this time.");

		//get tags, which set to look etc
		tag = getTag(addressAccessed);
		setNumber = findSet(addressAccessed);

		lookupEntry(setNumber, tag, true);

	}

	/*!
		called whenever an event takes place
	*/
	void CacheSimulator::event(const TraceEvent & event)
	{
		if(!event.memory_addresses.empty())
		{
			++_memoryAccess;

			if(event.instruction->opcode == ir::PTXInstruction::St) //write
			{
				TraceEvent::U32Vector::const_iterator
					size = event.memory_sizes.begin();
				for(TraceEvent::U64Vector::const_iterator
					i = event.memory_addresses.begin();
					i != event.memory_addresses.end()
					&& size != event.memory_sizes.end(); ++i, ++size)
				{
					write(*i, *size);
				}
			}
			else
			{
				TraceEvent::U32Vector::const_iterator
					size = event.memory_sizes.begin();
				for(TraceEvent::U64Vector::const_iterator
					i = event.memory_addresses.begin();
					i != event.memory_addresses.end()
					&& size != event.memory_sizes.end(); ++i, ++size)
				{
					read(*i, *size);
				}
			}
		}
		//report( "Default Trace: " << event.toString() );
	}

	void CacheSimulator::finish()
	{
		_cache.clear();
		std::cout << "Miss Rate: " << (_missCount+0.0)/_memoryAccess << "\n";
		std::cout << "Miss Latency: " << _missLatency<< "\n";
		std::cout << "Hit Latency: " << _hitLatency<< "\n";
		std::cout << "Total time for memory accesses: " << _time<< "\n";

		//a) Miss Rate (total_misses/total_accesses)
     //  b) Miss latency (total number of cycles servicing misses)
      // c) Hit latency (total number of cycles servicing hits)
	}

}

#endif

<commit_msg>Fixed the memory access count in CacheSimulator<commit_after>/*!
	\file CacheSimulator.cpp

	\author Sudnya Padalikar <sudnya.padalikar@gatech.edu>
	\date Sunday November 29, 2009
	\brief implementation of a CacheSimulator class to be the child of
	TraceGenerator class for evaluating performance of various STL Containers.
*/

#ifndef TRACE_CACHESIMULATOR_CPP_INCLUDED
#define TRACE_CACHESIMULATOR_CPP_INCLUDED

#include <ocelot/trace/interface/CacheSimulator.h>
#include <hydrazine/implementation/debug.h>
#include <cmath>

#ifdef REPORT_BASE
#undef REPORT_BASE
#endif

#define REPORT_BASE 0

namespace trace
{

	CacheSimulator::CacheSimulator()
	{

		writebackTime=50;
		addressWidth=64;
		cacheSize=8192;
		lineSize=64;
		hitTime=1;
		missTime=200;
		ways=1;
	}

	CacheSimulator::~CacheSimulator()
	{

	}

	/*!
		called when a traced kernel is launched to retrieve some parameters
		from the kernel
	*/
	void CacheSimulator::initialize(
		const executive::EmulatedKernel *kernel)
	{
		_time=0;

		_missCount=0;
		_hitCount=0;
		_missLatency=0;
		_hitLatency=0;

		_memoryAccess=0;

		_cache.resize(cacheSize / (lineSize * ways) );
		for(CacheContainer::iterator i = _cache.begin(); i!= _cache.end(); ++i)
		{
			i->valid = false;
			i->dirty = false;
		}
		//report( "Initializing trace generator for kernel " << kernel->name );
	}

	void CacheSimulator::lookupEntry(int setNumber, int tag, bool accessType)
	{
		if( ! _cache[setNumber].valid) //miss
		{
			++_missCount;

			_missLatency += missTime;
			_time +=missTime;

			_cache[setNumber].valid = true;
			_cache[setNumber].tag = tag;
			if(accessType)
			{
				_cache[setNumber].dirty = true;
			}
		}

		else if (_cache[setNumber].valid) // go ahead and check the tag
		{
			if(_cache[setNumber].tag == tag) //hit
			{
				++_hitCount;
				_hitLatency+=hitTime;
				_time +=hitTime;

				if(accessType)
				{
					_cache[setNumber].dirty = true;//maybe redundant here?
				}

			}

			else
			{
				++_missCount;
				_missLatency += missTime;
				_time +=missTime;
				_cache[setNumber].tag = tag;

				if(accessType) //write miss
				{
					if(_cache[setNumber].dirty)
					{
						_time+=writebackTime;//do we need this if we're over writing?
					}
					_cache[setNumber].dirty = true;
				}

				else //read miss
				{
					//check dirty bit, if set writeback and then load
					if(_cache[setNumber].dirty)
					{
						_time+=writebackTime; //???????????????
					}
				}

			}
		}
	}

	long long unsigned int CacheSimulator::getTag(ir::PTXU64 addressAccessed)
	{
		int shiftAmount = std::log2( cacheSize / lineSize )
			+ std::log2(lineSize);
		addressAccessed >>= shiftAmount;
		return addressAccessed;
	}

	int CacheSimulator::findSet(ir::PTXU64 addressAccessed)
	{
		addressAccessed >>= (int)std::log2(lineSize);

		ir::PTXU64 mask = (cacheSize / lineSize) - 1;
		return addressAccessed & mask;
	}

	int CacheSimulator::getOffset(ir::PTXU64 addressAccessed)
	{
		ir::PTXU64 mask = lineSize - 1;
		return addressAccessed & mask;
	}

	bool CacheSimulator::cachelineSplit(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		unsigned int offset;
		offset = getOffset(addressAccessed);
		if(offset+bytesAccessed > lineSize)
		{
			return false;
		}

		return true;
	}

	void CacheSimulator::read(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		long long unsigned int tag;
		int setNumber;
		assertM(cachelineSplit(addressAccessed, bytesAccessed),
			" Access accross split cache lines is not supported at this time.");
		//get tags, which set to look etc
		tag = getTag(addressAccessed);
		setNumber = findSet(addressAccessed);

		lookupEntry(setNumber, tag, false);

	}

	void CacheSimulator::write(ir::PTXU64 addressAccessed,
		ir::PTXU32 bytesAccessed)
	{
		long long unsigned int tag;
		int setNumber;
		assertM(cachelineSplit(addressAccessed, bytesAccessed),
			" Access accross split cache lines is not supported at this time.");

		//get tags, which set to look etc
		tag = getTag(addressAccessed);
		setNumber = findSet(addressAccessed);

		lookupEntry(setNumber, tag, true);

	}

	/*!
		called whenever an event takes place
	*/
	void CacheSimulator::event(const TraceEvent & event)
	{
		if(!event.memory_addresses.empty())
		{
			_memoryAccess+=event.memory_addresses.size();

			if(event.instruction->opcode == ir::PTXInstruction::St) //write
			{
				TraceEvent::U32Vector::const_iterator
					size = event.memory_sizes.begin();
				for(TraceEvent::U64Vector::const_iterator
					i = event.memory_addresses.begin();
					i != event.memory_addresses.end()
					&& size != event.memory_sizes.end(); ++i, ++size)
				{
					write(*i, *size);
				}
			}
			else
			{
				TraceEvent::U32Vector::const_iterator
					size = event.memory_sizes.begin();
				for(TraceEvent::U64Vector::const_iterator
					i = event.memory_addresses.begin();
					i != event.memory_addresses.end()
					&& size != event.memory_sizes.end(); ++i, ++size)
				{
					read(*i, *size);
				}
			}
		}
		//report( "Default Trace: " << event.toString() );
	}

	void CacheSimulator::finish()
	{
		_cache.clear();
		std::cout << "Miss Rate: " << (_missCount+0.0)/_memoryAccess << "\n";
		std::cout << "Miss Latency: " << _missLatency<< "\n";
		std::cout << "Hit Latency: " << _hitLatency<< "\n";
		std::cout << "Total time for memory accesses: " << _time<< "\n";

		//a) Miss Rate (total_misses/total_accesses)
     //  b) Miss latency (total number of cycles servicing misses)
      // c) Hit latency (total number of cycles servicing hits)
	}

}

#endif

<|endoftext|>


<commit_before>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*
 * Copyright (C) 2016 ScyllaDB.
 */
#include <vector>
#include <algorithm>

#include <seastar/core/do_with.hh>
#include <seastar/testing/test_case.hh>
#include <seastar/core/sstring.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/do_with.hh>
#include <seastar/core/future-util.hh>
#include <seastar/net/dns.hh>
#include <seastar/net/inet_address.hh>

using namespace seastar;
using namespace seastar::net;

static const inet_address google_addr = inet_address("216.58.201.164");
static const sstring google_name = "www.google.com";

static future<> test_resolve(dns_resolver::options opts) {
    auto d = ::make_lw_shared<dns_resolver>(std::move(opts));
    return d->get_host_by_name(google_name, inet_address::family::INET).then([d](hostent e) {
        //BOOST_REQUIRE(std::count(e.addr_list.begin(), e.addr_list.end(), google_addr));
        return d->get_host_by_addr(e.addr_list.front()).then([d, a = e.addr_list.front()](hostent e) {
            return d->get_host_by_name(e.names.front(), inet_address::family::INET).then([a](hostent e) {
                BOOST_REQUIRE(std::count(e.addr_list.begin(), e.addr_list.end(), a));
            });
        });
    }).finally([d]{
        return d->close();
    });
}

static future<> test_bad_name(dns_resolver::options opts) {
    auto d = ::make_lw_shared<dns_resolver>(std::move(opts));
    return d->get_host_by_name("apa.ninja.gnu", inet_address::family::INET).then_wrapped([d](future<hostent> f) {
        try {
            f.get();
            BOOST_FAIL("should not succeed");
        } catch (...) {
            // ok.
        }
    }).finally([d]{
        return d->close();
    });
}

SEASTAR_TEST_CASE(test_resolve_udp) {
    return test_resolve(dns_resolver::options());
}

SEASTAR_TEST_CASE(test_bad_name_udp) {
    return test_bad_name(dns_resolver::options());
}

SEASTAR_TEST_CASE(test_timeout_udp) {
    dns_resolver::options opts;
    opts.servers = std::vector<inet_address>({ inet_address("1.2.3.4") }); // not a server
    opts.timeout = std::chrono::milliseconds(500);

    auto d = ::make_lw_shared<dns_resolver>(engine().net(), opts);
    return d->get_host_by_name(google_name, inet_address::family::INET).then_wrapped([d](future<hostent> f) {
        try {
            f.get();
            BOOST_FAIL("should not succeed");
        } catch (...) {
            // ok.
        }
    }).finally([d]{
        return d->close();
    });
}

// Currently failing, disable until fixed (#521)
#if 0
SEASTAR_TEST_CASE(test_resolve_tcp) {
    dns_resolver::options opts;
    opts.use_tcp_query = true;
    return test_resolve(opts);
}
#endif

SEASTAR_TEST_CASE(test_bad_name_tcp) {
    dns_resolver::options opts;
    opts.use_tcp_query = true;
    return test_bad_name(opts);
}

static const sstring imaps_service = "imaps";
static const sstring gmail_domain = "gmail.com";

static future<> test_srv() {
    auto d = ::make_lw_shared<dns_resolver>();
    return d->get_srv_records(dns_resolver::srv_proto::tcp,
                              imaps_service,
                              gmail_domain).then([d](dns_resolver::srv_records records) {
        BOOST_REQUIRE(!records.empty());
        for (auto& record : records) {
            // record.target should end with "gmail.com"
            BOOST_REQUIRE_GT(record.target.size(), gmail_domain.size());
            BOOST_REQUIRE_EQUAL(record.target.compare(record.target.size() - gmail_domain.size(),
                                                      gmail_domain.size(),
                                                      gmail_domain),
                                0);
        }
    }).finally([d]{
        return d->close();
    });
}

SEASTAR_TEST_CASE(test_srv_tcp) {
    return test_srv();
}
<commit_msg>tests: dns: protect against home routers that grab and respond to all dns requests<commit_after>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*
 * Copyright (C) 2016 ScyllaDB.
 */
#include <vector>
#include <algorithm>

#include <seastar/core/do_with.hh>
#include <seastar/testing/test_case.hh>
#include <seastar/core/sstring.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/do_with.hh>
#include <seastar/core/future-util.hh>
#include <seastar/net/dns.hh>
#include <seastar/net/inet_address.hh>

using namespace seastar;
using namespace seastar::net;

static const inet_address google_addr = inet_address("216.58.201.164");
static const sstring google_name = "www.google.com";

static future<> test_resolve(dns_resolver::options opts) {
    auto d = ::make_lw_shared<dns_resolver>(std::move(opts));
    return d->get_host_by_name(google_name, inet_address::family::INET).then([d](hostent e) {
        //BOOST_REQUIRE(std::count(e.addr_list.begin(), e.addr_list.end(), google_addr));
        return d->get_host_by_addr(e.addr_list.front()).then([d, a = e.addr_list.front()](hostent e) {
            return d->get_host_by_name(e.names.front(), inet_address::family::INET).then([a](hostent e) {
                BOOST_REQUIRE(std::count(e.addr_list.begin(), e.addr_list.end(), a));
            });
        });
    }).finally([d]{
        return d->close();
    });
}

static future<> test_bad_name(dns_resolver::options opts) {
    auto d = ::make_lw_shared<dns_resolver>(std::move(opts));
    return d->get_host_by_name("apa.ninja.gnu", inet_address::family::INET).then_wrapped([d](future<hostent> f) {
        try {
            f.get();
            BOOST_FAIL("should not succeed");
        } catch (...) {
            // ok.
        }
    }).finally([d]{
        return d->close();
    });
}

SEASTAR_TEST_CASE(test_resolve_udp) {
    return test_resolve(dns_resolver::options());
}

SEASTAR_TEST_CASE(test_bad_name_udp) {
    return test_bad_name(dns_resolver::options());
}

SEASTAR_TEST_CASE(test_timeout_udp) {
    dns_resolver::options opts;
    opts.servers = std::vector<inet_address>({ inet_address("1.2.3.4") }); // not a server
    opts.udp_port = 29953; // not a dns port
    opts.timeout = std::chrono::milliseconds(500);

    auto d = ::make_lw_shared<dns_resolver>(engine().net(), opts);
    return d->get_host_by_name(google_name, inet_address::family::INET).then_wrapped([d](future<hostent> f) {
        try {
            f.get();
            BOOST_FAIL("should not succeed");
        } catch (...) {
            // ok.
        }
    }).finally([d]{
        return d->close();
    });
}

// Currently failing, disable until fixed (#521)
#if 0
SEASTAR_TEST_CASE(test_resolve_tcp) {
    dns_resolver::options opts;
    opts.use_tcp_query = true;
    return test_resolve(opts);
}
#endif

SEASTAR_TEST_CASE(test_bad_name_tcp) {
    dns_resolver::options opts;
    opts.use_tcp_query = true;
    return test_bad_name(opts);
}

static const sstring imaps_service = "imaps";
static const sstring gmail_domain = "gmail.com";

static future<> test_srv() {
    auto d = ::make_lw_shared<dns_resolver>();
    return d->get_srv_records(dns_resolver::srv_proto::tcp,
                              imaps_service,
                              gmail_domain).then([d](dns_resolver::srv_records records) {
        BOOST_REQUIRE(!records.empty());
        for (auto& record : records) {
            // record.target should end with "gmail.com"
            BOOST_REQUIRE_GT(record.target.size(), gmail_domain.size());
            BOOST_REQUIRE_EQUAL(record.target.compare(record.target.size() - gmail_domain.size(),
                                                      gmail_domain.size(),
                                                      gmail_domain),
                                0);
        }
    }).finally([d]{
        return d->close();
    });
}

SEASTAR_TEST_CASE(test_srv_tcp) {
    return test_srv();
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "thrift/lib/cpp/util/kerberos/Krb5CredentialsCacheManager.h"

#include <memory>
#include <set>

#include "folly/stats/BucketedTimeSeries-defs.h"
#include "folly/Memory.h"
#include "folly/ScopeGuard.h"

namespace apache { namespace thrift { namespace krb5 {
using namespace std;

const int Krb5CredentialsCacheManager::MANAGE_THREAD_SLEEP_PERIOD = 60*1000;
const int Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_NUM_BUCKETS = 10;
const int Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_PERIOD = 600;
const int Krb5CredentialsCacheManager::ABOUT_TO_EXPIRE_THRESHOLD = 300;
const int Krb5CredentialsCacheManager::NUM_ELEMENTS_TO_PERSIST_TO_FILE = 10000;

bool serviceCountCompare (
    const pair<string, uint64_t>& i,
    const pair<string, uint64_t>& j) {
  return (i.second > j.second);
}

Krb5CredentialsCacheManager::ServiceTimeSeries::ServiceTimeSeries() :
  timeSeries_(
    Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_NUM_BUCKETS,
    std::chrono::seconds(
      Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_PERIOD)) {
}

void Krb5CredentialsCacheManager::ServiceTimeSeries::bumpCount() {
  WriteLock guard(&serviceTimeseriesLock_);
  time_t now = time(nullptr);
  timeSeries_.addValue(std::chrono::seconds(now), 1);
}

uint64_t Krb5CredentialsCacheManager::ServiceTimeSeries::getCount() {
  ReadLock guard(serviceTimeseriesLock_);
  // Note that we don't have a need to call timeSeries_.update(<time>)
  // here because we don't care to have the exact count at the current
  // time. We're ok with grabbing the count at the last update.
  return timeSeries_.count();
}

Krb5CredentialsCacheManager::Krb5CredentialsCacheManager(
    const string& client)
  : clientString_(client)
  , client_(ctx_.get(), client)
  , stopManageThread_(false) {

  if (client.empty()) {
    client_ = Krb5Principal::snameToPrincipal(ctx_.get(), KRB5_NT_UNKNOWN);
  }

  manageThread_ = std::thread([=] {
    while(true) {
      MutexGuard l(manageThreadMutex_);
      if (stopManageThread_) {
        break;
      }

      // Catch all the exceptions. This thread should never die.
      try {
        auto mem = getCache();
        if (mem == nullptr) {
          try {
            mem = readInCache();
            LOG(INFO) << "Initialized the krb5 credentials cache from file";
          } catch(...) {
            // Failed reading in file cache, probably means it's not there.
            // Just get a new cache.
            mem = kInit();
            LOG(INFO) << "Initialized the krb5 credentials cache via kinit";
          }
        }

        auto lifetime = mem->getLifetime();
        time_t now;
        time(&now);
        bool reached_half_life = (uint64_t) now >
          (lifetime.first + (lifetime.second - lifetime.first) / 2);
        // about_to_expire is true if the cache will expire in 5 minutes,
        // or has already expired
        bool about_to_expire = ((uint64_t) now +
          Krb5CredentialsCacheManager::ABOUT_TO_EXPIRE_THRESHOLD) >
            lifetime.second;

        if (about_to_expire || lifetime.first == 0) {
          mem = kInit();
          importMemoryCache(mem);
          LOG(INFO) << "do kInit because CC is about to expire";
        } else if (reached_half_life) {
          // If we've reached half-life, but not about to expire, it means
          // the cache we just got is still valid, so we can use it while
          // we update the old one.
          if (getCache() == nullptr) {
            importMemoryCache(mem);
          }
          mem = buildRenewedCache();
          importMemoryCache(mem);
          LOG(INFO) << "renewed CC at half-life";
        }

        // If we still haven't persisted here, persist the cache to memory
        if (getCache() == nullptr) {
          importMemoryCache(mem);
        }

        // Persist cache to file
        writeOutCache(
          Krb5CredentialsCacheManager::NUM_ELEMENTS_TO_PERSIST_TO_FILE);
      } catch (const std::runtime_error& e) {
        LOG(ERROR) << "Failure in credential cache thread: " << e.what();
      }

      if (!stopManageThread_) {
        manageThreadCondVar_.wait_for(l, std::chrono::milliseconds(
          Krb5CredentialsCacheManager::MANAGE_THREAD_SLEEP_PERIOD));
      }
    }
  });
}

Krb5CredentialsCacheManager::~Krb5CredentialsCacheManager() {
  stopThread();
}

void Krb5CredentialsCacheManager::stopThread() {
  // Kill the manage thread
  MutexGuard l(manageThreadMutex_);
  if (!stopManageThread_) {
    stopManageThread_ = true;
    manageThreadCondVar_.notify_one();
    l.unlock();
    manageThread_.join();
  }
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::kInit() {
  // Make a new memory cache.
  auto mem = folly::make_unique<Krb5CCache>(
    Krb5CCache::makeNewUnique(ctx_.get(), "MEMORY"));
  // Initialize the new CC
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), mem->get(), client_.get());
  raiseIf(code, "initializing memory ccache");

  // Get default options and keytab objects
  Krb5InitCredsOpt options(ctx_.get());
  Krb5Keytab keytab(ctx_.get());

  // Grab the tgt
  krb5_creds creds;
  code = krb5_get_init_creds_keytab(
    ctx_.get(),
    &creds,
    client_.get(),
    keytab.get(),
    0,
    nullptr,
    options.get());
  raiseIf(code, "Getting new tgt ticket");
  SCOPE_EXIT { krb5_free_cred_contents(ctx_.get(), &creds); };

  code = krb5_cc_store_cred(ctx_.get(), mem->get(), &creds);
  raiseIf(code, "failed storing a credential into the new memory cache");

  return mem;
}

void Krb5CredentialsCacheManager::doTgsReq(
    krb5_principal server, Krb5CCache& cache) {
  // Make input credentials
  krb5_creds in_creds;
  memset(&in_creds, 0, sizeof(in_creds));
  in_creds.client = client_.get();
  in_creds.server = server;

  // Make options
  krb5_flags options;
  memset(&options, 0, sizeof(options));

  // To TGS request
  krb5_creds* out_creds;
  krb5_error_code code = krb5_get_credentials(ctx_.get(), options, cache.get(),
                                              &in_creds, &out_creds);
  raiseIf(code, "tgs request");
  // Drop the out credentials, we don't actually use these.
  krb5_free_cred_contents(ctx_.get(), out_creds);
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::buildRenewedCache() {

  // Read in the pointer to the current cache.
  auto cc_cur = getCache();
  if (cc_cur == nullptr) {
    throw std::runtime_error("Trying to refresh an empty cache");
  }

  // Get a new memory cache and initialize it.
  auto mem = kInit();

  for (auto& creds : *cc_cur) {
    Krb5Principal server(ctx_.get(), std::move(creds.server));
    if (!server.isTgt()) {
      doTgsReq(server.get(), *mem);
    }
  }

  return mem;
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::readInCache() {
  auto mem = folly::make_unique<Krb5CCache>(
    Krb5CCache::makeNewUnique(ctx_.get(), "MEMORY"));
  // Get the default local cache
  Krb5CCache file_cache = Krb5CCache::makeDefault(ctx_.get());
  Krb5Principal client = file_cache.getClientPrincipal();

  // Copy the cache into memory
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), mem->get(), client.get());
  raiseIf(code, "initializing memory ccache");
  code = krb5_cc_copy_creds(ctx_.get(), file_cache.get(), mem->get());
  raiseIf(code, "copying to memory cache");

  auto service_list = mem->getServicePrincipalList();
  for (auto& service : service_list)  {
    incUsedService(folly::to<string>(service));
  }

  return mem;
}

void Krb5CredentialsCacheManager::writeOutCache(size_t limit) {
  auto cc_mem = getCache();
  if (cc_mem == nullptr) {
    throw std::runtime_error("Trying to persist an empty cache");
  }

  // Get a default file cache and empty it out
  Krb5CCache file_cache = Krb5CCache::makeDefault(ctx_.get());
  Krb5Principal client = cc_mem->getClientPrincipal();
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), file_cache.get(), client.get());
  raiseIf(code, "Failed initializing file credentials cache");

  // Put 'limit' number of most frequently used credentials into the
  // top_services set.
  vector<pair<string, uint64_t>> count_vector;
  ReadLock readLock(&serviceCountLock_);
  for (auto& element : serviceCountMap_) {
    count_vector.push_back(pair<string, uint64_t>(
      element.first, element.second.getCount()));
  }
  readLock.reset();
  sort(count_vector.begin(), count_vector.end(), serviceCountCompare);

  std::set<string> top_services;
  int count = 0;
  for (auto& element : count_vector) {
    if (count >= limit) {
      break;
    }
    top_services.insert(element.first);
    count++;
  }

  // Iterate through the cc
  for (auto it = cc_mem->begin(true); it != cc_mem->end(); ++it) {
    krb5_principal borrowed_server = it->server;
    Krb5Principal server(ctx_.get(), std::move(borrowed_server));
    const string princ_string = folly::to<string>(server);
    SCOPE_EXIT { server.release(); };  // give back borrowed_server
    // Always persist config and tgt principals. And only persist
    // top 'limit' services.
    if (!it.isConfigEntry() && !server.isTgt() &&
        top_services.count(princ_string) == 0) {
      continue;
    }

    // Store the cred into a file
    code = krb5_cc_store_cred(ctx_.get(), file_cache.get(), &(*it));
    // Erase from top_services struct so we don't persist the same
    // principal more than once.
    top_services.erase(princ_string);
    raiseIf(code, "Failed storing a credential into a file");
  }
}

void Krb5CredentialsCacheManager::importMemoryCache(
    std::shared_ptr<Krb5CCache> cache) {
  MutexGuard guard(ccLock_);
  ccMemory_ = cache;
  ccMemoryCondVar_.notify_all();
}

std::shared_ptr<Krb5CCache> Krb5CredentialsCacheManager::waitForCache() {
  MutexGuard guard(ccLock_);
  while (ccMemory_ == nullptr) {
    ccMemoryCondVar_.wait(guard);
  }
  return ccMemory_;
}

std::shared_ptr<Krb5CCache> Krb5CredentialsCacheManager::getCache() {
  MutexGuard guard(ccLock_);
  return ccMemory_;
}

void Krb5CredentialsCacheManager::incUsedService(
    const string& service) {
  UpgradeLock readLock(&serviceCountLock_);
  auto found = serviceCountMap_.find(service);
  if (found != serviceCountMap_.end()) {
    // Here we already have the element, so we can just bump the count.
    found->second.bumpCount();
  } else {
    // Here let's upgrade the lock to a write lock and insert a new element
    // into the map.
    WriteLock writeLock(std::move(readLock));
    serviceCountMap_[service].bumpCount();
  }
}

}}}
<commit_msg>Smooth out cache renewal times<commit_after>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "thrift/lib/cpp/util/kerberos/Krb5CredentialsCacheManager.h"

#include <memory>
#include <set>

#include "folly/stats/BucketedTimeSeries-defs.h"
#include "folly/Memory.h"
#include "folly/ScopeGuard.h"

namespace apache { namespace thrift { namespace krb5 {
using namespace std;

const int Krb5CredentialsCacheManager::MANAGE_THREAD_SLEEP_PERIOD = 60*1000;
const int Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_NUM_BUCKETS = 10;
const int Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_PERIOD = 600;
const int Krb5CredentialsCacheManager::ABOUT_TO_EXPIRE_THRESHOLD = 300;
const int Krb5CredentialsCacheManager::NUM_ELEMENTS_TO_PERSIST_TO_FILE = 10000;

bool serviceCountCompare (
    const pair<string, uint64_t>& i,
    const pair<string, uint64_t>& j) {
  return (i.second > j.second);
}

Krb5CredentialsCacheManager::ServiceTimeSeries::ServiceTimeSeries() :
  timeSeries_(
    Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_NUM_BUCKETS,
    std::chrono::seconds(
      Krb5CredentialsCacheManager::SERVICE_HISTOGRAM_PERIOD)) {
}

void Krb5CredentialsCacheManager::ServiceTimeSeries::bumpCount() {
  WriteLock guard(&serviceTimeseriesLock_);
  time_t now = time(nullptr);
  timeSeries_.addValue(std::chrono::seconds(now), 1);
}

uint64_t Krb5CredentialsCacheManager::ServiceTimeSeries::getCount() {
  ReadLock guard(serviceTimeseriesLock_);
  // Note that we don't have a need to call timeSeries_.update(<time>)
  // here because we don't care to have the exact count at the current
  // time. We're ok with grabbing the count at the last update.
  return timeSeries_.count();
}

Krb5CredentialsCacheManager::Krb5CredentialsCacheManager(
    const string& client)
  : clientString_(client)
  , client_(ctx_.get(), client)
  , stopManageThread_(false) {

  if (client.empty()) {
    client_ = Krb5Principal::snameToPrincipal(ctx_.get(), KRB5_NT_UNKNOWN);
  }

  std::string sname = folly::to<string>(client_);
  std::hash<std::string> hash_fn;
  size_t sname_hash = hash_fn(sname);

  manageThread_ = std::thread([=] {
    while(true) {
      MutexGuard l(manageThreadMutex_);
      if (stopManageThread_) {
        break;
      }

      // Catch all the exceptions. This thread should never die.
      try {
        auto mem = getCache();
        if (mem == nullptr) {
          try {
            mem = readInCache();
            LOG(INFO) << "Initialized the krb5 credentials cache from file";
          } catch(...) {
            // Failed reading in file cache, probably means it's not there.
            // Just get a new cache.
            mem = kInit();
            LOG(INFO) << "Initialized the krb5 credentials cache via kinit";
          }
        }

        auto lifetime = mem->getLifetime();
        time_t now;
        time(&now);

        // Set the renew interval to be about 25% of lifetime
        uint64_t quarter_life_time = (lifetime.second - lifetime.first) / 4;
        uint64_t renew_offset = sname_hash % quarter_life_time;
        uint64_t half_life_time =
          (lifetime.first + (lifetime.second - lifetime.first) / 2);

        bool reached_renew_time = (uint64_t) now >
          (half_life_time + renew_offset);
        // about_to_expire is true if the cache will expire in 5 minutes,
        // or has already expired
        bool about_to_expire = ((uint64_t) now +
          Krb5CredentialsCacheManager::ABOUT_TO_EXPIRE_THRESHOLD) >
            lifetime.second;

        if (about_to_expire || lifetime.first == 0) {
          mem = kInit();
          importMemoryCache(mem);
          LOG(INFO) << "do kInit because CC is about to expire";
        } else if (reached_renew_time) {
          // If we've reached half-life, but not about to expire, it means
          // the cache we just got is still valid, so we can use it while
          // we update the old one.
          if (getCache() == nullptr) {
            importMemoryCache(mem);
          }
          mem = buildRenewedCache();
          importMemoryCache(mem);
          LOG(INFO) << "renewed CC at half-life";
        }

        // If we still haven't persisted here, persist the cache to memory
        if (getCache() == nullptr) {
          importMemoryCache(mem);
        }

        // Persist cache to file
        writeOutCache(
          Krb5CredentialsCacheManager::NUM_ELEMENTS_TO_PERSIST_TO_FILE);
      } catch (const std::runtime_error& e) {
        LOG(ERROR) << "Failure in credential cache thread: " << e.what();
      }

      if (!stopManageThread_) {
        manageThreadCondVar_.wait_for(l, std::chrono::milliseconds(
          Krb5CredentialsCacheManager::MANAGE_THREAD_SLEEP_PERIOD));
      }
    }
  });
}

Krb5CredentialsCacheManager::~Krb5CredentialsCacheManager() {
  stopThread();
}

void Krb5CredentialsCacheManager::stopThread() {
  // Kill the manage thread
  MutexGuard l(manageThreadMutex_);
  if (!stopManageThread_) {
    stopManageThread_ = true;
    manageThreadCondVar_.notify_one();
    l.unlock();
    manageThread_.join();
  }
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::kInit() {
  // Make a new memory cache.
  auto mem = folly::make_unique<Krb5CCache>(
    Krb5CCache::makeNewUnique(ctx_.get(), "MEMORY"));
  // Initialize the new CC
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), mem->get(), client_.get());
  raiseIf(code, "initializing memory ccache");

  // Get default options and keytab objects
  Krb5InitCredsOpt options(ctx_.get());
  Krb5Keytab keytab(ctx_.get());

  // Grab the tgt
  krb5_creds creds;
  code = krb5_get_init_creds_keytab(
    ctx_.get(),
    &creds,
    client_.get(),
    keytab.get(),
    0,
    nullptr,
    options.get());
  raiseIf(code, "Getting new tgt ticket");
  SCOPE_EXIT { krb5_free_cred_contents(ctx_.get(), &creds); };

  code = krb5_cc_store_cred(ctx_.get(), mem->get(), &creds);
  raiseIf(code, "failed storing a credential into the new memory cache");

  return mem;
}

void Krb5CredentialsCacheManager::doTgsReq(
    krb5_principal server, Krb5CCache& cache) {
  // Make input credentials
  krb5_creds in_creds;
  memset(&in_creds, 0, sizeof(in_creds));
  in_creds.client = client_.get();
  in_creds.server = server;

  // Make options
  krb5_flags options;
  memset(&options, 0, sizeof(options));

  // To TGS request
  krb5_creds* out_creds;
  krb5_error_code code = krb5_get_credentials(ctx_.get(), options, cache.get(),
                                              &in_creds, &out_creds);
  raiseIf(code, "tgs request");
  // Drop the out credentials, we don't actually use these.
  krb5_free_cred_contents(ctx_.get(), out_creds);
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::buildRenewedCache() {

  // Read in the pointer to the current cache.
  auto cc_cur = getCache();
  if (cc_cur == nullptr) {
    throw std::runtime_error("Trying to refresh an empty cache");
  }

  // Get a new memory cache and initialize it.
  auto mem = kInit();

  for (auto& creds : *cc_cur) {
    Krb5Principal server(ctx_.get(), std::move(creds.server));
    if (!server.isTgt()) {
      doTgsReq(server.get(), *mem);
    }
  }

  return mem;
}

std::unique_ptr<Krb5CCache> Krb5CredentialsCacheManager::readInCache() {
  auto mem = folly::make_unique<Krb5CCache>(
    Krb5CCache::makeNewUnique(ctx_.get(), "MEMORY"));
  // Get the default local cache
  Krb5CCache file_cache = Krb5CCache::makeDefault(ctx_.get());
  Krb5Principal client = file_cache.getClientPrincipal();

  // Copy the cache into memory
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), mem->get(), client.get());
  raiseIf(code, "initializing memory ccache");
  code = krb5_cc_copy_creds(ctx_.get(), file_cache.get(), mem->get());
  raiseIf(code, "copying to memory cache");

  auto service_list = mem->getServicePrincipalList();
  for (auto& service : service_list)  {
    incUsedService(folly::to<string>(service));
  }

  return mem;
}

void Krb5CredentialsCacheManager::writeOutCache(size_t limit) {
  auto cc_mem = getCache();
  if (cc_mem == nullptr) {
    throw std::runtime_error("Trying to persist an empty cache");
  }

  // Get a default file cache and empty it out
  Krb5CCache file_cache = Krb5CCache::makeDefault(ctx_.get());
  Krb5Principal client = cc_mem->getClientPrincipal();
  krb5_error_code code = krb5_cc_initialize(
    ctx_.get(), file_cache.get(), client.get());
  raiseIf(code, "Failed initializing file credentials cache");

  // Put 'limit' number of most frequently used credentials into the
  // top_services set.
  vector<pair<string, uint64_t>> count_vector;
  ReadLock readLock(&serviceCountLock_);
  for (auto& element : serviceCountMap_) {
    count_vector.push_back(pair<string, uint64_t>(
      element.first, element.second.getCount()));
  }
  readLock.reset();
  sort(count_vector.begin(), count_vector.end(), serviceCountCompare);

  std::set<string> top_services;
  int count = 0;
  for (auto& element : count_vector) {
    if (count >= limit) {
      break;
    }
    top_services.insert(element.first);
    count++;
  }

  // Iterate through the cc
  for (auto it = cc_mem->begin(true); it != cc_mem->end(); ++it) {
    krb5_principal borrowed_server = it->server;
    Krb5Principal server(ctx_.get(), std::move(borrowed_server));
    const string princ_string = folly::to<string>(server);
    SCOPE_EXIT { server.release(); };  // give back borrowed_server
    // Always persist config and tgt principals. And only persist
    // top 'limit' services.
    if (!it.isConfigEntry() && !server.isTgt() &&
        top_services.count(princ_string) == 0) {
      continue;
    }

    // Store the cred into a file
    code = krb5_cc_store_cred(ctx_.get(), file_cache.get(), &(*it));
    // Erase from top_services struct so we don't persist the same
    // principal more than once.
    top_services.erase(princ_string);
    raiseIf(code, "Failed storing a credential into a file");
  }
}

void Krb5CredentialsCacheManager::importMemoryCache(
    std::shared_ptr<Krb5CCache> cache) {
  MutexGuard guard(ccLock_);
  ccMemory_ = cache;
  ccMemoryCondVar_.notify_all();
}

std::shared_ptr<Krb5CCache> Krb5CredentialsCacheManager::waitForCache() {
  MutexGuard guard(ccLock_);
  while (ccMemory_ == nullptr) {
    ccMemoryCondVar_.wait(guard);
  }
  return ccMemory_;
}

std::shared_ptr<Krb5CCache> Krb5CredentialsCacheManager::getCache() {
  MutexGuard guard(ccLock_);
  return ccMemory_;
}

void Krb5CredentialsCacheManager::incUsedService(
    const string& service) {
  UpgradeLock readLock(&serviceCountLock_);
  auto found = serviceCountMap_.find(service);
  if (found != serviceCountMap_.end()) {
    // Here we already have the element, so we can just bump the count.
    found->second.bumpCount();
  } else {
    // Here let's upgrade the lock to a write lock and insert a new element
    // into the map.
    WriteLock writeLock(std::move(readLock));
    serviceCountMap_[service].bumpCount();
  }
}

}}}
<|endoftext|>


<commit_before>/**
 * @copyright
 * ====================================================================
 * Copyright (c) 2007 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 *
 * This software consists of voluntary contributions made by many
 * individuals.  For exact contribution history, see the revision
 * history and logs, available at http://subversion.tigris.org/.
 * ====================================================================
 * @endcopyright
 *
 * @file LogMessageCallback.cpp
 * @brief Implementation of the class LogMessageCallback
 */

#include "LogMessageCallback.h"
#include "JNIUtil.h"
#include "svn_time.h"
#include "svn_sorts.h"

/**
 * Create a LogMessageCallback object
 * @param jcallback the Java callback object.
 */
LogMessageCallback::LogMessageCallback(jobject jcallback)
{
    m_callback = jcallback;
}

/**
 * Destroy a LogMessageCallback object
 */
LogMessageCallback::~LogMessageCallback()
{
    // The m_callback does not need to be destroyed because it is the
    // passed in parameter to the Java SVNClientInterface.logMessages
    // method.
}

svn_error_t *
LogMessageCallback::callback(void *baton, apr_hash_t *changed_paths,
                             svn_revnum_t rev, const char *author,
                             const char *date, const char *msg,
                             apr_pool_t *pool)
{
    if (baton)
        return ((LogMessageCallback *)baton)->singleMessage(changed_paths, rev,
                                                            author, date, msg,
                                                            pool);

    return SVN_NO_ERROR;
}

/**
 * Callback called for a single log message
 * @param changed_paths a hash containing the paths that were changed in
 *                      the commit
 * @param revision      the revision number for the commit
 * @param author        the author of the commit
 * @param date          the date of the commit
 * @param msg           the log message for the commit
 * @param pool          memory pool for the use of this function
 */
svn_error_t *
LogMessageCallback::singleMessage(apr_hash_t *changed_paths,
                                  svn_revnum_t rev,
                                  const char *author, const char *date,
                                  const char *msg, apr_pool_t *pool)
{
    JNIEnv *env = JNIUtil::getEnv();

    // The method id will not change during the time this library is
    // loaded, so it can be cached.
    static jmethodID sm_mid = 0;
    if (sm_mid == 0)
    {
        jclass clazz = env->FindClass(JAVA_PACKAGE"/LogMessageCallback");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
        sm_mid = env->GetMethodID(clazz, "singleMessage",
            "([L"JAVA_PACKAGE"/ChangePath;JLjava/lang/String;"
            "Ljava/util/Date;Ljava/lang/String;)V");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
        env->DeleteLocalRef(clazz);
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jclass clazzCP = env->FindClass(JAVA_PACKAGE"/ChangePath");
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    static jmethodID midCP = 0;
    if (midCP == 0)
    {
        midCP = env->GetMethodID(clazzCP, "<init>",
            "(Ljava/lang/String;JLjava/lang/String;C)V");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jobject jdate = NULL;
    if (date != NULL && *date != '\0')
    {
        apr_time_t timeTemp;

        SVN_ERR(svn_time_from_cstring(&timeTemp, date, pool));
        jdate = JNIUtil::createDate(timeTemp);
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jstring jauthor = JNIUtil::makeJString(author);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    jobjectArray jChangedPaths = NULL;
    if (changed_paths)
    {
        apr_array_header_t *sorted_paths;
        int i;

        /* Get an array of sorted hash keys. */
        sorted_paths = svn_sort__hash(changed_paths,
                                      svn_sort_compare_items_as_paths,
                                      pool);

        jChangedPaths = env->NewObjectArray(sorted_paths->nelts,
                                            clazzCP,
                                            NULL);

        for (i = 0; i < sorted_paths->nelts; i++)
        {
            svn_sort__item_t *item = &(APR_ARRAY_IDX(sorted_paths, i,
                                                     svn_sort__item_t));
            const char *path = (const char *)item->key;
            svn_log_changed_path_t *log_item
                = (svn_log_changed_path_t *)
                    apr_hash_get(changed_paths, item->key, item->klen);

            jstring jpath = JNIUtil::makeJString(path);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            jstring jcopyFromPath =
                JNIUtil::makeJString(log_item->copyfrom_path);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            jlong jcopyFromRev = log_item->copyfrom_rev;
            jchar jaction = log_item->action;

            jobject cp = env->NewObject(clazzCP, midCP, jpath, jcopyFromRev,
                                        jcopyFromPath, jaction);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }

            env->SetObjectArrayElement(jChangedPaths, i, cp);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }

            env->DeleteLocalRef(cp);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            env->DeleteLocalRef(jpath);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            env->DeleteLocalRef(jcopyFromPath);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
        }
    }

    jstring jmessage = JNIUtil::makeJString(msg);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    env->CallVoidMethod(m_callback, sm_mid, jChangedPaths, (jlong)rev, jauthor,
                        jdate, jmessage);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    env->DeleteLocalRef(jauthor);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }
    env->DeleteLocalRef(jmessage);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    return SVN_NO_ERROR;
}
<commit_msg>* subversion/bindings/javahl/native/LogMessageCallback.cpp:   Formatting cleanup.<commit_after>/**
 * @copyright
 * ====================================================================
 * Copyright (c) 2007 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 *
 * This software consists of voluntary contributions made by many
 * individuals.  For exact contribution history, see the revision
 * history and logs, available at http://subversion.tigris.org/.
 * ====================================================================
 * @endcopyright
 *
 * @file LogMessageCallback.cpp
 * @brief Implementation of the class LogMessageCallback
 */

#include "LogMessageCallback.h"
#include "JNIUtil.h"
#include "svn_time.h"
#include "svn_sorts.h"

/**
 * Create a LogMessageCallback object
 * @param jcallback the Java callback object.
 */
LogMessageCallback::LogMessageCallback(jobject jcallback)
{
    m_callback = jcallback;
}

/**
 * Destroy a LogMessageCallback object
 */
LogMessageCallback::~LogMessageCallback()
{
    // The m_callback does not need to be destroyed because it is the
    // passed in parameter to the Java SVNClientInterface.logMessages
    // method.
}

svn_error_t *
LogMessageCallback::callback(void *baton, apr_hash_t *changed_paths,
                             svn_revnum_t rev, const char *author,
                             const char *date, const char *msg,
                             apr_pool_t *pool)
{
    if (baton)
        return ((LogMessageCallback *)baton)->singleMessage(changed_paths, rev,
                                                            author, date, msg,
                                                            pool);

    return SVN_NO_ERROR;
}

/**
 * Callback called for a single log message
 * @param changed_paths a hash containing the paths that were changed in
 *                      the commit
 * @param revision      the revision number for the commit
 * @param author        the author of the commit
 * @param date          the date of the commit
 * @param msg           the log message for the commit
 * @param pool          memory pool for the use of this function
 */
svn_error_t *
LogMessageCallback::singleMessage(apr_hash_t *changed_paths,
                                  svn_revnum_t rev,
                                  const char *author, const char *date,
                                  const char *msg, apr_pool_t *pool)
{
    JNIEnv *env = JNIUtil::getEnv();

    // The method id will not change during the time this library is
    // loaded, so it can be cached.
    static jmethodID sm_mid = 0;
    if (sm_mid == 0)
    {
        jclass clazz = env->FindClass(JAVA_PACKAGE"/LogMessageCallback");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
        sm_mid = env->GetMethodID(clazz,
                                  "singleMessage",
                                  "([L"JAVA_PACKAGE"/ChangePath;"
                                  "JLjava/lang/String;"
                                  "Ljava/util/Date;Ljava/lang/String;)V");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
        env->DeleteLocalRef(clazz);
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jclass clazzCP = env->FindClass(JAVA_PACKAGE"/ChangePath");
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    static jmethodID midCP = 0;
    if (midCP == 0)
    {
        midCP = env->GetMethodID(clazzCP,
                                 "<init>",
                                 "(Ljava/lang/String;JLjava/lang/String;C)V");
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jobject jdate = NULL;
    if (date != NULL && *date != '\0')
    {
        apr_time_t timeTemp;

        SVN_ERR(svn_time_from_cstring(&timeTemp, date, pool));
        jdate = JNIUtil::createDate(timeTemp);
        if (JNIUtil::isJavaExceptionThrown())
        {
           return SVN_NO_ERROR;
        }
    }

    jstring jauthor = JNIUtil::makeJString(author);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    jobjectArray jChangedPaths = NULL;
    if (changed_paths)
    {
        apr_array_header_t *sorted_paths;
        int i;

        /* Get an array of sorted hash keys. */
        sorted_paths = svn_sort__hash(changed_paths,
                                      svn_sort_compare_items_as_paths,
                                      pool);

        jChangedPaths = env->NewObjectArray(sorted_paths->nelts,
                                            clazzCP,
                                            NULL);

        for (i = 0; i < sorted_paths->nelts; i++)
        {
            svn_sort__item_t *item = &(APR_ARRAY_IDX(sorted_paths, i,
                                                     svn_sort__item_t));
            const char *path = (const char *)item->key;
            svn_log_changed_path_t *log_item
                = (svn_log_changed_path_t *)
                    apr_hash_get(changed_paths, item->key, item->klen);

            jstring jpath = JNIUtil::makeJString(path);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            jstring jcopyFromPath =
                JNIUtil::makeJString(log_item->copyfrom_path);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            jlong jcopyFromRev = log_item->copyfrom_rev;
            jchar jaction = log_item->action;

            jobject cp = env->NewObject(clazzCP, midCP, jpath, jcopyFromRev,
                                        jcopyFromPath, jaction);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }

            env->SetObjectArrayElement(jChangedPaths, i, cp);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }

            env->DeleteLocalRef(cp);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            env->DeleteLocalRef(jpath);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
            env->DeleteLocalRef(jcopyFromPath);
            if (JNIUtil::isJavaExceptionThrown())
            {
                return SVN_NO_ERROR;
            }
        }
    }

    jstring jmessage = JNIUtil::makeJString(msg);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    env->CallVoidMethod(m_callback, sm_mid, jChangedPaths, (jlong)rev, jauthor,
                        jdate, jmessage);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    env->DeleteLocalRef(jauthor);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }
    env->DeleteLocalRef(jmessage);
    if (JNIUtil::isJavaExceptionThrown())
    {
       return SVN_NO_ERROR;
    }

    return SVN_NO_ERROR;
}
<|endoftext|>