<commit_before>/**
 *  Eval.cpp
 *
 *  This file holds the implementation for the Php::eval() function
 *
 *  @author andot <https://github.com/andot>
 */

/**
 *  Dependencies
 */
#include "includes.h"

/**
 *  Open PHP namespace
 */
namespace Php {

/**
 *  Evaluate a PHP string
 *  @param  phpCode     The PHP code to evaluate
 *  @return Value       The result of the evaluation
 */
Value eval(const std::string &phpCode)
{
    // we need the tsrm_ls variable
    TSRMLS_FETCH();

    // the current exception
    zval* oldException = EG(exception);

    // the return zval
    zval* retval = nullptr;
    if (zend_eval_stringl_ex((char *)phpCode.c_str(), (int32_t)phpCode.length(), retval, (char *)"", 1 TSRMLS_CC) != SUCCESS)
    {
        // Do we want to throw an exception here? The original author of this code
        // did, but there are some reasons not to:
        //
        //  1. the PHP eval() function also does not throw exceptions.
        //
        //  2. the zend_eval_string() function already triggers a
        //     'PHP parse error' when an error occurs, which also has
        //     to be handled. If we also throw an exception here, the
        //     user will have to write two error checks: for the error
        //     and the exception.
        //
        // if we _do_ want to throw an exception, we will first have to
        // prevent the original zend_error to occur, and then turn it
        // into an exception. An exception would be nicer from a C++
        // point of view, but because of the extra complexity, we do not
        // this for now.
        return nullptr;
    }
    else
    {
        // was an exception thrown inside the eval()'ed code? In that case we
        // throw a C++ new exception to give the C++ code the chance to catch it
        if (oldException != EG(exception) && EG(exception)) throw OrigException(EG(exception) TSRMLS_CC);

        // no (additional) exception was thrown
        return retval ? Value(retval) : nullptr;
    }
}

/**
 *  Include a file
 *  @param  filename
 *  @return Value
 */
Value include(const std::string &filename)
{
    // we can simply execute a file
    return File(filename).execute();
}

/**
 *  Include a file only once
 *  @param  filename
 *  @return Value
 */
Value include_once(const std::string &filename)
{
    // we can simply execute a file
    return File(filename).execute();
}

/**
 *  Require a file
 *  This causes a fatal error if the file does not exist
 *  @param  filename
 *  @return Value
 */
Value require(const std::string &filename)
{
    // create the file
    File file(filename);

    // execute if it exists
    if (file.exists()) return file.execute();

    // trigger fatal error
    error << filename << " does not exist" << std::flush;

    // unreachable
    return nullptr;
}

/**
 *  Require a file only once
 *  This causes a fatal error if the file does not exist
 *  @param  filename
 *  @return Value
 */
Value require_once(const std::string &filename)
{
    // create the file
    File file(filename);

    // execute if it exists
    if (file.exists()) return file.once();

    // trigger fatal error
    error << filename << " does not exist" << std::flush;

    // unreachable
    return nullptr;
}

/**
 *  End of namespace
 */
}
<commit_msg>fixed return value problem in the Php::eval() function (also solved in issue #129)<commit_after>/**
 *  Eval.cpp
 *
 *  This file holds the implementation for the Php::eval() function
 *
 *  @author andot <https://github.com/andot>
 */

/**
 *  Dependencies
 */
#include "includes.h"

/**
 *  Open PHP namespace
 */
namespace Php {

/**
 *  Evaluate a PHP string
 *  @param  phpCode     The PHP code to evaluate
 *  @return Value       The result of the evaluation
 */
Value eval(const std::string &phpCode)
{
    // we need the tsrm_ls variable
    TSRMLS_FETCH();

    // the current exception
    zval* oldException = EG(exception);

    // the return va
    zval *retval = nullptr;
    MAKE_STD_ZVAL(retval);

    // evaluate the string
    if (zend_eval_stringl_ex((char *)phpCode.c_str(), (int32_t)phpCode.length(), retval, (char *)"", 1 TSRMLS_CC) != SUCCESS)
    {
        // Do we want to throw an exception here? The original author of this code
        // did, but there are some reasons not to:
        //
        //  1. the PHP eval() function also does not throw exceptions.
        //
        //  2. the zend_eval_string() function already triggers a
        //     'PHP parse error' when an error occurs, which also has
        //     to be handled. If we also throw an exception here, the
        //     user will have to write two error checks: for the error
        //     and the exception.
        //
        // if we _do_ want to throw an exception, we will first have to
        // prevent the original zend_error to occur, and then turn it
        // into an exception. An exception would be nicer from a C++
        // point of view, but because of the extra complexity, we do not
        // this for now.
        return nullptr;
    }
    else
    {
        // was an exception thrown inside the eval()'ed code? In that case we
        // throw a C++ new exception to give the C++ code the chance to catch it
        if (oldException != EG(exception) && EG(exception)) throw OrigException(EG(exception) TSRMLS_CC);

        // wrap the return value in a value object
        Value result(retval);

        // the retval should now have two references: the value object and the
        // retval itselves, so we can remove one of it (the zval_ptr_dtor only
        // decrements refcount and won't destruct anything because there still
        // is one reference left inside the Value object)
        zval_ptr_dtor(&retval);

        // done
        return result;
    }
}

/**
 *  Include a file
 *  @param  filename
 *  @return Value
 */
Value include(const std::string &filename)
{
    // we can simply execute a file
    return File(filename).execute();
}

/**
 *  Include a file only once
 *  @param  filename
 *  @return Value
 */
Value include_once(const std::string &filename)
{
    // we can simply execute a file
    return File(filename).execute();
}

/**
 *  Require a file
 *  This causes a fatal error if the file does not exist
 *  @param  filename
 *  @return Value
 */
Value require(const std::string &filename)
{
    // create the file
    File file(filename);

    // execute if it exists
    if (file.exists()) return file.execute();

    // trigger fatal error
    error << filename << " does not exist" << std::flush;

    // unreachable
    return nullptr;
}

/**
 *  Require a file only once
 *  This causes a fatal error if the file does not exist
 *  @param  filename
 *  @return Value
 */
Value require_once(const std::string &filename)
{
    // create the file
    File file(filename);

    // execute if it exists
    if (file.exists()) return file.once();

    // trigger fatal error
    error << filename << " does not exist" << std::flush;

    // unreachable
    return nullptr;
}

/**
 *  End of namespace
 */
}
<|endoftext|>


<commit_before>/** \file    apply_differential_update.cc
 *  \brief   A tool for applying a differential update to a complete MARC dump.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2018 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <map>
#include <stdexcept>
#include <unordered_set>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include "unistd.h"
#include "Archive.h"
#include "BSZUtil.h"
#include "Compiler.h"
#include "File.h"
#include "FileUtil.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [--min-log-level=log_level] [--keep-intermediate-files] input_archive "
              << "difference_archive output_archive\n"
              << "       Log levels are DEBUG, INFO, WARNING and ERROR with INFO being the default.\n\n";
    std::exit(EXIT_FAILURE);
}

void CopyAndCollectPPNs(MARC::Reader * const reader, MARC::Writer * const writer,
                        std::unordered_set<std::string> * const previously_seen_ppns)
{
    while (auto record = reader->read()) {
        if (previously_seen_ppns->find(record.getControlNumber()) == previously_seen_ppns->end()) {
            previously_seen_ppns->emplace(record.getControlNumber());
            if (record.getFirstField("ORI") == record.end())
                record.appendField("ORI", FileUtil::GetLastPathComponent(reader->getPath()));
            writer->write(record);
        }
    }
}

void CopySelectedTypes(const std::vector<std::string> &archive_members, MARC::Writer * const writer,
                       const std::set<BSZUtil::ArchiveType> &selected_types, std::unordered_set<std::string> * const previously_seen_ppns)
{
    for (const auto &archive_member : archive_members) {
        if (selected_types.find(BSZUtil::GetArchiveType(archive_member)) != selected_types.cend()) {
            const auto reader(MARC::Reader::Factory(archive_member, MARC::FileType::BINARY));
            CopyAndCollectPPNs(reader.get(), writer, previously_seen_ppns);
        }
    }
}

void PatchArchiveMembersAndCreateOutputArchive(const std::vector<std::string> &input_archive_members,
                                               const std::vector<std::string> &difference_archive_members, const std::string &output_archive)
{
    if (input_archive_members.empty())
        LOG_ERROR("no input archive members!");
    if (difference_archive_members.empty())
        LOG_WARNING("no difference archive members!");

    //
    // We process title data first and combine all inferior and superior records.
    //

    const auto title_writer(MARC::Writer::Factory(output_archive + "/tit.mrc", MARC::FileType::BINARY));
    std::unordered_set<std::string> previously_seen_title_ppns;
    CopySelectedTypes(difference_archive_members, title_writer.get(), { BSZUtil::TITLE_RECORDS, BSZUtil::SUPERIOR_TITLES },
                      &previously_seen_title_ppns);
    CopySelectedTypes(input_archive_members, title_writer.get(), { BSZUtil::TITLE_RECORDS, BSZUtil::SUPERIOR_TITLES },
                      &previously_seen_title_ppns);

    const auto authority_writer(MARC::Writer::Factory(output_archive + "/aut.mrc", MARC::FileType::BINARY));
    std::unordered_set<std::string> previously_seen_authority_ppns;
    CopySelectedTypes(difference_archive_members, authority_writer.get(), { BSZUtil::AUTHORITY_RECORDS },
                      &previously_seen_authority_ppns);
    CopySelectedTypes(input_archive_members, authority_writer.get(), { BSZUtil::AUTHORITY_RECORDS },
                      &previously_seen_authority_ppns);
}

void GetDirectoryContentsWithRelativepath(const std::string &archive_name, std::vector<std::string> * const archive_members) {
    const std::string directory_name(archive_name);
    FileUtil::GetFileNameList(".(raw|mrc)$", archive_members, directory_name);
    for (auto &archive_member : *archive_members)
        archive_member = directory_name + "/" + archive_member;
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 4)
        Usage();

    bool keep_intermediate_files(false);
    if (std::strcmp(argv[1], "--keep-intermediate-files") == 0) {
        keep_intermediate_files = true;
        --argc, ++argv;
    }

    if (argc != 4)
        Usage();

    const std::string input_archive(FileUtil::MakeAbsolutePath(argv[1]));
    const std::string difference_archive(FileUtil::MakeAbsolutePath(argv[2]));
    const std::string output_archive(FileUtil::MakeAbsolutePath(argv[3]));

    if (input_archive == difference_archive or input_archive == output_archive or difference_archive == output_archive)
        LOG_ERROR("all archive names must be distinct!");

    std::unique_ptr<FileUtil::AutoTempDirectory> working_directory;
    Archive::UnpackArchive(difference_archive, difference_archive);
    const auto directory_name(output_archive);
    if (not FileUtil::MakeDirectory(directory_name))
        LOG_ERROR("failed to create directory: \"" + directory_name + "\"!");

    std::vector<std::string> input_archive_members, difference_archive_members;
    GetDirectoryContentsWithRelativepath(input_archive, &input_archive_members);
    GetDirectoryContentsWithRelativepath(difference_archive, &difference_archive_members);

    PatchArchiveMembersAndCreateOutputArchive(input_archive_members, difference_archive_members, output_archive);

    if (not keep_intermediate_files and not FileUtil::RemoveDirectory(difference_archive))
        LOG_ERROR("failed to remove directory: \"" + difference_archive + "\"!");

    return EXIT_SUCCESS;
}
<commit_msg>Re-introduce .tar.gz name handling for the difference archive.<commit_after>/** \file    apply_differential_update.cc
 *  \brief   A tool for applying a differential update to a complete MARC dump.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2018 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <map>
#include <stdexcept>
#include <unordered_set>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include "unistd.h"
#include "Archive.h"
#include "BSZUtil.h"
#include "Compiler.h"
#include "File.h"
#include "FileUtil.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [--min-log-level=log_level] [--keep-intermediate-files] input_archive "
              << "difference_archive output_archive\n"
              << "       Log levels are DEBUG, INFO, WARNING and ERROR with INFO being the default.\n\n";
    std::exit(EXIT_FAILURE);
}

void CopyAndCollectPPNs(MARC::Reader * const reader, MARC::Writer * const writer,
                        std::unordered_set<std::string> * const previously_seen_ppns)
{
    while (auto record = reader->read()) {
        if (previously_seen_ppns->find(record.getControlNumber()) == previously_seen_ppns->end()) {
            previously_seen_ppns->emplace(record.getControlNumber());
            if (record.getFirstField("ORI") == record.end())
                record.appendField("ORI", FileUtil::GetLastPathComponent(reader->getPath()));
            writer->write(record);
        }
    }
}

void CopySelectedTypes(const std::vector<std::string> &archive_members, MARC::Writer * const writer,
                       const std::set<BSZUtil::ArchiveType> &selected_types, std::unordered_set<std::string> * const previously_seen_ppns)
{
    for (const auto &archive_member : archive_members) {
        if (selected_types.find(BSZUtil::GetArchiveType(archive_member)) != selected_types.cend()) {
            const auto reader(MARC::Reader::Factory(archive_member, MARC::FileType::BINARY));
            CopyAndCollectPPNs(reader.get(), writer, previously_seen_ppns);
        }
    }
}

void PatchArchiveMembersAndCreateOutputArchive(const std::vector<std::string> &input_archive_members,
                                               const std::vector<std::string> &difference_archive_members, const std::string &output_archive)
{
    if (input_archive_members.empty())
        LOG_ERROR("no input archive members!");
    if (difference_archive_members.empty())
        LOG_WARNING("no difference archive members!");

    //
    // We process title data first and combine all inferior and superior records.
    //

    const auto title_writer(MARC::Writer::Factory(output_archive + "/tit.mrc", MARC::FileType::BINARY));
    std::unordered_set<std::string> previously_seen_title_ppns;
    CopySelectedTypes(difference_archive_members, title_writer.get(), { BSZUtil::TITLE_RECORDS, BSZUtil::SUPERIOR_TITLES },
                      &previously_seen_title_ppns);
    CopySelectedTypes(input_archive_members, title_writer.get(), { BSZUtil::TITLE_RECORDS, BSZUtil::SUPERIOR_TITLES },
                      &previously_seen_title_ppns);

    const auto authority_writer(MARC::Writer::Factory(output_archive + "/aut.mrc", MARC::FileType::BINARY));
    std::unordered_set<std::string> previously_seen_authority_ppns;
    CopySelectedTypes(difference_archive_members, authority_writer.get(), { BSZUtil::AUTHORITY_RECORDS },
                      &previously_seen_authority_ppns);
    CopySelectedTypes(input_archive_members, authority_writer.get(), { BSZUtil::AUTHORITY_RECORDS },
                      &previously_seen_authority_ppns);
}

void GetDirectoryContentsWithRelativepath(const std::string &archive_name, std::vector<std::string> * const archive_members) {
    const std::string directory_name(archive_name);
    FileUtil::GetFileNameList(".(raw|mrc)$", archive_members, directory_name);
    for (auto &archive_member : *archive_members)
        archive_member = directory_name + "/" + archive_member;
}

std::string RemoveSuffix(const std::string &s, const std::string &suffix) {
    if (unlikely(not StringUtil::EndsWith(s, suffix)))
        LOG_ERROR("\"" + s + "\" does not end w/ \"" + suffix + "\"!");
    return s.substr(0, s.length() - suffix.length());
}

inline std::string StripTarGz(const std::string &archive_filename) {
    return RemoveSuffix(archive_filename, ".tar.gz");
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 4)
        Usage();

    bool keep_intermediate_files(false);
    if (std::strcmp(argv[1], "--keep-intermediate-files") == 0) {
        keep_intermediate_files = true;
        --argc, ++argv;
    }

    if (argc != 4)
        Usage();

    const std::string input_archive(FileUtil::MakeAbsolutePath(argv[1]));
    const std::string difference_archive(FileUtil::MakeAbsolutePath(argv[2]));
    const std::string output_archive(FileUtil::MakeAbsolutePath(argv[3]));

    if (input_archive == difference_archive or input_archive == output_archive or difference_archive == output_archive)
        LOG_ERROR("all archive names must be distinct!");

    std::unique_ptr<FileUtil::AutoTempDirectory> working_directory;
    Archive::UnpackArchive(difference_archive, StripTarGz(difference_archive));
    const auto directory_name(output_archive);
    if (not FileUtil::MakeDirectory(directory_name))
        LOG_ERROR("failed to create directory: \"" + directory_name + "\"!");

    std::vector<std::string> input_archive_members, difference_archive_members;
    GetDirectoryContentsWithRelativepath(input_archive, &input_archive_members);
    GetDirectoryContentsWithRelativepath(StripTarGz(difference_archive), &difference_archive_members);

    PatchArchiveMembersAndCreateOutputArchive(input_archive_members, difference_archive_members, output_archive);

    if (not keep_intermediate_files and not FileUtil::RemoveDirectory(difference_archive))
        LOG_ERROR("failed to remove directory: \"" + difference_archive + "\"!");

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 by Robert Schmidtke, Zuse Institute Berlin
 *
 * Licensed under the BSD License, see LICENSE file for details.
 *
 */

#include <gtest/gtest.h>

#include <boost/algorithm/string/predicate.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/thread.hpp>
#include <fstream>
#include <stdio.h>
#include <string>

#include "libxtreemfs/client.h"
#include "libxtreemfs/client_implementation.h"
#include "libxtreemfs/options.h"
#include "pbrpc/RPC.pb.h"
#include "util/logging.h"

using namespace xtreemfs::pbrpc;
using namespace xtreemfs::util;

namespace xtreemfs {
namespace rpc {

class ExternalService {
public:
  ExternalService(
      std::string config_file_name)
      : config_file_name_(config_file_name) {

    char *java_home = getenv("JAVA_HOME");
    if (java_home == NULL || (java_home_ = strdup(java_home)).empty()) {
      if (Logging::log->loggingActive(LEVEL_WARN)) {
        Logging::log->getLog(LEVEL_WARN) << "JAVA_HOME is empty."
            << std::endl;
      }
    }

    if (!boost::algorithm::ends_with(java_home_, "/")) {
      java_home_ += "/";
    }

    classpath_ = "java/servers/dist/XtreemFS.jar";
    classpath_ += ":java/foundation/dist/Foundation.jar";
    classpath_ += ":java/flease/dist/Flease.jar";
    classpath_ += ":java/lib/*";

    service_pid_ = -1;
  }

  void Start(std::string service_class) {
    char *argv[] = {
      strdup((java_home_ + "bin/java").c_str()),
      strdup("-ea"),
      strdup("-cp"),
      strdup(classpath_.c_str()),
      strdup(service_class.c_str()),
      strdup(config_file_name_.c_str()),
      NULL
    };

    char *envp[] = { NULL };

    service_pid_ = fork();
    if (service_pid_ == 0) {
      // Executed by the child.
      execve((java_home_ + "bin/java").c_str(), argv, envp);
      exit(EXIT_SUCCESS);
    }
  }

  void Shutdown() {
    if (service_pid_ > 0) {
      // Executed by the parent.
      kill(service_pid_, 2);
      waitpid(service_pid_, NULL, 0);
      service_pid_ = -1;
    }
  }

private:
  std::string config_file_name_;
  std::string java_home_;
  std::string classpath_;

  pid_t service_pid_;
};

class ExternalDIR : public ExternalService {
public:
  ExternalDIR(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.dir.DIR");
  }
};

class ExternalMRC : public ExternalService {
public:
  ExternalMRC(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.mrc.MRC");
  }
};

class ExternalOSD : public ExternalService {
public:
  ExternalOSD(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.osd.OSD");
  }
};

class ClientTest : public ::testing::Test {
protected:
  virtual void SetUp() {
    initialize_logger(options_.log_level_string,
                      options_.log_file_path,
                      LEVEL_WARN);

    external_dir_.reset(new ExternalDIR(dir_config_file_));
    external_dir_->Start();
    external_mrc_.reset(new ExternalMRC(mrc_config_file_));
    external_mrc_->Start();
    external_osd_.reset(new ExternalOSD(osd_config_file_));
    external_osd_->Start();

    auth_.set_auth_type(AUTH_NONE);
    user_credentials_.set_username("client_ssl_test");
    user_credentials_.add_groups("client_ssl_test");

    options_.retry_delay_s = 5;

    mrc_url_.ParseURL(kMRC);
    dir_url_.ParseURL(kDIR);
    client_.reset(xtreemfs::Client::CreateClient(dir_url_.service_addresses,
                                                 user_credentials_,
                                                 options_.GenerateSSLOptions(),
                                                 options_));

    client_->Start();
  }

  virtual void TearDown() {
    client_->Shutdown();

    external_osd_->Shutdown();
    external_mrc_->Shutdown();
    external_dir_->Shutdown();
  }

  void CreateOpenDeleteVolume(std::string volume_name) {
    client_->CreateVolume(mrc_url_.service_addresses,
                        auth_,
                        user_credentials_,
                        volume_name);
    client_->OpenVolume(volume_name,
                        options_.GenerateSSLOptions(),
                        options_);
    client_->DeleteVolume(mrc_url_.service_addresses,
                          auth_,
                          user_credentials_,
                          volume_name);
  }

  size_t count_occurrences_in_file(std::string file_path, std::string s) {
    std::ifstream in(file_path.c_str(), std::ios_base::in);
    size_t occurences = 0;
    while (!in.eof()) {
      std::string line;
      std::getline(in, line);
      occurences += line.find(s) == std::string::npos ? 0 : 1;
    }
    in.close();
    return occurences;
  }

  boost::scoped_ptr<ExternalDIR> external_dir_;
  boost::scoped_ptr<ExternalMRC> external_mrc_;
  boost::scoped_ptr<ExternalOSD> external_osd_;

  boost::scoped_ptr<xtreemfs::Client> client_;
  xtreemfs::Options options_;

  xtreemfs::Options dir_url_;
  xtreemfs::Options mrc_url_;

  std::string dir_config_file_;
  std::string mrc_config_file_;
  std::string osd_config_file_;

  xtreemfs::pbrpc::Auth auth_;
  xtreemfs::pbrpc::UserCredentials user_credentials_;
};

class ClientNoSSLTest : public ClientTest {
protected:
  virtual void SetUp() {
    dir_config_file_ = "tests/configs/dirconfig_nossl.test";
    mrc_config_file_ = "tests/configs/mrcconfig_nossl.test";
    osd_config_file_ = "tests/configs/osdconfig_nossl.test";

    dir_url_.xtreemfs_url = "pbrpc://localhost:42638/";
    mrc_url_.xtreemfs_url = "pbrpc://localhost:42636/";

    options_.log_level_string = "DEBUG";
    options_.log_file_path = "/tmp/xtreemfs_client_ssl_test_no_ssl";

    ClientTest::SetUp();
  }

  virtual void TearDown() {
    ClientTest::TearDown();
    unlink(options_.log_file_path.c_str());
  }
};

class ClientSSLTest : public ClientTest {
protected:
  virtual void SetUp() {
    // All service certificates are signed with Leaf CA, which is signed with
    // Intermediate CA, which is signed with Root CA. The keystore contains
    // only the Leaf CA.
    dir_config_file_ = "tests/configs/dirconfig_ssl.test";
    mrc_config_file_ = "tests/configs/mrcconfig_ssl.test";
    osd_config_file_ = "tests/configs/osdconfig_ssl.test";

    dir_url_.xtreemfs_url = "pbrpcs://localhost:42638/";
    mrc_url_.xtreemfs_url = "pbrpcs://localhost:42636/";

    options_.log_level_string = "DEBUG";
    options_.log_file_path = "/tmp/xtreemfs_client_ssl_test_with_chain";

    // Client certificate is signed with Leaf CA. Contains the entire chain
    // as additional certificates.
    options_.ssl_pkcs12_path = "tests/certs/client_ssl_test/Client_Leaf_Chain.p12";
    options_.ssl_verify_certificates = true;

    ClientTest::SetUp();
  }

  virtual void TearDown() {
    ClientTest::TearDown();
    unlink(options_.log_file_path.c_str());
  }
};

TEST_F(ClientNoSSLTest, TestNoSSL) {
  CreateOpenDeleteVolume("test_no_ssl");
  ASSERT_EQ(0, count_occurrences_in_file(options_.log_file_path, "SSL"));
}

TEST_F(ClientSSLTest, TestSSLWithChain) {
  CreateOpenDeleteVolume("test_ssl");

  // Once for MRC and once for DIR.
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support activated"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support using PKCS#12 file "
      "tests/certs/client_ssl_test/Client_Leaf_Chain.p12"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Writing 3 verification certificates to /tmp/ca"));

  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Root CA' "
      "was successful."));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Intermediate "
      "CA' was successful."));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Leaf CA' was "
      "successful."));

  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=MRC (Leaf)' "
      "was successful"));
  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=DIR (Leaf)' "
      "was successful."));
}

}  // namespace rpc
}  // namespace xtreemfs
<commit_msg>cpp: libxtreemfs: added short ssl chain verificationt test<commit_after>/*
 * Copyright (c) 2014 by Robert Schmidtke, Zuse Institute Berlin
 *
 * Licensed under the BSD License, see LICENSE file for details.
 *
 */

#include <gtest/gtest.h>

#include <boost/algorithm/string/predicate.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/thread.hpp>
#include <fstream>
#include <stdio.h>
#include <string>

#include "libxtreemfs/client.h"
#include "libxtreemfs/client_implementation.h"
#include "libxtreemfs/options.h"
#include "pbrpc/RPC.pb.h"
#include "util/logging.h"

using namespace xtreemfs::pbrpc;
using namespace xtreemfs::util;

namespace xtreemfs {
namespace rpc {

class ExternalService {
public:
  ExternalService(
      std::string config_file_name)
      : config_file_name_(config_file_name) {

    char *java_home = getenv("JAVA_HOME");
    if (java_home == NULL || (java_home_ = strdup(java_home)).empty()) {
      if (Logging::log->loggingActive(LEVEL_WARN)) {
        Logging::log->getLog(LEVEL_WARN) << "JAVA_HOME is empty."
            << std::endl;
      }
    }

    if (!boost::algorithm::ends_with(java_home_, "/")) {
      java_home_ += "/";
    }

    classpath_ = "java/servers/dist/XtreemFS.jar";
    classpath_ += ":java/foundation/dist/Foundation.jar";
    classpath_ += ":java/flease/dist/Flease.jar";
    classpath_ += ":java/lib/*";

    service_pid_ = -1;
  }

  void Start(std::string service_class) {
    char *argv[] = {
      strdup((java_home_ + "bin/java").c_str()),
      strdup("-ea"),
      strdup("-cp"),
      strdup(classpath_.c_str()),
      strdup(service_class.c_str()),
      strdup(config_file_name_.c_str()),
      NULL
    };

    char *envp[] = { NULL };

    service_pid_ = fork();
    if (service_pid_ == 0) {
      // Executed by the child.
      execve((java_home_ + "bin/java").c_str(), argv, envp);
      exit(EXIT_SUCCESS);
    }
  }

  void Shutdown() {
    if (service_pid_ > 0) {
      // Executed by the parent.
      kill(service_pid_, 2);
      waitpid(service_pid_, NULL, 0);
      service_pid_ = -1;
    }
  }

private:
  std::string config_file_name_;
  std::string java_home_;
  std::string classpath_;

  pid_t service_pid_;
};

class ExternalDIR : public ExternalService {
public:
  ExternalDIR(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.dir.DIR");
  }
};

class ExternalMRC : public ExternalService {
public:
  ExternalMRC(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.mrc.MRC");
  }
};

class ExternalOSD : public ExternalService {
public:
  ExternalOSD(std::string config_file_name)
  : ExternalService(config_file_name) {}

  void Start() {
    ExternalService::Start("org.xtreemfs.osd.OSD");
  }
};

class ClientTest : public ::testing::Test {
protected:
  virtual void SetUp() {
    initialize_logger(options_.log_level_string,
                      options_.log_file_path,
                      LEVEL_WARN);

    external_dir_.reset(new ExternalDIR(dir_config_file_));
    external_dir_->Start();
    external_mrc_.reset(new ExternalMRC(mrc_config_file_));
    external_mrc_->Start();
    external_osd_.reset(new ExternalOSD(osd_config_file_));
    external_osd_->Start();

    auth_.set_auth_type(AUTH_NONE);
    user_credentials_.set_username("client_ssl_test");
    user_credentials_.add_groups("client_ssl_test");

    options_.retry_delay_s = 5;

    mrc_url_.ParseURL(kMRC);
    dir_url_.ParseURL(kDIR);
    client_.reset(xtreemfs::Client::CreateClient(dir_url_.service_addresses,
                                                 user_credentials_,
                                                 options_.GenerateSSLOptions(),
                                                 options_));

    client_->Start();
  }

  virtual void TearDown() {
    client_->Shutdown();

    external_osd_->Shutdown();
    external_mrc_->Shutdown();
    external_dir_->Shutdown();
  }

  void CreateOpenDeleteVolume(std::string volume_name) {
    client_->CreateVolume(mrc_url_.service_addresses,
                        auth_,
                        user_credentials_,
                        volume_name);
    client_->OpenVolume(volume_name,
                        options_.GenerateSSLOptions(),
                        options_);
    client_->DeleteVolume(mrc_url_.service_addresses,
                          auth_,
                          user_credentials_,
                          volume_name);
  }

  size_t count_occurrences_in_file(std::string file_path, std::string s) {
    std::ifstream in(file_path.c_str(), std::ios_base::in);
    size_t occurences = 0;
    while (!in.eof()) {
      std::string line;
      std::getline(in, line);
      occurences += line.find(s) == std::string::npos ? 0 : 1;
    }
    in.close();
    return occurences;
  }

  boost::scoped_ptr<ExternalDIR> external_dir_;
  boost::scoped_ptr<ExternalMRC> external_mrc_;
  boost::scoped_ptr<ExternalOSD> external_osd_;

  boost::scoped_ptr<xtreemfs::Client> client_;
  xtreemfs::Options options_;

  xtreemfs::Options dir_url_;
  xtreemfs::Options mrc_url_;

  std::string dir_config_file_;
  std::string mrc_config_file_;
  std::string osd_config_file_;

  xtreemfs::pbrpc::Auth auth_;
  xtreemfs::pbrpc::UserCredentials user_credentials_;
};

class ClientNoSSLTest : public ClientTest {
protected:
  virtual void SetUp() {
    dir_config_file_ = "tests/configs/dirconfig_nossl.test";
    mrc_config_file_ = "tests/configs/mrcconfig_nossl.test";
    osd_config_file_ = "tests/configs/osdconfig_nossl.test";

    dir_url_.xtreemfs_url = "pbrpc://localhost:42638/";
    mrc_url_.xtreemfs_url = "pbrpc://localhost:42636/";

    options_.log_level_string = "DEBUG";
    options_.log_file_path = "/tmp/xtreemfs_client_ssl_test_no_ssl";

    ClientTest::SetUp();
  }

  virtual void TearDown() {
    ClientTest::TearDown();
    unlink(options_.log_file_path.c_str());
  }
};

class ClientSSLTestShortChain : public ClientTest {
protected:
  virtual void SetUp() {
    // Root signed, root trusted
    dir_config_file_ = "tests/configs/dirconfig_ssl_short_chain.test";
    mrc_config_file_ = "tests/configs/mrcconfig_ssl_short_chain.test";
    osd_config_file_ = "tests/configs/osdconfig_ssl_short_chain.test";

    dir_url_.xtreemfs_url = "pbrpcs://localhost:42638/";
    mrc_url_.xtreemfs_url = "pbrpcs://localhost:42636/";

    options_.log_level_string = "DEBUG";
    options_.log_file_path = "/tmp/xtreemfs_client_ssl_test_short_chain";

    // Root signed, only root as additional certificate.
    options_.ssl_pkcs12_path = "tests/certs/client_ssl_test/Client_Root_Root.p12";
    options_.ssl_verify_certificates = true;

    ClientTest::SetUp();
  }

  virtual void TearDown() {
    ClientTest::TearDown();
    unlink(options_.log_file_path.c_str());
  }
};

class ClientSSLTestLongChain : public ClientTest {
protected:
  virtual void SetUp() {
    // All service certificates are signed with Leaf CA, which is signed with
    // Intermediate CA, which is signed with Root CA. The keystore contains
    // only the Leaf CA.
    dir_config_file_ = "tests/configs/dirconfig_ssl.test";
    mrc_config_file_ = "tests/configs/mrcconfig_ssl.test";
    osd_config_file_ = "tests/configs/osdconfig_ssl.test";

    dir_url_.xtreemfs_url = "pbrpcs://localhost:42638/";
    mrc_url_.xtreemfs_url = "pbrpcs://localhost:42636/";

    options_.log_level_string = "DEBUG";
    options_.log_file_path = "/tmp/xtreemfs_client_ssl_test_with_chain";

    // Client certificate is signed with Leaf CA. Contains the entire chain
    // as additional certificates.
    options_.ssl_pkcs12_path = "tests/certs/client_ssl_test/Client_Leaf_Chain.p12";
    options_.ssl_verify_certificates = true;

    ClientTest::SetUp();
  }

  virtual void TearDown() {
    ClientTest::TearDown();
    unlink(options_.log_file_path.c_str());
  }
};

TEST_F(ClientNoSSLTest, TestNoSSL) {
  CreateOpenDeleteVolume("test_no_ssl");
  ASSERT_EQ(0, count_occurrences_in_file(options_.log_file_path, "SSL"));
}

TEST_F(ClientSSLTestShortChain, TestVerifyShortChain) {
  CreateOpenDeleteVolume("test_ssl_short_chain");

  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support activated"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support using PKCS#12 file "
      "tests/certs/client_ssl_test/Client_Root_Root.p12"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Writing 1 verification certificates to /tmp/ca"));

  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Root CA' "
      "was successful."));
  ASSERT_EQ(0, count_occurrences_in_file(
      options_.log_file_path,
      "/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Intermediate CA"));
  ASSERT_EQ(0, count_occurrences_in_file(
      options_.log_file_path,
      "/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Leaf CA"));

  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=MRC (Root)' "
      "was successful"));
  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=DIR (Root)' "
      "was successful."));
}

TEST_F(ClientSSLTestLongChain, TestVerifyLongChain) {
  CreateOpenDeleteVolume("test_ssl_long_chain");

  // Once for MRC and once for DIR.
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support activated"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "SSL support using PKCS#12 file "
      "tests/certs/client_ssl_test/Client_Leaf_Chain.p12"));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Writing 3 verification certificates to /tmp/ca"));

  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Root CA' "
      "was successful."));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Intermediate "
      "CA' was successful."));
  ASSERT_EQ(2, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=Leaf CA' was "
      "successful."));

  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=MRC (Leaf)' "
      "was successful"));
  ASSERT_EQ(1, count_occurrences_in_file(
      options_.log_file_path,
      "Verification of subject '/C=DE/ST=Berlin/L=Berlin/O=ZIB/CN=DIR (Leaf)' "
      "was successful."));
}

}  // namespace rpc
}  // namespace xtreemfs
<|endoftext|>


<commit_before>/**
 *  Copyright (C) 2015 Topology LP
 *  All rights reserved.
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a copy
 *  of this software and associated documentation files (the "Software"), to
 *  deal in the Software without restriction, including without limitation the
 *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 *  sell copies of the Software, and to permit persons to whom the Software is
 *  furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 *  IN THE SOFTWARE.
 */

#ifndef CPPCODEC_DETAIL_STREAM_CODEC
#define CPPCODEC_DETAIL_STREAM_CODEC

#include <stdlib.h> // for abort()
#include <stdint.h>

#include "../parse_error.hpp"

namespace cppcodec {
namespace detail {

template <typename Codec, typename CodecVariant>
class stream_codec
{
public:
    template <typename Result, typename ResultState> static void encode(
            Result& encoded_result, ResultState&, const uint8_t* binary, size_t binary_size);

    template <typename Result, typename ResultState> static void decode(
            Result& binary_result, ResultState&, const char* encoded, size_t encoded_size);

    static constexpr size_t encoded_size(size_t binary_size) noexcept;
    static constexpr size_t decoded_max_size(size_t encoded_size) noexcept;
};

template <typename Codec, typename CodecVariant>
template <typename Result, typename ResultState>
inline void stream_codec<Codec, CodecVariant>::encode(
        Result& encoded_result, ResultState& state,
        const uint8_t* src, size_t src_size)
{
    const uint8_t* src_end = src + src_size;

    if (src_size >= Codec::binary_block_size()) {
        src_end -= Codec::binary_block_size();

        for (; src <= src_end; src += Codec::binary_block_size()) {
            Codec::encode_block(encoded_result, state, src);
        }
        src_end += Codec::binary_block_size();
    }

    if (src_end > src) {
        auto remaining_src_len = src_end - src;
        if (!remaining_src_len || remaining_src_len >= Codec::binary_block_size()) {
            abort();
            return;
        }
        Codec::encode_tail(encoded_result, state, src, remaining_src_len);
        Codec::pad(encoded_result, state, remaining_src_len);
    }
}

template <typename Codec, typename CodecVariant>
template <typename Result, typename ResultState>
inline void stream_codec<Codec, CodecVariant>::decode(
        Result& binary_result, ResultState& state,
        const char* src_encoded, size_t src_size)
{
    const char* src = src_encoded;
    const char* src_end = src + src_size;

    uint8_t idx[Codec::encoded_block_size()] = {};
    uint8_t last_value_idx = 0;

    while (src < src_end) {
        if (CodecVariant::should_ignore(idx[last_value_idx] = CodecVariant::index_of(*(src++)))) {
            continue;
        }
        if (CodecVariant::is_special_character(idx[last_value_idx])) {
            break;
        }

        ++last_value_idx;
        if (last_value_idx == Codec::encoded_block_size()) {
            Codec::decode_block(binary_result, state, idx);
            last_value_idx = 0;
        }
    }

    uint8_t last_idx = last_value_idx;
    if (CodecVariant::is_padding_symbol(idx[last_idx])) {
        // We're in here because we just read a (first) padding character. Try to read more.
        ++last_idx;
        while (src < src_end) {
            if (CodecVariant::is_eof(idx[last_idx] = CodecVariant::index_of(*(src++)))) {
                break;
            }
            if (!CodecVariant::is_padding_symbol(idx[last_idx])) {
                throw padding_error();
            }

            ++last_idx;
            if (last_idx > Codec::encoded_block_size()) {
                throw padding_error();
            }
        }
    }

    if (last_idx)  {
        if (CodecVariant::requires_padding() && last_idx != Codec::encoded_block_size()) {
            // If the input is not a multiple of the block size then the input is incorrect.
            throw padding_error();
        }
        if (last_value_idx >= Codec::encoded_block_size()) {
            abort();
            return;
        }
        Codec::decode_tail(binary_result, state, idx, last_value_idx);
    }
}

template <typename Codec, typename CodecVariant>
inline constexpr size_t stream_codec<Codec, CodecVariant>::encoded_size(size_t binary_size) noexcept
{
    using C = Codec;

    // constexpr rules make this a lot harder to read than it actually is.
    return CodecVariant::generates_padding()
            // With padding, the encoded size is a multiple of the encoded block size.
            // To calculate that, round the binary size up to multiple of the binary block size,
            // then convert to encoded by multiplying with { base32: 8/5, base64: 4/3 }.
            ? (binary_size + (C::binary_block_size() - 1)
                    - ((binary_size + (C::binary_block_size() - 1)) % C::binary_block_size()))
                    * C::encoded_block_size() / C::binary_block_size()
            // No padding: only pad to the next multiple of 5 bits, i.e. at most a single extra byte.
            : (binary_size * C::encoded_block_size() / C::binary_block_size())
                    + (((binary_size * C::encoded_block_size()) % C::binary_block_size()) ? 1 : 0);
}

template <typename Codec, typename CodecVariant>
inline constexpr size_t stream_codec<Codec, CodecVariant>::decoded_max_size(size_t encoded_size) noexcept
{
    using C = Codec;

    return CodecVariant::requires_padding()
            ? encoded_size * C::binary_block_size() / C::encoded_block_size()
            : (encoded_size * C::binary_block_size() / C::encoded_block_size())
                    + (((encoded_size * C::binary_block_size()) % C::encoded_block_size()) ? 1 : 0);
}

} // namespace detail
} // namespace cppcodec

#endif // CPPCODEC_DETAIL_STREAM_CODEC
<commit_msg>Fix decoded_max_size() for unpadded stream codecs.<commit_after>/**
 *  Copyright (C) 2015 Topology LP
 *  All rights reserved.
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a copy
 *  of this software and associated documentation files (the "Software"), to
 *  deal in the Software without restriction, including without limitation the
 *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 *  sell copies of the Software, and to permit persons to whom the Software is
 *  furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 *  IN THE SOFTWARE.
 */

#ifndef CPPCODEC_DETAIL_STREAM_CODEC
#define CPPCODEC_DETAIL_STREAM_CODEC

#include <stdlib.h> // for abort()
#include <stdint.h>

#include "../parse_error.hpp"

namespace cppcodec {
namespace detail {

template <typename Codec, typename CodecVariant>
class stream_codec
{
public:
    template <typename Result, typename ResultState> static void encode(
            Result& encoded_result, ResultState&, const uint8_t* binary, size_t binary_size);

    template <typename Result, typename ResultState> static void decode(
            Result& binary_result, ResultState&, const char* encoded, size_t encoded_size);

    static constexpr size_t encoded_size(size_t binary_size) noexcept;
    static constexpr size_t decoded_max_size(size_t encoded_size) noexcept;
};

template <typename Codec, typename CodecVariant>
template <typename Result, typename ResultState>
inline void stream_codec<Codec, CodecVariant>::encode(
        Result& encoded_result, ResultState& state,
        const uint8_t* src, size_t src_size)
{
    const uint8_t* src_end = src + src_size;

    if (src_size >= Codec::binary_block_size()) {
        src_end -= Codec::binary_block_size();

        for (; src <= src_end; src += Codec::binary_block_size()) {
            Codec::encode_block(encoded_result, state, src);
        }
        src_end += Codec::binary_block_size();
    }

    if (src_end > src) {
        auto remaining_src_len = src_end - src;
        if (!remaining_src_len || remaining_src_len >= Codec::binary_block_size()) {
            abort();
            return;
        }
        Codec::encode_tail(encoded_result, state, src, remaining_src_len);
        Codec::pad(encoded_result, state, remaining_src_len);
    }
}

template <typename Codec, typename CodecVariant>
template <typename Result, typename ResultState>
inline void stream_codec<Codec, CodecVariant>::decode(
        Result& binary_result, ResultState& state,
        const char* src_encoded, size_t src_size)
{
    const char* src = src_encoded;
    const char* src_end = src + src_size;

    uint8_t idx[Codec::encoded_block_size()] = {};
    uint8_t last_value_idx = 0;

    while (src < src_end) {
        if (CodecVariant::should_ignore(idx[last_value_idx] = CodecVariant::index_of(*(src++)))) {
            continue;
        }
        if (CodecVariant::is_special_character(idx[last_value_idx])) {
            break;
        }

        ++last_value_idx;
        if (last_value_idx == Codec::encoded_block_size()) {
            Codec::decode_block(binary_result, state, idx);
            last_value_idx = 0;
        }
    }

    uint8_t last_idx = last_value_idx;
    if (CodecVariant::is_padding_symbol(idx[last_idx])) {
        // We're in here because we just read a (first) padding character. Try to read more.
        ++last_idx;
        while (src < src_end) {
            if (CodecVariant::is_eof(idx[last_idx] = CodecVariant::index_of(*(src++)))) {
                break;
            }
            if (!CodecVariant::is_padding_symbol(idx[last_idx])) {
                throw padding_error();
            }

            ++last_idx;
            if (last_idx > Codec::encoded_block_size()) {
                throw padding_error();
            }
        }
    }

    if (last_idx)  {
        if (CodecVariant::requires_padding() && last_idx != Codec::encoded_block_size()) {
            // If the input is not a multiple of the block size then the input is incorrect.
            throw padding_error();
        }
        if (last_value_idx >= Codec::encoded_block_size()) {
            abort();
            return;
        }
        Codec::decode_tail(binary_result, state, idx, last_value_idx);
    }
}

template <typename Codec, typename CodecVariant>
inline constexpr size_t stream_codec<Codec, CodecVariant>::encoded_size(size_t binary_size) noexcept
{
    using C = Codec;

    // constexpr rules make this a lot harder to read than it actually is.
    return CodecVariant::generates_padding()
            // With padding, the encoded size is a multiple of the encoded block size.
            // To calculate that, round the binary size up to multiple of the binary block size,
            // then convert to encoded by multiplying with { base32: 8/5, base64: 4/3 }.
            ? (binary_size + (C::binary_block_size() - 1)
                    - ((binary_size + (C::binary_block_size() - 1)) % C::binary_block_size()))
                    * C::encoded_block_size() / C::binary_block_size()
            // No padding: only pad to the next multiple of 5 bits, i.e. at most a single extra byte.
            : (binary_size * C::encoded_block_size() / C::binary_block_size())
                    + (((binary_size * C::encoded_block_size()) % C::binary_block_size()) ? 1 : 0);
}

template <typename Codec, typename CodecVariant>
inline constexpr size_t stream_codec<Codec, CodecVariant>::decoded_max_size(size_t encoded_size) noexcept
{
    using C = Codec;

    return CodecVariant::requires_padding()
            ? (encoded_size / C::encoded_block_size() * C::binary_block_size())
            : (encoded_size / C::encoded_block_size() * C::binary_block_size())
                    + ((encoded_size % C::encoded_block_size())
                            * C::binary_block_size() / C::encoded_block_size());
}

} // namespace detail
} // namespace cppcodec

#endif // CPPCODEC_DETAIL_STREAM_CODEC
<|endoftext|>


<commit_before>#include "RaZ/Render/Mesh.hpp"

namespace Raz {

Mesh::Mesh(const Triangle& triangle) : Mesh() {
  // TODO: check if vertices are defined counterclockwise

  const Vec3f& firstPos  = triangle.getFirstPos();
  const Vec3f& secondPos = triangle.getSecondPos();
  const Vec3f& thirdPos  = triangle.getThirdPos();

  Vertex firstVert {};
  firstVert.position  = firstPos;
  firstVert.texcoords = Vec2f({ 0.f, 0.f });

  Vertex secondVert {};
  secondVert.position  = secondPos;
  secondVert.texcoords = Vec2f({ 0.5f, 1.f });

  Vertex thirdVert {};
  thirdVert.position  = thirdPos;
  thirdVert.texcoords = Vec2f({ 1.f, 0.f });

  // Computing normals
  firstVert.normal  = (firstPos - secondPos).cross(firstPos - thirdPos).normalize();
  secondVert.normal = (secondPos - thirdPos).cross(secondPos - firstPos).normalize();
  thirdVert.normal  = (thirdPos - firstPos).cross(thirdPos - secondPos).normalize();

  std::vector<Vertex>& vertices = m_submeshes.front()->getVbo().getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getEbo().getIndices();

  vertices.resize(3);
  indices.resize(3);

  vertices[0] = firstVert;
  vertices[1] = secondVert;
  vertices[2] = thirdVert;

  indices[0] = 1;
  indices[1] = 0;
  indices[2] = 2;

  load();
}

Mesh::Mesh(const Quad& quad) : Mesh() {
  const Vec3f& leftTopPos     = quad.getLeftTopPos();
  const Vec3f& rightTopPos    = quad.getRightTopPos();
  const Vec3f& rightBottomPos = quad.getRightBottomPos();
  const Vec3f& leftBottomPos  = quad.getLeftBottomPos();

  Vertex leftTop {};
  leftTop.position  = leftTopPos;
  leftTop.texcoords = Vec2f({ 0.f, 1.f });

  Vertex rightTop {};
  rightTop.position  = rightTopPos;
  rightTop.texcoords = Vec2f({ 1.f, 1.f });

  Vertex rightBottom {};
  rightBottom.position  = rightBottomPos;
  rightBottom.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftBottom {};
  leftBottom.position  = leftBottomPos;
  leftBottom.texcoords = Vec2f({ 0.f, 0.f });

  // Computing normals
  // TODO: normals should not be computed (or even exist) for simple display quads like a framebuffer
  leftTop.normal     = (leftTopPos - rightTopPos).cross(leftTopPos - leftBottomPos).normalize();
  rightTop.normal    = (rightTopPos - rightBottomPos).cross(rightTopPos - leftTopPos).normalize();
  rightBottom.normal = (rightBottomPos - leftBottomPos).cross(rightBottomPos - rightTopPos).normalize();
  leftBottom.normal  = (leftBottomPos - leftTopPos).cross(leftBottomPos - rightBottomPos).normalize();

  std::vector<Vertex>& vertices = m_submeshes.front()->getVbo().getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getEbo().getIndices();

  vertices.resize(4);
  indices.resize(6);

  vertices[0] = leftTop;
  vertices[1] = leftBottom;
  vertices[2] = rightBottom;
  vertices[3] = rightTop;

  indices[0] = 0;
  indices[1] = 1;
  indices[2] = 2;

  indices[3] = 0;
  indices[4] = 2;
  indices[5] = 3;

  load();
}

Mesh::Mesh(const AABB& box) : Mesh() {
  const Vec3f& rightTopFrontPos  = box.getRightTopFrontPos();
  const Vec3f& leftBottomBackPos = box.getLeftBottomBackPos();

  const float rightPos  = rightTopFrontPos[0];
  const float leftPos   = leftBottomBackPos[0];
  const float topPos    = rightTopFrontPos[1];
  const float bottomPos = leftBottomBackPos[1];
  const float frontPos  = rightTopFrontPos[2];
  const float backPos   = leftBottomBackPos[2];

  // TODO: texcoords should not exist for simple display cubes like a skybox

  Vertex rightTopBack {};
  rightTopBack.position  = Vec3f({ rightPos, topPos, backPos });
  rightTopBack.texcoords = Vec2f({ 0.f, 1.f });

  Vertex rightTopFront {};
  rightTopFront.position  = rightTopFrontPos;
  rightTopFront.texcoords = Vec2f({ 1.f, 1.f });

  Vertex rightBottomBack {};
  rightBottomBack.position  = Vec3f({ rightPos, bottomPos, backPos });
  rightBottomBack.texcoords = Vec2f({ 0.f, 0.f });

  Vertex rightBottomFront {};
  rightBottomFront.position  = Vec3f({ rightPos, bottomPos, frontPos });
  rightBottomFront.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftTopBack {};
  leftTopBack.position  = Vec3f({ leftPos, topPos, backPos });
  leftTopBack.texcoords = Vec2f({ 1.f, 1.f });

  Vertex leftTopFront {};
  leftTopFront.position  = Vec3f({ leftPos, topPos, frontPos });
  leftTopFront.texcoords = Vec2f({ 0.f, 1.f });

  Vertex leftBottomBack {};
  leftBottomBack.position  = leftBottomBackPos;
  leftBottomBack.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftBottomFront {};
  leftBottomFront.position  = Vec3f({ leftPos, bottomPos, frontPos });
  leftBottomFront.texcoords = Vec2f({ 0.f, 0.f });

  // Computing normals
  // TODO: normals should not be computed (or even exist) for simple display cubes like a skybox
  // TODO: compute normals

  std::vector<Vertex>& vertices = m_submeshes.front()->getVbo().getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getEbo().getIndices();

  vertices.resize(8);
  indices.resize(36);

  vertices[0] = rightTopBack;
  vertices[1] = rightTopFront;

  vertices[2] = rightBottomBack;
  vertices[3] = rightBottomFront;

  vertices[4] = leftTopBack;
  vertices[5] = leftTopFront;

  vertices[6] = leftBottomBack;
  vertices[7] = leftBottomFront;

  // Right face
  indices[0] = 1;
  indices[1] = 0;
  indices[2] = 2;

  indices[3] = 1;
  indices[4] = 2;
  indices[5] = 3;

  // Left face
  indices[6] = 4;
  indices[7] = 5;
  indices[8] = 7;

  indices[9]  = 4;
  indices[10] = 7;
  indices[11] = 6;

  // Top face
  indices[12] = 4;
  indices[13] = 0;
  indices[14] = 1;

  indices[15] = 4;
  indices[16] = 1;
  indices[17] = 5;

  // Bottom face
  indices[18] = 7;
  indices[19] = 3;
  indices[20] = 2;

  indices[21] = 7;
  indices[22] = 2;
  indices[23] = 6;

  // Front face
  indices[24] = 5;
  indices[25] = 1;
  indices[26] = 3;

  indices[27] = 5;
  indices[28] = 3;
  indices[29] = 7;

  // Back face
  indices[30] = 0;
  indices[31] = 4;
  indices[32] = 6;

  indices[33] = 0;
  indices[34] = 6;
  indices[35] = 2;

  load();
}

} // namespace Raz
<commit_msg>[Render/MeshShape] MeshShape's funcs don't use getVbo()/getEbo() anymore<commit_after>#include "RaZ/Render/Mesh.hpp"

namespace Raz {

Mesh::Mesh(const Triangle& triangle) : Mesh() {
  // TODO: check if vertices are defined counterclockwise

  const Vec3f& firstPos  = triangle.getFirstPos();
  const Vec3f& secondPos = triangle.getSecondPos();
  const Vec3f& thirdPos  = triangle.getThirdPos();

  Vertex firstVert {};
  firstVert.position  = firstPos;
  firstVert.texcoords = Vec2f({ 0.f, 0.f });

  Vertex secondVert {};
  secondVert.position  = secondPos;
  secondVert.texcoords = Vec2f({ 0.5f, 1.f });

  Vertex thirdVert {};
  thirdVert.position  = thirdPos;
  thirdVert.texcoords = Vec2f({ 1.f, 0.f });

  // Computing normals
  firstVert.normal  = (firstPos - secondPos).cross(firstPos - thirdPos).normalize();
  secondVert.normal = (secondPos - thirdPos).cross(secondPos - firstPos).normalize();
  thirdVert.normal  = (thirdPos - firstPos).cross(thirdPos - secondPos).normalize();

  std::vector<Vertex>& vertices = m_submeshes.front()->getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getIndices();

  vertices.resize(3);
  indices.resize(3);

  vertices[0] = firstVert;
  vertices[1] = secondVert;
  vertices[2] = thirdVert;

  indices[0] = 1;
  indices[1] = 0;
  indices[2] = 2;

  load();
}

Mesh::Mesh(const Quad& quad) : Mesh() {
  const Vec3f& leftTopPos     = quad.getLeftTopPos();
  const Vec3f& rightTopPos    = quad.getRightTopPos();
  const Vec3f& rightBottomPos = quad.getRightBottomPos();
  const Vec3f& leftBottomPos  = quad.getLeftBottomPos();

  Vertex leftTop {};
  leftTop.position  = leftTopPos;
  leftTop.texcoords = Vec2f({ 0.f, 1.f });

  Vertex rightTop {};
  rightTop.position  = rightTopPos;
  rightTop.texcoords = Vec2f({ 1.f, 1.f });

  Vertex rightBottom {};
  rightBottom.position  = rightBottomPos;
  rightBottom.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftBottom {};
  leftBottom.position  = leftBottomPos;
  leftBottom.texcoords = Vec2f({ 0.f, 0.f });

  // Computing normals
  // TODO: normals should not be computed (or even exist) for simple display quads like a framebuffer
  leftTop.normal     = (leftTopPos - rightTopPos).cross(leftTopPos - leftBottomPos).normalize();
  rightTop.normal    = (rightTopPos - rightBottomPos).cross(rightTopPos - leftTopPos).normalize();
  rightBottom.normal = (rightBottomPos - leftBottomPos).cross(rightBottomPos - rightTopPos).normalize();
  leftBottom.normal  = (leftBottomPos - leftTopPos).cross(leftBottomPos - rightBottomPos).normalize();

  std::vector<Vertex>& vertices = m_submeshes.front()->getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getIndices();

  vertices.resize(4);
  indices.resize(6);

  vertices[0] = leftTop;
  vertices[1] = leftBottom;
  vertices[2] = rightBottom;
  vertices[3] = rightTop;

  indices[0] = 0;
  indices[1] = 1;
  indices[2] = 2;

  indices[3] = 0;
  indices[4] = 2;
  indices[5] = 3;

  load();
}

Mesh::Mesh(const AABB& box) : Mesh() {
  const Vec3f& rightTopFrontPos  = box.getRightTopFrontPos();
  const Vec3f& leftBottomBackPos = box.getLeftBottomBackPos();

  const float rightPos  = rightTopFrontPos[0];
  const float leftPos   = leftBottomBackPos[0];
  const float topPos    = rightTopFrontPos[1];
  const float bottomPos = leftBottomBackPos[1];
  const float frontPos  = rightTopFrontPos[2];
  const float backPos   = leftBottomBackPos[2];

  // TODO: texcoords should not exist for simple display cubes like a skybox

  Vertex rightTopBack {};
  rightTopBack.position  = Vec3f({ rightPos, topPos, backPos });
  rightTopBack.texcoords = Vec2f({ 0.f, 1.f });

  Vertex rightTopFront {};
  rightTopFront.position  = rightTopFrontPos;
  rightTopFront.texcoords = Vec2f({ 1.f, 1.f });

  Vertex rightBottomBack {};
  rightBottomBack.position  = Vec3f({ rightPos, bottomPos, backPos });
  rightBottomBack.texcoords = Vec2f({ 0.f, 0.f });

  Vertex rightBottomFront {};
  rightBottomFront.position  = Vec3f({ rightPos, bottomPos, frontPos });
  rightBottomFront.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftTopBack {};
  leftTopBack.position  = Vec3f({ leftPos, topPos, backPos });
  leftTopBack.texcoords = Vec2f({ 1.f, 1.f });

  Vertex leftTopFront {};
  leftTopFront.position  = Vec3f({ leftPos, topPos, frontPos });
  leftTopFront.texcoords = Vec2f({ 0.f, 1.f });

  Vertex leftBottomBack {};
  leftBottomBack.position  = leftBottomBackPos;
  leftBottomBack.texcoords = Vec2f({ 1.f, 0.f });

  Vertex leftBottomFront {};
  leftBottomFront.position  = Vec3f({ leftPos, bottomPos, frontPos });
  leftBottomFront.texcoords = Vec2f({ 0.f, 0.f });

  // Computing normals
  // TODO: normals should not be computed (or even exist) for simple display cubes like a skybox
  // TODO: compute normals

  std::vector<Vertex>& vertices = m_submeshes.front()->getVertices();
  std::vector<unsigned int>& indices = m_submeshes.front()->getIndices();

  vertices.resize(8);
  indices.resize(36);

  vertices[0] = rightTopBack;
  vertices[1] = rightTopFront;

  vertices[2] = rightBottomBack;
  vertices[3] = rightBottomFront;

  vertices[4] = leftTopBack;
  vertices[5] = leftTopFront;

  vertices[6] = leftBottomBack;
  vertices[7] = leftBottomFront;

  // Right face
  indices[0] = 1;
  indices[1] = 0;
  indices[2] = 2;

  indices[3] = 1;
  indices[4] = 2;
  indices[5] = 3;

  // Left face
  indices[6] = 4;
  indices[7] = 5;
  indices[8] = 7;

  indices[9]  = 4;
  indices[10] = 7;
  indices[11] = 6;

  // Top face
  indices[12] = 4;
  indices[13] = 0;
  indices[14] = 1;

  indices[15] = 4;
  indices[16] = 1;
  indices[17] = 5;

  // Bottom face
  indices[18] = 7;
  indices[19] = 3;
  indices[20] = 2;

  indices[21] = 7;
  indices[22] = 2;
  indices[23] = 6;

  // Front face
  indices[24] = 5;
  indices[25] = 1;
  indices[26] = 3;

  indices[27] = 5;
  indices[28] = 3;
  indices[29] = 7;

  // Back face
  indices[30] = 0;
  indices[31] = 4;
  indices[32] = 6;

  indices[33] = 0;
  indices[34] = 6;
  indices[35] = 2;

  load();
}

} // namespace Raz
<|endoftext|>


<commit_before>// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================

#include <dlfcn.h>

#include "absl/strings/str_format.h"
#include "absl/time/time.h"
#include "tensorflow/compiler/xla/python/tpu_driver/client/c_api.h"
#include "tensorflow/compiler/xla/python/tpu_driver/tpu_driver.h"
#include "tensorflow/compiler/xla/python/tpu_driver/tpu_driver.pb.h"
#include "tensorflow/compiler/xla/statusor.h"
#include "tensorflow/compiler/xla/util.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"

namespace tpu_driver {
namespace {

class ExternalTpuDriver;

class ExternalEvent : public Event {
 public:
  explicit ExternalEvent(::TpuDriverFn* driver_fn, ::TpuEvent* event)
      : driver_fn_(driver_fn), event_(event) {}

  ~ExternalEvent() override { driver_fn_->TpuDriver_FreeEvent(event_); }

  xla::Status Await() override {
    auto tpu_status = driver_fn_->TpuDriver_EventAwait(event_, -1);
    auto ret = xla::Status(tensorflow::error::Code(tpu_status->code),
                           absl::StrFormat("%s", tpu_status->msg));
    driver_fn_->TpuDriver_FreeStatus(tpu_status);
    return ret;
  }

  absl::optional<xla::Status> AwaitWithTimeout(
      absl::Duration duration) override {
    auto tpu_status_or = driver_fn_->TpuDriver_EventAwait(
        event_, absl::ToInt64Microseconds(duration));
    if (tpu_status_or == nullptr) {
      return absl::nullopt;
    } else {
      auto ret = xla::Status(tensorflow::error::Code(tpu_status_or->code),
                             absl::StrFormat("%s", tpu_status_or->msg));
      driver_fn_->TpuDriver_FreeStatus(tpu_status_or);
      return ret;
    }
  }

  void AddCallback(std::function<void(xla::Status)> callback) override {
    // We have to create a new copy of the fn on the heap to make it persist.
    std::function<void(xla::Status)>* callback_addr =
        new std::function<void(xla::Status)>(callback);

    // Using the callback_addr instead of capturing because C++11 lambdas with
    // variable captures cannot be converted to C function pointers.
    driver_fn_->TpuDriver_EventAddCallback(
        event_,
        [](struct TpuStatus* status, void* additional_info) {
          auto callback_addr =
              static_cast<std::function<void(xla::Status)>*>(additional_info);
          auto xla_status = xla::Status(tensorflow::error::Code(status->code),
                                        absl::StrFormat("%s", status->msg));
          (*callback_addr)(xla_status);
          delete callback_addr;
        },
        callback_addr);
  }

 private:
  ::TpuDriverFn* driver_fn_;
  ::TpuEvent* event_;

  friend ExternalTpuDriver;
};

class ExternalBufferHandle : public BufferHandle {
 public:
  explicit ExternalBufferHandle(::TpuDriverFn* driver_fn,
                                ::TpuBufferHandle* handle)
      : handle_(handle), event_(new ExternalEvent(driver_fn, handle->event)) {}

  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override { return handle_->size_in_bytes; }

  absl::optional<xla::ShapeProto> shape() override {
    LOG(FATAL) << "Unimplemented.";
    return absl::nullopt;
  }

 private:
  ::TpuBufferHandle* handle_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalCompiledProgramHandle : public CompiledProgramHandle {
 public:
  explicit ExternalCompiledProgramHandle(::TpuDriverFn* driver_fn,
                                         ::TpuCompiledProgramHandle* handle)
      : handle_(handle),
        driver_fn_(driver_fn),
        event_(new ExternalEvent(driver_fn, handle->event)) {}

  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override {
    LOG(FATAL) << "Unimplemented.";
    return 0;
  }

  xla::Status program_shape(xla::ProgramShapeProto* program_shape) override {
    struct CompiledProgramShape* shape =
        driver_fn_->TpuDriver_GetCompiledProgramShape(handle_);
    program_shape->ParseFromArray(shape->bytes, shape->size);

    auto status = xla::Status(tensorflow::error::Code(shape->status->code),
                              absl::StrFormat("%s", shape->status->msg));
    driver_fn_->TpuDriver_FreeCompiledProgramShape(shape);

    return status;
  }

 private:
  ::TpuCompiledProgramHandle* handle_;
  ::TpuDriverFn* driver_fn_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalLoadedProgramHandle : public LoadedProgramHandle {
 public:
  explicit ExternalLoadedProgramHandle(::TpuDriverFn* driver_fn,
                                       ::TpuLoadedProgramHandle* handle)
      : handle_(handle), event_(new ExternalEvent(driver_fn, handle->event)) {}
  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override {
    LOG(FATAL) << "Unimplemented.";
    return 0;
  }

 private:
  ::TpuLoadedProgramHandle* handle_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalTpuDriver : public TpuDriver {
 public:
  explicit ExternalTpuDriver(const std::string& so_path) {
    void* handle;
    handle = dlopen(so_path.c_str(), RTLD_NOW);
    if (!handle) {
      LOG(FATAL) << "Unable to load shared library: " << dlerror();
    }

    PrototypeTpuDriver_Initialize* initialize_fn;
    *reinterpret_cast<void**>(&initialize_fn) =
        dlsym(handle, "TpuDriver_Initialize");
    initialize_fn(&driver_fn_);

    driver_ = driver_fn_.TpuDriver_Open("local://");
  }

  ~ExternalTpuDriver() override {}

  void QuerySystemInfo(SystemInfo* system_info) override {
    LOG(FATAL) << "Unimplemented.";
  }

  xla::Status Reset() override { LOG(FATAL) << "Unimplemented."; }

  std::unique_ptr<BufferHandle> Allocate(
      int32_t core_id, MemoryRegion region, int64_t num_bytes,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto bh = absl::make_unique<ExternalBufferHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_Allocate(driver_, core_id, region, num_bytes,
                                      wait_for.size(), tpu_events));
    delete tpu_events;
    return bh;
  }

  std::unique_ptr<BufferHandle> Allocate(
      int32_t core_id, MemoryRegion region, const xla::ShapeProto& shape,
      absl::Span<Event* const> wait_for) override {
    LOG(FATAL) << "Unimplemented.";
    return nullptr;
  }

  std::unique_ptr<BufferHandle> AllocateTuple(
      int32_t core_id, MemoryRegion region,
      absl::Span<BufferHandle* const> children,
      absl::Span<Event* const> wait_for) override {
    LOG(FATAL) << "Unimplemented.";
    return nullptr;
  }

  std::shared_ptr<Event> Deallocate(
      std::unique_ptr<BufferHandle> handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_Deallocate(
            driver_, static_cast<ExternalBufferHandle*>(handle.get())->handle_,
            wait_for.size(), tpu_events));
    delete tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferToDevice(
      const void* src, BufferHandle* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferToDevice(
            driver_, src, static_cast<ExternalBufferHandle*>(dst)->handle_,
            wait_for.size(), tpu_events));
    delete tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferFromDevice(
      const BufferHandle* src, void* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferFromDevice(
            driver_, static_cast<const ExternalBufferHandle*>(src)->handle_,
            dst, wait_for.size(), tpu_events));
    delete tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferFromDeviceToDevice(
      const BufferHandle* src, BufferHandle* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferFromDeviceToDevice(
            driver_, static_cast<const ExternalBufferHandle*>(src)->handle_,
            static_cast<ExternalBufferHandle*>(dst)->handle_, wait_for.size(),
            tpu_events));
    delete tpu_events;
    return event;
  }

  std::unique_ptr<CompiledProgramHandle> CompileProgram(
      const xla::HloProto& source, int32_t num_replicas,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    struct HloProto hlo;
    hlo.size = source.ByteSizeLong();
    hlo.buffer = malloc(hlo.size);
    if (!source.SerializeToArray(hlo.buffer, hlo.size)) {
      LOG(ERROR) << "Unable to serialize HLO to array.";
      return nullptr;
    }

    auto handle = absl::make_unique<ExternalCompiledProgramHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_CompileProgram(driver_, hlo, num_replicas,
                                            wait_for.size(), tpu_events));

    free(hlo.buffer);
    delete tpu_events;
    return handle;
  }
  std::unique_ptr<LoadedProgramHandle> LoadProgram(
      int32_t core_id, const CompiledProgramHandle* handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    auto loaded_handle = absl::make_unique<ExternalLoadedProgramHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_LoadProgram(
            driver_, core_id,
            static_cast<const ExternalCompiledProgramHandle*>(handle)->handle_,
            wait_for.size(), tpu_events));

    delete tpu_events;
    return loaded_handle;
  }

  std::shared_ptr<Event> UnloadProgram(
      std::unique_ptr<LoadedProgramHandle> handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_UnloadProgram(
            driver_,
            static_cast<ExternalLoadedProgramHandle*>(handle.get())->handle_,
            wait_for.size(), tpu_events));
    delete tpu_events;
    return event;
  }

  std::shared_ptr<Event> ExecuteProgram(
      LoadedProgramHandle* program, absl::Span<BufferHandle* const> inputs,
      absl::Span<BufferHandle* const> outputs,
      const xla::DeviceAssignmentProto& device_assignment,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    std::vector<::TpuBufferHandle*> inputv;
    inputv.reserve(inputs.size());
    for (int i = 0; i < inputs.size(); i++) {
      inputv.push_back(
          static_cast<ExternalBufferHandle* const>(inputs[i])->handle_);
    }
    std::vector<::TpuBufferHandle*> outputv;
    outputv.reserve(outputs.size());
    for (int i = 0; i < outputs.size(); i++) {
      outputv.push_back(
          static_cast<ExternalBufferHandle* const>(outputs[i])->handle_);
    }

    struct DeviceAssignment da = {device_assignment.replica_count(),
                                  device_assignment.computation_count()};
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_ExecuteProgram(
            driver_,
            static_cast<ExternalLoadedProgramHandle*>(program)->handle_,
            inputs.size(), inputv.data(), outputs.size(), outputv.data(), da,
            wait_for.size(), tpu_events));

    return event;
  }

  std::unique_ptr<TpuLinearizer> GetLinearizer() override { return nullptr; }

 private:
  ::TpuDriverFn driver_fn_;
  ::TpuDriver* driver_;

  ::TpuEvent** MakeEventArray(absl::Span<Event* const> wait_for) {
    if (wait_for.empty()) return nullptr;
    ::TpuEvent** ret = new ::TpuEvent*[wait_for.size()];
    for (int i = 0; i < wait_for.size(); i++) {
      ret[i] = static_cast<ExternalEvent* const>(wait_for[i])->event_;
    }
    return ret;
  }
};

xla::StatusOr<std::unique_ptr<TpuDriver>> RegisterExternalTpuDriver(
    const TpuDriverConfig& config) {
  std::string shared_lib = config.worker().substr(strlen("external://"));
  return xla::StatusOr<std::unique_ptr<TpuDriver>>(
      absl::make_unique<ExternalTpuDriver>(shared_lib));
}

REGISTER_TPU_DRIVER("external://", RegisterExternalTpuDriver);

}  // namespace
}  // namespace tpu_driver
<commit_msg>Add implementation for AllocateTuple to external TPU driver<commit_after>// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================

#include <dlfcn.h>

#include "absl/strings/str_format.h"
#include "absl/time/time.h"
#include "tensorflow/compiler/xla/python/tpu_driver/client/c_api.h"
#include "tensorflow/compiler/xla/python/tpu_driver/tpu_driver.h"
#include "tensorflow/compiler/xla/python/tpu_driver/tpu_driver.pb.h"
#include "tensorflow/compiler/xla/statusor.h"
#include "tensorflow/compiler/xla/util.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"

namespace tpu_driver {
namespace {

class ExternalTpuDriver;

class ExternalEvent : public Event {
 public:
  explicit ExternalEvent(::TpuDriverFn* driver_fn, ::TpuEvent* event)
      : driver_fn_(driver_fn), event_(event) {}

  ~ExternalEvent() override { driver_fn_->TpuDriver_FreeEvent(event_); }

  xla::Status Await() override {
    auto tpu_status = driver_fn_->TpuDriver_EventAwait(event_, -1);
    auto ret = xla::Status(tensorflow::error::Code(tpu_status->code),
                           absl::StrFormat("%s", tpu_status->msg));
    driver_fn_->TpuDriver_FreeStatus(tpu_status);
    return ret;
  }

  absl::optional<xla::Status> AwaitWithTimeout(
      absl::Duration duration) override {
    auto tpu_status_or = driver_fn_->TpuDriver_EventAwait(
        event_, absl::ToInt64Microseconds(duration));
    if (tpu_status_or == nullptr) {
      return absl::nullopt;
    } else {
      auto ret = xla::Status(tensorflow::error::Code(tpu_status_or->code),
                             absl::StrFormat("%s", tpu_status_or->msg));
      driver_fn_->TpuDriver_FreeStatus(tpu_status_or);
      return ret;
    }
  }

  void AddCallback(std::function<void(xla::Status)> callback) override {
    // We have to create a new copy of the fn on the heap to make it persist.
    std::function<void(xla::Status)>* callback_addr =
        new std::function<void(xla::Status)>(callback);

    // Using the callback_addr instead of capturing because C++11 lambdas with
    // variable captures cannot be converted to C function pointers.
    driver_fn_->TpuDriver_EventAddCallback(
        event_,
        [](struct TpuStatus* status, void* additional_info) {
          auto callback_addr =
              static_cast<std::function<void(xla::Status)>*>(additional_info);
          auto xla_status = xla::Status(tensorflow::error::Code(status->code),
                                        absl::StrFormat("%s", status->msg));
          (*callback_addr)(xla_status);
          delete callback_addr;
        },
        callback_addr);
  }

 private:
  ::TpuDriverFn* driver_fn_;
  ::TpuEvent* event_;

  friend ExternalTpuDriver;
};

class ExternalBufferHandle : public BufferHandle {
 public:
  explicit ExternalBufferHandle(::TpuDriverFn* driver_fn,
                                ::TpuBufferHandle* handle)
      : handle_(handle), event_(new ExternalEvent(driver_fn, handle->event)) {}

  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override { return handle_->size_in_bytes; }

  absl::optional<xla::ShapeProto> shape() override {
    LOG(FATAL) << "Unimplemented.";
    return absl::nullopt;
  }

 private:
  ::TpuBufferHandle* handle_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalCompiledProgramHandle : public CompiledProgramHandle {
 public:
  explicit ExternalCompiledProgramHandle(::TpuDriverFn* driver_fn,
                                         ::TpuCompiledProgramHandle* handle)
      : handle_(handle),
        driver_fn_(driver_fn),
        event_(new ExternalEvent(driver_fn, handle->event)) {}

  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override {
    LOG(FATAL) << "Unimplemented.";
    return 0;
  }

  xla::Status program_shape(xla::ProgramShapeProto* program_shape) override {
    struct CompiledProgramShape* shape =
        driver_fn_->TpuDriver_GetCompiledProgramShape(handle_);
    program_shape->ParseFromArray(shape->bytes, shape->size);

    auto status = xla::Status(tensorflow::error::Code(shape->status->code),
                              absl::StrFormat("%s", shape->status->msg));
    driver_fn_->TpuDriver_FreeCompiledProgramShape(shape);

    return status;
  }

 private:
  ::TpuCompiledProgramHandle* handle_;
  ::TpuDriverFn* driver_fn_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalLoadedProgramHandle : public LoadedProgramHandle {
 public:
  explicit ExternalLoadedProgramHandle(::TpuDriverFn* driver_fn,
                                       ::TpuLoadedProgramHandle* handle)
      : handle_(handle), event_(new ExternalEvent(driver_fn, handle->event)) {}
  std::shared_ptr<Event> OnReady() override { return event_; }

  int64_t size_in_bytes() override {
    LOG(FATAL) << "Unimplemented.";
    return 0;
  }

 private:
  ::TpuLoadedProgramHandle* handle_;
  std::shared_ptr<ExternalEvent> event_;

  friend ExternalTpuDriver;
};

class ExternalTpuDriver : public TpuDriver {
 public:
  explicit ExternalTpuDriver(const std::string& so_path) {
    void* handle;
    handle = dlopen(so_path.c_str(), RTLD_NOW);
    if (!handle) {
      LOG(FATAL) << "Unable to load shared library: " << dlerror();
    }

    PrototypeTpuDriver_Initialize* initialize_fn;
    *reinterpret_cast<void**>(&initialize_fn) =
        dlsym(handle, "TpuDriver_Initialize");
    initialize_fn(&driver_fn_);

    driver_ = driver_fn_.TpuDriver_Open("local://");
  }

  ~ExternalTpuDriver() override {}

  void QuerySystemInfo(SystemInfo* system_info) override {
    LOG(FATAL) << "Unimplemented.";
  }

  xla::Status Reset() override { LOG(FATAL) << "Unimplemented."; }

  std::unique_ptr<BufferHandle> Allocate(
      int32_t core_id, MemoryRegion region, int64_t num_bytes,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto bh = absl::make_unique<ExternalBufferHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_Allocate(driver_, core_id, region, num_bytes,
                                      wait_for.size(), tpu_events));
    delete[] tpu_events;
    return bh;
  }

  std::unique_ptr<BufferHandle> Allocate(
      int32_t core_id, MemoryRegion region, const xla::ShapeProto& shape,
      absl::Span<Event* const> wait_for) override {
    LOG(FATAL) << "Unimplemented.";
    return nullptr;
  }

  std::unique_ptr<BufferHandle> AllocateTuple(
      int32_t core_id, MemoryRegion region,
      absl::Span<BufferHandle* const> children,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    ::TpuBufferHandle** childbuf = new ::TpuBufferHandle*[children.size()];
    for (int i = 0; i < children.size(); i++) {
      childbuf[i] =
          static_cast<ExternalBufferHandle* const>(children[i])->handle_;
    }

    auto bh = absl::make_unique<ExternalBufferHandle>(
        &driver_fn_, driver_fn_.TpuDriver_AllocateTuple(
                         driver_, core_id, region, children.size(), childbuf,
                         wait_for.size(), tpu_events));
    delete[] tpu_events;
    delete[] childbuf;

    return bh;
  }

  std::shared_ptr<Event> Deallocate(
      std::unique_ptr<BufferHandle> handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_Deallocate(
            driver_, static_cast<ExternalBufferHandle*>(handle.get())->handle_,
            wait_for.size(), tpu_events));
    delete[] tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferToDevice(
      const void* src, BufferHandle* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferToDevice(
            driver_, src, static_cast<ExternalBufferHandle*>(dst)->handle_,
            wait_for.size(), tpu_events));
    delete[] tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferFromDevice(
      const BufferHandle* src, void* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferFromDevice(
            driver_, static_cast<const ExternalBufferHandle*>(src)->handle_,
            dst, wait_for.size(), tpu_events));
    delete[] tpu_events;
    return event;
  }

  std::shared_ptr<Event> TransferFromDeviceToDevice(
      const BufferHandle* src, BufferHandle* dst,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_TransferFromDeviceToDevice(
            driver_, static_cast<const ExternalBufferHandle*>(src)->handle_,
            static_cast<ExternalBufferHandle*>(dst)->handle_, wait_for.size(),
            tpu_events));
    delete[] tpu_events;
    return event;
  }

  std::unique_ptr<CompiledProgramHandle> CompileProgram(
      const xla::HloProto& source, int32_t num_replicas,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    struct HloProto hlo;
    hlo.size = source.ByteSizeLong();
    hlo.buffer = malloc(hlo.size);
    if (!source.SerializeToArray(hlo.buffer, hlo.size)) {
      LOG(ERROR) << "Unable to serialize HLO to array.";
      return nullptr;
    }

    auto handle = absl::make_unique<ExternalCompiledProgramHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_CompileProgram(driver_, hlo, num_replicas,
                                            wait_for.size(), tpu_events));

    free(hlo.buffer);
    delete[] tpu_events;
    return handle;
  }
  std::unique_ptr<LoadedProgramHandle> LoadProgram(
      int32_t core_id, const CompiledProgramHandle* handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    auto loaded_handle = absl::make_unique<ExternalLoadedProgramHandle>(
        &driver_fn_,
        driver_fn_.TpuDriver_LoadProgram(
            driver_, core_id,
            static_cast<const ExternalCompiledProgramHandle*>(handle)->handle_,
            wait_for.size(), tpu_events));

    delete[] tpu_events;
    return loaded_handle;
  }

  std::shared_ptr<Event> UnloadProgram(
      std::unique_ptr<LoadedProgramHandle> handle,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_UnloadProgram(
            driver_,
            static_cast<ExternalLoadedProgramHandle*>(handle.get())->handle_,
            wait_for.size(), tpu_events));
    delete[] tpu_events;
    return event;
  }

  std::shared_ptr<Event> ExecuteProgram(
      LoadedProgramHandle* program, absl::Span<BufferHandle* const> inputs,
      absl::Span<BufferHandle* const> outputs,
      const xla::DeviceAssignmentProto& device_assignment,
      absl::Span<Event* const> wait_for) override {
    auto tpu_events = MakeEventArray(wait_for);

    std::vector<::TpuBufferHandle*> inputv;
    inputv.reserve(inputs.size());
    for (int i = 0; i < inputs.size(); i++) {
      inputv.push_back(
          static_cast<ExternalBufferHandle* const>(inputs[i])->handle_);
    }
    std::vector<::TpuBufferHandle*> outputv;
    outputv.reserve(outputs.size());
    for (int i = 0; i < outputs.size(); i++) {
      outputv.push_back(
          static_cast<ExternalBufferHandle* const>(outputs[i])->handle_);
    }

    struct DeviceAssignment da = {device_assignment.replica_count(),
                                  device_assignment.computation_count()};
    auto event = std::make_shared<ExternalEvent>(
        &driver_fn_,
        driver_fn_.TpuDriver_ExecuteProgram(
            driver_,
            static_cast<ExternalLoadedProgramHandle*>(program)->handle_,
            inputs.size(), inputv.data(), outputs.size(), outputv.data(), da,
            wait_for.size(), tpu_events));

    delete[] tpu_events;
    return event;
  }

  std::unique_ptr<TpuLinearizer> GetLinearizer() override { return nullptr; }

 private:
  ::TpuDriverFn driver_fn_;
  ::TpuDriver* driver_;

  ::TpuEvent** MakeEventArray(absl::Span<Event* const> wait_for) {
    if (wait_for.empty()) return nullptr;
    ::TpuEvent** ret = new ::TpuEvent*[wait_for.size()];
    for (int i = 0; i < wait_for.size(); i++) {
      ret[i] = static_cast<ExternalEvent* const>(wait_for[i])->event_;
    }
    return ret;
  }
};

xla::StatusOr<std::unique_ptr<TpuDriver>> RegisterExternalTpuDriver(
    const TpuDriverConfig& config) {
  std::string shared_lib = config.worker().substr(strlen("external://"));
  return xla::StatusOr<std::unique_ptr<TpuDriver>>(
      absl::make_unique<ExternalTpuDriver>(shared_lib));
}

REGISTER_TPU_DRIVER("external://", RegisterExternalTpuDriver);

}  // namespace
}  // namespace tpu_driver
<|endoftext|>