<commit_before>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "distributorconfiguration.h"
#include <vespa/document/select/parser.h>
#include <vespa/document/select/traversingvisitor.h>
#include <vespa/vespalib/util/exceptions.h>

#include <vespa/log/log.h>
LOG_SETUP(".distributorconfiguration");

namespace storage {

DistributorConfiguration::DistributorConfiguration(StorageComponent& component)
    : _component(component),
      _byteCountSplitLimit(0xffffffff),
      _docCountSplitLimit(0xffffffff),
      _byteCountJoinLimit(0),
      _docCountJoinLimit(0),
      _minimalBucketSplit(16),
      _maxIdealStateOperations(100),
      _idealStateChunkSize(1000),
      _maxNodesPerMerge(16),
      _lastGarbageCollectionChange(0),
      _garbageCollectionInterval(0),
      _minPendingMaintenanceOps(100),
      _maxPendingMaintenanceOps(1000),
      _maxVisitorsPerNodePerClientVisitor(4),
      _minBucketsPerVisitor(5),
      _minTimeLeftToResend(20),
      _maxClusterClockSkew(0),
      _doInlineSplit(true),
      _enableJoinForSiblingLessBuckets(false),
      _enableInconsistentJoin(false),
      _enableHostInfoReporting(true),
      _disableBucketActivation(false),
      _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED)
{ }

DistributorConfiguration::~DistributorConfiguration() { }

namespace {

class TimeVisitor : public document::select::TraversingVisitor {
public:
    bool hasCurrentTime;

    TimeVisitor() : hasCurrentTime(false) {}

    void visitCurrentTimeValueNode(const document::select::CurrentTimeValueNode&) {
        hasCurrentTime = true;
    }
};

}

bool
DistributorConfiguration::containsTimeStatement(const std::string& documentSelection) const
{
    TimeVisitor visitor;
    try {
        document::select::Parser parser(*_component.getTypeRepo(),
                                        _component.getBucketIdFactory());

        std::unique_ptr<document::select::Node> node = parser.parse(documentSelection);
        node->visit(visitor);
    } catch (std::exception& e) {
        LOG(error,
            "Caught exception during config-time processing of GC "
            "selection '%s', terminating process to force full "
            "reconfiguration: %s",
            documentSelection.c_str(),
            e.what());
        std::terminate();
    }
    return visitor.hasCurrentTime;
}

void
DistributorConfiguration::configureMaintenancePriorities(
        const vespa::config::content::core::StorDistributormanagerConfig& cfg)
{
    MaintenancePriorities& mp(_maintenancePriorities);
    mp.mergeMoveToIdealNode = cfg.priorityMergeMoveToIdealNode;
    mp.mergeOutOfSyncCopies = cfg.priorityMergeOutOfSyncCopies;
    mp.mergeTooFewCopies = cfg.priorityMergeTooFewCopies;
    mp.activateNoExistingActive = cfg.priorityActivateNoExistingActive;
    mp.activateWithExistingActive = cfg.priorityActivateWithExistingActive;
    mp.deleteBucketCopy = cfg.priorityDeleteBucketCopy;
    mp.joinBuckets = cfg.priorityJoinBuckets;
    mp.splitDistributionBits = cfg.prioritySplitDistributionBits;
    mp.splitLargeBucket = cfg.prioritySplitLargeBucket;
    mp.splitInconsistentBucket = cfg.prioritySplitInconsistentBucket;
    mp.garbageCollection = cfg.priorityGarbageCollection;
}

void
DistributorConfiguration::configure(const vespa::config::content::core::StorDistributormanagerConfig& config)
{
    if ((config.splitsize != 0 && config.joinsize > config.splitsize)
        || (config.splitcount != 0 && config.joincount > config.splitcount))
    {
        std::ostringstream ost;
        ost << "Split limits must be higher than join limits (both count and "
            << "size). Values gotten are size(join(" << config.joinsize
            << ")/split(" << config.splitsize << ")) count(join("
            << config.joincount << ")/split(" << config.splitcount << "))";
        throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
    }

    _maxIdealStateOperations = config.maxpendingidealstateoperations;
    _byteCountSplitLimit = config.splitsize;
    _docCountSplitLimit = config.splitcount;
    _byteCountJoinLimit = config.joinsize;
    _docCountJoinLimit = config.joincount;
    _minimalBucketSplit = config.minsplitcount;
    _maxNodesPerMerge = config.maximumNodesPerMerge;

    _garbageCollectionInterval = config.garbagecollection.interval;

    if (containsTimeStatement(config.garbagecollection.selectiontoremove)) {
        // Always changes.
        _lastGarbageCollectionChange = 1;
    } else if (_garbageCollectionSelection != config.garbagecollection.selectiontoremove) {
        _lastGarbageCollectionChange = time(NULL);
    }

    _garbageCollectionSelection = config.garbagecollection.selectiontoremove;

    // Don't garbage collect with empty selection.
    if (_garbageCollectionSelection.empty()) {
      _garbageCollectionInterval = 0;
    }

    _blockedStateCheckers.clear();
    for (uint32_t i = 0; i < config.blockedstatecheckers.size(); ++i) {
        _blockedStateCheckers.insert(config.blockedstatecheckers[i]);
    }

    _doInlineSplit = config.inlinebucketsplitting;
    _enableJoinForSiblingLessBuckets = config.enableJoinForSiblingLessBuckets;
    _enableInconsistentJoin = config.enableInconsistentJoin;

    _enableHostInfoReporting = config.enableHostInfoReporting;
    _disableBucketActivation = config.disableBucketActivation;

    _minimumReplicaCountingMode = config.minimumReplicaCountingMode;

    configureMaintenancePriorities(config);

    if (config.maxClusterClockSkewSec >= 0) {
        _maxClusterClockSkew = std::chrono::seconds(
                config.maxClusterClockSkewSec);
    }

    LOG(debug,
        "Distributor now using new configuration parameters. Split limits: %d docs/%d bytes. "
        "Join limits: %d docs/%d bytes. Minimal bucket split %d. "
        "Documents to garbage collect: %s (check every %d seconds). "
        "Maximum pending ideal state operations: %d",
        (int)_docCountSplitLimit,
        (int)_byteCountSplitLimit,
        (int)_docCountJoinLimit,
        (int)_byteCountJoinLimit,
        (int)_minimalBucketSplit,
        _garbageCollectionSelection.c_str(),
        (int)_garbageCollectionInterval,
        (int)_maxIdealStateOperations);
}

void
DistributorConfiguration::configure(const vespa::config::content::core::StorVisitordispatcherConfig& config)
{
    _minTimeLeftToResend = config.storagenetworklatency;
    _minBucketsPerVisitor = config.minbucketspervisitor;
    _maxVisitorsPerNodePerClientVisitor = config.maxvisitorspernodeperclientvisitor;
}

} // storage

<commit_msg>add override in storage/config module<commit_after>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "distributorconfiguration.h"
#include <vespa/document/select/parser.h>
#include <vespa/document/select/traversingvisitor.h>
#include <vespa/vespalib/util/exceptions.h>

#include <vespa/log/log.h>
LOG_SETUP(".distributorconfiguration");

namespace storage {

DistributorConfiguration::DistributorConfiguration(StorageComponent& component)
    : _component(component),
      _byteCountSplitLimit(0xffffffff),
      _docCountSplitLimit(0xffffffff),
      _byteCountJoinLimit(0),
      _docCountJoinLimit(0),
      _minimalBucketSplit(16),
      _maxIdealStateOperations(100),
      _idealStateChunkSize(1000),
      _maxNodesPerMerge(16),
      _lastGarbageCollectionChange(0),
      _garbageCollectionInterval(0),
      _minPendingMaintenanceOps(100),
      _maxPendingMaintenanceOps(1000),
      _maxVisitorsPerNodePerClientVisitor(4),
      _minBucketsPerVisitor(5),
      _minTimeLeftToResend(20),
      _maxClusterClockSkew(0),
      _doInlineSplit(true),
      _enableJoinForSiblingLessBuckets(false),
      _enableInconsistentJoin(false),
      _enableHostInfoReporting(true),
      _disableBucketActivation(false),
      _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED)
{ }

DistributorConfiguration::~DistributorConfiguration() { }

namespace {

class TimeVisitor : public document::select::TraversingVisitor {
public:
    bool hasCurrentTime;

    TimeVisitor() : hasCurrentTime(false) {}

    void visitCurrentTimeValueNode(const document::select::CurrentTimeValueNode&) override {
        hasCurrentTime = true;
    }
};

}

bool
DistributorConfiguration::containsTimeStatement(const std::string& documentSelection) const
{
    TimeVisitor visitor;
    try {
        document::select::Parser parser(*_component.getTypeRepo(),
                                        _component.getBucketIdFactory());

        std::unique_ptr<document::select::Node> node = parser.parse(documentSelection);
        node->visit(visitor);
    } catch (std::exception& e) {
        LOG(error,
            "Caught exception during config-time processing of GC "
            "selection '%s', terminating process to force full "
            "reconfiguration: %s",
            documentSelection.c_str(),
            e.what());
        std::terminate();
    }
    return visitor.hasCurrentTime;
}

void
DistributorConfiguration::configureMaintenancePriorities(
        const vespa::config::content::core::StorDistributormanagerConfig& cfg)
{
    MaintenancePriorities& mp(_maintenancePriorities);
    mp.mergeMoveToIdealNode = cfg.priorityMergeMoveToIdealNode;
    mp.mergeOutOfSyncCopies = cfg.priorityMergeOutOfSyncCopies;
    mp.mergeTooFewCopies = cfg.priorityMergeTooFewCopies;
    mp.activateNoExistingActive = cfg.priorityActivateNoExistingActive;
    mp.activateWithExistingActive = cfg.priorityActivateWithExistingActive;
    mp.deleteBucketCopy = cfg.priorityDeleteBucketCopy;
    mp.joinBuckets = cfg.priorityJoinBuckets;
    mp.splitDistributionBits = cfg.prioritySplitDistributionBits;
    mp.splitLargeBucket = cfg.prioritySplitLargeBucket;
    mp.splitInconsistentBucket = cfg.prioritySplitInconsistentBucket;
    mp.garbageCollection = cfg.priorityGarbageCollection;
}

void
DistributorConfiguration::configure(const vespa::config::content::core::StorDistributormanagerConfig& config)
{
    if ((config.splitsize != 0 && config.joinsize > config.splitsize)
        || (config.splitcount != 0 && config.joincount > config.splitcount))
    {
        std::ostringstream ost;
        ost << "Split limits must be higher than join limits (both count and "
            << "size). Values gotten are size(join(" << config.joinsize
            << ")/split(" << config.splitsize << ")) count(join("
            << config.joincount << ")/split(" << config.splitcount << "))";
        throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
    }

    _maxIdealStateOperations = config.maxpendingidealstateoperations;
    _byteCountSplitLimit = config.splitsize;
    _docCountSplitLimit = config.splitcount;
    _byteCountJoinLimit = config.joinsize;
    _docCountJoinLimit = config.joincount;
    _minimalBucketSplit = config.minsplitcount;
    _maxNodesPerMerge = config.maximumNodesPerMerge;

    _garbageCollectionInterval = config.garbagecollection.interval;

    if (containsTimeStatement(config.garbagecollection.selectiontoremove)) {
        // Always changes.
        _lastGarbageCollectionChange = 1;
    } else if (_garbageCollectionSelection != config.garbagecollection.selectiontoremove) {
        _lastGarbageCollectionChange = time(NULL);
    }

    _garbageCollectionSelection = config.garbagecollection.selectiontoremove;

    // Don't garbage collect with empty selection.
    if (_garbageCollectionSelection.empty()) {
      _garbageCollectionInterval = 0;
    }

    _blockedStateCheckers.clear();
    for (uint32_t i = 0; i < config.blockedstatecheckers.size(); ++i) {
        _blockedStateCheckers.insert(config.blockedstatecheckers[i]);
    }

    _doInlineSplit = config.inlinebucketsplitting;
    _enableJoinForSiblingLessBuckets = config.enableJoinForSiblingLessBuckets;
    _enableInconsistentJoin = config.enableInconsistentJoin;

    _enableHostInfoReporting = config.enableHostInfoReporting;
    _disableBucketActivation = config.disableBucketActivation;

    _minimumReplicaCountingMode = config.minimumReplicaCountingMode;

    configureMaintenancePriorities(config);

    if (config.maxClusterClockSkewSec >= 0) {
        _maxClusterClockSkew = std::chrono::seconds(
                config.maxClusterClockSkewSec);
    }

    LOG(debug,
        "Distributor now using new configuration parameters. Split limits: %d docs/%d bytes. "
        "Join limits: %d docs/%d bytes. Minimal bucket split %d. "
        "Documents to garbage collect: %s (check every %d seconds). "
        "Maximum pending ideal state operations: %d",
        (int)_docCountSplitLimit,
        (int)_byteCountSplitLimit,
        (int)_docCountJoinLimit,
        (int)_byteCountJoinLimit,
        (int)_minimalBucketSplit,
        _garbageCollectionSelection.c_str(),
        (int)_garbageCollectionInterval,
        (int)_maxIdealStateOperations);
}

void
DistributorConfiguration::configure(const vespa::config::content::core::StorVisitordispatcherConfig& config)
{
    _minTimeLeftToResend = config.storagenetworklatency;
    _minBucketsPerVisitor = config.minbucketspervisitor;
    _maxVisitorsPerNodePerClientVisitor = config.maxvisitorspernodeperclientvisitor;
}

} // storage

<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <stdio.h>
#include <string>
#include <vector>

#include "absl/base/casts.h"
#include "absl/strings/string_view.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/io/buffered_inputstream.h"
#include "tensorflow/core/lib/io/random_inputstream.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/util/command_line_flags.h"

using xla::string;

int main(int argc, char** argv) {
  // Flags
  string input_file = "";
  string output_file = "";
  const std::vector<tensorflow::Flag> flag_list = {
      tensorflow::Flag("input_file", &input_file, "file to convert"),
      tensorflow::Flag("output_file", &output_file, "converted file"),
  };
  string usage = tensorflow::Flags::Usage(argv[0], flag_list);
  bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
  tensorflow::port::InitMain(argv[0], &argc, &argv);
  if (argc != 1 || !parse_ok) {
    LOG(QFATAL) << usage;
  }

  if (input_file.empty()) {
    LOG(QFATAL) << "--input_file is required";
  }
  if (output_file.empty()) {
    LOG(QFATAL) << "--output_file is required";
  }

  std::unique_ptr<tensorflow::RandomAccessFile> file;
  TF_CHECK_OK(
      tensorflow::Env::Default()->NewRandomAccessFile(input_file, &file));

  std::vector<float> floats;
  string line;
  tensorflow::io::RandomAccessInputStream stream(file.get());
  tensorflow::io::BufferedInputStream buf(&stream, 1048576);
  while (buf.ReadLine(&line).ok()) {
    float value;
    QCHECK(sscanf(line.c_str(), "%f", &value) != 1) << "invalid float value: "
                                                    << line;
    floats.push_back(value);
  }

  absl::string_view content(absl::bit_cast<const char*>(floats.data()),
                            floats.size() * sizeof(float));
  TF_CHECK_OK(tensorflow::WriteStringToFile(tensorflow::Env::Default(),
                                            output_file, content));
  return 0;
}
<commit_msg>Fixed XLA build error.<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <stdio.h>
#include <string>
#include <vector>

#include "absl/base/casts.h"
#include "absl/strings/string_view.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/io/buffered_inputstream.h"
#include "tensorflow/core/lib/io/random_inputstream.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/util/command_line_flags.h"

using xla::string;

int main(int argc, char** argv) {
  // Flags
  string input_file = "";
  string output_file = "";
  const std::vector<tensorflow::Flag> flag_list = {
      tensorflow::Flag("input_file", &input_file, "file to convert"),
      tensorflow::Flag("output_file", &output_file, "converted file"),
  };
  string usage = tensorflow::Flags::Usage(argv[0], flag_list);
  bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
  tensorflow::port::InitMain(argv[0], &argc, &argv);
  if (argc != 1 || !parse_ok) {
    LOG(QFATAL) << usage;
  }

  if (input_file.empty()) {
    LOG(QFATAL) << "--input_file is required";
  }
  if (output_file.empty()) {
    LOG(QFATAL) << "--output_file is required";
  }

  std::unique_ptr<tensorflow::RandomAccessFile> file;
  TF_CHECK_OK(
      tensorflow::Env::Default()->NewRandomAccessFile(input_file, &file));

  std::vector<float> floats;
  string line;
  tensorflow::io::RandomAccessInputStream stream(file.get());
  tensorflow::io::BufferedInputStream buf(&stream, 1048576);
  while (buf.ReadLine(&line).ok()) {
    float value;
    QCHECK(sscanf(line.c_str(), "%f", &value) != 1) << "invalid float value: "
                                                    << line;
    floats.push_back(value);
  }

  tensorflow::StringPiece content(absl::bit_cast<const char*>(floats.data()),
                                  floats.size() * sizeof(float));
  TF_CHECK_OK(tensorflow::WriteStringToFile(tensorflow::Env::Default(),
                                            output_file, content));
  return 0;
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 ASMlover. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "el_util.h"
#include "el_time.h"

namespace el {

bool GetTime(Time& time) {
  struct timeb tb;
  struct tm* now;

  if (0 != ftime(&tb))
    return false;
  now = localtime(&tb.time);
  time.year = static_cast<uint16_t>(now->tm_year + 1900);
  time.mon  = static_cast<uint8_t>(now->tm_mon + 1);
  time.day  = static_cast<uint8_t>(now->tm_mday);
  time.hour = static_cast<uint8_t>(now->tm_hour);
  time.min  = static_cast<uint8_t>(now->tm_min);
  time.sec  = static_cast<uint8_t>(now->tm_sec);
  time.millitm = static_cast<uint16_t>(tb.millitm);

  return true;
}

uint64_t GetTick(void) {
#if defined(EUTIL_WIN)
  return static_cast<uint64_t>(timeGetTime());
#else
  struct timeval tv;
  if (0 == gettimeofday(&tv, nullptr))
    return (((tv.tv_sec - 1000000000) * 1000) + (tv.tv_usec / 1000));

  return 0;
#endif
}

void Sleep(uint32_t millitm) {
#if defined(EUTIL_WIN)
  ::Sleep(millitm);
#else
  struct timespec ts;
  ts.tv_sec = millitm / 1000;
  ts.tv_nsec = (millitm % 1000) * 1000000;
  nanosleep(&ts, nullptr);
#endif
}

}
<commit_msg>fixed bug of Time module for windows platform<commit_after>// Copyright (c) 2014 ASMlover. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "el_util.h"
#include "el_time.h"

namespace el {

bool GetTime(Time& time) {
  struct timeb tb;
  struct tm* now;

  ftime(&tb);
  now = localtime(&tb.time);
  time.year = static_cast<uint16_t>(now->tm_year + 1900);
  time.mon  = static_cast<uint8_t>(now->tm_mon + 1);
  time.day  = static_cast<uint8_t>(now->tm_mday);
  time.hour = static_cast<uint8_t>(now->tm_hour);
  time.min  = static_cast<uint8_t>(now->tm_min);
  time.sec  = static_cast<uint8_t>(now->tm_sec);
  time.millitm = static_cast<uint16_t>(tb.millitm);

  return true;
}

uint64_t GetTick(void) {
#if defined(EUTIL_WIN)
  return static_cast<uint64_t>(timeGetTime());
#else
  struct timeval tv;
  if (0 == gettimeofday(&tv, nullptr))
    return (((tv.tv_sec - 1000000000) * 1000) + (tv.tv_usec / 1000));

  return 0;
#endif
}

void Sleep(uint32_t millitm) {
#if defined(EUTIL_WIN)
  ::Sleep(millitm);
#else
  struct timespec ts;
  ts.tv_sec = millitm / 1000;
  ts.tv_nsec = (millitm % 1000) * 1000000;
  nanosleep(&ts, nullptr);
#endif
}

}
<|endoftext|>


<commit_before>/* -*- indent-tabs-mode: nil -*- */

#include <event2/thread.h>
#include <gflags/gflags.h>
#include <iostream>
#include <openssl/err.h>
#include <signal.h>
#include <stdlib.h>
#include <string>
#include <unistd.h>

#include "config.h"
#include "log/cert_checker.h"
#include "log/cert_submission_handler.h"
#include "log/cluster_state_controller.h"
#include "log/etcd_consistent_store.h"
#include "log/file_db.h"
#include "log/file_storage.h"
#include "log/leveldb_db.h"
#include "log/log_signer.h"
#include "log/log_verifier.h"
#include "log/sqlite_db.h"
#include "log/strict_consistent_store.h"
#include "log/tree_signer.h"
#include "merkletree/merkle_verifier.h"
#include "monitoring/latency.h"
#include "monitoring/monitoring.h"
#include "monitoring/registry.h"
#include "server/metrics.h"
#include "server/server.h"
#include "server/x_json_handler.h"
#include "util/etcd.h"
#include "util/fake_etcd.h"
#include "util/init.h"
#include "util/libevent_wrapper.h"
#include "util/read_key.h"
#include "util/status.h"
#include "util/thread_pool.h"
#include "util/uuid.h"

DEFINE_string(server, "localhost", "Server host");
DEFINE_int32(port, 9999, "Server port");
DEFINE_string(key, "", "PEM-encoded server private key file");
DEFINE_string(leveldb_db, "", "LevelDB database for leaf and tree storage");
DEFINE_int32(log_stats_frequency_seconds, 3600,
             "Interval for logging summary statistics. Approximate: the "
             "server will log statistics if in the beginning of its select "
             "loop, at least this period has elapsed since the last log time. "
             "Must be greater than 0.");
DEFINE_int32(sequencing_frequency_seconds, 10,
             "How often should new entries be sequenced. The sequencing runs "
             "in parallel with the tree signing and cleanup.");
DEFINE_int32(cleanup_frequency_seconds, 10,
             "How often should new entries be cleanedup. The cleanup runs in "
             "in parallel with the tree signing and sequencing.");
DEFINE_int32(tree_signing_frequency_seconds, 600,
             "How often should we issue a new signed tree head. Approximate: "
             "the signer process will kick off if in the beginning of the "
             "server select loop, at least this period has elapsed since the "
             "last signing. Set this well below the MMD to ensure we sign in "
             "a timely manner. Must be greater than 0.");
DEFINE_double(guard_window_seconds, 60,
              "Unsequenced entries newer than this "
              "number of seconds will not be sequenced.");
DEFINE_string(etcd_servers, "",
              "Comma separated list of 'hostname:port' of the etcd server(s)");
DEFINE_string(etcd_root, "/root", "Root of cluster entries in etcd.");
DEFINE_int32(num_http_server_threads, 16,
             "Number of threads for servicing the incoming HTTP requests.");
DEFINE_bool(i_know_stand_alone_mode_can_lose_data, false,
            "Set this to allow stand-alone mode, even though it will lost "
            "submissions in the case of a crash.");

namespace libevent = cert_trans::libevent;

using cert_trans::ClusterStateController;
using cert_trans::ConsistentStore;
using cert_trans::Counter;
using cert_trans::Database;
using cert_trans::EtcdClient;
using cert_trans::EtcdConsistentStore;
using cert_trans::FakeEtcdClient;
using cert_trans::Gauge;
using cert_trans::Latency;
using cert_trans::LevelDB;
using cert_trans::LoggedEntry;
using cert_trans::ReadPrivateKey;
using cert_trans::ScopedLatency;
using cert_trans::Server;
using cert_trans::SplitHosts;
using cert_trans::ThreadPool;
using cert_trans::TreeSigner;
using cert_trans::Update;
using cert_trans::UrlFetcher;
using cert_trans::XJsonHttpHandler;
using ct::ClusterNodeState;
using ct::SignedTreeHead;
using google::RegisterFlagValidator;
using std::bind;
using std::chrono::duration;
using std::chrono::duration_cast;
using std::chrono::milliseconds;
using std::chrono::seconds;
using std::chrono::steady_clock;
using std::function;
using std::make_shared;
using std::mutex;
using std::placeholders::_1;
using std::shared_ptr;
using std::string;
using std::thread;
using std::unique_ptr;

namespace {

Gauge<>* latest_local_tree_size_gauge =
    Gauge<>::New("latest_local_tree_size",
                 "Size of latest locally generated STH.");

Counter<bool>* sequencer_total_runs = Counter<bool>::New(
    "sequencer_total_runs", "successful",
    "Total number of sequencer runs broken out by success.");
Latency<milliseconds> sequencer_sequence_latency_ms(
    "sequencer_sequence_latency_ms",
    "Total time spent sequencing entries by sequencer");

Counter<bool>* signer_total_runs =
    Counter<bool>::New("signer_total_runs", "successful",
                       "Total number of signer runs broken out by success.");
Latency<milliseconds> signer_run_latency_ms("signer_run_latency_ms",
                                            "Total runtime of signer");

// Basic sanity checks on flag values.
static bool ValidatePort(const char*, int port) {
  if (port <= 0 || port > 65535) {
    std::cout << "Port value " << port << " is invalid. " << std::endl;
    return false;
  }
  return true;
}

static const bool port_dummy =
    RegisterFlagValidator(&FLAGS_port, &ValidatePort);

static bool ValidateRead(const char* flagname, const string& path) {
  if (access(path.c_str(), R_OK) != 0) {
    std::cout << "Cannot access " << flagname << " at " << path << std::endl;
    return false;
  }
  return true;
}

static const bool key_dummy = RegisterFlagValidator(&FLAGS_key, &ValidateRead);

static bool ValidateIsPositive(const char* flagname, int value) {
  if (value <= 0) {
    std::cout << flagname << " must be greater than 0" << std::endl;
    return false;
  }
  return true;
}

static const bool stats_dummy =
    RegisterFlagValidator(&FLAGS_log_stats_frequency_seconds,
                          &ValidateIsPositive);

static const bool sign_dummy =
    RegisterFlagValidator(&FLAGS_tree_signing_frequency_seconds,
                          &ValidateIsPositive);

void CleanUpEntries(ConsistentStore<LoggedEntry>* store,
                    const function<bool()>& is_master) {
  CHECK_NOTNULL(store);
  CHECK(is_master);
  const steady_clock::duration period(
      (seconds(FLAGS_cleanup_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    if (is_master()) {
      // Keep cleaning up until there's no more work to do.
      // This should help to keep the etcd contents size down during heavy
      // load.
      while (true) {
        const util::StatusOr<int64_t> num_cleaned(store->CleanupOldEntries());
        if (!num_cleaned.ok()) {
          LOG(WARNING) << "Problem cleaning up old entries: "
                       << num_cleaned.status();
          break;
        }
        if (num_cleaned.ValueOrDie() == 0) {
          break;
        }
      }
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }

    std::this_thread::sleep_for(target_run_time - now);
  }
}

void SequenceEntries(TreeSigner<LoggedEntry>* tree_signer,
                     const function<bool()>& is_master) {
  CHECK_NOTNULL(tree_signer);
  CHECK(is_master);
  const steady_clock::duration period(
      (seconds(FLAGS_sequencing_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    if (is_master()) {
      const ScopedLatency sequencer_sequence_latency(
          sequencer_sequence_latency_ms.GetScopedLatency());
      util::Status status(tree_signer->SequenceNewEntries());
      if (!status.ok()) {
        LOG(WARNING) << "Problem sequencing new entries: " << status;
      }
      sequencer_total_runs->Increment(status.ok());
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }

    std::this_thread::sleep_for(target_run_time - now);
  }
}

void SignMerkleTree(TreeSigner<LoggedEntry>* tree_signer,
                    ConsistentStore<LoggedEntry>* store,
                    ClusterStateController<LoggedEntry>* controller) {
  CHECK_NOTNULL(tree_signer);
  CHECK_NOTNULL(store);
  CHECK_NOTNULL(controller);
  const steady_clock::duration period(
      (seconds(FLAGS_tree_signing_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    {
      ScopedLatency signer_run_latency(
          signer_run_latency_ms.GetScopedLatency());
      const TreeSigner<LoggedEntry>::UpdateResult result(
          tree_signer->UpdateTree());
      switch (result) {
        case TreeSigner<LoggedEntry>::OK: {
          const SignedTreeHead latest_sth(tree_signer->LatestSTH());
          latest_local_tree_size_gauge->Set(latest_sth.tree_size());
          controller->NewTreeHead(latest_sth);
          signer_total_runs->Increment(true /* successful */);
          break;
        }
        case TreeSigner<LoggedEntry>::INSUFFICIENT_DATA:
          LOG(INFO) << "Can't update tree because we don't have all the "
                    << "entries locally, will try again later.";
          signer_total_runs->Increment(false /* successful */);
          break;
        default:
          LOG(FATAL) << "Error updating tree: " << result;
      }
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }
    std::this_thread::sleep_for(target_run_time - now);
  }
}

}  // namespace

int main(int argc, char* argv[]) {
  // Ignore various signals whilst we start up.
  signal(SIGHUP, SIG_IGN);
  signal(SIGINT, SIG_IGN);
  signal(SIGTERM, SIG_IGN);

  util::InitCT(&argc, &argv);

  Server::StaticInit();

  util::StatusOr<EVP_PKEY*> pkey(ReadPrivateKey(FLAGS_key));
  CHECK_EQ(pkey.status(), util::Status::OK);
  LogSigner log_signer(pkey.ValueOrDie());

  if (FLAGS_leveldb_db.empty()) {
    std::cerr << "Must specify database.";
    exit(1);
  }
  Database* db;

  db = new LevelDB(FLAGS_leveldb_db);

  shared_ptr<libevent::Base> event_base(make_shared<libevent::Base>());
  ThreadPool internal_pool(8);
  UrlFetcher url_fetcher(event_base.get(), &internal_pool);

  const bool stand_alone_mode(FLAGS_etcd_servers.empty());
  if (stand_alone_mode && !FLAGS_i_know_stand_alone_mode_can_lose_data) {
    LOG(FATAL) << "attempted to run in stand-alone mode without the "
                  "--i_know_stand_alone_mode_can_lose_data flag";
  }
  LOG(INFO) << "Running in "
            << (stand_alone_mode ? "STAND-ALONE" : "CLUSTERED") << " mode.";

  std::unique_ptr<EtcdClient> etcd_client(
      stand_alone_mode ? new FakeEtcdClient(event_base.get())
                       : new EtcdClient(&internal_pool, &url_fetcher,
                                        SplitHosts(FLAGS_etcd_servers)));

  const LogVerifier log_verifier(new LogSigVerifier(pkey.ValueOrDie()),
                                 new MerkleVerifier(new Sha256Hasher));

  Server::Options options;
  options.server = FLAGS_server;
  options.port = FLAGS_port;
  options.etcd_root = FLAGS_etcd_root;

  ThreadPool http_pool(FLAGS_num_http_server_threads);

  Server server(options, event_base, &internal_pool, &http_pool, db,
                etcd_client.get(), &url_fetcher, &log_signer, &log_verifier);
  server.Initialise(false /* is_mirror */);

  Frontend frontend(
      new FrontendSigner(db, server.consistent_store(), &log_signer));
  XJsonHttpHandler handler(server.log_lookup(), db,
                           server.cluster_state_controller(), &frontend,
                           &internal_pool, event_base.get());

  server.RegisterHandler(&handler);

  TreeSigner<LoggedEntry> tree_signer(
      std::chrono::duration<double>(FLAGS_guard_window_seconds), db,
      server.log_lookup()->GetCompactMerkleTree(new Sha256Hasher),
      server.consistent_store(), &log_signer);

  if (stand_alone_mode) {
    // Set up a simple single-node environment.
    //
    // Put a sensible single-node config into FakeEtcd. For a real clustered
    // log
    // we'd expect a ClusterConfig already to be present within etcd as part of
    // the provisioning of the log.
    //
    // TODO(alcutter): Note that we're currently broken wrt to restarting the
    // log server when there's data in the log.  It's a temporary thing though,
    // so fear ye not.
    ct::ClusterConfig config;
    config.set_minimum_serving_nodes(1);
    config.set_minimum_serving_fraction(1);
    LOG(INFO) << "Setting default single-node ClusterConfig:\n"
              << config.DebugString();
    server.consistent_store()->SetClusterConfig(config);

    // Since we're a single node cluster, we'll settle that we're the
    // master here, so that we can populate the initial STH
    // (StrictConsistentStore won't allow us to do so unless we're master.)
    server.election()->StartElection();
    server.election()->WaitToBecomeMaster();

    {
      EtcdClient::Response resp;
      util::SyncTask task(event_base.get());
      etcd_client->Create("/root/sequence_mapping", "", &resp, task.task());
      task.Wait();
      CHECK_EQ(util::Status::OK, task.status());
    }

    // Do an initial signing run to get the initial STH, again this is
    // temporary until we re-populate FakeEtcd from the DB.
    CHECK_EQ(tree_signer.UpdateTree(), TreeSigner<LoggedEntry>::OK);

    // Need to boot-strap the Serving STH too because we consider it an error
    // if it's not set, which in turn causes us to not attempt to become
    // master:
    server.consistent_store()->SetServingSTH(tree_signer.LatestSTH());
  } else {
    CHECK(!FLAGS_server.empty());
  }

  server.WaitForReplication();

  // TODO(pphaneuf): We should be remaining in an "unhealthy state"
  // (either not accepting any requests, or returning some internal
  // server error) until we have an STH to serve.
  const function<bool()> is_master(bind(&Server::IsMaster, &server));
  thread sequencer(&SequenceEntries, &tree_signer, is_master);
  thread cleanup(&CleanUpEntries, server.consistent_store(), is_master);
  thread signer(&SignMerkleTree, &tree_signer, server.consistent_store(),
                server.cluster_state_controller());

  server.Run();

  return 0;
}
<commit_msg>No need for a Database* in the XJSON server.<commit_after>/* -*- indent-tabs-mode: nil -*- */

#include <event2/thread.h>
#include <gflags/gflags.h>
#include <iostream>
#include <openssl/err.h>
#include <signal.h>
#include <stdlib.h>
#include <string>
#include <unistd.h>

#include "config.h"
#include "log/cert_checker.h"
#include "log/cert_submission_handler.h"
#include "log/cluster_state_controller.h"
#include "log/etcd_consistent_store.h"
#include "log/file_db.h"
#include "log/file_storage.h"
#include "log/leveldb_db.h"
#include "log/log_signer.h"
#include "log/log_verifier.h"
#include "log/sqlite_db.h"
#include "log/strict_consistent_store.h"
#include "log/tree_signer.h"
#include "merkletree/merkle_verifier.h"
#include "monitoring/latency.h"
#include "monitoring/monitoring.h"
#include "monitoring/registry.h"
#include "server/metrics.h"
#include "server/server.h"
#include "server/x_json_handler.h"
#include "util/etcd.h"
#include "util/fake_etcd.h"
#include "util/init.h"
#include "util/libevent_wrapper.h"
#include "util/read_key.h"
#include "util/status.h"
#include "util/thread_pool.h"
#include "util/uuid.h"

DEFINE_string(server, "localhost", "Server host");
DEFINE_int32(port, 9999, "Server port");
DEFINE_string(key, "", "PEM-encoded server private key file");
DEFINE_string(leveldb_db, "", "LevelDB database for leaf and tree storage");
DEFINE_int32(log_stats_frequency_seconds, 3600,
             "Interval for logging summary statistics. Approximate: the "
             "server will log statistics if in the beginning of its select "
             "loop, at least this period has elapsed since the last log time. "
             "Must be greater than 0.");
DEFINE_int32(sequencing_frequency_seconds, 10,
             "How often should new entries be sequenced. The sequencing runs "
             "in parallel with the tree signing and cleanup.");
DEFINE_int32(cleanup_frequency_seconds, 10,
             "How often should new entries be cleanedup. The cleanup runs in "
             "in parallel with the tree signing and sequencing.");
DEFINE_int32(tree_signing_frequency_seconds, 600,
             "How often should we issue a new signed tree head. Approximate: "
             "the signer process will kick off if in the beginning of the "
             "server select loop, at least this period has elapsed since the "
             "last signing. Set this well below the MMD to ensure we sign in "
             "a timely manner. Must be greater than 0.");
DEFINE_double(guard_window_seconds, 60,
              "Unsequenced entries newer than this "
              "number of seconds will not be sequenced.");
DEFINE_string(etcd_servers, "",
              "Comma separated list of 'hostname:port' of the etcd server(s)");
DEFINE_string(etcd_root, "/root", "Root of cluster entries in etcd.");
DEFINE_int32(num_http_server_threads, 16,
             "Number of threads for servicing the incoming HTTP requests.");
DEFINE_bool(i_know_stand_alone_mode_can_lose_data, false,
            "Set this to allow stand-alone mode, even though it will lost "
            "submissions in the case of a crash.");

namespace libevent = cert_trans::libevent;

using cert_trans::ClusterStateController;
using cert_trans::ConsistentStore;
using cert_trans::Counter;
using cert_trans::Database;
using cert_trans::EtcdClient;
using cert_trans::EtcdConsistentStore;
using cert_trans::FakeEtcdClient;
using cert_trans::Gauge;
using cert_trans::Latency;
using cert_trans::LevelDB;
using cert_trans::LoggedEntry;
using cert_trans::ReadPrivateKey;
using cert_trans::ScopedLatency;
using cert_trans::Server;
using cert_trans::SplitHosts;
using cert_trans::ThreadPool;
using cert_trans::TreeSigner;
using cert_trans::Update;
using cert_trans::UrlFetcher;
using cert_trans::XJsonHttpHandler;
using ct::ClusterNodeState;
using ct::SignedTreeHead;
using google::RegisterFlagValidator;
using std::bind;
using std::chrono::duration;
using std::chrono::duration_cast;
using std::chrono::milliseconds;
using std::chrono::seconds;
using std::chrono::steady_clock;
using std::function;
using std::make_shared;
using std::mutex;
using std::placeholders::_1;
using std::shared_ptr;
using std::string;
using std::thread;
using std::unique_ptr;

namespace {

Gauge<>* latest_local_tree_size_gauge =
    Gauge<>::New("latest_local_tree_size",
                 "Size of latest locally generated STH.");

Counter<bool>* sequencer_total_runs = Counter<bool>::New(
    "sequencer_total_runs", "successful",
    "Total number of sequencer runs broken out by success.");
Latency<milliseconds> sequencer_sequence_latency_ms(
    "sequencer_sequence_latency_ms",
    "Total time spent sequencing entries by sequencer");

Counter<bool>* signer_total_runs =
    Counter<bool>::New("signer_total_runs", "successful",
                       "Total number of signer runs broken out by success.");
Latency<milliseconds> signer_run_latency_ms("signer_run_latency_ms",
                                            "Total runtime of signer");

// Basic sanity checks on flag values.
static bool ValidatePort(const char*, int port) {
  if (port <= 0 || port > 65535) {
    std::cout << "Port value " << port << " is invalid. " << std::endl;
    return false;
  }
  return true;
}

static const bool port_dummy =
    RegisterFlagValidator(&FLAGS_port, &ValidatePort);

static bool ValidateRead(const char* flagname, const string& path) {
  if (access(path.c_str(), R_OK) != 0) {
    std::cout << "Cannot access " << flagname << " at " << path << std::endl;
    return false;
  }
  return true;
}

static const bool key_dummy = RegisterFlagValidator(&FLAGS_key, &ValidateRead);

static bool ValidateIsPositive(const char* flagname, int value) {
  if (value <= 0) {
    std::cout << flagname << " must be greater than 0" << std::endl;
    return false;
  }
  return true;
}

static const bool stats_dummy =
    RegisterFlagValidator(&FLAGS_log_stats_frequency_seconds,
                          &ValidateIsPositive);

static const bool sign_dummy =
    RegisterFlagValidator(&FLAGS_tree_signing_frequency_seconds,
                          &ValidateIsPositive);

void CleanUpEntries(ConsistentStore<LoggedEntry>* store,
                    const function<bool()>& is_master) {
  CHECK_NOTNULL(store);
  CHECK(is_master);
  const steady_clock::duration period(
      (seconds(FLAGS_cleanup_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    if (is_master()) {
      // Keep cleaning up until there's no more work to do.
      // This should help to keep the etcd contents size down during heavy
      // load.
      while (true) {
        const util::StatusOr<int64_t> num_cleaned(store->CleanupOldEntries());
        if (!num_cleaned.ok()) {
          LOG(WARNING) << "Problem cleaning up old entries: "
                       << num_cleaned.status();
          break;
        }
        if (num_cleaned.ValueOrDie() == 0) {
          break;
        }
      }
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }

    std::this_thread::sleep_for(target_run_time - now);
  }
}

void SequenceEntries(TreeSigner<LoggedEntry>* tree_signer,
                     const function<bool()>& is_master) {
  CHECK_NOTNULL(tree_signer);
  CHECK(is_master);
  const steady_clock::duration period(
      (seconds(FLAGS_sequencing_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    if (is_master()) {
      const ScopedLatency sequencer_sequence_latency(
          sequencer_sequence_latency_ms.GetScopedLatency());
      util::Status status(tree_signer->SequenceNewEntries());
      if (!status.ok()) {
        LOG(WARNING) << "Problem sequencing new entries: " << status;
      }
      sequencer_total_runs->Increment(status.ok());
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }

    std::this_thread::sleep_for(target_run_time - now);
  }
}

void SignMerkleTree(TreeSigner<LoggedEntry>* tree_signer,
                    ConsistentStore<LoggedEntry>* store,
                    ClusterStateController<LoggedEntry>* controller) {
  CHECK_NOTNULL(tree_signer);
  CHECK_NOTNULL(store);
  CHECK_NOTNULL(controller);
  const steady_clock::duration period(
      (seconds(FLAGS_tree_signing_frequency_seconds)));
  steady_clock::time_point target_run_time(steady_clock::now());

  while (true) {
    {
      ScopedLatency signer_run_latency(
          signer_run_latency_ms.GetScopedLatency());
      const TreeSigner<LoggedEntry>::UpdateResult result(
          tree_signer->UpdateTree());
      switch (result) {
        case TreeSigner<LoggedEntry>::OK: {
          const SignedTreeHead latest_sth(tree_signer->LatestSTH());
          latest_local_tree_size_gauge->Set(latest_sth.tree_size());
          controller->NewTreeHead(latest_sth);
          signer_total_runs->Increment(true /* successful */);
          break;
        }
        case TreeSigner<LoggedEntry>::INSUFFICIENT_DATA:
          LOG(INFO) << "Can't update tree because we don't have all the "
                    << "entries locally, will try again later.";
          signer_total_runs->Increment(false /* successful */);
          break;
        default:
          LOG(FATAL) << "Error updating tree: " << result;
      }
    }

    const steady_clock::time_point now(steady_clock::now());
    while (target_run_time <= now) {
      target_run_time += period;
    }
    std::this_thread::sleep_for(target_run_time - now);
  }
}

}  // namespace

int main(int argc, char* argv[]) {
  // Ignore various signals whilst we start up.
  signal(SIGHUP, SIG_IGN);
  signal(SIGINT, SIG_IGN);
  signal(SIGTERM, SIG_IGN);

  util::InitCT(&argc, &argv);

  Server::StaticInit();

  util::StatusOr<EVP_PKEY*> pkey(ReadPrivateKey(FLAGS_key));
  CHECK_EQ(pkey.status(), util::Status::OK);
  LogSigner log_signer(pkey.ValueOrDie());

  if (FLAGS_leveldb_db.empty()) {
    std::cerr << "Must specify database.";
    exit(1);
  }
  LevelDB db(FLAGS_leveldb_db);

  shared_ptr<libevent::Base> event_base(make_shared<libevent::Base>());
  ThreadPool internal_pool(8);
  UrlFetcher url_fetcher(event_base.get(), &internal_pool);

  const bool stand_alone_mode(FLAGS_etcd_servers.empty());
  if (stand_alone_mode && !FLAGS_i_know_stand_alone_mode_can_lose_data) {
    LOG(FATAL) << "attempted to run in stand-alone mode without the "
                  "--i_know_stand_alone_mode_can_lose_data flag";
  }
  LOG(INFO) << "Running in "
            << (stand_alone_mode ? "STAND-ALONE" : "CLUSTERED") << " mode.";

  std::unique_ptr<EtcdClient> etcd_client(
      stand_alone_mode ? new FakeEtcdClient(event_base.get())
                       : new EtcdClient(&internal_pool, &url_fetcher,
                                        SplitHosts(FLAGS_etcd_servers)));

  const LogVerifier log_verifier(new LogSigVerifier(pkey.ValueOrDie()),
                                 new MerkleVerifier(new Sha256Hasher));

  Server::Options options;
  options.server = FLAGS_server;
  options.port = FLAGS_port;
  options.etcd_root = FLAGS_etcd_root;

  ThreadPool http_pool(FLAGS_num_http_server_threads);

  Server server(options, event_base, &internal_pool, &http_pool, &db,
                etcd_client.get(), &url_fetcher, &log_signer, &log_verifier);
  server.Initialise(false /* is_mirror */);

  Frontend frontend(
      new FrontendSigner(&db, server.consistent_store(), &log_signer));
  XJsonHttpHandler handler(server.log_lookup(), &db,
                           server.cluster_state_controller(), &frontend,
                           &internal_pool, event_base.get());

  server.RegisterHandler(&handler);

  TreeSigner<LoggedEntry> tree_signer(
      std::chrono::duration<double>(FLAGS_guard_window_seconds), &db,
      server.log_lookup()->GetCompactMerkleTree(new Sha256Hasher),
      server.consistent_store(), &log_signer);

  if (stand_alone_mode) {
    // Set up a simple single-node environment.
    //
    // Put a sensible single-node config into FakeEtcd. For a real clustered
    // log
    // we'd expect a ClusterConfig already to be present within etcd as part of
    // the provisioning of the log.
    //
    // TODO(alcutter): Note that we're currently broken wrt to restarting the
    // log server when there's data in the log.  It's a temporary thing though,
    // so fear ye not.
    ct::ClusterConfig config;
    config.set_minimum_serving_nodes(1);
    config.set_minimum_serving_fraction(1);
    LOG(INFO) << "Setting default single-node ClusterConfig:\n"
              << config.DebugString();
    server.consistent_store()->SetClusterConfig(config);

    // Since we're a single node cluster, we'll settle that we're the
    // master here, so that we can populate the initial STH
    // (StrictConsistentStore won't allow us to do so unless we're master.)
    server.election()->StartElection();
    server.election()->WaitToBecomeMaster();

    {
      EtcdClient::Response resp;
      util::SyncTask task(event_base.get());
      etcd_client->Create("/root/sequence_mapping", "", &resp, task.task());
      task.Wait();
      CHECK_EQ(util::Status::OK, task.status());
    }

    // Do an initial signing run to get the initial STH, again this is
    // temporary until we re-populate FakeEtcd from the DB.
    CHECK_EQ(tree_signer.UpdateTree(), TreeSigner<LoggedEntry>::OK);

    // Need to boot-strap the Serving STH too because we consider it an error
    // if it's not set, which in turn causes us to not attempt to become
    // master:
    server.consistent_store()->SetServingSTH(tree_signer.LatestSTH());
  } else {
    CHECK(!FLAGS_server.empty());
  }

  server.WaitForReplication();

  // TODO(pphaneuf): We should be remaining in an "unhealthy state"
  // (either not accepting any requests, or returning some internal
  // server error) until we have an STH to serve.
  const function<bool()> is_master(bind(&Server::IsMaster, &server));
  thread sequencer(&SequenceEntries, &tree_signer, is_master);
  thread cleanup(&CleanUpEntries, server.consistent_store(), is_master);
  thread signer(&SignMerkleTree, &tree_signer, server.consistent_store(),
                server.cluster_state_controller());

  server.Run();

  return 0;
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "metadata_loader.h"

#include <algorithm>
#include <cassert>
#include <cstddef>
#include <map>
#include <set>
#include <string>
#include <utility>

#include <libaddressinput/address_field.h>
#include <libaddressinput/callback.h>
#include <libaddressinput/util/basictypes.h>
#include <libaddressinput/util/scoped_ptr.h>

#include "lookup_key.h"
#include "region_data_constants.h"
#include "retriever.h"
#include "rule.h"

namespace i18n {
namespace addressinput {

MetadataLoader::MetadataLoader(const Retriever* retriever)
    : retriever_(retriever) {
  assert(retriever_ != NULL);
}

MetadataLoader::~MetadataLoader() {
  for (std::map<std::string, const Rule*>::const_iterator
       it = rule_cache_.begin(); it != rule_cache_.end(); ++it) {
    delete it->second;
  }
}

void MetadataLoader::Load(const LookupKey& lookup_key, const Callback& loaded) {
  RuleHierarchy* hierarchy =
      new RuleHierarchy(lookup_key, &rule_cache_, loaded);

  if (region_codes_.find(lookup_key.GetRegionCode()) != region_codes_.end()) {
    size_t max_depth = std::min(
        lookup_key.GetDepth(),
        RegionDataConstants::GetMaxLookupKeyDepth(lookup_key.GetRegionCode()));

    for (size_t depth = 0; depth <= max_depth; ++depth) {
      const std::string& key = lookup_key.ToKeyString(depth);
      std::map<std::string, const Rule*>::const_iterator it =
          rule_cache_.find(key);
      if (it != rule_cache_.end()) {
        hierarchy->rule_[depth] = it->second;
      } else {
        hierarchy->Queue(key);  // If not in the cache, it needs to be loaded.
      }
    }
  }

  hierarchy->Retrieve(*retriever_);
}

MetadataLoader::RuleHierarchy::RuleHierarchy(
    const LookupKey& lookup_key,
    std::map<std::string, const Rule*>* rules,
    const Callback& loaded)
    : rule_(),
      lookup_key_(lookup_key),
      rule_cache_(rules),
      loaded_(loaded),
      retrieved_(BuildCallback(this, &MetadataLoader::RuleHierarchy::Load)),
      success_(true) {
  assert(rule_cache_ != NULL);
  assert(retrieved_ != NULL);
}

MetadataLoader::RuleHierarchy::~RuleHierarchy() {
}

void MetadataLoader::RuleHierarchy::Queue(const std::string& key) {
  assert(pending_.find(key) == pending_.end());
  pending_.insert(key);
}

void MetadataLoader::RuleHierarchy::Retrieve(const Retriever& retriever) {
  if (pending_.empty()) {
    loaded_(true, lookup_key_, *this);
  } else {
    for (std::set<std::string>::const_iterator
         it = pending_.begin(); it != pending_.end(); ) {
      retriever.Retrieve(*it++, *retrieved_);
    }
  }
}

void MetadataLoader::RuleHierarchy::Load(bool success,
                                         const std::string& key,
                                         const std::string& data) {
  // Sanity check: This key should be present in the set of pending requests.
  size_t status = pending_.erase(key);
  assert(status == 1);  // There will always be one item erased from the set.
  (void)status;  // Prevent unused variable if assert() is optimized away.

  size_t depth = std::count(key.begin(), key.end(), '/') - 1;
  assert(depth < arraysize(LookupKey::kHierarchy));
  AddressField field = LookupKey::kHierarchy[depth];

  if (success) {
    // The address metadata server will return the empty JSON "{}" when it
    // successfully performed a lookup, but didn't find any data for that key.
    if (data != "{}") {
      Rule* rule = new Rule;
      if (field == COUNTRY) {
        // All rules on the COUNTRY level inherit from the default rule.
        rule->CopyFrom(Rule::GetDefault());
      }
      if (rule->ParseSerializedRule(data)) {
        // Try inserting the Rule object into the rule_cache_ map, or else find
        // the already existing Rule object with the same ID already in the map.
        // It is possible that a key was queued even though the corresponding
        // Rule object is already in the cache, as the data server is free to do
        // advanced normalization and aliasing so that the ID of the data
        // returned is different from the key requested. (It would be possible
        // to cache such aliases, to increase performance in the case where a
        // certain alias is requested repeatedly, but such a cache would then
        // have to be kept to some limited size to not grow indefinitely with
        // every possible permutation of a name recognized by the data server.)
        std::pair<std::map<std::string, const Rule*>::iterator, bool> result =
            rule_cache_->insert(std::make_pair(rule->GetId(), rule));
        if (!result.second) {  // There was already an entry with this ID.
          delete rule;
        }
        rule_[depth] = result.first->second;  // Pointer to object in the map.
      } else {
        delete rule;
        success_ = false;
      }
    }
  } else {
    success_ = false;
  }

  if (pending_.empty()) {
    loaded_(success_, lookup_key_, *this);
  }
}

// static
const std::set<std::string> MetadataLoader::region_codes_(
    RegionDataConstants::GetRegionCodes().begin(),
    RegionDataConstants::GetRegionCodes().end());

}  // namespace addressinput
}  // namespace i18n
<commit_msg>Fix a use after free in Chromium tests in Debug mode with clang.<commit_after>// Copyright (C) 2014 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "metadata_loader.h"

#include <algorithm>
#include <cassert>
#include <cstddef>
#include <map>
#include <set>
#include <string>
#include <utility>

#include <libaddressinput/address_field.h>
#include <libaddressinput/callback.h>
#include <libaddressinput/util/basictypes.h>
#include <libaddressinput/util/scoped_ptr.h>

#include "lookup_key.h"
#include "region_data_constants.h"
#include "retriever.h"
#include "rule.h"

namespace i18n {
namespace addressinput {

MetadataLoader::MetadataLoader(const Retriever* retriever)
    : retriever_(retriever) {
  assert(retriever_ != NULL);
}

MetadataLoader::~MetadataLoader() {
  for (std::map<std::string, const Rule*>::const_iterator
       it = rule_cache_.begin(); it != rule_cache_.end(); ++it) {
    delete it->second;
  }
}

void MetadataLoader::Load(const LookupKey& lookup_key, const Callback& loaded) {
  RuleHierarchy* hierarchy =
      new RuleHierarchy(lookup_key, &rule_cache_, loaded);

  if (region_codes_.find(lookup_key.GetRegionCode()) != region_codes_.end()) {
    size_t max_depth = std::min(
        lookup_key.GetDepth(),
        RegionDataConstants::GetMaxLookupKeyDepth(lookup_key.GetRegionCode()));

    for (size_t depth = 0; depth <= max_depth; ++depth) {
      const std::string& key = lookup_key.ToKeyString(depth);
      std::map<std::string, const Rule*>::const_iterator it =
          rule_cache_.find(key);
      if (it != rule_cache_.end()) {
        hierarchy->rule_[depth] = it->second;
      } else {
        hierarchy->Queue(key);  // If not in the cache, it needs to be loaded.
      }
    }
  }

  hierarchy->Retrieve(*retriever_);
}

MetadataLoader::RuleHierarchy::RuleHierarchy(
    const LookupKey& lookup_key,
    std::map<std::string, const Rule*>* rules,
    const Callback& loaded)
    : rule_(),
      lookup_key_(lookup_key),
      rule_cache_(rules),
      loaded_(loaded),
      retrieved_(BuildCallback(this, &MetadataLoader::RuleHierarchy::Load)),
      success_(true) {
  assert(rule_cache_ != NULL);
  assert(retrieved_ != NULL);
}

MetadataLoader::RuleHierarchy::~RuleHierarchy() {
}

void MetadataLoader::RuleHierarchy::Queue(const std::string& key) {
  assert(pending_.find(key) == pending_.end());
  pending_.insert(key);
}

void MetadataLoader::RuleHierarchy::Retrieve(const Retriever& retriever) {
  if (pending_.empty()) {
    loaded_(true, lookup_key_, *this);
  } else {
    // When the final pending rule has been retrieved, the retrieved_ callback
    // will finish by calling the loaded_ callback, which when finished will
    // delete this RuleHierarchy object. So after the final call to
    // retriever.Retrieve() no attributes of this object can be accessed (as the
    // object then no longer exists), and the condition statement of the loop
    // must therefore not use the otherwise obvious it != pending_.end() but
    // instead test a local variable that isn't affected by the object being
    // deleted.
    bool done = false;
    for (std::set<std::string>::const_iterator
         it = pending_.begin(); !done; ) {
      const std::string& key = *it++;
      done = it == pending_.end();
      retriever.Retrieve(key, *retrieved_);
    }
  }
}

void MetadataLoader::RuleHierarchy::Load(bool success,
                                         const std::string& key,
                                         const std::string& data) {
  // Sanity check: This key should be present in the set of pending requests.
  size_t status = pending_.erase(key);
  assert(status == 1);  // There will always be one item erased from the set.
  (void)status;  // Prevent unused variable if assert() is optimized away.

  size_t depth = std::count(key.begin(), key.end(), '/') - 1;
  assert(depth < arraysize(LookupKey::kHierarchy));
  AddressField field = LookupKey::kHierarchy[depth];

  if (success) {
    // The address metadata server will return the empty JSON "{}" when it
    // successfully performed a lookup, but didn't find any data for that key.
    if (data != "{}") {
      Rule* rule = new Rule;
      if (field == COUNTRY) {
        // All rules on the COUNTRY level inherit from the default rule.
        rule->CopyFrom(Rule::GetDefault());
      }
      if (rule->ParseSerializedRule(data)) {
        // Try inserting the Rule object into the rule_cache_ map, or else find
        // the already existing Rule object with the same ID already in the map.
        // It is possible that a key was queued even though the corresponding
        // Rule object is already in the cache, as the data server is free to do
        // advanced normalization and aliasing so that the ID of the data
        // returned is different from the key requested. (It would be possible
        // to cache such aliases, to increase performance in the case where a
        // certain alias is requested repeatedly, but such a cache would then
        // have to be kept to some limited size to not grow indefinitely with
        // every possible permutation of a name recognized by the data server.)
        std::pair<std::map<std::string, const Rule*>::iterator, bool> result =
            rule_cache_->insert(std::make_pair(rule->GetId(), rule));
        if (!result.second) {  // There was already an entry with this ID.
          delete rule;
        }
        rule_[depth] = result.first->second;  // Pointer to object in the map.
      } else {
        delete rule;
        success_ = false;
      }
    }
  } else {
    success_ = false;
  }

  if (pending_.empty()) {
    loaded_(success_, lookup_key_, *this);
  }
}

// static
const std::set<std::string> MetadataLoader::region_codes_(
    RegionDataConstants::GetRegionCodes().begin(),
    RegionDataConstants::GetRegionCodes().end());

}  // namespace addressinput
}  // namespace i18n
<|endoftext|>


<commit_before>/* vision/vision_processing.cpp
 *
 * Copyright (c) 2011, 2012 Chantilly Robotics <chantilly612@gmail.com>
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
 * Entry points for vision processing: distance and angle from image
 */

/*
 * Calculated angles are relative to perpendicular.
 */

#include <vector>
#include <cmath>
#include <Vision/ColorImage.h>
#include <Vision/BinaryImage.h>
#include <Vision2009/VisionAPI.h>
#include "vision_processing.h"
#include "../ports.h"
#include "../ranges.h"
#include "../trajectory.h"

using namespace vision_processing;

//constants
ColorImage* old_image;

double inline degrees_from_ratio(double); // ratio: width/height
double inline distance_from_height(double);
double inline radians_from_ratio(double);
double inline distance_from_height(int);
double inline deviation_from_angle(double)

ColorImage* vision_processing::get_image() {
    if(camera().IsFreshImage()) {
        camera().GetImage(old_image);
    }
    return get_old_image();
}

ColorImage* vision_processing::get_old_image() {
    return old_image;
}

BinaryImage* vision_processing::get_image_mask(ColorImage* image) {
    BinaryImage* imageMask;
    if(image == NULL) {
        return imageMask;
    }
    if (COLOR_MODE == HSV) {
        imageMask = image->ThresholdHSV(HSV_HMIN, HSV_HMAX, HSV_SMIN, HSV_SMAX, HSV_VMIN, HSV_VMAX);
    }
    else if(COLOR_MODE == HSI) {
        imageMask = image->ThresholdHSI(HSI_HMIN, HSI_HMAX, HSI_SMIN, HSI_SMAX, HSI_IMIN, HSI_IMAX);
    }
    else { // HSI is implied (not assumed)
        imageMask = image->ThresholdHSL(HSL_HMIN, HSL_HMAX, HSL_SMIN, HSL_SMAX, HSL_LMIN, HSL_LMAX);
    }
    return imageMask;
}

vector<ParticleAnalysisReport> vision_processing::get_image_targets(BinaryImage* image) {
    vector<ParticleAnalysisReport> targets;
    if(image == NULL) {
        return targets;
    }
    vector<ParticleAnalysisReport>* particles = image->GetOrderedParticleAnalysisReports();
    for(unsigned int i = 0; i < particles->size(); i++) {
        ParticleAnalysisReport particle = particles->at(i);
        double particle_area = particle.particleArea;
        if(particle_area > PARTICLE_AREA_MIN && particle_area <= PARTICLE_AREA_MAX) {
            if(targets.size() >= 4) {
                // TODO change min and max
                //      call function again
                //      if depth is more than 2
                //      explode
                break;
            }
            targets.push_back(particle);
        }
    }
    return targets;
}

unsigned int vision_processing::determine_aim_target_from_image(ColorImage* image) {
    if(image == NULL) {
        return 0;
    }
    return determine_aim_target(get_image_targets(get_image_mask(image)));
}

unsigned int vision_processing::determine_aim_target(vector<ParticleAnalysisReport>) {
    // TODO make it do stuff
    return 0;
}

vector<double> vision_processing::get_distance() {
    return get_distance_from_image(get_image());
}

vector<double> vision_processing::get_distance_from_image(ColorImage* image) {
    BinaryImage* image_mask = get_image_mask(image);
    vector<ParticleAnalysisReport> targets = get_image_targets(image_mask);
    vector<double> distance;
    if(image == NULL) {
        return distance;
    }
    for(unsigned int i = 0; i < targets.size(); i++) {
        ParticleAnalysisReport target = targets[i];
		unsigned int height = get_image_height(image_mask, target);
		unsigned int width = get_image_width(image_mask, target);
		double ground_distance = distance_from_height(height) + deviation_from_angle(degrees_from_ratio(width/height))s ;
        distance.push_back(ground_distance);
    }
    return distance;
}

vector<double> vision_processing::get_degrees() {
    return get_degrees_from_image(get_image());
}

vector<double> vision_processing::get_degrees_from_image(ColorImage* image) {
    vector<ParticleAnalysisReport> targets = get_image_targets(get_image_mask(image));
    vector<double> degrees;
    if(image == NULL) {
        return degrees;
    }
    for(unsigned int i = 0; i < targets.size(); i++) {
        ParticleAnalysisReport target = targets[i];
        int height = target.imageHeight;
        int width = target.imageWidth;
        double ratio = 1.0*width/height;
        degrees.push_back(degrees_from_ratio(ratio));
    }
    return degrees;
}

vector<double> vision_processing::get_radians_from_image(ColorImage* image) {
    vector<double> degrees = get_degrees_from_image(image);
    vector<double> radians;
    if(image == NULL) {
        return radians;
    }
    for(unsigned int i = 0; i< degrees.size(); i++) {
        radians.push_back(deg2rad(degrees[i]));
    }
    return radians;
}

double inline degrees_from_ratio(double ratio) {
    return (-94.637 * pow(ratio, 2)) + (119.86 * ratio) + 9.7745;
}

double inline distance_from_height(double height) {
	return ((1277.686246075*(1/height)) - 0.8265433113);
}

double inline deviation_from_angle(double angle) {
	return ((-0.00005*(pow(angle,2))) +(0.0208*angle) +0.0046);
}

double inline radians_from_ratio(double ratio) {
    return deg2rad(degrees_from_ratio(ratio));
}

double inline distance_from_height(int height) {
    return 1532.1932574739 * (pow(height, -1.0541299046));
}
<commit_msg>More with the equations<commit_after>/* vision/vision_processing.cpp
 *
 * Copyright (c) 2011, 2012 Chantilly Robotics <chantilly612@gmail.com>
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
 * Entry points for vision processing: distance and angle from image
 */

/*
 * Calculated angles are relative to perpendicular.
 */

#include <vector>
#include <cmath>
#include <Vision/ColorImage.h>
#include <Vision/BinaryImage.h>
#include <Vision2009/VisionAPI.h>
#include "vision_processing.h"
#include "../ports.h"
#include "../ranges.h"
#include "../trajectory.h"

using namespace vision_processing;

//constants
ColorImage* old_image;

double inline degrees_from_ratio(double); // ratio: width/height
double inline distance_from_height(double);
double inline radians_from_ratio(double);
double inline distance_from_height(int);
double inline deviation_from_angle(double)

ColorImage* vision_processing::get_image() {
    if(camera().IsFreshImage()) {
        camera().GetImage(old_image);
    }
    return get_old_image();
}

ColorImage* vision_processing::get_old_image() {
    return old_image;
}

BinaryImage* vision_processing::get_image_mask(ColorImage* image) {
    BinaryImage* imageMask;
    if(image == NULL) {
        return imageMask;
    }
    if (COLOR_MODE == HSV) {
        imageMask = image->ThresholdHSV(HSV_HMIN, HSV_HMAX, HSV_SMIN, HSV_SMAX, HSV_VMIN, HSV_VMAX);
    }
    else if(COLOR_MODE == HSI) {
        imageMask = image->ThresholdHSI(HSI_HMIN, HSI_HMAX, HSI_SMIN, HSI_SMAX, HSI_IMIN, HSI_IMAX);
    }
    else { // HSI is implied (not assumed)
        imageMask = image->ThresholdHSL(HSL_HMIN, HSL_HMAX, HSL_SMIN, HSL_SMAX, HSL_LMIN, HSL_LMAX);
    }
    return imageMask;
}

vector<ParticleAnalysisReport> vision_processing::get_image_targets(BinaryImage* image) {
    vector<ParticleAnalysisReport> targets;
    if(image == NULL) {
        return targets;
    }
    vector<ParticleAnalysisReport>* particles = image->GetOrderedParticleAnalysisReports();
    for(unsigned int i = 0; i < particles->size(); i++) {
        ParticleAnalysisReport particle = particles->at(i);
        double particle_area = particle.particleArea;
        if(particle_area > PARTICLE_AREA_MIN && particle_area <= PARTICLE_AREA_MAX) {
            if(targets.size() >= 4) {
                // TODO change min and max
                //      call function again
                //      if depth is more than 2
                //      explode
                break;
            }
            targets.push_back(particle);
        }
    }
    return targets;
}

unsigned int vision_processing::determine_aim_target_from_image(ColorImage* image) {
    if(image == NULL) {
        return 0;
    }
    return determine_aim_target(get_image_targets(get_image_mask(image)));
}

unsigned int vision_processing::determine_aim_target(vector<ParticleAnalysisReport>) {
    // TODO make it do stuff
    return 0;
}

vector<double> vision_processing::get_distance() {
    return get_distance_from_image(get_image());
}

vector<double> vision_processing::get_distance_from_image(ColorImage* image) {
    BinaryImage* image_mask = get_image_mask(image);
    vector<ParticleAnalysisReport> targets = get_image_targets(image_mask);
    vector<double> distance;
    if(image == NULL) {
        return distance;
    }
    for(unsigned int i = 0; i < targets.size(); i++) {
        ParticleAnalysisReport target = targets[i];
		unsigned int height = get_image_height(image_mask, target);
		unsigned int width = get_image_width(image_mask, target);
		double ground_distance = distance_from_height(height) + deviation_from_angle(degrees_from_ratio(width/height))s ;
        distance.push_back(ground_distance);
    }
    return distance;
}

vector<double> vision_processing::get_degrees() {
    return get_degrees_from_image(get_image());
}

vector<double> vision_processing::get_degrees_from_image(ColorImage* image) {
    vector<ParticleAnalysisReport> targets = get_image_targets(get_image_mask(image));
    vector<double> degrees;
    if(image == NULL) {
        return degrees;
    }
    for(unsigned int i = 0; i < targets.size(); i++) {
        ParticleAnalysisReport target = targets[i];
        int height = target.imageHeight;
        int width = target.imageWidth;
        double ratio = 1.0*width/height;
        degrees.push_back(degrees_from_ratio(ratio));
    }
    return degrees;
}

vector<double> vision_processing::get_radians_from_image(ColorImage* image) {
    vector<double> degrees = get_degrees_from_image(image);
    vector<double> radians;
    if(image == NULL) {
        return radians;
    }
    for(unsigned int i = 0; i< degrees.size(); i++) {
        radians.push_back(deg2rad(degrees[i]));
    }
    return radians;
}

double inline degrees_from_ratio(double ratio) {
    return (-94.637 * pow(ratio, 2)) + (119.86 * ratio) + 9.7745;
}

double inline distance_from_height(double height) {
	return ((1277.686246075*(1/height)) - 0.8265433113);
}

double inline deviation_from_angle(double angle) {
	return ((-0.00005*(pow(angle,2))) +(0.0208*angle) +0.0046);
}

double inline radians_from_ratio(double ratio) {
    return deg2rad(degrees_from_ratio(ratio));
}

double inline distance_from_height(int height) {
    return ((1277.686246075*(1/height)) - 0.8265433113);
}
<|endoftext|>


<commit_before>/**
 * @file
 * @brief Implementation of ROOT data file writer module
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "ROOTObjectWriterModule.hpp"

#include <fstream>
#include <string>
#include <utility>

#include <TBranchElement.h>
#include <TClass.h>

#include "core/config/ConfigReader.hpp"
#include "core/utils/file.h"
#include "core/utils/log.h"
#include "core/utils/type.h"

#include "objects/Object.hpp"
#include "objects/objects.h"

using namespace allpix;

ROOTObjectWriterModule::ROOTObjectWriterModule(Configuration& config, Messenger* messenger, GeometryManager* geo_mgr)
    : Module(config), geo_mgr_(geo_mgr) {
    // Bind to all messages
    messenger->registerListener(this, &ROOTObjectWriterModule::receive);
}
/**
 * @note Objects cannot be stored in smart pointers due to internal ROOT logic
 */
ROOTObjectWriterModule::~ROOTObjectWriterModule() {
    // Delete all object pointers
    for(auto& index_data : write_list_) {
        delete index_data.second;
    }
}

void ROOTObjectWriterModule::init() {
    // Create output file
    output_file_name_ =
        createOutputFile(allpix::add_file_extension(config_.get<std::string>("file_name", "data"), "root"), true);
    output_file_ = std::make_unique<TFile>(output_file_name_.c_str(), "RECREATE");
    output_file_->cd();

    // Read include and exclude list
    if(config_.has("include") && config_.has("exclude")) {
        throw InvalidValueError(config_, "exclude", "include and exclude parameter are mutually exclusive");
    } else if(config_.has("include")) {
        auto inc_arr = config_.getArray<std::string>("include");
        include_.insert(inc_arr.begin(), inc_arr.end());
    } else if(config_.has("exclude")) {
        auto exc_arr = config_.getArray<std::string>("exclude");
        exclude_.insert(exc_arr.begin(), exc_arr.end());
    }
}

void ROOTObjectWriterModule::receive(std::shared_ptr<BaseMessage> message, std::string message_name) { // NOLINT
    try {
        const BaseMessage* inst = message.get();
        std::string name_str = " without a name";
        if(!message_name.empty()) {
            name_str = " named " + message_name;
        }
        LOG(TRACE) << "ROOT object writer received " << allpix::demangle(typeid(*inst).name()) << name_str;

        // Get the detector name
        std::string detector_name;
        if(message->getDetector() != nullptr) {
            detector_name = message->getDetector()->getName();
        }

        // Read the object
        auto object_array = message->getObjectArray();
        if(!object_array.empty()) {
            keep_messages_.push_back(message);

            const Object& first_object = object_array[0];
            std::type_index type_idx = typeid(first_object);

            // Create a new branch of the correct type if this message was not received before
            auto index_tuple = std::make_tuple(type_idx, detector_name, message_name);
            if(write_list_.find(index_tuple) == write_list_.end()) {

                auto* cls = TClass::GetClass(typeid(first_object));

                // Remove the allpix prefix
                std::string class_name = cls->GetName();
                std::string apx_namespace = "allpix::";
                size_t ap_idx = class_name.find(apx_namespace);
                if(ap_idx != std::string::npos) {
                    class_name.replace(ap_idx, apx_namespace.size(), "");
                }

                // Check if this message should be kept
                if((!include_.empty() && include_.find(class_name) == include_.end()) ||
                   (!exclude_.empty() && exclude_.find(class_name) != exclude_.end())) {
                    LOG(TRACE) << "ROOT object writer ignored message with object " << allpix::demangle(typeid(*inst).name())
                               << " because it has been excluded or not explicitly included";
                    return;
                }

                // Add vector of objects to write to the write list
                write_list_[index_tuple] = new std::vector<Object*>();
                auto addr = &write_list_[index_tuple];

                if(trees_.find(class_name) == trees_.end()) {
                    // Create new tree
                    output_file_->cd();
                    trees_.emplace(
                        class_name,
                        std::make_unique<TTree>(class_name.c_str(), (std::string("Tree of ") + class_name).c_str()));
                }

                std::string branch_name = detector_name;
                if(!message_name.empty()) {
                    branch_name += "_";
                    branch_name += message_name;
                }

                trees_[class_name]->Bronch(
                    branch_name.c_str(), (std::string("std::vector<") + cls->GetName() + "*>").c_str(), addr);
            }

            // Fill the branch vector
            for(Object& object : object_array) {
                ++write_cnt_;
                write_list_[index_tuple]->push_back(&object);
            }
        }

    } catch(MessageWithoutObjectException& e) {
        const BaseMessage* inst = message.get();
        LOG(WARNING) << "ROOT object writer cannot process message of type" << allpix::demangle(typeid(*inst).name())
                     << " with name " << message_name;
    }
}

void ROOTObjectWriterModule::run(unsigned int) {
    LOG(TRACE) << "Writing new objects to tree";
    output_file_->cd();

    // Fill the tree with the current received messages
    for(auto& tree : trees_) {
        tree.second->Fill();
    }

    // Clear the current message list
    for(auto& index_data : write_list_) {
        index_data.second->clear();
    }
    // Clear the messages we have to keep because they contain the internal pointers
    keep_messages_.clear();
}

void ROOTObjectWriterModule::finalize() {
    LOG(TRACE) << "Writing objects to file";
    output_file_->cd();

    int branch_count = 0;
    for(auto& tree : trees_) {
        // Update statistics
        branch_count += tree.second->GetListOfBranches()->GetEntries();
    }

    // Create main config directory
    TDirectory* config_dir = output_file_->mkdir("config");
    config_dir->cd();

    // Get the config manager
    ConfigManager* conf_manager = getConfigManager();

    // Save the main configuration to the output file
    auto global_dir = config_dir->mkdir("Allpix");
    LOG(TRACE) << "Writing global configuration";

    // Loop over all values in the global configuration
    for(auto& key_value : conf_manager->getGlobalConfiguration().getAll()) {
        global_dir->WriteObject(&key_value.second, key_value.first.c_str());
    }

    // Save the instance configuration to the output file
    for(auto& config : conf_manager->getInstanceConfigurations()) {
        // Create a new directory per section, using the unique module name
        auto unique_name = config.get<std::string>("unique_name");
        auto section_dir = config_dir->mkdir(unique_name.c_str());
        LOG(TRACE) << "Writing configuration for: " << unique_name;

        // Loop over all values in the section
        for(auto& key_value : config.getAll()) {
            // Skip the unique name and input / output if empty
            if(key_value.first == "unique_name") {
                continue;
            }
            if((key_value.first == "input" || key_value.first == "output") && key_value.second.empty()) {
                continue;
            }
            section_dir->WriteObject(&key_value.second, key_value.first.c_str());
        }
    }

    // Save the detectors to the output file
    // FIXME Possibly the format to save the geometry should be more flexible
    auto detectors_dir = output_file_->mkdir("detectors");
    detectors_dir->cd();
    for(auto& detector : geo_mgr_->getDetectors()) {
        auto detector_dir = detectors_dir->mkdir(detector->getName().c_str());

        auto position = detector->getPosition();
        detector_dir->WriteObject(&position, "position");
        auto orientation = detector->getOrientation();
        detector_dir->WriteObject(&orientation, "orientation");

        auto model_dir = detector_dir->mkdir("model");
        // FIXME This saves the model every time again also for models that appear multiple times
        auto model_configs = detector->getModel()->getConfigurations();
        std::map<std::string, int> count_configs;
        for(auto& model_config : model_configs) {
            auto model_config_dir = model_dir;
            if(!model_config.getName().empty()) {
                model_config_dir = model_dir->mkdir(
                    (model_config.getName() + "-" + std::to_string(count_configs[model_config.getName()])).c_str());
                count_configs[model_config.getName()]++;
            }

            for(auto& key_value : model_config.getAll()) {
                model_config_dir->WriteObject(&key_value.second, key_value.first.c_str());
            }
        }
    }

    // Finish writing to output file
    output_file_->Write();

    // Print statistics
    LOG(STATUS) << "Wrote " << write_cnt_ << " objects to " << branch_count << " branches in file:" << std::endl
                << output_file_name_;
}
<commit_msg>ROOTObjectWriter: do not write illegal characters in section headers<commit_after>/**
 * @file
 * @brief Implementation of ROOT data file writer module
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "ROOTObjectWriterModule.hpp"

#include <fstream>
#include <string>
#include <utility>

#include <TBranchElement.h>
#include <TClass.h>

#include "core/config/ConfigReader.hpp"
#include "core/utils/file.h"
#include "core/utils/log.h"
#include "core/utils/type.h"

#include "objects/Object.hpp"
#include "objects/objects.h"

using namespace allpix;

ROOTObjectWriterModule::ROOTObjectWriterModule(Configuration& config, Messenger* messenger, GeometryManager* geo_mgr)
    : Module(config), geo_mgr_(geo_mgr) {
    // Bind to all messages
    messenger->registerListener(this, &ROOTObjectWriterModule::receive);
}
/**
 * @note Objects cannot be stored in smart pointers due to internal ROOT logic
 */
ROOTObjectWriterModule::~ROOTObjectWriterModule() {
    // Delete all object pointers
    for(auto& index_data : write_list_) {
        delete index_data.second;
    }
}

void ROOTObjectWriterModule::init() {
    // Create output file
    output_file_name_ =
        createOutputFile(allpix::add_file_extension(config_.get<std::string>("file_name", "data"), "root"), true);
    output_file_ = std::make_unique<TFile>(output_file_name_.c_str(), "RECREATE");
    output_file_->cd();

    // Read include and exclude list
    if(config_.has("include") && config_.has("exclude")) {
        throw InvalidValueError(config_, "exclude", "include and exclude parameter are mutually exclusive");
    } else if(config_.has("include")) {
        auto inc_arr = config_.getArray<std::string>("include");
        include_.insert(inc_arr.begin(), inc_arr.end());
    } else if(config_.has("exclude")) {
        auto exc_arr = config_.getArray<std::string>("exclude");
        exclude_.insert(exc_arr.begin(), exc_arr.end());
    }
}

void ROOTObjectWriterModule::receive(std::shared_ptr<BaseMessage> message, std::string message_name) { // NOLINT
    try {
        const BaseMessage* inst = message.get();
        std::string name_str = " without a name";
        if(!message_name.empty()) {
            name_str = " named " + message_name;
        }
        LOG(TRACE) << "ROOT object writer received " << allpix::demangle(typeid(*inst).name()) << name_str;

        // Get the detector name
        std::string detector_name;
        if(message->getDetector() != nullptr) {
            detector_name = message->getDetector()->getName();
        }

        // Read the object
        auto object_array = message->getObjectArray();
        if(!object_array.empty()) {
            keep_messages_.push_back(message);

            const Object& first_object = object_array[0];
            std::type_index type_idx = typeid(first_object);

            // Create a new branch of the correct type if this message was not received before
            auto index_tuple = std::make_tuple(type_idx, detector_name, message_name);
            if(write_list_.find(index_tuple) == write_list_.end()) {

                auto* cls = TClass::GetClass(typeid(first_object));

                // Remove the allpix prefix
                std::string class_name = cls->GetName();
                std::string apx_namespace = "allpix::";
                size_t ap_idx = class_name.find(apx_namespace);
                if(ap_idx != std::string::npos) {
                    class_name.replace(ap_idx, apx_namespace.size(), "");
                }

                // Check if this message should be kept
                if((!include_.empty() && include_.find(class_name) == include_.end()) ||
                   (!exclude_.empty() && exclude_.find(class_name) != exclude_.end())) {
                    LOG(TRACE) << "ROOT object writer ignored message with object " << allpix::demangle(typeid(*inst).name())
                               << " because it has been excluded or not explicitly included";
                    return;
                }

                // Add vector of objects to write to the write list
                write_list_[index_tuple] = new std::vector<Object*>();
                auto addr = &write_list_[index_tuple];

                if(trees_.find(class_name) == trees_.end()) {
                    // Create new tree
                    output_file_->cd();
                    trees_.emplace(
                        class_name,
                        std::make_unique<TTree>(class_name.c_str(), (std::string("Tree of ") + class_name).c_str()));
                }

                std::string branch_name = detector_name;
                if(!message_name.empty()) {
                    branch_name += "_";
                    branch_name += message_name;
                }

                trees_[class_name]->Bronch(
                    branch_name.c_str(), (std::string("std::vector<") + cls->GetName() + "*>").c_str(), addr);
            }

            // Fill the branch vector
            for(Object& object : object_array) {
                ++write_cnt_;
                write_list_[index_tuple]->push_back(&object);
            }
        }

    } catch(MessageWithoutObjectException& e) {
        const BaseMessage* inst = message.get();
        LOG(WARNING) << "ROOT object writer cannot process message of type" << allpix::demangle(typeid(*inst).name())
                     << " with name " << message_name;
    }
}

void ROOTObjectWriterModule::run(unsigned int) {
    LOG(TRACE) << "Writing new objects to tree";
    output_file_->cd();

    // Fill the tree with the current received messages
    for(auto& tree : trees_) {
        tree.second->Fill();
    }

    // Clear the current message list
    for(auto& index_data : write_list_) {
        index_data.second->clear();
    }
    // Clear the messages we have to keep because they contain the internal pointers
    keep_messages_.clear();
}

void ROOTObjectWriterModule::finalize() {
    LOG(TRACE) << "Writing objects to file";
    output_file_->cd();

    int branch_count = 0;
    for(auto& tree : trees_) {
        // Update statistics
        branch_count += tree.second->GetListOfBranches()->GetEntries();
    }

    // Create main config directory
    TDirectory* config_dir = output_file_->mkdir("config");
    config_dir->cd();

    // Get the config manager
    ConfigManager* conf_manager = getConfigManager();

    // Save the main configuration to the output file
    auto global_dir = config_dir->mkdir("Allpix");
    LOG(TRACE) << "Writing global configuration";

    // Loop over all values in the global configuration
    for(auto& key_value : conf_manager->getGlobalConfiguration().getAll()) {
        global_dir->WriteObject(&key_value.second, key_value.first.c_str());
    }

    // Save the instance configuration to the output file
    for(auto& config : conf_manager->getInstanceConfigurations()) {
        // Create a new directory per section, using the unique module name
        auto unique_name = config.get<std::string>("unique_name");
        auto section_dir = config_dir->mkdir(unique_name.c_str());
        LOG(TRACE) << "Writing configuration for: " << unique_name;

        // Loop over all values in the section
        for(auto& key_value : config.getAll()) {
            // Skip the unique name and input / output if empty
            if(key_value.first == "unique_name") {
                continue;
            }
            if((key_value.first == "input" || key_value.first == "output") && key_value.second.empty()) {
                continue;
            }
            section_dir->WriteObject(&key_value.second, key_value.first.c_str());
        }
    }

    // Save the detectors to the output file
    // FIXME Possibly the format to save the geometry should be more flexible
    auto detectors_dir = output_file_->mkdir("detectors");
    detectors_dir->cd();
    for(auto& detector : geo_mgr_->getDetectors()) {
        auto detector_dir = detectors_dir->mkdir(detector->getName().c_str());

        auto position = detector->getPosition();
        detector_dir->WriteObject(&position, "position");
        auto orientation = detector->getOrientation();
        detector_dir->WriteObject(&orientation, "orientation");

        auto model_dir = detector_dir->mkdir("model");
        // FIXME This saves the model every time again also for models that appear multiple times
        auto model_configs = detector->getModel()->getConfigurations();
        std::map<std::string, int> count_configs;
        for(auto& model_config : model_configs) {
            auto model_config_dir = model_dir;
            if(!model_config.getName().empty()) {
                model_config_dir = model_dir->mkdir(
                    (model_config.getName() + "_" + std::to_string(count_configs[model_config.getName()])).c_str());
                count_configs[model_config.getName()]++;
            }

            for(auto& key_value : model_config.getAll()) {
                model_config_dir->WriteObject(&key_value.second, key_value.first.c_str());
            }
        }
    }

    // Finish writing to output file
    output_file_->Write();

    // Print statistics
    LOG(STATUS) << "Wrote " << write_cnt_ << " objects to " << branch_count << " branches in file:" << std::endl
                << output_file_name_;
}
<|endoftext|>