<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FirstOrderOptimizer.h"
#include "paddle/math/TrainingAlgorithmOp.h"
#include "paddle/utils/Flags.h"
#include "paddle/utils/Util.h"

#include <cmath>

DEFINE_bool(log_clipping, false, "enable log clipping or not");

namespace paddle {

SparseMomentumParameterOptimizer::SparseMomentumParameterOptimizer(
    const OptimizationConfig& optConfig)
    : ParameterOptimizer(optConfig) {
  addParameterType(PARAMETER_MOMENTUM);
  addParameterType(PARAMETER_MOMENTUM_UT);
  addParameterType(PARAMETER_MOMENTUM_VT);
  alpha_ = 1;
  beta_ = 1;
  tau_ = -1;
  threshold_ = 1e+06;
}

void SparseMomentumParameterOptimizer::init(size_t numRows,
                                            const ParameterConfig* config) {
  isParameterSparse_ = numRows != 0;
  t0Vec_.resize(numRows);
  t0Vec_.assign(t0Vec_.size(), 0);
  timer_ = 0;
  momentum_ = config->momentum();
  decayRate_ = config->decay_rate();
  gamma_ = config->learning_rate();
}

void SparseMomentumParameterOptimizer::startBatch(int64_t numSamplesProcessed) {
  learningRate_ = calcLearningRate(numSamplesProcessed, pass_);
  if (isParameterSparse_) {
    tau_ = tau_ + beta_ / alpha_;
    alpha_ = alpha_ / momentum_;
    beta_ = beta_ / (1 + decayRate_ * gamma_ * learningRate_);
  }
}

void SparseMomentumParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& paraConfig,
                                              size_t sparseId) const {
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    if (t0Vec_[sparseId] == 0) {
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
      t0Vec_[sparseId] = 1;
    }
    vecs[PARAMETER_MOMENTUM_UT]->add(*vecs[PARAMETER_GRADIENT],
                                     -alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_MOMENTUM_VT]->add(*vecs[PARAMETER_GRADIENT],
                                     tau_ * alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_VALUE]->add(*vecs[PARAMETER_MOMENTUM_UT],
                               tau_ / beta_ + 1.0 / alpha_,
                               *vecs[PARAMETER_MOMENTUM_VT],
                               1.0 / beta_);

  } else {
    vecs[PARAMETER_VALUE]->sgdUpdate(*vecs[PARAMETER_GRADIENT],
                                     *vecs[PARAMETER_MOMENTUM],
                                     learningRate_ * paraConfig.learning_rate(),
                                     paraConfig.momentum(),
                                     applyDecay_ ? paraConfig.decay_rate() : 0);
  }
}

ParameterOptimizer::TraverseCallback
SparseMomentumParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (alpha_ > threshold_ && isParameterSparse_) {
    //  Restart to avoid large value multiplication
    //  1. \alpha = 1, \beta = 1, \tau = 0
    //  2. Note that \tau * u_t + v_t = \beta \theta_t, therefore:
    //     u_t should be rescaled to u_t/alpha_
    //     v_t should be reset to \theta_t
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_MOMENTUM_UT]->divScalar(alpha_);
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
    };
  } else {
    return nullptr;
  }
}

void SparseMomentumParameterOptimizer::finishBatch() {
  timer_++;
  if (!isParameterSparse_) return;
  if (alpha_ > threshold_) {
    alpha_ = 1;
    beta_ = 1;
    tau_ = -1;
  }
}

void AdagradParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum_buffer = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adagradApply(value,
               grad,
               mom,
               accum_buffer,
               accum,
               lr,
               epsilon,
               learningRate,
               momentum,
               decayRate);
}

ParameterOptimizer::TraverseCallback
AdagradParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (numUpdates_ % kMaxNumAccumulates == 0) {
    // Move the sum to a different buffer to avoid loss of precision
    // due to too many sums.
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_GRADIENT_SQURESUM]->add(
          *vecs[PARAMETER_GRADIENT_SQURESUM1]);
      vecs[PARAMETER_GRADIENT_SQURESUM1]->zeroMem();
    };
  } else {
    return nullptr;
  }
}

void AdaDeltaParameterOptimizer::update(const VectorPtr vecs[],
                                        const ParameterConfig& config,
                                        size_t sparseId) const {
  CHECK(sparseId == -1LU) << "Sparse update is not supported";

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum_update = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adadeltaApply(value,
                grad,
                mom,
                accum,
                accum_update,
                lr,
                rou_,
                epsilon_,
                learningRate,
                momentum,
                decayRate);
}

void RMSPropParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& sum1 = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  rmspropApply(value,
               grad,
               mom,
               sum,
               sum1,
               lr,
               accumulatedRou,
               rou_,
               epsilon,
               learningRate,
               momentum,
               decayRate,
               firstTime);
}

void DecayedAdagradParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& config,
                                              size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  decayedAdagradApply(value,
                      grad,
                      mom,
                      sum,
                      lr,
                      accumulatedRou,
                      rou_,
                      epsilon,
                      learningRate,
                      momentum,
                      decayRate,
                      firstTime);
}

void AdamParameterOptimizer::update(const VectorPtr vecs[],
                                    const ParameterConfig& config,
                                    size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";

  real beta1_power = std::pow(beta1_, step_);
  real beta2_power = std::pow(beta2_, step_);
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& v = *vecs[PARAMETER_SECOND_MOMENTUM];

  adamApply(value,
            grad,
            mom,
            v,
            beta1_,
            beta2_,
            beta1_power,
            beta2_power,
            epsilon_,
            learningRate);
}

void AdamaxParameterOptimizer::update(const VectorPtr vecs[],
                                      const ParameterConfig& config,
                                      size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& u = *vecs[PARAMETER_WEIGHTED_INFINITY_NORM];

  adamaxApply(value, grad, mom, u, beta1_, beta2_, step_, learningRate);
}

void OptimizerWithGradientClipping::update(const VectorPtr vecs[],
                                           const ParameterConfig& config,
                                           size_t sparseId) const {
  real global_thres_ = optConfig_.gradient_clipping_threshold();
  real local_thres_ = config.gradient_clipping_threshold();

  real threshold;
  std::string field;
  // Get the minimum of local and global threshold
  // as the real threshold for clipping
  if (global_thres_ > 0.0f && local_thres_ > 0.0f) {
    threshold = global_thres_ < local_thres_ ? global_thres_ : local_thres_;
    field = global_thres_ < local_thres_ ? "global" : "local";
  } else if (global_thres_ > 0.0f) {
    threshold = global_thres_;
    field = "global";
  } else {
    threshold = local_thres_;
    field = "local";
  }

  real maxAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsMax();
  if (maxAbsGrad > threshold) {
    if (FLAGS_log_clipping) {
      real avgAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsSum() /
                        vecs[PARAMETER_GRADIENT]->getSize();
      LOG(INFO) << "parameter=" << config.name() << " need clipping by "
                << field << " threshold=" << threshold
                << ", max grad=" << maxAbsGrad << ", avg grad=" << avgAbsGrad;
    }
    vecs[PARAMETER_GRADIENT]->clip(-threshold, threshold);
  }
  optimizer_->update(vecs, config, sparseId);
}

}  // namespace paddle
<commit_msg>rename two variables<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FirstOrderOptimizer.h"
#include "paddle/math/TrainingAlgorithmOp.h"
#include "paddle/utils/Flags.h"
#include "paddle/utils/Util.h"

#include <cmath>

DEFINE_bool(log_clipping, false, "enable log clipping or not");

namespace paddle {

SparseMomentumParameterOptimizer::SparseMomentumParameterOptimizer(
    const OptimizationConfig& optConfig)
    : ParameterOptimizer(optConfig) {
  addParameterType(PARAMETER_MOMENTUM);
  addParameterType(PARAMETER_MOMENTUM_UT);
  addParameterType(PARAMETER_MOMENTUM_VT);
  alpha_ = 1;
  beta_ = 1;
  tau_ = -1;
  threshold_ = 1e+06;
}

void SparseMomentumParameterOptimizer::init(size_t numRows,
                                            const ParameterConfig* config) {
  isParameterSparse_ = numRows != 0;
  t0Vec_.resize(numRows);
  t0Vec_.assign(t0Vec_.size(), 0);
  timer_ = 0;
  momentum_ = config->momentum();
  decayRate_ = config->decay_rate();
  gamma_ = config->learning_rate();
}

void SparseMomentumParameterOptimizer::startBatch(int64_t numSamplesProcessed) {
  learningRate_ = calcLearningRate(numSamplesProcessed, pass_);
  if (isParameterSparse_) {
    tau_ = tau_ + beta_ / alpha_;
    alpha_ = alpha_ / momentum_;
    beta_ = beta_ / (1 + decayRate_ * gamma_ * learningRate_);
  }
}

void SparseMomentumParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& paraConfig,
                                              size_t sparseId) const {
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    if (t0Vec_[sparseId] == 0) {
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
      t0Vec_[sparseId] = 1;
    }
    vecs[PARAMETER_MOMENTUM_UT]->add(*vecs[PARAMETER_GRADIENT],
                                     -alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_MOMENTUM_VT]->add(*vecs[PARAMETER_GRADIENT],
                                     tau_ * alpha_ * gamma_ * learningRate_);
    vecs[PARAMETER_VALUE]->add(*vecs[PARAMETER_MOMENTUM_UT],
                               tau_ / beta_ + 1.0 / alpha_,
                               *vecs[PARAMETER_MOMENTUM_VT],
                               1.0 / beta_);

  } else {
    vecs[PARAMETER_VALUE]->sgdUpdate(*vecs[PARAMETER_GRADIENT],
                                     *vecs[PARAMETER_MOMENTUM],
                                     learningRate_ * paraConfig.learning_rate(),
                                     paraConfig.momentum(),
                                     applyDecay_ ? paraConfig.decay_rate() : 0);
  }
}

ParameterOptimizer::TraverseCallback
SparseMomentumParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (alpha_ > threshold_ && isParameterSparse_) {
    //  Restart to avoid large value multiplication
    //  1. \alpha = 1, \beta = 1, \tau = 0
    //  2. Note that \tau * u_t + v_t = \beta \theta_t, therefore:
    //     u_t should be rescaled to u_t/alpha_
    //     v_t should be reset to \theta_t
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_MOMENTUM_UT]->divScalar(alpha_);
      vecs[PARAMETER_MOMENTUM_VT]->assign(*vecs[PARAMETER_VALUE]);
    };
  } else {
    return nullptr;
  }
}

void SparseMomentumParameterOptimizer::finishBatch() {
  timer_++;
  if (!isParameterSparse_) return;
  if (alpha_ > threshold_) {
    alpha_ = 1;
    beta_ = 1;
    tau_ = -1;
  }
}

void AdagradParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum_buffer = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adagradApply(value,
               grad,
               mom,
               accum_buffer,
               accum,
               lr,
               epsilon,
               learningRate,
               momentum,
               decayRate);
}

ParameterOptimizer::TraverseCallback
AdagradParameterOptimizer::needSpecialTraversal(
    const ParameterConfig& config) const {
  if (numUpdates_ % kMaxNumAccumulates == 0) {
    // Move the sum to a different buffer to avoid loss of precision
    // due to too many sums.
    return [this](const VectorPtr vecs[],
                  const ParameterConfig& config,
                  size_t sparseId) {
      vecs[PARAMETER_GRADIENT_SQURESUM]->add(
          *vecs[PARAMETER_GRADIENT_SQURESUM1]);
      vecs[PARAMETER_GRADIENT_SQURESUM1]->zeroMem();
    };
  } else {
    return nullptr;
  }
}

void AdaDeltaParameterOptimizer::update(const VectorPtr vecs[],
                                        const ParameterConfig& config,
                                        size_t sparseId) const {
  CHECK(sparseId == -1LU) << "Sparse update is not supported";

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& accum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& accum_update = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  adadeltaApply(value,
                grad,
                mom,
                accum,
                accum_update,
                lr,
                rou_,
                epsilon_,
                learningRate,
                momentum,
                decayRate);
}

void RMSPropParameterOptimizer::update(const VectorPtr vecs[],
                                       const ParameterConfig& config,
                                       size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& sum1 = *vecs[PARAMETER_GRADIENT_SQURESUM1];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  rmspropApply(value,
               grad,
               mom,
               sum,
               sum1,
               lr,
               accumulatedRou,
               rou_,
               epsilon,
               learningRate,
               momentum,
               decayRate,
               firstTime);
}

void DecayedAdagradParameterOptimizer::update(const VectorPtr vecs[],
                                              const ParameterConfig& config,
                                              size_t sparseId) const {
  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& sum = *vecs[PARAMETER_GRADIENT_SQURESUM];
  BaseMatrix& lr = *vecs[PARAMETER_LEARNING_RATE];

  real accumulatedRou = rou_;
  bool firstTime = timer_ == 0;
  if (sparseId != -1LU) {
    CHECK_LT(sparseId, t0Vec_.size());
    accumulatedRou = std::pow(rou_, timer_ + 1 - t0Vec_[sparseId]);
    firstTime = t0Vec_[sparseId] == 0;
    t0Vec_[sparseId] = timer_ + 1;
  }

  real epsilon = optConfig_.ada_epsilon();
  real learningRate = learningRate_ * config.learning_rate();
  real momentum = config.momentum();
  real decayRate = applyDecay_ ? config.decay_rate() : 0;

  decayedAdagradApply(value,
                      grad,
                      mom,
                      sum,
                      lr,
                      accumulatedRou,
                      rou_,
                      epsilon,
                      learningRate,
                      momentum,
                      decayRate,
                      firstTime);
}

void AdamParameterOptimizer::update(const VectorPtr vecs[],
                                    const ParameterConfig& config,
                                    size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";

  real beta1_power = std::pow(beta1_, step_);
  real beta2_power = std::pow(beta2_, step_);
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& v = *vecs[PARAMETER_SECOND_MOMENTUM];

  adamApply(value,
            grad,
            mom,
            v,
            beta1_,
            beta2_,
            beta1_power,
            beta2_power,
            epsilon_,
            learningRate);
}

void AdamaxParameterOptimizer::update(const VectorPtr vecs[],
                                      const ParameterConfig& config,
                                      size_t sparseId) const {
  CHECK(sparseId == -1UL) << "Sparse update is not supported";
  real learningRate = config.learning_rate() * learningRate_;

  BaseMatrix& value = *vecs[PARAMETER_VALUE];
  BaseMatrix& grad = *vecs[PARAMETER_GRADIENT];
  BaseMatrix& mom = *vecs[PARAMETER_MOMENTUM];
  BaseMatrix& u = *vecs[PARAMETER_WEIGHTED_INFINITY_NORM];

  adamaxApply(value, grad, mom, u, beta1_, beta2_, step_, learningRate);
}

void OptimizerWithGradientClipping::update(const VectorPtr vecs[],
                                           const ParameterConfig& config,
                                           size_t sparseId) const {
  real globalThreshold = optConfig_.gradient_clipping_threshold();
  real localThreshold = config.gradient_clipping_threshold();

  real threshold;
  std::string field;
  // Get the minimum of local and global threshold
  // as the real threshold for clipping
  if (globalThreshold > 0.0f && localThreshold > 0.0f) {
    threshold =
        globalThreshold < localThreshold ? globalThreshold : localThreshold;
    field = globalThreshold < localThreshold ? "global" : "local";
  } else if (globalThreshold > 0.0f) {
    threshold = globalThreshold;
    field = "global";
  } else {
    threshold = localThreshold;
    field = "local";
  }

  real maxAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsMax();
  if (maxAbsGrad > threshold) {
    if (FLAGS_log_clipping) {
      real avgAbsGrad = vecs[PARAMETER_GRADIENT]->getAbsSum() /
                        vecs[PARAMETER_GRADIENT]->getSize();
      LOG(INFO) << "parameter=" << config.name() << " need clipping by "
                << field << " threshold=" << threshold
                << ", max grad=" << maxAbsGrad << ", avg grad=" << avgAbsGrad;
    }
    vecs[PARAMETER_GRADIENT]->clip(-threshold, threshold);
  }
  optimizer_->update(vecs, config, sparseId);
}

}  // namespace paddle
<|endoftext|>


<commit_before>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: OOXMLDocument.hxx,v $
 * $Revision: 1.15 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/
#ifndef INCLUDED_OOXML_DOCUMENT_HXX
#define INCLUDED_OOXML_DOCUMENT_HXX

#include <sal/types.h>
#include <com/sun/star/uno/Reference.hxx>
#include <com/sun/star/io/XInputStream.hpp>
#ifndef _COM_SUN_STAR_UNO_XCOMPONENTCONTEX_HPP_
#include <com/sun/star/uno/XComponentContext.hpp>
#endif
#ifndef INCLUDED_WW8_RESOURCE_MODEL_HXX
#include <resourcemodel/WW8ResourceModel.hxx>
#endif
#ifndef _COM_SUN_STAR_XML_SAX_XPARSER_HOO_
#include <com/sun/star/xml/sax/XParser.hpp>
#endif
#include <com/sun/star/xml/sax/XFastParser.hpp>
#include <com/sun/star/xml/sax/XFastTokenHandler.hpp>
#include <com/sun/star/frame/XModel.hpp>
#include <com/sun/star/drawing/XShapes.hpp>

/**
   @file OOXMLDocument.hxx

Import of OOXML WordprocessingML Documents

   The following picture shows the classes involved in importing OOXML
   WordprocessingML documents.

   @image html ooxmlimportchain.png

   The DOCX consists of parts. Each part is an XML document.  The
   OOXMLDocument opens the DOCX and creates a SAX parser for the part
   containing the main document content. The OOXMLDocument creates a
   SAX handler, too. This handler is set as the handler for the events
   created by the parser. Finally the OOXMLDocument initiates the
   parsing process.

   The SAX handler hosts a stack of contexts. Each context is an
   instance of a class derived from OOXMLContext. There is a context
   class for each <define> in the model.xml.

   For a detailed information about how the contexts are handled see
   the documentation for OOXMLContext.

   The contexts know how to convert an element in OOXML to the
   intermediate format that the domain mapper understands. They
   enumerate the according entity in OOXML by sending the according
   events to the domain mapper.

   The domain mapper knows how to convert the intermediate format to
   API calls. It takes the events sent by the contexts and uses the
   core API to insert the according elements to the core.
 */

namespace writerfilter {
namespace ooxml
{

using namespace com::sun::star;

class WRITERFILTER_DLLPUBLIC OOXMLStream
{
public:
    enum StreamType_t { UNKNOWN, DOCUMENT, STYLES, FONTTABLE, NUMBERING, FOOTNOTES, ENDNOTES, COMMENTS, THEME };
    typedef boost::shared_ptr<OOXMLStream> Pointer_t;

    virtual ~OOXMLStream() {}

    /**
       Returns parser for this stream.
     */
    virtual uno::Reference<xml::sax::XParser> getParser() = 0;

    /**
       Returns fast parser for this stream.
     */
    virtual uno::Reference<xml::sax::XFastParser> getFastParser() = 0;

    /**
       Returns input stream for this stream.
     */
    virtual uno::Reference<io::XInputStream> getInputStream() = 0;

    virtual uno::Reference<io::XInputStream> getDocumentStream() = 0;

    /**
       Returns component context for this stream.
     */
    virtual uno::Reference<uno::XComponentContext> getContext() = 0;

    /**
       Returns target URL from relationships for a given id.

       @param rId           the id to look for

       @return the URL found or an empty string
     */
    virtual ::rtl::OUString getTargetForId(const ::rtl::OUString & rId) = 0;

    virtual uno::Reference<xml::sax::XFastTokenHandler>
    getFastTokenHandler(uno::Reference<uno::XComponentContext> rContext) = 0;

};

class WRITERFILTER_DLLPUBLIC OOXMLDocument : public writerfilter::Reference<Stream>
{
public:
    /**
       Pointer to this stream.
    */
    typedef boost::shared_ptr<OOXMLDocument> Pointer_t;

    virtual ~OOXMLDocument() {}

    /**
       Resolves this document to a stream handler.

       @param rStream     stream handler to resolve this document to
     */
    virtual void resolve(Stream & rStream) = 0;

    /**
       Returns string representation of the type of this reference.

       DEBUGGING PURPOSE ONLY.
     */
    virtual string getType() const = 0;

    /**
       Resolves a footnote to a stream handler.

       @param rStream       stream handler to resolve to
       @param rNoteId       id of the footnote to resolve
     */
    virtual void resolveFootnote(Stream & rStream,
                                 const rtl::OUString & rNoteId) = 0;

    /**
       Resolves an endnote to a stream handler.

       @param rStream       stream handler to resolve to
       @param rNoteId       id of the endnote to resolve
     */
    virtual void resolveEndnote(Stream & rStream,
                                const rtl::OUString & rNoteId) = 0;

    /**
       Resolves a comment to a stream handler.

       @param rStream       stream handler to resolve to
       @param rComment      id of the comment to resolve
     */
    virtual void resolveComment(Stream & rStream,
                                const rtl::OUString & rCommentId) = 0;

    /**
       Resolves a picture to a stream handler.

       @param rStream       stream handler to resolve to
       @param rPictureId    id of the picture to resolve
     */
    virtual void resolvePicture(Stream & rStream,
                                const rtl::OUString & rPictureId) = 0;

    /**
       Resolves a header to a stream handler.

       @param rStream       stream handler to resolve to
       @param type          type of header to resolve:
                            NS_ooxml::LN_Value_ST_HrdFtr_even     header on even page
                            NS_ooxml::LN_Value_ST_HrdFtr_default  header on right page
                            NS_ooxml::LN_Value_ST_HrdFtr_first    header on first page

       @param rId           id of the header
     */
    virtual void resolveHeader(Stream & rStream,
                               const sal_Int32 type,
                               const rtl::OUString & rId) = 0;

    /**
       Resolves a footer to a stream handler.

       @param rStream       stream handler to resolve to
       @param type          type of footer to resolve:
                            NS_ooxml::LN_Value_ST_HrdFtr_even     header on even page
                            NS_ooxml::LN_Value_ST_HrdFtr_default  header on right page
                            NS_ooxml::LN_Value_ST_HrdFtr_first    header on first page

       @param rId           id of the header
    */
    virtual void resolveFooter(Stream & rStream,
                               const sal_Int32 type,
                               const rtl::OUString & rId) = 0;

    /**
       Returns target URL from relationships for a given id.

       @param rId           the id to look for

       @return the URL found or an empty string
     */
    virtual ::rtl::OUString getTargetForId(const ::rtl::OUString & rId) = 0;

    virtual void setModel(uno::Reference<frame::XModel> xModel) = 0;
    virtual uno::Reference<frame::XModel> getModel() = 0;
    virtual void setShapes(uno::Reference<drawing::XShapes> xShapes) = 0;
    virtual uno::Reference<drawing::XShapes> getShapes() = 0;
    virtual uno::Reference<io::XInputStream> getInputStream() = 0;
    virtual uno::Reference<io::XInputStream> getInputStreamForId
    (const ::rtl::OUString & rId) = 0;
    virtual void setXNoteId(const rtl::OUString & rId) = 0;
    virtual const ::rtl::OUString & getXNoteId() const = 0;
};

class WRITERFILTER_DLLPUBLIC OOXMLDocumentFactory
{
public:
    static OOXMLStream::Pointer_t
    createStream(uno::Reference<uno::XComponentContext> rContext,
                 uno::Reference<io::XInputStream> rStream,
                 OOXMLStream::StreamType_t nStreamType = OOXMLStream::DOCUMENT);

    static OOXMLStream::Pointer_t
    createStream(OOXMLStream::Pointer_t pStream,
                 OOXMLStream::StreamType_t nStreamType = OOXMLStream::DOCUMENT);

    static OOXMLStream::Pointer_t
    createStream(OOXMLStream::Pointer_t pStream, const rtl::OUString & rId);

    static OOXMLDocument *
    createDocument(OOXMLStream::Pointer_t pStream);

};

void ooxmlidsToXML(::std::iostream & out);

}}
#endif // INCLUDED_OOXML_DOCUMENT_HXX
<commit_msg>INTEGRATION: CWS xmlfilter04 (1.13.14); FILE MERGED 2008/03/20 10:32:22 hbrinkm 1.13.14.5: resync cleanup 2008/03/19 14:33:16 hbrinkm 1.13.14.4: RESYNC: (1.13-1.14); FILE MERGED 2008/03/06 08:09:08 hbrinkm 1.13.14.3: removed getInputStream, new: getStorageStream, getTarget 2008/02/26 08:36:38 hbrinkm 1.13.14.2: additionally use type of footnote/endnote to specify which footnote/endnote to use 2008/02/21 12:33:22 hbrinkm 1.13.14.1: joined changes from xmlfilter03<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: OOXMLDocument.hxx,v $
 * $Revision: 1.16 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/
#ifndef INCLUDED_OOXML_DOCUMENT_HXX
#define INCLUDED_OOXML_DOCUMENT_HXX

#include <sal/types.h>
#include <com/sun/star/uno/Reference.hxx>
#include <com/sun/star/io/XInputStream.hpp>
#ifndef _COM_SUN_STAR_UNO_XCOMPONENTCONTEX_HPP_
#include <com/sun/star/uno/XComponentContext.hpp>
#endif
#ifndef INCLUDED_WW8_RESOURCE_MODEL_HXX
#include <resourcemodel/WW8ResourceModel.hxx>
#endif
#ifndef _COM_SUN_STAR_XML_SAX_XPARSER_HOO_
#include <com/sun/star/xml/sax/XParser.hpp>
#endif
#include <com/sun/star/xml/sax/XFastParser.hpp>
#include <com/sun/star/xml/sax/XFastTokenHandler.hpp>
#include <com/sun/star/frame/XModel.hpp>
#include <com/sun/star/drawing/XShapes.hpp>

/**
   @file OOXMLDocument.hxx

Import of OOXML WordprocessingML Documents

   The following picture shows the classes involved in importing OOXML
   WordprocessingML documents.

   @image html ooxmlimportchain.png

   The DOCX consists of parts. Each part is an XML document.  The
   OOXMLDocument opens the DOCX and creates a SAX parser for the part
   containing the main document content. The OOXMLDocument creates a
   SAX handler, too. This handler is set as the handler for the events
   created by the parser. Finally the OOXMLDocument initiates the
   parsing process.

   The SAX handler hosts a stack of contexts. Each context is an
   instance of a class derived from OOXMLContext. There is a context
   class for each <define> in the model.xml.

   For a detailed information about how the contexts are handled see
   the documentation for OOXMLContext.

   The contexts know how to convert an element in OOXML to the
   intermediate format that the domain mapper understands. They
   enumerate the according entity in OOXML by sending the according
   events to the domain mapper.

   The domain mapper knows how to convert the intermediate format to
   API calls. It takes the events sent by the contexts and uses the
   core API to insert the according elements to the core.
 */

namespace writerfilter {
namespace ooxml
{

using namespace com::sun::star;

class WRITERFILTER_DLLPUBLIC OOXMLStream
{
public:
    enum StreamType_t { UNKNOWN, DOCUMENT, STYLES, FONTTABLE, NUMBERING, FOOTNOTES, ENDNOTES, COMMENTS, THEME };
    typedef boost::shared_ptr<OOXMLStream> Pointer_t;

    virtual ~OOXMLStream() {}

    /**
       Returns parser for this stream.
     */
    virtual uno::Reference<xml::sax::XParser> getParser() = 0;

    /**
       Returns fast parser for this stream.
     */
    virtual uno::Reference<xml::sax::XFastParser> getFastParser() = 0;

    virtual uno::Reference<io::XInputStream> getDocumentStream() = 0;

    virtual uno::Reference<io::XInputStream> getStorageStream() = 0;

    /**
       Returns component context for this stream.
     */
    virtual uno::Reference<uno::XComponentContext> getContext() = 0;

    /**
       Returns target URL from relationships for a given id.

       @param rId           the id to look for

       @return the URL found or an empty string
     */
    virtual ::rtl::OUString getTargetForId(const ::rtl::OUString & rId) = 0;

    virtual const ::rtl::OUString & getTarget() const = 0;

    virtual uno::Reference<xml::sax::XFastTokenHandler>
    getFastTokenHandler(uno::Reference<uno::XComponentContext> rContext) = 0;

};

class WRITERFILTER_DLLPUBLIC OOXMLDocument : public writerfilter::Reference<Stream>
{
public:
    /**
       Pointer to this stream.
    */
    typedef boost::shared_ptr<OOXMLDocument> Pointer_t;

    virtual ~OOXMLDocument() {}

    /**
       Resolves this document to a stream handler.

       @param rStream     stream handler to resolve this document to
     */
    virtual void resolve(Stream & rStream) = 0;

    /**
       Returns string representation of the type of this reference.

       DEBUGGING PURPOSE ONLY.
     */
    virtual string getType() const = 0;

    /**
       Resolves a footnote to a stream handler.

       A footnote is resolved if either the note type or
       note id matches.

       @param rStream       stream handler to resolve to
       @param rNoteType     type of footnote to resolve
       @param rNoteId       id of the footnote to resolve
     */
    virtual void resolveFootnote(Stream & rStream,
                                 const Id & rNoteType,
                                 const rtl::OUString & rNoteId) = 0;
    /**
       Resolves an endnote to a stream handler.

       An endnote is resolved if either the note type or
       note id matches.

       @param rStream       stream handler to resolve to
       @param rNoteType     type of footnote to resolve
       @param rNoteId       id of the endnote to resolve
     */
    virtual void resolveEndnote(Stream & rStream,
                                const Id & rNoteType,
                                const rtl::OUString & rNoteId) = 0;

    /**
       Resolves a comment to a stream handler.

       @param rStream       stream handler to resolve to
       @param rComment      id of the comment to resolve
     */
    virtual void resolveComment(Stream & rStream,
                                const rtl::OUString & rCommentId) = 0;

    /**
       Resolves a picture to a stream handler.

       @param rStream       stream handler to resolve to
       @param rPictureId    id of the picture to resolve
     */
    virtual void resolvePicture(Stream & rStream,
                                const rtl::OUString & rPictureId) = 0;

    /**
       Resolves a header to a stream handler.

       @param rStream       stream handler to resolve to
       @param type          type of header to resolve:
                            NS_ooxml::LN_Value_ST_HrdFtr_even     header on even page
                            NS_ooxml::LN_Value_ST_HrdFtr_default  header on right page
                            NS_ooxml::LN_Value_ST_HrdFtr_first    header on first page

       @param rId           id of the header
     */
    virtual void resolveHeader(Stream & rStream,
                               const sal_Int32 type,
                               const rtl::OUString & rId) = 0;

    /**
       Resolves a footer to a stream handler.

       @param rStream       stream handler to resolve to
       @param type          type of footer to resolve:
                            NS_ooxml::LN_Value_ST_HrdFtr_even     header on even page
                            NS_ooxml::LN_Value_ST_HrdFtr_default  header on right page
                            NS_ooxml::LN_Value_ST_HrdFtr_first    header on first page

       @param rId           id of the header
    */
    virtual void resolveFooter(Stream & rStream,
                               const sal_Int32 type,
                               const rtl::OUString & rId) = 0;

    /**
       Returns target URL from relationships for a given id.

       @param rId           the id to look for

       @return the URL found or an empty string
     */
    virtual ::rtl::OUString getTargetForId(const ::rtl::OUString & rId) = 0;

    virtual void setModel(uno::Reference<frame::XModel> xModel) = 0;
    virtual uno::Reference<frame::XModel> getModel() = 0;
    virtual void setShapes(uno::Reference<drawing::XShapes> xShapes) = 0;
    virtual uno::Reference<drawing::XShapes> getShapes() = 0;
    virtual uno::Reference<io::XInputStream> getInputStream() = 0;
    virtual uno::Reference<io::XInputStream> getStorageStream() = 0;
    virtual uno::Reference<io::XInputStream> getInputStreamForId
    (const ::rtl::OUString & rId) = 0;
    virtual void setXNoteId(const rtl::OUString & rId) = 0;
    virtual const ::rtl::OUString & getXNoteId() const = 0;
    virtual void setXNoteType(const Id & nId) = 0;
    virtual const Id & getXNoteType() const = 0;
    virtual const ::rtl::OUString & getTarget() const = 0;
};

class WRITERFILTER_DLLPUBLIC OOXMLDocumentFactory
{
public:
    static OOXMLStream::Pointer_t
    createStream(uno::Reference<uno::XComponentContext> rContext,
                 uno::Reference<io::XInputStream> rStream,
                 OOXMLStream::StreamType_t nStreamType = OOXMLStream::DOCUMENT);

    static OOXMLStream::Pointer_t
    createStream(OOXMLStream::Pointer_t pStream,
                 OOXMLStream::StreamType_t nStreamType = OOXMLStream::DOCUMENT);

    static OOXMLStream::Pointer_t
    createStream(OOXMLStream::Pointer_t pStream, const rtl::OUString & rId);

    static OOXMLDocument *
    createDocument(OOXMLStream::Pointer_t pStream);

};

void ooxmlidsToXML(::std::iostream & out);

}}
#endif // INCLUDED_OOXML_DOCUMENT_HXX
<|endoftext|>


<commit_before>#include <pcl/point_types.h>
#include <pcl/io/pcd_io.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/filters/statistical_outlier_removal.h>
#include <pcl/kdtree/kdtree_flann.h>
#include <pcl/features/normal_3d.h>
#include <pcl/surface/poisson.h>
#include <pcl/io/vtk_io.h>

typedef pcl::Normal Normal;
//typedef pcl::PointXYZ PointType;
//typedef pcl::PointNormal PointTypeN;
typedef pcl::PointXYZRGB PointType;
typedef pcl::PointXYZRGBNormal PointTypeN;

void downsample (int argc, char* argv[]);
void remove_outliers (int argc, char* argv[]);
void reconstruct_mesh (int argc, char* argv[]);

int main (int argc, char *argv[])
{
    downsample (argc, argv);
    remove_outliers (argc, argv);
    reconstruct_mesh (argc, argv);

    return 0;
}

void downsample (int argc, char* argv[])
{
    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud.pcd", *cloud);
    }
    else {
        reader.read (argv[1], *cloud);
    }

    std::cout << "PointCloud before filtering: " << cloud->width *
        cloud->height << " data points (" << pcl::getFieldsList (*cloud)
        << ")." << std::endl;

    // Create the filtering object
    pcl::VoxelGrid<pcl::PCLPointCloud2> sor;
    sor.setInputCloud (cloud);
    sor.setLeafSize (0.01f, 0.01f, 0.01f);
    sor.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: " <<
            cloud_filtered->width * cloud_filtered->height
            << " data points (" << pcl::getFieldsList (*cloud_filtered)
            << ")." << std::endl;

    pcl::PCDWriter writer;
    if (argc < 2){
        writer.write ("pointcloud-downsampled.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        writer.write (str, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

}

void remove_outliers (int argc, char* argv[])
{
    pcl::PointCloud<pcl::PointXYZRGB>::Ptr cloud2 (new
            pcl::PointCloud<pcl::PointXYZRGB>);
    pcl::PointCloud<pcl::PointXYZRGB>::Ptr cloud_filtered2 (new
            pcl::PointCloud<pcl::PointXYZRGB>);

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read<pcl::PointXYZRGB> ("pointcloud-downsampled.pcd",
                *cloud2);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        reader.read (str, *cloud2);
    }

    std::cerr << "Cloud before filtering: " << std::endl;
    std::cerr << *cloud2 << std::endl;

    // Create the filtering object
    pcl::StatisticalOutlierRemoval<pcl::PointXYZRGB> sor2;
    sor2.setInputCloud (cloud2);
    sor2.setMeanK (50);
    sor2.setStddevMulThresh (1.0);
    sor2.filter (*cloud_filtered2);

    std::cout << "Cloud after filtering: " << std::endl;
    std::cout << *cloud_filtered2 << std::endl;

    pcl::PCDWriter writer;

    if (argc < 2){
        writer.write<pcl::PointXYZRGB>
            ("pointcloud-downsampled-inliers.pcd",
             *cloud_filtered2, false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered2);
        writer.write<pcl::PointXYZRGB>
            ("pointcloud-downsampled-outliers.pcd",
             *cloud_filtered2, false);
    }
    else {
        std::string str1;
        str1.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        std::string str2;
        str2.append(argv[1]).append("-downsampled.pcd-outliers.pcd");
        writer.write<pcl::PointXYZRGB> (str1, *cloud_filtered2, false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered2);
        writer.write<pcl::PointXYZRGB> (str2, *cloud_filtered2, false);
    }

}

void reconstruct_mesh (int argc, char* argv[])
{
    // Load input file into a PointCloud<T> with an appropriate type
    pcl::PointCloud<PointType>::Ptr cloud3 (new
            pcl::PointCloud<PointType>);
    pcl::PCLPointCloud2 cloud_blob;

    if (argc < 2) {
        pcl::io::loadPCDFile
            ("pointcloud-downsampled-inliers.pcd", cloud_blob);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        pcl::io::loadPCDFile (str, cloud_blob);
    }

    pcl::fromPCLPointCloud2 (cloud_blob, *cloud3);
    //* the data should be available in cloud
    std::cout << "cloud loaded " << std::endl;
    std::cout << cloud3->size() << std::endl;

    // Normal estimation*
    pcl::NormalEstimation<PointType, Normal> normEst;
    pcl::PointCloud<Normal>::Ptr normals (new pcl::PointCloud<Normal>);
    pcl::search::KdTree<PointType>::Ptr tree (new
            pcl::search::KdTree<PointType>);
    tree->setInputCloud (cloud3);
    normEst.setInputCloud (cloud3);
    normEst.setSearchMethod (tree);
    normEst.setKSearch (20);
    normEst.compute (*normals);
    //* normals should not contain the point normals + surface
    //curvatures

    // Concatenate the XYZ and normal fields*
    pcl::PointCloud<PointTypeN>::Ptr cloud_with_normals (new
            pcl::PointCloud<PointTypeN>);
    pcl::concatenateFields (*cloud3, *normals, *cloud_with_normals);
    //* cloud_with_normals = cloud + normals

    // Create search tree*
    pcl::search::KdTree<PointTypeN>::Ptr tree2 (new
            pcl::search::KdTree<PointTypeN>);
    tree2->setInputCloud (cloud_with_normals);

    // Initialize objects
    pcl::Poisson<PointTypeN> poissn;
    pcl::PolygonMesh triangles;

    std::cout << cloud_with_normals->size() << std::endl;

    poissn.setInputCloud(cloud_with_normals);
    poissn.setSearchMethod(tree2);
    poissn.reconstruct (triangles);
    poissn.setOutputPolygons(false);

    if (argc < 2){
        pcl::io::saveVTKFile
            ("pointcloud-downsampled-outliers-mesh.vtk",
             triangles);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-mesh.vtk");
        pcl::io::saveVTKFile (str, triangles);
    }
}

<commit_msg>added discriptive comments<commit_after>#include <pcl/point_types.h>
#include <pcl/io/pcd_io.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/filters/statistical_outlier_removal.h>
#include <pcl/kdtree/kdtree_flann.h>
#include <pcl/features/normal_3d.h>
#include <pcl/surface/poisson.h>
#include <pcl/io/vtk_io.h>

typedef pcl::Normal Normal;
//typedef pcl::PointXYZ PointType;
//typedef pcl::PointNormal PointTypeN;
typedef pcl::PointXYZRGB PointType;
typedef pcl::PointXYZRGBNormal PointTypeN;

void downsample (int argc, char* argv[]);
void remove_outliers (int argc, char* argv[]);
void reconstruct_mesh (int argc, char* argv[]);

int main (int argc, char *argv[])
{
    downsample (argc, argv);
    remove_outliers (argc, argv);
    reconstruct_mesh (argc, argv);

    return 0;
}

/* Downsampling with VoxelGrid class.
 * Voxel grid is a set of tiny 3D boxes in space.
 * In each voxel (i.e. 3D box) all points present will be approximated
 * (i.e. downsampled) with their centeroid.
 */

void downsample (int argc, char* argv[])
{
    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud.pcd", *cloud);
    }
    else {
        reader.read (argv[1], *cloud);
    }

    std::cout << "PointCloud before filtering: " << cloud->width *
        cloud->height << " data points (" << pcl::getFieldsList (*cloud)
        << ")." << std::endl;

    // Create the filtering object
    pcl::VoxelGrid<pcl::PCLPointCloud2> sor;
    sor.setInputCloud (cloud);
    // voxel size to be 1cm^3
    sor.setLeafSize (0.01f, 0.01f, 0.01f);
    sor.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: " <<
            cloud_filtered->width * cloud_filtered->height
            << " data points (" << pcl::getFieldsList (*cloud_filtered)
            << ")." << std::endl;

    pcl::PCDWriter writer;
    if (argc < 2){
        writer.write ("pointcloud-downsampled.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        writer.write (str, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

}

/* Removing noisy data (i.e. outliers) from measurements using statistical
 * analysis technique
 */

void remove_outliers (int argc, char* argv[])
{
    pcl::PointCloud<pcl::PointXYZRGB>::Ptr cloud (new
            pcl::PointCloud<pcl::PointXYZRGB>);
    pcl::PointCloud<pcl::PointXYZRGB>::Ptr cloud_filtered2 (new
            pcl::PointCloud<pcl::PointXYZRGB>);

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read<pcl::PointXYZRGB> ("pointcloud-downsampled.pcd",
                *cloud);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        reader.read (str, *cloud);
    }

    std::cout << "Cloud before filtering: " << std::endl;
    std::cout << *cloud << std::endl;

    // Create the filtering object
    pcl::StatisticalOutlierRemoval<pcl::PointXYZRGB> sor2;
    sor2.setInputCloud (cloud);
    // Set number of neighbors to analyze
    sor2.setMeanK (50);
    sor2.setStddevMulThresh (1.0);
    sor2.filter (*cloud_filtered2);

    std::cout << "Cloud after filtering: " << std::endl;
    std::cout << *cloud_filtered2 << std::endl;

    pcl::PCDWriter writer;

    // Write both inliers and outliers
    if (argc < 2){
        writer.write<pcl::PointXYZRGB>
            ("pointcloud-downsampled-inliers.pcd",
             *cloud_filtered2, false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered2);
        writer.write<pcl::PointXYZRGB>
            ("pointcloud-downsampled-outliers.pcd",
             *cloud_filtered2, false);
    }
    else {
        std::string str1;
        str1.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        std::string str2;
        str2.append(argv[1]).append("-downsampled.pcd-outliers.pcd");
        writer.write<pcl::PointXYZRGB> (str1, *cloud_filtered2, false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered2);
        writer.write<pcl::PointXYZRGB> (str2, *cloud_filtered2, false);
    }

}

/* Construct mesh of triangles from input pointcloud using Poisson's
 * surface reconstruction algorithm
 */

void reconstruct_mesh (int argc, char* argv[])
{
    // Load input file into a PointCloud<T> with an appropriate type
    pcl::PointCloud<PointType>::Ptr cloud (new
            pcl::PointCloud<PointType>);
    pcl::PCLPointCloud2 cloud_blob;

    if (argc < 2) {
        pcl::io::loadPCDFile
            ("pointcloud-downsampled-inliers.pcd", cloud_blob);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        pcl::io::loadPCDFile (str, cloud_blob);
    }

    pcl::fromPCLPointCloud2 (cloud_blob, *cloud);
    // the data should be available in cloud
    std::cout << "cloud loaded " << std::endl;
    std::cout << cloud->size() << std::endl;

    // Normal estimation
    pcl::NormalEstimation<PointType, Normal> normEst;
    pcl::PointCloud<Normal>::Ptr normals (new pcl::PointCloud<Normal>);

    // Create kdtree representation of cloud,
    // and pass it to the normal estimation object.
    pcl::search::KdTree<PointType>::Ptr tree (new
            pcl::search::KdTree<PointType>);
    tree->setInputCloud (cloud);
    normEst.setInputCloud (cloud);
    normEst.setSearchMethod (tree);
    // Use 20 neighbor points for estimating normal
    normEst.setKSearch (20);
    normEst.compute (*normals);
    // normals should not contain the point normals + surface
    // curvatures

    // Concatenate the XYZ and normal fields
    pcl::PointCloud<PointTypeN>::Ptr cloud_with_normals (new
            pcl::PointCloud<PointTypeN>);
    pcl::concatenateFields (*cloud, *normals, *cloud_with_normals);
    // cloud_with_normals = cloud + normals

    // Create search tree
    pcl::search::KdTree<PointTypeN>::Ptr tree2 (new
            pcl::search::KdTree<PointTypeN>);
    tree2->setInputCloud (cloud_with_normals);

    // Initialize objects
    // psn - for surface reconstruction algorithm
    // triangles - for storage of reconstructed triangles
    pcl::Poisson<PointTypeN> psn;
    pcl::PolygonMesh triangles;

    std::cout << cloud_with_normals->size() << std::endl;

    psn.setInputCloud(cloud_with_normals);
    psn.setSearchMethod(tree2);
    psn.reconstruct (triangles);
    psn.setOutputPolygons(false);

    // Write reconstructed mesh
    if (argc < 2){
        pcl::io::saveVTKFile
            ("pointcloud-downsampled-outliers-mesh.vtk",
             triangles);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-mesh.vtk");
        pcl::io::saveVTKFile (str, triangles);
    }
}

<|endoftext|>


<commit_before>/*
 * Jamoma Hash Table Class
 * Copyright © 2008, Timothy Place
 *
 * License: This code is licensed under the terms of the "New BSD License"
 * http://creativecommons.org/licenses/BSD/
 */

#include "TTHash.h"
#include "TTSymbolTable.h"
#include "TTMutex.h"

#ifdef TT_PLATFORM_WIN
	#include <hash_map>
	using namespace stdext;	// Visual Studio 2008 puts the hash_map in this namespace
	typedef hash_map<TTPtrSizedInt,TTValue>			TTHashMap;
#else
	#include "boost/unordered_map.hpp"
	using namespace boost;
	typedef unordered_map<TTPtrSizedInt,TTValue>	TTHashMap;
#endif

typedef TTHashMap::const_iterator	TTHashMapIter;
#define HASHMAP  ((TTHashMap*)(mHashMap))
#define mHASHMAP (*HASHMAP)

/****************************************************************************************************/

TTHash::TTHash()
			:mThreadProtection(NO)
{
	mHashMap = new TTHashMap;
	mMutex = new TTMutex(false);
}

TTHash::~TTHash()
{
	delete mMutex;
	delete HASHMAP;
}

TTHash::TTHash(TTHash& that)
{
	mHashMap = new TTHashMap;
	mMutex = new TTMutex(false);

	*HASHMAP = *((TTHashMap*)that.mHashMap);
}

TTErr TTHash::append(const TTSymbolPtr key, const TTValue& value)
{
	lock();
	HASHMAP->insert(TTKeyVal(TTPtrSizedInt(key), value));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::lookup(const TTSymbolPtr key, TTValue& value)
{
	lock();
	TTHashMapIter iter = HASHMAP->find(TTPtrSizedInt(key));

	if (iter == HASHMAP->end()) {
		unlock();
		return kTTErrValueNotFound;
	}
	else {
		value = iter->second;
		unlock();
		return kTTErrNone;
	}
}

TTErr TTHash::remove(const TTSymbolPtr key)
{
	lock();
	HASHMAP->erase(TTPtrSizedInt(key));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::clear()
{
	lock();
	HASHMAP->clear();
	unlock();
	return kTTErrNone;
}

TTErr TTHash::getKeys(TTValue& hashKeys)
{
	lock();
	hashKeys.clear();

	for (TTHashMapIter iter = HASHMAP->begin(); iter != HASHMAP->end(); iter++)
		hashKeys.append(TTSymbolPtr(iter->first));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::iterate(const TTPtr target, const TTHashIteratorType callback)
{
	lock();

	for (TTHashMapIter iter = HASHMAP->begin(); iter != HASHMAP->end(); iter++)
		callback(target, *iter);
	unlock();
	return kTTErrNone;
}

TTUInt32 TTHash::getSize()
{
	return HASHMAP->size();
}

TTBoolean TTHash::isEmpty()
{
	return HASHMAP->empty();
}

void TTHash::lock()
{
	if (mThreadProtection)
		mMutex->lock();
}

void TTHash::unlock()
{
	if (mThreadProtection)
		mMutex->unlock();
}

<commit_msg>TTHash: on Linux we use the C++0x standard unordered_map instead of the Boost unordered_map. The Boost unordered_map is now deprecated in favor of the C++0x and also it doesn't compile on the BeagleBoard.<commit_after>/*
 * Jamoma Hash Table Class
 * Copyright © 2008, Timothy Place
 *
 * License: This code is licensed under the terms of the "New BSD License"
 * http://creativecommons.org/licenses/BSD/
 */

#include "TTHash.h"
#include "TTSymbolTable.h"
#include "TTMutex.h"

#ifdef TT_PLATFORM_WIN
	#include <hash_map>
	using namespace stdext;	// Visual Studio 2008 puts the hash_map in this namespace
	typedef hash_map<TTPtrSizedInt,TTValue>			TTHashMap;
#else
	#ifdef TT_PLATFORM_LINUX
		// at least for GCC 4.6 on the BeagleBoard, the unordered map is standard
		#include <unordered_map>
	#else
		#include "boost/unordered_map.hpp"
		using namespace boost;
	#endif
	typedef unordered_map<TTPtrSizedInt,TTValue>	TTHashMap;
#endif

typedef TTHashMap::const_iterator	TTHashMapIter;
#define HASHMAP  ((TTHashMap*)(mHashMap))
#define mHASHMAP (*HASHMAP)

/****************************************************************************************************/

TTHash::TTHash()
			:mThreadProtection(NO)
{
	mHashMap = new TTHashMap;
	mMutex = new TTMutex(false);
}

TTHash::~TTHash()
{
	delete mMutex;
	delete HASHMAP;
}

TTHash::TTHash(TTHash& that)
{
	mHashMap = new TTHashMap;
	mMutex = new TTMutex(false);

	*HASHMAP = *((TTHashMap*)that.mHashMap);
}

TTErr TTHash::append(const TTSymbolPtr key, const TTValue& value)
{
	lock();
	HASHMAP->insert(TTKeyVal(TTPtrSizedInt(key), value));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::lookup(const TTSymbolPtr key, TTValue& value)
{
	lock();
	TTHashMapIter iter = HASHMAP->find(TTPtrSizedInt(key));

	if (iter == HASHMAP->end()) {
		unlock();
		return kTTErrValueNotFound;
	}
	else {
		value = iter->second;
		unlock();
		return kTTErrNone;
	}
}

TTErr TTHash::remove(const TTSymbolPtr key)
{
	lock();
	HASHMAP->erase(TTPtrSizedInt(key));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::clear()
{
	lock();
	HASHMAP->clear();
	unlock();
	return kTTErrNone;
}

TTErr TTHash::getKeys(TTValue& hashKeys)
{
	lock();
	hashKeys.clear();

	for (TTHashMapIter iter = HASHMAP->begin(); iter != HASHMAP->end(); iter++)
		hashKeys.append(TTSymbolPtr(iter->first));
	unlock();
	return kTTErrNone;
}

TTErr TTHash::iterate(const TTPtr target, const TTHashIteratorType callback)
{
	lock();

	for (TTHashMapIter iter = HASHMAP->begin(); iter != HASHMAP->end(); iter++)
		callback(target, *iter);
	unlock();
	return kTTErrNone;
}

TTUInt32 TTHash::getSize()
{
	return HASHMAP->size();
}

TTBoolean TTHash::isEmpty()
{
	return HASHMAP->empty();
}

void TTHash::lock()
{
	if (mThreadProtection)
		mMutex->lock();
}

void TTHash::unlock()
{
	if (mThreadProtection)
		mMutex->unlock();
}

<|endoftext|>


<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/cpu-info.h"

#ifdef __APPLE__
#include <sys/sysctl.h>
#endif

#include <boost/algorithm/string.hpp>
#include <iostream>
#include <fstream>
#include <gutil/strings/substitute.h>
#include <mmintrin.h>
#include <sstream>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "util/pretty-printer.h"

#include "common/names.h"

using boost::algorithm::contains;
using boost::algorithm::trim;
using std::max;
using strings::Substitute;

DECLARE_bool(abort_on_config_error);
DEFINE_int32(num_cores, 0, "(Advanced) If > 0, it sets the number of cores available to"
    " Impala. Setting it to 0 means Impala will use all available cores on the machine"
    " according to /proc/cpuinfo.");

namespace impala {

bool CpuInfo::initialized_ = false;
int64_t CpuInfo::hardware_flags_ = 0;
int64_t CpuInfo::original_hardware_flags_;
int64_t CpuInfo::cycles_per_ms_;
int CpuInfo::num_cores_ = 1;
string CpuInfo::model_name_ = "unknown";

static struct {
  string name;
  int64_t flag;
} flag_mappings[] =
{
  { "ssse3",  CpuInfo::SSSE3 },
  { "sse4_1", CpuInfo::SSE4_1 },
  { "sse4_2", CpuInfo::SSE4_2 },
  { "popcnt", CpuInfo::POPCNT },
};
static const long num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]);

// Helper function to parse for hardware flags.
// values contains a list of space-seperated flags.  check to see if the flags we
// care about are present.
// Returns a bitmap of flags.
int64_t ParseCPUFlags(const string& values) {
  int64_t flags = 0;
  for (int i = 0; i < num_flags; ++i) {
    if (contains(values, flag_mappings[i].name)) {
      flags |= flag_mappings[i].flag;
    }
  }
  return flags;
}

void CpuInfo::Init() {
  string line;
  string name;
  string value;

  float max_mhz = 0;
  int num_cores = 0;

  // Read from /proc/cpuinfo
  ifstream cpuinfo("/proc/cpuinfo", ios::in);
  while (cpuinfo) {
    getline(cpuinfo, line);
    size_t colon = line.find(':');
    if (colon != string::npos) {
      name = line.substr(0, colon - 1);
      value = line.substr(colon + 1, string::npos);
      trim(name);
      trim(value);
      if (name.compare("flags") == 0) {
        hardware_flags_ |= ParseCPUFlags(value);
      } else if (name.compare("cpu MHz") == 0) {
        // Every core will report a different speed.  We'll take the max, assuming
        // that when impala is running, the core will not be in a lower power state.
        // TODO: is there a more robust way to do this, such as
        // Window's QueryPerformanceFrequency()
        float mhz = atof(value.c_str());
        max_mhz = max(mhz, max_mhz);
      } else if (name.compare("processor") == 0) {
        ++num_cores;
      } else if (name.compare("model name") == 0) {
        model_name_ = value;
      }
    }
  }
  if (cpuinfo.is_open()) cpuinfo.close();

  if (max_mhz != 0) {
    cycles_per_ms_ = max_mhz * 1000;
  } else {
    cycles_per_ms_ = 1000000;
  }
  original_hardware_flags_ = hardware_flags_;

  if (num_cores > 0) {
    num_cores_ = num_cores;
  } else {
    num_cores_ = 1;
  }

  if (FLAGS_num_cores > 0) num_cores_ = FLAGS_num_cores;

  initialized_ = true;
}

void CpuInfo::VerifyCpuRequirements() {
  if (!CpuInfo::IsSupported(CpuInfo::SSSE3)) {
    LOG(ERROR) << "CPU does not support the Supplemental SSE3 (SSSE3) instruction set, "
               << "which is required. Exiting if Supplemental SSE3 is not functional...";
  }
}

void CpuInfo::EnableFeature(long flag, bool enable) {
  DCHECK(initialized_);
  if (!enable) {
    hardware_flags_ &= ~flag;
  } else {
    // Can't turn something on that can't be supported
    DCHECK((original_hardware_flags_ & flag) != 0);
    hardware_flags_ |= flag;
  }
}

void CpuInfo::GetCacheInfo(long cache_sizes[NUM_CACHE_LEVELS],
      long cache_line_sizes[NUM_CACHE_LEVELS]) {
#ifdef __APPLE__
  // On Mac OS X use sysctl() to get the cache sizes
  size_t len = 0;
  sysctlbyname("hw.cachesize", NULL, &len, NULL, 0);
  uint64_t* data = static_cast<uint64_t*>(malloc(len));
  sysctlbyname("hw.cachesize", data, &len, NULL, 0);
  DCHECK(len / sizeof(uint64_t) >= 3);
  for (size_t i = 0; i < NUM_CACHE_LEVELS; ++i) {
    cache_sizes[i] = data[i];
  }
  size_t linesize;
  size_t sizeof_linesize = sizeof(linesize);
  sysctlbyname("hw.cachelinesize", &linesize, &sizeof_linesize, NULL, 0);
  for (size_t i = 0; i < NUM_CACHE_LEVELS; ++i) cache_line_sizes[i] = linesize;
#else
  // Call sysconf to query for the cache sizes
  // Note: on some systems (e.g. RHEL 5 on AWS EC2), this returns 0 instead of the
  // actual cache line size.
  cache_sizes[L1_CACHE] = sysconf(_SC_LEVEL1_DCACHE_SIZE);
  cache_sizes[L2_CACHE] = sysconf(_SC_LEVEL2_CACHE_SIZE);
  cache_sizes[L3_CACHE] = sysconf(_SC_LEVEL3_CACHE_SIZE);

  cache_line_sizes[L1_CACHE] = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
  cache_line_sizes[L2_CACHE] = sysconf(_SC_LEVEL2_CACHE_LINESIZE);
  cache_line_sizes[L3_CACHE] = sysconf(_SC_LEVEL3_CACHE_LINESIZE);
#endif
}

string CpuInfo::DebugString() {
  DCHECK(initialized_);
  stringstream stream;
  long cache_sizes[NUM_CACHE_LEVELS];
  long cache_line_sizes[NUM_CACHE_LEVELS];
  GetCacheInfo(cache_sizes, cache_line_sizes);

  string L1 = Substitute("L1 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L1_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L1_CACHE], TUnit::BYTES));
  string L2 = Substitute("L1 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L2_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L2_CACHE], TUnit::BYTES));
  string L3 = Substitute("L1 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L3_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L3_CACHE], TUnit::BYTES));
  stream << "Cpu Info:" << endl
         << "  Model: " << model_name_ << endl
         << "  Cores: " << num_cores_ << endl
         << "  " << L1 << endl
         << "  " << L2 << endl
         << "  " << L3 << endl
         << "  Hardware Supports:" << endl;
  for (int i = 0; i < num_flags; ++i) {
    if (IsSupported(flag_mappings[i].flag)) {
      stream << "    " << flag_mappings[i].name << endl;
    }
  }
  return stream.str();
}

}
<commit_msg>IMPALA-3597: mislabelled cache levels on debug webpage<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/cpu-info.h"

#ifdef __APPLE__
#include <sys/sysctl.h>
#endif

#include <boost/algorithm/string.hpp>
#include <iostream>
#include <fstream>
#include <gutil/strings/substitute.h>
#include <mmintrin.h>
#include <sstream>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "util/pretty-printer.h"

#include "common/names.h"

using boost::algorithm::contains;
using boost::algorithm::trim;
using std::max;
using strings::Substitute;

DECLARE_bool(abort_on_config_error);
DEFINE_int32(num_cores, 0, "(Advanced) If > 0, it sets the number of cores available to"
    " Impala. Setting it to 0 means Impala will use all available cores on the machine"
    " according to /proc/cpuinfo.");

namespace impala {

bool CpuInfo::initialized_ = false;
int64_t CpuInfo::hardware_flags_ = 0;
int64_t CpuInfo::original_hardware_flags_;
int64_t CpuInfo::cycles_per_ms_;
int CpuInfo::num_cores_ = 1;
string CpuInfo::model_name_ = "unknown";

static struct {
  string name;
  int64_t flag;
} flag_mappings[] =
{
  { "ssse3",  CpuInfo::SSSE3 },
  { "sse4_1", CpuInfo::SSE4_1 },
  { "sse4_2", CpuInfo::SSE4_2 },
  { "popcnt", CpuInfo::POPCNT },
};
static const long num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]);

// Helper function to parse for hardware flags.
// values contains a list of space-seperated flags.  check to see if the flags we
// care about are present.
// Returns a bitmap of flags.
int64_t ParseCPUFlags(const string& values) {
  int64_t flags = 0;
  for (int i = 0; i < num_flags; ++i) {
    if (contains(values, flag_mappings[i].name)) {
      flags |= flag_mappings[i].flag;
    }
  }
  return flags;
}

void CpuInfo::Init() {
  string line;
  string name;
  string value;

  float max_mhz = 0;
  int num_cores = 0;

  // Read from /proc/cpuinfo
  ifstream cpuinfo("/proc/cpuinfo", ios::in);
  while (cpuinfo) {
    getline(cpuinfo, line);
    size_t colon = line.find(':');
    if (colon != string::npos) {
      name = line.substr(0, colon - 1);
      value = line.substr(colon + 1, string::npos);
      trim(name);
      trim(value);
      if (name.compare("flags") == 0) {
        hardware_flags_ |= ParseCPUFlags(value);
      } else if (name.compare("cpu MHz") == 0) {
        // Every core will report a different speed.  We'll take the max, assuming
        // that when impala is running, the core will not be in a lower power state.
        // TODO: is there a more robust way to do this, such as
        // Window's QueryPerformanceFrequency()
        float mhz = atof(value.c_str());
        max_mhz = max(mhz, max_mhz);
      } else if (name.compare("processor") == 0) {
        ++num_cores;
      } else if (name.compare("model name") == 0) {
        model_name_ = value;
      }
    }
  }
  if (cpuinfo.is_open()) cpuinfo.close();

  if (max_mhz != 0) {
    cycles_per_ms_ = max_mhz * 1000;
  } else {
    cycles_per_ms_ = 1000000;
  }
  original_hardware_flags_ = hardware_flags_;

  if (num_cores > 0) {
    num_cores_ = num_cores;
  } else {
    num_cores_ = 1;
  }

  if (FLAGS_num_cores > 0) num_cores_ = FLAGS_num_cores;

  initialized_ = true;
}

void CpuInfo::VerifyCpuRequirements() {
  if (!CpuInfo::IsSupported(CpuInfo::SSSE3)) {
    LOG(ERROR) << "CPU does not support the Supplemental SSE3 (SSSE3) instruction set, "
               << "which is required. Exiting if Supplemental SSE3 is not functional...";
  }
}

void CpuInfo::EnableFeature(long flag, bool enable) {
  DCHECK(initialized_);
  if (!enable) {
    hardware_flags_ &= ~flag;
  } else {
    // Can't turn something on that can't be supported
    DCHECK((original_hardware_flags_ & flag) != 0);
    hardware_flags_ |= flag;
  }
}

void CpuInfo::GetCacheInfo(long cache_sizes[NUM_CACHE_LEVELS],
      long cache_line_sizes[NUM_CACHE_LEVELS]) {
#ifdef __APPLE__
  // On Mac OS X use sysctl() to get the cache sizes
  size_t len = 0;
  sysctlbyname("hw.cachesize", NULL, &len, NULL, 0);
  uint64_t* data = static_cast<uint64_t*>(malloc(len));
  sysctlbyname("hw.cachesize", data, &len, NULL, 0);
  DCHECK(len / sizeof(uint64_t) >= 3);
  for (size_t i = 0; i < NUM_CACHE_LEVELS; ++i) {
    cache_sizes[i] = data[i];
  }
  size_t linesize;
  size_t sizeof_linesize = sizeof(linesize);
  sysctlbyname("hw.cachelinesize", &linesize, &sizeof_linesize, NULL, 0);
  for (size_t i = 0; i < NUM_CACHE_LEVELS; ++i) cache_line_sizes[i] = linesize;
#else
  // Call sysconf to query for the cache sizes
  // Note: on some systems (e.g. RHEL 5 on AWS EC2), this returns 0 instead of the
  // actual cache line size.
  cache_sizes[L1_CACHE] = sysconf(_SC_LEVEL1_DCACHE_SIZE);
  cache_sizes[L2_CACHE] = sysconf(_SC_LEVEL2_CACHE_SIZE);
  cache_sizes[L3_CACHE] = sysconf(_SC_LEVEL3_CACHE_SIZE);

  cache_line_sizes[L1_CACHE] = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
  cache_line_sizes[L2_CACHE] = sysconf(_SC_LEVEL2_CACHE_LINESIZE);
  cache_line_sizes[L3_CACHE] = sysconf(_SC_LEVEL3_CACHE_LINESIZE);
#endif
}

string CpuInfo::DebugString() {
  DCHECK(initialized_);
  stringstream stream;
  long cache_sizes[NUM_CACHE_LEVELS];
  long cache_line_sizes[NUM_CACHE_LEVELS];
  GetCacheInfo(cache_sizes, cache_line_sizes);

  string L1 = Substitute("L1 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L1_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L1_CACHE], TUnit::BYTES));
  string L2 = Substitute("L2 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L2_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L2_CACHE], TUnit::BYTES));
  string L3 = Substitute("L3 Cache: $0 (Line: $1)",
      PrettyPrinter::Print(cache_sizes[L3_CACHE], TUnit::BYTES),
      PrettyPrinter::Print(cache_line_sizes[L3_CACHE], TUnit::BYTES));
  stream << "Cpu Info:" << endl
         << "  Model: " << model_name_ << endl
         << "  Cores: " << num_cores_ << endl
         << "  " << L1 << endl
         << "  " << L2 << endl
         << "  " << L3 << endl
         << "  Hardware Supports:" << endl;
  for (int i = 0; i < num_flags; ++i) {
    if (IsSupported(flag_mappings[i].flag)) {
      stream << "    " << flag_mappings[i].name << endl;
    }
  }
  return stream.str();
}

}
<|endoftext|>


<commit_before><commit_msg>Remember the server certificate on Mac OS X even if the handshake fails BUG=23569 TEST=https://rbacpro.sftcomp.ru/ should report "invalid certificate" rather than crash Review URL: http://codereview.chromium.org/270034<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2011-2013 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "bitcoinunits.h"

#include <QStringList>

BitcoinUnits::BitcoinUnits(QObject *parent):
        QAbstractListModel(parent),
        unitlist(availableUnits())
{
}

QList<BitcoinUnits::Unit> BitcoinUnits::availableUnits()
{
    QList<BitcoinUnits::Unit> unitlist;
    unitlist.append(MDOGE);
    unitlist.append(kDOGE);
    unitlist.append(DOGE);
    unitlist.append(Koinu);
    return unitlist;
}

bool BitcoinUnits::valid(int unit)
{
    switch(unit)
    {
    case MDOGE:
    case kDOGE:
    case DOGE:
    case Koinu:
        return true;
    default:
        return false;
    }
}

QString BitcoinUnits::name(int unit)
{
    switch(unit)
    {
    case MDOGE: return QString("MDOGE");
    case kDOGE: return QString("kDOGE");
    case DOGE: return QString("DOGE");
    case Koinu: return QString("Koinu");
    default: return QString("???");
    }
}

QString BitcoinUnits::description(int unit)
{
    switch(unit)
    {
    case MDOGE: return QString("Mega-Dogecoin (1,000,000 DOGE)");
    case kDOGE: return QString("Kilo-Dogecoin (1000 DOGE)");
    case DOGE: return QString("Dogecoin");
    case Koinu: return QString("Koinu (1 / 100,000,000");
    default: return QString("???");
    }
}

qint64 BitcoinUnits::factor(int unit)
{
    switch(unit)
    {
    case MDOGE: return Q_INT64_C(100000000000000);
    case kDOGE: return Q_INT64_C(100000000000);
    case DOGE:  return Q_INT64_C(100000000);
    case Koinu: return Q_INT64_C(1);
    default:    return Q_INT64_C(100000000);
    }
}

qint64 BitcoinUnits::maxAmount(int unit)
{
    switch(unit)
    {
    case MDOGE: return Q_INT64_C(900000);
    case kDOGE: return Q_INT64_C(900000000);
    case DOGE:  return Q_INT64_C(900000000000);    //less than the coin supply until the year 2170
    case Koinu: return Q_INT64_C(90000000000000000000);
    default:   return 0;
    }
}

int BitcoinUnits::amountDigits(int unit)
{
    switch(unit)
    {
    case MDOGE: return 6;  // 900,000 (# digits, without commas)
    case kDOGE: return 9;  // 900,000,000
    case DOGE:  return 12; // 900,000,000,000
    case Koinu: return 20; // 90,000,000,000,000,000,000
    default: return 0;
    }
}

int BitcoinUnits::decimals(int unit)
{
    switch(unit)
    {
    case MDOGE: return 14;
    case kDOGE: return 11;
    case DOGE: return 8;
    case Koinu: return 0;
    default: return 0;
    }
}

QString BitcoinUnits::format(int unit, qint64 n, bool fPlus)
{
    // Note: not using straight sprintf here because we do NOT want
    // localized number formatting.
    if(!valid(unit))
        return QString(); // Refuse to format invalid unit
    qint64 coin = factor(unit);
    int num_decimals = decimals(unit);
    qint64 n_abs = (n > 0 ? n : -n);
    qint64 quotient = n_abs / coin;
    qint64 remainder = n_abs % coin;
    QString quotient_str = QString::number(quotient);
    QString remainder_str = QString::number(remainder).rightJustified(num_decimals, '0');

    // Right-trim excess zeros after the decimal point
    int nTrim = 0;
    for (int i = remainder_str.size()-1; i>=2 && (remainder_str.at(i) == '0'); --i)
        ++nTrim;
    remainder_str.chop(nTrim);

    if (n < 0)
        quotient_str.insert(0, '-');
    else if (fPlus && n > 0)
        quotient_str.insert(0, '+');
    return quotient_str + QString(".") + remainder_str;
}

QString BitcoinUnits::formatWithUnit(int unit, qint64 amount, bool plussign)
{
    return format(unit, amount, plussign) + QString(" ") + name(unit);
}

bool BitcoinUnits::parse(int unit, const QString &value, qint64 *val_out)
{
    if(!valid(unit) || value.isEmpty())
        return false; // Refuse to parse invalid unit or empty string
    int num_decimals = decimals(unit);
    QStringList parts = value.split(".");

    if(parts.size() > 2)
    {
        return false; // More than one dot
    }
    QString whole = parts[0];
    QString decimals;

    if(parts.size() > 1)
    {
        decimals = parts[1];
    }
    if(decimals.size() > num_decimals)
    {
        return false; // Exceeds max precision
    }
    bool ok = false;
    QString str = whole + decimals.leftJustified(num_decimals, '0');

    if(str.size() > 18)
    {
        return false; // Longer numbers will exceed 63 bits
    }
    qint64 retvalue = str.toLongLong(&ok);
    if(val_out)
    {
        *val_out = retvalue;
    }
    return ok;
}

int BitcoinUnits::rowCount(const QModelIndex &parent) const
{
    Q_UNUSED(parent);
    return unitlist.size();
}

QVariant BitcoinUnits::data(const QModelIndex &index, int role) const
{
    int row = index.row();
    if(row >= 0 && row < unitlist.size())
    {
        Unit unit = unitlist.at(row);
        switch(role)
        {
        case Qt::EditRole:
        case Qt::DisplayRole:
            return QVariant(name(unit));
        case Qt::ToolTipRole:
            return QVariant(description(unit));
        case UnitRole:
            return QVariant(static_cast<int>(unit));
        }
    }
    return QVariant();
}
<commit_msg>Took an order of magnitude off maximum value for Koinu, and precision, so it fits into a 64bit integer value.<commit_after>// Copyright (c) 2011-2013 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "bitcoinunits.h"

#include <QStringList>

BitcoinUnits::BitcoinUnits(QObject *parent):
        QAbstractListModel(parent),
        unitlist(availableUnits())
{
}

QList<BitcoinUnits::Unit> BitcoinUnits::availableUnits()
{
    QList<BitcoinUnits::Unit> unitlist;
    unitlist.append(MDOGE);
    unitlist.append(kDOGE);
    unitlist.append(DOGE);
    unitlist.append(Koinu);
    return unitlist;
}

bool BitcoinUnits::valid(int unit)
{
    switch(unit)
    {
    case MDOGE:
    case kDOGE:
    case DOGE:
    case Koinu:
        return true;
    default:
        return false;
    }
}

QString BitcoinUnits::name(int unit)
{
    switch(unit)
    {
    case MDOGE: return QString("MDOGE");
    case kDOGE: return QString("kDOGE");
    case DOGE: return QString("DOGE");
    case Koinu: return QString("Koinu");
    default: return QString("???");
    }
}

QString BitcoinUnits::description(int unit)
{
    switch(unit)
    {
    case MDOGE: return QString("Mega-Dogecoin (1,000,000 DOGE)");
    case kDOGE: return QString("Kilo-Dogecoin (1000 DOGE)");
    case DOGE: return QString("Dogecoin");
    case Koinu: return QString("Koinu (1 / 100,000,000");
    default: return QString("???");
    }
}

qint64 BitcoinUnits::factor(int unit)
{
    switch(unit)
    {
    case MDOGE: return Q_INT64_C(100000000000000);
    case kDOGE: return Q_INT64_C(100000000000);
    case DOGE:  return Q_INT64_C(100000000);
    case Koinu: return Q_INT64_C(1);
    default:    return Q_INT64_C(100000000);
    }
}

qint64 BitcoinUnits::maxAmount(int unit)
{
    switch(unit)
    {
    case MDOGE: return Q_INT64_C(900000);
    case kDOGE: return Q_INT64_C(900000000);
    case DOGE:  return Q_INT64_C(900000000000);    //less than the coin supply until the year 2170
    case Koinu: return Q_INT64_C(9000000000000000000); // Slightly under max value for int64
    default:   return 0;
    }
}

int BitcoinUnits::amountDigits(int unit)
{
    switch(unit)
    {
    case MDOGE: return 6;  // 900,000 (# digits, without commas)
    case kDOGE: return 9;  // 900,000,000
    case DOGE:  return 12; // 900,000,000,000
    case Koinu: return 19; // 9,000,000,000,000,000,000
    default: return 0;
    }
}

int BitcoinUnits::decimals(int unit)
{
    switch(unit)
    {
    case MDOGE: return 14;
    case kDOGE: return 11;
    case DOGE: return 8;
    case Koinu: return 0;
    default: return 0;
    }
}

QString BitcoinUnits::format(int unit, qint64 n, bool fPlus)
{
    // Note: not using straight sprintf here because we do NOT want
    // localized number formatting.
    if(!valid(unit))
        return QString(); // Refuse to format invalid unit
    qint64 coin = factor(unit);
    int num_decimals = decimals(unit);
    qint64 n_abs = (n > 0 ? n : -n);
    qint64 quotient = n_abs / coin;
    qint64 remainder = n_abs % coin;
    QString quotient_str = QString::number(quotient);
    QString remainder_str = QString::number(remainder).rightJustified(num_decimals, '0');

    // Right-trim excess zeros after the decimal point
    int nTrim = 0;
    for (int i = remainder_str.size()-1; i>=2 && (remainder_str.at(i) == '0'); --i)
        ++nTrim;
    remainder_str.chop(nTrim);

    if (n < 0)
        quotient_str.insert(0, '-');
    else if (fPlus && n > 0)
        quotient_str.insert(0, '+');
    return quotient_str + QString(".") + remainder_str;
}

QString BitcoinUnits::formatWithUnit(int unit, qint64 amount, bool plussign)
{
    return format(unit, amount, plussign) + QString(" ") + name(unit);
}

bool BitcoinUnits::parse(int unit, const QString &value, qint64 *val_out)
{
    if(!valid(unit) || value.isEmpty())
        return false; // Refuse to parse invalid unit or empty string
    int num_decimals = decimals(unit);
    QStringList parts = value.split(".");

    if(parts.size() > 2)
    {
        return false; // More than one dot
    }
    QString whole = parts[0];
    QString decimals;

    if(parts.size() > 1)
    {
        decimals = parts[1];
    }
    if(decimals.size() > num_decimals)
    {
        return false; // Exceeds max precision
    }
    bool ok = false;
    QString str = whole + decimals.leftJustified(num_decimals, '0');

    if(str.size() > 18)
    {
        return false; // Longer numbers will exceed 63 bits
    }
    qint64 retvalue = str.toLongLong(&ok);
    if(val_out)
    {
        *val_out = retvalue;
    }
    return ok;
}

int BitcoinUnits::rowCount(const QModelIndex &parent) const
{
    Q_UNUSED(parent);
    return unitlist.size();
}

QVariant BitcoinUnits::data(const QModelIndex &index, int role) const
{
    int row = index.row();
    if(row >= 0 && row < unitlist.size())
    {
        Unit unit = unitlist.at(row);
        switch(role)
        {
        case Qt::EditRole:
        case Qt::DisplayRole:
            return QVariant(name(unit));
        case Qt::ToolTipRole:
            return QVariant(description(unit));
        case UnitRole:
            return QVariant(static_cast<int>(unit));
        }
    }
    return QVariant();
}
<|endoftext|>