<commit_before>/**
 * @file hoeffding_split_impl.hpp
 * @author Ryan Curtin
 *
 * Implementation of the HoeffdingSplit class.
 */
#ifndef __MLPACK_METHODS_HOEFFDING_TREES_HOEFFDING_SPLIT_IMPL_HPP
#define __MLPACK_METHODS_HOEFFDING_TREES_HOEFFDING_SPLIT_IMPL_HPP

namespace mlpack {
namespace tree {

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::HoeffdingSplit(const size_t dimensionality,
                  const size_t numClasses,
                  const data::DatasetInfo& datasetInfo,
                  const double successProbability,
                  const size_t maxSamples,
                  std::unordered_map<size_t, std::pair<size_t, size_t>>*
                      dimensionMappingsIn) :
    dimensionMappings((dimensionMappingsIn != NULL) ? dimensionMappingsIn :
        new std::unordered_map<size_t, std::pair<size_t, size_t>>()),
    ownsMappings(dimensionMappingsIn == NULL),
    numSamples(0),
    numClasses(numClasses),
    maxSamples(maxSamples),
    datasetInfo(datasetInfo),
    successProbability(successProbability),
    splitDimension(size_t(-1)),
    categoricalSplit(0),
    numericSplit()
{
  // Do we need to generate the mappings too?
  if (ownsMappings)
  {
    for (size_t i = 0; i < dimensionality; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
      {
        categoricalSplits.push_back(
            CategoricalSplitType(datasetInfo.NumMappings(i), numClasses));
        (*dimensionMappings)[i] = std::make_pair(data::Datatype::categorical,
            categoricalSplits.size() - 1);
      }
      else
      {
        numericSplits.push_back(NumericSplitType(numClasses));
        (*dimensionMappings)[i] = std::make_pair(data::Datatype::numeric,
            numericSplits.size() - 1);
      }
    }
  }
  else
  {
    for (size_t i = 0; i < dimensionality; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
      {
        categoricalSplits.push_back(
            CategoricalSplitType(datasetInfo.NumMappings(i), numClasses));
      }
      else
      {
        numericSplits.push_back(NumericSplitType(numClasses));
      }
    }
  }
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
HoeffdingSplit<FitnessFunction, NumericSplitType, CategoricalSplitType>::
    ~HoeffdingSplit()
{
  if (ownsMappings)
    delete dimensionMappings;
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
template<typename VecType>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Train(const VecType& point, const size_t label)
{
  if (splitDimension == size_t(-1))
  {
    ++numSamples;
    size_t numericIndex = 0;
    size_t categoricalIndex = 0;
    for (size_t i = 0; i < point.n_rows; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
        categoricalSplits[categoricalIndex++].Train(point[i], label);
      else if (datasetInfo.Type(i) == data::Datatype::numeric)
        numericSplits[numericIndex++].Train(point[i], label);
    }
  }
  else
  {
    // Already split.
    // But we should probably pass it down anyway.
  }
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::SplitCheck()
{
  // Do nothing if we've already split.
  if (splitDimension != size_t(-1))
    return 0;

  // Check the fitness of each dimension.  Then we'll use a Hoeffding bound
  // somehow.

  // Calculate epsilon, the value we need things to be greater than.
  const double rSquared = std::pow(FitnessFunction::Range(numClasses), 2.0);
  const double epsilon = std::sqrt(rSquared *
      std::log(1.0 / (1.0 - successProbability)) / (2 * numSamples));

  arma::vec gains(categoricalSplits.size() + numericSplits.size());
  for (size_t i = 0; i < gains.n_elem; ++i)
  {
    size_t type = dimensionMappings->at(i).first;
    size_t index = dimensionMappings->at(i).second;
    if (type == data::Datatype::categorical)
      gains[i] = categoricalSplits[index].EvaluateFitnessFunction();
    else if (type == data::Datatype::numeric)
      gains[i] = numericSplits[index].EvaluateFitnessFunction();
  }

  // Now find the largest and second-largest.
  double largest = -DBL_MAX;
  size_t largestIndex = 0;
  double secondLargest = -DBL_MAX;
  for (size_t i = 0; i < gains.n_elem; ++i)
  {
    if (gains[i] > largest)
    {
      secondLargest = largest;
      largest = gains[i];
      largestIndex = i;
    }
    else if (gains[i] > secondLargest)
    {
      secondLargest = gains[i];
    }
  }

  // Are these far enough apart to split?
  if (largest - secondLargest > epsilon || numSamples > maxSamples)
  {
    // Split!
    splitDimension = largestIndex;
    if (datasetInfo.Type(largestIndex) == data::Datatype::categorical)
    {
      // I don't know if this should be here.
      majorityClass = categoricalSplits[largestIndex].MajorityClass();
      return datasetInfo.NumMappings(largestIndex);
    }
    else
    {
      majorityClass = numericSplits[largestIndex].MajorityClass();
      return numericSplits[largestIndex].Bins();
    }
  }
  else
  {
    return 0; // Don't split.
  }
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::MajorityClass() const
{
  // If the node is not split yet, we have to grab the majority class from any
  // of the structures figuring out what to split on.
  if (splitDimension == size_t(-1))
  {
    // Grab majority class from splits.
    if (categoricalSplits.size() > 0)
      majorityClass = categoricalSplits[0].MajorityClass();
    else
      majorityClass = numericSplits[0].MajorityClass();
  }

  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
size_t& HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::MajorityClass()
{
  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename VecType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::CalculateDirection(const VecType& point) const
{
  // Don't call this before the node is split...
  if (datasetInfo.Type(splitDimension) == data::Datatype::numeric)
    return numericSplit.CalculateDirection(point[splitDimension]);
  else if (datasetInfo.Type(splitDimension) == data::Datatype::categorical)
    return categoricalSplit.CalculateDirection(point[splitDimension]);
  else
    return 0; // Not sure what to do here...
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename VecType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Classify(const VecType& /* point */) const
{
  // We're a leaf (or being considered a leaf), so classify based on what we
  // know.
  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename StreamingDecisionTreeType>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::CreateChildren(std::vector<StreamingDecisionTreeType>& children)
{
  // Create the children.
  arma::Col<size_t> childMajorities;
  if (dimensionMappings->at(splitDimension).first ==
      data::Datatype::categorical)
  {
    categoricalSplits[dimensionMappings->at(splitDimension).second].Split(
        childMajorities, categoricalSplit);
  }
  else if (dimensionMappings->at(splitDimension).first ==
           data::Datatype::numeric)
  {
    numericSplits[dimensionMappings->at(splitDimension).second].Split(
        childMajorities, numericSplit);
  }

  // We already know what the splitDimension will be.
  const size_t dimensionality = numericSplits.size() + categoricalSplits.size();
  for (size_t i = 0; i < childMajorities.n_elem; ++i)
  {
    children.push_back(StreamingDecisionTreeType(datasetInfo, dimensionality,
        numClasses, successProbability, numSamples, dimensionMappings));
    children[i].MajorityClass() = childMajorities[i];
  }

  // Eliminate now-unnecessary split information.
  numericSplits.clear();
  categoricalSplits.clear();
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename Archive>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Serialize(Archive& ar, const unsigned int /* version */)
{
  using data::CreateNVP;

  ar & CreateNVP(splitDimension, "splitDimension");
  ar & CreateNVP(dimensionMappings, "dimensionMappings");
  // What to do here about ownership...?
  if (Archive::is_loading::value)
    ownsMappings = true;

  // Depending on whether or not we have split yet, we may need to save
  // different things.
  if (splitDimension == size_t(-1))
  {
    // We have not yet split.  So we have to serialize the splits.
    ar & CreateNVP(numericSplits, "numericSplits");
    ar & CreateNVP(categoricalSplits, "categoricalSplits");

    ar & CreateNVP(numSamples, "numSamples");
    ar & CreateNVP(numClasses, "numClasses");
    ar & CreateNVP(maxSamples, "maxSamples");
    ar & CreateNVP(successProbability, "successProbability");

    if (Archive::is_loading::value)
    {
      // Clear things we don't need.
      majorityClass = 0;
      categoricalSplit = CategoricalSplitType::SplitInfo();
      numericSplit = NumericSplitType::SplitInfo();
    }
  }
  else
  {
    // We have split, so we only need to cache the numeric and categorical
    // split.
    ar & CreateNVP(categoricalSplit, "categoricalSplit");
    ar & CreateNVP(numericSplit, "numericSplit");
    ar & CreateNVP(majorityClass, "majorityClass");

    if (Archive::is_loading::value)
    {
      numericSplits.clear();
      categoricalSplits.clear();

      numSamples = 0;
      numClasses = 0;
      maxSamples = 0;
      successProbability = 0.0;
    }
  }
}

} // namespace tree
} // namespace mlpack

#endif
<commit_msg>Fix compilation issue.<commit_after>/**
 * @file hoeffding_split_impl.hpp
 * @author Ryan Curtin
 *
 * Implementation of the HoeffdingSplit class.
 */
#ifndef __MLPACK_METHODS_HOEFFDING_TREES_HOEFFDING_SPLIT_IMPL_HPP
#define __MLPACK_METHODS_HOEFFDING_TREES_HOEFFDING_SPLIT_IMPL_HPP

namespace mlpack {
namespace tree {

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::HoeffdingSplit(const size_t dimensionality,
                  const size_t numClasses,
                  const data::DatasetInfo& datasetInfo,
                  const double successProbability,
                  const size_t maxSamples,
                  std::unordered_map<size_t, std::pair<size_t, size_t>>*
                      dimensionMappingsIn) :
    dimensionMappings((dimensionMappingsIn != NULL) ? dimensionMappingsIn :
        new std::unordered_map<size_t, std::pair<size_t, size_t>>()),
    ownsMappings(dimensionMappingsIn == NULL),
    numSamples(0),
    numClasses(numClasses),
    maxSamples(maxSamples),
    datasetInfo(datasetInfo),
    successProbability(successProbability),
    splitDimension(size_t(-1)),
    categoricalSplit(0),
    numericSplit()
{
  // Do we need to generate the mappings too?
  if (ownsMappings)
  {
    for (size_t i = 0; i < dimensionality; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
      {
        categoricalSplits.push_back(
            CategoricalSplitType(datasetInfo.NumMappings(i), numClasses));
        (*dimensionMappings)[i] = std::make_pair(data::Datatype::categorical,
            categoricalSplits.size() - 1);
      }
      else
      {
        numericSplits.push_back(NumericSplitType(numClasses));
        (*dimensionMappings)[i] = std::make_pair(data::Datatype::numeric,
            numericSplits.size() - 1);
      }
    }
  }
  else
  {
    for (size_t i = 0; i < dimensionality; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
      {
        categoricalSplits.push_back(
            CategoricalSplitType(datasetInfo.NumMappings(i), numClasses));
      }
      else
      {
        numericSplits.push_back(NumericSplitType(numClasses));
      }
    }
  }
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
HoeffdingSplit<FitnessFunction, NumericSplitType, CategoricalSplitType>::
    ~HoeffdingSplit()
{
  if (ownsMappings)
    delete dimensionMappings;
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
template<typename VecType>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Train(const VecType& point, const size_t label)
{
  if (splitDimension == size_t(-1))
  {
    ++numSamples;
    size_t numericIndex = 0;
    size_t categoricalIndex = 0;
    for (size_t i = 0; i < point.n_rows; ++i)
    {
      if (datasetInfo.Type(i) == data::Datatype::categorical)
        categoricalSplits[categoricalIndex++].Train(point[i], label);
      else if (datasetInfo.Type(i) == data::Datatype::numeric)
        numericSplits[numericIndex++].Train(point[i], label);
    }
  }
  else
  {
    // Already split.
    // But we should probably pass it down anyway.
  }
}

template<typename FitnessFunction,
         typename NumericSplitType,
         typename CategoricalSplitType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::SplitCheck()
{
  // Do nothing if we've already split.
  if (splitDimension != size_t(-1))
    return 0;

  // Check the fitness of each dimension.  Then we'll use a Hoeffding bound
  // somehow.

  // Calculate epsilon, the value we need things to be greater than.
  const double rSquared = std::pow(FitnessFunction::Range(numClasses), 2.0);
  const double epsilon = std::sqrt(rSquared *
      std::log(1.0 / (1.0 - successProbability)) / (2 * numSamples));

  arma::vec gains(categoricalSplits.size() + numericSplits.size());
  for (size_t i = 0; i < gains.n_elem; ++i)
  {
    size_t type = dimensionMappings->at(i).first;
    size_t index = dimensionMappings->at(i).second;
    if (type == data::Datatype::categorical)
      gains[i] = categoricalSplits[index].EvaluateFitnessFunction();
    else if (type == data::Datatype::numeric)
      gains[i] = numericSplits[index].EvaluateFitnessFunction();
  }

  // Now find the largest and second-largest.
  double largest = -DBL_MAX;
  size_t largestIndex = 0;
  double secondLargest = -DBL_MAX;
  for (size_t i = 0; i < gains.n_elem; ++i)
  {
    if (gains[i] > largest)
    {
      secondLargest = largest;
      largest = gains[i];
      largestIndex = i;
    }
    else if (gains[i] > secondLargest)
    {
      secondLargest = gains[i];
    }
  }

  // Are these far enough apart to split?
  if (largest - secondLargest > epsilon || numSamples > maxSamples)
  {
    // Split!
    splitDimension = largestIndex;
    if (datasetInfo.Type(largestIndex) == data::Datatype::categorical)
    {
      // I don't know if this should be here.
      majorityClass = categoricalSplits[largestIndex].MajorityClass();
      return datasetInfo.NumMappings(largestIndex);
    }
    else
    {
      majorityClass = numericSplits[largestIndex].MajorityClass();
      return numericSplits[largestIndex].Bins();
    }
  }
  else
  {
    return 0; // Don't split.
  }
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::MajorityClass() const
{
  // If the node is not split yet, we have to grab the majority class from any
  // of the structures figuring out what to split on.
  if (splitDimension == size_t(-1))
  {
    // Grab majority class from splits.
    if (categoricalSplits.size() > 0)
      majorityClass = categoricalSplits[0].MajorityClass();
    else
      majorityClass = numericSplits[0].MajorityClass();
  }

  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
size_t& HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::MajorityClass()
{
  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename VecType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::CalculateDirection(const VecType& point) const
{
  // Don't call this before the node is split...
  if (datasetInfo.Type(splitDimension) == data::Datatype::numeric)
    return numericSplit.CalculateDirection(point[splitDimension]);
  else if (datasetInfo.Type(splitDimension) == data::Datatype::categorical)
    return categoricalSplit.CalculateDirection(point[splitDimension]);
  else
    return 0; // Not sure what to do here...
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename VecType>
size_t HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Classify(const VecType& /* point */) const
{
  // We're a leaf (or being considered a leaf), so classify based on what we
  // know.
  return majorityClass;
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename StreamingDecisionTreeType>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::CreateChildren(std::vector<StreamingDecisionTreeType>& children)
{
  // Create the children.
  arma::Col<size_t> childMajorities;
  if (dimensionMappings->at(splitDimension).first ==
      data::Datatype::categorical)
  {
    categoricalSplits[dimensionMappings->at(splitDimension).second].Split(
        childMajorities, categoricalSplit);
  }
  else if (dimensionMappings->at(splitDimension).first ==
           data::Datatype::numeric)
  {
    numericSplits[dimensionMappings->at(splitDimension).second].Split(
        childMajorities, numericSplit);
  }

  // We already know what the splitDimension will be.
  const size_t dimensionality = numericSplits.size() + categoricalSplits.size();
  for (size_t i = 0; i < childMajorities.n_elem; ++i)
  {
    children.push_back(StreamingDecisionTreeType(datasetInfo, dimensionality,
        numClasses, successProbability, numSamples, dimensionMappings));
    children[i].MajorityClass() = childMajorities[i];
  }

  // Eliminate now-unnecessary split information.
  numericSplits.clear();
  categoricalSplits.clear();
}

template<
    typename FitnessFunction,
    typename NumericSplitType,
    typename CategoricalSplitType
>
template<typename Archive>
void HoeffdingSplit<
    FitnessFunction,
    NumericSplitType,
    CategoricalSplitType
>::Serialize(Archive& ar, const unsigned int /* version */)
{
  using data::CreateNVP;

  ar & CreateNVP(splitDimension, "splitDimension");
  ar & CreateNVP(dimensionMappings, "dimensionMappings");
  // What to do here about ownership...?
  if (Archive::is_loading::value)
    ownsMappings = true;

  // Depending on whether or not we have split yet, we may need to save
  // different things.
  if (splitDimension == size_t(-1))
  {
    // We have not yet split.  So we have to serialize the splits.
    ar & CreateNVP(numericSplits, "numericSplits");
    ar & CreateNVP(categoricalSplits, "categoricalSplits");

    ar & CreateNVP(numSamples, "numSamples");
    ar & CreateNVP(numClasses, "numClasses");
    ar & CreateNVP(maxSamples, "maxSamples");
    ar & CreateNVP(successProbability, "successProbability");

    if (Archive::is_loading::value)
    {
      // Clear things we don't need.
      majorityClass = 0;
      categoricalSplit = typename CategoricalSplitType::SplitInfo(numClasses);
      numericSplit = typename NumericSplitType::SplitInfo();
    }
  }
  else
  {
    // We have split, so we only need to cache the numeric and categorical
    // split.
    ar & CreateNVP(categoricalSplit, "categoricalSplit");
    ar & CreateNVP(numericSplit, "numericSplit");
    ar & CreateNVP(majorityClass, "majorityClass");

    if (Archive::is_loading::value)
    {
      numericSplits.clear();
      categoricalSplits.clear();

      numSamples = 0;
      numClasses = 0;
      maxSamples = 0;
      successProbability = 0.0;
    }
  }
}

} // namespace tree
} // namespace mlpack

#endif
<|endoftext|>


<commit_before>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <ctime>

#include <ArgumentList.hpp>
#include <Observation.hpp>
#include <utils.hpp>
#include <SNR.hpp>

int main(int argc, char *argv[]) {
  std::string typeName;
	AstroData::Observation observation;
  PulsarSearch::snrDedispersedConf dConf;

	try {
    isa::utils::ArgumentList args(argc, argv);
    typeName = args.getSwitchArgument< std::string >("-type");
    observation.setPadding(args.getSwitchArgument< unsigned int >("-padding"));
    dConf.setNrSamplesPerBlock(args.getSwitchArgument< unsigned int >("-db"));
    observation.setNrSamplesPerSecond(args.getSwitchArgument< unsigned int >("-samples"));
		observation.setDMRange(args.getSwitchArgument< unsigned int >("-dms"), 0.0, 0.0);
	} catch  ( isa::utils::SwitchNotFound &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  } catch ( std::exception &err ) {
    std::cerr << "Usage: " << argv[0] << " -type ... -padding ... -db ... -dms ... -samples ..." << std::endl;
		return 1;
	}

  // Generate kernel
  std::string * code;
  code = PulsarSearch::getSNRDedispersedOpenCL(dConf, typeName, observation);
  std::cout << *code << std::endl;

	return 0;
}

<commit_msg>Format fix.<commit_after>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <ctime>

#include <ArgumentList.hpp>
#include <Observation.hpp>
#include <utils.hpp>
#include <SNR.hpp>

int main(int argc, char *argv[]) {
  std::string typeName;
	AstroData::Observation observation;
  PulsarSearch::snrDedispersedConf dConf;

	try {
    isa::utils::ArgumentList args(argc, argv);
    typeName = args.getSwitchArgument< std::string >("-type");
    observation.setPadding(args.getSwitchArgument< unsigned int >("-padding"));
    dConf.setNrSamplesPerBlock(args.getSwitchArgument< unsigned int >("-sb"));
    observation.setNrSamplesPerSecond(args.getSwitchArgument< unsigned int >("-samples"));
		observation.setDMRange(args.getSwitchArgument< unsigned int >("-dms"), 0.0, 0.0);
	} catch  ( isa::utils::SwitchNotFound &err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  } catch ( std::exception &err ) {
    std::cerr << "Usage: " << argv[0] << " -type ... -padding ... -sb ... -dms ... -samples ..." << std::endl;
		return 1;
	}

  // Generate kernel
  std::string * code;
  code = PulsarSearch::getSNRDedispersedOpenCL(dConf, typeName, observation);
  std::cout << *code << std::endl;

	return 0;
}

<|endoftext|>


<commit_before>#pragma once

struct padded_length {
  volatile u64 v_ __mpalign__;
  __padout__;
};

// Compile-time max number of workers
#define NWORKERS (NCPU-1)
struct uwq_ipcbuf {
  // Run-time max number of workers
  u64 maxworkers __mpalign__;
  padded_length len[NWORKERS]__mpalign__;
};

#if defined (XV6_KERNEL)
bool uwq_trywork(void);

class uwq;

class uwq_worker {
  uwq_worker(uwq*, proc*);
  long wait();
  void exit();

  uwq* uwq_;
  proc *proc_;
  bool running_;
  struct spinlock lock_;
  struct condvar cv_;

  NEW_DELETE_OPS(uwq_worker);
};

class uwq : public referenced, public rcu_freed {
  friend class uwq_worker;

  static uwq* alloc(proc_pgmap* pgmap, vmap* vmap,
                    filetable *ftable, uptr uentry);
  bool  haswork() const;
  bool  tryworker();

  virtual void do_gc(void) { delete this; }

protected:
  virtual void onzero() const;

private:
  uwq(proc_pgmap* pgmap, vmap* vmap,
      filetable* ftable, uwq_ipcbuf *ipc, uptr uentry);
  ~uwq();
  uwq& operator=(const uwq&);
  uwq(const uwq& x);
  proc* allocworker();
  void  finish();
  NEW_DELETE_OPS(uwq);

  struct spinlock lock_;
  proc_pgmap *pgmap_;
  vmap* vmap_;
  filetable* ftable_;
  uwq_ipcbuf* ipc_;
  uptr uentry_;
  std::atomic<uptr> ustack_;
  std::atomic<u64> uref_;

  std::atomic<uwq_worker*> worker_[NWORKERS];
};
#endif
<commit_msg>Fix class struct mismatch in uwq.hh<commit_after>#pragma once

struct padded_length {
  volatile u64 v_ __mpalign__;
  __padout__;
};

// Compile-time max number of workers
#define NWORKERS (NCPU-1)
struct uwq_ipcbuf {
  // Run-time max number of workers
  u64 maxworkers __mpalign__;
  padded_length len[NWORKERS]__mpalign__;
};

#if defined (XV6_KERNEL)
bool uwq_trywork(void);

class uwq;

class uwq_worker {
public:
  uwq_worker(uwq*, proc*);
  long wait();
  void exit();

  uwq* uwq_;
  proc *proc_;
  bool running_;
  struct spinlock lock_;
  struct condvar cv_;

  NEW_DELETE_OPS(uwq_worker);
};

class uwq : public referenced, public rcu_freed {
  friend class uwq_worker;

public:
  static uwq* alloc(proc_pgmap* pgmap, vmap* vmap,
                    filetable *ftable, uptr uentry);
  bool  haswork() const;
  bool  tryworker();

  virtual void do_gc(void) { delete this; }

protected:
  virtual void onzero() const;

private:
  uwq(proc_pgmap* pgmap, vmap* vmap,
      filetable* ftable, uwq_ipcbuf *ipc, uptr uentry);
  ~uwq();
  uwq& operator=(const uwq&);
  uwq(const uwq& x);
  proc* allocworker();
  void  finish();
  NEW_DELETE_OPS(uwq);

  struct spinlock lock_;
  proc_pgmap *pgmap_;
  vmap* vmap_;
  filetable* ftable_;
  uwq_ipcbuf* ipc_;
  uptr uentry_;
  std::atomic<uptr> ustack_;
  std::atomic<u64> uref_;

  std::atomic<uwq_worker*> worker_[NWORKERS];
};
#endif
<|endoftext|>


<commit_before>/**
 * @file
 * @brief Implementation of CorryvreckanWriter module
 * @copyright Copyright (c) 2017-2019 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "CorryvreckanWriterModule.hpp"

#include <Math/RotationZYX.h>

#include <fstream>
#include <string>
#include <utility>

#include "core/utils/file.h"
#include "core/utils/log.h"

using namespace allpix;

CorryvreckanWriterModule::CorryvreckanWriterModule(Configuration& config, Messenger* messenger, GeometryManager* geoManager)
    : Module(config), messenger_(messenger), geometryManager_(geoManager) {

    // Require PixelCharge messages for single detector
    messenger_->bindMulti(this, &CorryvreckanWriterModule::pixel_messages_, MsgFlags::REQUIRED);

    config_.setDefault("file_name", "corryvreckanOutput.root");
    config_.setDefault("geometry_file", "corryvreckanGeometry.conf");
    config_.setDefault("output_mctruth", true);
}

// Set up the output trees
void CorryvreckanWriterModule::init() {

    // Check if MC data to be saved
    output_mc_truth_ = config_.get<bool>("output_mctruth");

    reference_ = config_.get<std::string>("reference");
    if(!geometryManager_->hasDetector(reference_)) {
        throw InvalidValueError(config_, "reference", "detector not defined");
    }
    dut_ = config_.getArray<std::string>("dut", std::vector<std::string>());
    for(auto& dut : dut_) {
        if(!geometryManager_->hasDetector(dut)) {
            throw InvalidValueError(config_, "dut", "detector not defined");
        }
    }

    // Create output file and directories
    fileName_ = createOutputFile(allpix::add_file_extension(config_.get<std::string>("file_name"), "root"));
    output_file_ = std::make_unique<TFile>(fileName_.c_str(), "RECREATE");
    output_file_->cd();

    // Create geometry file:
    geometryFileName_ = createOutputFile(allpix::add_file_extension(config_.get<std::string>("geometry_file"), "conf"));

    // Create trees:
    event_tree_ = std::make_unique<TTree>("Event", (std::string("Tree of Events").c_str()));
    event_tree_->Bronch("global", "corryvreckan::Event", &event_);

    pixel_tree_ = std::make_unique<TTree>("Pixel", (std::string("Tree of Pixels").c_str()));

    if(output_mc_truth_) {
        mcparticle_tree_ = std::make_unique<TTree>("MCParticle", (std::string("Tree of MCParticles").c_str()));
    }

    // Initialise the time
    time_ = 0;
}

// Make instantiations of Corryvreckan pixels, and store these in the trees during run time
void CorryvreckanWriterModule::run(unsigned int event) {

    // Create and store a new Event:
    event_ = new corryvreckan::Event(time_, time_ + 5);
    LOG(DEBUG) << "Defining event for Corryvreckan: [" << Units::display(event_->start(), {"ns", "um"}) << ","
               << Units::display(event_->end(), {"ns", "um"}) << "]";
    event_tree_->Fill();

    // Loop through all receieved messages
    for(auto& message : pixel_messages_) {

        auto detector_name = message->getDetector()->getName();
        LOG(DEBUG) << "Receieved " << message->getData().size() << " pixel hits from detector " << detector_name;

        // Events start with 1, pre-filling only with empty events before:
        event--;

        if(write_list_px_.find(detector_name) == write_list_px_.end()) {
            write_list_px_[detector_name] = new std::vector<corryvreckan::Pixel*>();
            pixel_tree_->Bronch(detector_name.c_str(),
                                std::string("std::vector<corryvreckan::Pixel*>").c_str(),
                                &write_list_px_[detector_name]);

            LOG(DEBUG) << "Pre-filling new branch " << detector_name << " of corryvreckan::Pixel with " << event
                       << " empty events";
            auto* branch = pixel_tree_->GetBranch(detector_name.c_str());
            for(unsigned int i = 0; i < event; ++i) {
                branch->Fill();
            }
        }

        if(output_mc_truth_ && write_list_mcp_.find(detector_name) == write_list_mcp_.end()) {
            write_list_mcp_[detector_name] = new std::vector<corryvreckan::MCParticle*>();
            mcparticle_tree_->Bronch(detector_name.c_str(),
                                     std::string("std::vector<corryvreckan::MCParticle*>").c_str(),
                                     &write_list_mcp_[detector_name]);

            LOG(DEBUG) << "Pre-filling new branch " << detector_name << " of corryvreckan::MCParticle with " << event
                       << " empty events";
            auto* branch = mcparticle_tree_->GetBranch(detector_name.c_str());
            for(unsigned int i = 0; i < event; ++i) {
                branch->Fill();
            }
        }

        // Fill the branch vector
        for(auto& apx_pixel : message->getData()) {
            auto corry_pixel = new corryvreckan::Pixel(detector_name,
                                                       static_cast<int>(apx_pixel.getPixel().getIndex().X()),
                                                       static_cast<int>(apx_pixel.getPixel().getIndex().Y()),
                                                       static_cast<int>(apx_pixel.getSignal()),
                                                       apx_pixel.getSignal(),
                                                       event_->start());
            write_list_px_[detector_name]->push_back(corry_pixel);

            // If writing MC truth then also write out associated particle info
            if(!output_mc_truth_) {
                continue;
            }

            // Get all associated particles
            auto mcp = apx_pixel.getMCParticles();
            for(auto& particle : mcp) {
                auto mcParticle = new corryvreckan::MCParticle(detector_name,
                                                               particle->getParticleID(),
                                                               particle->getLocalStartPoint(),
                                                               particle->getLocalEndPoint(),
                                                               event_->start());
                write_list_mcp_[detector_name]->push_back(mcParticle);
            }
        }
    }

    LOG(TRACE) << "Writing new objects to tree";
    output_file_->cd();

    pixel_tree_->Fill();
    mcparticle_tree_->Fill();

    // Clear the current write lists
    for(auto& index_data : write_list_px_) {
        for(auto& pixel : (*index_data.second)) {
            delete pixel;
        }
        index_data.second->clear();
    }

    // Clear the current write lists
    for(auto& index_data : write_list_mcp_) {
        for(auto& mcp : (*index_data.second)) {
            delete mcp;
        }
        index_data.second->clear();
    }

    // Increment the time till the next event
    time_ += 10;
}

// Save the output trees to file
void CorryvreckanWriterModule::finalize() {

    // Finish writing to output file
    output_file_->Write();

    // Print statistics
    LOG(STATUS) << "Wrote output data to file:" << std::endl << fileName_;

    // Loop over all detectors and store the geometry:
    // Write geometry:
    std::ofstream geometry_file;
    if(!geometryFileName_.empty()) {
        geometry_file.open(geometryFileName_, std::ios_base::out | std::ios_base::trunc);
        if(!geometry_file.good()) {
            throw ModuleError("Cannot write to Corryvreckan geometry file");
        }

        geometry_file << "# Allpix Squared detector geometry - https://cern.ch/allpix-squared" << std::endl << std::endl;

        auto detectors = geometryManager_->getDetectors();
        for(auto& detector : detectors) {
            geometry_file << "[" << detector->getName() << "]" << std::endl;
            geometry_file << "position = " << Units::display(detector->getPosition().x(), {"mm", "um"}) << ", "
                          << Units::display(detector->getPosition().y(), {"mm", "um"}) << ", "
                          << Units::display(detector->getPosition().z(), {"mm", "um"}) << std::endl;

            // Transform the rotation matrix to a ZYX rotation and invert it to get a XYZ rotation
            // This way we stay compatible to old Corryvreckan versions which only support XYZ.
            geometry_file << "orientation_mode = \"xyz\"" << std::endl;
            ROOT::Math::RotationZYX rotations(detector->getOrientation().Inverse());
            geometry_file << "orientation = " << Units::display(-rotations.Psi(), "deg") << ", "
                          << Units::display(-rotations.Theta(), "deg") << ", " << Units::display(-rotations.Phi(), "deg")
                          << std::endl;

            auto model = detector->getModel();
            geometry_file << "type = \"" << model->getType() << "\"" << std::endl;
            geometry_file << "pixel_pitch = " << Units::display(model->getPixelSize().x(), "um") << ", "
                          << Units::display(model->getPixelSize().y(), "um") << std::endl;
            geometry_file << "number_of_pixels = " << model->getNPixels().x() << ", " << model->getNPixels().y()
                          << std::endl;

            std::string roles;
            if(detector->getName() == reference_) {
                roles += "reference";
            }
            if(std::find(dut_.begin(), dut_.end(), detector->getName()) != dut_.end()) {
                if(!roles.empty()) {
                    roles += ",";
                }
                roles += "dut";
            }
            geometry_file << "role = " << roles;
            geometry_file << std::endl;
        }
    }
}
<commit_msg>CorryvreckanWriter: write MCP tree only if created<commit_after>/**
 * @file
 * @brief Implementation of CorryvreckanWriter module
 * @copyright Copyright (c) 2017-2019 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "CorryvreckanWriterModule.hpp"

#include <Math/RotationZYX.h>

#include <fstream>
#include <string>
#include <utility>

#include "core/utils/file.h"
#include "core/utils/log.h"

using namespace allpix;

CorryvreckanWriterModule::CorryvreckanWriterModule(Configuration& config, Messenger* messenger, GeometryManager* geoManager)
    : Module(config), messenger_(messenger), geometryManager_(geoManager) {

    // Require PixelCharge messages for single detector
    messenger_->bindMulti(this, &CorryvreckanWriterModule::pixel_messages_, MsgFlags::REQUIRED);

    config_.setDefault("file_name", "corryvreckanOutput.root");
    config_.setDefault("geometry_file", "corryvreckanGeometry.conf");
    config_.setDefault("output_mctruth", true);
}

// Set up the output trees
void CorryvreckanWriterModule::init() {

    // Check if MC data to be saved
    output_mc_truth_ = config_.get<bool>("output_mctruth");

    reference_ = config_.get<std::string>("reference");
    if(!geometryManager_->hasDetector(reference_)) {
        throw InvalidValueError(config_, "reference", "detector not defined");
    }
    dut_ = config_.getArray<std::string>("dut", std::vector<std::string>());
    for(auto& dut : dut_) {
        if(!geometryManager_->hasDetector(dut)) {
            throw InvalidValueError(config_, "dut", "detector not defined");
        }
    }

    // Create output file and directories
    fileName_ = createOutputFile(allpix::add_file_extension(config_.get<std::string>("file_name"), "root"));
    output_file_ = std::make_unique<TFile>(fileName_.c_str(), "RECREATE");
    output_file_->cd();

    // Create geometry file:
    geometryFileName_ = createOutputFile(allpix::add_file_extension(config_.get<std::string>("geometry_file"), "conf"));

    // Create trees:
    event_tree_ = std::make_unique<TTree>("Event", (std::string("Tree of Events").c_str()));
    event_tree_->Bronch("global", "corryvreckan::Event", &event_);

    pixel_tree_ = std::make_unique<TTree>("Pixel", (std::string("Tree of Pixels").c_str()));

    if(output_mc_truth_) {
        mcparticle_tree_ = std::make_unique<TTree>("MCParticle", (std::string("Tree of MCParticles").c_str()));
    }

    // Initialise the time
    time_ = 0;
}

// Make instantiations of Corryvreckan pixels, and store these in the trees during run time
void CorryvreckanWriterModule::run(unsigned int event) {

    // Create and store a new Event:
    event_ = new corryvreckan::Event(time_, time_ + 5);
    LOG(DEBUG) << "Defining event for Corryvreckan: [" << Units::display(event_->start(), {"ns", "um"}) << ","
               << Units::display(event_->end(), {"ns", "um"}) << "]";
    event_tree_->Fill();

    // Loop through all receieved messages
    for(auto& message : pixel_messages_) {

        auto detector_name = message->getDetector()->getName();
        LOG(DEBUG) << "Receieved " << message->getData().size() << " pixel hits from detector " << detector_name;

        // Events start with 1, pre-filling only with empty events before:
        event--;

        if(write_list_px_.find(detector_name) == write_list_px_.end()) {
            write_list_px_[detector_name] = new std::vector<corryvreckan::Pixel*>();
            pixel_tree_->Bronch(detector_name.c_str(),
                                std::string("std::vector<corryvreckan::Pixel*>").c_str(),
                                &write_list_px_[detector_name]);

            LOG(DEBUG) << "Pre-filling new branch " << detector_name << " of corryvreckan::Pixel with " << event
                       << " empty events";
            auto* branch = pixel_tree_->GetBranch(detector_name.c_str());
            for(unsigned int i = 0; i < event; ++i) {
                branch->Fill();
            }
        }

        if(output_mc_truth_ && write_list_mcp_.find(detector_name) == write_list_mcp_.end()) {
            write_list_mcp_[detector_name] = new std::vector<corryvreckan::MCParticle*>();
            mcparticle_tree_->Bronch(detector_name.c_str(),
                                     std::string("std::vector<corryvreckan::MCParticle*>").c_str(),
                                     &write_list_mcp_[detector_name]);

            LOG(DEBUG) << "Pre-filling new branch " << detector_name << " of corryvreckan::MCParticle with " << event
                       << " empty events";
            auto* branch = mcparticle_tree_->GetBranch(detector_name.c_str());
            for(unsigned int i = 0; i < event; ++i) {
                branch->Fill();
            }
        }

        // Fill the branch vector
        for(auto& apx_pixel : message->getData()) {
            auto corry_pixel = new corryvreckan::Pixel(detector_name,
                                                       static_cast<int>(apx_pixel.getPixel().getIndex().X()),
                                                       static_cast<int>(apx_pixel.getPixel().getIndex().Y()),
                                                       static_cast<int>(apx_pixel.getSignal()),
                                                       apx_pixel.getSignal(),
                                                       event_->start());
            write_list_px_[detector_name]->push_back(corry_pixel);

            // If writing MC truth then also write out associated particle info
            if(!output_mc_truth_) {
                continue;
            }

            // Get all associated particles
            auto mcp = apx_pixel.getMCParticles();
            for(auto& particle : mcp) {
                auto mcParticle = new corryvreckan::MCParticle(detector_name,
                                                               particle->getParticleID(),
                                                               particle->getLocalStartPoint(),
                                                               particle->getLocalEndPoint(),
                                                               event_->start());
                write_list_mcp_[detector_name]->push_back(mcParticle);
            }
        }
    }

    LOG(TRACE) << "Writing new objects to tree";
    output_file_->cd();

    pixel_tree_->Fill();
    if(output_mc_truth_) {
        mcparticle_tree_->Fill();
    }

    // Clear the current write lists
    for(auto& index_data : write_list_px_) {
        for(auto& pixel : (*index_data.second)) {
            delete pixel;
        }
        index_data.second->clear();
    }

    // Clear the current write lists
    for(auto& index_data : write_list_mcp_) {
        for(auto& mcp : (*index_data.second)) {
            delete mcp;
        }
        index_data.second->clear();
    }

    // Increment the time till the next event
    time_ += 10;
}

// Save the output trees to file
void CorryvreckanWriterModule::finalize() {

    // Finish writing to output file
    output_file_->Write();

    // Print statistics
    LOG(STATUS) << "Wrote output data to file:" << std::endl << fileName_;

    // Loop over all detectors and store the geometry:
    // Write geometry:
    std::ofstream geometry_file;
    if(!geometryFileName_.empty()) {
        geometry_file.open(geometryFileName_, std::ios_base::out | std::ios_base::trunc);
        if(!geometry_file.good()) {
            throw ModuleError("Cannot write to Corryvreckan geometry file");
        }

        geometry_file << "# Allpix Squared detector geometry - https://cern.ch/allpix-squared" << std::endl << std::endl;

        auto detectors = geometryManager_->getDetectors();
        for(auto& detector : detectors) {
            geometry_file << "[" << detector->getName() << "]" << std::endl;
            geometry_file << "position = " << Units::display(detector->getPosition().x(), {"mm", "um"}) << ", "
                          << Units::display(detector->getPosition().y(), {"mm", "um"}) << ", "
                          << Units::display(detector->getPosition().z(), {"mm", "um"}) << std::endl;

            // Transform the rotation matrix to a ZYX rotation and invert it to get a XYZ rotation
            // This way we stay compatible to old Corryvreckan versions which only support XYZ.
            geometry_file << "orientation_mode = \"xyz\"" << std::endl;
            ROOT::Math::RotationZYX rotations(detector->getOrientation().Inverse());
            geometry_file << "orientation = " << Units::display(-rotations.Psi(), "deg") << ", "
                          << Units::display(-rotations.Theta(), "deg") << ", " << Units::display(-rotations.Phi(), "deg")
                          << std::endl;

            auto model = detector->getModel();
            geometry_file << "type = \"" << model->getType() << "\"" << std::endl;
            geometry_file << "pixel_pitch = " << Units::display(model->getPixelSize().x(), "um") << ", "
                          << Units::display(model->getPixelSize().y(), "um") << std::endl;
            geometry_file << "number_of_pixels = " << model->getNPixels().x() << ", " << model->getNPixels().y()
                          << std::endl;

            std::string roles;
            if(detector->getName() == reference_) {
                roles += "reference";
            }
            if(std::find(dut_.begin(), dut_.end(), detector->getName()) != dut_.end()) {
                if(!roles.empty()) {
                    roles += ",";
                }
                roles += "dut";
            }
            geometry_file << "role = " << roles;
            geometry_file << std::endl;
        }
    }
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <grpc/support/port_platform.h>

#include "src/core/lib/iomgr/port.h"

#ifdef GRPC_CFSTREAM
#import <CoreFoundation/CoreFoundation.h>
#import "src/core/lib/iomgr/cfstream_handle.h"

#include <grpc/support/atm.h>
#include <grpc/support/sync.h>

#include "src/core/lib/debug/trace.h"
#include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error_cfstream.h"
#include "src/core/lib/iomgr/exec_ctx.h"

extern grpc_core::TraceFlag grpc_tcp_trace;

void* CFStreamHandle::Retain(void* info) {
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(info);
  CFSTREAM_HANDLE_REF(handle, "retain");
  return info;
}

void CFStreamHandle::Release(void* info) {
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(info);
  CFSTREAM_HANDLE_UNREF(handle, "release");
}

CFStreamHandle* CFStreamHandle::CreateStreamHandle(
    CFReadStreamRef read_stream, CFWriteStreamRef write_stream) {
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(gpr_malloc(sizeof(CFStreamHandle)));
  return new (handle) CFStreamHandle(read_stream, write_stream);
}

void CFStreamHandle::ReadCallback(CFReadStreamRef stream,
                                  CFStreamEventType type,
                                  void* client_callback_info) {
  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
  grpc_core::ExecCtx exec_ctx;
  grpc_error* error;
  CFErrorRef stream_error;
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(client_callback_info);
  if (grpc_tcp_trace.enabled()) {
    gpr_log(GPR_DEBUG, "CFStream ReadCallback (%p, %p, %lu, %p)", handle,
            stream, type, client_callback_info);
  }
  switch (type) {
    case kCFStreamEventOpenCompleted:
      handle->open_event_.SetReady();
      break;
    case kCFStreamEventHasBytesAvailable:
    case kCFStreamEventEndEncountered:
      handle->read_event_.SetReady();
      break;
    case kCFStreamEventErrorOccurred:
      stream_error = CFReadStreamCopyError(stream);
      error = grpc_error_set_int(
          GRPC_ERROR_CREATE_FROM_CFERROR(stream_error, "read error"),
          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
      CFRelease(stream_error);
      handle->open_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->write_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->read_event_.SetShutdown(GRPC_ERROR_REF(error));
      GRPC_ERROR_UNREF(error);
      break;
    default:
      GPR_UNREACHABLE_CODE(return );
  }
}
void CFStreamHandle::WriteCallback(CFWriteStreamRef stream,
                                   CFStreamEventType type,
                                   void* clientCallBackInfo) {
  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
  grpc_core::ExecCtx exec_ctx;
  grpc_error* error;
  CFErrorRef stream_error;
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(clientCallBackInfo);
  if (grpc_tcp_trace.enabled()) {
    gpr_log(GPR_DEBUG, "CFStream WriteCallback (%p, %p, %lu, %p)", handle,
            stream, type, clientCallBackInfo);
  }
  switch (type) {
    case kCFStreamEventOpenCompleted:
      handle->open_event_.SetReady();
      break;
    case kCFStreamEventCanAcceptBytes:
    case kCFStreamEventEndEncountered:
      handle->write_event_.SetReady();
      break;
    case kCFStreamEventErrorOccurred:
      stream_error = CFWriteStreamCopyError(stream);
      error = grpc_error_set_int(
          GRPC_ERROR_CREATE_FROM_CFERROR(stream_error, "write error"),
          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
      CFRelease(stream_error);
      handle->open_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->write_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->read_event_.SetShutdown(GRPC_ERROR_REF(error));
      GRPC_ERROR_UNREF(error);
      break;
    default:
      GPR_UNREACHABLE_CODE(return );
  }
}

CFStreamHandle::CFStreamHandle(CFReadStreamRef read_stream,
                               CFWriteStreamRef write_stream) {
  gpr_ref_init(&refcount_, 1);
  open_event_.InitEvent();
  read_event_.InitEvent();
  write_event_.InitEvent();
  dispatch_queue_ = dispatch_queue_create(nullptr, DISPATCH_QUEUE_SERIAL);
  CFStreamClientContext ctx = {0, static_cast<void*>(this),
                               CFStreamHandle::Retain, CFStreamHandle::Release,
                               nil};
  CFReadStreamSetClient(
      read_stream,
      kCFStreamEventOpenCompleted | kCFStreamEventHasBytesAvailable |
          kCFStreamEventErrorOccurred | kCFStreamEventEndEncountered,
      CFStreamHandle::ReadCallback, &ctx);
  CFWriteStreamSetClient(
      write_stream,
      kCFStreamEventOpenCompleted | kCFStreamEventCanAcceptBytes |
          kCFStreamEventErrorOccurred | kCFStreamEventEndEncountered,
      CFStreamHandle::WriteCallback, &ctx);
  CFReadStreamSetDispatchQueue(read_stream, dispatch_queue_);
  CFWriteStreamSetDispatchQueue(write_stream, dispatch_queue_);
}

CFStreamHandle::~CFStreamHandle() {
  open_event_.DestroyEvent();
  read_event_.DestroyEvent();
  write_event_.DestroyEvent();
}

void CFStreamHandle::NotifyOnOpen(grpc_closure* closure) {
  open_event_.NotifyOn(closure);
}

void CFStreamHandle::NotifyOnRead(grpc_closure* closure) {
  read_event_.NotifyOn(closure);
}

void CFStreamHandle::NotifyOnWrite(grpc_closure* closure) {
  write_event_.NotifyOn(closure);
}

void CFStreamHandle::Shutdown(grpc_error* error) {
  open_event_.SetShutdown(GRPC_ERROR_REF(error));
  read_event_.SetShutdown(GRPC_ERROR_REF(error));
  write_event_.SetShutdown(GRPC_ERROR_REF(error));
  GRPC_ERROR_UNREF(error);
}

void CFStreamHandle::Ref(const char* file, int line, const char* reason) {
  if (grpc_tcp_trace.enabled()) {
    gpr_atm val = gpr_atm_no_barrier_load(&refcount_.count);
    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
            "CFStream Handle ref %p : %s %" PRIdPTR " -> %" PRIdPTR, this,
            reason, val, val + 1);
  }
  gpr_ref(&refcount_);
}

void CFStreamHandle::Unref(const char* file, int line, const char* reason) {
  if (grpc_tcp_trace.enabled()) {
    gpr_atm val = gpr_atm_no_barrier_load(&refcount_.count);
    gpr_log(GPR_ERROR,
            "CFStream Handle unref %p : %s %" PRIdPTR " -> %" PRIdPTR, this,
            reason, val, val - 1);
  }
  if (gpr_unref(&refcount_)) {
    this->~CFStreamHandle();
    gpr_free(this);
  }
}

#else

/* Creating a dummy function so that the grpc_cfstream library will be
 * non-empty.
 */
void CFStreamDummy() {}

#endif
<commit_msg>Use grpc_core::New and grpc_core::Delete<commit_after>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <grpc/support/port_platform.h>

#include "src/core/lib/iomgr/port.h"
#include "src/core/lib/gprpp/memory.h"

#ifdef GRPC_CFSTREAM
#import <CoreFoundation/CoreFoundation.h>
#import "src/core/lib/iomgr/cfstream_handle.h"

#include <grpc/support/atm.h>
#include <grpc/support/sync.h>

#include "src/core/lib/debug/trace.h"
#include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error_cfstream.h"
#include "src/core/lib/iomgr/exec_ctx.h"

extern grpc_core::TraceFlag grpc_tcp_trace;

void* CFStreamHandle::Retain(void* info) {
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(info);
  CFSTREAM_HANDLE_REF(handle, "retain");
  return info;
}

void CFStreamHandle::Release(void* info) {
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(info);
  CFSTREAM_HANDLE_UNREF(handle, "release");
}

CFStreamHandle* CFStreamHandle::CreateStreamHandle(
    CFReadStreamRef read_stream, CFWriteStreamRef write_stream) {
  return grpc_core::New<CFStreamHandle>(read_stream, write_stream);
}

void CFStreamHandle::ReadCallback(CFReadStreamRef stream,
                                  CFStreamEventType type,
                                  void* client_callback_info) {
  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
  grpc_core::ExecCtx exec_ctx;
  grpc_error* error;
  CFErrorRef stream_error;
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(client_callback_info);
  if (grpc_tcp_trace.enabled()) {
    gpr_log(GPR_DEBUG, "CFStream ReadCallback (%p, %p, %lu, %p)", handle,
            stream, type, client_callback_info);
  }
  switch (type) {
    case kCFStreamEventOpenCompleted:
      handle->open_event_.SetReady();
      break;
    case kCFStreamEventHasBytesAvailable:
    case kCFStreamEventEndEncountered:
      handle->read_event_.SetReady();
      break;
    case kCFStreamEventErrorOccurred:
      stream_error = CFReadStreamCopyError(stream);
      error = grpc_error_set_int(
          GRPC_ERROR_CREATE_FROM_CFERROR(stream_error, "read error"),
          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
      CFRelease(stream_error);
      handle->open_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->write_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->read_event_.SetShutdown(GRPC_ERROR_REF(error));
      GRPC_ERROR_UNREF(error);
      break;
    default:
      GPR_UNREACHABLE_CODE(return );
  }
}
void CFStreamHandle::WriteCallback(CFWriteStreamRef stream,
                                   CFStreamEventType type,
                                   void* clientCallBackInfo) {
  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
  grpc_core::ExecCtx exec_ctx;
  grpc_error* error;
  CFErrorRef stream_error;
  CFStreamHandle* handle = static_cast<CFStreamHandle*>(clientCallBackInfo);
  if (grpc_tcp_trace.enabled()) {
    gpr_log(GPR_DEBUG, "CFStream WriteCallback (%p, %p, %lu, %p)", handle,
            stream, type, clientCallBackInfo);
  }
  switch (type) {
    case kCFStreamEventOpenCompleted:
      handle->open_event_.SetReady();
      break;
    case kCFStreamEventCanAcceptBytes:
    case kCFStreamEventEndEncountered:
      handle->write_event_.SetReady();
      break;
    case kCFStreamEventErrorOccurred:
      stream_error = CFWriteStreamCopyError(stream);
      error = grpc_error_set_int(
          GRPC_ERROR_CREATE_FROM_CFERROR(stream_error, "write error"),
          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
      CFRelease(stream_error);
      handle->open_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->write_event_.SetShutdown(GRPC_ERROR_REF(error));
      handle->read_event_.SetShutdown(GRPC_ERROR_REF(error));
      GRPC_ERROR_UNREF(error);
      break;
    default:
      GPR_UNREACHABLE_CODE(return );
  }
}

CFStreamHandle::CFStreamHandle(CFReadStreamRef read_stream,
                               CFWriteStreamRef write_stream) {
  gpr_ref_init(&refcount_, 1);
  open_event_.InitEvent();
  read_event_.InitEvent();
  write_event_.InitEvent();
  dispatch_queue_ = dispatch_queue_create(nullptr, DISPATCH_QUEUE_SERIAL);
  CFStreamClientContext ctx = {0, static_cast<void*>(this),
                               CFStreamHandle::Retain, CFStreamHandle::Release,
                               nil};
  CFReadStreamSetClient(
      read_stream,
      kCFStreamEventOpenCompleted | kCFStreamEventHasBytesAvailable |
          kCFStreamEventErrorOccurred | kCFStreamEventEndEncountered,
      CFStreamHandle::ReadCallback, &ctx);
  CFWriteStreamSetClient(
      write_stream,
      kCFStreamEventOpenCompleted | kCFStreamEventCanAcceptBytes |
          kCFStreamEventErrorOccurred | kCFStreamEventEndEncountered,
      CFStreamHandle::WriteCallback, &ctx);
  CFReadStreamSetDispatchQueue(read_stream, dispatch_queue_);
  CFWriteStreamSetDispatchQueue(write_stream, dispatch_queue_);
}

CFStreamHandle::~CFStreamHandle() {
  open_event_.DestroyEvent();
  read_event_.DestroyEvent();
  write_event_.DestroyEvent();
}

void CFStreamHandle::NotifyOnOpen(grpc_closure* closure) {
  open_event_.NotifyOn(closure);
}

void CFStreamHandle::NotifyOnRead(grpc_closure* closure) {
  read_event_.NotifyOn(closure);
}

void CFStreamHandle::NotifyOnWrite(grpc_closure* closure) {
  write_event_.NotifyOn(closure);
}

void CFStreamHandle::Shutdown(grpc_error* error) {
  open_event_.SetShutdown(GRPC_ERROR_REF(error));
  read_event_.SetShutdown(GRPC_ERROR_REF(error));
  write_event_.SetShutdown(GRPC_ERROR_REF(error));
  GRPC_ERROR_UNREF(error);
}

void CFStreamHandle::Ref(const char* file, int line, const char* reason) {
  if (grpc_tcp_trace.enabled()) {
    gpr_atm val = gpr_atm_no_barrier_load(&refcount_.count);
    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
            "CFStream Handle ref %p : %s %" PRIdPTR " -> %" PRIdPTR, this,
            reason, val, val + 1);
  }
  gpr_ref(&refcount_);
}

void CFStreamHandle::Unref(const char* file, int line, const char* reason) {
  if (grpc_tcp_trace.enabled()) {
    gpr_atm val = gpr_atm_no_barrier_load(&refcount_.count);
    gpr_log(GPR_ERROR,
            "CFStream Handle unref %p : %s %" PRIdPTR " -> %" PRIdPTR, this,
            reason, val, val - 1);
  }
  if (gpr_unref(&refcount_)) {
    grpc_core::Delete<CFStreamHandle>(this);
  }
}

#else

/* Creating a dummy function so that the grpc_cfstream library will be
 * non-empty.
 */
void CFStreamDummy() {}

#endif
<|endoftext|>


<commit_before>/*
 *  This file is part of Poedit (https://poedit.net)
 *
 *  Copyright (C) 2017-2018 Vaclav Slavik
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a
 *  copy of this software and associated documentation files (the "Software"),
 *  to deal in the Software without restriction, including without limitation
 *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
 *  and/or sell copies of the Software, and to permit persons to whom the
 *  Software is furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 *  DEALINGS IN THE SOFTWARE.
 *
 */

#include "qa_checks.h"

#include <unicode/uchar.h>

#include <wx/translation.h>

// -------------------------------------------------------------
// QACheck implementations
// -------------------------------------------------------------

namespace QA
{

class NotAllPlurals : public QACheck
{
public:
    bool CheckItem(CatalogItemPtr item) override
    {
        if (!item->HasPlural())
            return false;

        bool foundTranslated = false;
        bool foundEmpty = false;
        for (auto& s: item->GetTranslations())
        {
            if (s.empty())
                foundEmpty = true;
            else
                foundTranslated = true;
        }

        if (foundEmpty && foundTranslated)
        {
            item->SetIssue(CatalogItem::Issue::Warning, _("Not all plural forms are translated."));
            return true;
        }

        return false;
    }
};

class CaseMismatch : public QACheck
{
public:
    CaseMismatch(Language lang) : m_lang(lang.Lang())
    {
    }

    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        if (u_isupper(source[0]) && u_islower(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _("The translation should start as a sentence."));
            return true;
        }

        if (u_islower(source[0]) && u_isupper(translation[0]))
        {
            if (m_lang != "de")
            {
                item->SetIssue(CatalogItem::Issue::Warning, _("The translation should start with a lowercase character."));
                return true;
            }
            // else: German nouns start uppercased, this would cause too many false positives
        }

        return false;
    }

private:
    std::string m_lang;
};

class WhitespaceMismatch : public QACheck
{
public:
    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        if (u_isspace(source[0]) && !u_isspace(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation doesn’t start with a space."));
            return true;
        }

        if (!u_isspace(source[0]) && u_isspace(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation starts with a space, but the source text doesn’t."));
            return true;
        }

        if (source.Last() == '\n' && translation.Last() != '\n')
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation is missing a newline at the end."));
            return true;
        }

        if (source.Last() != '\n' && translation.Last() == '\n')
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation ends with a newline, but the source text doesn’t."));
            return true;
        }

        if (u_isspace(source.Last()) && !u_isspace(translation.Last()))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation is missing a space at the end."));
            return true;
        }

        if (!u_isspace(source.Last()) && u_isspace(translation.Last()))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation ends with a space, but the source text doesn’t."));
            return true;
        }

        return false;
    }
};

class PunctuationMismatch : public QACheck
{
public:
    PunctuationMismatch(Language lang) : m_lang(lang.Lang())
    {
    }

    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        const UChar32 s_last = source.Last();
        const UChar32 t_last = translation.Last();
        const bool s_punct = u_ispunct(s_last);
        const bool t_punct = u_ispunct(t_last);

        if (u_getIntPropertyValue(s_last, UCHAR_BIDI_PAIRED_BRACKET_TYPE) == U_BPT_CLOSE ||
            u_getIntPropertyValue(t_last, UCHAR_BIDI_PAIRED_BRACKET_TYPE) == U_BPT_CLOSE)
        {
            // too many reordering related false positives for brackets
            // e.g. "your {site} account" -> "váš účet na {site}"
            if ((wchar_t)u_getBidiPairedBracket(s_last) != (wchar_t)source[0])
            {
                return false;
            }
            else
            {
                // OTOH, it's desirable to check strings fully enclosed in brackets like "(unsaved)"
                if (source.find_first_of((wchar_t)s_last, 1) != source.size() - 1)
                {
                    // it's more complicated, possibly something like "your {foo} on {bar}"
                    return false;
                }
            }
        }

        if (s_punct && !t_punct)
        {
            item->SetIssue(CatalogItem::Issue::Warning,
                           wxString::Format(_(L"The translation should end with “%s”."), wxString(wxUniChar(s_last))));
            return true;
        }
        else if (!s_punct && t_punct)
        {
            item->SetIssue(CatalogItem::Issue::Warning,
                           wxString::Format(_(L"The translation should not end with “%s”."), wxString(wxUniChar(t_last))));
            return true;
        }
        else if (s_punct && t_punct && s_last != t_last)
        {
            if (t_last == L'…' && source.EndsWith("..."))
            {
                // as a special case, allow translating ... (3 dots) as … (ellipsis)
            }
            else if (u_hasBinaryProperty(s_last, UCHAR_QUOTATION_MARK) || u_hasBinaryProperty(t_last, UCHAR_QUOTATION_MARK))
            {
                // quoted fragments can move around, e.g., so ignore quotes in reporting:
                //      >> Invalid value for ‘{fieldName}’ field
                //      >> Valor inválido para el campo ‘{fieldName}’
                // TODO: count quote characters to check if used correctly in translation; don't check position
                // (don't check for correct quotes for now, accept any quotations marks as equal)

            }
            else if (IsEquivalent(s_last, t_last))
            {
                // some characters are mostly equivalent and we shouldn't warn about them
            }
            else
            {
                item->SetIssue(CatalogItem::Issue::Warning,
                               wxString::Format(_(L"The translation ends with “%s”, but the source text ends with “%s”."),
                                                wxString(wxUniChar(t_last)), wxString(wxUniChar(s_last))));
                return true;
            }
        }

        return false;
    }

private:
    bool IsEquivalent(UChar32 src, UChar32 trans) const
    {
        if (src == trans)
            return true;

        if (m_lang == "zh" || m_lang == "ja")
        {
            // Chinese uses full-width punctuation.
            // See https://en.wikipedia.org/wiki/Chinese_punctuation
            switch (src)
            {
                case '.':
                    return trans == L'。';
                case ',':
                    return trans == L'、';
                case '!':
                    return trans == L'！';
                case '?':
                    return trans == L'？';
                case ':':
                    return trans == L'：';
                case '(':
                    return trans == L'（';
                case ')':
                    return trans == L'）';
                default:
                    break;
            }
        }
        else if (m_lang == "ar" || m_lang == "fa")
        {
            // In Arabic (but not other RTL languages), some punctuation is mirrored.
            switch (src)
            {
                case ';':
                    return trans == L'؛';
                case '?':
                    return trans == L'؟';
                case ',':
                    return trans == L'،';
                default:
                    break;
            }
        }
        else if (m_lang == "el")
        {
            // In Greek, questions end with ';' and not '?'.
            if (src == '?')
                return trans == ';';
        }

        return false;
    }

    std::string m_lang;
};

} // namespace QA

// -------------------------------------------------------------
// QACheck support code
// -------------------------------------------------------------

bool QACheck::CheckItem(CatalogItemPtr item)
{
    if (!item->GetTranslation().empty() && CheckString(item, item->GetString(), item->GetTranslation()))
        return true;

    if (item->HasPlural())
    {
        unsigned count = item->GetNumberOfTranslations();
        for (unsigned i = 1; i < count; i++)
        {
            auto t = item->GetTranslation(i);
            if (!t.empty() && CheckString(item, item->GetPluralString(), t))
                return true;
        }
    }

    return false;
}

bool QACheck::CheckString(CatalogItemPtr /*item*/, const wxString& /*source*/, const wxString& /*translation*/)
{
    wxFAIL_MSG("not implemented - must override CheckString OR CheckItem");
    return false;
}

// -------------------------------------------------------------
// QAChecker
// -------------------------------------------------------------

int QAChecker::Check(Catalog& catalog)
{
    // TODO: parallelize this (make async tasks for chunks of the catalog)
    //       doing it per-checker is MT-unsafe with API that calls SetIssue()!

    int issues = 0;

    for (auto& i: catalog.items())
        issues += Check(i);

    return issues;
}

int QAChecker::Check(CatalogItemPtr item)
{
    int issues = 0;

    for (auto& c: m_checks)
    {
        if (item->GetString().empty() || (item->HasPlural() && item->GetPluralString().empty()))
            continue;

        if (c->CheckItem(item))
            issues++;
    }

    return issues;
}

std::shared_ptr<QAChecker> QAChecker::GetFor(Catalog& catalog)
{
    auto lang = catalog.GetLanguage();
    auto c = std::make_shared<QAChecker>();
    c->AddCheck<QA::NotAllPlurals>();
    c->AddCheck<QA::CaseMismatch>(lang);
    c->AddCheck<QA::WhitespaceMismatch>();
    c->AddCheck<QA::PunctuationMismatch>(lang);
    return c;
}
<commit_msg>Fix logic error in quotes QA check<commit_after>/*
 *  This file is part of Poedit (https://poedit.net)
 *
 *  Copyright (C) 2017-2018 Vaclav Slavik
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a
 *  copy of this software and associated documentation files (the "Software"),
 *  to deal in the Software without restriction, including without limitation
 *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
 *  and/or sell copies of the Software, and to permit persons to whom the
 *  Software is furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 *  DEALINGS IN THE SOFTWARE.
 *
 */

#include "qa_checks.h"

#include <unicode/uchar.h>

#include <wx/translation.h>

// -------------------------------------------------------------
// QACheck implementations
// -------------------------------------------------------------

namespace QA
{

class NotAllPlurals : public QACheck
{
public:
    bool CheckItem(CatalogItemPtr item) override
    {
        if (!item->HasPlural())
            return false;

        bool foundTranslated = false;
        bool foundEmpty = false;
        for (auto& s: item->GetTranslations())
        {
            if (s.empty())
                foundEmpty = true;
            else
                foundTranslated = true;
        }

        if (foundEmpty && foundTranslated)
        {
            item->SetIssue(CatalogItem::Issue::Warning, _("Not all plural forms are translated."));
            return true;
        }

        return false;
    }
};

class CaseMismatch : public QACheck
{
public:
    CaseMismatch(Language lang) : m_lang(lang.Lang())
    {
    }

    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        if (u_isupper(source[0]) && u_islower(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _("The translation should start as a sentence."));
            return true;
        }

        if (u_islower(source[0]) && u_isupper(translation[0]))
        {
            if (m_lang != "de")
            {
                item->SetIssue(CatalogItem::Issue::Warning, _("The translation should start with a lowercase character."));
                return true;
            }
            // else: German nouns start uppercased, this would cause too many false positives
        }

        return false;
    }

private:
    std::string m_lang;
};

class WhitespaceMismatch : public QACheck
{
public:
    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        if (u_isspace(source[0]) && !u_isspace(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation doesn’t start with a space."));
            return true;
        }

        if (!u_isspace(source[0]) && u_isspace(translation[0]))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation starts with a space, but the source text doesn’t."));
            return true;
        }

        if (source.Last() == '\n' && translation.Last() != '\n')
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation is missing a newline at the end."));
            return true;
        }

        if (source.Last() != '\n' && translation.Last() == '\n')
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation ends with a newline, but the source text doesn’t."));
            return true;
        }

        if (u_isspace(source.Last()) && !u_isspace(translation.Last()))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation is missing a space at the end."));
            return true;
        }

        if (!u_isspace(source.Last()) && u_isspace(translation.Last()))
        {
            item->SetIssue(CatalogItem::Issue::Warning, _(L"The translation ends with a space, but the source text doesn’t."));
            return true;
        }

        return false;
    }
};

class PunctuationMismatch : public QACheck
{
public:
    PunctuationMismatch(Language lang) : m_lang(lang.Lang())
    {
    }

    bool CheckString(CatalogItemPtr item, const wxString& source, const wxString& translation) override
    {
        const UChar32 s_last = source.Last();
        const UChar32 t_last = translation.Last();
        const bool s_punct = u_ispunct(s_last);
        const bool t_punct = u_ispunct(t_last);

        if (u_getIntPropertyValue(s_last, UCHAR_BIDI_PAIRED_BRACKET_TYPE) == U_BPT_CLOSE ||
            u_getIntPropertyValue(t_last, UCHAR_BIDI_PAIRED_BRACKET_TYPE) == U_BPT_CLOSE)
        {
            // too many reordering related false positives for brackets
            // e.g. "your {site} account" -> "váš účet na {site}"
            if ((wchar_t)u_getBidiPairedBracket(s_last) != (wchar_t)source[0])
            {
                return false;
            }
            else
            {
                // OTOH, it's desirable to check strings fully enclosed in brackets like "(unsaved)"
                if (source.find_first_of((wchar_t)s_last, 1) != source.size() - 1)
                {
                    // it's more complicated, possibly something like "your {foo} on {bar}"
                    return false;
                }
            }
        }

        if (u_hasBinaryProperty(s_last, UCHAR_QUOTATION_MARK) || (!s_punct && u_hasBinaryProperty(t_last, UCHAR_QUOTATION_MARK)))
        {
            // quoted fragments can move around, e.g., so ignore quotes in reporting:
            //      >> Invalid value for ‘{fieldName}’ field
            //      >> Valor inválido para el campo ‘{fieldName}’
            // TODO: count quote characters to check if used correctly in translation; don't check position
            return false;
        }

        if (s_punct && !t_punct)
        {
            item->SetIssue(CatalogItem::Issue::Warning,
                           wxString::Format(_(L"The translation should end with “%s”."), wxString(wxUniChar(s_last))));
            return true;
        }
        else if (!s_punct && t_punct)
        {
            item->SetIssue(CatalogItem::Issue::Warning,
                           wxString::Format(_(L"The translation should not end with “%s”."), wxString(wxUniChar(t_last))));
            return true;
        }
        else if (s_punct && t_punct && s_last != t_last)
        {
            if (t_last == L'…' && source.EndsWith("..."))
            {
                // as a special case, allow translating ... (3 dots) as … (ellipsis)
            }
            else if (u_hasBinaryProperty(s_last, UCHAR_QUOTATION_MARK) && u_hasBinaryProperty(t_last, UCHAR_QUOTATION_MARK))
            {
                // don't check for correct quotes for now, accept any quotations marks as equal
            }
            else if (IsEquivalent(s_last, t_last))
            {
                // some characters are mostly equivalent and we shouldn't warn about them
            }
            else
            {
                item->SetIssue(CatalogItem::Issue::Warning,
                               wxString::Format(_(L"The translation ends with “%s”, but the source text ends with “%s”."),
                                                wxString(wxUniChar(t_last)), wxString(wxUniChar(s_last))));
                return true;
            }
        }

        return false;
    }

private:
    bool IsEquivalent(UChar32 src, UChar32 trans) const
    {
        if (src == trans)
            return true;

        if (m_lang == "zh" || m_lang == "ja")
        {
            // Chinese uses full-width punctuation.
            // See https://en.wikipedia.org/wiki/Chinese_punctuation
            switch (src)
            {
                case '.':
                    return trans == L'。';
                case ',':
                    return trans == L'、';
                case '!':
                    return trans == L'！';
                case '?':
                    return trans == L'？';
                case ':':
                    return trans == L'：';
                case '(':
                    return trans == L'（';
                case ')':
                    return trans == L'）';
                default:
                    break;
            }
        }
        else if (m_lang == "ar" || m_lang == "fa")
        {
            // In Arabic (but not other RTL languages), some punctuation is mirrored.
            switch (src)
            {
                case ';':
                    return trans == L'؛';
                case '?':
                    return trans == L'؟';
                case ',':
                    return trans == L'،';
                default:
                    break;
            }
        }
        else if (m_lang == "el")
        {
            // In Greek, questions end with ';' and not '?'.
            if (src == '?')
                return trans == ';';
        }

        return false;
    }

    std::string m_lang;
};

} // namespace QA

// -------------------------------------------------------------
// QACheck support code
// -------------------------------------------------------------

bool QACheck::CheckItem(CatalogItemPtr item)
{
    if (!item->GetTranslation().empty() && CheckString(item, item->GetString(), item->GetTranslation()))
        return true;

    if (item->HasPlural())
    {
        unsigned count = item->GetNumberOfTranslations();
        for (unsigned i = 1; i < count; i++)
        {
            auto t = item->GetTranslation(i);
            if (!t.empty() && CheckString(item, item->GetPluralString(), t))
                return true;
        }
    }

    return false;
}

bool QACheck::CheckString(CatalogItemPtr /*item*/, const wxString& /*source*/, const wxString& /*translation*/)
{
    wxFAIL_MSG("not implemented - must override CheckString OR CheckItem");
    return false;
}

// -------------------------------------------------------------
// QAChecker
// -------------------------------------------------------------

int QAChecker::Check(Catalog& catalog)
{
    // TODO: parallelize this (make async tasks for chunks of the catalog)
    //       doing it per-checker is MT-unsafe with API that calls SetIssue()!

    int issues = 0;

    for (auto& i: catalog.items())
        issues += Check(i);

    return issues;
}

int QAChecker::Check(CatalogItemPtr item)
{
    int issues = 0;

    for (auto& c: m_checks)
    {
        if (item->GetString().empty() || (item->HasPlural() && item->GetPluralString().empty()))
            continue;

        if (c->CheckItem(item))
            issues++;
    }

    return issues;
}

std::shared_ptr<QAChecker> QAChecker::GetFor(Catalog& catalog)
{
    auto lang = catalog.GetLanguage();
    auto c = std::make_shared<QAChecker>();
    c->AddCheck<QA::NotAllPlurals>();
    c->AddCheck<QA::CaseMismatch>(lang);
    c->AddCheck<QA::WhitespaceMismatch>();
    c->AddCheck<QA::PunctuationMismatch>(lang);
    return c;
}
<|endoftext|>


<commit_before>/*
 * ============================================================================
 *
 *       Filename:  qc-adaptor.cc
 *    Description:  Trim adaptor read-through from reads
 *        License:  LGPL-3+
 *         Author:  Kevin Murray, spam@kdmurray.id.au
 *
 * ============================================================================
 */

/* Copyright (c) 2015 Kevin Murray
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <yaml-cpp/yaml.h>

#include <seqan/modifier.h>
#include <seqan/align.h>

#include "qc-adaptor.hh"

namespace qcpp
{

AdaptorTrimPE::
AdaptorTrimPE(const std::string &name, int min_overlap):
    ReadProcessor(name),
    _num_pairs_trimmed(0),
    _num_pairs_joined(0),
    _min_overlap(min_overlap)
{
}

void
AdaptorTrimPE::
process_read_pair(ReadPair &the_read_pair)
{
    seqan::Align<std::string, seqan::ArrayGaps> aligner;
    std::string r2_rc = the_read_pair.second.sequence;
    int score = 0;

    seqan::reverseComplement(r2_rc);
    resize(rows(aligner), 2);
    assignSource(row(aligner, 0), the_read_pair.first.sequence);
    assignSource(row(aligner, 1), r2_rc);

    score = seqan::globalAlignment(aligner,
                           seqan::Score<int, seqan::Simple>(1, -3, -3, -3),
                           seqan::AlignConfig<true, true, true, true>());

    if (score >= _min_overlap) {
        size_t r1_len = the_read_pair.first.size();
        size_t r2_len = the_read_pair.second.size();
        ssize_t read_len_diff = r1_len - r2_len;
        size_t r1_start = toViewPosition(row(aligner, 0), 0);
        size_t r2_start = toViewPosition(row(aligner, 1), 0);
        std::string &r1_seq = the_read_pair.first.sequence;
        std::string &r2_seq = the_read_pair.second.sequence;
        std::string &r1_qual = the_read_pair.first.quality;
        std::string &r2_qual = the_read_pair.second.quality;

        // Complement R2, as we use it to correct R1 below or concatenation it
        // to R1 if it's the read needs merging
        seqan::complement(r2_seq);

        if (r1_start > r2_start) {
            // Adaptor read-through, trim R1, truncate R2.
            size_t new_seq_len = r1_len - read_len_diff - r1_start + 1;

            // If R1 base is lower quality than R2, use the base from R2
            for (size_t i = 0; i < new_seq_len; i++) {
                size_t r2_pos = r2_len + read_len_diff - r1_start - i;
                if (r1_qual[i] < r2_qual[r2_pos]) {
                    r1_seq[i] = r2_seq[r2_pos];
                    r1_qual[i] = r2_qual[r2_pos];
                }
            }

            // Trim the read to its new length
            r1_seq.erase(new_seq_len - 1);
            r1_qual.erase(new_seq_len - 1);

            // Remove R2, as it's just duplicated
            r2_seq.erase();
            r2_qual.erase();

            // Mark read as trimmed
            //the_read_pair.first.name += " AdaptorTrimPE_trimmed";
            //the_read_pair.second.name += " AdaptorTrimPE_trimmed";

            _num_pairs_trimmed++;
        } else {
            // Read-through into acutal read, needs merging
            size_t overlap_starts = r2_start - read_len_diff;
            size_t overlap_size = r1_len - r2_start;

            // If R1 base is lower quality than R2, use the base from R2
            for (size_t i = 0; i < overlap_size; i++) {
                size_t r1_pos = overlap_starts + i + read_len_diff;
                size_t r2_pos = r2_len + read_len_diff - i;
                if (r1_qual[r1_pos] < r2_qual[r2_pos]) {
                    r1_seq[r1_pos] = r2_seq[r2_pos];
                    r1_qual[r1_pos] = r2_qual[r2_pos];
                }
            }

            // Remove the overlap from the read
            r2_seq.erase(overlap_starts);
            r2_qual.erase(overlap_starts);

            // Reverse the second read, so we can append it directly to the
            // first read
            std::reverse(r2_seq.begin(), r2_seq.end());
            std::reverse(r2_qual.begin(), r2_qual.end());

            // Append to form one psuedo read
            r1_seq += r2_seq;
            r1_qual += r2_qual;

            // Remove the second read from the read pair, so it doesn't get
            // printed.
            r2_seq.erase();
            r2_qual.erase();

            // Mark the read as having been merged
            //the_read_pair.first.name += " AdaptorTrimPE_merged";
            //the_read_pair.second.name += " AdaptorTrimPE_merged";

            _num_pairs_joined++;
        }
    }

    _num_reads += 2;
}

std::string
AdaptorTrimPE::
report()
{
    std::ostringstream ss;
    YAML::Emitter yml;
    float percent_trimmed = (_num_pairs_trimmed * 2 / (float) _num_reads ) * 100;
    float percent_merged = (_num_pairs_joined * 2 / (float) _num_reads ) * 100;

    yml << YAML::BeginSeq;
    yml << YAML::BeginMap;
    yml << YAML::Key   << "AdaptorTrimPE"
        << YAML::Value
        << YAML::BeginMap
        << YAML::Key   << "name"
        << YAML::Value << _name
        << YAML::Key   << "parameters"
        << YAML::Value << YAML::BeginMap
                       << YAML::EndMap
        << YAML::Key   << "output"
        << YAML::Value << YAML::BeginMap
                       << YAML::Key << "num_reads"
                       << YAML::Value << _num_reads
                       << YAML::Key << "num_trimmed"
                       << YAML::Value << (_num_pairs_trimmed * 2)
                       << YAML::Key << "num_merged"
                       << YAML::Value << (_num_pairs_joined * 2)
                       << YAML::Key << "percent_trimmed"
                       << YAML::Value << percent_trimmed
                       << YAML::Key << "percent_merged"
                       << YAML::Value << percent_merged
                       << YAML::EndMap
        << YAML::EndMap;
    yml << YAML::EndMap;
    yml << YAML::EndSeq;
    ss << yml.c_str() << "\n";
    return ss.str();

}

} // namespace qcpp
<commit_msg>Fix rare bug in AdaptorTrimPE<commit_after>/*
 * ============================================================================
 *
 *       Filename:  qc-adaptor.cc
 *    Description:  Trim adaptor read-through from reads
 *        License:  LGPL-3+
 *         Author:  Kevin Murray, spam@kdmurray.id.au
 *
 * ============================================================================
 */

/* Copyright (c) 2015 Kevin Murray
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <yaml-cpp/yaml.h>

#include <seqan/modifier.h>
#include <seqan/align.h>

#include "qc-adaptor.hh"

namespace qcpp
{

AdaptorTrimPE::
AdaptorTrimPE(const std::string &name, int min_overlap):
    ReadProcessor(name),
    _num_pairs_trimmed(0),
    _num_pairs_joined(0),
    _min_overlap(min_overlap)
{
}

void
AdaptorTrimPE::
process_read_pair(ReadPair &the_read_pair)
{
    seqan::Align<std::string, seqan::ArrayGaps> aligner;
    std::string r2_rc = the_read_pair.second.sequence;
    int score = 0;

    seqan::reverseComplement(r2_rc);
    resize(rows(aligner), 2);
    assignSource(row(aligner, 0), the_read_pair.first.sequence);
    assignSource(row(aligner, 1), r2_rc);

    score = seqan::globalAlignment(aligner,
                           seqan::Score<int, seqan::Simple>(1, -3, -3, -3),
                           seqan::AlignConfig<true, true, true, true>());

    if (score >= _min_overlap) {
        size_t r1_len = the_read_pair.first.size();
        size_t r2_len = the_read_pair.second.size();
        ssize_t read_len_diff = r1_len - r2_len;
        size_t r1_start = toViewPosition(row(aligner, 0), 0);
        size_t r2_start = toViewPosition(row(aligner, 1), 0);
        std::string &r1_seq = the_read_pair.first.sequence;
        std::string &r2_seq = the_read_pair.second.sequence;
        std::string &r1_qual = the_read_pair.first.quality;
        std::string &r2_qual = the_read_pair.second.quality;

        // Complement R2, as we use it to correct R1 below or concatenation it
        // to R1 if it's the read needs merging.
        seqan::complement(r2_seq);

        if (r1_start > r2_start) {
            // Adaptor read-through, trim R1, remove R2.
            size_t new_len = r1_len - read_len_diff - r1_start;
            new_len = new_len > r1_seq.size() ?  r1_seq.size() : new_len;

            for (size_t i = 0; i < new_len; i++) {
                size_t r2_pos = r2_len + read_len_diff - r1_start - i;
                if (r1_qual[i] < r2_qual[r2_pos]) {
                    r1_seq[i] = r2_seq[r2_pos];
                    r1_qual[i] = r2_qual[r2_pos];
                }
            }

            // Trim the read to its new length
            r1_seq.erase(new_len);
            r1_qual.erase(new_len);

            // Remove R2, as it's just duplicated
            r2_seq.erase();
            r2_qual.erase();

            _num_pairs_trimmed++;
        } else {
            // Read-through into acutal read, needs merging
            size_t overlap_starts = r2_start - read_len_diff;
            size_t overlap_size = r1_len - r2_start;

            // If R1 base is lower quality than R2, use the base from R2
            for (size_t i = 0; i < overlap_size; i++) {
                size_t r1_pos = overlap_starts + i + read_len_diff;
                size_t r2_pos = r2_len + read_len_diff - i;
                if (r1_qual[r1_pos] < r2_qual[r2_pos]) {
                    r1_seq[r1_pos] = r2_seq[r2_pos];
                    r1_qual[r1_pos] = r2_qual[r2_pos];
                }
            }

            // Remove the overlap from the read
            r2_seq.erase(overlap_starts);
            r2_qual.erase(overlap_starts);

            // Reverse the second read, so we can append it directly to the
            // first read
            std::reverse(r2_seq.begin(), r2_seq.end());
            std::reverse(r2_qual.begin(), r2_qual.end());

            // Append to form one psuedo read
            r1_seq += r2_seq;
            r1_qual += r2_qual;

            // Remove the second read from the read pair, so it doesn't get
            // printed.
            r2_seq.erase();
            r2_qual.erase();

            _num_pairs_joined++;
        }
    }

    _num_reads += 2;
}

std::string
AdaptorTrimPE::
report()
{
    std::ostringstream ss;
    YAML::Emitter yml;
    float percent_trimmed = (_num_pairs_trimmed * 2 / (float) _num_reads ) * 100;
    float percent_merged = (_num_pairs_joined * 2 / (float) _num_reads ) * 100;

    yml << YAML::BeginSeq;
    yml << YAML::BeginMap;
    yml << YAML::Key   << "AdaptorTrimPE"
        << YAML::Value
        << YAML::BeginMap
        << YAML::Key   << "name"
        << YAML::Value << _name
        << YAML::Key   << "parameters"
        << YAML::Value << YAML::BeginMap
                       << YAML::EndMap
        << YAML::Key   << "output"
        << YAML::Value << YAML::BeginMap
                       << YAML::Key << "num_reads"
                       << YAML::Value << _num_reads
                       << YAML::Key << "num_trimmed"
                       << YAML::Value << (_num_pairs_trimmed * 2)
                       << YAML::Key << "num_merged"
                       << YAML::Value << (_num_pairs_joined * 2)
                       << YAML::Key << "percent_trimmed"
                       << YAML::Value << percent_trimmed
                       << YAML::Key << "percent_merged"
                       << YAML::Value << percent_merged
                       << YAML::EndMap
        << YAML::EndMap;
    yml << YAML::EndMap;
    yml << YAML::EndSeq;
    ss << yml.c_str() << "\n";
    return ss.str();

}

} // namespace qcpp
<|endoftext|>