<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "NeonDepthwiseConv.h"
#include "paddle/function/ConvOp.h"
#include "paddle/function/Im2Col.h"

namespace paddle {

namespace neon {

#if defined(__ARM_NEON__) || defined(__ARM_NEON)

template <DeviceType Device>
class NeonDepthwiseConvFunction : public ConvFunctionBase {
public:
  void init(const FuncConfig& config) override {
    ConvFunctionBase::init(config);
  }

  void check(const BufferArgs& inputs, const BufferArgs& outputs) override {
    const TensorShape& input = inputs[0].shape();
    const TensorShape& filter = inputs[1].shape();
    const TensorShape& output = outputs[0].shape();
    checkShape(input, filter, output);
  }

  void calc(const BufferArgs& inputs, const BufferArgs& outputs) override {
    CHECK_EQ(numInputs_, inputs.size());
    CHECK_EQ(numOutputs_, outputs.size());
    check(inputs, outputs);

    const TensorShape& input = inputs[0].shape();
    const TensorShape& filter = inputs[1].shape();
    const TensorShape& output = outputs[0].shape();

    size_t batchSize = input[0];
    size_t inputChannels = input[1];
    size_t inputHeight = input[2];
    size_t inputWidth = input[3];
    size_t filterHeight = getFilterHeight(filter);
    size_t filterWidth = getFilterWidth(filter);
    size_t outputChannels = output[1];
    size_t outputHeight = output[2];
    size_t outputWidth = output[3];
    size_t filterMultiplier = outputChannels / groups_;
    CHECK_EQ(inputChannels, groups_);

    // only support strideH() == strideW() and filterHeight == filterWidth.
    CHECK_EQ(strideH(), strideW());
    CHECK_EQ(filterHeight, filterWidth);

    float* inputData = inputs[0].data<float>();
    float* filterData = inputs[1].data<float>();
    float* outputData = outputs[0].data<float>();

    // padding the input
    float* inputPadding = inputData;
    if (paddingH() > 0 || paddingW() > 0) {
      int newSize = batchSize * inputChannels * (inputHeight + 2 * paddingH()) *
                    (inputWidth + 2 * paddingW());
      resizeBuffer<Device>(newSize);
      inputPadding = reinterpret_cast<float*>(memory_->getBuf());
      Padding<float>::run(inputData,
                          inputPadding,
                          batchSize * inputChannels,
                          inputHeight,
                          inputWidth,
                          paddingH(),
                          paddingW());

      // height and width of padding data
      inputHeight += 2 * paddingH();
      inputWidth += 2 * paddingW();
    }

    std::function<void(
        const float*, const float*, int, int, int, int, int, int, float*)>
        DepthWiseConv;

    if (filterWidth == 3 && strideW() == 1) {
      DepthWiseConv = DepthwiseConvKernel<3, 1>::run;
    } else if (filterWidth == 3 && strideW() == 2) {
      DepthWiseConv = DepthwiseConvKernel<3, 2>::run;
    } else if (filterWidth == 4 && strideW() == 1) {
      DepthWiseConv = DepthwiseConvKernel<4, 1>::run;
    } else if (filterWidth == 4 && strideW() == 2) {
      DepthWiseConv = DepthwiseConvKernel<4, 2>::run;
    } else {
      LOG(FATAL) << "Not supported";
    }

    for (size_t i = 0; i < batchSize; i++) {
      DepthWiseConv(inputPadding,
                    filterData,
                    inputHeight,
                    inputWidth,
                    outputChannels,
                    outputHeight,
                    outputWidth,
                    filterMultiplier,
                    outputData);
      inputPadding += inputChannels * inputHeight * inputWidth;
      outputData += outputChannels * outputHeight * outputWidth;
    }
  }
};

REGISTER_TYPED_FUNC(NeonDepthwiseConv, CPU, NeonDepthwiseConvFunction);

#endif

}  // namespace neon
}  // namespace paddle
<commit_msg>Refine NeonDepthwiseConv.<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "NeonDepthwiseConv.h"
#include "paddle/function/ConvOp.h"
#include "paddle/function/Im2Col.h"

namespace paddle {

#if defined(__ARM_NEON__) || defined(__ARM_NEON)

template <DeviceType Device>
class NeonDepthwiseConvFunction : public ConvFunctionBase {
public:
  void init(const FuncConfig& config) override {
    ConvFunctionBase::init(config);
  }

  void check(const BufferArgs& inputs, const BufferArgs& outputs) override {
    const TensorShape& input = inputs[0].shape();
    const TensorShape& filter = inputs[1].shape();
    const TensorShape& output = outputs[0].shape();
    checkShape(input, filter, output);
  }

  void calc(const BufferArgs& inputs, const BufferArgs& outputs) override {
    CHECK_EQ(numInputs_, inputs.size());
    CHECK_EQ(numOutputs_, outputs.size());
    check(inputs, outputs);

    const TensorShape& input = inputs[0].shape();
    const TensorShape& filter = inputs[1].shape();
    const TensorShape& output = outputs[0].shape();

    int batchSize = input[0];
    int inputChannels = input[1];
    int inputHeight = input[2];
    int inputWidth = input[3];
    int filterHeight = getFilterHeight(filter);
    int filterWidth = getFilterWidth(filter);
    int outputChannels = output[1];
    int outputHeight = output[2];
    int outputWidth = output[3];
    int filterMultiplier = outputChannels / groups_;
    CHECK_EQ(inputChannels, groups_);

    // only support strideH() == strideW() and filterHeight == filterWidth.
    CHECK_EQ(strideH(), strideW());
    CHECK_EQ(filterHeight, filterWidth);

    float* inputData = inputs[0].data<float>();
    float* filterData = inputs[1].data<float>();
    float* outputData = outputs[0].data<float>();

    // padding the input
    float* inputPadding = inputData;
    if (paddingH() > 0 || paddingW() > 0) {
      int newSize = batchSize * inputChannels * (inputHeight + 2 * paddingH()) *
                    (inputWidth + 2 * paddingW());
      resizeBuffer<Device>(newSize);
      inputPadding = reinterpret_cast<float*>(memory_->getBuf());
      Padding<float>::run(inputData,
                          inputPadding,
                          batchSize * inputChannels,
                          inputHeight,
                          inputWidth,
                          paddingH(),
                          paddingW());

      // height and width of padding data
      inputHeight += 2 * paddingH();
      inputWidth += 2 * paddingW();
    }

    std::function<void(
        const float*, const float*, int, int, int, int, int, int, float*)>
        DepthWiseConv;

    if (filterWidth == 3 && strideW() == 1) {
      DepthWiseConv = neon::DepthwiseConvKernel<3, 1>::run;
    } else if (filterWidth == 3 && strideW() == 2) {
      DepthWiseConv = neon::DepthwiseConvKernel<3, 2>::run;
    } else if (filterWidth == 4 && strideW() == 1) {
      DepthWiseConv = neon::DepthwiseConvKernel<4, 1>::run;
    } else if (filterWidth == 4 && strideW() == 2) {
      DepthWiseConv = neon::DepthwiseConvKernel<4, 2>::run;
    } else {
      LOG(FATAL) << "Not supported";
    }

    for (int i = 0; i < batchSize; i++) {
      DepthWiseConv(inputPadding,
                    filterData,
                    inputHeight,
                    inputWidth,
                    outputChannels,
                    outputHeight,
                    outputWidth,
                    filterMultiplier,
                    outputData);
      inputPadding += inputChannels * inputHeight * inputWidth;
      outputData += outputChannels * outputHeight * outputWidth;
    }
  }
};

#ifndef PADDLE_TYPE_DOUBLE
REGISTER_TYPED_FUNC(NeonDepthwiseConv, CPU, NeonDepthwiseConvFunction);
#endif

#endif

}  // namespace paddle
<|endoftext|>


<commit_before>/*
 *  Copyright 2019-2020 Diligent Graphics LLC
 *  Copyright 2015-2019 Egor Yusov
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  In no event and under no legal theory, whether in tort (including negligence),
 *  contract, or otherwise, unless required by applicable law (such as deliberate
 *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
 *  liable for any damages, including any direct, indirect, special, incidental,
 *  or consequential damages of any character arising as a result of this License or
 *  out of the use or inability to use the software (including but not limited to damages
 *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and
 *  all other commercial damages or losses), even if such Contributor has been advised
 *  of the possibility of such damages.
 */

#include "DefaultRawMemoryAllocator.hpp"
#include "FixedBlockMemoryAllocator.hpp"

#include "gtest/gtest.h"

using namespace Diligent;

namespace
{

TEST(Common_FixedBlockMemoryAllocator, AllocDealloc)
{
    constexpr Uint32 AllocSize             = 32;
    constexpr Uint32 NumAllocationsPerPage = 16;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    void* Allocations[NumAllocationsPerPage][2] = {};
    for (int p = 0; p < 2; ++p)
    {
        for (int a = 1; a < NumAllocationsPerPage; ++a)
        {
            for (int i = 0; i < a; ++i)
                Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);

            for (int i = a - 1; i >= 0; --i)
                TestAllocator.Free(Allocations[i][p]);

            for (int i = 0; i < a; ++i)
            {
                auto* NewAlloc = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);
                EXPECT_EQ(Allocations[i][p], NewAlloc);
            }

            for (int i = a - 1; i >= 0; --i)
                TestAllocator.Free(Allocations[i][p]);
        }
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);
    }

    for (int p = 0; p < 2; ++p)
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            TestAllocator.Free(Allocations[i][p]);

    for (int p = 0; p < 2; ++p)
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);

    for (int p = 0; p < 2; ++p)
        for (int s = 0; s < 5; ++s)
            for (int i = s; i < NumAllocationsPerPage; i += 5)
                TestAllocator.Free(Allocations[i][p]);
}

TEST(Common_FixedBlockMemoryAllocator, SmallObject)
{
    constexpr Uint32 AllocSize             = 4;
    constexpr Uint32 NumAllocationsPerPage = 1;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
    }

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        void* pRawMem1 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
        TestAllocator.Free(pRawMem1);
    }
}

TEST(Common_FixedBlockMemoryAllocator, UnalignedSize)
{
    constexpr Uint32 AllocSize             = 10;
    constexpr Uint32 NumAllocationsPerPage = 1;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
    }

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        void* pRawMem1 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
        TestAllocator.Free(pRawMem1);
    }
}

} // namespace
<commit_msg>Added test for LinearAllocator<commit_after>/*
 *  Copyright 2019-2020 Diligent Graphics LLC
 *  Copyright 2015-2019 Egor Yusov
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  In no event and under no legal theory, whether in tort (including negligence),
 *  contract, or otherwise, unless required by applicable law (such as deliberate
 *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
 *  liable for any damages, including any direct, indirect, special, incidental,
 *  or consequential damages of any character arising as a result of this License or
 *  out of the use or inability to use the software (including but not limited to damages
 *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and
 *  all other commercial damages or losses), even if such Contributor has been advised
 *  of the possibility of such damages.
 */

#include "DefaultRawMemoryAllocator.hpp"
#include "FixedBlockMemoryAllocator.hpp"
#include "LinearAllocator.hpp"

#include "gtest/gtest.h"

using namespace Diligent;

namespace
{

TEST(Common_FixedBlockMemoryAllocator, AllocDealloc)
{
    constexpr Uint32 AllocSize             = 32;
    constexpr Uint32 NumAllocationsPerPage = 16;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    void* Allocations[NumAllocationsPerPage][2] = {};
    for (int p = 0; p < 2; ++p)
    {
        for (int a = 1; a < NumAllocationsPerPage; ++a)
        {
            for (int i = 0; i < a; ++i)
                Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);

            for (int i = a - 1; i >= 0; --i)
                TestAllocator.Free(Allocations[i][p]);

            for (int i = 0; i < a; ++i)
            {
                auto* NewAlloc = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);
                EXPECT_EQ(Allocations[i][p], NewAlloc);
            }

            for (int i = a - 1; i >= 0; --i)
                TestAllocator.Free(Allocations[i][p]);
        }
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);
    }

    for (int p = 0; p < 2; ++p)
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            TestAllocator.Free(Allocations[i][p]);

    for (int p = 0; p < 2; ++p)
        for (int i = 0; i < NumAllocationsPerPage; ++i)
            Allocations[i][p] = TestAllocator.Allocate(AllocSize, "Fixed block allocator test", __FILE__, __LINE__);

    for (int p = 0; p < 2; ++p)
        for (int s = 0; s < 5; ++s)
            for (int i = s; i < NumAllocationsPerPage; i += 5)
                TestAllocator.Free(Allocations[i][p]);
}

TEST(Common_FixedBlockMemoryAllocator, SmallObject)
{
    constexpr Uint32 AllocSize             = 4;
    constexpr Uint32 NumAllocationsPerPage = 1;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
    }

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        void* pRawMem1 = TestAllocator.Allocate(AllocSize, "Small object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
        TestAllocator.Free(pRawMem1);
    }
}

TEST(Common_FixedBlockMemoryAllocator, UnalignedSize)
{
    constexpr Uint32 AllocSize             = 10;
    constexpr Uint32 NumAllocationsPerPage = 1;

    FixedBlockMemoryAllocator TestAllocator(DefaultRawMemoryAllocator::GetAllocator(), AllocSize, NumAllocationsPerPage);

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
    }

    {
        void* pRawMem0 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        void* pRawMem1 = TestAllocator.Allocate(AllocSize, "Unaligned-size object allocation test", __FILE__, __LINE__);
        TestAllocator.Free(pRawMem0);
        TestAllocator.Free(pRawMem1);
    }
}

TEST(Common_LinearAllocator, PointerAlignment)
{
    LinearAllocator Allocator{DefaultRawMemoryAllocator::GetAllocator()};

    const std::string SrcStr = "123456789";

    Allocator.AddRequiredSize<char>(SrcStr.length() + 1);
    Allocator.AddRequiredSize<uint32_t>(5);
    Allocator.AddRequiredSize<uint64_t>(3);
    Allocator.AddRequiredSize<float>(6);

    Allocator.Reserve();

    char* DstStr = Allocator.CopyString(SrcStr.c_str());
    size_t IntPtrUint32 = reinterpret_cast<size_t>(Allocator.Allocate<uint32_t>(5));
    size_t IntPtrUint64 = reinterpret_cast<size_t>(Allocator.Allocate<uint64_t>(3));
    size_t IntPtrFloat  = reinterpret_cast<size_t>(Allocator.Allocate<float>(6));

    EXPECT_EQ(SrcStr, DstStr);
    EXPECT_EQ(IntPtrUint32, Align(IntPtrUint32, alignof(uint32_t)));
    EXPECT_EQ(IntPtrUint64, Align(IntPtrUint64, alignof(uint64_t)));
    EXPECT_EQ(IntPtrFloat, Align(IntPtrFloat, alignof(float)));
}

} // namespace
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "SwitchOrderLayer.h"
#include "paddle/utils/Stat.h"

namespace paddle {

REGISTER_LAYER(switch_order, SwitchOrderLayer);

bool SwitchOrderLayer::init(const LayerMap& layerMap,
                            const ParameterMap& parameterMap) {
  /* Initialize the basic parent class */
  Layer::init(layerMap, parameterMap);
  auto& img_conf = config_.inputs(0).image_conf();
  size_t inH =
      img_conf.has_img_size_y() ? img_conf.img_size_y() : img_conf.img_size();
  size_t inW = img_conf.img_size();
  size_t inC = img_conf.channels();
  inDims_ = TensorShape({0, inC, inH, inW});
  outDims_ = TensorShape(4);

  auto& reshape_conf = config_.reshape_conf();
  for (size_t i = 0; i < reshape_conf.heightaxis_size(); i++) {
    LOG(INFO) << "reshape height axis: " << reshape_conf.heightaxis(i);
    heightAxis_.push_back(reshape_conf.heightaxis(i));
  }
  for (size_t i = 0; i < reshape_conf.widthaxis_size(); i++) {
    LOG(INFO) << "reshape width axis: " << reshape_conf.widthaxis(i);
    widthAxis_.push_back(reshape_conf.widthaxis(i));
  }
  createFunction(nchw2nhwc_, "NCHW2NHWC", FuncConfig());
  createFunction(nhwc2nchw_, "NHWC2NCHW", FuncConfig());
  return true;
}

void SwitchOrderLayer::setOutDims() {
  outDims_.setDim(0, inDims_[0]);
  outDims_.setDim(1, inDims_[2]);
  outDims_.setDim(2, inDims_[3]);
  outDims_.setDim(3, inDims_[1]);
  reshapeHeight_ = 1;
  for (size_t i = 0; i < heightAxis_.size(); i++) {
    reshapeHeight_ *= outDims_[heightAxis_[i]];
  }
  output_.setFrameHeight(reshapeHeight_);
  reshapeWidth_ = 1;
  for (size_t i = 0; i < widthAxis_.size(); i++) {
    reshapeWidth_ *= outDims_[widthAxis_[i]];
  }
  output_.setFrameWidth(reshapeWidth_);
  LOG(INFO) << "outDims: " << outDims_[0] << "; " << outDims_[1] << ";"
            << outDims_[2] << ";" << outDims_[3];
}

void SwitchOrderLayer::setInDims() {
  MatrixPtr input = inputLayers_[0]->getOutputValue();
  size_t batchSize = input->getHeight();
  inDims_.setDim(0, batchSize);

  int h = inputLayers_[0]->getOutput().getFrameHeight();
  if (h != 0) inDims_.setDim(2, h);
  int w = inputLayers_[0]->getOutput().getFrameWidth();
  if (w != 0) inDims_.setDim(3, w);
  int totalCount = input->getElementCnt();
  int channels = totalCount / (inDims_[0] * inDims_[2] * inDims_[3]);
  if (channels != 0) inDims_.setDim(1, channels);
  LOG(INFO) << "inDims: " << inDims_[0] << "; " << inDims_[1] << ";"
            << inDims_[2] << ";" << inDims_[3];
}

void SwitchOrderLayer::forward(PassType passType) {
  Layer::forward(passType);
  setInDims();
  setOutDims();
  resetOutput(outDims_[0], outDims_[1] * outDims_[2] * outDims_[3]);
  if (heightAxis_.size() > 0) {
    getOutputValue()->reshape(reshapeHeight_, reshapeWidth_);
  }

  // switch NCHW to NHWC
  BufferArgs inputs;
  BufferArgs outputs;
  inputs.addArg(*getInputValue(0), inDims_);
  outputs.addArg(*getOutputValue(), outDims_);
  nchw2nhwc_[0]->calc(inputs, outputs);
  // forwardActivation();
}

void SwitchOrderLayer::backward(const UpdateCallback& callback) {
  (void)callback;
  // backwardActivation();

  // switch NHWC to NCHW
  BufferArgs inputs;
  BufferArgs outputs;
  inputs.addArg(*getOutputGrad(), outDims_);
  outputs.addArg(*getInputGrad(0), inDims_, ADD_TO);
  nhwc2nchw_[0]->calc(inputs, outputs);
}
}  // namespace paddle
<commit_msg>Delete debug log<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "SwitchOrderLayer.h"
#include "paddle/utils/Stat.h"

namespace paddle {

REGISTER_LAYER(switch_order, SwitchOrderLayer);

bool SwitchOrderLayer::init(const LayerMap& layerMap,
                            const ParameterMap& parameterMap) {
  /* Initialize the basic parent class */
  Layer::init(layerMap, parameterMap);
  auto& img_conf = config_.inputs(0).image_conf();
  size_t inH =
      img_conf.has_img_size_y() ? img_conf.img_size_y() : img_conf.img_size();
  size_t inW = img_conf.img_size();
  size_t inC = img_conf.channels();
  inDims_ = TensorShape({0, inC, inH, inW});
  outDims_ = TensorShape(4);

  auto& reshape_conf = config_.reshape_conf();
  for (size_t i = 0; i < reshape_conf.heightaxis_size(); i++) {
    heightAxis_.push_back(reshape_conf.heightaxis(i));
  }
  for (size_t i = 0; i < reshape_conf.widthaxis_size(); i++) {
    widthAxis_.push_back(reshape_conf.widthaxis(i));
  }
  createFunction(nchw2nhwc_, "NCHW2NHWC", FuncConfig());
  createFunction(nhwc2nchw_, "NHWC2NCHW", FuncConfig());
  return true;
}

void SwitchOrderLayer::setOutDims() {
  outDims_.setDim(0, inDims_[0]);
  outDims_.setDim(1, inDims_[2]);
  outDims_.setDim(2, inDims_[3]);
  outDims_.setDim(3, inDims_[1]);
  reshapeHeight_ = 1;
  for (size_t i = 0; i < heightAxis_.size(); i++) {
    reshapeHeight_ *= outDims_[heightAxis_[i]];
  }
  output_.setFrameHeight(reshapeHeight_);
  reshapeWidth_ = 1;
  for (size_t i = 0; i < widthAxis_.size(); i++) {
    reshapeWidth_ *= outDims_[widthAxis_[i]];
  }
  output_.setFrameWidth(reshapeWidth_);
}

void SwitchOrderLayer::setInDims() {
  MatrixPtr input = inputLayers_[0]->getOutputValue();
  size_t batchSize = input->getHeight();
  inDims_.setDim(0, batchSize);

  int h = inputLayers_[0]->getOutput().getFrameHeight();
  if (h != 0) inDims_.setDim(2, h);
  int w = inputLayers_[0]->getOutput().getFrameWidth();
  if (w != 0) inDims_.setDim(3, w);
  int totalCount = input->getElementCnt();
  int channels = totalCount / (inDims_[0] * inDims_[2] * inDims_[3]);
  if (channels != 0) inDims_.setDim(1, channels);
}

void SwitchOrderLayer::forward(PassType passType) {
  Layer::forward(passType);
  setInDims();
  setOutDims();
  resetOutput(outDims_[0], outDims_[1] * outDims_[2] * outDims_[3]);
  if (heightAxis_.size() > 0) {
    getOutputValue()->reshape(reshapeHeight_, reshapeWidth_);
  }

  // switch NCHW to NHWC
  BufferArgs inputs;
  BufferArgs outputs;
  inputs.addArg(*getInputValue(0), inDims_);
  outputs.addArg(*getOutputValue(), outDims_);
  nchw2nhwc_[0]->calc(inputs, outputs);
  forwardActivation();
}

void SwitchOrderLayer::backward(const UpdateCallback& callback) {
  (void)callback;
  backwardActivation();

  // switch NHWC to NCHW
  BufferArgs inputs;
  BufferArgs outputs;
  inputs.addArg(*getOutputGrad(), outDims_);
  outputs.addArg(*getInputGrad(0), inDims_, ADD_TO);
  nhwc2nchw_[0]->calc(inputs, outputs);
}
}  // namespace paddle
<|endoftext|>


<commit_before>#include <cstdlib>
#include <atomic>
#include <thread>
#include <future>
#include <mutex>
#include <cassert>
#include <iostream>
#include <functional>
#include <type_traits>
#include <utility>

#include "../../common/macro.hpp"
#include "../../common/timer.hpp"

#include <libprecisegc/details/utils/scope_guard.hpp>

#ifdef BDW_GC
    #define GC_THREADS
    #include <gc/gc.h>
#endif

#ifdef PRECISE_GC
    #include <libprecisegc/libprecisegc.hpp>
    #include <libprecisegc/details/threads/managed_thread.hpp>
    using namespace precisegc;
#endif

static const int node_size    = 64;
static const int lists_count  = 512;
static const int lists_length = 512;

static std::mutex threads_mutex;
static std::condition_variable threads_cond;
static size_t threads_count = 0;

template <typename Function, typename... Args>
std::thread create_thread(Function&& f, Args&&... args)
{
    #ifdef PRECISE_GC
        return details::threads::managed_thread::create(std::forward<Function>(f), std::forward<Args>(args)...);
    #else
        return std::thread(std::forward<Function>(f), std::forward<Args>(args)...);
    #endif
};

template <typename F, typename... Args>
std::future<typename std::result_of<F(Args...)>::type> launch_task(F&& f, Args&&... args)
{
    typedef typename std::result_of<F(Args...)>::type result_type;

    std::function<result_type(Args...)> functor(f);
    std::packaged_task<result_type(bool, Args...)> task([functor] (bool parallel_flag, Args&&... task_args) {
        auto guard = precisegc::details::utils::make_scope_guard([parallel_flag, &threads_mutex, &threads_cond, &threads_count] {
            if (parallel_flag) {
                std::lock_guard<std::mutex> lock(threads_mutex);
                assert(threads_count != 0);
                --threads_count;
                threads_cond.notify_one();
            }
        });
        return functor(std::forward<Args>(task_args)...);
    });
    auto future = task.get_future();

    std::unique_lock<std::mutex> lock(threads_mutex);
    if (threads_count < std::thread::hardware_concurrency()) {
        ++threads_count;
        std::thread thread = create_thread(std::move(task), true, std::forward<Args>(args)...);
        thread.detach();
    } else {
        lock.unlock();
        task(false, std::forward<Args>(args)...);
    }

    return future;
};

void wait_for_tasks_complete()
{
    std::unique_lock<std::mutex> lock(threads_mutex);
    threads_cond.wait(lock, [&threads_count] { return threads_count == 0; });
}

struct Node
{
    int data;
    ptr_t(Node) next;
};

struct List
{
    ptr_t(Node) head;
    size_t length;
};

ptr_t(Node) advance(ptr_t(Node) node, size_t n)
{
    while (n > 0) {
        node = node->next;
        --n;
    }
    return node;
}

List merge(const List& fst, const List& snd)
{
    if (fst.length == 0 && snd.length == 0) {
        return {.head = null_ptr(Node), .length = 0};
    }

    ptr_t(Node) res;
    ptr_t(Node) it1 = fst.head;
    ptr_t(Node) it2 = snd.head;
    size_t l1 = fst.length;
    size_t l2 = snd.length;
    size_t length = l1 + l2;

    if (it1->data < it2->data) {
        res = it1;
        it1 = it1->next;
        l1--;
    } else {
        res = it2;
        it2 = it2->next;
        l2--;
    }

    ptr_t(Node) dst = res;
    while (l1 > 0 && l2 > 0) {
        if (it1->data < it2->data) {
            dst->next = it1;
            dst = dst->next;
            it1 = it1->next;
            --l1;
        } else {
            dst->next = it2;
            dst = dst->next;
            it2 = it2->next;
            --l2;
        }
    }

    if (l1 > 0) {
        dst->next = it1;
    }
    if (l2 > 0) {
        dst->next = it2;
    }

    return {.head = res, .length = length};
}

List merge_sort(const List& list)
{
    if (list.length == 0) {
        return {.head = null_ptr(Node), .length = 0};
    } else if (list.length == 1) {
        ptr_t(Node) node = new_(Node);
        node->data = list.head->data;
        set_null(node->next);
        return {.head = node, .length = 1};
    }

    size_t m = list.length / 2;
    ptr_t(Node) mid = advance(list.head, m);

    List lpart = {.head = list.head, .length = m};
    List rpart = {.head = mid,       .length = list.length - m};

    std::future<List> lsorted = launch_task(merge_sort, std::ref(lpart));
    List rsorted = merge_sort(rpart);

    return merge(lsorted.get(), rsorted);
}

List create_list(size_t n, int mod)
{
    if (n == 0) {
        return {.head = null_ptr(Node), .length = 0};
    }
    size_t length = n;
    ptr_t(Node) head = new_(Node);
    ptr_t(Node) it = head;
    head->data = rand() % mod;
    --n;
    while (n > 0) {
        it->next = new_(Node);
        it = it->next;
        it->data = rand() % mod;
        --n;
    }
    return {.head = head, .length = length};
}

void clear_list(List& list)
{
    #ifdef NO_GC
        ptr_t(Node) it = list.head;
        while (list.length > 0) {
            ptr_t(Node) next = it->next;
            delete_(it);
            it = next;
            --list.length;
        }
    #endif
    set_null(list.head);
    list.length = 0;
}

void routine()
{
    List list = create_list(lists_length, lists_length);
    List sorted = merge_sort(list);

    ptr_t(Node) it = sorted.head;
    assert(sorted.length == lists_length);
    for (size_t i = 0; i < sorted.length - 1; ++i) {
        assert(it->data <= it->next->data);
    }

    clear_list(list);
    clear_list(sorted);
}

int main(int argc, const char* argv[])
{
    bool incremental_flag = false;
    bool compacting_flag = false;
    for (int i = 1; i < argc; ++i) {
        auto arg = std::string(argv[i]);
        if (arg == "--incremental") {
            incremental_flag = true;
        } else if (arg == "--compacting") {
            compacting_flag = true;
        }
    }

    #if defined(PRECISE_GC)
        gc_options ops;
        ops.heapsize            = 2 * 1024 * 1024;      // 2 Mb
        ops.threads_available   = 1;
        ops.type                = incremental_flag ? gc_type::INCREMENTAL : gc_type::SERIAL;
        ops.init                = gc_init_strategy::SPACE_BASED;
        ops.compacting          = compacting_flag ? gc_compacting::ENABLED : gc_compacting::DISABLED;
        ops.loglevel            = gc_loglevel::SILENT;
        ops.print_stat          = false;
        gc_init(ops);
    #elif defined(BDW_GC)
        GC_INIT();
        if (incremental_flag) {
            GC_enable_incremental();
        }
    #endif

    std::cout << "Sorting " << lists_count << " lists with length " << lists_length << std::endl;
    std::cout << "Size of each list " << lists_length * sizeof(Node) << " b" << std::endl;
    std::cout << "Total memory usage " << 2 * lists_length * sizeof(Node) * lists_count << " b" << std::endl;

    timer tm;
    for (int i = 0; i < lists_count; ++i) {
        if ((i+1) % 32 == 0) {
            std::cout << (i+1) * 100 / lists_count << "%" << std::endl;
        }
        routine();
    }
    wait_for_tasks_complete();

    std::cout << "Completed in " << tm.elapsed<std::chrono::milliseconds>() << " ms" << std::endl;
    #if defined(BDW_GC)
        std::cout << "Completed " << GC_get_gc_no() << " collections" << std::endl;
        std::cout << "Heap size is " << GC_get_heap_size() << std::endl;
    #elif defined(PRECISE_GC)
        gc_stat stat = gc_stats();
        std::cout << "Completed " << stat.gc_count << " collections" << std::endl;
        std::cout << "Time spent in gc " << std::chrono::duration_cast<std::chrono::microseconds>(stat.gc_time).count() << " us" << std::endl;
        std::cout << "Average pause time " << std::chrono::duration_cast<std::chrono::microseconds>(stat.gc_time / stat.gc_count).count() << " us" << std::endl;
    #endif
}

<commit_msg>fix par_merge_sort benchmark for current gc root impl<commit_after>#include <cstdlib>
#include <atomic>
#include <thread>
#include <future>
#include <mutex>
#include <cassert>
#include <iostream>
#include <functional>
#include <type_traits>
#include <utility>

#include "../../common/macro.hpp"
#include "../../common/timer.hpp"

#include <libprecisegc/details/utils/scope_guard.hpp>

#ifdef BDW_GC
    #define GC_THREADS
    #include <gc/gc.h>
#endif

#ifdef PRECISE_GC
    #include <libprecisegc/libprecisegc.hpp>
    #include <libprecisegc/details/threads/managed_thread.hpp>
    using namespace precisegc;
#endif

static const int node_size    = 64;
static const int lists_count  = 512;
static const int lists_length = 512;

static std::mutex threads_mutex;
static std::condition_variable threads_cond;
static size_t threads_count = 0;

template <typename Function, typename... Args>
std::thread create_thread(Function&& f, Args&&... args)
{
    #ifdef PRECISE_GC
        return details::threads::managed_thread::create(std::forward<Function>(f), std::forward<Args>(args)...);
    #else
        return std::thread(std::forward<Function>(f), std::forward<Args>(args)...);
    #endif
};

template <typename F, typename... Args>
std::future<typename std::result_of<F(Args...)>::type> launch_task(F&& f, Args&&... args)
{
    using precisegc::details::utils::make_scope_guard;
    typedef typename std::result_of<F(Args...)>::type result_type;

    std::function<result_type(Args...)> functor(std::forward<F>(f));
    std::packaged_task<result_type(bool, Args...)> task([functor] (bool parallel_flag, Args&&... task_args) {
        auto guard = make_scope_guard([parallel_flag, &threads_mutex, &threads_cond, &threads_count] {
            if (parallel_flag) {
                std::lock_guard<std::mutex> lock(threads_mutex);
                assert(threads_count != 0);
                --threads_count;
                threads_cond.notify_one();
            }
        });
        return functor(std::forward<Args>(task_args)...);
    });
    auto future = task.get_future();

    std::unique_lock<std::mutex> lock(threads_mutex);
    if (threads_count < std::thread::hardware_concurrency()) {
        ++threads_count;
        std::thread thread = create_thread(std::move(task), true, std::forward<Args>(args)...);
        thread.detach();
    } else {
        lock.unlock();
        task(false, std::forward<Args>(args)...);
    }

    return future;
};

void wait_for_tasks_complete()
{
    std::unique_lock<std::mutex> lock(threads_mutex);
    threads_cond.wait(lock, [&threads_count] { return threads_count == 0; });
}

struct Node
{
    int data;
    ptr_t(Node) next;
};

struct List
{
    ptr_t(Node) head;
    size_t length;
};

ptr_t(Node) advance(ptr_t(Node) node, size_t n)
{
    while (n > 0) {
        node = node->next;
        --n;
    }
    return node;
}

List merge(const List& fst, const List& snd)
{
    if (fst.length == 0 && snd.length == 0) {
        return {.head = null_ptr(Node), .length = 0};
    }

    ptr_t(Node) res;
    ptr_t(Node) it1 = fst.head;
    ptr_t(Node) it2 = snd.head;
    size_t l1 = fst.length;
    size_t l2 = snd.length;
    size_t length = l1 + l2;

    if (it1->data < it2->data) {
        res = it1;
        it1 = it1->next;
        l1--;
    } else {
        res = it2;
        it2 = it2->next;
        l2--;
    }

    ptr_t(Node) dst = res;
    while (l1 > 0 && l2 > 0) {
        if (it1->data < it2->data) {
            dst->next = it1;
            dst = dst->next;
            it1 = it1->next;
            --l1;
        } else {
            dst->next = it2;
            dst = dst->next;
            it2 = it2->next;
            --l2;
        }
    }

    if (l1 > 0) {
        dst->next = it1;
    }
    if (l2 > 0) {
        dst->next = it2;
    }

    return {.head = res, .length = length};
}

void merge_sort_helper(const List& list, List& sorted);

List merge_sort(const List& list)
{
    if (list.length == 0) {
        return {.head = null_ptr(Node), .length = 0};
    } else if (list.length == 1) {
        ptr_t(Node) node = new_(Node);
        node->data = list.head->data;
        set_null(node->next);
        return {.head = node, .length = 1};
    }

    size_t m = list.length / 2;
    ptr_t(Node) mid = advance(list.head, m);

    List lpart = {.head = list.head, .length = m};
    List rpart = {.head = mid,       .length = list.length - m};

    List lsorted;
    std::future<void> future = launch_task(merge_sort_helper, std::ref(lpart), std::ref(lsorted));
    List rsorted = merge_sort(rpart);
    future.get();

    return merge(lsorted, rsorted);
}

void merge_sort_helper(const List& list, List& sorted)
{
    sorted = merge_sort(list);
}

List create_list(size_t n, int mod)
{
    if (n == 0) {
        return {.head = null_ptr(Node), .length = 0};
    }
    size_t length = n;
    ptr_t(Node) head = new_(Node);
    ptr_t(Node) it = head;
    head->data = rand() % mod;
    --n;
    while (n > 0) {
        it->next = new_(Node);
        it = it->next;
        it->data = rand() % mod;
        --n;
    }
    return {.head = head, .length = length};
}

void clear_list(List& list)
{
    #ifdef NO_GC
        ptr_t(Node) it = list.head;
        while (list.length > 0) {
            ptr_t(Node) next = it->next;
            delete_(it);
            it = next;
            --list.length;
        }
    #endif
    set_null(list.head);
    list.length = 0;
}

void routine()
{
    List list = create_list(lists_length, lists_length);
    List sorted = merge_sort(list);

    ptr_t(Node) it = sorted.head;
    assert(sorted.length == lists_length);
    for (size_t i = 0; i < sorted.length - 1; ++i) {
        assert(it->data <= it->next->data);
    }

    clear_list(list);
    clear_list(sorted);
}

int main(int argc, const char* argv[])
{
    bool incremental_flag = false;
    bool compacting_flag = false;
    for (int i = 1; i < argc; ++i) {
        auto arg = std::string(argv[i]);
        if (arg == "--incremental") {
            incremental_flag = true;
        } else if (arg == "--compacting") {
            compacting_flag = true;
        }
    }

    #if defined(PRECISE_GC)
        gc_options ops;
        ops.heapsize            = 2 * 1024 * 1024;      // 2 Mb
        ops.threads_available   = 1;
        ops.type                = incremental_flag ? gc_type::INCREMENTAL : gc_type::SERIAL;
        ops.init                = gc_init_strategy::SPACE_BASED;
        ops.compacting          = compacting_flag ? gc_compacting::ENABLED : gc_compacting::DISABLED;
        ops.loglevel            = gc_loglevel::SILENT;
        ops.print_stat          = false;
        gc_init(ops);
    #elif defined(BDW_GC)
        GC_INIT();
        if (incremental_flag) {
            GC_enable_incremental();
        }
    #endif

    std::cout << "Sorting " << lists_count << " lists with length " << lists_length << std::endl;
    std::cout << "Size of each list " << lists_length * sizeof(Node) << " b" << std::endl;
    std::cout << "Total memory usage " << 2 * lists_length * sizeof(Node) * lists_count << " b" << std::endl;

    timer tm;
    for (int i = 0; i < lists_count; ++i) {
        if ((i+1) % 32 == 0) {
            std::cout << (i+1) * 100 / lists_count << "%" << std::endl;
        }
        routine();
    }
    wait_for_tasks_complete();

    std::cout << "Completed in " << tm.elapsed<std::chrono::milliseconds>() << " ms" << std::endl;
    #if defined(BDW_GC)
        std::cout << "Completed " << GC_get_gc_no() << " collections" << std::endl;
        std::cout << "Heap size is " << GC_get_heap_size() << std::endl;
    #elif defined(PRECISE_GC)
        gc_stat stat = gc_stats();
        std::cout << "Completed " << stat.gc_count << " collections" << std::endl;
        std::cout << "Time spent in gc " << std::chrono::duration_cast<std::chrono::microseconds>(stat.gc_time).count() << " us" << std::endl;
        std::cout << "Average pause time " << std::chrono::duration_cast<std::chrono::microseconds>(stat.gc_time / stat.gc_count).count() << " us" << std::endl;
    #endif
}

<|endoftext|>


<commit_before>#include <sstream>
#include <iostream>
#include <set>

#include "contexts_corpus.hh"
#include "gzstream.hh"
#include "contexts_lexer.h"

#include <boost/tuple/tuple.hpp>

using namespace std;

//////////////////////////////////////////////////
// ContextsCorpus
//////////////////////////////////////////////////

void read_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) {
  assert(new_contexts.contexts.size() == new_contexts.counts.size());

  boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* extra_pair
    = static_cast< boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* >(extra);

  ContextsCorpus* corpus_ptr = extra_pair->get<0>();
  BackoffGenerator* backoff_gen = extra_pair->get<1>();
  map<string,int>* counts = extra_pair->get<2>();

  Document* doc(new Document());

  //cout << "READ: " << new_contexts.phrase << "\t";
  for (int i=0; i < new_contexts.contexts.size(); ++i) {
    int cache_word_count = corpus_ptr->m_dict.max();
    string context_str = corpus_ptr->m_dict.toString(new_contexts.contexts[i]);

    // filter out singleton contexts
    if (!counts->empty()) {
      map<string,int>::const_iterator find_it = counts->find(context_str);
      if (find_it == counts->end() || find_it->second == 1)
        continue;
    }

    WordID id = corpus_ptr->m_dict.Convert(context_str);
    if (cache_word_count != corpus_ptr->m_dict.max()) {
      corpus_ptr->m_backoff->terms_at_level(0)++;
      corpus_ptr->m_num_types++;
    }

    int count = new_contexts.counts[i];
    for (int j=0; j<count; ++j)
      doc->push_back(id);
    corpus_ptr->m_num_terms += count;

    // generate the backoff map
    if (backoff_gen) {
      int order = 1;
      WordID backoff_id = id;
      ContextsLexer::Context backedoff_context = new_contexts.contexts[i];
      while (true) {
        if (!corpus_ptr->m_backoff->has_backoff(backoff_id)) {
          //cerr << "Backing off from " << corpus_ptr->m_dict.Convert(backoff_id) << " to ";
          backedoff_context = (*backoff_gen)(backedoff_context);

          if (backedoff_context.empty()) {
            //cerr << "Nothing." << endl;
            (*corpus_ptr->m_backoff)[backoff_id] = -1;
            break;
          }

          if (++order > corpus_ptr->m_backoff->order())
            corpus_ptr->m_backoff->order(order);

          int cache_word_count = corpus_ptr->m_dict.max();
          int new_backoff_id = corpus_ptr->m_dict.Convert(backedoff_context);
          if (cache_word_count != corpus_ptr->m_dict.max())
            corpus_ptr->m_backoff->terms_at_level(order-1)++;

          //cerr << corpus_ptr->m_dict.Convert(new_backoff_id) << " ." << endl;

          backoff_id = ((*corpus_ptr->m_backoff)[backoff_id] = new_backoff_id);
        }
        else break;
      }
    }
    //cout << context_str << " (" << id << ") ||| C=" << count << " ||| ";
  }
  //cout << endl;

  if (!doc->empty()) {
    corpus_ptr->m_documents.push_back(doc);
    corpus_ptr->m_keys.push_back(new_contexts.phrase);
  }
}

void filter_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) {
  assert(new_contexts.contexts.size() == new_contexts.counts.size());

  map<string,int>* context_counts = (static_cast<map<string,int>*>(extra));

  for (int i=0; i < new_contexts.contexts.size(); ++i) {
    int count = new_contexts.counts[i];
    pair<map<string,int>::iterator,bool> result
      = context_counts->insert(make_pair(Dict::toString(new_contexts.contexts[i]),count));
    if (!result.second)
      result.first->second += count;
  }
}

unsigned ContextsCorpus::read_contexts(const string &filename,
                                       BackoffGenerator* backoff_gen_ptr,
                                       bool filter_singeltons) {
  map<string,int> counts;
  if (filter_singeltons) {
    cerr << "--- Filtering singleton contexts ---" << endl;
    igzstream in(filename.c_str());
    ContextsLexer::ReadContexts(&in, filter_callback, &counts);
  }

  m_num_terms = 0;
  m_num_types = 0;

  igzstream in(filename.c_str());
  boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* > extra_pair(this,backoff_gen_ptr,&counts);
  ContextsLexer::ReadContexts(&in, read_callback, &extra_pair);

  //m_num_types = m_dict.max();

  cerr << "Read backoff with order " << m_backoff->order() << "\n";
  for (int o=0; o<m_backoff->order(); o++)
    cerr << "  Terms at " << o << " = " << m_backoff->terms_at_level(o) << endl;
  cerr << endl;

  return m_documents.size();
}
<commit_msg>Added singleton filter.<commit_after>#include <sstream>
#include <iostream>
#include <set>

#include "contexts_corpus.hh"
#include "gzstream.hh"
#include "contexts_lexer.h"

#include <boost/tuple/tuple.hpp>

using namespace std;

//////////////////////////////////////////////////
// ContextsCorpus
//////////////////////////////////////////////////

void read_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) {
  assert(new_contexts.contexts.size() == new_contexts.counts.size());

  boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* extra_pair
    = static_cast< boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* >(extra);

  ContextsCorpus* corpus_ptr = extra_pair->get<0>();
  BackoffGenerator* backoff_gen = extra_pair->get<1>();
  map<string,int>* counts = extra_pair->get<2>();

  Document* doc(new Document());

  //cout << "READ: " << new_contexts.phrase << "\t";
  for (int i=0; i < new_contexts.contexts.size(); ++i) {
    int cache_word_count = corpus_ptr->m_dict.max();
    string context_str = corpus_ptr->m_dict.toString(new_contexts.contexts[i]);

    // filter out singleton contexts
    if (!counts->empty()) {
      map<string,int>::const_iterator find_it = counts->find(context_str);
      if (find_it == counts->end() || find_it->second < 5)
        continue;
    }

    WordID id = corpus_ptr->m_dict.Convert(context_str);
    if (cache_word_count != corpus_ptr->m_dict.max()) {
      corpus_ptr->m_backoff->terms_at_level(0)++;
      corpus_ptr->m_num_types++;
    }

    int count = new_contexts.counts[i];
    for (int j=0; j<count; ++j)
      doc->push_back(id);
    corpus_ptr->m_num_terms += count;

    // generate the backoff map
    if (backoff_gen) {
      int order = 1;
      WordID backoff_id = id;
      ContextsLexer::Context backedoff_context = new_contexts.contexts[i];
      while (true) {
        if (!corpus_ptr->m_backoff->has_backoff(backoff_id)) {
          //cerr << "Backing off from " << corpus_ptr->m_dict.Convert(backoff_id) << " to ";
          backedoff_context = (*backoff_gen)(backedoff_context);

          if (backedoff_context.empty()) {
            //cerr << "Nothing." << endl;
            (*corpus_ptr->m_backoff)[backoff_id] = -1;
            break;
          }

          if (++order > corpus_ptr->m_backoff->order())
            corpus_ptr->m_backoff->order(order);

          int cache_word_count = corpus_ptr->m_dict.max();
          int new_backoff_id = corpus_ptr->m_dict.Convert(backedoff_context);
          if (cache_word_count != corpus_ptr->m_dict.max())
            corpus_ptr->m_backoff->terms_at_level(order-1)++;

          //cerr << corpus_ptr->m_dict.Convert(new_backoff_id) << " ." << endl;

          backoff_id = ((*corpus_ptr->m_backoff)[backoff_id] = new_backoff_id);
        }
        else break;
      }
    }
    //cout << context_str << " (" << id << ") ||| C=" << count << " ||| ";
  }
  //cout << endl;

  if (!doc->empty()) {
    corpus_ptr->m_documents.push_back(doc);
    corpus_ptr->m_keys.push_back(new_contexts.phrase);
  }
}

void filter_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) {
  assert(new_contexts.contexts.size() == new_contexts.counts.size());

  map<string,int>* context_counts = (static_cast<map<string,int>*>(extra));

  for (int i=0; i < new_contexts.contexts.size(); ++i) {
    int count = new_contexts.counts[i];
    pair<map<string,int>::iterator,bool> result
      = context_counts->insert(make_pair(Dict::toString(new_contexts.contexts[i]),count));
    if (!result.second)
      result.first->second += count;
  }
}

unsigned ContextsCorpus::read_contexts(const string &filename,
                                       BackoffGenerator* backoff_gen_ptr,
                                       bool filter_singeltons) {
  map<string,int> counts;
  if (filter_singeltons) {
    cerr << "--- Filtering singleton contexts ---" << endl;
    igzstream in(filename.c_str());
    ContextsLexer::ReadContexts(&in, filter_callback, &counts);
  }

  m_num_terms = 0;
  m_num_types = 0;

  igzstream in(filename.c_str());
  boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* > extra_pair(this,backoff_gen_ptr,&counts);
  ContextsLexer::ReadContexts(&in, read_callback, &extra_pair);

  //m_num_types = m_dict.max();

  cerr << "Read backoff with order " << m_backoff->order() << "\n";
  for (int o=0; o<m_backoff->order(); o++)
    cerr << "  Terms at " << o << " = " << m_backoff->terms_at_level(o) << endl;
  cerr << endl;

  return m_documents.size();
}
<|endoftext|>


<commit_before>// Copyright 2012 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file contains code imported from the v8 project, at trunk/src/d8.cc, and
// then modified by Aaron Jacobs. The original copyright notice is as follows.

// Copyright 2012 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "gjstest/internal/cpp/typed_arrays.h"

#include "base/logging.h"

using v8::Arguments;
using v8::ExternalArrayType;
using v8::FunctionTemplate;
using v8::Handle;
using v8::HandleScope;
using v8::Int32;
using v8::Local;
using v8::Object;
using v8::ObjectTemplate;
using v8::Persistent;
using v8::ReadOnly;
using v8::String;
using v8::True;
using v8::TryCatch;
using v8::Value;
using v8::kExternalByteArray;
using v8::kExternalDoubleArray;
using v8::kExternalFloatArray;
using v8::kExternalIntArray;
using v8::kExternalShortArray;
using v8::kExternalUnsignedByteArray;
using v8::kExternalUnsignedIntArray;
using v8::kExternalUnsignedShortArray;

namespace gjstest {

static const char kArrayBufferReferencePropName[] = "_is_array_buffer_";
static const char kArrayBufferMarkerPropName[] = "_array_buffer_ref_";

static size_t ConvertToUint(
    const Handle<Value>& value_in,
    TryCatch* try_catch) {
  CHECK(!value_in.IsEmpty());

  if (value_in->IsUint32()) {
    return value_in->Uint32Value();
  }

  Local<Value> number = value_in->ToNumber();
  if (try_catch->HasCaught()) return 0;

  CHECK(number->IsNumber());
  Local<Int32> int32 = number->ToInt32();
  if (try_catch->HasCaught() || int32.IsEmpty()) return 0;

  int32_t raw_value = int32->Int32Value();
  if (try_catch->HasCaught()) return 0;

  if (raw_value < 0) {
    ThrowException(String::New("Array length must not be negative."));
    return 0;
  }

  static const int kMaxLength = 0x3fffffff;
  if (raw_value > static_cast<int32_t>(kMaxLength)) {
    ThrowException(
        String::New("Array length exceeds maximum length."));
  }

  return static_cast<size_t>(raw_value);
}

static void ExternalArrayWeakCallback(Persistent<Value> object, void* data) {
  HandleScope scope;
  Handle<String> prop_name = String::New(kArrayBufferReferencePropName);
  Handle<Object> converted_object = object->ToObject();
  Local<Value> prop_value = converted_object->Get(prop_name);
  if (data != NULL && !prop_value->IsObject()) {
    free(data);
  }
  object.Dispose();
}

// Create an external array from an underlying set of data. Takes ownership of
// the data.
static Handle<Object> CreateExternalArray(
    uint8_t* data,
    size_t num_elements,
    size_t element_size,
    ExternalArrayType element_type) {
  Handle<Object> result = Object::New();

  // Create a weak reference to the handle that will delete the underlying data
  // when it is disposed of.
  Persistent<Object> persistent = Persistent<Object>::New(result);
  persistent.MakeWeak(data, ExternalArrayWeakCallback);
  persistent.MarkIndependent();

  // Set the backing store for indexed elements.
  result->SetIndexedPropertiesToExternalArrayData(
      data,
      element_type,
      num_elements);

  // Set up the length and BYTES_PER_ELEMENT properties on the result.
  result->Set(
      String::New("length"),
      Int32::New(num_elements),
      ReadOnly);

  result->Set(
      String::New("BYTES_PER_ELEMENT"),
      Int32::New(element_size));

  return result;
}

static bool IsArrayBuffer(const Handle<Value>& val) {
  return val->IsObject() &&
         val->ToObject()->Get(String::New(kArrayBufferMarkerPropName))->
             IsTrue();
}

// Create a typed array from an existing array buffer. The array_buffer
// reference must be non-empty, but the other two arguments are optional.
static Handle<Value> CreateExternalArrayFromArrayBuffer(
    ExternalArrayType element_type,
    size_t element_size,
    const Handle<Object>& array_buffer,
    const Handle<Value>& byte_offset_arg,
    const Handle<Value>& length_arg) {
  CHECK(!array_buffer.IsEmpty());
  CHECK(IsArrayBuffer(array_buffer));

  TryCatch try_catch;

  // Figure out what the length of the existing array buffer is.
  const Local<Value> array_buffer_length_property =
      array_buffer->Get(String::New("length"));
  CHECK(!array_buffer_length_property.IsEmpty());

  const size_t array_buffer_length =
      ConvertToUint(array_buffer_length_property, &try_catch);

  if (try_catch.HasCaught()) {
    return try_catch.Exception();
  }

  // Figure out what the offset into the array buffer should be.
  size_t byte_offset = 0;
  if (!byte_offset_arg.IsEmpty()) {
    byte_offset = ConvertToUint(byte_offset_arg, &try_catch);
    if (try_catch.HasCaught()) {
      return try_catch.Exception();
    }
  }

  // Make sure the offset is legal.
  if (byte_offset % element_size) {
    return ThrowException(
        String::New("Offset must be a multiple of element size."));
  }

  if (byte_offset > array_buffer_length) {
    return ThrowException(
        String::New("Offset must be less than the array buffer length."));
  }

  // Figure out what the length of the resulting array should be (in elements,
  // not bytes)
  size_t length = 0;
  if (length_arg.IsEmpty()) {
    // If the length arg is omitted, the array spans from the byte offset to the
    // end of the array buffer. The size of the range spanned must be a multiple
    // of the element size.
    if ((array_buffer_length - byte_offset) % element_size) {
      return ThrowException(
          String::New(
              "Array buffer length minus the byte offset must be a "
                  "multiple of the element size"));
    }

    length = (array_buffer_length - byte_offset) / element_size;
  } else {
    length = ConvertToUint(length_arg, &try_catch);
    if (try_catch.HasCaught()) {
      return try_catch.Exception();
    }
  }

  // Make sure the length is legal.
  if (byte_offset + (length * element_size) > array_buffer_length) {
    return ThrowException(
        String::New(
            "length references an area beyond the end of the array buffer."));
  }

  // Grab the data property from the array buffer.
  uint8_t* const data =
      static_cast<uint8_t*>(
          array_buffer->GetIndexedPropertiesExternalArrayData());

  if (!data) {
    return ThrowException(String::New("ArrayBuffer doesn't have data."));
  }

  // Create the resulting object.
  const Handle<Object> result =
      CreateExternalArray(
          data + byte_offset,
          length,
          element_size,
          element_type);

  // Hold a reference to the ArrayBuffer so its buffer doesn't get collected.
  result->Set(
      String::New(kArrayBufferReferencePropName),
      array_buffer,
      ReadOnly);

  return result;
}

// Implement the constructor with this signature:
//
//     TypedArray(unsigned long length)
//
static Handle<Value> CreateExternalArrayWithLengthArg(
    ExternalArrayType element_type,
    size_t element_size,
    const Handle<Value>& length_arg) {
  CHECK(!length_arg.IsEmpty());
  CHECK(element_size);

  TryCatch try_catch;

  // Convert the length to a useful value.
  const size_t length = ConvertToUint(length_arg, &try_catch);
  if (try_catch.HasCaught()) {
    return try_catch.Exception();
  }

  // Create the underlying data buffer.
  uint8_t* const data =
      static_cast<uint8_t*>(
          calloc(length, element_size));

  const size_t offset = 0;

  if (!data) {
    return ThrowException(String::New("Memory allocation failed."));
  }

  return CreateExternalArray(data, length, element_size, element_type);
}

// Implement the ArrayBuffer constructor.
static Handle<Value> CreateArrayBuffer(
    const Handle<Value>& length_arg) {
  const ExternalArrayType element_type = v8::kExternalByteArray;
  const size_t element_size = 1;

  const Handle<Value> result =
      CreateExternalArrayWithLengthArg(
          element_type,
          element_size,
          length_arg);

  if (!result->IsObject()) {
    return result;
  }

  // Mark this as an array buffer, for use by other code in this file.
  result->ToObject()->Set(
      String::New(kArrayBufferMarkerPropName),
      True(),
      ReadOnly);

  return result;
}

// Common constructor code for all typed arrays. The following signatures are
// supported:
//
//     TypedArray(
//         ArrayBuffer buffer,
//         optional unsigned long byteOffset,
//         optional unsigned long length)
//
//     TypedArray(unsigned long length)
//
//     TypedArray(type[] array)
//
static Handle<Value> CreateExternalArray(
    const Arguments& args,
    ExternalArrayType element_type,
    size_t element_size) {
  const size_t num_args = args.Length();

  // Of the functions that defer to this one, the only one with element_size
  // equal to zero is the constructor for ArrayBuffer.
  if (element_size == 0) {
    if (num_args != 1) {
      return ThrowException(String::New("Expected exactly one argument."));
    }

    return CreateArrayBuffer(args[0]);
  }

  // We only support these element sizes.
  CHECK(
      element_size == 1 ||
      element_size == 2 ||
      element_size == 4 ||
      element_size == 8);

  // We require at least one arg.
  if (num_args == 0) {
    return ThrowException(String::New("Expected at least one argument."));
  }

  // Is this the constructor with the following signature?
  //
  //     TypedArray(
  //         ArrayBuffer buffer,
  //         optional unsigned long byteOffset,
  //         optional unsigned long length)
  //
  if (IsArrayBuffer(args[0])) {
    if (num_args > 3) {
      return ThrowException(
          String::New(
              "Array constructor from ArrayBuffer must have 1-3 parameters."));
    }

    return CreateExternalArrayFromArrayBuffer(
        element_type,
        element_size,
        args[0]->ToObject(),
        num_args > 1 ? args[1] : Handle<Value>(),
        num_args > 2 ? args[2] : Handle<Value>());
  }

  // Otherwise, there should be exactly one argument.
  if (num_args != 1) {
    return ThrowException(String::New("Expected exactly one argument."));
  }

  return CreateExternalArrayWithLengthArg(
      element_type,
      element_size,
      args[0]);
}

Handle<Value> ArrayBuffer(const Arguments& args) {
  return CreateExternalArray(args, kExternalByteArray, 0);
}

Handle<Value> Int8Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalByteArray, sizeof(int8_t));
}

Handle<Value> Int16Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalShortArray, sizeof(int16_t));
}

Handle<Value> Int32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalIntArray, sizeof(int32_t));
}

Handle<Value> Uint8Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedByteArray, sizeof(uint8_t));
}

Handle<Value> Uint16Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedShortArray, sizeof(uint16_t));
}

Handle<Value> Uint32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedIntArray, sizeof(uint32_t));
}

Handle<Value> Float32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalFloatArray, sizeof(float));
}

Handle<Value> Float64Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalDoubleArray, sizeof(double));
}

void ExportTypedArrays(
    const Handle<ObjectTemplate>& global_template) {
  global_template->Set(
      String::New("ArrayBuffer"),
      FunctionTemplate::New(ArrayBuffer));

  // Signed integers.
  global_template->Set(
      String::New("Int8Array"),
      FunctionTemplate::New(Int8Array));

  global_template->Set(
      String::New("Int16Array"),
      FunctionTemplate::New(Int16Array));

  global_template->Set(
      String::New("Int32Array"),
      FunctionTemplate::New(Int32Array));

  // Unigned integers.
  global_template->Set(
      String::New("Uint8Array"),
      FunctionTemplate::New(Uint8Array));

  global_template->Set(
      String::New("Uint16Array"),
      FunctionTemplate::New(Uint16Array));

  global_template->Set(
      String::New("Uint32Array"),
      FunctionTemplate::New(Uint32Array));

  // Floats.
  global_template->Set(
      String::New("Float32Array"),
      FunctionTemplate::New(Float32Array));

  global_template->Set(
      String::New("Float64Array"),
      FunctionTemplate::New(Float64Array));
}

}  // namespace gjstest
<commit_msg>Added a helper for the array constructor.<commit_after>// Copyright 2012 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file contains code imported from the v8 project, at trunk/src/d8.cc, and
// then modified by Aaron Jacobs. The original copyright notice is as follows.

// Copyright 2012 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "gjstest/internal/cpp/typed_arrays.h"

#include "base/logging.h"

using v8::Arguments;
using v8::Array;
using v8::ExternalArrayType;
using v8::FunctionTemplate;
using v8::Handle;
using v8::HandleScope;
using v8::Int32;
using v8::Local;
using v8::Object;
using v8::ObjectTemplate;
using v8::Persistent;
using v8::ReadOnly;
using v8::String;
using v8::True;
using v8::TryCatch;
using v8::Value;
using v8::kExternalByteArray;
using v8::kExternalDoubleArray;
using v8::kExternalFloatArray;
using v8::kExternalIntArray;
using v8::kExternalShortArray;
using v8::kExternalUnsignedByteArray;
using v8::kExternalUnsignedIntArray;
using v8::kExternalUnsignedShortArray;

namespace gjstest {

static const char kArrayBufferReferencePropName[] = "_is_array_buffer_";
static const char kArrayBufferMarkerPropName[] = "_array_buffer_ref_";

static size_t ConvertToUint(
    const Handle<Value>& value_in,
    TryCatch* try_catch) {
  CHECK(!value_in.IsEmpty());

  if (value_in->IsUint32()) {
    return value_in->Uint32Value();
  }

  Local<Value> number = value_in->ToNumber();
  if (try_catch->HasCaught()) return 0;

  CHECK(number->IsNumber());
  Local<Int32> int32 = number->ToInt32();
  if (try_catch->HasCaught() || int32.IsEmpty()) return 0;

  int32_t raw_value = int32->Int32Value();
  if (try_catch->HasCaught()) return 0;

  if (raw_value < 0) {
    ThrowException(String::New("Array length must not be negative."));
    return 0;
  }

  static const int kMaxLength = 0x3fffffff;
  if (raw_value > static_cast<int32_t>(kMaxLength)) {
    ThrowException(
        String::New("Array length exceeds maximum length."));
  }

  return static_cast<size_t>(raw_value);
}

static void ExternalArrayWeakCallback(Persistent<Value> object, void* data) {
  HandleScope scope;
  Handle<String> prop_name = String::New(kArrayBufferReferencePropName);
  Handle<Object> converted_object = object->ToObject();
  Local<Value> prop_value = converted_object->Get(prop_name);
  if (data != NULL && !prop_value->IsObject()) {
    free(data);
  }
  object.Dispose();
}

// Create an external array from an underlying set of data. Takes ownership of
// the data.
static Handle<Object> CreateExternalArray(
    uint8_t* data,
    size_t num_elements,
    size_t element_size,
    ExternalArrayType element_type) {
  Handle<Object> result = Object::New();

  // Create a weak reference to the handle that will delete the underlying data
  // when it is disposed of.
  Persistent<Object> persistent = Persistent<Object>::New(result);
  persistent.MakeWeak(data, ExternalArrayWeakCallback);
  persistent.MarkIndependent();

  // Set the backing store for indexed elements.
  result->SetIndexedPropertiesToExternalArrayData(
      data,
      element_type,
      num_elements);

  // Set up the length and BYTES_PER_ELEMENT properties on the result.
  result->Set(
      String::New("length"),
      Int32::New(num_elements),
      ReadOnly);

  result->Set(
      String::New("BYTES_PER_ELEMENT"),
      Int32::New(element_size));

  return result;
}

static bool IsArrayBuffer(const Handle<Value>& val) {
  return val->IsObject() &&
         val->ToObject()->Get(String::New(kArrayBufferMarkerPropName))->
             IsTrue();
}

// Create a typed array from an existing array buffer. The array_buffer
// reference must be non-empty, but the other two arguments are optional.
static Handle<Value> CreateExternalArrayFromArrayBuffer(
    ExternalArrayType element_type,
    size_t element_size,
    const Handle<Object>& array_buffer,
    const Handle<Value>& byte_offset_arg,
    const Handle<Value>& length_arg) {
  CHECK(!array_buffer.IsEmpty());
  CHECK(IsArrayBuffer(array_buffer));

  TryCatch try_catch;

  // Figure out what the length of the existing array buffer is.
  const Local<Value> array_buffer_length_property =
      array_buffer->Get(String::New("length"));
  CHECK(!array_buffer_length_property.IsEmpty());

  const size_t array_buffer_length =
      ConvertToUint(array_buffer_length_property, &try_catch);

  if (try_catch.HasCaught()) {
    return try_catch.Exception();
  }

  // Figure out what the offset into the array buffer should be.
  size_t byte_offset = 0;
  if (!byte_offset_arg.IsEmpty()) {
    byte_offset = ConvertToUint(byte_offset_arg, &try_catch);
    if (try_catch.HasCaught()) {
      return try_catch.Exception();
    }
  }

  // Make sure the offset is legal.
  if (byte_offset % element_size) {
    return ThrowException(
        String::New("Offset must be a multiple of element size."));
  }

  if (byte_offset > array_buffer_length) {
    return ThrowException(
        String::New("Offset must be less than the array buffer length."));
  }

  // Figure out what the length of the resulting array should be (in elements,
  // not bytes)
  size_t length = 0;
  if (length_arg.IsEmpty()) {
    // If the length arg is omitted, the array spans from the byte offset to the
    // end of the array buffer. The size of the range spanned must be a multiple
    // of the element size.
    if ((array_buffer_length - byte_offset) % element_size) {
      return ThrowException(
          String::New(
              "Array buffer length minus the byte offset must be a "
                  "multiple of the element size"));
    }

    length = (array_buffer_length - byte_offset) / element_size;
  } else {
    length = ConvertToUint(length_arg, &try_catch);
    if (try_catch.HasCaught()) {
      return try_catch.Exception();
    }
  }

  // Make sure the length is legal.
  if (byte_offset + (length * element_size) > array_buffer_length) {
    return ThrowException(
        String::New(
            "length references an area beyond the end of the array buffer."));
  }

  // Grab the data property from the array buffer.
  uint8_t* const data =
      static_cast<uint8_t*>(
          array_buffer->GetIndexedPropertiesExternalArrayData());

  if (!data) {
    return ThrowException(String::New("ArrayBuffer doesn't have data."));
  }

  // Create the resulting object.
  const Handle<Object> result =
      CreateExternalArray(
          data + byte_offset,
          length,
          element_size,
          element_type);

  // Hold a reference to the ArrayBuffer so its buffer doesn't get collected.
  result->Set(
      String::New(kArrayBufferReferencePropName),
      array_buffer,
      ReadOnly);

  return result;
}

// Implement the constructor with this signature:
//
//     TypedArray(unsigned long length)
//
static Handle<Value> CreateExternalArrayWithLengthArg(
    ExternalArrayType element_type,
    size_t element_size,
    const Handle<Value>& length_arg) {
  CHECK(!length_arg.IsEmpty());
  CHECK(element_size);

  TryCatch try_catch;

  // Convert the length to a useful value.
  const size_t length = ConvertToUint(length_arg, &try_catch);
  if (try_catch.HasCaught()) {
    return try_catch.Exception();
  }

  // Create the underlying data buffer.
  uint8_t* const data =
      static_cast<uint8_t*>(
          calloc(length, element_size));

  const size_t offset = 0;

  if (!data) {
    return ThrowException(String::New("Memory allocation failed."));
  }

  return CreateExternalArray(data, length, element_size, element_type);
}

// Implement the ArrayBuffer constructor.
static Handle<Value> CreateArrayBuffer(
    const Handle<Value>& length_arg) {
  const ExternalArrayType element_type = v8::kExternalByteArray;
  const size_t element_size = 1;

  const Handle<Value> result =
      CreateExternalArrayWithLengthArg(
          element_type,
          element_size,
          length_arg);

  if (!result->IsObject()) {
    return result;
  }

  // Mark this as an array buffer, for use by other code in this file.
  result->ToObject()->Set(
      String::New(kArrayBufferMarkerPropName),
      True(),
      ReadOnly);

  return result;
}

// Implement the constructor with this signature:
//
//     TypedArray(type[] array)
//
template <typename T>
static Handle<Value> CreateExternalArrayWithArrayArg(
    ExternalArrayType element_type,
    const Handle<Array>& array) {
  CHECK(!array.IsEmpty());
  TryCatch try_catch;

  const size_t element_size = sizeof(T);
  const uint32_t num_elements = array->Length();

  // Create the underlying data.
  T* const data =
      static_cast<T*>(
          calloc(num_elements, element_size));

  const size_t offset = 0;

  if (!data) {
    return ThrowException(String::New("Memory allocation failed."));
  }

  // Convert each element.
  for (uint32_t i = 0; i < num_elements; ++i) {
    data[i] = array->Get(i)->ToInteger()->Value();
  }

  return CreateExternalArray(
      data,
      num_elements,
      element_size,
      element_type);
}

// Common constructor code for all typed arrays. The following signatures are
// supported:
//
//     TypedArray(
//         ArrayBuffer buffer,
//         optional unsigned long byteOffset,
//         optional unsigned long length)
//
//     TypedArray(unsigned long length)
//
//     TypedArray(type[] array)
//
static Handle<Value> CreateExternalArray(
    const Arguments& args,
    ExternalArrayType element_type,
    size_t element_size) {
  const size_t num_args = args.Length();

  // Of the functions that defer to this one, the only one with element_size
  // equal to zero is the constructor for ArrayBuffer.
  if (element_size == 0) {
    if (num_args != 1) {
      return ThrowException(String::New("Expected exactly one argument."));
    }

    return CreateArrayBuffer(args[0]);
  }

  // We only support these element sizes.
  CHECK(
      element_size == 1 ||
      element_size == 2 ||
      element_size == 4 ||
      element_size == 8);

  // We require at least one arg.
  if (num_args == 0) {
    return ThrowException(String::New("Expected at least one argument."));
  }

  // Is this the constructor with the following signature?
  //
  //     TypedArray(
  //         ArrayBuffer buffer,
  //         optional unsigned long byteOffset,
  //         optional unsigned long length)
  //
  if (IsArrayBuffer(args[0])) {
    if (num_args > 3) {
      return ThrowException(
          String::New(
              "Array constructor from ArrayBuffer must have 1-3 parameters."));
    }

    return CreateExternalArrayFromArrayBuffer(
        element_type,
        element_size,
        args[0]->ToObject(),
        num_args > 1 ? args[1] : Handle<Value>(),
        num_args > 2 ? args[2] : Handle<Value>());
  }

  // Otherwise, there should be exactly one argument.
  if (num_args != 1) {
    return ThrowException(String::New("Expected exactly one argument."));
  }

  return CreateExternalArrayWithLengthArg(
      element_type,
      element_size,
      args[0]);
}

Handle<Value> ArrayBuffer(const Arguments& args) {
  return CreateExternalArray(args, kExternalByteArray, 0);
}

Handle<Value> Int8Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalByteArray, sizeof(int8_t));
}

Handle<Value> Int16Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalShortArray, sizeof(int16_t));
}

Handle<Value> Int32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalIntArray, sizeof(int32_t));
}

Handle<Value> Uint8Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedByteArray, sizeof(uint8_t));
}

Handle<Value> Uint16Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedShortArray, sizeof(uint16_t));
}

Handle<Value> Uint32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalUnsignedIntArray, sizeof(uint32_t));
}

Handle<Value> Float32Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalFloatArray, sizeof(float));
}

Handle<Value> Float64Array(const Arguments& args) {
  return CreateExternalArray(args, kExternalDoubleArray, sizeof(double));
}

void ExportTypedArrays(
    const Handle<ObjectTemplate>& global_template) {
  global_template->Set(
      String::New("ArrayBuffer"),
      FunctionTemplate::New(ArrayBuffer));

  // Signed integers.
  global_template->Set(
      String::New("Int8Array"),
      FunctionTemplate::New(Int8Array));

  global_template->Set(
      String::New("Int16Array"),
      FunctionTemplate::New(Int16Array));

  global_template->Set(
      String::New("Int32Array"),
      FunctionTemplate::New(Int32Array));

  // Unigned integers.
  global_template->Set(
      String::New("Uint8Array"),
      FunctionTemplate::New(Uint8Array));

  global_template->Set(
      String::New("Uint16Array"),
      FunctionTemplate::New(Uint16Array));

  global_template->Set(
      String::New("Uint32Array"),
      FunctionTemplate::New(Uint32Array));

  // Floats.
  global_template->Set(
      String::New("Float32Array"),
      FunctionTemplate::New(Float32Array));

  global_template->Set(
      String::New("Float64Array"),
      FunctionTemplate::New(Float64Array));
}

}  // namespace gjstest
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "gpu/config/gpu_info_collector.h"

#include <X11/Xlib.h>
#include <vector>

#include "base/command_line.h"
#include "base/debug/trace_event.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/message_loop/message_loop.h"
#include "base/strings/string_piece.h"
#include "base/strings/string_split.h"
#include "base/strings/string_tokenizer.h"
#include "base/strings/string_util.h"
#include "library_loaders/libpci.h"
#include "third_party/libXNVCtrl/NVCtrl.h"
#include "third_party/libXNVCtrl/NVCtrlLib.h"
#include "ui/gfx/x/x11_types.h"
#include "ui/gl/gl_bindings.h"
#include "ui/gl/gl_context.h"
#include "ui/gl/gl_implementation.h"
#include "ui/gl/gl_surface.h"
#include "ui/gl/gl_switches.h"

namespace gpu {

namespace {

// This checks if a system supports PCI bus.
// We check the existence of /sys/bus/pci or /sys/bug/pci_express.
bool IsPciSupported() {
  const base::FilePath pci_path("/sys/bus/pci/");
  const base::FilePath pcie_path("/sys/bus/pci_express/");
  return (base::PathExists(pci_path) ||
          base::PathExists(pcie_path));
}

// Scan /etc/ati/amdpcsdb.default for "ReleaseVersion".
// Return empty string on failing.
std::string CollectDriverVersionATI() {
  const base::FilePath::CharType kATIFileName[] =
      FILE_PATH_LITERAL("/etc/ati/amdpcsdb.default");
  base::FilePath ati_file_path(kATIFileName);
  if (!base::PathExists(ati_file_path))
    return std::string();
  std::string contents;
  if (!base::ReadFileToString(ati_file_path, &contents))
    return std::string();
  base::StringTokenizer t(contents, "\r\n");
  while (t.GetNext()) {
    std::string line = t.token();
    if (StartsWithASCII(line, "ReleaseVersion=", true)) {
      size_t begin = line.find_first_of("0123456789");
      if (begin != std::string::npos) {
        size_t end = line.find_first_not_of("0123456789.", begin);
        if (end == std::string::npos)
          return line.substr(begin);
        else
          return line.substr(begin, end - begin);
      }
    }
  }
  return std::string();
}

// Use NVCtrl extention to query NV driver version.
// Return empty string on failing.
std::string CollectDriverVersionNVidia() {
  Display* display = gfx::GetXDisplay();
  if (!display) {
    LOG(ERROR) << "XOpenDisplay failed.";
    return std::string();
  }
  int event_base = 0, error_base = 0;
  if (!XNVCTRLQueryExtension(display, &event_base, &error_base)) {
    VLOG(1) << "NVCtrl extension does not exist.";
    return std::string();
  }
  int screen_count = ScreenCount(display);
  for (int screen = 0; screen < screen_count; ++screen) {
    char* buffer = NULL;
    if (XNVCTRLIsNvScreen(display, screen) &&
        XNVCTRLQueryStringAttribute(display, screen, 0,
                                    NV_CTRL_STRING_NVIDIA_DRIVER_VERSION,
                                    &buffer)) {
      std::string driver_version(buffer);
      XFree(buffer);
      return driver_version;
    }
  }
  return std::string();
}

const uint32 kVendorIDIntel = 0x8086;
const uint32 kVendorIDNVidia = 0x10de;
const uint32 kVendorIDAMD = 0x1002;

bool CollectPCIVideoCardInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  if (IsPciSupported() == false) {
    VLOG(1) << "PCI bus scanning is not supported";
    return false;
  }

  // TODO(zmo): be more flexible about library name.
  LibPciLoader libpci_loader;
  if (!libpci_loader.Load("libpci.so.3") &&
      !libpci_loader.Load("libpci.so")) {
    VLOG(1) << "Failed to locate libpci";
    return false;
  }

  pci_access* access = (libpci_loader.pci_alloc)();
  DCHECK(access != NULL);
  (libpci_loader.pci_init)(access);
  (libpci_loader.pci_scan_bus)(access);
  bool primary_gpu_identified = false;
  for (pci_dev* device = access->devices;
       device != NULL; device = device->next) {
    // Fill the IDs and class fields.
    (libpci_loader.pci_fill_info)(device, 33);
    bool is_gpu = false;
    switch (device->device_class) {
      case PCI_CLASS_DISPLAY_VGA:
      case PCI_CLASS_DISPLAY_XGA:
      case PCI_CLASS_DISPLAY_3D:
        is_gpu = true;
        break;
      case PCI_CLASS_DISPLAY_OTHER:
      default:
        break;
    }
    if (!is_gpu)
      continue;

    GPUInfo::GPUDevice gpu;
    gpu.vendor_id = device->vendor_id;
    gpu.device_id = device->device_id;

    if (!primary_gpu_identified) {
      primary_gpu_identified = true;
      gpu_info->gpu = gpu;
    } else {
      // TODO(zmo): if there are multiple GPUs, we assume the non Intel
      // one is primary. Revisit this logic because we actually don't know
      // which GPU we are using at this point.
      if (gpu_info->gpu.vendor_id == kVendorIDIntel &&
          gpu.vendor_id != kVendorIDIntel) {
        gpu_info->secondary_gpus.push_back(gpu_info->gpu);
        gpu_info->gpu = gpu;
      } else {
        gpu_info->secondary_gpus.push_back(gpu);
      }
    }
  }

  // Detect Optimus or AMD Switchable GPU.
  if (gpu_info->secondary_gpus.size() == 1 &&
      gpu_info->secondary_gpus[0].vendor_id == kVendorIDIntel) {
    if (gpu_info->gpu.vendor_id == kVendorIDNVidia)
      gpu_info->optimus = true;
    if (gpu_info->gpu.vendor_id == kVendorIDAMD)
      gpu_info->amd_switchable = true;
  }

  (libpci_loader.pci_cleanup)(access);
  return (primary_gpu_identified);
}

}  // namespace anonymous

CollectInfoResult CollectContextGraphicsInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  TRACE_EVENT0("gpu", "gpu_info_collector::CollectGraphicsInfo");

  if (CommandLine::ForCurrentProcess()->HasSwitch(
          switches::kGpuNoContextLost)) {
    gpu_info->can_lose_context = false;
  } else {
#if defined(OS_CHROMEOS)
    gpu_info->can_lose_context = false;
#else
    // TODO(zmo): need to consider the case where we are running on top
    // of desktop GL and GL_ARB_robustness extension is available.
    gpu_info->can_lose_context =
        (gfx::GetGLImplementation() == gfx::kGLImplementationEGLGLES2);
#endif
  }

  CollectInfoResult result = CollectGraphicsInfoGL(gpu_info);
  gpu_info->finalized = true;
  return result;
}

GpuIDResult CollectGpuID(uint32* vendor_id, uint32* device_id) {
  DCHECK(vendor_id && device_id);
  *vendor_id = 0;
  *device_id = 0;

  GPUInfo gpu_info;
  if (CollectPCIVideoCardInfo(&gpu_info)) {
    *vendor_id = gpu_info.gpu.vendor_id;
    *device_id = gpu_info.gpu.device_id;
    return kGpuIDSuccess;
  }
  return kGpuIDFailure;
}

CollectInfoResult CollectBasicGraphicsInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  bool rt = CollectPCIVideoCardInfo(gpu_info);

  std::string driver_version;
  switch (gpu_info->gpu.vendor_id) {
    case kVendorIDAMD:
      driver_version = CollectDriverVersionATI();
      if (!driver_version.empty()) {
        gpu_info->driver_vendor = "ATI / AMD";
        gpu_info->driver_version = driver_version;
      }
      break;
    case kVendorIDNVidia:
      driver_version = CollectDriverVersionNVidia();
      if (!driver_version.empty()) {
        gpu_info->driver_vendor = "NVIDIA";
        gpu_info->driver_version = driver_version;
      }
      break;
    case kVendorIDIntel:
      // In dual-GPU cases, sometimes PCI scan only gives us the
      // integrated GPU (i.e., the Intel one).
      driver_version = CollectDriverVersionNVidia();
      if (!driver_version.empty()) {
        gpu_info->driver_vendor = "NVIDIA";
        gpu_info->driver_version = driver_version;
        // Machines with more than two GPUs are not handled.
        if (gpu_info->secondary_gpus.size() <= 1)
          gpu_info->optimus = true;
      }
      break;
  }

  return rt ? kCollectInfoSuccess : kCollectInfoNonFatalFailure;
}

CollectInfoResult CollectDriverInfoGL(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  std::string gl_version = gpu_info->gl_version;
  if (StartsWithASCII(gl_version, "OpenGL ES", true))
    gl_version = gl_version.substr(10);
  std::vector<std::string> pieces;
  base::SplitStringAlongWhitespace(gl_version, &pieces);
  // In linux, the gl version string might be in the format of
  //   GLVersion DriverVendor DriverVersion
  if (pieces.size() < 3)
    return kCollectInfoNonFatalFailure;

  std::string driver_version = pieces[2];
  size_t pos = driver_version.find_first_not_of("0123456789.");
  if (pos == 0)
    return kCollectInfoNonFatalFailure;
  if (pos != std::string::npos)
    driver_version = driver_version.substr(0, pos);

  gpu_info->driver_vendor = pieces[1];
  gpu_info->driver_version = driver_version;
  return kCollectInfoSuccess;
}

void MergeGPUInfo(GPUInfo* basic_gpu_info,
                  const GPUInfo& context_gpu_info) {
  MergeGPUInfoGL(basic_gpu_info, context_gpu_info);
}

}  // namespace gpu
<commit_msg>Be more strict with detecting Optimus on Linux.<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "gpu/config/gpu_info_collector.h"

#include <X11/Xlib.h>
#include <vector>

#include "base/command_line.h"
#include "base/debug/trace_event.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/message_loop/message_loop.h"
#include "base/strings/string_piece.h"
#include "base/strings/string_split.h"
#include "base/strings/string_tokenizer.h"
#include "base/strings/string_util.h"
#include "library_loaders/libpci.h"
#include "third_party/libXNVCtrl/NVCtrl.h"
#include "third_party/libXNVCtrl/NVCtrlLib.h"
#include "ui/gfx/x/x11_types.h"
#include "ui/gl/gl_bindings.h"
#include "ui/gl/gl_context.h"
#include "ui/gl/gl_implementation.h"
#include "ui/gl/gl_surface.h"
#include "ui/gl/gl_switches.h"

namespace gpu {

namespace {

// This checks if a system supports PCI bus.
// We check the existence of /sys/bus/pci or /sys/bug/pci_express.
bool IsPciSupported() {
  const base::FilePath pci_path("/sys/bus/pci/");
  const base::FilePath pcie_path("/sys/bus/pci_express/");
  return (base::PathExists(pci_path) ||
          base::PathExists(pcie_path));
}

// Scan /etc/ati/amdpcsdb.default for "ReleaseVersion".
// Return empty string on failing.
std::string CollectDriverVersionATI() {
  const base::FilePath::CharType kATIFileName[] =
      FILE_PATH_LITERAL("/etc/ati/amdpcsdb.default");
  base::FilePath ati_file_path(kATIFileName);
  if (!base::PathExists(ati_file_path))
    return std::string();
  std::string contents;
  if (!base::ReadFileToString(ati_file_path, &contents))
    return std::string();
  base::StringTokenizer t(contents, "\r\n");
  while (t.GetNext()) {
    std::string line = t.token();
    if (StartsWithASCII(line, "ReleaseVersion=", true)) {
      size_t begin = line.find_first_of("0123456789");
      if (begin != std::string::npos) {
        size_t end = line.find_first_not_of("0123456789.", begin);
        if (end == std::string::npos)
          return line.substr(begin);
        else
          return line.substr(begin, end - begin);
      }
    }
  }
  return std::string();
}

// Use NVCtrl extention to query NV driver version.
// Return empty string on failing.
std::string CollectDriverVersionNVidia() {
  Display* display = gfx::GetXDisplay();
  if (!display) {
    LOG(ERROR) << "XOpenDisplay failed.";
    return std::string();
  }
  int event_base = 0, error_base = 0;
  if (!XNVCTRLQueryExtension(display, &event_base, &error_base)) {
    VLOG(1) << "NVCtrl extension does not exist.";
    return std::string();
  }
  int screen_count = ScreenCount(display);
  for (int screen = 0; screen < screen_count; ++screen) {
    char* buffer = NULL;
    if (XNVCTRLIsNvScreen(display, screen) &&
        XNVCTRLQueryStringAttribute(display, screen, 0,
                                    NV_CTRL_STRING_NVIDIA_DRIVER_VERSION,
                                    &buffer)) {
      std::string driver_version(buffer);
      XFree(buffer);
      return driver_version;
    }
  }
  return std::string();
}

const uint32 kVendorIDIntel = 0x8086;
const uint32 kVendorIDNVidia = 0x10de;
const uint32 kVendorIDAMD = 0x1002;

bool CollectPCIVideoCardInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  if (IsPciSupported() == false) {
    VLOG(1) << "PCI bus scanning is not supported";
    return false;
  }

  // TODO(zmo): be more flexible about library name.
  LibPciLoader libpci_loader;
  if (!libpci_loader.Load("libpci.so.3") &&
      !libpci_loader.Load("libpci.so")) {
    VLOG(1) << "Failed to locate libpci";
    return false;
  }

  pci_access* access = (libpci_loader.pci_alloc)();
  DCHECK(access != NULL);
  (libpci_loader.pci_init)(access);
  (libpci_loader.pci_scan_bus)(access);
  bool primary_gpu_identified = false;
  for (pci_dev* device = access->devices;
       device != NULL; device = device->next) {
    // Fill the IDs and class fields.
    (libpci_loader.pci_fill_info)(device, 33);
    bool is_gpu = false;
    switch (device->device_class) {
      case PCI_CLASS_DISPLAY_VGA:
      case PCI_CLASS_DISPLAY_XGA:
      case PCI_CLASS_DISPLAY_3D:
        is_gpu = true;
        break;
      case PCI_CLASS_DISPLAY_OTHER:
      default:
        break;
    }
    if (!is_gpu)
      continue;

    GPUInfo::GPUDevice gpu;
    gpu.vendor_id = device->vendor_id;
    gpu.device_id = device->device_id;

    if (!primary_gpu_identified) {
      primary_gpu_identified = true;
      gpu_info->gpu = gpu;
    } else {
      // TODO(zmo): if there are multiple GPUs, we assume the non Intel
      // one is primary. Revisit this logic because we actually don't know
      // which GPU we are using at this point.
      if (gpu_info->gpu.vendor_id == kVendorIDIntel &&
          gpu.vendor_id != kVendorIDIntel) {
        gpu_info->secondary_gpus.push_back(gpu_info->gpu);
        gpu_info->gpu = gpu;
      } else {
        gpu_info->secondary_gpus.push_back(gpu);
      }
    }
  }

  // Detect Optimus or AMD Switchable GPU.
  if (gpu_info->secondary_gpus.size() == 1 &&
      gpu_info->secondary_gpus[0].vendor_id == kVendorIDIntel) {
    if (gpu_info->gpu.vendor_id == kVendorIDNVidia)
      gpu_info->optimus = true;
    if (gpu_info->gpu.vendor_id == kVendorIDAMD)
      gpu_info->amd_switchable = true;
  }

  (libpci_loader.pci_cleanup)(access);
  return (primary_gpu_identified);
}

}  // namespace anonymous

CollectInfoResult CollectContextGraphicsInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  TRACE_EVENT0("gpu", "gpu_info_collector::CollectGraphicsInfo");

  if (CommandLine::ForCurrentProcess()->HasSwitch(
          switches::kGpuNoContextLost)) {
    gpu_info->can_lose_context = false;
  } else {
#if defined(OS_CHROMEOS)
    gpu_info->can_lose_context = false;
#else
    // TODO(zmo): need to consider the case where we are running on top
    // of desktop GL and GL_ARB_robustness extension is available.
    gpu_info->can_lose_context =
        (gfx::GetGLImplementation() == gfx::kGLImplementationEGLGLES2);
#endif
  }

  CollectInfoResult result = CollectGraphicsInfoGL(gpu_info);
  gpu_info->finalized = true;
  return result;
}

GpuIDResult CollectGpuID(uint32* vendor_id, uint32* device_id) {
  DCHECK(vendor_id && device_id);
  *vendor_id = 0;
  *device_id = 0;

  GPUInfo gpu_info;
  if (CollectPCIVideoCardInfo(&gpu_info)) {
    *vendor_id = gpu_info.gpu.vendor_id;
    *device_id = gpu_info.gpu.device_id;
    return kGpuIDSuccess;
  }
  return kGpuIDFailure;
}

CollectInfoResult CollectBasicGraphicsInfo(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  bool rt = CollectPCIVideoCardInfo(gpu_info);

  std::string driver_version;
  switch (gpu_info->gpu.vendor_id) {
    case kVendorIDAMD:
      driver_version = CollectDriverVersionATI();
      if (!driver_version.empty()) {
        gpu_info->driver_vendor = "ATI / AMD";
        gpu_info->driver_version = driver_version;
      }
      break;
    case kVendorIDNVidia:
      driver_version = CollectDriverVersionNVidia();
      if (!driver_version.empty()) {
        gpu_info->driver_vendor = "NVIDIA";
        gpu_info->driver_version = driver_version;
      }
      break;
    case kVendorIDIntel:
      // In dual-GPU cases, sometimes PCI scan only gives us the
      // integrated GPU (i.e., the Intel one).
      if (gpu_info->secondary_gpus.size() == 0) {
        driver_version = CollectDriverVersionNVidia();
        if (!driver_version.empty()) {
          gpu_info->driver_vendor = "NVIDIA";
          gpu_info->driver_version = driver_version;
          gpu_info->optimus = true;
          // Put Intel to the secondary GPU list.
          gpu_info->secondary_gpus.push_back(gpu_info->gpu);
          // Put NVIDIA as the primary GPU.
          gpu_info->gpu.vendor_id = kVendorIDNVidia;
          gpu_info->gpu.device_id = 0;  // Unknown Device.
        }
      }
      break;
  }

  return rt ? kCollectInfoSuccess : kCollectInfoNonFatalFailure;
}

CollectInfoResult CollectDriverInfoGL(GPUInfo* gpu_info) {
  DCHECK(gpu_info);

  std::string gl_version = gpu_info->gl_version;
  if (StartsWithASCII(gl_version, "OpenGL ES", true))
    gl_version = gl_version.substr(10);
  std::vector<std::string> pieces;
  base::SplitStringAlongWhitespace(gl_version, &pieces);
  // In linux, the gl version string might be in the format of
  //   GLVersion DriverVendor DriverVersion
  if (pieces.size() < 3)
    return kCollectInfoNonFatalFailure;

  std::string driver_version = pieces[2];
  size_t pos = driver_version.find_first_not_of("0123456789.");
  if (pos == 0)
    return kCollectInfoNonFatalFailure;
  if (pos != std::string::npos)
    driver_version = driver_version.substr(0, pos);

  gpu_info->driver_vendor = pieces[1];
  gpu_info->driver_version = driver_version;
  return kCollectInfoSuccess;
}

void MergeGPUInfo(GPUInfo* basic_gpu_info,
                  const GPUInfo& context_gpu_info) {
  MergeGPUInfoGL(basic_gpu_info, context_gpu_info);
}

}  // namespace gpu
<|endoftext|>