<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_sal.hxx"

 #ifndef _OSL_UUNXAPI_H_
 #include "uunxapi.h"
 #endif

 #ifndef __OSL_SYSTEM_H__
 #include "system.h"
 #endif

 #ifndef _LIMITS_H
 #include <limits.h>
 #endif

 #ifndef _RTL_USTRING_HXX_
 #include <rtl/ustring.hxx>
 #endif

 #ifndef _OSL_THREAD_H_
 #include <osl/thread.h>
 #endif

 //###########################
 inline rtl::OString OUStringToOString(const rtl_uString* s)
 {
    return rtl::OUStringToOString(
        rtl::OUString(const_cast<rtl_uString*>(s)),
        osl_getThreadTextEncoding());
 }

 //###########################
#ifdef MACOSX
/*
 * Helper function for resolving Mac native alias files (not the same as unix alias files)
 * and to return the resolved alias as rtl::OString
 */
 inline rtl::OString macxp_resolveAliasAndConvert(const rtl_uString* s)
 {
  rtl::OString p = OUStringToOString(s);
  sal_Char path[PATH_MAX];
  if (p.getLength() < PATH_MAX)
    {
      strcpy(path, p.getStr());
      macxp_resolveAlias(path, PATH_MAX);
      p = rtl::OString(path);
    }
  return p;
 }
#endif /* MACOSX */

 //###########################
 //access_u
 int access_u(const rtl_uString* pustrPath, int mode)
 {
#ifndef MACOSX // not MACOSX
    return access(OUStringToOString(pustrPath).getStr(), mode);
#else
    return access(macxp_resolveAliasAndConvert(pustrPath).getStr(), mode);
#endif
 }

 //#########################
 //realpath_u
 sal_Bool realpath_u(const rtl_uString* pustrFileName, rtl_uString** ppustrResolvedName)
 {
#ifndef MACOSX // not MACOSX
        rtl::OString fn = OUStringToOString(pustrFileName);
#else
    rtl::OString fn = macxp_resolveAliasAndConvert(pustrFileName);
#endif
    char  rp[PATH_MAX];
    bool  bRet = realpath(fn.getStr(), rp);

    if (bRet)
    {
        rtl::OUString resolved = rtl::OStringToOUString(
            rtl::OString(static_cast<sal_Char*>(rp)),
            osl_getThreadTextEncoding());

        rtl_uString_assign(ppustrResolvedName, resolved.pData);
    }
    return bRet;
 }

 //#########################
 //lstat_u
  int lstat_u(const rtl_uString* pustrPath, struct stat* buf)
 {
#ifndef MACOSX  // not MACOSX
    return lstat(OUStringToOString(pustrPath).getStr(), buf);
#else
    return lstat(macxp_resolveAliasAndConvert(pustrPath).getStr(), buf);
#endif
 }

 //#########################
 // @see mkdir
 int mkdir_u(const rtl_uString* path, mode_t mode)
 {
    return mkdir(OUStringToOString(path).getStr(), mode);
 }

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>fix indent<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_sal.hxx"

 #ifndef _OSL_UUNXAPI_H_
 #include "uunxapi.h"
 #endif

 #ifndef __OSL_SYSTEM_H__
 #include "system.h"
 #endif

 #ifndef _LIMITS_H
 #include <limits.h>
 #endif

 #ifndef _RTL_USTRING_HXX_
 #include <rtl/ustring.hxx>
 #endif

 #ifndef _OSL_THREAD_H_
 #include <osl/thread.h>
 #endif

 //###########################
 inline rtl::OString OUStringToOString(const rtl_uString* s)
 {
    return rtl::OUStringToOString(
        rtl::OUString(const_cast<rtl_uString*>(s)),
        osl_getThreadTextEncoding());
 }

 //###########################
#ifdef MACOSX
/*
 * Helper function for resolving Mac native alias files (not the same as unix alias files)
 * and to return the resolved alias as rtl::OString
 */
 inline rtl::OString macxp_resolveAliasAndConvert(const rtl_uString* s)
 {
  rtl::OString p = OUStringToOString(s);
  sal_Char path[PATH_MAX];
  if (p.getLength() < PATH_MAX)
    {
      strcpy(path, p.getStr());
      macxp_resolveAlias(path, PATH_MAX);
      p = rtl::OString(path);
    }
  return p;
 }
#endif /* MACOSX */

 //###########################
 //access_u
 int access_u(const rtl_uString* pustrPath, int mode)
 {
#ifndef MACOSX // not MACOSX
    return access(OUStringToOString(pustrPath).getStr(), mode);
#else
    return access(macxp_resolveAliasAndConvert(pustrPath).getStr(), mode);
#endif
 }

 //#########################
 //realpath_u
 sal_Bool realpath_u(const rtl_uString* pustrFileName, rtl_uString** ppustrResolvedName)
 {
#ifndef MACOSX // not MACOSX
    rtl::OString fn = OUStringToOString(pustrFileName);
#else
    rtl::OString fn = macxp_resolveAliasAndConvert(pustrFileName);
#endif
    char  rp[PATH_MAX];
    bool  bRet = realpath(fn.getStr(), rp);

    if (bRet)
    {
        rtl::OUString resolved = rtl::OStringToOUString(
            rtl::OString(static_cast<sal_Char*>(rp)),
            osl_getThreadTextEncoding());

        rtl_uString_assign(ppustrResolvedName, resolved.pData);
    }
    return bRet;
 }

 //#########################
 //lstat_u
  int lstat_u(const rtl_uString* pustrPath, struct stat* buf)
 {
#ifndef MACOSX  // not MACOSX
    return lstat(OUStringToOString(pustrPath).getStr(), buf);
#else
    return lstat(macxp_resolveAliasAndConvert(pustrPath).getStr(), buf);
#endif
 }

 //#########################
 // @see mkdir
 int mkdir_u(const rtl_uString* path, mode_t mode)
 {
    return mkdir(OUStringToOString(path).getStr(), mode);
 }

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <thread>
#include "paddle/fluid/framework/channel.h"
#include "paddle/fluid/operators/reader/reader_op_registry.h"

namespace paddle {
namespace operators {
namespace reader {

static constexpr size_t kChannelSize = 2;
static constexpr size_t kCacheSize = 4;  // kChannelSize + 2

class DoubleBufferReader : public framework::DecoratedReader {
 public:
  struct Item {
    Item() : ctx_(nullptr) {}

    std::vector<framework::LoDTensor> payloads_;
    platform::DeviceContext* ctx_;
  };

  explicit DoubleBufferReader(
      ReaderBase* reader, platform::Place target_place = platform::CPUPlace())
      : DecoratedReader(reader), place_(target_place) {
#ifdef PADDLE_WITH_CUDA
    for (size_t i = 0; i < kChannelSize + 2; ++i) {
      if (platform::is_gpu_place(place_)) {
        ctxs_.emplace_back(new platform::CUDADeviceContext(
            boost::get<platform::CUDAPlace>(place_)));
      }
    }
#endif
    StartPrefetcher();
  }

  bool HasNext() const override;
  void ReadNext(std::vector<framework::LoDTensor>* out) override;
  void ReInit() override;

  void StartPrefetcher() {
    buffer_ = framework::MakeChannel<Item>(kChannelSize);
    prefetcher_ = std::thread([this] { PrefetchThreadFunc(); });
  }

  void EndPrefetcher() {
    buffer_->Close();
    if (prefecther_.joinable()) {
      prefetcher_.join();
    }
    delete buffer_;
    buffer_ = nullptr;
  }

  ~DoubleBufferReader() { EndPrefetcher(); }

 private:
  void PrefetchThreadFunc();

  std::thread prefetcher_;
  framework::Channel<Item>* buffer_;
  platform::Place place_;
  std::vector<std::unique_ptr<platform::DeviceContext>> ctxs_;
};

class CreateDoubleBufferReaderOp : public framework::OperatorBase {
 public:
  using framework::OperatorBase::OperatorBase;

 private:
  void RunImpl(const framework::Scope& scope,
               const platform::Place& dev_place) const override {
    const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader"))
                                        ->Get<framework::ReaderHolder>();
    auto* out = scope.FindVar(Output("Out"))
                    ->template GetMutable<framework::ReaderHolder>();

    auto place_str = Attr<std::string>("place");
    platform::Place place;
    if (place_str == "CPU") {
      place = platform::CPUPlace();
    } else {
      std::istringstream sin(place_str);
      sin.seekg(std::string("CUDA:").size(), std::ios::beg);
      size_t num;
      sin >> num;
      place = platform::CUDAPlace(static_cast<int>(num));
    }

    out->Reset(new DoubleBufferReader(underlying_reader.Get(), place));
  }
};

class CreateDoubleBufferReaderOpMaker : public DecoratedReaderMakerBase {
 public:
  CreateDoubleBufferReaderOpMaker(OpProto* op_proto, OpAttrChecker* op_checker)
      : DecoratedReaderMakerBase(op_proto, op_checker) {
    AddComment(R"DOC(
      CreateDoubleBufferReader Operator

      A double buffer reader takes another reader as its 'underlying reader'.
      It launches another thread to execute the 'underlying reader' asynchronously,
      which prevents reading process from blocking subsequent training.
    )DOC");
    std::unordered_set<std::string> enum_range;
    constexpr size_t kMaxCUDADevs = 128;
    for (size_t i = 0; i < kMaxCUDADevs; ++i) {
      enum_range.insert(string::Sprintf("CUDA:%d", i));
    }
    enum_range.insert("CPU");
    AddAttr<std::string>("place", "The double buffer place, default is CPU")
        .SetDefault("CPU")
        .InEnum({enum_range});
  }
};

bool DoubleBufferReader::HasNext() const {
  while (!buffer_->IsClosed() && !buffer_->CanReceive()) {
  }
  return buffer_->CanReceive()
}

void DoubleBufferReader::ReadNext(std::vector<framework::LoDTensor>* out) {
  if (!HasNext()) {
    PADDLE_THROW("There is no next data!");
  }

  Item batch;
  buffer_->Receive(&batch);
  *out = batch.payload_;
  if (batch.ctx_) {
    batch.ctx_->Wait();
  }
}

void DoubleBufferReader::ReInit() {
  reader_->ReInit();
  EndPrefetcher();
  StartPrefetcher();
}

void DoubleBufferReader::PrefetchThreadFunc() {
  VLOG(5) << "A new prefetch thread starts.";
  std::vector<std::vector<framework::LoDTensor>> cpu_tensor_cache(kCacheSize);
  std::vector<std::vector<framework::LoDTensor>> gpu_tensor_cache(kCacheSize);
  size_t cached_tensor_id = 0;

  while (reader_->HasNext()) {
    Item batch;
    auto& cpu_batch = cpu_tensor_cache[cached_tensor_id];
    reader_->ReadNext(&cpu_batch);
    if (platform::is_gpu_place(place_)) {
      auto& gpu_batch = gpu_tensor_cache[cached_tensor_id];
      auto* gpu_ctx = ctxs_[cached_tensor_id].get();
      gpu_batch.resize(cpu_batch.size());
      for (size_t i = 0; i < cpu_batch.size(); ++i) {
        framework::TensorCopy(cpu_batch[i], place_, *gpu_ctx, &gpu_batch[i]);
        gpu_batch[i].set_lod(batch.payloads_[i].lod());
      }
      batch.payload_ = gpu_batch;
      batch.ctx_ = gpu_ctx;
    } else {
      // CPUPlace
      batch.payload_ = cpu_batch;
    }
    ++cached_tensor_id;
    cached_tensor_id %= kCacheSize;

    try {
      buffer_->Send(&batch);
    } catch (paddle::platform::EnforceNotMet e) {
      VLOG(5) << "WARNING: The double buffer channel has been closed. The "
                 "prefetch thread will terminate.";
      break;
    }
  }
  buffer_->Close();
  VLOG(5) << "Prefetch thread terminates.";
}

}  // namespace reader
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators::reader;
REGISTER_DECORATED_READER_OPERATOR(create_double_buffer_reader,
                                   ops::CreateDoubleBufferReaderOp,
                                   ops::CreateDoubleBufferReaderOpMaker);
<commit_msg>fix compile errors<commit_after>//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <thread>
#include "paddle/fluid/framework/channel.h"
#include "paddle/fluid/operators/reader/reader_op_registry.h"

namespace paddle {
namespace operators {
namespace reader {

static constexpr size_t kCacheSize = 2;
static constexpr size_t kChannelSize = 0;  // kCacheSize - 2

class DoubleBufferReader : public framework::DecoratedReader {
 public:
  struct Item {
    Item() : ctx_(nullptr) {}

    std::vector<framework::LoDTensor> payloads_;
    platform::DeviceContext* ctx_;
  };

  explicit DoubleBufferReader(
      ReaderBase* reader, platform::Place target_place = platform::CPUPlace())
      : DecoratedReader(reader), place_(target_place) {
#ifdef PADDLE_WITH_CUDA
    for (size_t i = 0; i < kCacheSize; ++i) {
      if (platform::is_gpu_place(place_)) {
        ctxs_.emplace_back(new platform::CUDADeviceContext(
            boost::get<platform::CUDAPlace>(place_)));
      }
    }
#endif
    StartPrefetcher();
  }

  bool HasNext() const override;
  void ReadNext(std::vector<framework::LoDTensor>* out) override;
  void ReInit() override;

  void StartPrefetcher() {
    channel_ = framework::MakeChannel<Item>(kChannelSize);
    prefetcher_ = std::thread([this] { PrefetchThreadFunc(); });
  }

  void EndPrefetcher() {
    channel_->Close();
    if (prefetcher_.joinable()) {
      prefetcher_.join();
    }
    delete channel_;
    channel_ = nullptr;
  }

  ~DoubleBufferReader() { EndPrefetcher(); }

 private:
  void PrefetchThreadFunc();

  std::thread prefetcher_;
  framework::Channel<Item>* channel_;
  platform::Place place_;
  std::vector<std::unique_ptr<platform::DeviceContext>> ctxs_;
};

class CreateDoubleBufferReaderOp : public framework::OperatorBase {
 public:
  using framework::OperatorBase::OperatorBase;

 private:
  void RunImpl(const framework::Scope& scope,
               const platform::Place& dev_place) const override {
    const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader"))
                                        ->Get<framework::ReaderHolder>();
    auto* out = scope.FindVar(Output("Out"))
                    ->template GetMutable<framework::ReaderHolder>();

    auto place_str = Attr<std::string>("place");
    platform::Place place;
    if (place_str == "CPU") {
      place = platform::CPUPlace();
    } else {
      std::istringstream sin(place_str);
      sin.seekg(std::string("CUDA:").size(), std::ios::beg);
      size_t num;
      sin >> num;
      place = platform::CUDAPlace(static_cast<int>(num));
    }

    out->Reset(new DoubleBufferReader(underlying_reader.Get(), place));
  }
};

class CreateDoubleBufferReaderOpMaker : public DecoratedReaderMakerBase {
 public:
  CreateDoubleBufferReaderOpMaker(OpProto* op_proto, OpAttrChecker* op_checker)
      : DecoratedReaderMakerBase(op_proto, op_checker) {
    AddComment(R"DOC(
      CreateDoubleBufferReader Operator

      A double buffer reader takes another reader as its 'underlying reader'.
      It launches another thread to execute the 'underlying reader' asynchronously,
      which prevents reading process from blocking subsequent training.
    )DOC");
    std::unordered_set<std::string> enum_range;
    constexpr size_t kMaxCUDADevs = 128;
    for (size_t i = 0; i < kMaxCUDADevs; ++i) {
      enum_range.insert(string::Sprintf("CUDA:%d", i));
    }
    enum_range.insert("CPU");
    AddAttr<std::string>("place", "The double buffer place, default is CPU")
        .SetDefault("CPU")
        .InEnum({enum_range});
  }
};

bool DoubleBufferReader::HasNext() const {
  while (!channel_->IsClosed() && !channel_->CanReceive()) {
  }
  return channel_->CanReceive();
}

void DoubleBufferReader::ReadNext(std::vector<framework::LoDTensor>* out) {
  if (!HasNext()) {
    PADDLE_THROW("There is no next data!");
  }

  Item batch;
  channel_->Receive(&batch);
  *out = batch.payloads_;
  if (batch.ctx_) {
    batch.ctx_->Wait();
  }
}

void DoubleBufferReader::ReInit() {
  reader_->ReInit();
  EndPrefetcher();
  StartPrefetcher();
}

void DoubleBufferReader::PrefetchThreadFunc() {
  VLOG(5) << "A new prefetch thread starts.";
  std::vector<std::vector<framework::LoDTensor>> cpu_tensor_cache(kCacheSize);
  std::vector<std::vector<framework::LoDTensor>> gpu_tensor_cache(kCacheSize);
  size_t cached_tensor_id = 0;

  while (reader_->HasNext()) {
    Item batch;
    auto& cpu_batch = cpu_tensor_cache[cached_tensor_id];
    reader_->ReadNext(&cpu_batch);
    if (platform::is_gpu_place(place_)) {
      auto& gpu_batch = gpu_tensor_cache[cached_tensor_id];
      auto* gpu_ctx = ctxs_[cached_tensor_id].get();
      gpu_batch.resize(cpu_batch.size());
      for (size_t i = 0; i < cpu_batch.size(); ++i) {
        framework::TensorCopy(cpu_batch[i], place_, *gpu_ctx, &gpu_batch[i]);
        gpu_batch[i].set_lod(cpu_batch[i].lod());
      }
      batch.payloads_ = gpu_batch;
      batch.ctx_ = gpu_ctx;
    } else {
      // CPUPlace
      batch.payloads_ = cpu_batch;
    }
    ++cached_tensor_id;
    cached_tensor_id %= kCacheSize;

    try {
      channel_->Send(&batch);
    } catch (paddle::platform::EnforceNotMet e) {
      VLOG(5) << "WARNING: The double buffer channel has been closed. The "
                 "prefetch thread will terminate.";
      break;
    }
  }
  channel_->Close();
  VLOG(5) << "Prefetch thread terminates.";
}

}  // namespace reader
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators::reader;
REGISTER_DECORATED_READER_OPERATOR(create_double_buffer_reader,
                                   ops::CreateDoubleBufferReaderOp,
                                   ops::CreateDoubleBufferReaderOpMaker);
<|endoftext|>


<commit_before>#include "chip8.h"
#include <QFileDialog>
#include <qbitmap.h>
#include <qpainter.h>
#include <qtimer.h>
#include <qkeyevent>
#include <qkeysequence>

Chip8::Chip8(QWidget *parent)
: QMainWindow( parent ),
  _emuThread( &_emu )
{
  ui.setupUi(this);

  connect(ui.actionOpenGame, SIGNAL(triggered()), this, SLOT(openGame()));
  connect(ui.actionZoomIn, SIGNAL(triggered()), this, SLOT(zoomIn()));
  connect(ui.actionZoomOut, SIGNAL(triggered()), this, SLOT(zoomOut()));
  // toolbar
  connect(ui.actionStartEmulator, SIGNAL(triggered()), this, SLOT(play()));
  connect(ui.actionPauseEmulator, SIGNAL(triggered()), this, SLOT(pause()));
  //thread
  connect(&_emuThread, SIGNAL(screenInvalidated()), this, SLOT(screenInvalidated()), Qt::BlockingQueuedConnection);
  connect(&_emuThread, SIGNAL(threadExit()), this, SLOT(threadExit()), Qt::BlockingQueuedConnection);

  initPallette();
  initBitmap();

  _scale = 5;

  // set up timer
  _timer = new QTimer(this);
  connect(_timer, SIGNAL(timeout()), this, SLOT(timerTick()));
  _timer->start(100);

  // key event handler
  QApplication::instance()->installEventFilter(this);
}

void Chip8::initPallette()
{
  _pallette.clear();
  _pallette.append(QRgb(0xFF000000));
  _pallette.append(QRgb(0xFFFFFFFF));
  for (int dummy = 2; dummy < 256; dummy++)
    _pallette.append(QRgb(0xFF800000));

}

void Chip8::initBitmap()
{
  _scr = QImage(64, 32, QImage::Format::Format_Indexed8);
  _scr.setColorTable(_pallette);
}

Chip8::~Chip8()
{
  // key event handler
  QApplication::instance()->removeEventFilter(this);
}

// painting

void Chip8::paintEvent(QPaintEvent *event)
{
  QPainter pnt(this);
  QRect target(10, 80, _scale*_scr.width(), _scale*_scr.height());
  pnt.drawImage(target, _scr);
}

//slot
void Chip8::screenInvalidated()
{
  _scr = QImage(
    _emu.SCR.Data(),
    _emu.SCR.Width(), _emu.SCR.Height(), _emu.SCR.BytesPerLine(),
    QImage::Format::Format_Indexed8
    );
  _scr.setColorTable(_pallette);

  update();
}

// key event filter and key handling

void Chip8::registerKey(bool down, int key)
{
}

bool Chip8::eventFilter(QObject * /*object*/, QEvent *event){

  if (event->type() == QEvent::KeyPress) {
    QKeyEvent *keyEvent = static_cast<QKeyEvent *>(event);
    registerKey(true, keyEvent->key());
    return true;
  }

  if (event->type() == QEvent::KeyRelease) {
    QKeyEvent *keyEvent = static_cast<QKeyEvent *>(event);
    registerKey(false, keyEvent->key());
    return true;
  }

  return false;
}

// thread stuff

void Chip8::timerTick()
{
  _emu.DecreaseTimers();
}

void Chip8::openGame()
{
  QString fileName;

  fileName = QFileDialog::getOpenFileName(this, tr("Open File"),
    "",
    tr("Chip 8 Files (*.ch8);;All files (*.*)")
    );

  if (!fileName.isEmpty())
  {
    QFile progFile(fileName);
    if (progFile.open(QIODevice::ReadOnly))
    {
      QByteArray progData = progFile.readAll();
      _emu.Init(Emulator::CHIP8);
      _emu.storeProgram((uint8_t*)(progData.data()), progData.size());
    }
  }

}

void Chip8::zoomIn()
{
}

void Chip8::zoomOut()
{
}

void Chip8::play()
{
  if (!_emuThread.isRunning()) {
    _emuThread.start();
    UpdateUI();
    // thread
  }
}

void Chip8::pause()
{
  if (_emuThread.isRunning()) {
    _emuThread.stop();
    UpdateUI();
  }
}

void Chip8::UpdateUI()
{
  ui.actionStartEmulator->setChecked(_emuThread.isRunning());
  ui.actionPauseEmulator->setChecked(_emuThread.isFinished());
}<commit_msg>Set timer to correct value for 60Hz.<commit_after>#include "chip8.h"
#include <QFileDialog>
#include <qbitmap.h>
#include <qpainter.h>
#include <qtimer.h>
#include <qkeyevent>
#include <qkeysequence>

Chip8::Chip8(QWidget *parent)
: QMainWindow( parent ),
  _emuThread( &_emu )
{
  ui.setupUi(this);

  connect(ui.actionOpenGame, SIGNAL(triggered()), this, SLOT(openGame()));
  connect(ui.actionZoomIn, SIGNAL(triggered()), this, SLOT(zoomIn()));
  connect(ui.actionZoomOut, SIGNAL(triggered()), this, SLOT(zoomOut()));
  // toolbar
  connect(ui.actionStartEmulator, SIGNAL(triggered()), this, SLOT(play()));
  connect(ui.actionPauseEmulator, SIGNAL(triggered()), this, SLOT(pause()));
  //thread
  connect(&_emuThread, SIGNAL(screenInvalidated()), this, SLOT(screenInvalidated()), Qt::BlockingQueuedConnection);
  connect(&_emuThread, SIGNAL(threadExit()), this, SLOT(threadExit()), Qt::BlockingQueuedConnection);

  initPallette();
  initBitmap();

  _scale = 5;

  // set up timer
  _timer = new QTimer(this);
  connect(_timer, SIGNAL(timeout()), this, SLOT(timerTick()));
  _timer->start(17);

  // key event handler
  QApplication::instance()->installEventFilter(this);
}

void Chip8::initPallette()
{
  _pallette.clear();
  _pallette.append(QRgb(0xFF000000));
  _pallette.append(QRgb(0xFFFFFFFF));
  for (int dummy = 2; dummy < 256; dummy++)
    _pallette.append(QRgb(0xFF800000));

}

void Chip8::initBitmap()
{
  _scr = QImage(64, 32, QImage::Format::Format_Indexed8);
  _scr.setColorTable(_pallette);
}

Chip8::~Chip8()
{
  // key event handler
  QApplication::instance()->removeEventFilter(this);
}

// painting

void Chip8::paintEvent(QPaintEvent *event)
{
  QPainter pnt(this);
  QRect target(10, 80, _scale*_scr.width(), _scale*_scr.height());
  pnt.drawImage(target, _scr);
}

//slot
void Chip8::screenInvalidated()
{
  _scr = QImage(
    _emu.SCR.Data(),
    _emu.SCR.Width(), _emu.SCR.Height(), _emu.SCR.BytesPerLine(),
    QImage::Format::Format_Indexed8
    );
  _scr.setColorTable(_pallette);

  update();
}

// key event filter and key handling

void Chip8::registerKey(bool down, int key)
{
}

bool Chip8::eventFilter(QObject * /*object*/, QEvent *event){

  if (event->type() == QEvent::KeyPress) {
    QKeyEvent *keyEvent = static_cast<QKeyEvent *>(event);
    registerKey(true, keyEvent->key());
    return true;
  }

  if (event->type() == QEvent::KeyRelease) {
    QKeyEvent *keyEvent = static_cast<QKeyEvent *>(event);
    registerKey(false, keyEvent->key());
    return true;
  }

  return false;
}

// thread stuff

void Chip8::timerTick()
{
  _emu.DecreaseTimers();
}

void Chip8::openGame()
{
  QString fileName;

  fileName = QFileDialog::getOpenFileName(this, tr("Open File"),
    "",
    tr("Chip 8 Files (*.ch8);;All files (*.*)")
    );

  if (!fileName.isEmpty())
  {
    QFile progFile(fileName);
    if (progFile.open(QIODevice::ReadOnly))
    {
      QByteArray progData = progFile.readAll();
      _emu.Init(Emulator::CHIP8);
      _emu.storeProgram((uint8_t*)(progData.data()), progData.size());
    }
  }

}

void Chip8::zoomIn()
{
}

void Chip8::zoomOut()
{
}

void Chip8::play()
{
  if (!_emuThread.isRunning()) {
    _emuThread.start();
    UpdateUI();
    // thread
  }
}

void Chip8::pause()
{
  if (_emuThread.isRunning()) {
    _emuThread.stop();
    UpdateUI();
  }
}

void Chip8::UpdateUI()
{
  ui.actionStartEmulator->setChecked(_emuThread.isRunning());
  ui.actionPauseEmulator->setChecked(_emuThread.isFinished());
}<|endoftext|>


<commit_before>// vim: set ai shiftwidth=4 softtabstop=4 tabstop=4
#include "probability.h"

#include "logger_factory.h"
#include "plugin_factory.h"

#include "fetcher.h"
#include "writer.h"

#include "ensemble.h"
#include "util.h"

#include <boost/lexical_cast.hpp>
#include <boost/thread.hpp>

#include <algorithm>
#include <exception>
#include <iostream>

#include <math.h>

#include <stdio.h>

namespace himan
{
namespace plugin
{

static std::mutex singleFileWriteMutex;

static const std::string kClassName = "himan::plugin::probability";

/// @brief Used for calculating wind vector magnitude
static inline double Magnitude(double u, double v) { return sqrt(u * u + v * v); }

probability::probability()
{
	itsClearTextFormula = "???";
	itsCudaEnabledCalculation = false;
	itsLogger = logger_factory::Instance()->GetLog("probability");

	itsEnsembleSize = 0;
	itsMaximumMissingForecasts = 0;
	itsUseNormalizedResult = false;
}

probability::~probability() {}

/// @brief Configuration reading
/// @param outParamConfig is modified to have information about the threshold value and input parameters
/// @returns param to be pushed in the calculatedParams vector in Process()
static param GetConfigurationParameter(const std::string& name, const std::shared_ptr<const plugin_configuration> conf,
                                       param_configuration* outParamConfig)
{
	long univId = 0;

	if (conf->ParameterExists(name))
	{
		const auto paramOpts = conf->GetParameterOptions(name);

		param param1;
		param param2;

		// NOTE These are plugin dependent
		for (auto&& p : paramOpts)
		{
			if (p.first == "univ_id")
			{
				univId = boost::lexical_cast<long>(p.second);
			}
			else if (p.first == "threshold")
			{
				outParamConfig->threshold = boost::lexical_cast<double>(p.second);
			}
			else if (p.first == "input_param1")
			{
				param1.Name(p.second);
			}
			else if (p.first == "input_param2")
			{
				param2.Name(p.second);
			}
			else if (p.first == "input_id1")
			{
				param1.UnivId(boost::lexical_cast<long>(p.second));
			}
			else if (p.first == "input_id2")
			{
				param2.UnivId(boost::lexical_cast<long>(p.second));
			}
			else
			{
				std::cout << "?? " << p.first << " " << p.second << "\n";
			}
		}

		if (param1.Name() == "XX-X" || param1.UnivId() == static_cast<long>(kHPMissingInt))
		{
			throw std::runtime_error("probability : configuration error:: input parameter not specified for '" + name +
			                         "'");
		}
		outParamConfig->parameter = param1;

		// NOTE param2 is used only with wind calculation at the moment
		if (param2.Name() != "XX-X" && param2.UnivId() != static_cast<long>(kHPMissingInt))
		{
			outParamConfig->parameter2 = param2;
		}
	}
	else
	{
		throw std::runtime_error(
		    "probability : configuration error:: requested parameter doesn't exist in the configuration file '" + name +
		    "'");
	}

	assert(univId != 0);

	return param(name, univId);
}

void probability::Process(const std::shared_ptr<const plugin_configuration> conf)
{
	Init(conf);

	//
	// 1. Parse json configuration specific to this plugin
	//

	// Most of the plugin operates by the configuration in the json file.
	// Here we collect all the parameters we want to calculate, and the parameters that are used for the calculation.
	// If the configuration is invalid we will bail out asap!

	// Get the number of forecasts this ensemble has from plugin configuration
	if (itsConfiguration->Exists("ensemble_size"))
	{
		const int ensembleSize = boost::lexical_cast<int>(itsConfiguration->GetValue("ensemble_size"));
		if (ensembleSize <= 0)
		{
			throw std::runtime_error(ClassName() + " invalid ensemble_size in plugin configuration");
		}
		itsEnsembleSize = ensembleSize;
	}
	else
	{
		throw std::runtime_error(ClassName() + " ensemble_size not specified in plugin configuration");
	}

	// Find out whether we want probabilities in [0,1] range or [0,100] range
	if (itsConfiguration->Exists("normalized_results"))
	{
		const std::string useNormalized = itsConfiguration->GetValue("normalized_results");

		itsUseNormalizedResult = (useNormalized == "true") ? true : false;
	}
	else
	{
		// default to [0,100] for compatibility
		itsUseNormalizedResult = false;
		itsLogger->Info(
		    "'normalized_results' not found from the configuration, results will be written in [0,100] range");
	}

	// Maximum number of missing forecasts for an ensemble
	if (itsConfiguration->Exists("max_missing_forecasts"))
	{
		const int maxMissingForecasts = boost::lexical_cast<int>(itsConfiguration->GetValue("max_missing_forecasts"));
		if (maxMissingForecasts < 0)
		{
			throw std::runtime_error(ClassName() +
			                         " invalid max_missing_forecasts value specified in plugin configuration");
		}
		itsMaximumMissingForecasts = maxMissingForecasts;
	}

	//
	// 2. Setup input and output parameters from the json configuration.
	//    `calculatedParams' will hold the output parameter, it's inputs,
	//    and the 'threshold' value.
	//
	params calculatedParams;

	int targetInfoIndex = 0;
	const auto& names = conf->GetParameterNames();
	for (const std::string& name : names)
	{
		param_configuration config;

		config.targetInfoIndex = targetInfoIndex;
		config.output.Name(name);

		param p = GetConfigurationParameter(name, conf, &config);

		if (p.Name() == "" || p.UnivId() == 0)
		{
			throw std::runtime_error(ClassName() + "Misconfigured parameter definition in JSON");
		}

		itsParamConfigurations.push_back(config);
		calculatedParams.push_back(p);
		targetInfoIndex++;
	}

	SetParams(calculatedParams);

	// NOTE
	// NOTE We circumvent the usual Himan Start => Run => RunAll / RunTimeDimension - control flow structure
	// NOTE

	//
	// 3. Process parameters in LIFO order by spawning threads for each parameter.
	//	  (We don't divide timesteps between threads)
	//
	boost::thread_group g;
	auto paramConfigurations = itsParamConfigurations;

	// Set iterators at this stage to avoid invalid indexing when loading from auxiliary files
	itsInfo->First();

	while (!paramConfigurations.empty())
	{
		for (int threadIdx = 0; threadIdx < itsThreadCount; threadIdx++)
		{
			if (paramConfigurations.empty())
			{
				break;
			}

			const auto& pc = paramConfigurations.back();
			paramConfigurations.pop_back();

			g.add_thread(new boost::thread(&himan::plugin::probability::Calculate, this, threadIdx, pc));
		}

		g.join_all();
	}

	Finish();
}

static void CalculateNormal(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                            const int infoIndex, const bool normalized, ensemble& ens);

static void CalculateNegative(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                              const int infoIndex, const bool normalized, ensemble& ens);

static void CalculateWind(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                          const int infoIndex, const bool normalized, ensemble& ens1, ensemble& ens2);

void probability::Calculate(uint16_t threadIndex, const param_configuration& pc)
{
	info myTargetInfo = *itsInfo;

	auto threadedLogger =
	    logger_factory::Instance()->GetLog("probabilityThread # " + boost::lexical_cast<std::string>(threadIndex));
	const std::string deviceType = "CPU";

	const double threshold = pc.threshold;
	const int infoIndex = pc.targetInfoIndex;
	const int ensembleSize = itsEnsembleSize;
	const bool normalized = itsUseNormalizedResult;

	// used with all calculations
	ensemble ens1(pc.parameter, ensembleSize);
	ens1.MaximumMissingForecasts(itsMaximumMissingForecasts);

	// used with wind calculation
	ensemble ens2;

	if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
	{
		// Wind
		ens2 = ensemble(pc.parameter2, ensembleSize);
		ens2.MaximumMissingForecasts(itsMaximumMissingForecasts);
	}

	myTargetInfo.First();

	// NOTE we only loop through the time steps here
	do
	{
		threadedLogger->Info("Calculating " + pc.output.Name() + " time " +
		                     static_cast<std::string>(myTargetInfo.Time().ValidDateTime()) + " threshold '" +
		                     boost::lexical_cast<std::string>(threshold) + "' infoIndex " +
		                     boost::lexical_cast<std::string>(infoIndex));

		//
		// Setup input data, data fetching
		//
		ens1.Fetch(itsConfiguration, myTargetInfo.Time(), myTargetInfo.Level());

		if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
		{
			ens2.Fetch(itsConfiguration, myTargetInfo.Time(), myTargetInfo.Level());
		}

		// Output memory
		if (itsConfiguration->UseDynamicMemoryAllocation())
		{
			AllocateMemory(myTargetInfo);
		}
		assert(myTargetInfo.Data().Size() > 0);

		//
		// Choose the correct calculation function for this parameter and do the actual calculation
		//
		// Unfortunately we use both the input parameter name and output parameter name for doing this.
		//
		if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
		{
			CalculateWind(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized, ens1,
			              ens2);
		}
		else if (pc.output.Name() == "PROB-TC-0" || pc.output.Name() == "PROB-TC-1" ||
		         pc.output.Name() == "PROB-TC-2" || pc.output.Name() == "PROB-TC-3" || pc.output.Name() == "PROB-TC-4")
		{
			CalculateNegative(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized,
			                  ens1);
		}
		else
		{
			CalculateNormal(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized, ens1);
		}

		if (itsConfiguration->StatisticsEnabled())
		{
			itsConfiguration->Statistics()->AddToMissingCount(myTargetInfo.Data().MissingCount());
			itsConfiguration->Statistics()->AddToValueCount(myTargetInfo.Data().Size());
		}

		// Finally write this parameter out
		WriteToFile(myTargetInfo, pc.targetInfoIndex);

	} while (myTargetInfo.NextTime());

	threadedLogger->Info("[" + deviceType + "] Missing values: " +
	                     boost::lexical_cast<std::string>(myTargetInfo.Data().MissingCount()) + "/" +
	                     boost::lexical_cast<std::string>(myTargetInfo.Data().Size()));
}

// Usually himan writes all the parameters out on a call to WriteToFile, but probability calculates
// each parameter separately in separate threads so this makes no sense (writing all out if we've only
// calculated one parameter)
void probability::WriteToFile(const info& targetInfo, const size_t targetInfoIndex, write_options opts)
{
	auto writer = GET_PLUGIN(writer);

	writer->WriteOptions(opts);

	auto info = targetInfo;

	info.ResetParam();
	info.ParamIndex(targetInfoIndex);

	if (itsConfiguration->FileWriteOption() == kDatabase || itsConfiguration->FileWriteOption() == kMultipleFiles)
	{
		writer->ToFile(info, itsConfiguration);
	}
	else
	{
		std::lock_guard<std::mutex> lock(singleFileWriteMutex);

		writer->ToFile(info, itsConfiguration, itsConfiguration->ConfigurationFile());
	}

	if (itsConfiguration->UseDynamicMemoryAllocation())
	{
		DeallocateMemory(info);
	}
}

void CalculateWind(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold, const int infoIndex,
                   const bool normalized, ensemble& ens1, ensemble& ens2)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens1.ResetLocation();
	ens2.ResetLocation();

	while (targetInfo->NextLocation() && ens1.NextLocation() && ens2.NextLocation())
	{
		const auto ensembleValuesU = ens1.Values();
		const auto ensembleValuesV = ens2.Values();

		const size_t ensembleSize = ensembleValuesU.size();
		if (ensembleSize != ensembleValuesV.size())
		{
			throw std::runtime_error(kClassName +
			                         "::CalculateWind(): U and V ensembles are of different size, aborting");
		}

		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto&& uv : zip_range(ensembleValuesU, ensembleValuesV))
		{
			if (Magnitude(uv.get<0>(), uv.get<1>()) >= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

void CalculateNegative(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                       const int infoIndex, const bool normalized, ensemble& ens)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens.ResetLocation();

	while (targetInfo->NextLocation() && ens.NextLocation())
	{
		const auto ensembleValues = ens.Values();
		const size_t ensembleSize = ensembleValues.size();
		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto& x : ensembleValues)
		{
			if (x <= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

void CalculateNormal(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                     const int infoIndex, const bool normalized, ensemble& ens)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens.ResetLocation();

	while (targetInfo->NextLocation() && ens.NextLocation())
	{
		const auto ensembleValues = ens.Values();
		const size_t ensembleSize = ensembleValues.size();
		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto& x : ensembleValues)
		{
			if (x >= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

}  // plugin

}  // namespace
<commit_msg>removed unnecessary include from probability<commit_after>// vim: set ai shiftwidth=4 softtabstop=4 tabstop=4
#include "probability.h"

#include "logger_factory.h"
#include "plugin_factory.h"

#include "fetcher.h"
#include "writer.h"

#include "ensemble.h"
#include "util.h"

#include <boost/lexical_cast.hpp>
#include <boost/thread.hpp>

#include <algorithm>
#include <exception>
#include <iostream>

#include <math.h>

namespace himan
{
namespace plugin
{

static std::mutex singleFileWriteMutex;

static const std::string kClassName = "himan::plugin::probability";

/// @brief Used for calculating wind vector magnitude
static inline double Magnitude(double u, double v) { return sqrt(u * u + v * v); }

probability::probability()
{
	itsClearTextFormula = "???";
	itsCudaEnabledCalculation = false;
	itsLogger = logger_factory::Instance()->GetLog("probability");

	itsEnsembleSize = 0;
	itsMaximumMissingForecasts = 0;
	itsUseNormalizedResult = false;
}

probability::~probability() {}

/// @brief Configuration reading
/// @param outParamConfig is modified to have information about the threshold value and input parameters
/// @returns param to be pushed in the calculatedParams vector in Process()
static param GetConfigurationParameter(const std::string& name, const std::shared_ptr<const plugin_configuration> conf,
                                       param_configuration* outParamConfig)
{
	long univId = 0;

	if (conf->ParameterExists(name))
	{
		const auto paramOpts = conf->GetParameterOptions(name);

		param param1;
		param param2;

		// NOTE These are plugin dependent
		for (auto&& p : paramOpts)
		{
			if (p.first == "univ_id")
			{
				univId = boost::lexical_cast<long>(p.second);
			}
			else if (p.first == "threshold")
			{
				outParamConfig->threshold = boost::lexical_cast<double>(p.second);
			}
			else if (p.first == "input_param1")
			{
				param1.Name(p.second);
			}
			else if (p.first == "input_param2")
			{
				param2.Name(p.second);
			}
			else if (p.first == "input_id1")
			{
				param1.UnivId(boost::lexical_cast<long>(p.second));
			}
			else if (p.first == "input_id2")
			{
				param2.UnivId(boost::lexical_cast<long>(p.second));
			}
			else
			{
				std::cout << "?? " << p.first << " " << p.second << "\n";
			}
		}

		if (param1.Name() == "XX-X" || param1.UnivId() == static_cast<long>(kHPMissingInt))
		{
			throw std::runtime_error("probability : configuration error:: input parameter not specified for '" + name +
			                         "'");
		}
		outParamConfig->parameter = param1;

		// NOTE param2 is used only with wind calculation at the moment
		if (param2.Name() != "XX-X" && param2.UnivId() != static_cast<long>(kHPMissingInt))
		{
			outParamConfig->parameter2 = param2;
		}
	}
	else
	{
		throw std::runtime_error(
		    "probability : configuration error:: requested parameter doesn't exist in the configuration file '" + name +
		    "'");
	}

	assert(univId != 0);

	return param(name, univId);
}

void probability::Process(const std::shared_ptr<const plugin_configuration> conf)
{
	Init(conf);

	//
	// 1. Parse json configuration specific to this plugin
	//

	// Most of the plugin operates by the configuration in the json file.
	// Here we collect all the parameters we want to calculate, and the parameters that are used for the calculation.
	// If the configuration is invalid we will bail out asap!

	// Get the number of forecasts this ensemble has from plugin configuration
	if (itsConfiguration->Exists("ensemble_size"))
	{
		const int ensembleSize = boost::lexical_cast<int>(itsConfiguration->GetValue("ensemble_size"));
		if (ensembleSize <= 0)
		{
			throw std::runtime_error(ClassName() + " invalid ensemble_size in plugin configuration");
		}
		itsEnsembleSize = ensembleSize;
	}
	else
	{
		throw std::runtime_error(ClassName() + " ensemble_size not specified in plugin configuration");
	}

	// Find out whether we want probabilities in [0,1] range or [0,100] range
	if (itsConfiguration->Exists("normalized_results"))
	{
		const std::string useNormalized = itsConfiguration->GetValue("normalized_results");

		itsUseNormalizedResult = (useNormalized == "true") ? true : false;
	}
	else
	{
		// default to [0,100] for compatibility
		itsUseNormalizedResult = false;
		itsLogger->Info(
		    "'normalized_results' not found from the configuration, results will be written in [0,100] range");
	}

	// Maximum number of missing forecasts for an ensemble
	if (itsConfiguration->Exists("max_missing_forecasts"))
	{
		const int maxMissingForecasts = boost::lexical_cast<int>(itsConfiguration->GetValue("max_missing_forecasts"));
		if (maxMissingForecasts < 0)
		{
			throw std::runtime_error(ClassName() +
			                         " invalid max_missing_forecasts value specified in plugin configuration");
		}
		itsMaximumMissingForecasts = maxMissingForecasts;
	}

	//
	// 2. Setup input and output parameters from the json configuration.
	//    `calculatedParams' will hold the output parameter, it's inputs,
	//    and the 'threshold' value.
	//
	params calculatedParams;

	int targetInfoIndex = 0;
	const auto& names = conf->GetParameterNames();
	for (const std::string& name : names)
	{
		param_configuration config;

		config.targetInfoIndex = targetInfoIndex;
		config.output.Name(name);

		param p = GetConfigurationParameter(name, conf, &config);

		if (p.Name() == "" || p.UnivId() == 0)
		{
			throw std::runtime_error(ClassName() + "Misconfigured parameter definition in JSON");
		}

		itsParamConfigurations.push_back(config);
		calculatedParams.push_back(p);
		targetInfoIndex++;
	}

	SetParams(calculatedParams);

	// NOTE
	// NOTE We circumvent the usual Himan Start => Run => RunAll / RunTimeDimension - control flow structure
	// NOTE

	//
	// 3. Process parameters in LIFO order by spawning threads for each parameter.
	//	  (We don't divide timesteps between threads)
	//
	boost::thread_group g;
	auto paramConfigurations = itsParamConfigurations;

	// Set iterators at this stage to avoid invalid indexing when loading from auxiliary files
	itsInfo->First();

	while (!paramConfigurations.empty())
	{
		for (int threadIdx = 0; threadIdx < itsThreadCount; threadIdx++)
		{
			if (paramConfigurations.empty())
			{
				break;
			}

			const auto& pc = paramConfigurations.back();
			paramConfigurations.pop_back();

			g.add_thread(new boost::thread(&himan::plugin::probability::Calculate, this, threadIdx, pc));
		}

		g.join_all();
	}

	Finish();
}

static void CalculateNormal(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                            const int infoIndex, const bool normalized, ensemble& ens);

static void CalculateNegative(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                              const int infoIndex, const bool normalized, ensemble& ens);

static void CalculateWind(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                          const int infoIndex, const bool normalized, ensemble& ens1, ensemble& ens2);

void probability::Calculate(uint16_t threadIndex, const param_configuration& pc)
{
	info myTargetInfo = *itsInfo;

	auto threadedLogger =
	    logger_factory::Instance()->GetLog("probabilityThread # " + boost::lexical_cast<std::string>(threadIndex));
	const std::string deviceType = "CPU";

	const double threshold = pc.threshold;
	const int infoIndex = pc.targetInfoIndex;
	const int ensembleSize = itsEnsembleSize;
	const bool normalized = itsUseNormalizedResult;

	// used with all calculations
	ensemble ens1(pc.parameter, ensembleSize);
	ens1.MaximumMissingForecasts(itsMaximumMissingForecasts);

	// used with wind calculation
	ensemble ens2;

	if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
	{
		// Wind
		ens2 = ensemble(pc.parameter2, ensembleSize);
		ens2.MaximumMissingForecasts(itsMaximumMissingForecasts);
	}

	myTargetInfo.First();

	// NOTE we only loop through the time steps here
	do
	{
		threadedLogger->Info("Calculating " + pc.output.Name() + " time " +
		                     static_cast<std::string>(myTargetInfo.Time().ValidDateTime()) + " threshold '" +
		                     boost::lexical_cast<std::string>(threshold) + "' infoIndex " +
		                     boost::lexical_cast<std::string>(infoIndex));

		//
		// Setup input data, data fetching
		//
		ens1.Fetch(itsConfiguration, myTargetInfo.Time(), myTargetInfo.Level());

		if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
		{
			ens2.Fetch(itsConfiguration, myTargetInfo.Time(), myTargetInfo.Level());
		}

		// Output memory
		if (itsConfiguration->UseDynamicMemoryAllocation())
		{
			AllocateMemory(myTargetInfo);
		}
		assert(myTargetInfo.Data().Size() > 0);

		//
		// Choose the correct calculation function for this parameter and do the actual calculation
		//
		// Unfortunately we use both the input parameter name and output parameter name for doing this.
		//
		if (pc.parameter.Name() == "U-MS" || pc.parameter.Name() == "V-MS")
		{
			CalculateWind(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized, ens1,
			              ens2);
		}
		else if (pc.output.Name() == "PROB-TC-0" || pc.output.Name() == "PROB-TC-1" ||
		         pc.output.Name() == "PROB-TC-2" || pc.output.Name() == "PROB-TC-3" || pc.output.Name() == "PROB-TC-4")
		{
			CalculateNegative(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized,
			                  ens1);
		}
		else
		{
			CalculateNormal(std::make_shared<info>(myTargetInfo), threadIndex, threshold, infoIndex, normalized, ens1);
		}

		if (itsConfiguration->StatisticsEnabled())
		{
			itsConfiguration->Statistics()->AddToMissingCount(myTargetInfo.Data().MissingCount());
			itsConfiguration->Statistics()->AddToValueCount(myTargetInfo.Data().Size());
		}

		// Finally write this parameter out
		WriteToFile(myTargetInfo, pc.targetInfoIndex);

	} while (myTargetInfo.NextTime());

	threadedLogger->Info("[" + deviceType + "] Missing values: " +
	                     boost::lexical_cast<std::string>(myTargetInfo.Data().MissingCount()) + "/" +
	                     boost::lexical_cast<std::string>(myTargetInfo.Data().Size()));
}

// Usually himan writes all the parameters out on a call to WriteToFile, but probability calculates
// each parameter separately in separate threads so this makes no sense (writing all out if we've only
// calculated one parameter)
void probability::WriteToFile(const info& targetInfo, const size_t targetInfoIndex, write_options opts)
{
	auto writer = GET_PLUGIN(writer);

	writer->WriteOptions(opts);

	auto info = targetInfo;

	info.ResetParam();
	info.ParamIndex(targetInfoIndex);

	if (itsConfiguration->FileWriteOption() == kDatabase || itsConfiguration->FileWriteOption() == kMultipleFiles)
	{
		writer->ToFile(info, itsConfiguration);
	}
	else
	{
		std::lock_guard<std::mutex> lock(singleFileWriteMutex);

		writer->ToFile(info, itsConfiguration, itsConfiguration->ConfigurationFile());
	}

	if (itsConfiguration->UseDynamicMemoryAllocation())
	{
		DeallocateMemory(info);
	}
}

void CalculateWind(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold, const int infoIndex,
                   const bool normalized, ensemble& ens1, ensemble& ens2)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens1.ResetLocation();
	ens2.ResetLocation();

	while (targetInfo->NextLocation() && ens1.NextLocation() && ens2.NextLocation())
	{
		const auto ensembleValuesU = ens1.Values();
		const auto ensembleValuesV = ens2.Values();

		const size_t ensembleSize = ensembleValuesU.size();
		if (ensembleSize != ensembleValuesV.size())
		{
			throw std::runtime_error(kClassName +
			                         "::CalculateWind(): U and V ensembles are of different size, aborting");
		}

		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto&& uv : zip_range(ensembleValuesU, ensembleValuesV))
		{
			if (Magnitude(uv.get<0>(), uv.get<1>()) >= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

void CalculateNegative(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                       const int infoIndex, const bool normalized, ensemble& ens)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens.ResetLocation();

	while (targetInfo->NextLocation() && ens.NextLocation())
	{
		const auto ensembleValues = ens.Values();
		const size_t ensembleSize = ensembleValues.size();
		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto& x : ensembleValues)
		{
			if (x <= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

void CalculateNormal(std::shared_ptr<info> targetInfo, uint16_t threadIndex, const double threshold,
                     const int infoIndex, const bool normalized, ensemble& ens)
{
	targetInfo->ParamIndex(infoIndex);
	targetInfo->ResetLocation();
	ens.ResetLocation();

	while (targetInfo->NextLocation() && ens.NextLocation())
	{
		const auto ensembleValues = ens.Values();
		const size_t ensembleSize = ensembleValues.size();
		const double invN =
		    normalized ? 1.0 / static_cast<double>(ensembleSize) : 100.0 / static_cast<double>(ensembleSize);

		double probability = 0.0;

		for (const auto& x : ensembleValues)
		{
			if (x >= threshold)
			{
				probability += invN;
			}
		}
		targetInfo->Value(probability);
	}
}

}  // plugin

}  // namespace
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/path_service.h"
#include "base/string_split.h"
#include "base/string_util.h"
#include "base/threading/sequenced_worker_pool.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/page_cycler/page_cycler.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/ui/browser_list.h"
#include "chrome/common/chrome_notification_types.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/testing_profile.h"
#include "chrome/test/base/ui_test_utils.h"
#include "content/public/browser/notification_observer.h"
#include "content/public/browser/notification_registrar.h"
#include "content/public/browser/notification_service.h"
#include "content/public/common/content_switches.h"
#include "content/public/common/url_constants.h"
#include "googleurl/src/gurl.h"

// Basic PageCyclerBrowserTest structure; used in testing most of PageCycler's
// functionality.
class PageCyclerBrowserTest : public content::NotificationObserver,
                              public InProcessBrowserTest {
 public:
  PageCyclerBrowserTest() : page_cycler_(NULL) {
  }

  virtual ~PageCyclerBrowserTest() {
  }

  // Initialize file paths within a temporary directory; this should be
  // empty and nonexistent.
  virtual void InitFilePaths(FilePath temp_path) {
    temp_path_ = temp_path;
    urls_file_ = temp_path.AppendASCII("urls_file");
    errors_file_ = temp_path.AppendASCII("errors");
    stats_file_ = temp_path.AppendASCII("stats");

    ASSERT_FALSE(file_util::PathExists(urls_file_));
    ASSERT_FALSE(file_util::PathExists(errors_file_));
    ASSERT_FALSE(file_util::PathExists(stats_file_));
  }

  // Initialize a PageCycler using either the base fields, or using provided
  // ones.
  void InitPageCycler() {
    page_cycler_ = new PageCycler(browser(), urls_file());
    page_cycler_->set_errors_file(errors_file());
    page_cycler_->set_stats_file(stats_file());
  }

  void InitPageCycler(FilePath urls_file,
                      FilePath errors_file,
                      FilePath stats_file) {
    page_cycler_ = new PageCycler(browser(), urls_file);
    page_cycler_->set_errors_file(errors_file);
    page_cycler_->set_stats_file(stats_file);
  }

  void RegisterForNotifications() {
    registrar_.Add(this, chrome::NOTIFICATION_BROWSER_CLOSED,
                   content::NotificationService::AllSources());
  }

  // Get a collection of basic urls which are stored in the test directory.
  // NOTE: |test_server| must be started first!
  std::vector<GURL> GetURLs() {
    std::vector<GURL> urls;
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_html.html"));
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_js.html"));
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_css.html"));
    return urls;
  }

  // Read the errors file, and generate a vector of error strings.
  std::vector<std::string> GetErrorsFromFile() {
    std::string error_file_contents;
    CHECK(file_util::ReadFileToString(errors_file_,
                                      &error_file_contents));
    if (error_file_contents[error_file_contents.size() - 1] == '\n')
      error_file_contents.resize(error_file_contents.size() - 1);

    std::vector<std::string> errors;
    base::SplitString(error_file_contents, '\n', &errors);

    return errors;
  }

  // Convert a vector of GURLs into a newline-separated string, ready to be
  // written to the urls file for PageCycler to use.
  std::string GetStringFromURLs(std::vector<GURL> urls) {
    std::string urls_string;
    for (std::vector<GURL>::const_iterator iter = urls.begin();
         iter != urls.end(); ++iter)
      urls_string.append(iter->spec() + "\n");
    return urls_string;
  }

  // content::NotificationObserver.
  virtual void Observe(int type,
                       const content::NotificationSource& source,
                       const content::NotificationDetails& details) {
    switch (type) {
      case chrome::NOTIFICATION_BROWSER_CLOSED:
        MessageLoop::current()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
        break;
      default:
        NOTREACHED();
        break;
    }
  }

  FilePath urls_file() { return urls_file_; }
  FilePath errors_file() { return errors_file_; }
  FilePath stats_file() { return stats_file_; }
  PageCycler* page_cycler() { return page_cycler_; }

 protected:
  FilePath temp_path_;
  FilePath urls_file_;
  FilePath errors_file_;
  FilePath stats_file_;
  PageCycler* page_cycler_;
  content::NotificationRegistrar registrar_;
};

// Structure used for testing PageCycler's ability to playback a series of
// URLs given a cache directory.
class PageCyclerCachedBrowserTest : public PageCyclerBrowserTest {
 public:
  // For a cached test, we use the provided user data directory from the test
  // directory.
  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    InProcessBrowserTest::SetUpCommandLine(command_line);

    FilePath test_dir;
    ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &test_dir));
    test_dir = test_dir.AppendASCII("page_cycler");

    FilePath source_data_dir = test_dir.AppendASCII("cached_data_dir");
    CHECK(file_util::PathExists(source_data_dir));

    CHECK(user_data_dir_.CreateUniqueTempDir());

    FilePath dest_data_dir =
        user_data_dir_.path().AppendASCII("cached_data_dir");
    CHECK(!file_util::PathExists(dest_data_dir));

    CHECK(file_util::CopyDirectory(source_data_dir,
                                   user_data_dir_.path(),
                                   true));  // recursive.
    CHECK(file_util::PathExists(dest_data_dir));

    command_line->AppendSwitchPath(switches::kUserDataDir,
                                   dest_data_dir);
    command_line->AppendSwitch(switches::kPlaybackMode);
  }

  // Initialize the file paths to use the UserDataDir's urls file, instead
  // of one to be written.
  virtual void InitFilePaths(FilePath temp_path) OVERRIDE {
    urls_file_ = user_data_dir_.path().AppendASCII("cached_data_dir")
                                      .AppendASCII("urls");
    errors_file_ = temp_path.AppendASCII("errors");
    stats_file_ = temp_path.AppendASCII("stats");

    ASSERT_TRUE(file_util::PathExists(urls_file_));
    ASSERT_FALSE(file_util::PathExists(errors_file_));
    ASSERT_FALSE(file_util::PathExists(stats_file_));
  }

 private:
  // The directory storing the copy of the UserDataDir.
  ScopedTempDir user_data_dir_;
};

// Sanity check; iterate through a series of URLs and make sure there are no
// errors.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, BasicTest) {
  const size_t kNumIterations = 3;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::string urls_string = GetStringFromURLs(GetURLs());;

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_FALSE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));
}

// Test to make sure that PageCycler will recognize unvisitable URLs, and will
// handle them appropriately.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, UnvisitableURL) {
  const size_t kNumIterations = 3;
  const char kFakeURL[] = "http://www.pleasenoonehavethisurlanytimeinthenext"
                          "century.com/gibberish";
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::vector<GURL> urls = GetURLs();
  urls.push_back(GURL(kFakeURL));
  std::string urls_string = GetStringFromURLs(urls);

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();

  size_t num_errors = errors.size();
  ASSERT_EQ(kNumIterations, num_errors);

  // Check that each error message contains the fake URL (i.e., that it wasn't
  // from a valid URL, and that the fake URL was caught each time).
  for (std::vector<std::string>::const_iterator iter = errors.begin();
       iter != errors.end(); ++iter) {
    ASSERT_NE(iter->find(kFakeURL), std::string::npos);
  }
}

// Test that PageCycler will remove an invalid URL prior to running.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, InvalidURL) {
  const size_t kNumIterations = 1;
  const char kBadURL[] = "notarealurl";

  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::string urls_string = GetStringFromURLs(GetURLs());
  urls_string.append(kBadURL).append("\n");

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error = "Omitting invalid URL: ";
  expected_error.append(kBadURL).append(".");

  ASSERT_FALSE(errors[0].compare(expected_error));
}

// Test that PageCycler will remove a Chrome Error URL prior to running.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, ChromeErrorURL) {
  const size_t kNumIterations = 1;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::vector<GURL> urls = GetURLs();
  urls.push_back(GURL(content::kUnreachableWebDataURL));
  std::string urls_string = GetStringFromURLs(urls);

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error = "Chrome error pages are not allowed as urls. "
                               "Omitting url: ";
  expected_error.append(content::kUnreachableWebDataURL).append(".");

  ASSERT_FALSE(errors[0].compare(expected_error));
}

// Test that PageCycler will visit all the urls from a cache directory
// successfully while in playback mode.
IN_PROC_BROWSER_TEST_F(PageCyclerCachedBrowserTest, PlaybackMode) {
  const size_t kNumIterations = 1;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  InitPageCycler();

  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(stats_file()));
  ASSERT_FALSE(file_util::PathExists(errors_file()));
}

// Test that PageCycler will have a cache miss if a URL is missing from the
// cache directory while in playback mode.
IN_PROC_BROWSER_TEST_F(PageCyclerCachedBrowserTest, URLNotInCache) {
  const size_t kNumIterations = 1;
  const char kCacheMissURL[] = "http://www.images.google.com/";

  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  std::string urls_string;
  ASSERT_TRUE(file_util::ReadFileToString(urls_file(),
                                          &urls_string));

  urls_string.append("\n").append(kCacheMissURL);
  FilePath new_urls_file = temp.path().AppendASCII("urls");
  ASSERT_FALSE(file_util::PathExists(new_urls_file));

  ASSERT_TRUE(file_util::WriteFile(new_urls_file, urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler(new_urls_file, errors_file(), stats_file());
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error;
  expected_error.append("Failed to load the page at: ")
      .append(kCacheMissURL)
      .append(": The requested entry was not found in the cache.");

  ASSERT_FALSE(errors[0].compare(expected_error));
}
<commit_msg>Disable two page cycler tests that fail on Chrome OS.<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/path_service.h"
#include "base/string_split.h"
#include "base/string_util.h"
#include "base/threading/sequenced_worker_pool.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/page_cycler/page_cycler.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/ui/browser_list.h"
#include "chrome/common/chrome_notification_types.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/testing_profile.h"
#include "chrome/test/base/ui_test_utils.h"
#include "content/public/browser/notification_observer.h"
#include "content/public/browser/notification_registrar.h"
#include "content/public/browser/notification_service.h"
#include "content/public/common/content_switches.h"
#include "content/public/common/url_constants.h"
#include "googleurl/src/gurl.h"

// Basic PageCyclerBrowserTest structure; used in testing most of PageCycler's
// functionality.
class PageCyclerBrowserTest : public content::NotificationObserver,
                              public InProcessBrowserTest {
 public:
  PageCyclerBrowserTest() : page_cycler_(NULL) {
  }

  virtual ~PageCyclerBrowserTest() {
  }

  // Initialize file paths within a temporary directory; this should be
  // empty and nonexistent.
  virtual void InitFilePaths(FilePath temp_path) {
    temp_path_ = temp_path;
    urls_file_ = temp_path.AppendASCII("urls_file");
    errors_file_ = temp_path.AppendASCII("errors");
    stats_file_ = temp_path.AppendASCII("stats");

    ASSERT_FALSE(file_util::PathExists(urls_file_));
    ASSERT_FALSE(file_util::PathExists(errors_file_));
    ASSERT_FALSE(file_util::PathExists(stats_file_));
  }

  // Initialize a PageCycler using either the base fields, or using provided
  // ones.
  void InitPageCycler() {
    page_cycler_ = new PageCycler(browser(), urls_file());
    page_cycler_->set_errors_file(errors_file());
    page_cycler_->set_stats_file(stats_file());
  }

  void InitPageCycler(FilePath urls_file,
                      FilePath errors_file,
                      FilePath stats_file) {
    page_cycler_ = new PageCycler(browser(), urls_file);
    page_cycler_->set_errors_file(errors_file);
    page_cycler_->set_stats_file(stats_file);
  }

  void RegisterForNotifications() {
    registrar_.Add(this, chrome::NOTIFICATION_BROWSER_CLOSED,
                   content::NotificationService::AllSources());
  }

  // Get a collection of basic urls which are stored in the test directory.
  // NOTE: |test_server| must be started first!
  std::vector<GURL> GetURLs() {
    std::vector<GURL> urls;
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_html.html"));
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_js.html"));
    urls.push_back(test_server()->GetURL("files/page_cycler/basic_css.html"));
    return urls;
  }

  // Read the errors file, and generate a vector of error strings.
  std::vector<std::string> GetErrorsFromFile() {
    std::string error_file_contents;
    CHECK(file_util::ReadFileToString(errors_file_,
                                      &error_file_contents));
    if (error_file_contents[error_file_contents.size() - 1] == '\n')
      error_file_contents.resize(error_file_contents.size() - 1);

    std::vector<std::string> errors;
    base::SplitString(error_file_contents, '\n', &errors);

    return errors;
  }

  // Convert a vector of GURLs into a newline-separated string, ready to be
  // written to the urls file for PageCycler to use.
  std::string GetStringFromURLs(std::vector<GURL> urls) {
    std::string urls_string;
    for (std::vector<GURL>::const_iterator iter = urls.begin();
         iter != urls.end(); ++iter)
      urls_string.append(iter->spec() + "\n");
    return urls_string;
  }

  // content::NotificationObserver.
  virtual void Observe(int type,
                       const content::NotificationSource& source,
                       const content::NotificationDetails& details) {
    switch (type) {
      case chrome::NOTIFICATION_BROWSER_CLOSED:
        MessageLoop::current()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
        break;
      default:
        NOTREACHED();
        break;
    }
  }

  FilePath urls_file() { return urls_file_; }
  FilePath errors_file() { return errors_file_; }
  FilePath stats_file() { return stats_file_; }
  PageCycler* page_cycler() { return page_cycler_; }

 protected:
  FilePath temp_path_;
  FilePath urls_file_;
  FilePath errors_file_;
  FilePath stats_file_;
  PageCycler* page_cycler_;
  content::NotificationRegistrar registrar_;
};

// Structure used for testing PageCycler's ability to playback a series of
// URLs given a cache directory.
class PageCyclerCachedBrowserTest : public PageCyclerBrowserTest {
 public:
  // For a cached test, we use the provided user data directory from the test
  // directory.
  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    InProcessBrowserTest::SetUpCommandLine(command_line);

    FilePath test_dir;
    ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &test_dir));
    test_dir = test_dir.AppendASCII("page_cycler");

    FilePath source_data_dir = test_dir.AppendASCII("cached_data_dir");
    CHECK(file_util::PathExists(source_data_dir));

    CHECK(user_data_dir_.CreateUniqueTempDir());

    FilePath dest_data_dir =
        user_data_dir_.path().AppendASCII("cached_data_dir");
    CHECK(!file_util::PathExists(dest_data_dir));

    CHECK(file_util::CopyDirectory(source_data_dir,
                                   user_data_dir_.path(),
                                   true));  // recursive.
    CHECK(file_util::PathExists(dest_data_dir));

    command_line->AppendSwitchPath(switches::kUserDataDir,
                                   dest_data_dir);
    command_line->AppendSwitch(switches::kPlaybackMode);
  }

  // Initialize the file paths to use the UserDataDir's urls file, instead
  // of one to be written.
  virtual void InitFilePaths(FilePath temp_path) OVERRIDE {
    urls_file_ = user_data_dir_.path().AppendASCII("cached_data_dir")
                                      .AppendASCII("urls");
    errors_file_ = temp_path.AppendASCII("errors");
    stats_file_ = temp_path.AppendASCII("stats");

    ASSERT_TRUE(file_util::PathExists(urls_file_));
    ASSERT_FALSE(file_util::PathExists(errors_file_));
    ASSERT_FALSE(file_util::PathExists(stats_file_));
  }

 private:
  // The directory storing the copy of the UserDataDir.
  ScopedTempDir user_data_dir_;
};

// Sanity check; iterate through a series of URLs and make sure there are no
// errors.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, BasicTest) {
  const size_t kNumIterations = 3;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::string urls_string = GetStringFromURLs(GetURLs());;

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_FALSE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));
}

// Test to make sure that PageCycler will recognize unvisitable URLs, and will
// handle them appropriately.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, UnvisitableURL) {
  const size_t kNumIterations = 3;
  const char kFakeURL[] = "http://www.pleasenoonehavethisurlanytimeinthenext"
                          "century.com/gibberish";
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::vector<GURL> urls = GetURLs();
  urls.push_back(GURL(kFakeURL));
  std::string urls_string = GetStringFromURLs(urls);

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();

  size_t num_errors = errors.size();
  ASSERT_EQ(kNumIterations, num_errors);

  // Check that each error message contains the fake URL (i.e., that it wasn't
  // from a valid URL, and that the fake URL was caught each time).
  for (std::vector<std::string>::const_iterator iter = errors.begin();
       iter != errors.end(); ++iter) {
    ASSERT_NE(iter->find(kFakeURL), std::string::npos);
  }
}

// Test that PageCycler will remove an invalid URL prior to running.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, InvalidURL) {
  const size_t kNumIterations = 1;
  const char kBadURL[] = "notarealurl";

  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::string urls_string = GetStringFromURLs(GetURLs());
  urls_string.append(kBadURL).append("\n");

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error = "Omitting invalid URL: ";
  expected_error.append(kBadURL).append(".");

  ASSERT_FALSE(errors[0].compare(expected_error));
}

// Test that PageCycler will remove a Chrome Error URL prior to running.
IN_PROC_BROWSER_TEST_F(PageCyclerBrowserTest, ChromeErrorURL) {
  const size_t kNumIterations = 1;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  ASSERT_TRUE(test_server()->Start());

  std::vector<GURL> urls = GetURLs();
  urls.push_back(GURL(content::kUnreachableWebDataURL));
  std::string urls_string = GetStringFromURLs(urls);

  ASSERT_TRUE(file_util::WriteFile(urls_file(), urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler();
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error = "Chrome error pages are not allowed as urls. "
                               "Omitting url: ";
  expected_error.append(content::kUnreachableWebDataURL).append(".");

  ASSERT_FALSE(errors[0].compare(expected_error));
}

// Test that PageCycler will visit all the urls from a cache directory
// successfully while in playback mode.
#if defined(OS_CHROMEOS)
// TODO(rdevlin.cronin): Perhaps page cycler isn't completely implemented on
// ChromeOS?
#else
IN_PROC_BROWSER_TEST_F(PageCyclerCachedBrowserTest, PlaybackMode) {
  const size_t kNumIterations = 1;
  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  InitPageCycler();

  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(stats_file()));
  ASSERT_FALSE(file_util::PathExists(errors_file()));
}

// Test that PageCycler will have a cache miss if a URL is missing from the
// cache directory while in playback mode.
IN_PROC_BROWSER_TEST_F(PageCyclerCachedBrowserTest, URLNotInCache) {
  const size_t kNumIterations = 1;
  const char kCacheMissURL[] = "http://www.images.google.com/";

  ScopedTempDir temp;
  ASSERT_TRUE(temp.CreateUniqueTempDir());

  RegisterForNotifications();
  InitFilePaths(temp.path());

  std::string urls_string;
  ASSERT_TRUE(file_util::ReadFileToString(urls_file(),
                                          &urls_string));

  urls_string.append("\n").append(kCacheMissURL);
  FilePath new_urls_file = temp.path().AppendASCII("urls");
  ASSERT_FALSE(file_util::PathExists(new_urls_file));

  ASSERT_TRUE(file_util::WriteFile(new_urls_file, urls_string.c_str(),
                                   urls_string.size()));

  InitPageCycler(new_urls_file, errors_file(), stats_file());
  page_cycler()->Run(kNumIterations);

  ui_test_utils::RunMessageLoop();
  ASSERT_TRUE(file_util::PathExists(errors_file()));
  ASSERT_TRUE(file_util::PathExists(stats_file()));

  std::vector<std::string> errors = GetErrorsFromFile();
  ASSERT_EQ(1u, errors.size());

  std::string expected_error;
  expected_error.append("Failed to load the page at: ")
      .append(kCacheMissURL)
      .append(": The requested entry was not found in the cache.");

  ASSERT_FALSE(errors[0].compare(expected_error));
}
#endif
<|endoftext|>


<commit_before><commit_msg>Perform password import from Keychain in the muted mode.<commit_after><|endoftext|>


<commit_before>/*
 * Copyright (c) 2021 Travis Geiseblrecht
 *
 * Use of this source code is governed by a MIT-style
 * license that can be found in the LICENSE file or at
 * https://opensource.org/licenses/MIT
 */
#include "resource.h"

#include <dev/bus/pci.h>
#include <lk/err.h>
#include <lk/trace.h>
#include <stdint.h>

#define LOCAL_TRACE 0

namespace pci {

resource_range &resource_allocator::type_to_range(pci_resource_type type, bool prefetchable) {
    if (prefetchable) {
        switch (type) {
            case PCI_RESOURCE_MMIO_RANGE:
                return ranges_.mmio_prefetchable;
            case PCI_RESOURCE_MMIO64_RANGE:
                return ranges_.mmio64_prefetchable;
            default:
                DEBUG_ASSERT_MSG(0, "uhandled prefetchable pci resource type %d\n", type);
        }
    } else {
        switch (type) {
            case PCI_RESOURCE_IO_RANGE:
                return ranges_.io;
            case PCI_RESOURCE_MMIO_RANGE:
                return ranges_.mmio;
            case PCI_RESOURCE_MMIO64_RANGE:
                return ranges_.mmio64;
            default:
                DEBUG_ASSERT_MSG(0, "uhandled pci resource type %d\n", type);
        }
    }
}

status_t resource_allocator::set_range(const resource_range &range, bool prefetchable) {
    LTRACEF("range base %#llx size %#llx type %d prefetchable %d\n", range.base, range.size, range.type, prefetchable);
    type_to_range(range.type, prefetchable) = range;
    return NO_ERROR;
}

status_t resource_allocator::allocate_mmio(bool can_be_64bit, bool prefetchable, uint64_t size, uint8_t align, uint64_t *out) {
    pci_resource_type type;

    for (;;) {
        if (can_be_64bit) {
            type = PCI_RESOURCE_MMIO64_RANGE;
        } else {
            type = PCI_RESOURCE_MMIO_RANGE;
        }

        auto &range = type_to_range(type, prefetchable);

        LTRACEF("range base %#llx size %#llx. request size %#llx align %u prefetchable %d can_be_64 %d\n",
                range.base, range.size, size, align, prefetchable, can_be_64bit);

        // TODO: make sure align is honored or removed
        if (range.base + size <= range.base + range.size) {
            *out = range.base;
            range.base += size;
            range.size -= size;
            return NO_ERROR;
        }

        if (can_be_64bit) {
            can_be_64bit = false;
            continue;
        }
        break;
    }

    return ERR_NO_RESOURCES;
}

status_t resource_allocator::allocate_io(uint32_t size, uint8_t align, uint32_t *out) {
    auto &range = type_to_range(PCI_RESOURCE_IO_RANGE, false);

    LTRACEF("range base %#llx size %#llx. request size %#x align %u\n", range.base, range.size, size, align);

    // TODO: make sure align is honored or removed
    if (range.base + size <= range.base + range.size) {
        *out = range.base;
        range.base += size;
        range.size -= size;
        return NO_ERROR;
    }

    return ERR_NO_RESOURCES;
}

} // namespace pci
<commit_msg>[bus][pci] Make sure there's a fallback range in release mode<commit_after>/*
 * Copyright (c) 2021 Travis Geiseblrecht
 *
 * Use of this source code is governed by a MIT-style
 * license that can be found in the LICENSE file or at
 * https://opensource.org/licenses/MIT
 */
#include "resource.h"

#include <dev/bus/pci.h>
#include <lk/err.h>
#include <lk/trace.h>
#include <stdint.h>

#define LOCAL_TRACE 0

namespace pci {

resource_range &resource_allocator::type_to_range(pci_resource_type type, bool prefetchable) {
    if (prefetchable) {
        switch (type) {
            case PCI_RESOURCE_MMIO_RANGE:
                return ranges_.mmio_prefetchable;
            case PCI_RESOURCE_MMIO64_RANGE:
                return ranges_.mmio64_prefetchable;
            default:
                DEBUG_ASSERT_MSG(0, "uhandled prefetchable pci resource type %d\n", type);
        }
    } else {
        switch (type) {
            case PCI_RESOURCE_IO_RANGE:
                return ranges_.io;
            case PCI_RESOURCE_MMIO_RANGE:
                return ranges_.mmio;
            case PCI_RESOURCE_MMIO64_RANGE:
                return ranges_.mmio64;
            default:
                DEBUG_ASSERT_MSG(0, "uhandled pci resource type %d\n", type);
        }
    }
    static resource_range zero = {};
    return zero;
}

status_t resource_allocator::set_range(const resource_range &range, bool prefetchable) {
    LTRACEF("range base %#llx size %#llx type %d prefetchable %d\n", range.base, range.size, range.type, prefetchable);
    type_to_range(range.type, prefetchable) = range;
    return NO_ERROR;
}

status_t resource_allocator::allocate_mmio(bool can_be_64bit, bool prefetchable, uint64_t size, uint8_t align, uint64_t *out) {
    pci_resource_type type;

    for (;;) {
        if (can_be_64bit) {
            type = PCI_RESOURCE_MMIO64_RANGE;
        } else {
            type = PCI_RESOURCE_MMIO_RANGE;
        }

        auto &range = type_to_range(type, prefetchable);

        LTRACEF("range base %#llx size %#llx. request size %#llx align %u prefetchable %d can_be_64 %d\n",
                range.base, range.size, size, align, prefetchable, can_be_64bit);

        // TODO: make sure align is honored or removed
        if (range.base + size <= range.base + range.size) {
            *out = range.base;
            range.base += size;
            range.size -= size;
            return NO_ERROR;
        }

        if (can_be_64bit) {
            can_be_64bit = false;
            continue;
        }
        break;
    }

    return ERR_NO_RESOURCES;
}

status_t resource_allocator::allocate_io(uint32_t size, uint8_t align, uint32_t *out) {
    auto &range = type_to_range(PCI_RESOURCE_IO_RANGE, false);

    LTRACEF("range base %#llx size %#llx. request size %#x align %u\n", range.base, range.size, size, align);

    // TODO: make sure align is honored or removed
    if (range.base + size <= range.base + range.size) {
        *out = range.base;
        range.base += size;
        range.size -= size;
        return NO_ERROR;
    }

    return ERR_NO_RESOURCES;
}

} // namespace pci
<|endoftext|>