<commit_before>// ===============================
//  PC-BSD REST API Server
// Available under the 3-clause BSD License
// Written by: Ken Moore <ken@pcbsd.org> 2015-2016
// =================================
#include "Dispatcher.h"

#include "globals.h"

// ================================
//  DProcess Class (Internal)
// ================================
DProcess::DProcess(QObject *parent) : QProcess(parent){
    //Setup the process
    bool notify = false;
    uptimer = new QTimer(this);
      uptimer->setSingleShot(true);
      uptimer->setInterval(1000); //1 second intervals
    connect(uptimer, SIGNAL(timeout()), this, SLOT(emitUpdate()) );
    this->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
    this->setProcessChannelMode(QProcess::MergedChannels);
    connect(this, SIGNAL(readyReadStandardOutput()), this, SLOT(updateLog()) );
}

DProcess::~DProcess(){
  if( this->state()!=QProcess::NotRunning ){
    this->terminate();
  }
}

void DProcess::startProc(){
  cmds.removeAll(""); //make sure no empty commands
  if(cmds.isEmpty()){
    proclog.insert("state","finished");
    proclog.insert("time_finished", QDateTime::currentDateTime().toString(Qt::ISODate));
    proclog.remove("current_cmd");
    emit ProcFinished(ID, proclog);
    return;
  }
  if(proclog.isEmpty()){
    //first cmd started
    proclog.insert("time_started", QDateTime::currentDateTime().toString(Qt::ISODate));
    proclog.insert("cmd_list",QJsonArray::fromStringList(cmds));
    proclog.insert("process_id",ID);
    proclog.insert("state","running");
    //rawcmds = cmds; rawcmds.prepend(cCmd);
    //setup internal connections
    connect(this, SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(cmdFinished(int, QProcess::ExitStatus)) );
    connect(this, SIGNAL(error(QProcess::ProcessError)), this, SLOT(cmdError(QProcess::ProcessError)) );
  }
  cCmd = cmds.takeFirst();
  success = false; //not finished yet
  proclog.insert("current_cmd",cCmd);
  //qDebug() << "Proc Starting:" << ID << cmd;
  this->start(cCmd);
}

bool DProcess::isRunning(){
  return (this->state()!=QProcess::NotRunning);
}

bool DProcess::isDone(){
  return (!this->isRunning() && !proclog.isEmpty());
}

QJsonObject DProcess::getProcLog(){
  //Now return the current version of the log
  return proclog;
}

void DProcess::cmdError(QProcess::ProcessError err){
  //qDebug() << "Process Error:" << err;
  cmdFinished(-1, QProcess::NormalExit);
}

void DProcess::cmdFinished(int ret, QProcess::ExitStatus status){
  if(uptimer->isActive()){ uptimer->stop(); } //use the finished signal instead
  //determine success/failure
  success = (status==QProcess::NormalExit && ret==0);
  //update the log before starting another command
  proclog.insert(cCmd, proclog.value(cCmd).toString().append(this->readAllStandardOutput()) );
  proclog.insert("return_codes/"+cCmd, QString::number(ret));

  //Now run any additional commands
  //qDebug() << "Proc Finished:" << ID << success << proclog;
  if(success && !cmds.isEmpty()){
    emit ProcUpdate(ID, proclog);
    startProc();
  }else{
    proclog.insert("state","finished");
    proclog.remove("current_cmd");
    proclog.insert("time_finished", QDateTime::currentDateTime().toString(Qt::ISODate));
    emit ProcFinished(ID, proclog);
  }
}

void DProcess::updateLog(){
  proclog.insert(cCmd, proclog.value(cCmd).toString().append(this->readAllStandardOutput()) );
  if(!uptimer->isActive()){ uptimer->start(); }
}

void DProcess::emitUpdate(){
  emit ProcUpdate(ID, proclog);
}

// ================================
// Dispatcher Class
// ================================
Dispatcher::Dispatcher(){
  qRegisterMetaType<Dispatcher::PROC_QUEUE>("Dispatcher::PROC_QUEUE");
  //connect(this, SIGNAL(mkprocs(Dispatcher::PROC_QUEUE, DProcess*)), this, SLOT(mkProcs(Dispatcher::PROC_QUEUE, DProcess*)) );
}

Dispatcher::~Dispatcher(){

}

void Dispatcher::start(QString queuefile){
  //Setup connections here (in case it was moved to different thread after creation)
  connect(this, SIGNAL(mkprocs(Dispatcher::PROC_QUEUE, DProcess*)), this, SLOT(mkProcs(Dispatcher::PROC_QUEUE, DProcess*)) );
  connect(this, SIGNAL(checkProcs()), this, SLOT(checkQueues()) );
  //load any previously-unrun processes
  // TO DO
}

void Dispatcher::stop(){
  //save any currently-unrun processes for next time the server starts
  // TO DO
}

//Overloaded Main Calling Functions (single command, or multiple in-order commands)
DProcess* Dispatcher::queueProcess(QString ID, QString cmd){
  return queueProcess(NO_QUEUE, ID, QStringList() << cmd);
}
DProcess* Dispatcher::queueProcess(QString ID, QStringList cmds){
  return queueProcess(NO_QUEUE, ID, cmds);
}
DProcess* Dispatcher::queueProcess(Dispatcher::PROC_QUEUE queue, QString ID, QString cmd){
  return queueProcess(queue, ID, QStringList() << cmd);
}
DProcess* Dispatcher::queueProcess(Dispatcher::PROC_QUEUE queue, QString ID, QStringList cmds){
  //This is the primary queueProcess() function - all the overloads end up here to do the actual work
  //For multi-threading, need to emit a signal/slot for this action (object creations need to be in same thread as parent)
  //qDebug() << "Queue Process:" << queue << ID << cmds;
  DProcess *P = createProcess(ID, cmds);
  emit mkprocs(queue, P);
  return P;
}

// === PRIVATE ===
//Simplification routine for setting up a process
DProcess* Dispatcher::createProcess(QString ID, QStringList cmds){
  DProcess *P = new DProcess();
    P->moveToThread(this->thread());
    P->cmds = cmds;
    P->ID = ID;
  return P;
}

// === PRIVATE SLOTS ===
void Dispatcher::mkProcs(Dispatcher::PROC_QUEUE queue, DProcess *P){
  //qDebug() << "mkProcs()";
  //P->moveToThread(this->thread());
  connect(P, SIGNAL(ProcFinished(QString, QJsonObject)), this, SLOT(ProcFinished(QString, QJsonObject)) );
  connect(P, SIGNAL(ProcUpdate(QString, QJsonObject)), this, SLOT(ProcUpdated(QString, QJsonObject)) );
  QList<DProcess*> list;
  if(!HASH.contains(queue)){ HASH.insert(queue, list); } //insert an empty list
  HASH[queue] << P; //add this proc to the end of the list
  emit checkProcs();
}

void Dispatcher::ProcFinished(QString ID, QJsonObject log){
  //Find the process with this ID and close it down (with proper events)
  //qDebug() << " - Got Proc Finished Signal:" << ID;
  LogManager::log(LogManager::DISPATCH, log);
  //First emit any subsystem-specific event, falling back on the raw log
  QJsonObject ev = CreateDispatcherEventNotification(ID,log);
  if(!ev.isEmpty()){
    emit DispatchEvent(ev);
  }else{
    emit DispatchEvent(log);
  }

  emit checkProcs();
}

void Dispatcher::ProcUpdated(QString ID, QJsonObject log){
  //See if this needs to generate an event
  QJsonObject ev = CreateDispatcherEventNotification(ID,log);
  if(!ev.isEmpty()){
    emit DispatchEvent(ev);
  }
}

void Dispatcher::CheckQueues(){
for(int i=0; i<enum_length; i++){
    PROC_QUEUE queue = static_cast<PROC_QUEUE>(i);
    if(HASH.contains(queue)){
      QList<DProcess*> list = HASH[queue];
      for(int j=0; j<list.length(); j++){
	if(j>0 && queue!=NO_QUEUE){ break; } //done with this - only first item in these queues should run at a time
	if( !list[j]->isRunning() ){
	  if(list[j]->isDone() ){
	    list.takeAt(j)->deleteLater();
	    j--;
	  }else{
	    //Need to start this one - has not run yet
	    //qDebug() << "Call Start Proc:" << list[j]->ID;
	    emit DispatchStarting(list[j]->ID);
	    list[j]->startProc();
	    //QTimer::singleShot(0,list[j], SLOT(startProc()) );
	  }
	}
      } //end loop over list
    }

  } //end loop over queue types
}
<commit_msg>Fix up the dispatcher queueing system (for real this time). Now the processes will properly get removed when they are finished, and the next process started up.<commit_after>// ===============================
//  PC-BSD REST API Server
// Available under the 3-clause BSD License
// Written by: Ken Moore <ken@pcbsd.org> 2015-2016
// =================================
#include "Dispatcher.h"

#include "globals.h"

// ================================
//  DProcess Class (Internal)
// ================================
DProcess::DProcess(QObject *parent) : QProcess(parent){
    //Setup the process
    bool notify = false;
    uptimer = new QTimer(this);
      uptimer->setSingleShot(true);
      uptimer->setInterval(1000); //1 second intervals
    connect(uptimer, SIGNAL(timeout()), this, SLOT(emitUpdate()) );
    this->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
    this->setProcessChannelMode(QProcess::MergedChannels);
    connect(this, SIGNAL(readyReadStandardOutput()), this, SLOT(updateLog()) );
}

DProcess::~DProcess(){
  if( this->state()!=QProcess::NotRunning ){
    this->terminate();
  }
}

void DProcess::startProc(){
  cmds.removeAll(""); //make sure no empty commands
  if(cmds.isEmpty()){
    proclog.insert("state","finished");
    proclog.insert("time_finished", QDateTime::currentDateTime().toString(Qt::ISODate));
    proclog.remove("current_cmd");
    emit ProcFinished(ID, proclog);
    return;
  }
  if(proclog.isEmpty()){
    //first cmd started
    proclog.insert("time_started", QDateTime::currentDateTime().toString(Qt::ISODate));
    proclog.insert("cmd_list",QJsonArray::fromStringList(cmds));
    proclog.insert("process_id",ID);
    proclog.insert("state","running");
    //rawcmds = cmds; rawcmds.prepend(cCmd);
    //setup internal connections
    connect(this, SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(cmdFinished(int, QProcess::ExitStatus)) );
    connect(this, SIGNAL(error(QProcess::ProcessError)), this, SLOT(cmdError(QProcess::ProcessError)) );
  }
  cCmd = cmds.takeFirst();
  success = false; //not finished yet
  proclog.insert("current_cmd",cCmd);
  //qDebug() << "Proc Starting:" << ID << cmd;
  this->start(cCmd);
}

bool DProcess::isRunning(){
  return (this->state()!=QProcess::NotRunning);
}

bool DProcess::isDone(){
  return (!this->isRunning() && !proclog.isEmpty());
}

QJsonObject DProcess::getProcLog(){
  //Now return the current version of the log
  return proclog;
}

void DProcess::cmdError(QProcess::ProcessError err){
  //qDebug() << "Process Error:" << err;
  cmdFinished(-1, QProcess::NormalExit);
}

void DProcess::cmdFinished(int ret, QProcess::ExitStatus status){
  if(uptimer->isActive()){ uptimer->stop(); } //use the finished signal instead
  //determine success/failure
  success = (status==QProcess::NormalExit && ret==0);
  //update the log before starting another command
  proclog.insert(cCmd, proclog.value(cCmd).toString().append(this->readAllStandardOutput()) );
  proclog.insert("return_codes/"+cCmd, QString::number(ret));

  //Now run any additional commands
  //qDebug() << "Proc Finished:" << ID << success << proclog;
  if(success && !cmds.isEmpty()){
    emit ProcUpdate(ID, proclog);
    startProc();
  }else{
    proclog.insert("state","finished");
    proclog.remove("current_cmd");
    proclog.insert("time_finished", QDateTime::currentDateTime().toString(Qt::ISODate));
    emit ProcFinished(ID, proclog);
  }
}

void DProcess::updateLog(){
  proclog.insert(cCmd, proclog.value(cCmd).toString().append(this->readAllStandardOutput()) );
  if(!uptimer->isActive()){ uptimer->start(); }
}

void DProcess::emitUpdate(){
  emit ProcUpdate(ID, proclog);
}

// ================================
// Dispatcher Class
// ================================
Dispatcher::Dispatcher(){
  qRegisterMetaType<Dispatcher::PROC_QUEUE>("Dispatcher::PROC_QUEUE");
  connect(this, SIGNAL(mkprocs(Dispatcher::PROC_QUEUE, DProcess*)), this, SLOT(mkProcs(Dispatcher::PROC_QUEUE, DProcess*)) );
  connect(this, SIGNAL(checkProcs()), this, SLOT(CheckQueues()) );
}

Dispatcher::~Dispatcher(){

}

void Dispatcher::start(QString queuefile){
  //Setup connections here (in case it was moved to different thread after creation)
  //connect(this, SIGNAL(mkprocs(Dispatcher::PROC_QUEUE, DProcess*)), this, SLOT(mkProcs(Dispatcher::PROC_QUEUE, DProcess*)) );
  //connect(this, SIGNAL(checkProcs()), this, SLOT(checkQueues()) );
  //load any previously-unrun processes
  // TO DO
}

void Dispatcher::stop(){
  //save any currently-unrun processes for next time the server starts
  // TO DO
}

//Overloaded Main Calling Functions (single command, or multiple in-order commands)
DProcess* Dispatcher::queueProcess(QString ID, QString cmd){
  return queueProcess(NO_QUEUE, ID, QStringList() << cmd);
}
DProcess* Dispatcher::queueProcess(QString ID, QStringList cmds){
  return queueProcess(NO_QUEUE, ID, cmds);
}
DProcess* Dispatcher::queueProcess(Dispatcher::PROC_QUEUE queue, QString ID, QString cmd){
  return queueProcess(queue, ID, QStringList() << cmd);
}
DProcess* Dispatcher::queueProcess(Dispatcher::PROC_QUEUE queue, QString ID, QStringList cmds){
  //This is the primary queueProcess() function - all the overloads end up here to do the actual work
  //For multi-threading, need to emit a signal/slot for this action (object creations need to be in same thread as parent)
  //qDebug() << "Queue Process:" << queue << ID << cmds;
  DProcess *P = createProcess(ID, cmds);
  this->emit mkprocs(queue, P);
  return P;
}

// === PRIVATE ===
//Simplification routine for setting up a process
DProcess* Dispatcher::createProcess(QString ID, QStringList cmds){
  DProcess *P = new DProcess();
    P->moveToThread(this->thread());
    P->cmds = cmds;
    P->ID = ID;
  return P;
}

// === PRIVATE SLOTS ===
void Dispatcher::mkProcs(Dispatcher::PROC_QUEUE queue, DProcess *P){
  //qDebug() << "mkProcs()";
  //P->moveToThread(this->thread());
  QList<DProcess*> list = HASH.value(queue);
  list << P;
  //qDebug() << " - add to queue:" << queue;
  HASH.insert(queue,list);
  connect(P, SIGNAL(ProcFinished(QString, QJsonObject)), this, SLOT(ProcFinished(QString, QJsonObject)) );
  connect(P, SIGNAL(ProcUpdate(QString, QJsonObject)), this, SLOT(ProcUpdated(QString, QJsonObject)) );
  QTimer::singleShot(30, this, SIGNAL(checkProcs()) );
  //this->emit checkProcs();
}

void Dispatcher::ProcFinished(QString ID, QJsonObject log){
  //Find the process with this ID and close it down (with proper events)
  //qDebug() << " - Got Proc Finished Signal:" << ID;
  LogManager::log(LogManager::DISPATCH, log);
  //First emit any subsystem-specific event, falling back on the raw log
  QJsonObject ev = CreateDispatcherEventNotification(ID,log);
  if(!ev.isEmpty()){
    emit DispatchEvent(ev);
  }else{
    emit DispatchEvent(log);
  }

  QTimer::singleShot(30, this, SIGNAL(checkProcs()) );
  //this->emit checkProcs();
}

void Dispatcher::ProcUpdated(QString ID, QJsonObject log){
  //See if this needs to generate an event
  QJsonObject ev = CreateDispatcherEventNotification(ID,log);
  if(!ev.isEmpty()){
    emit DispatchEvent(ev);
  }
}

void Dispatcher::CheckQueues(){
//qDebug() << "Check Queues...";
for(int i=0; i<enum_length; i++){
    PROC_QUEUE queue = static_cast<PROC_QUEUE>(i);
    //qDebug() << "Got queue:" << queue;
    if(HASH.contains(queue)){
      //qDebug() << "Hash has queue";
      QList<DProcess*> list = HASH[queue];
      //qDebug() << "Length:" << list.length();
      for(int j=0; j<list.length(); j++){
	//qDebug() << "Check Proc:" << list[j]->ID;
	if(j>0 && queue!=NO_QUEUE){ break; } //done with this - only first item in these queues should run at a time
	if( !list[j]->isRunning() ){
	  if(list[j]->isDone() ){
	    //qDebug() << "Remove Finished Proc:" << list[j]->ID;
	    list.takeAt(j)->deleteLater();
	    HASH.insert(queue, list); //replace the list in the hash since it changed
	    j--;
	  }else{
	    //Need to start this one - has not run yet
	    //qDebug() << "Call Start Proc:" << list[j]->ID;
	    emit DispatchStarting(list[j]->ID);
	    list[j]->startProc();
	    //QTimer::singleShot(0,list[j], SLOT(startProc()) );
	  }
	}
      } //end loop over list
    }

  } //end loop over queue types
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/system_wrappers/source/thread_posix.h"

#include <algorithm>

#include <errno.h>
#include <unistd.h>
#ifdef WEBRTC_LINUX
#include <linux/unistd.h>
#include <sched.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <sys/types.h>
#endif

#include "webrtc/base/checks.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/event_wrapper.h"
#include "webrtc/system_wrappers/interface/sleep.h"
#include "webrtc/system_wrappers/interface/trace.h"

namespace webrtc {
namespace {
struct ThreadAttributes {
  ThreadAttributes() { pthread_attr_init(&attr); }
  ~ThreadAttributes() { pthread_attr_destroy(&attr); }
  pthread_attr_t* operator&() { return &attr; }
  pthread_attr_t attr;
};
}  // namespace

int ConvertToSystemPriority(ThreadPriority priority, int min_prio,
                            int max_prio) {
  DCHECK(max_prio - min_prio > 2);
  const int top_prio = max_prio - 1;
  const int low_prio = min_prio + 1;

  switch (priority) {
    case kLowPriority:
      return low_prio;
    case kNormalPriority:
      // The -1 ensures that the kHighPriority is always greater or equal to
      // kNormalPriority.
      return (low_prio + top_prio - 1) / 2;
    case kHighPriority:
      return std::max(top_prio - 2, low_prio);
    case kHighestPriority:
      return std::max(top_prio - 1, low_prio);
    case kRealtimePriority:
      return top_prio;
  }
  DCHECK(false);
  return low_prio;
}

struct ThreadPosix::InitParams {
  InitParams(ThreadPosix* thread)
      : me(thread), started(EventWrapper::Create()) {
  }
  ThreadPosix* me;
  rtc::scoped_ptr<EventWrapper> started;
};

// static
void* ThreadPosix::StartThread(void* param) {
  auto params = static_cast<InitParams*>(param);
  params->me->Run(params);
  return 0;
}

ThreadPosix::ThreadPosix(ThreadRunFunction func, ThreadObj obj,
                         ThreadPriority prio, const char* thread_name)
    : run_function_(func),
      obj_(obj),
      prio_(prio),
      stop_event_(EventWrapper::Create()),
      name_(thread_name ? thread_name : "webrtc"),
      thread_id_(0),
      thread_(0) {
  DCHECK(name_.length() < kThreadMaxNameLength);
}

uint32_t ThreadWrapper::GetThreadId() {
  return rtc::CurrentThreadId();
}

ThreadPosix::~ThreadPosix() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

bool ThreadPosix::Start(unsigned int& thread_id) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_id_) << "Thread already started?";

  ThreadAttributes attr;
  // Set the stack stack size to 1M.
  pthread_attr_setstacksize(&attr, 1024 * 1024);

  InitParams params(this);
  int result = pthread_create(&thread_, &attr, &StartThread, &params);
  if (result != 0)
    return false;

  CHECK_EQ(kEventSignaled, params.started->Wait(WEBRTC_EVENT_INFINITE));
  DCHECK_NE(thread_id_, 0);

  thread_id = thread_id_;

  return true;
}

bool ThreadPosix::Stop() {
  DCHECK(thread_checker_.CalledOnValidThread());
  if (!thread_id_)
    return true;

  stop_event_->Set();
  CHECK_EQ(0, pthread_join(thread_, nullptr));
  thread_id_ = 0;
  stop_event_->Reset();

  return true;
}

void ThreadPosix::Run(ThreadPosix::InitParams* params) {
  thread_id_ = rtc::CurrentThreadId();
  params->started->Set();

  if (!name_.empty()) {
    // Setting the thread name may fail (harmlessly) if running inside a
    // sandbox. Ignore failures if they happen.
#if defined(WEBRTC_LINUX) || defined(WEBRTC_ANDROID)
    prctl(PR_SET_NAME, reinterpret_cast<unsigned long>(name_.c_str()));
#elif defined(WEBRTC_MAC) || defined(WEBRTC_IOS)
    pthread_setname_np(name_.substr(0, 63).c_str());
#endif
  }

#ifdef WEBRTC_THREAD_RR
  const int policy = SCHED_RR;
#else
  const int policy = SCHED_FIFO;
#endif
  const int min_prio = sched_get_priority_min(policy);
  const int max_prio = sched_get_priority_max(policy);
  if ((min_prio == -1) || (max_prio == -1)) {
    WEBRTC_TRACE(kTraceError, kTraceUtility, -1,
                 "unable to retreive min or max priority for threads");
  }

  if (max_prio - min_prio > 2) {
    sched_param param;
    param.sched_priority = ConvertToSystemPriority(prio_, min_prio, max_prio);
    if (pthread_setschedparam(pthread_self(), policy, &param) != 0) {
      WEBRTC_TRACE(
          kTraceError, kTraceUtility, -1, "unable to set thread priority");
    }
  }

  // It's a requirement that for successful thread creation that the run
  // function be called at least once (see RunFunctionIsCalled unit test),
  // so to fullfill that requirement, we use a |do| loop and not |while|.
  do {
    if (!run_function_(obj_))
      break;
  } while (stop_event_->Wait(0) == kEventTimeout);
}

}  // namespace webrtc
<commit_msg>Temporarily change ThreadPosix to CHECK (crash) if we ever spend more than 30 seconds waiting for thread shutdown. There are cases on build bots where it looks like we're hitting this problem, but reproducing locally has been a struggle.<commit_after>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/system_wrappers/source/thread_posix.h"

#include <algorithm>

#include <errno.h>
#include <unistd.h>
#ifdef WEBRTC_LINUX
#include <linux/unistd.h>
#include <sched.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <sys/types.h>
#endif

#include "webrtc/base/checks.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/event_wrapper.h"
#include "webrtc/system_wrappers/interface/sleep.h"
#include "webrtc/system_wrappers/interface/trace.h"

namespace webrtc {
namespace {
struct ThreadAttributes {
  ThreadAttributes() { pthread_attr_init(&attr); }
  ~ThreadAttributes() { pthread_attr_destroy(&attr); }
  pthread_attr_t* operator&() { return &attr; }
  pthread_attr_t attr;
};
}  // namespace

int ConvertToSystemPriority(ThreadPriority priority, int min_prio,
                            int max_prio) {
  DCHECK(max_prio - min_prio > 2);
  const int top_prio = max_prio - 1;
  const int low_prio = min_prio + 1;

  switch (priority) {
    case kLowPriority:
      return low_prio;
    case kNormalPriority:
      // The -1 ensures that the kHighPriority is always greater or equal to
      // kNormalPriority.
      return (low_prio + top_prio - 1) / 2;
    case kHighPriority:
      return std::max(top_prio - 2, low_prio);
    case kHighestPriority:
      return std::max(top_prio - 1, low_prio);
    case kRealtimePriority:
      return top_prio;
  }
  DCHECK(false);
  return low_prio;
}

struct ThreadPosix::InitParams {
  InitParams(ThreadPosix* thread)
      : me(thread), started(EventWrapper::Create()) {
  }
  ThreadPosix* me;
  rtc::scoped_ptr<EventWrapper> started;
};

// static
void* ThreadPosix::StartThread(void* param) {
  auto params = static_cast<InitParams*>(param);
  params->me->Run(params);
  return 0;
}

ThreadPosix::ThreadPosix(ThreadRunFunction func, ThreadObj obj,
                         ThreadPriority prio, const char* thread_name)
    : run_function_(func),
      obj_(obj),
      prio_(prio),
      stop_event_(EventWrapper::Create()),
      name_(thread_name ? thread_name : "webrtc"),
      thread_id_(0),
      thread_(0) {
  DCHECK(name_.length() < kThreadMaxNameLength);
}

uint32_t ThreadWrapper::GetThreadId() {
  return rtc::CurrentThreadId();
}

ThreadPosix::~ThreadPosix() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

bool ThreadPosix::Start(unsigned int& thread_id) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_id_) << "Thread already started?";

  ThreadAttributes attr;
  // Set the stack stack size to 1M.
  pthread_attr_setstacksize(&attr, 1024 * 1024);

  InitParams params(this);
  int result = pthread_create(&thread_, &attr, &StartThread, &params);
  if (result != 0)
    return false;

  // TODO(tommi): Change the timeout back to WEBRTC_EVENT_INFINITE when we've
  // figured out where we're seeing tests hang.
  auto state = params.started->Wait(30 * 1000);
  CHECK_EQ(kEventSignaled, state);
  DCHECK_NE(thread_id_, 0);

  thread_id = thread_id_;

  return true;
}

bool ThreadPosix::Stop() {
  DCHECK(thread_checker_.CalledOnValidThread());
  if (!thread_id_)
    return true;

  stop_event_->Set();
  CHECK_EQ(0, pthread_join(thread_, nullptr));
  thread_id_ = 0;
  stop_event_->Reset();

  return true;
}

void ThreadPosix::Run(ThreadPosix::InitParams* params) {
  thread_id_ = rtc::CurrentThreadId();
  params->started->Set();

  if (!name_.empty()) {
    // Setting the thread name may fail (harmlessly) if running inside a
    // sandbox. Ignore failures if they happen.
#if defined(WEBRTC_LINUX) || defined(WEBRTC_ANDROID)
    prctl(PR_SET_NAME, reinterpret_cast<unsigned long>(name_.c_str()));
#elif defined(WEBRTC_MAC) || defined(WEBRTC_IOS)
    pthread_setname_np(name_.substr(0, 63).c_str());
#endif
  }

#ifdef WEBRTC_THREAD_RR
  const int policy = SCHED_RR;
#else
  const int policy = SCHED_FIFO;
#endif
  const int min_prio = sched_get_priority_min(policy);
  const int max_prio = sched_get_priority_max(policy);
  if ((min_prio == -1) || (max_prio == -1)) {
    WEBRTC_TRACE(kTraceError, kTraceUtility, -1,
                 "unable to retreive min or max priority for threads");
  }

  if (max_prio - min_prio > 2) {
    sched_param param;
    param.sched_priority = ConvertToSystemPriority(prio_, min_prio, max_prio);
    if (pthread_setschedparam(pthread_self(), policy, &param) != 0) {
      WEBRTC_TRACE(
          kTraceError, kTraceUtility, -1, "unable to set thread priority");
    }
  }

  // It's a requirement that for successful thread creation that the run
  // function be called at least once (see RunFunctionIsCalled unit test),
  // so to fullfill that requirement, we use a |do| loop and not |while|.
  do {
    if (!run_function_(obj_))
      break;
  } while (stop_event_->Wait(0) == kEventTimeout);
}

}  // namespace webrtc
<|endoftext|>


<commit_before>#include "GLManager.h"

/**
 * initGL() - create and configure OpenGL resources
 *
 * @return bool true if initialization is a success
 */
bool GLManager::initGL()
{
    // success flag
    bool success = true;

    // generate program
    programID = glCreateProgram();

    // enable z-buffer face culling
    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LEQUAL);

    // transparency
    //glEnable(GL_BLEND);
    //glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

    // enable vsync rate
    if (SDL_GL_SetSwapInterval(1) < 0)
        std::cout << "Swap Interval could not be set!" << std::endl;

    // vertex & fragment shaders
    vertexShader   = 0;
    fragmentShader = 0;
    ShaderLoader loader;

    // load vertex shader
    int failCount = 0;
    while (vertexShader == 0) {
        vertexShader = loader.loadShader( OsType::osConvert("shaders/phongAttenuation.vert"), programID );
        ++failCount;
        if (failCount >= 1000) {
            std::cout << "Loading vertexShader failed!" << std::endl;
            failCount = 0;
            break;
        }
    }

    // load fragment shader
    while (fragmentShader == 0) {
        fragmentShader = loader.loadShader( OsType::osConvert("shaders/phongAttenuation.frag"), programID );
        ++failCount;
        if (failCount >= 1000) {
            std::cout << "Loading fragmentShader failed!" << std::endl;
            failCount = 0;
            break;
        }
    }

    // create VAO
    glGenVertexArrays(1, &VAO);

    // link OpenGL program
    glLinkProgram(programID);

    //Initialize clear color
    glClearColor(0.0, 0.0, 0.0, 1.0f);

    return success;
}

/**
 * initSDL() - create and configure native windowing systemDimensions
 * @return bool true if initialization is a success
 */
bool GLManager::initSDL()
{
    bool success = true;

    //Initialize SDL
    if (SDL_Init(SDL_INIT_EVERYTHING) < 0) {
        printf("SDL could not initialize! SDL Error: %s\n", SDL_GetError());
        success = false;

    //Use OpenGL 3.3 core
    } else {
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 3);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
        SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);
        SDL_GL_SetAttribute(SDL_GL_MULTISAMPLEBUFFERS, 1);
        SDL_GL_SetAttribute(SDL_GL_MULTISAMPLESAMPLES, 8);

        // allow for multisampling of frame buffer
        glEnable(GL_MULTISAMPLE);

        // use native width and height of screen if possible
        SDL_Rect r;
        if (SDL_GetDisplayBounds(0, &r) == 0) {
            this->screenWidth = r.w;// * 0.85f;
            this->screenHeight = r.h;// * 0.85f;
        }

        //Create window
        window = SDL_CreateWindow("opulence v1.0",
                                   SDL_WINDOWPOS_UNDEFINED,
                                   SDL_WINDOWPOS_UNDEFINED,
                                   this->screenWidth, this->screenHeight,
                                   SDL_WINDOW_OPENGL | SDL_WINDOW_FULLSCREEN);

        surface = SDL_GetWindowSurface(window);

        SDL_GL_CreateContext(window);

        // initialize GLEW system independence
        glewExperimental = GL_TRUE;
        GLenum glewError = glewInit();
        if (glewError != GLEW_OK) {
            printf("Error: Glew failed to initialize!\n");
            exit(EXIT_FAILURE);
        }

        //Use Vsync
        if (SDL_GL_SetSwapInterval(1) < 0) {
            printf("Warning: Unable to set VSync! SDL Error: %s\n", SDL_GetError());
        }

    }

    return success;
}

/**
 * shutDown() - free resources maintaining OpenGL context
 *              and free event handlers
 */
void GLManager::shutDown()
{
    //Deallocate program
    glDeleteProgram(programID);
    glDeleteShader(vertexShader);
    glDeleteShader(fragmentShader);

    //Destroy window
    SDL_DestroyWindow(window);

    //Disable text input
    SDL_StopTextInput();

    //Quit SDL subsystems
    SDL_Quit();
}

/**
 * getID() - get OpenGL context's id
 *
 * @return GLuint current OpenGL context
 */
GLuint GLManager::getID()
{
    return programID;
}

/**
 * getVertexShader() - get current vertex shader's handle in the OpenGL context
 *
 * @return GLuint current vertex shader in use
 */
GLuint GLManager::getVertexShader()
{
    return vertexShader;
}

/**
 * getFragmentShader() - get current fragment shader's handle in the OpenGL context
 *
 * @return GLuint current fragment shader in use
 */
GLuint GLManager::getFragmentShader()
{
    return fragmentShader;
}

/**
 * getVAO() - get current vertex array object's handle in the OpenGL context
 *
 * @return GLuint current vertex array object in use
 */
GLuint GLManager::getVAO()
{
    return VAO;
}

GLfloat GLManager::getScreenWidth()
{
    return screenWidth;
}

GLfloat GLManager::getScreenHeight()
{
    return screenHeight;
}

/**
 * getWindow() - get current SDL window containing the current
 *               frame buffer and native windowing system settings
 *
 * @return SDL_Window* current SDL window in use
 */
SDL_Window * GLManager::getWindow()
{
    return window;
}

/**
 * getSurface() - get current SDL window's frame buffer
 *
 * @return SDL_Surface* current SDL window's frame buffer
 */
SDL_Surface * GLManager::getSurface()
{
    return surface;
}

/**
 * setScreenSize() - set resolution of opulence window
 *
 * @params w,h width and height in pixels
 */
void GLManager::setScreenSize(GLfloat w, GLfloat h)
{
    SDL_SetWindowSize(window , w, h);
    screenWidth = w;
    screenHeight = h;
}
<commit_msg>increaed depth buffer to 24 bits<commit_after>#include "GLManager.h"

/**
 * initGL() - create and configure OpenGL resources
 *
 * @return bool true if initialization is a success
 */
bool GLManager::initGL()
{
    // success flag
    bool success = true;

    // generate program
    programID = glCreateProgram();

    // enable z-buffer face culling
    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);

    // transparency
    //glEnable(GL_BLEND);
    //glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

    // enable vsync rate
    if (SDL_GL_SetSwapInterval(1) < 0)
        std::cout << "Swap Interval could not be set!" << std::endl;

    // vertex & fragment shaders
    vertexShader   = 0;
    fragmentShader = 0;
    ShaderLoader loader;

    // load vertex shader
    int failCount = 0;
    while (vertexShader == 0) {
        vertexShader = loader.loadShader( OsType::osConvert("shaders/phongAttenuation.vert"), programID );
        ++failCount;
        if (failCount >= 1000) {
            std::cout << "Loading vertexShader failed!" << std::endl;
            failCount = 0;
            break;
        }
    }

    // load fragment shader
    while (fragmentShader == 0) {
        fragmentShader = loader.loadShader( OsType::osConvert("shaders/phongAttenuation.frag"), programID );
        ++failCount;
        if (failCount >= 1000) {
            std::cout << "Loading fragmentShader failed!" << std::endl;
            failCount = 0;
            break;
        }
    }

    // create VAO
    glGenVertexArrays(1, &VAO);

    // link OpenGL program
    glLinkProgram(programID);

    //Initialize clear color
    glClearColor(0.0, 0.0, 0.0, 1.0f);

    return success;
}

/**
 * initSDL() - create and configure native windowing systemDimensions
 * @return bool true if initialization is a success
 */
bool GLManager::initSDL()
{
    bool success = true;

    //Initialize SDL
    if (SDL_Init(SDL_INIT_EVERYTHING) < 0) {
        printf("SDL could not initialize! SDL Error: %s\n", SDL_GetError());
        success = false;

    //Use OpenGL 3.3 core
    } else {
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 3);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
        SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);
        SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 24);
        SDL_GL_SetAttribute(SDL_GL_MULTISAMPLEBUFFERS, 1);
        SDL_GL_SetAttribute(SDL_GL_MULTISAMPLESAMPLES, 8);

        // allow for multisampling of frame buffer
        glEnable(GL_MULTISAMPLE);

        // use native width and height of screen if possible
        SDL_Rect r;
        if (SDL_GetDisplayBounds(0, &r) == 0) {
            this->screenWidth = r.w;// * 0.85f;
            this->screenHeight = r.h;// * 0.85f;
        }

        //Create window
        window = SDL_CreateWindow("opulence v1.0",
                                   SDL_WINDOWPOS_UNDEFINED,
                                   SDL_WINDOWPOS_UNDEFINED,
                                   this->screenWidth, this->screenHeight,
                                   SDL_WINDOW_OPENGL | SDL_WINDOW_FULLSCREEN);

        surface = SDL_GetWindowSurface(window);

        SDL_GL_CreateContext(window);

        // initialize GLEW system independence
        glewExperimental = GL_TRUE;
        GLenum glewError = glewInit();
        if (glewError != GLEW_OK) {
            printf("Error: Glew failed to initialize!\n");
            exit(EXIT_FAILURE);
        }

        //Use Vsync
        if (SDL_GL_SetSwapInterval(1) < 0) {
            printf("Warning: Unable to set VSync! SDL Error: %s\n", SDL_GetError());
        }

    }

    return success;
}

/**
 * shutDown() - free resources maintaining OpenGL context
 *              and free event handlers
 */
void GLManager::shutDown()
{
    //Deallocate program
    glDeleteProgram(programID);
    glDeleteShader(vertexShader);
    glDeleteShader(fragmentShader);

    //Destroy window
    SDL_DestroyWindow(window);

    //Disable text input
    SDL_StopTextInput();

    //Quit SDL subsystems
    SDL_Quit();
}

/**
 * getID() - get OpenGL context's id
 *
 * @return GLuint current OpenGL context
 */
GLuint GLManager::getID()
{
    return programID;
}

/**
 * getVertexShader() - get current vertex shader's handle in the OpenGL context
 *
 * @return GLuint current vertex shader in use
 */
GLuint GLManager::getVertexShader()
{
    return vertexShader;
}

/**
 * getFragmentShader() - get current fragment shader's handle in the OpenGL context
 *
 * @return GLuint current fragment shader in use
 */
GLuint GLManager::getFragmentShader()
{
    return fragmentShader;
}

/**
 * getVAO() - get current vertex array object's handle in the OpenGL context
 *
 * @return GLuint current vertex array object in use
 */
GLuint GLManager::getVAO()
{
    return VAO;
}

GLfloat GLManager::getScreenWidth()
{
    return screenWidth;
}

GLfloat GLManager::getScreenHeight()
{
    return screenHeight;
}

/**
 * getWindow() - get current SDL window containing the current
 *               frame buffer and native windowing system settings
 *
 * @return SDL_Window* current SDL window in use
 */
SDL_Window * GLManager::getWindow()
{
    return window;
}

/**
 * getSurface() - get current SDL window's frame buffer
 *
 * @return SDL_Surface* current SDL window's frame buffer
 */
SDL_Surface * GLManager::getSurface()
{
    return surface;
}

/**
 * setScreenSize() - set resolution of opulence window
 *
 * @params w,h width and height in pixels
 */
void GLManager::setScreenSize(GLfloat w, GLfloat h)
{
    SDL_SetWindowSize(window , w, h);
    screenWidth = w;
    screenHeight = h;
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2016 Anton Kozhevnikov, Thomas Schulthess
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that
// the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the
//    following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
//    and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

/** \file augment.cpp
 *
 *  \brief Contains implementation of sirius::Density::augment function.
 */

#include "density.h"

namespace sirius {

#ifdef __GPU
extern "C" void generate_dm_pw_gpu(int num_atoms__,
                                   int num_gvec_loc__,
                                   int num_beta__,
                                   double const* atom_pos__,
                                   int const* gvec__,
                                   double_complex const* dm__,
                                   double_complex* dm_pw__);

extern "C" void sum_q_pw_dm_pw_gpu(int num_gvec_loc__,
                                   int nbf__,
                                   double_complex const* q_pw_t__,
                                   double_complex const* dm_pw__,
                                   double_complex* rho_pw__);
#endif

void Density::augment(K_set& ks__)
{
    PROFILE_WITH_TIMER("sirius::Density::augment");

    /* If we have ud and du spin blocks, don't compute one of them (du in this implementation)
     * because density matrix is symmetric. */
    int ndm = (ctx_.num_mag_dims() == 3) ? 3 : ctx_.num_spins();

    /* complex density matrix */
    mdarray<double_complex, 4> density_matrix(unit_cell_.max_mt_basis_size(), unit_cell_.max_mt_basis_size(),
                                              ndm, unit_cell_.num_atoms());
    density_matrix.zero();

    /* add k-point contribution */
    for (int ikloc = 0; ikloc < ks__.spl_num_kpoints().local_size(); ikloc++)
    {
        int ik = ks__.spl_num_kpoints(ikloc);
        if (ctx_.gamma_point())
        {
            add_k_point_contribution<double>(ks__[ik], density_matrix);
        }
        else
        {
            add_k_point_contribution<double_complex>(ks__[ik], density_matrix);
        }
    }
    ctx_.comm().allreduce(density_matrix.at<CPU>(), static_cast<int>(density_matrix.size()));
    #ifdef __PRINT_OBJECT_CHECKSUM
    double_complex cs1 = density_matrix.checksum();
    DUMP("checksum(density_matrix): %20.14f %20.14f", cs1.real(), cs1.imag());
    #endif

    /* split G-vectors between ranks */
    splindex<block> spl_gvec(ctx_.gvec().num_gvec(), ctx_.comm().size(), ctx_.comm().rank());

    /* collect density and magnetization into single array */
    std::vector<Periodic_function<double>*> rho_vec(ctx_.num_mag_dims() + 1);
    rho_vec[0] = rho_;
    for (int j = 0; j < ctx_.num_mag_dims(); j++) rho_vec[1 + j] = magnetization_[j];

    #ifdef __PRINT_OBJECT_CHECKSUM
    for (auto e: rho_vec)
    {
        double_complex cs = e->checksum_pw();
        DUMP("checksum(rho_vec_pw): %20.14f %20.14f", cs.real(), cs.imag());
    }
    #endif

    for (int iat = 0; iat < ctx_.unit_cell().num_atom_types(); iat++)
        ctx_.augmentation_op(iat).prepare();

    #ifdef __GPU
    mdarray<int, 2> gvec;
    mdarray<double_complex, 2> rho_pw_gpu;
    if (ctx_.processing_unit() == GPU)
    {
        gvec = mdarray<int, 2>(3, spl_gvec.local_size());
        for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
        {
            for (int x: {0, 1, 2}) gvec(x, igloc) = ctx_.gvec()[spl_gvec[igloc]][x];
        }
        gvec.allocate_on_device();
        gvec.copy_to_device();

        rho_pw_gpu = mdarray<double_complex, 2>(spl_gvec.local_size(), ctx_.num_mag_dims() + 1);
        rho_pw_gpu.allocate_on_device();
        rho_pw_gpu.zero_on_device();
    }
    #endif

    //mdarray<double, 2> timers(3, Platform::max_num_threads());
    //timers.zero();

    for (int iat = 0; iat < unit_cell_.num_atom_types(); iat++)
    {
        auto& atom_type = unit_cell_.atom_type(iat);
        int nbf = atom_type.mt_basis_size();

        mdarray<double_complex, 3> dm(nbf * nbf, atom_type.num_atoms(), ndm);
        #pragma omp parallel for
        for (int i = 0; i < atom_type.num_atoms(); i++)
        {
            int ia = atom_type.atom_id(i);

            for (int xi2 = 0; xi2 < nbf; xi2++)
            {
                for (int xi1 = 0; xi1 < nbf; xi1++)
                {
                    switch (ctx_.num_mag_dims())
                    {
                        case 0:
                        {
                            dm(xi2 * nbf + xi1, i, 0) = density_matrix(xi2, xi1, 0, ia);
                            break;
                        }
                        case 1:
                        {
                            dm(xi2 * nbf + xi1, i, 0) = density_matrix(xi2, xi1, 0, ia) + density_matrix(xi2, xi1, 1, ia);
                            dm(xi2 * nbf + xi1, i, 1) = density_matrix(xi2, xi1, 0, ia) - density_matrix(xi2, xi1, 1, ia);
                            break;
                        }
                    }
                }
            }
        }

        if (ctx_.processing_unit() == CPU)
        {
            mdarray<double_complex, 2> phase_factors(atom_type.num_atoms(), spl_gvec.local_size());

            #pragma omp parallel for
            for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
            {
                int ig = spl_gvec[igloc];
                for (int i = 0; i < atom_type.num_atoms(); i++)
                {
                    int ia = atom_type.atom_id(i);
                    phase_factors(i, igloc) = std::conj(ctx_.gvec_phase_factor(ig, ia));
                }
            }

            mdarray<double_complex, 2> dm_pw(nbf * nbf, spl_gvec.local_size());

            for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
            {
                linalg<CPU>::gemm(0, 0, nbf * nbf, spl_gvec.local_size(), atom_type.num_atoms(),
                                  &dm(0, 0, iv), dm.ld(), &phase_factors(0, 0), phase_factors.ld(),
                                  &dm_pw(0, 0), dm.ld());

                #ifdef __PRINT_OBJECT_CHECKSUM
                auto cs = dm_pw.checksum();
                ctx_.comm().allreduce(&cs, 1);
                DUMP("checksum(dm_pw) : %18.10f %18.10f", cs.real(), cs.imag());
                #endif
                #pragma omp parallel for
                for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
                {
                    int ig = spl_gvec[igloc];
                    double_complex z(0, 0);

                    /* remember that dm_pw is not a Hermitian matrix in xi1,xi2 indices */
                    for (int xi2 = 0; xi2 < nbf; xi2++)
                    {
                        int idx12 = xi2 * (xi2 + 1) / 2;

                        /* add diagonal term */
                        /* D_{xi2,xi2} * Q(G)_{xi2, xi2} */
                        z += dm_pw(xi2 * nbf + xi2, igloc) * ctx_.augmentation_op(iat).q_pw(idx12 + xi2, igloc);

                        /* add non-diagonal terms */
                        for (int xi1 = 0; xi1 < xi2; xi1++, idx12++)
                        {
                            double_complex q = ctx_.augmentation_op(iat).q_pw(idx12, igloc);

                            /* D_{xi2,xi1} * Q(G)_{xi1, xi2} + D_{xi1,xi2} * Q(G)_{xix, xi1}^{+} */
                            z += (dm_pw(xi2 * nbf + xi1, igloc) * q + dm_pw(xi1 * nbf + xi2, igloc) * std::conj(q));
                        }
                    }
                    rho_vec[iv]->f_pw(ig) += z;
                }
            }
        }

        #ifdef __GPU
        if (ctx_.processing_unit() == GPU)
        {
            mdarray<double, 2> atom_pos(3, atom_type.num_atoms());
            #pragma omp parallel for
            for (int i = 0; i < atom_type.num_atoms(); i++)
            {
                int ia = atom_type.atom_id(i);
                auto pos = unit_cell_.atom(ia).position();
                for (int x: {0, 1, 2}) atom_pos(x, i) = pos[x];
            }
            atom_pos.allocate_on_device();
            atom_pos.copy_to_device();

            dm.allocate_on_device();
            dm.copy_to_device();

            mdarray<double_complex, 2> dm_pw_gpu(nullptr, nbf * nbf, spl_gvec.local_size());
            dm_pw_gpu.allocate_on_device();

            for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
            {
                generate_dm_pw_gpu(atom_type.num_atoms(),
                                   spl_gvec.local_size(),
                                   nbf,
                                   atom_pos.at<GPU>(),
                                   gvec.at<GPU>(),
                                   dm.at<GPU>(0, 0, iv),
                                   dm_pw_gpu.at<GPU>());

                sum_q_pw_dm_pw_gpu(spl_gvec.local_size(),
                                   nbf,
                                   ctx_.augmentation_op(iat).q_pw().at<GPU>(),
                                   dm_pw_gpu.at<GPU>(),
                                   rho_pw_gpu.at<GPU>(0, iv));
            }
        }
        #endif
    }

    #ifdef __GPU
    if (ctx_.processing_unit() == GPU)
    {
        rho_pw_gpu.copy_to_host();
        for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
        {
            #pragma omp parallel for
            for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
            {
                int ig = spl_gvec[igloc];
                rho_vec[iv]->f_pw(ig) += rho_pw_gpu(igloc, iv);
            }
        }
    }
    #endif

    //if (ctx_.comm().rank() == 0)
    //{
    //    std::cout << "-------------------------------------------" << std::endl;
    //    std::cout << "thread_id  | phase    | zgemm    | update  " << std::endl;
    //    std::cout << "-------------------------------------------" << std::endl;
    //    for (int i = 0; i < Platform::max_num_threads(); i++)
    //    {
    //        printf("   %2i      | %8.4f | %8.4f | %8.4f \n", i, timers(0, i), timers(1, i), timers(2, i));
    //    }
    //    std::cout << "-------------------------------------------" << std::endl;
    //}

    for (auto e: rho_vec)
    {
        ctx_.comm().allgather(&e->f_pw(0), spl_gvec.global_offset(), spl_gvec.local_size());

        #ifdef __PRINT_OBJECT_CHECKSUM
        double_complex cs = e->checksum_pw();
        DUMP("checksum(rho_vec_pw): %20.14f %20.14f", cs.real(), cs.imag());
        #endif
    }

    for (int iat = 0; iat < ctx_.unit_cell().num_atom_types(); iat++)
        ctx_.augmentation_op(iat).dismiss();
}

};

<commit_msg>more timers in augment<commit_after>// Copyright (c) 2013-2016 Anton Kozhevnikov, Thomas Schulthess
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that
// the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the
//    following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
//    and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

/** \file augment.cpp
 *
 *  \brief Contains implementation of sirius::Density::augment function.
 */

#include "density.h"

namespace sirius {

#ifdef __GPU
extern "C" void generate_dm_pw_gpu(int num_atoms__,
                                   int num_gvec_loc__,
                                   int num_beta__,
                                   double const* atom_pos__,
                                   int const* gvec__,
                                   double_complex const* dm__,
                                   double_complex* dm_pw__);

extern "C" void sum_q_pw_dm_pw_gpu(int num_gvec_loc__,
                                   int nbf__,
                                   double_complex const* q_pw_t__,
                                   double_complex const* dm_pw__,
                                   double_complex* rho_pw__);
#endif

void Density::augment(K_set& ks__)
{
    PROFILE_WITH_TIMER("sirius::Density::augment");

    /* If we have ud and du spin blocks, don't compute one of them (du in this implementation)
     * because density matrix is symmetric. */
    int ndm = (ctx_.num_mag_dims() == 3) ? 3 : ctx_.num_spins();

    runtime::Timer t1("sirius::Density::augment|dm");
    /* complex density matrix */
    mdarray<double_complex, 4> density_matrix(unit_cell_.max_mt_basis_size(), unit_cell_.max_mt_basis_size(),
                                              ndm, unit_cell_.num_atoms());
    density_matrix.zero();

    /* add k-point contribution */
    for (int ikloc = 0; ikloc < ks__.spl_num_kpoints().local_size(); ikloc++)
    {
        int ik = ks__.spl_num_kpoints(ikloc);
        if (ctx_.gamma_point())
        {
            add_k_point_contribution<double>(ks__[ik], density_matrix);
        }
        else
        {
            add_k_point_contribution<double_complex>(ks__[ik], density_matrix);
        }
    }
    ctx_.comm().allreduce(density_matrix.at<CPU>(), static_cast<int>(density_matrix.size()));
    #ifdef __PRINT_OBJECT_CHECKSUM
    double_complex cs1 = density_matrix.checksum();
    DUMP("checksum(density_matrix): %20.14f %20.14f", cs1.real(), cs1.imag());
    #endif
    t1.stop();

    /* split G-vectors between ranks */
    splindex<block> spl_gvec(ctx_.gvec().num_gvec(), ctx_.comm().size(), ctx_.comm().rank());

    /* collect density and magnetization into single array */
    std::vector<Periodic_function<double>*> rho_vec(ctx_.num_mag_dims() + 1);
    rho_vec[0] = rho_;
    for (int j = 0; j < ctx_.num_mag_dims(); j++) rho_vec[1 + j] = magnetization_[j];

    #ifdef __PRINT_OBJECT_CHECKSUM
    for (auto e: rho_vec)
    {
        double_complex cs = e->checksum_pw();
        DUMP("checksum(rho_vec_pw): %20.14f %20.14f", cs.real(), cs.imag());
    }
    #endif

    for (int iat = 0; iat < ctx_.unit_cell().num_atom_types(); iat++)
        ctx_.augmentation_op(iat).prepare();

    #ifdef __GPU
    mdarray<int, 2> gvec;
    mdarray<double_complex, 2> rho_pw_gpu;
    if (ctx_.processing_unit() == GPU)
    {
        gvec = mdarray<int, 2>(3, spl_gvec.local_size());
        for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
        {
            for (int x: {0, 1, 2}) gvec(x, igloc) = ctx_.gvec()[spl_gvec[igloc]][x];
        }
        gvec.allocate_on_device();
        gvec.copy_to_device();

        rho_pw_gpu = mdarray<double_complex, 2>(spl_gvec.local_size(), ctx_.num_mag_dims() + 1);
        rho_pw_gpu.allocate_on_device();
        rho_pw_gpu.zero_on_device();
    }
    #endif

    //mdarray<double, 2> timers(3, Platform::max_num_threads());
    //timers.zero();

    for (int iat = 0; iat < unit_cell_.num_atom_types(); iat++)
    {
        auto& atom_type = unit_cell_.atom_type(iat);
        int nbf = atom_type.mt_basis_size();

        mdarray<double_complex, 3> dm(nbf * nbf, atom_type.num_atoms(), ndm);
        #pragma omp parallel for
        for (int i = 0; i < atom_type.num_atoms(); i++)
        {
            int ia = atom_type.atom_id(i);

            for (int xi2 = 0; xi2 < nbf; xi2++)
            {
                for (int xi1 = 0; xi1 < nbf; xi1++)
                {
                    switch (ctx_.num_mag_dims())
                    {
                        case 0:
                        {
                            dm(xi2 * nbf + xi1, i, 0) = density_matrix(xi2, xi1, 0, ia);
                            break;
                        }
                        case 1:
                        {
                            dm(xi2 * nbf + xi1, i, 0) = density_matrix(xi2, xi1, 0, ia) + density_matrix(xi2, xi1, 1, ia);
                            dm(xi2 * nbf + xi1, i, 1) = density_matrix(xi2, xi1, 0, ia) - density_matrix(xi2, xi1, 1, ia);
                            break;
                        }
                    }
                }
            }
        }

        if (ctx_.processing_unit() == CPU)
        {
            runtime::Timer t2("sirius::Density::augment|phase_fac");
            mdarray<double_complex, 2> phase_factors(atom_type.num_atoms(), spl_gvec.local_size());

            #pragma omp parallel for
            for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
            {
                int ig = spl_gvec[igloc];
                for (int i = 0; i < atom_type.num_atoms(); i++)
                {
                    int ia = atom_type.atom_id(i);
                    phase_factors(i, igloc) = std::conj(ctx_.gvec_phase_factor(ig, ia));
                }
            }
            t2.stop();

            mdarray<double_complex, 2> dm_pw(nbf * nbf, spl_gvec.local_size());

            for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
            {
                runtime::Timer t3("sirius::Density::augment|gemm");
                linalg<CPU>::gemm(0, 0, nbf * nbf, spl_gvec.local_size(), atom_type.num_atoms(),
                                  &dm(0, 0, iv), dm.ld(), &phase_factors(0, 0), phase_factors.ld(),
                                  &dm_pw(0, 0), dm.ld());
                t3.stop();

                #ifdef __PRINT_OBJECT_CHECKSUM
                auto cs = dm_pw.checksum();
                ctx_.comm().allreduce(&cs, 1);
                DUMP("checksum(dm_pw) : %18.10f %18.10f", cs.real(), cs.imag());
                #endif

                runtime::Timer t4("sirius::Density::augment|sum");
                #pragma omp parallel for
                for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
                {
                    int ig = spl_gvec[igloc];
                    double_complex z(0, 0);

                    /* remember that dm_pw is not a Hermitian matrix in xi1,xi2 indices */
                    for (int xi2 = 0; xi2 < nbf; xi2++)
                    {
                        int idx12 = xi2 * (xi2 + 1) / 2;

                        /* add diagonal term */
                        /* D_{xi2,xi2} * Q(G)_{xi2, xi2} */
                        z += dm_pw(xi2 * nbf + xi2, igloc) * ctx_.augmentation_op(iat).q_pw(idx12 + xi2, igloc);

                        /* add non-diagonal terms */
                        for (int xi1 = 0; xi1 < xi2; xi1++, idx12++)
                        {
                            double_complex q = ctx_.augmentation_op(iat).q_pw(idx12, igloc);

                            /* D_{xi2,xi1} * Q(G)_{xi1, xi2} + D_{xi1,xi2} * Q(G)_{xix, xi1}^{+} */
                            z += (dm_pw(xi2 * nbf + xi1, igloc) * q + dm_pw(xi1 * nbf + xi2, igloc) * std::conj(q));
                        }
                    }
                    rho_vec[iv]->f_pw(ig) += z;
                }
                t4.stop();
            }
        }

        #ifdef __GPU
        if (ctx_.processing_unit() == GPU)
        {
            mdarray<double, 2> atom_pos(3, atom_type.num_atoms());
            #pragma omp parallel for
            for (int i = 0; i < atom_type.num_atoms(); i++)
            {
                int ia = atom_type.atom_id(i);
                auto pos = unit_cell_.atom(ia).position();
                for (int x: {0, 1, 2}) atom_pos(x, i) = pos[x];
            }
            atom_pos.allocate_on_device();
            atom_pos.copy_to_device();

            dm.allocate_on_device();
            dm.copy_to_device();

            mdarray<double_complex, 2> dm_pw_gpu(nullptr, nbf * nbf, spl_gvec.local_size());
            dm_pw_gpu.allocate_on_device();

            for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
            {
                generate_dm_pw_gpu(atom_type.num_atoms(),
                                   spl_gvec.local_size(),
                                   nbf,
                                   atom_pos.at<GPU>(),
                                   gvec.at<GPU>(),
                                   dm.at<GPU>(0, 0, iv),
                                   dm_pw_gpu.at<GPU>());

                sum_q_pw_dm_pw_gpu(spl_gvec.local_size(),
                                   nbf,
                                   ctx_.augmentation_op(iat).q_pw().at<GPU>(),
                                   dm_pw_gpu.at<GPU>(),
                                   rho_pw_gpu.at<GPU>(0, iv));
            }
        }
        #endif
    }

    #ifdef __GPU
    if (ctx_.processing_unit() == GPU)
    {
        rho_pw_gpu.copy_to_host();
        for (int iv = 0; iv < ctx_.num_mag_dims() + 1; iv++)
        {
            #pragma omp parallel for
            for (int igloc = 0; igloc < spl_gvec.local_size(); igloc++)
            {
                int ig = spl_gvec[igloc];
                rho_vec[iv]->f_pw(ig) += rho_pw_gpu(igloc, iv);
            }
        }
    }
    #endif

    //if (ctx_.comm().rank() == 0)
    //{
    //    std::cout << "-------------------------------------------" << std::endl;
    //    std::cout << "thread_id  | phase    | zgemm    | update  " << std::endl;
    //    std::cout << "-------------------------------------------" << std::endl;
    //    for (int i = 0; i < Platform::max_num_threads(); i++)
    //    {
    //        printf("   %2i      | %8.4f | %8.4f | %8.4f \n", i, timers(0, i), timers(1, i), timers(2, i));
    //    }
    //    std::cout << "-------------------------------------------" << std::endl;
    //}

    runtime::Timer t5("sirius::Density::augment|mpi");
    for (auto e: rho_vec)
    {
        ctx_.comm().allgather(&e->f_pw(0), spl_gvec.global_offset(), spl_gvec.local_size());

        #ifdef __PRINT_OBJECT_CHECKSUM
        double_complex cs = e->checksum_pw();
        DUMP("checksum(rho_vec_pw): %20.14f %20.14f", cs.real(), cs.imag());
        #endif
    }
    t5.stop();

    for (int iat = 0; iat < ctx_.unit_cell().num_atom_types(); iat++)
        ctx_.augmentation_op(iat).dismiss();
}

};

<|endoftext|>


<commit_before>#include <QDebug>
#include <QMutexLocker>
#include <QCoreApplication>

#include <sstream>
#include <iostream>

#include "DownloadManager.h"

#include "Util.h"
#include "Downloader.h"
using namespace efdl;

DownloadManager::DownloadManager(bool connProg)
  : connProg{connProg}, chksum{false}, conns{0}, chunksAmount{0},
  chunksFinished{0}, size{0}, offset{0}, bytesDown{0},
  hashAlg{QCryptographicHash::Sha3_512}, downloader{nullptr}
{ }

DownloadManager::~DownloadManager() {
  cleanup();
  qDeleteAll(queue);
}

void DownloadManager::add(Downloader *entry) {
  queue << entry;
}

void DownloadManager::createChecksum(QCryptographicHash::Algorithm hashAlg) {
  this->hashAlg = hashAlg;
  chksum = true;
}

void DownloadManager::start() {
  next();
}

void DownloadManager::next() {
  static bool first{true};
  if (!first) {
    // Update progress with total download time with no connection
    // lines.
    bool tmp{connProg};
    connProg = false;
    lastProgress = QDateTime(); // Force update.
    updateProgress();
    connProg = tmp;

    if (chksum) printChecksum();

    // Separate each download with a newline.
    if (!queue.isEmpty()) qDebug();
  }
  if (first) first = false;

  if (queue.isEmpty()) {
    emit finished();
    return;
  }

  cleanup();

  downloader = queue.dequeue();
  qDebug() << "Downloading" << qPrintable(downloader->getUrl().toString());
  connect(downloader, &Downloader::finished, this, &DownloadManager::next);
  connect(downloader, &Downloader::information,
          this, &DownloadManager::onInformation);
  connect(downloader, &Downloader::chunkStarted,
          this, &DownloadManager::onChunkStarted);
  connect(downloader, &Downloader::chunkProgress,
          this, &DownloadManager::onChunkProgress);
  connect(downloader, &Downloader::chunkFinished,
          this, &DownloadManager::onChunkFinished);
  connect(downloader, &Downloader::chunkFailed,
          this, &DownloadManager::onChunkFailed);
  downloader->start();

  started = QDateTime::currentDateTime();
}
void DownloadManager::onInformation(const QString &outputPath, qint64 size,
                                    int chunksAmount, int conns,
                                    qint64 offset) {
  this->outputPath = outputPath;
  this->size = size;
  this->chunksAmount = chunksAmount;
  this->conns = conns;
  this->offset = offset;
  updateProgress();
}

void DownloadManager::onChunkStarted(int num) {
  QMutexLocker locker{&chunkMutex};
  chunkMap[num] = new Chunk{Range{0, 0}, QDateTime::currentDateTime()};
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkProgress(int num, qint64 received, qint64 total) {
  QMutexLocker locker{&chunkMutex};
  Chunk *chunk = chunkMap[num];
  bytesDown += received - chunk->range.first;
  chunk->range = Range{received, total};
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkFinished(int num, Range range) {
  QMutexLocker locker{&chunkMutex};
  chunksFinished++;
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkFailed(int num, Range range, int httpCode,
                                    QNetworkReply::NetworkError error) {
  QMutexLocker locker{&chunkMutex};
  qCritical() << "Chunk" << num << "failed on range" << range;
  qCritical() << "HTTP code:" << httpCode;
  qCritical() << "Error:" << qPrintable(Util::getErrorString(error));
  qCritical() << "Aborting..";

  cleanup();
  QCoreApplication::exit(-1);
}

void DownloadManager::cleanup() {
  chunksAmount = chunksFinished = size = offset = bytesDown = 0;

  qDeleteAll(chunkMap);
  chunkMap.clear();

  if (downloader) {
    downloader->disconnect();
    downloader->deleteLater();
    downloader = nullptr;
  }
}

void DownloadManager::updateChunkMap() {
  if (chunkMap.size() <= conns) {
    return;
  }

  bool rem = false;
  foreach (const auto &key, chunkMap.keys()) {
    const auto *chunk = chunkMap[key];
    if (chunk->isDone()) {
      rem = true;
      chunkMap.remove(key);
      delete chunk;
      break;
    }
  }
  if (!rem) {
    delete chunkMap.take(chunkMap.firstKey());
  }

  if (chunkMap.size() > conns) {
    updateChunkMap();
  }
}

void DownloadManager::updateProgress() {
  // Only update progress every half second.
  QDateTime now{QDateTime::currentDateTime()};
  if (!lastProgress.isNull() && lastProgress.msecsTo(now) < 500) {
    return;
  }
  lastProgress = now;

  using namespace std;
  stringstream sstream;

  // Set fixed float formatting to one decimal digit.
  sstream.precision(1);
  sstream.setf(ios::fixed, ios::floatfield);

  sstream << "[ ";

  if (bytesDown == 0) {
    sstream << "Starting up download..";
  }
  else {
    bool done = (bytesDown + offset == size);
    qint64 secs{started.secsTo(now)}, bytesPrSec{0}, secsLeft{0};
    if (secs > 0) {
      bytesPrSec = bytesDown / secs;
      if (bytesPrSec > 0) {
        secsLeft = (!done ? (size - bytesDown - offset) / bytesPrSec
                    : secs);
      }
    }

    float perc = (long double)(bytesDown + offset) / (long double)size * 100.0;

    sstream << perc << "% | "
            << (!done
                ? Util::formatSize(bytesDown + offset, 1).toStdString()+ " / "
                : "")
            << Util::formatSize(size, 1).toStdString() << " @ "
            << Util::formatSize(bytesPrSec, 1).toStdString() << "/s | "
            << (!done
                ? QString("chunk %1 / %2").arg(chunksFinished).arg(chunksAmount).toStdString()
                : QString("%1 chunks").arg(chunksAmount).toStdString())
            << " | "
            << Util::formatTime(secsLeft).toStdString() << " "
            << (!done ? "left" : "total");
  }

  sstream << " ]";

  if (connProg) {
    foreach (const int &num, chunkMap.keys()) {
      auto *chunk = chunkMap[num];
      qint64 received = chunk->range.first, total = chunk->range.second;

      bool done = chunk->isDone();
      if (done && chunk->ended.isNull()) {
        chunk->ended = now;
      }

      qint64 secs{chunk->started.secsTo(done ? chunk->ended : now)},
        bytesPrSec{0}, secsLeft{0};
      if (secs > 0) {
        bytesPrSec = received / secs;
        if (bytesPrSec > 0) {
          secsLeft = (!done ? (total - received) / bytesPrSec
                      : secs);
        }
      }

      float perc{0};
      if (received > 0 && total > 0) {
        perc = (long double) received / (long double) total * 100.0;
      }
      sstream << "\n{ chunk #" << num << ": " << perc << "% ";
      if (total > 0) {
        sstream << "| "
                << (!done
                    ? Util::formatSize(received, 1).toStdString() + " / "
                    : "")
                << Util::formatSize(total, 1).toStdString() << " @ "
                << Util::formatSize(bytesPrSec, 1).toStdString() << "/s | "
                << Util::formatTime(secsLeft).toStdString() << " "
                << (!done ? "left" : "total") << " ";
      }
      sstream << "}";
    }
  }

  sstream << '\n';

  static int lastLines{0};
  string msg{sstream.str()};

  // Remove additional lines, if any.
  for (int i = 0; i < lastLines; i++) {
    cout << "\033[A" // Go up a line (\033 = ESC, [ = CTRL).
         << "\033[2K"; // Kill line.
  }

  // Rewind to beginning with carriage return and write actual
  // message.
  cout << '\r' << msg;
  cout.flush();

  lastLines = QString(msg.c_str()).split("\n").size() - 1;
}

void DownloadManager::printChecksum() {
  QCryptographicHash hasher{hashAlg};
  QFile file{outputPath};
  if (!file.open(QIODevice::ReadOnly)) {
    qCritical() << "ERROR Checksum generation failed: could not open output"
                << "file for reading.";
     return;
  }
  if (!hasher.addData(&file)) {
    qCritical() << "ERROR Failed to do checksum of file.";
     return;
  }
  qDebug() << "Checksum:" << qPrintable(hasher.result().toHex());
}
<commit_msg>Added color escape codes making percentages, speed and ETA bold faced.<commit_after>#include <QDebug>
#include <QMutexLocker>
#include <QCoreApplication>

#include <sstream>
#include <iostream>

#include "DownloadManager.h"

#include "Util.h"
#include "Downloader.h"
using namespace efdl;

DownloadManager::DownloadManager(bool connProg)
  : connProg{connProg}, chksum{false}, conns{0}, chunksAmount{0},
  chunksFinished{0}, size{0}, offset{0}, bytesDown{0},
  hashAlg{QCryptographicHash::Sha3_512}, downloader{nullptr}
{ }

DownloadManager::~DownloadManager() {
  cleanup();
  qDeleteAll(queue);
}

void DownloadManager::add(Downloader *entry) {
  queue << entry;
}

void DownloadManager::createChecksum(QCryptographicHash::Algorithm hashAlg) {
  this->hashAlg = hashAlg;
  chksum = true;
}

void DownloadManager::start() {
  next();
}

void DownloadManager::next() {
  static bool first{true};
  if (!first) {
    // Update progress with total download time with no connection
    // lines.
    bool tmp{connProg};
    connProg = false;
    lastProgress = QDateTime(); // Force update.
    updateProgress();
    connProg = tmp;

    if (chksum) printChecksum();

    // Separate each download with a newline.
    if (!queue.isEmpty()) qDebug();
  }
  if (first) first = false;

  if (queue.isEmpty()) {
    emit finished();
    return;
  }

  cleanup();

  downloader = queue.dequeue();
  qDebug() << "Downloading" << qPrintable(downloader->getUrl().toString());
  connect(downloader, &Downloader::finished, this, &DownloadManager::next);
  connect(downloader, &Downloader::information,
          this, &DownloadManager::onInformation);
  connect(downloader, &Downloader::chunkStarted,
          this, &DownloadManager::onChunkStarted);
  connect(downloader, &Downloader::chunkProgress,
          this, &DownloadManager::onChunkProgress);
  connect(downloader, &Downloader::chunkFinished,
          this, &DownloadManager::onChunkFinished);
  connect(downloader, &Downloader::chunkFailed,
          this, &DownloadManager::onChunkFailed);
  downloader->start();

  started = QDateTime::currentDateTime();
}
void DownloadManager::onInformation(const QString &outputPath, qint64 size,
                                    int chunksAmount, int conns,
                                    qint64 offset) {
  this->outputPath = outputPath;
  this->size = size;
  this->chunksAmount = chunksAmount;
  this->conns = conns;
  this->offset = offset;
  updateProgress();
}

void DownloadManager::onChunkStarted(int num) {
  QMutexLocker locker{&chunkMutex};
  chunkMap[num] = new Chunk{Range{0, 0}, QDateTime::currentDateTime()};
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkProgress(int num, qint64 received, qint64 total) {
  QMutexLocker locker{&chunkMutex};
  Chunk *chunk = chunkMap[num];
  bytesDown += received - chunk->range.first;
  chunk->range = Range{received, total};
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkFinished(int num, Range range) {
  QMutexLocker locker{&chunkMutex};
  chunksFinished++;
  updateChunkMap();
  updateProgress();
}

void DownloadManager::onChunkFailed(int num, Range range, int httpCode,
                                    QNetworkReply::NetworkError error) {
  QMutexLocker locker{&chunkMutex};
  qCritical() << "Chunk" << num << "failed on range" << range;
  qCritical() << "HTTP code:" << httpCode;
  qCritical() << "Error:" << qPrintable(Util::getErrorString(error));
  qCritical() << "Aborting..";

  cleanup();
  QCoreApplication::exit(-1);
}

void DownloadManager::cleanup() {
  chunksAmount = chunksFinished = size = offset = bytesDown = 0;

  qDeleteAll(chunkMap);
  chunkMap.clear();

  if (downloader) {
    downloader->disconnect();
    downloader->deleteLater();
    downloader = nullptr;
  }
}

void DownloadManager::updateChunkMap() {
  if (chunkMap.size() <= conns) {
    return;
  }

  bool rem = false;
  foreach (const auto &key, chunkMap.keys()) {
    const auto *chunk = chunkMap[key];
    if (chunk->isDone()) {
      rem = true;
      chunkMap.remove(key);
      delete chunk;
      break;
    }
  }
  if (!rem) {
    delete chunkMap.take(chunkMap.firstKey());
  }

  if (chunkMap.size() > conns) {
    updateChunkMap();
  }
}

void DownloadManager::updateProgress() {
  // Only update progress every half second.
  QDateTime now{QDateTime::currentDateTime()};
  if (!lastProgress.isNull() && lastProgress.msecsTo(now) < 500) {
    return;
  }
  lastProgress = now;

  using namespace std;
  stringstream sstream;

  // Set fixed float formatting to one decimal digit.
  sstream.precision(1);
  sstream.setf(ios::fixed, ios::floatfield);

  // Color escape codes.
  static const string nw{"\033[0;37m"}, // normal, white
    bw{"\033[1;37m"}; // bold, white

  sstream << nw << "[ ";

  if (bytesDown == 0) {
    sstream << "Starting up download..";
  }
  else {
    bool done = (bytesDown + offset == size);
    qint64 secs{started.secsTo(now)}, bytesPrSec{0}, secsLeft{0};
    if (secs > 0) {
      bytesPrSec = bytesDown / secs;
      if (bytesPrSec > 0) {
        secsLeft = (!done ? (size - bytesDown - offset) / bytesPrSec
                    : secs);
      }
    }

    float perc = (long double)(bytesDown + offset) / (long double)size * 100.0;

    sstream << bw << perc << "%" << nw << " | "
            << (!done
                ? Util::formatSize(bytesDown + offset, 1).toStdString()+ " / "
                : "")
            << Util::formatSize(size, 1).toStdString() << " @ "
            << bw
            << Util::formatSize(bytesPrSec, 1).toStdString() << "/s"
            << nw
            << " | "
            << (!done
                ? QString("chunk %1 / %2").arg(chunksFinished).arg(chunksAmount).toStdString()
                : QString("%1 chunks").arg(chunksAmount).toStdString())
            << " | "
            << bw
            << Util::formatTime(secsLeft).toStdString() << " "
            << nw
            << (!done ? "left" : "total");
  }

  sstream << " ]";

  if (connProg) {
    foreach (const int &num, chunkMap.keys()) {
      auto *chunk = chunkMap[num];
      qint64 received = chunk->range.first, total = chunk->range.second;

      bool done = chunk->isDone();
      if (done && chunk->ended.isNull()) {
        chunk->ended = now;
      }

      qint64 secs{chunk->started.secsTo(done ? chunk->ended : now)},
        bytesPrSec{0}, secsLeft{0};
      if (secs > 0) {
        bytesPrSec = received / secs;
        if (bytesPrSec > 0) {
          secsLeft = (!done ? (total - received) / bytesPrSec
                      : secs);
        }
      }

      float perc{0};
      if (received > 0 && total > 0) {
        perc = (long double) received / (long double) total * 100.0;
      }
      sstream << "\n{ chunk #" << num << ": " << perc << "% ";
      if (total > 0) {
        sstream << "| "
                << (!done
                    ? Util::formatSize(received, 1).toStdString() + " / "
                    : "")
                << Util::formatSize(total, 1).toStdString() << " @ "
                << Util::formatSize(bytesPrSec, 1).toStdString() << "/s | "
                << Util::formatTime(secsLeft).toStdString() << " "
                << (!done ? "left" : "total") << " ";
      }
      sstream << "}";
    }
  }

  sstream << '\n';

  static int lastLines{0};
  string msg{sstream.str()};

  // Remove additional lines, if any.
  for (int i = 0; i < lastLines; i++) {
    cout << "\033[A" // Go up a line (\033 = ESC, [ = CTRL).
         << "\033[2K"; // Kill line.
  }

  // Rewind to beginning with carriage return and write actual
  // message.
  cout << '\r' << msg;
  cout.flush();

  lastLines = QString(msg.c_str()).split("\n").size() - 1;
}

void DownloadManager::printChecksum() {
  QCryptographicHash hasher{hashAlg};
  QFile file{outputPath};
  if (!file.open(QIODevice::ReadOnly)) {
    qCritical() << "ERROR Checksum generation failed: could not open output"
                << "file for reading.";
     return;
  }
  if (!hasher.addData(&file)) {
    qCritical() << "ERROR Failed to do checksum of file.";
     return;
  }
  qDebug() << "Checksum:" << qPrintable(hasher.result().toHex());
}
<|endoftext|>