<commit_before>// @(#)root/proofplayer:$Id$
// Author: Long Tran-Thanh    22/07/07

/*************************************************************************
 * Copyright (C) 1995-2002, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TPacketizerUnit                                                      //
//                                                                      //
// This packetizer generates packets of generic units, representing the //
// number of times an operation cycle has to be repeated by the worker  //
// node, e.g. the number of Monte carlo events to be generated.         //
// Packets sizes are generated taking into account the performance of   //
// worker nodes, based on the time needed to process previous packets.  //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TPacketizerUnit.h"

#include "Riostream.h"
#include "TDSet.h"
#include "TError.h"
#include "TEventList.h"
#include "TMap.h"
#include "TMessage.h"
#include "TMonitor.h"
#include "TNtupleD.h"
#include "TObject.h"
#include "TParameter.h"
#include "TPerfStats.h"
#include "TProofDebug.h"
#include "TProof.h"
#include "TProofPlayer.h"
#include "TProofServ.h"
#include "TSlave.h"
#include "TSocket.h"
#include "TStopwatch.h"
#include "TTimer.h"
#include "TUrl.h"
#include "TClass.h"
#include "TMath.h"
#include "TObjString.h"

using namespace TMath;
//
// The following utility class manage the state of the
// work to be performed and the slaves involved in the process.
//
// The list of TSlaveStat(s) keep track of the work (being) done
// by each slave
//

//------------------------------------------------------------------------------

class TPacketizerUnit::TSlaveStat : public TObject {

friend class TPacketizerUnit;

private:
   TSlave   *fSlave;         // corresponding TSlave record
   Long64_t  fProcessed;     // number of entries processed
   Long64_t  fLastProcessed; // number of processed entries of the last packet
   Double_t  fSpeed;         // estimated current average speed of the processing slave
   Double_t  fTimeInstant;   // stores the time instant when the current packet started
   TNtupleD *fCircNtp;       // Keeps circular info for speed calculations
   Long_t    fCircLvl;       // Circularity level

public:
   TSlaveStat(TSlave *sl, TList *input);
   ~TSlaveStat();

   void        GetCurrentTime();

   const char *GetName() const { return fSlave->GetName(); }
   Long64_t    GetEntriesProcessed() const { return fProcessed; }

   void        UpdatePerformance(Double_t time);
};

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::TSlaveStat(TSlave *slave, TList *input)
                            : fSlave(slave), fProcessed(0), fLastProcessed(0),
                              fSpeed(0), fTimeInstant(0), fCircLvl(5)
{
   // Main constructor

   // Initialize the circularity ntple for speed calculations
   fCircNtp = new TNtupleD("Speed Circ Ntp", "Circular process info","tm:ev");
   TProof::GetParameter(input, "PROOF_TPacketizerUnitCircularity", fCircLvl);
   fCircLvl = (fCircLvl > 0) ? fCircLvl : 5;
   fCircNtp->SetCircular(fCircLvl);
}

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::~TSlaveStat()
{
   // Destructor

   SafeDelete(fCircNtp);
}

//______________________________________________________________________________
void TPacketizerUnit::TSlaveStat::UpdatePerformance(Double_t time)
{
   // Update the circular ntple

   Double_t ttot = time;
   Double_t *ar = fCircNtp->GetArgs();
   Int_t ne = fCircNtp->GetEntries();
   if (ne <= 0) {
      // First call: just fill one ref entry and return
      fCircNtp->Fill(0., 0);
      fSpeed = 0.;
      return;
   }
   // Fill the entry
   fCircNtp->GetEntry(ne-1);
   ttot = ar[0] + time;
   fCircNtp->Fill(ttot, fProcessed);

   // Calculate the speed
   fCircNtp->GetEntry(0);
   Double_t dtime = (ttot > ar[0]) ? ttot - ar[0] : ne+1 ;
   Long64_t nevts = fProcessed - (Long64_t)ar[1];
   fSpeed = nevts / dtime;
   PDB(kPacketizer,2)
      Info("UpdatePerformance", "time:%f, dtime:%f, nevts:%lld, speed: %f",
                                time, dtime, nevts, fSpeed);

}

//------------------------------------------------------------------------------

ClassImp(TPacketizerUnit)

//______________________________________________________________________________
TPacketizerUnit::TPacketizerUnit(TList *slaves, Long64_t num, TList *input,
                                 TProofProgressStatus *st)
                : TVirtualPacketizer(input, st)
{
   // Constructor

   PDB(kPacketizer,1) Info("TPacketizerUnit", "enter (num %lld)", num);

   // Init pointer members
   fSlaveStats = 0;
   fPackets = 0;

   fTimeLimit = 1;
   TProof::GetParameter(input, "PROOF_PacketizerTimeLimit", fTimeLimit);
   PDB(kPacketizer,1)
      Info("TPacketizerUnit", "time limit is %lf", fTimeLimit);
   fProcessing = 0;

   fStopwatch = new TStopwatch();

   fPackets = new TList;
   fPackets->SetOwner();

   fSlaveStats = new TMap;
   fSlaveStats->SetOwner(kFALSE);

   TSlave *slave;
   TIter si(slaves);
   while ((slave = (TSlave*) si.Next()))
      fSlaveStats->Add(slave, new TSlaveStat(slave, input));

   fTotalEntries = num;

   fStopwatch->Start();
   PDB(kPacketizer,1) Info("TPacketizerUnit", "return");
}

//______________________________________________________________________________
TPacketizerUnit::~TPacketizerUnit()
{
   // Destructor.

   if (fSlaveStats)
      fSlaveStats->DeleteValues();
   SafeDelete(fSlaveStats);
   SafeDelete(fPackets);
   SafeDelete(fStopwatch);
}

//______________________________________________________________________________
Double_t TPacketizerUnit::GetCurrentTime()
{
   // Get current time

   Double_t retValue = fStopwatch->RealTime();
   fStopwatch->Continue();
   return retValue;
}

//______________________________________________________________________________
TDSetElement *TPacketizerUnit::GetNextPacket(TSlave *sl, TMessage *r)
{
   // Get next packet

   if (!fValid)
      return 0;

   // Find slave
   TSlaveStat *slstat = (TSlaveStat*) fSlaveStats->GetValue(sl);
   R__ASSERT(slstat != 0);

   // Update stats & free old element
   Double_t latency, proctime, proccpu;
   Long64_t bytesRead = -1;
   Long64_t totalEntries = -1;
   Long64_t totev = 0;
   Long64_t numev = -1;

   if (!gProofServ || gProofServ->GetProtocol() > 18) {
      TProofProgressStatus *status = 0;
      (*r) >> latency;
      (*r) >> status;

      proctime = status ? status->GetProcTime() : -1.;
      proccpu  = status ? status->GetCPUTime() : -1.;
      totev  = status ? status->GetEntries() : 0;
      bytesRead  = status ? status->GetBytesRead() : 0;

      delete status;
   } else {

      (*r) >> latency >> proctime >> proccpu;

      // only read new info if available
      if (r->BufferSize() > r->Length()) (*r) >> bytesRead;
      if (r->BufferSize() > r->Length()) (*r) >> totalEntries;
      if (r->BufferSize() > r->Length()) (*r) >> totev;
   }
   numev = totev - slstat->fProcessed;

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s: fProcessed %lld\t", sl->GetOrdinal(), GetEntriesProcessed());
   fProcessing = 0;

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s (%s): %lld %7.3lf %7.3lf %7.3lf %lld",
                           sl->GetOrdinal(), sl->GetName(),
                           numev, latency, proctime, proccpu, bytesRead);

   if (gPerfStats != 0) {
      gPerfStats->PacketEvent(sl->GetOrdinal(), sl->GetName(), "", numev,
                              latency, proctime, proccpu, bytesRead);
   }

   if (GetEntriesProcessed() == fTotalEntries) {
      // Send last timer message
      HandleTimer(0);
      SafeDelete(fProgress);
   }

   if (fStop) {
      // Send last timer message
      HandleTimer(0);
      return 0;
   }

   Long64_t num;

   // Get the current time
   Double_t cTime = GetCurrentTime();

   if (slstat->fCircNtp->GetEntries() <= 0) {
      // The calibration phase
      PDB(kPacketizer,2) Info("GetNextPacket"," calibration: "
                              "total entries %lld, workers %d",
                              fTotalEntries, fSlaveStats->GetSize());
      Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
      num = (avg > 5) ? 5 : avg;

      // Create a reference entry
      slstat->UpdatePerformance(0.);

   } else {
      // Schedule tasks for workers based on the currently estimated processing speeds

      // Update worker stats and performances
      slstat->fProcessed += slstat->fLastProcessed;
      slstat->UpdatePerformance(proctime);

      TMapIter *iter = (TMapIter *)fSlaveStats->MakeIterator();
      TSlave * tmpSlave;
      TSlaveStat * tmpStat;

      Double_t sumSpeed = 0;
      Double_t sumBusy = 0;

      // The basic idea is to estimate the optimal finishing time for the process, assuming
      // that the currently measured processing speeds of the slaves remain the same in the future.
      // The optTime can be calculated as follows.
      // Let s_i be the speed of worker i. We assume that worker i will be busy in the
      // next b_i time instant. Therefore we have the following equation:
      //                 SUM((optTime-b_i)*s_i) = (remaining_entries),
      // Hence optTime can be calculated easily:
      //                 optTime = ((remaining_entries) + SUM(b_i*s_i))/SUM(s_i)

      while ((tmpSlave = (TSlave *)iter->Next())) {
         tmpStat = (TSlaveStat *)fSlaveStats->GetValue(tmpSlave);
         // If the slave doesn't response for a long time, its service rate will be considered as 0
         if ((cTime - tmpStat->fTimeInstant) > 4*fTimeLimit)
            tmpStat->fSpeed = 0;
         PDB(kPacketizer,2)
            Info("GetNextPacket",
                 "worker-%s: speed %lf", tmpSlave->GetOrdinal(), tmpStat->fSpeed);
         if (tmpStat->fSpeed > 0) {
            // Calculating the SUM(s_i)
            sumSpeed += tmpStat->fSpeed;
            // There is nothing to do if slave_i is not calibrated or slave i is the current slave
            if ((tmpStat->fTimeInstant) && (cTime - tmpStat->fTimeInstant > 0)) {
               // Calculating the SUM(b_i*s_i)
               //      s_i = tmpStat->fSpeed
               //      b_i = tmpStat->fTimeInstant + tmpStat->fLastProcessed/s_i - cTime)
               //  b_i*s_i = (tmpStat->fTimeInstant - cTime)*s_i + tmpStat->fLastProcessed
               Double_t busyspeed =
                  ((tmpStat->fTimeInstant - cTime)*tmpStat->fSpeed + tmpStat->fLastProcessed);
               if (busyspeed > 0)
                  sumBusy += busyspeed;
            }
         }
      }
      PDB(kPacketizer,2)
         Info("GetNextPacket", "worker-%s: sum speed: %lf, sum busy: %f",
                               sl->GetOrdinal(), sumSpeed, sumBusy);
      // firstly the slave will try to get all of the remaining entries
      if (slstat->fSpeed > 0 &&
         (fTotalEntries - GetEntriesProcessed())/(slstat->fSpeed) < fTimeLimit) {
         num = (fTotalEntries - GetEntriesProcessed());
      } else {
         if (slstat->fSpeed > 0) {
            // calculating the optTime
            Double_t optTime = (fTotalEntries - GetEntriesProcessed() + sumBusy)/sumSpeed;
            // if optTime is greater than the official time limit, then the slave gets a number
            // of entries that still fit into the time limit, otherwise it uses the optTime as
            // a time limit
            num = (optTime > fTimeLimit) ? Nint(fTimeLimit*(slstat->fSpeed))
                                         : Nint(optTime*(slstat->fSpeed));
           PDB(kPacketizer,2)
               Info("GetNextPacket", "opTime %lf num %lld", optTime, num);
         } else {
            Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
            num = (avg > 5) ? 5 : avg;
         }
      }
   }
   fProcessing = (num < (fTotalEntries - GetEntriesProcessed())) ? num : (fTotalEntries - GetEntriesProcessed());

   // Set the informations of the current slave
   slstat->fLastProcessed = fProcessing;
   // Set the start time of the current packet
   slstat->fTimeInstant = cTime;

   PDB(kPacketizer,2)
      Info("GetNextPacket", "worker-%s: num %lld, processing %lld, remaining %lld",sl->GetOrdinal(),
                            num, fProcessing, (fTotalEntries - GetEntriesProcessed() - fProcessing));
   TDSetElement *elem = new TDSetElement("", "", "", 0, fProcessing);
   elem->SetBit(TDSetElement::kEmpty);

   // Update the total counter
   fProgressStatus->IncEntries(slstat->fLastProcessed);

   return elem;
}
<commit_msg>Make sure that the packet contains at least 1 event<commit_after>// @(#)root/proofplayer:$Id$
// Author: Long Tran-Thanh    22/07/07

/*************************************************************************
 * Copyright (C) 1995-2002, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TPacketizerUnit                                                      //
//                                                                      //
// This packetizer generates packets of generic units, representing the //
// number of times an operation cycle has to be repeated by the worker  //
// node, e.g. the number of Monte carlo events to be generated.         //
// Packets sizes are generated taking into account the performance of   //
// worker nodes, based on the time needed to process previous packets.  //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TPacketizerUnit.h"

#include "Riostream.h"
#include "TDSet.h"
#include "TError.h"
#include "TEventList.h"
#include "TMap.h"
#include "TMessage.h"
#include "TMonitor.h"
#include "TNtupleD.h"
#include "TObject.h"
#include "TParameter.h"
#include "TPerfStats.h"
#include "TProofDebug.h"
#include "TProof.h"
#include "TProofPlayer.h"
#include "TProofServ.h"
#include "TSlave.h"
#include "TSocket.h"
#include "TStopwatch.h"
#include "TTimer.h"
#include "TUrl.h"
#include "TClass.h"
#include "TMath.h"
#include "TObjString.h"

using namespace TMath;
//
// The following utility class manage the state of the
// work to be performed and the slaves involved in the process.
//
// The list of TSlaveStat(s) keep track of the work (being) done
// by each slave
//

//------------------------------------------------------------------------------

class TPacketizerUnit::TSlaveStat : public TObject {

friend class TPacketizerUnit;

private:
   TSlave   *fSlave;         // corresponding TSlave record
   Long64_t  fProcessed;     // number of entries processed
   Long64_t  fLastProcessed; // number of processed entries of the last packet
   Double_t  fSpeed;         // estimated current average speed of the processing slave
   Double_t  fTimeInstant;   // stores the time instant when the current packet started
   TNtupleD *fCircNtp;       // Keeps circular info for speed calculations
   Long_t    fCircLvl;       // Circularity level

public:
   TSlaveStat(TSlave *sl, TList *input);
   ~TSlaveStat();

   void        GetCurrentTime();

   const char *GetName() const { return fSlave->GetName(); }
   Long64_t    GetEntriesProcessed() const { return fProcessed; }

   void        UpdatePerformance(Double_t time);
};

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::TSlaveStat(TSlave *slave, TList *input)
                            : fSlave(slave), fProcessed(0), fLastProcessed(0),
                              fSpeed(0), fTimeInstant(0), fCircLvl(5)
{
   // Main constructor

   // Initialize the circularity ntple for speed calculations
   fCircNtp = new TNtupleD("Speed Circ Ntp", "Circular process info","tm:ev");
   TProof::GetParameter(input, "PROOF_TPacketizerUnitCircularity", fCircLvl);
   fCircLvl = (fCircLvl > 0) ? fCircLvl : 5;
   fCircNtp->SetCircular(fCircLvl);
}

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::~TSlaveStat()
{
   // Destructor

   SafeDelete(fCircNtp);
}

//______________________________________________________________________________
void TPacketizerUnit::TSlaveStat::UpdatePerformance(Double_t time)
{
   // Update the circular ntple

   Double_t ttot = time;
   Double_t *ar = fCircNtp->GetArgs();
   Int_t ne = fCircNtp->GetEntries();
   if (ne <= 0) {
      // First call: just fill one ref entry and return
      fCircNtp->Fill(0., 0);
      fSpeed = 0.;
      return;
   }
   // Fill the entry
   fCircNtp->GetEntry(ne-1);
   ttot = ar[0] + time;
   fCircNtp->Fill(ttot, fProcessed);

   // Calculate the speed
   fCircNtp->GetEntry(0);
   Double_t dtime = (ttot > ar[0]) ? ttot - ar[0] : ne+1 ;
   Long64_t nevts = fProcessed - (Long64_t)ar[1];
   fSpeed = nevts / dtime;
   PDB(kPacketizer,2)
      Info("UpdatePerformance", "time:%f, dtime:%f, nevts:%lld, speed: %f",
                                time, dtime, nevts, fSpeed);

}

//------------------------------------------------------------------------------

ClassImp(TPacketizerUnit)

//______________________________________________________________________________
TPacketizerUnit::TPacketizerUnit(TList *slaves, Long64_t num, TList *input,
                                 TProofProgressStatus *st)
                : TVirtualPacketizer(input, st)
{
   // Constructor

   PDB(kPacketizer,1) Info("TPacketizerUnit", "enter (num %lld)", num);

   // Init pointer members
   fSlaveStats = 0;
   fPackets = 0;

   fTimeLimit = 1;
   TProof::GetParameter(input, "PROOF_PacketizerTimeLimit", fTimeLimit);
   PDB(kPacketizer,1)
      Info("TPacketizerUnit", "time limit is %lf", fTimeLimit);
   fProcessing = 0;

   fStopwatch = new TStopwatch();

   fPackets = new TList;
   fPackets->SetOwner();

   fSlaveStats = new TMap;
   fSlaveStats->SetOwner(kFALSE);

   TSlave *slave;
   TIter si(slaves);
   while ((slave = (TSlave*) si.Next()))
      fSlaveStats->Add(slave, new TSlaveStat(slave, input));

   fTotalEntries = num;

   fStopwatch->Start();
   PDB(kPacketizer,1) Info("TPacketizerUnit", "return");
}

//______________________________________________________________________________
TPacketizerUnit::~TPacketizerUnit()
{
   // Destructor.

   if (fSlaveStats)
      fSlaveStats->DeleteValues();
   SafeDelete(fSlaveStats);
   SafeDelete(fPackets);
   SafeDelete(fStopwatch);
}

//______________________________________________________________________________
Double_t TPacketizerUnit::GetCurrentTime()
{
   // Get current time

   Double_t retValue = fStopwatch->RealTime();
   fStopwatch->Continue();
   return retValue;
}

//______________________________________________________________________________
TDSetElement *TPacketizerUnit::GetNextPacket(TSlave *sl, TMessage *r)
{
   // Get next packet

   if (!fValid)
      return 0;

   // Find slave
   TSlaveStat *slstat = (TSlaveStat*) fSlaveStats->GetValue(sl);
   R__ASSERT(slstat != 0);

   // Update stats & free old element
   Double_t latency, proctime, proccpu;
   Long64_t bytesRead = -1;
   Long64_t totalEntries = -1;
   Long64_t totev = 0;
   Long64_t numev = -1;

   if (!gProofServ || gProofServ->GetProtocol() > 18) {
      TProofProgressStatus *status = 0;
      (*r) >> latency;
      (*r) >> status;

      proctime = status ? status->GetProcTime() : -1.;
      proccpu  = status ? status->GetCPUTime() : -1.;
      totev  = status ? status->GetEntries() : 0;
      bytesRead  = status ? status->GetBytesRead() : 0;

      delete status;
   } else {

      (*r) >> latency >> proctime >> proccpu;

      // only read new info if available
      if (r->BufferSize() > r->Length()) (*r) >> bytesRead;
      if (r->BufferSize() > r->Length()) (*r) >> totalEntries;
      if (r->BufferSize() > r->Length()) (*r) >> totev;
   }
   numev = totev - slstat->fProcessed;

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s: fProcessed %lld\t", sl->GetOrdinal(), GetEntriesProcessed());
   fProcessing = 0;

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s (%s): %lld %7.3lf %7.3lf %7.3lf %lld",
                           sl->GetOrdinal(), sl->GetName(),
                           numev, latency, proctime, proccpu, bytesRead);

   if (gPerfStats != 0) {
      gPerfStats->PacketEvent(sl->GetOrdinal(), sl->GetName(), "", numev,
                              latency, proctime, proccpu, bytesRead);
   }

   if (GetEntriesProcessed() == fTotalEntries) {
      // Send last timer message
      HandleTimer(0);
      SafeDelete(fProgress);
   }

   if (fStop) {
      // Send last timer message
      HandleTimer(0);
      return 0;
   }

   Long64_t num;

   // Get the current time
   Double_t cTime = GetCurrentTime();

   if (slstat->fCircNtp->GetEntries() <= 0) {
      // The calibration phase
      PDB(kPacketizer,2) Info("GetNextPacket"," calibration: "
                              "total entries %lld, workers %d",
                              fTotalEntries, fSlaveStats->GetSize());
      Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
      num = (avg > 5) ? 5 : avg;

      // Create a reference entry
      slstat->UpdatePerformance(0.);

   } else {
      // Schedule tasks for workers based on the currently estimated processing speeds

      // Update worker stats and performances
      slstat->fProcessed += slstat->fLastProcessed;
      slstat->UpdatePerformance(proctime);

      TMapIter *iter = (TMapIter *)fSlaveStats->MakeIterator();
      TSlave * tmpSlave;
      TSlaveStat * tmpStat;

      Double_t sumSpeed = 0;
      Double_t sumBusy = 0;

      // The basic idea is to estimate the optimal finishing time for the process, assuming
      // that the currently measured processing speeds of the slaves remain the same in the future.
      // The optTime can be calculated as follows.
      // Let s_i be the speed of worker i. We assume that worker i will be busy in the
      // next b_i time instant. Therefore we have the following equation:
      //                 SUM((optTime-b_i)*s_i) = (remaining_entries),
      // Hence optTime can be calculated easily:
      //                 optTime = ((remaining_entries) + SUM(b_i*s_i))/SUM(s_i)

      while ((tmpSlave = (TSlave *)iter->Next())) {
         tmpStat = (TSlaveStat *)fSlaveStats->GetValue(tmpSlave);
         // If the slave doesn't response for a long time, its service rate will be considered as 0
         if ((cTime - tmpStat->fTimeInstant) > 4*fTimeLimit)
            tmpStat->fSpeed = 0;
         PDB(kPacketizer,2)
            Info("GetNextPacket",
                 "worker-%s: speed %lf", tmpSlave->GetOrdinal(), tmpStat->fSpeed);
         if (tmpStat->fSpeed > 0) {
            // Calculating the SUM(s_i)
            sumSpeed += tmpStat->fSpeed;
            // There is nothing to do if slave_i is not calibrated or slave i is the current slave
            if ((tmpStat->fTimeInstant) && (cTime - tmpStat->fTimeInstant > 0)) {
               // Calculating the SUM(b_i*s_i)
               //      s_i = tmpStat->fSpeed
               //      b_i = tmpStat->fTimeInstant + tmpStat->fLastProcessed/s_i - cTime)
               //  b_i*s_i = (tmpStat->fTimeInstant - cTime)*s_i + tmpStat->fLastProcessed
               Double_t busyspeed =
                  ((tmpStat->fTimeInstant - cTime)*tmpStat->fSpeed + tmpStat->fLastProcessed);
               if (busyspeed > 0)
                  sumBusy += busyspeed;
            }
         }
      }
      PDB(kPacketizer,2)
         Info("GetNextPacket", "worker-%s: sum speed: %lf, sum busy: %f",
                               sl->GetOrdinal(), sumSpeed, sumBusy);
      // firstly the slave will try to get all of the remaining entries
      if (slstat->fSpeed > 0 &&
         (fTotalEntries - GetEntriesProcessed())/(slstat->fSpeed) < fTimeLimit) {
         num = (fTotalEntries - GetEntriesProcessed());
      } else {
         if (slstat->fSpeed > 0) {
            // calculating the optTime
            Double_t optTime = (fTotalEntries - GetEntriesProcessed() + sumBusy)/sumSpeed;
            // if optTime is greater than the official time limit, then the slave gets a number
            // of entries that still fit into the time limit, otherwise it uses the optTime as
            // a time limit
            num = (optTime > fTimeLimit) ? Nint(fTimeLimit*(slstat->fSpeed))
                                         : Nint(optTime*(slstat->fSpeed));
           PDB(kPacketizer,2)
              Info("GetNextPacket", "opTime %lf num %lld speed %lf", optTime, num, slstat->fSpeed);
         } else {
            Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
            num = (avg > 5) ? 5 : avg;
         }
      }
   }
   num = (num > 1) ? num : 1;
   fProcessing = (num < (fTotalEntries - GetEntriesProcessed())) ? num : (fTotalEntries - GetEntriesProcessed());

   // Set the informations of the current slave
   slstat->fLastProcessed = fProcessing;
   // Set the start time of the current packet
   slstat->fTimeInstant = cTime;

   PDB(kPacketizer,2)
      Info("GetNextPacket", "worker-%s: num %lld, processing %lld, remaining %lld",sl->GetOrdinal(),
                            num, fProcessing, (fTotalEntries - GetEntriesProcessed() - fProcessing));
   TDSetElement *elem = new TDSetElement("", "", "", 0, fProcessing);
   elem->SetBit(TDSetElement::kEmpty);

   // Update the total counter
   fProgressStatus->IncEntries(slstat->fLastProcessed);

   return elem;
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

#include "packager/media/event/vod_media_info_dump_muxer_listener.h"

#include <google/protobuf/text_format.h>

#include "packager/base/logging.h"
#include "packager/media/base/muxer_options.h"
#include "packager/media/base/stream_info.h"
#include "packager/media/event/muxer_listener_internal.h"
#include "packager/media/file/file.h"
#include "packager/mpd/base/media_info.pb.h"

namespace edash_packager {
namespace media {
namespace event {

VodMediaInfoDumpMuxerListener::VodMediaInfoDumpMuxerListener(
    const std::string& output_file_name)
    : output_file_name_(output_file_name) {}

VodMediaInfoDumpMuxerListener::~VodMediaInfoDumpMuxerListener() {}

void VodMediaInfoDumpMuxerListener::SetContentProtectionSchemeIdUri(
    const std::string& scheme_id_uri) {
  scheme_id_uri_ = scheme_id_uri;
}

void VodMediaInfoDumpMuxerListener::OnMediaStart(
    const MuxerOptions& muxer_options,
    const std::vector<StreamInfo*>& stream_infos,
    uint32_t time_scale,
    ContainerType container_type,
    bool is_encrypted) {
  DCHECK(muxer_options.single_segment);
  media_info_.reset(new MediaInfo());
  if (!internal::GenerateMediaInfo(muxer_options,
                                   stream_infos,
                                   time_scale,
                                   container_type,
                                   media_info_.get())) {
    LOG(ERROR) << "Failed to generate MediaInfo from input.";
    return;
  }

  if (is_encrypted) {
    if (!internal::AddContentProtectionElements(
            container_type, scheme_id_uri_, media_info_.get())) {
      LOG(ERROR) << "Failed to add content protection elements.";
      return;
    }
  }
}

void VodMediaInfoDumpMuxerListener::OnMediaEnd(bool has_init_range,
                                               uint64_t init_range_start,
                                               uint64_t init_range_end,
                                               bool has_index_range,
                                               uint64_t index_range_start,
                                               uint64_t index_range_end,
                                               float duration_seconds,
                                               uint64_t file_size) {
  DCHECK(media_info_);
  if (!internal::SetVodInformation(has_init_range,
                                   init_range_start,
                                   init_range_end,
                                   has_index_range,
                                   index_range_start,
                                   index_range_end,
                                   duration_seconds,
                                   file_size,
                                   media_info_.get())) {
    LOG(ERROR) << "Failed to generate VOD information from input.";
    return;
  }
  SerializeMediaInfoToFile();
}

void VodMediaInfoDumpMuxerListener::OnNewSegment(uint64_t start_time,
                                                 uint64_t duration,
                                                 uint64_t segment_file_size) {
  NOTIMPLEMENTED();
}

bool VodMediaInfoDumpMuxerListener::SerializeMediaInfoToFile() {
  std::string output_string;
  if (!google::protobuf::TextFormat::PrintToString(*media_info_,
                                                   &output_string)) {
    LOG(ERROR) << "Failed to serialize MediaInfo to string.";
    return false;
  }

  media::File* file = File::Open(output_file_name_.c_str(), "w");
  if (!file) {
    LOG(ERROR) << "Failed to open " << output_file_name_;
    return false;
  }
  if (file->Write(output_string.data(), output_string.size()) <= 0) {
    LOG(ERROR) << "Failed to write MediaInfo to file.";
    file->Close();
    return false;
  }
  if (!file->Close()) {
    LOG(ERROR) << "Failed to close " << output_file_name_;
    return false;
  }
  return true;
}

}  // namespace event
}  // namespace media
}  // namespace edash_packager
<commit_msg>Bug fix for VodMediaInfoDumpMuxerListener::OnNewSegment<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

#include "packager/media/event/vod_media_info_dump_muxer_listener.h"

#include <google/protobuf/text_format.h>

#include "packager/base/logging.h"
#include "packager/media/base/muxer_options.h"
#include "packager/media/base/stream_info.h"
#include "packager/media/event/muxer_listener_internal.h"
#include "packager/media/file/file.h"
#include "packager/mpd/base/media_info.pb.h"

namespace edash_packager {
namespace media {
namespace event {

VodMediaInfoDumpMuxerListener::VodMediaInfoDumpMuxerListener(
    const std::string& output_file_name)
    : output_file_name_(output_file_name) {}

VodMediaInfoDumpMuxerListener::~VodMediaInfoDumpMuxerListener() {}

void VodMediaInfoDumpMuxerListener::SetContentProtectionSchemeIdUri(
    const std::string& scheme_id_uri) {
  scheme_id_uri_ = scheme_id_uri;
}

void VodMediaInfoDumpMuxerListener::OnMediaStart(
    const MuxerOptions& muxer_options,
    const std::vector<StreamInfo*>& stream_infos,
    uint32_t time_scale,
    ContainerType container_type,
    bool is_encrypted) {
  DCHECK(muxer_options.single_segment);
  media_info_.reset(new MediaInfo());
  if (!internal::GenerateMediaInfo(muxer_options,
                                   stream_infos,
                                   time_scale,
                                   container_type,
                                   media_info_.get())) {
    LOG(ERROR) << "Failed to generate MediaInfo from input.";
    return;
  }

  if (is_encrypted) {
    if (!internal::AddContentProtectionElements(
            container_type, scheme_id_uri_, media_info_.get())) {
      LOG(ERROR) << "Failed to add content protection elements.";
      return;
    }
  }
}

void VodMediaInfoDumpMuxerListener::OnMediaEnd(bool has_init_range,
                                               uint64_t init_range_start,
                                               uint64_t init_range_end,
                                               bool has_index_range,
                                               uint64_t index_range_start,
                                               uint64_t index_range_end,
                                               float duration_seconds,
                                               uint64_t file_size) {
  DCHECK(media_info_);
  if (!internal::SetVodInformation(has_init_range,
                                   init_range_start,
                                   init_range_end,
                                   has_index_range,
                                   index_range_start,
                                   index_range_end,
                                   duration_seconds,
                                   file_size,
                                   media_info_.get())) {
    LOG(ERROR) << "Failed to generate VOD information from input.";
    return;
  }
  SerializeMediaInfoToFile();
}

void VodMediaInfoDumpMuxerListener::OnNewSegment(uint64_t start_time,
                                                 uint64_t duration,
                                                 uint64_t segment_file_size) {
}

bool VodMediaInfoDumpMuxerListener::SerializeMediaInfoToFile() {
  std::string output_string;
  if (!google::protobuf::TextFormat::PrintToString(*media_info_,
                                                   &output_string)) {
    LOG(ERROR) << "Failed to serialize MediaInfo to string.";
    return false;
  }

  media::File* file = File::Open(output_file_name_.c_str(), "w");
  if (!file) {
    LOG(ERROR) << "Failed to open " << output_file_name_;
    return false;
  }
  if (file->Write(output_string.data(), output_string.size()) <= 0) {
    LOG(ERROR) << "Failed to write MediaInfo to file.";
    file->Close();
    return false;
  }
  if (!file->Close()) {
    LOG(ERROR) << "Failed to close " << output_file_name_;
    return false;
  }
  return true;
}

}  // namespace event
}  // namespace media
}  // namespace edash_packager
<|endoftext|>


<commit_before>#include "parser.h"
#include "platform.h"
#include <assert.h>

extern "C" void parse(const char* json, command* wifiRC_command);

#ifndef DESKTOP
#include "edison_platform.h"
#else
#include "desktop_platform.h"
#endif

#define RC_ADDRESS_BASE "http://192.168.1."
#define RC_JSON_ADDRESS_SUFFIX ":8080/CommandServer/currentJsonCommand"

static void getAddress(char* address_buffer, int size)
{
    assert(size == 64);

//Need to do something smarter here like search for servers on the local network
    char buffer[4];
    memset(buffer, '\0', 4);
    memset(address_buffer, '\0', 64);
    strcat(address_buffer, RC_ADDRESS_BASE);
    sprintf(buffer, "%i", DEV);
    strcat(address_buffer, buffer);
    strcat(address_buffer, RC_JSON_ADDRESS_SUFFIX);
}

#ifndef DESKTOP
void platform_setup()
{
    pthread_mutex_init(&gHttpMutex, NULL);

    pinMode(LIGHTS_ENABLE_PIN, OUTPUT);
    pinMode(DRIVE_MOTOR_PIN, OUTPUT);
    pinMode(REVERSE_ENGAGE_PIN, OUTPUT);
    pinMode(RIGHT_ENGAGE_PIN, OUTPUT);

    while(gStatus != WL_CONNECTED)
    {
        gStatus = WiFi.begin("", "");
        delay(DEFAULT_WAIT_TIME_MS);
    }
}
const char* platform_getName()
{
    return "Intel Edison";
}
void platform_send(void* params)
{
    http_request("POST /CommandServer/currentJsonCommand HTTP/1.1\nHost: 192.168.1.6:8080\nAccept: */*\nContent-Length: 15\nContent-Type: application/x-www-form-urlencoded\nConnection: close\n\nPLATFORM:EDISON\n", NULL, 0);
}
char gResponse[256];
int platform_getCommand()
{
    memset(gResponse, '\0', 256);
    http_request("GET /CommandServer/currentJsonCommand HTTP/1.1\nHost: 192.168.1.6:8080\nAccept: */*\n", gResponse, 256);

    return IDLE;
}
void platform_cleanup()
{
}
#else
void platform_setup()
{
    curl_global_init(CURL_GLOBAL_DEFAULT);
}
const char* platform_getName()
{
    return "Desktop";
}
void platform_send(void* params)
{
    CURL* pCURL = curl_easy_init();
    if(pCURL)
    {
        char address_buffer[64];
        getAddress(address_buffer, 64);
        curl_easy_setopt(pCURL, CURLOPT_URL, address_buffer);
        curl_easy_setopt(pCURL, CURLOPT_POSTFIELDS, params);
        CURLcode result = curl_easy_perform(pCURL);
        if(result == CURLE_OK)
        {
        }
        curl_easy_cleanup(pCURL);
    }
}
int platform_getCommand()
{
    int command = IDLE;
    CURL* pCURL = curl_easy_init();
    if(pCURL)
    {
        int temp;
        char address_buffer[64];
        getAddress(address_buffer, 64);
        curl_easy_setopt(pCURL, CURLOPT_URL, address_buffer);
        curl_easy_setopt(pCURL, CURLOPT_WRITEFUNCTION, curl_write_json_function);
        curl_easy_setopt(pCURL, CURLOPT_WRITEDATA, &temp);
        CURLcode result = curl_easy_perform(pCURL);
        if(result == CURLE_OK)
        {
            command = temp;
        }
        curl_easy_cleanup(pCURL);
    }

    return command;
}
void platform_cleanup()
{
    curl_global_cleanup();
}
#endif

<commit_msg>Parse HttpResponse<commit_after>#include "parser.h"
#include "platform.h"
#include <assert.h>

extern "C" void parse(const char* json, command* wifiRC_command);

#ifndef DESKTOP
#include "edison_platform.h"
#else
#include "desktop_platform.h"
#endif

#define RC_ADDRESS_BASE "http://192.168.1."
#define RC_JSON_ADDRESS_SUFFIX ":8080/CommandServer/currentJsonCommand"

static void getAddress(char* address_buffer, int size)
{
    assert(size == 64);

//Need to do something smarter here like search for servers on the local network
    char buffer[4];
    memset(buffer, '\0', 4);
    memset(address_buffer, '\0', 64);
    strcat(address_buffer, RC_ADDRESS_BASE);
    sprintf(buffer, "%i", DEV);
    strcat(address_buffer, buffer);
    strcat(address_buffer, RC_JSON_ADDRESS_SUFFIX);
}

#ifndef DESKTOP
void platform_setup()
{
    pthread_mutex_init(&gHttpMutex, NULL);

    pinMode(LIGHTS_ENABLE_PIN, OUTPUT);
    pinMode(DRIVE_MOTOR_PIN, OUTPUT);
    pinMode(REVERSE_ENGAGE_PIN, OUTPUT);
    pinMode(RIGHT_ENGAGE_PIN, OUTPUT);

    while(gStatus != WL_CONNECTED)
    {
        gStatus = WiFi.begin("", "");
        delay(DEFAULT_WAIT_TIME_MS);
    }
}
const char* platform_getName()
{
    return "Intel Edison";
}
void platform_send(void* params)
{
    http_request("POST /CommandServer/currentJsonCommand HTTP/1.1\nHost: 192.168.1.6:8080\nAccept: */*\nContent-Length: 15\nContent-Type: application/x-www-form-urlencoded\nConnection: close\n\nPLATFORM:EDISON\n", NULL, 0);
}
char gJson[128];
char gResponse[256];
int platform_getCommand()
{
    memset(gJson, '\0', 128);
    memset(gResponse, '\0', 256);
    http_request("GET /CommandServer/currentJsonCommand HTTP/1.1\nHost: 192.168.1.6:8080\nAccept: */*\n", gResponse, 256);

    char* start = NULL;
    char* length = NULL;
    start = strchr(gResponse, '\n')+1;
    start = strchr(start, '\n')+1;
    length = strchr(start, ':')+1;
    int size = atoi(length);
    start = strchr(gResponse, '{');
    for(int i = 0; i < size; i++)
    {
        gJson[i] = *start;
        start++;
    }

    set_command command;
    parse(start, &command);

    return command.get_direction();

}
void platform_cleanup()
{
}
#else
void platform_setup()
{
    curl_global_init(CURL_GLOBAL_DEFAULT);
}
const char* platform_getName()
{
    return "Desktop";
}
void platform_send(void* params)
{
    CURL* pCURL = curl_easy_init();
    if(pCURL)
    {
        char address_buffer[64];
        getAddress(address_buffer, 64);
        curl_easy_setopt(pCURL, CURLOPT_URL, address_buffer);
        curl_easy_setopt(pCURL, CURLOPT_POSTFIELDS, params);
        CURLcode result = curl_easy_perform(pCURL);
        if(result == CURLE_OK)
        {
        }
        curl_easy_cleanup(pCURL);
    }
}
int platform_getCommand()
{
    int command = IDLE;
    CURL* pCURL = curl_easy_init();
    if(pCURL)
    {
        int temp;
        char address_buffer[64];
        getAddress(address_buffer, 64);
        curl_easy_setopt(pCURL, CURLOPT_URL, address_buffer);
        curl_easy_setopt(pCURL, CURLOPT_WRITEFUNCTION, curl_write_json_function);
        curl_easy_setopt(pCURL, CURLOPT_WRITEDATA, &temp);
        CURLcode result = curl_easy_perform(pCURL);
        if(result == CURLE_OK)
        {
            command = temp;
        }
        curl_easy_cleanup(pCURL);
    }

    return command;
}
void platform_cleanup()
{
    curl_global_cleanup();
}
#endif

<|endoftext|>


<commit_before>/*
 * Copyright 2019 Benoy Bose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software
 * and associated documentation files (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy, modify, merge, publish, distribute,
 * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef HOOLANG_OPCODEPREFIXES_HH
#define HOOLANG_OPCODEPREFIXES_HH

#include <cstdint>

namespace hooc {
    namespace emitter {
        namespace x86 {
            const uint8_t PREFIX_LOCK = 0xF0;
            const uint8_t PREFIX_REPNE = 0xF2;
            const uint8_t PREFIX_REPNZ = 0xF2;
            const uint8_t PREFIX_REP = 0xF3;
            const uint8_t PREFIX_REPE = 0xF3;
            const uint8_t PREFIX_REPZ = 0xF3;

            const uint8_t PREFIX_SEGMENT_CS = 0x2E;
            const uint8_t PREFIX_SEGMENT_SS = 0x36;
            const uint8_t PREFIX_SEGMENT_DS = 0x3E;
            const uint8_t PREFIX_SEGMENT_ES = 0x26;
            const uint8_t PREFIX_SEGMENT_FS = 0x64;
            const uint8_t PREFIX_SEGMENT_GS = 0x65;
            const uint8_t PREFIX_BRANCH_NOT_TAKEN = 0x2E;
            const uint8_t PREFIX_BRANCH_TAKEN = 0x3E;

            const uint8_t PREFIX_OPERAND_SIZE_OVERRIDE = 0x66;
            const uint8_t PREFIX_ADDRESS_SIZE_OVERRIDE = 0x67;
        }
    }
}

#endif //HOOLANG_OPCODEPREFIXES_HH
<commit_msg>Opcode prefix constants are changed to enum values.<commit_after>/*
 * Copyright 2019 Benoy Bose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software
 * and associated documentation files (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy, modify, merge, publish, distribute,
 * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef HOOLANG_OPCODEPREFIXES_HH
#define HOOLANG_OPCODEPREFIXES_HH

#include <cstdint>

namespace hooc {
    namespace emitter {
        namespace x86 {
            typedef enum {
                PREFIX_G1_LOCK = 0xF0,
                PREFIX_G1_REPN = 0xF2, // REPNE, REPNZ
                PREFIX_G1_REP = 0xF3, // REP, REPE, REPZ
                PREFIX_G2_SEGMENT_CS = 0x2E,
                PREFIX_G2_SEGMENT_SS = 0x36,
                PREFIX_G2_SEGMENT_DS = 0x3E,
                PREFIX_G2_SEGMENT_ES = 0x26,
                PREFIX_G2_SEGMENT_FS = 0x64,
                PREFIX_G2_SEGMENT_GS = 0x65,
                PREFIX_G2_BRANCH_NOT_TAKEN = 0x2E,
                PREFIX_G2_BRANCH_TAKEN = 0x3E,
                PREFIX_G3_OPERAND_SIZE_OVERRIDE = 0x66,
                PREFIX_G4_ADDRESS_SIZE_OVERRIDE = 0x67
            } OpcodePrefix;
        }
    }
}

#endif //HOOLANG_OPCODEPREFIXES_HH
<|endoftext|>


<commit_before>#pragma once
#include <stdint.h>
#include <string.h>
#include <cassert>
#include <iostream>
#include <math.h>

namespace gemmbitserial {

// Utility function to increment-and-align "in" to "af"
inline uint64_t alignTo(uint64_t in, uint64_t af) {
  if(in % af != 0) {
    return in + (af - (in % af));
  } else {
    return in;
  }
}

class BitSerialMatrix {
public:
  // static member functions for working with BitSerialMatrix

  /* Allocate buffer space for a BitSerialMatrix */
  static BitSerialMatrix alloc(uint64_t nbits, uint64_t nrows, uint64_t ncols, bool issigned, uint64_t rowalign = 1, uint64_t colalign = 64) {
    BitSerialMatrix bsm;
    bsm.nbits = nbits;
    bsm.nrows = nrows;
    bsm.ncols = ncols;
    bsm.nrows_a = alignTo(nrows, rowalign);
    bsm.ncols_a = alignTo(ncols, colalign);
    bsm.issigned = issigned;
    uint64_t wordsPerBitplane = bsm.wordsPerBitplane();
    bsm.data = new uint64_t[nbits * wordsPerBitplane];
    return bsm;
  }

  /* Deallocate buffers for a BitSerialMatrix */
  static void dealloc(BitSerialMatrix bsm) {
    delete [] bsm.data;
  }

public:
  // actual member variables and functions of BitSerialMatrix instances
  bool issigned;        // whether highest order bit pos is negative
  uint64_t nbits;       // bits of precision
  uint64_t nrows;       // number of real (actual) rows
  uint64_t ncols;       // number of real (actual) columns
  uint64_t nrows_a;     // number of allocated rows
  uint64_t ncols_a;     // number of allocated columns
  uint64_t * data;      // data buffer, layout [nbits][nrows_a][ncols_a/64]

  // print key statistics about BitSerialMatrix to stdout
  void printSummary() {
    std::cout << "BitSerialMatrix" << std::endl;
    std::cout << "Bits of precision: " << nbits << " signed: " << issigned << std::endl;
    std::cout << "Actual size: " << nrows << " x " << ncols << std::endl;
    std::cout << "Allocated size: " << nrows_a << " x " << ncols_a << std::endl;
  }

  // number of storage words needed for each row
  inline uint64_t wordsPerRow() const {
    const uint64_t bitsPerWord = sizeof(uint64_t) * 8;
    return ncols_a / bitsPerWord;
  }

  // number of storage words needed for each bitplane (bit matrix)
  inline uint64_t wordsPerBitplane() const {
    return nrows_a * wordsPerRow();
  }

  // get given bit. true if set, false if unset.
  inline bool get(uint64_t bit, uint64_t row, uint64_t col) {
    return ((word(bit, row, col) >> bitpos(col)) & 1L) == 1;
  }

  // set all bits to zero
  inline void clearAll() {
    memset(data, 0, nbits * wordsPerBitplane() * sizeof(uint64_t));
  }

  // set given bit to one
  inline void set(uint64_t bit, uint64_t row, uint64_t col) {
    word(bit, row, col) |= (1L << bitpos(col));
  }

  // set given bit to zero
  inline void unset(uint64_t bit, uint64_t row, uint64_t col) {
    word(bit, row, col) &= ~(1L << bitpos(col));
  }

  // access the container word for a given bit
  inline uint64_t & word(uint64_t bit, uint64_t row, uint64_t col) {
    // right shift by log2(bits per word) to get word index
    uint64_t colw = col >> 6;
    return data[bit * wordsPerBitplane() + row * wordsPerRow() + colw];
  }

  // get a pointer to a particular row
  inline uint64_t * rowptr(uint64_t bit, uint64_t row) {
    return &data[bit * wordsPerBitplane() + row * wordsPerRow()];
  }

  // get a pointer to a particular bit plane
  inline uint64_t * bitplaneptr(uint64_t bit) {
    return &data[bit * wordsPerBitplane()];
  }

  uint64_t bitpos(uint64_t col) {
    // return modulo 64 of col by using a bitmask
    return col & ((1 << 6) - 1);
  }

  /* Imports a regular matrix into this BitSerialMatrix.
  */
  template <typename T>
  void importRegular(T * matrix, bool readColMajor=false) {
    // TODO add support for transposed reading
    assert(!readColMajor);
    this->clearAll();
    for(uint64_t r = 0; r < this->nrows; r++) {
      for(uint64_t c = 0; c < this->ncols; c++) {
        uint8_t currentElem = (uint8_t) matrix[r * this->ncols + c];
        for(uint64_t b = 0; b < this->nbits; b++) {
          if(currentElem & (1 << b)) {
            this->set(b, r, c);
          }
        }
      }
    }
  }

  /* Convert this BitSerialMatrix back to a regular matrix.
  */
  template <typename T>
  void exportRegular(T * matrix) {
    for(uint64_t r = 0; r < this->nrows; r++) {
      for(uint64_t c = 0; c < this->ncols; c++) {
        uint8_t currentElem = 0;
        for(uint64_t b = 0; b < this->nbits; b++) {
          if(this->get(b, r, c)) {
            currentElem |= 1 << b;
          }
        }
        matrix[r * this->ncols + c] = (T) currentElem;
      }
    }
  }
};

/* Utility function to find block size under the following assumptions:
   - size of lhs block + rhs block + result block <= cacheBits
   - no blocking along depth (i.e. only entire rows of dBits bits)
   - lhsMult and rhsMult determine the ratio for lhs and rhs rows in cache
   - returned lhsRows and rhsRows are divisible by lhsMult and rhsMult, respectively
   - each result elem takes bitsPerRes bits
*/
void computeBlockSize(float lhsMult, float rhsMult, float cacheBits, float dBits, uint64_t & lhsBlock, uint64_t & rhsBlock) {
  float a = sizeof(int32_t) * lhsMult * rhsMult;
  float b = dBits*(lhsMult + rhsMult);
  float c = -cacheBits;
  float discr = sqrt(b*b - 4 * a * c);
  assert(discr > 0);
  int64_t x0 = floor((-b + discr) / (2*a));
  int64_t x1 = floor((-b - discr) / (2*a));
  int64_t x = x0 > x1 ? x0 : x1;
  assert(x > 0);
  lhsBlock = lhsMult * x;
  rhsBlock = rhsMult * x;
};

class GEMMContext {
public:
  BitSerialMatrix lhs, rhs;
  uint64_t lhsBlock, rhsBlock;
  int32_t * res;

  void printSummary() {
    std::cout << "GEMMContext" << std::endl;
    std::cout << "LHS: ";
    lhs.printSummary();
    std::cout << "Block size: " << lhsBlock << std::endl;
    std::cout << "RHS: ";
    rhs.printSummary();
    std::cout << "Block size: " << rhsBlock << std::endl;
  }
};

void deallocGEMMContext(GEMMContext ctx) {
  delete [] ctx.res;
  BitSerialMatrix::dealloc(ctx.lhs);
  BitSerialMatrix::dealloc(ctx.rhs);
};

// generic implementations using regular & and __builtin_popcountll
#include "arch-generic.hpp"

// select the implementations to be used based on architecture
#if defined(__ARM_NEON) || defined(__aarch64__)
#warning "Compiling with ARM NEON"
#include "arch-neon.hpp"
// ARM NEON-specific implementations
#define gemmBitSerial   gemmBitSerial_neon_usingBinary
// TODO context def
#else
#warning "Compiling using generic popcount"
#define gemmBitSerial     gemmBitSerial_generic_usingBinary
#define allocGEMMContext  allocGEMMContext_generic
#endif

}
<commit_msg>print actual, allocated and wasted ops<commit_after>#pragma once
#include <stdint.h>
#include <string.h>
#include <cassert>
#include <iostream>
#include <math.h>

namespace gemmbitserial {

// Utility function to increment-and-align "in" to "af"
inline uint64_t alignTo(uint64_t in, uint64_t af) {
  if(in % af != 0) {
    return in + (af - (in % af));
  } else {
    return in;
  }
}

class BitSerialMatrix {
public:
  // static member functions for working with BitSerialMatrix

  /* Allocate buffer space for a BitSerialMatrix */
  static BitSerialMatrix alloc(uint64_t nbits, uint64_t nrows, uint64_t ncols, bool issigned, uint64_t rowalign = 1, uint64_t colalign = 64) {
    BitSerialMatrix bsm;
    bsm.nbits = nbits;
    bsm.nrows = nrows;
    bsm.ncols = ncols;
    bsm.nrows_a = alignTo(nrows, rowalign);
    bsm.ncols_a = alignTo(ncols, colalign);
    bsm.issigned = issigned;
    uint64_t wordsPerBitplane = bsm.wordsPerBitplane();
    bsm.data = new uint64_t[nbits * wordsPerBitplane];
    return bsm;
  }

  /* Deallocate buffers for a BitSerialMatrix */
  static void dealloc(BitSerialMatrix bsm) {
    delete [] bsm.data;
  }

public:
  // actual member variables and functions of BitSerialMatrix instances
  bool issigned;        // whether highest order bit pos is negative
  uint64_t nbits;       // bits of precision
  uint64_t nrows;       // number of real (actual) rows
  uint64_t ncols;       // number of real (actual) columns
  uint64_t nrows_a;     // number of allocated rows
  uint64_t ncols_a;     // number of allocated columns
  uint64_t * data;      // data buffer, layout [nbits][nrows_a][ncols_a/64]

  // print key statistics about BitSerialMatrix to stdout
  void printSummary() {
    std::cout << "BitSerialMatrix" << std::endl;
    std::cout << "Bits of precision: " << nbits << " signed: " << issigned << std::endl;
    std::cout << "Actual size: " << nrows << " x " << ncols << std::endl;
    std::cout << "Allocated size: " << nrows_a << " x " << ncols_a << std::endl;
    std::cout << "Actual ops: " << 2*nrows*ncols << std::endl;
    std::cout << "Allocated ops: " << 2*nrows_a*ncols_a << std::endl;
    std::cout << "Wasted ops: " << 2*nrows_a*ncols_a-2*nrows*ncols << std::endl;
  }

  // number of storage words needed for each row
  inline uint64_t wordsPerRow() const {
    const uint64_t bitsPerWord = sizeof(uint64_t) * 8;
    return ncols_a / bitsPerWord;
  }

  // number of storage words needed for each bitplane (bit matrix)
  inline uint64_t wordsPerBitplane() const {
    return nrows_a * wordsPerRow();
  }

  // get given bit. true if set, false if unset.
  inline bool get(uint64_t bit, uint64_t row, uint64_t col) {
    return ((word(bit, row, col) >> bitpos(col)) & 1L) == 1;
  }

  // set all bits to zero
  inline void clearAll() {
    memset(data, 0, nbits * wordsPerBitplane() * sizeof(uint64_t));
  }

  // set given bit to one
  inline void set(uint64_t bit, uint64_t row, uint64_t col) {
    word(bit, row, col) |= (1L << bitpos(col));
  }

  // set given bit to zero
  inline void unset(uint64_t bit, uint64_t row, uint64_t col) {
    word(bit, row, col) &= ~(1L << bitpos(col));
  }

  // access the container word for a given bit
  inline uint64_t & word(uint64_t bit, uint64_t row, uint64_t col) {
    // right shift by log2(bits per word) to get word index
    uint64_t colw = col >> 6;
    return data[bit * wordsPerBitplane() + row * wordsPerRow() + colw];
  }

  // get a pointer to a particular row
  inline uint64_t * rowptr(uint64_t bit, uint64_t row) {
    return &data[bit * wordsPerBitplane() + row * wordsPerRow()];
  }

  // get a pointer to a particular bit plane
  inline uint64_t * bitplaneptr(uint64_t bit) {
    return &data[bit * wordsPerBitplane()];
  }

  uint64_t bitpos(uint64_t col) {
    // return modulo 64 of col by using a bitmask
    return col & ((1 << 6) - 1);
  }

  /* Imports a regular matrix into this BitSerialMatrix.
  */
  template <typename T>
  void importRegular(T * matrix, bool readColMajor=false) {
    // TODO add support for transposed reading
    assert(!readColMajor);
    this->clearAll();
    for(uint64_t r = 0; r < this->nrows; r++) {
      for(uint64_t c = 0; c < this->ncols; c++) {
        uint8_t currentElem = (uint8_t) matrix[r * this->ncols + c];
        for(uint64_t b = 0; b < this->nbits; b++) {
          if(currentElem & (1 << b)) {
            this->set(b, r, c);
          }
        }
      }
    }
  }

  /* Convert this BitSerialMatrix back to a regular matrix.
  */
  template <typename T>
  void exportRegular(T * matrix) {
    for(uint64_t r = 0; r < this->nrows; r++) {
      for(uint64_t c = 0; c < this->ncols; c++) {
        uint8_t currentElem = 0;
        for(uint64_t b = 0; b < this->nbits; b++) {
          if(this->get(b, r, c)) {
            currentElem |= 1 << b;
          }
        }
        matrix[r * this->ncols + c] = (T) currentElem;
      }
    }
  }
};

/* Utility function to find block size under the following assumptions:
   - size of lhs block + rhs block + result block <= cacheBits
   - no blocking along depth (i.e. only entire rows of dBits bits)
   - lhsMult and rhsMult determine the ratio for lhs and rhs rows in cache
   - returned lhsRows and rhsRows are divisible by lhsMult and rhsMult, respectively
   - each result elem takes bitsPerRes bits
*/
void computeBlockSize(float lhsMult, float rhsMult, float cacheBits, float dBits, uint64_t & lhsBlock, uint64_t & rhsBlock) {
  float a = sizeof(int32_t) * lhsMult * rhsMult;
  float b = dBits*(lhsMult + rhsMult);
  float c = -cacheBits;
  float discr = sqrt(b*b - 4 * a * c);
  assert(discr > 0);
  int64_t x0 = floor((-b + discr) / (2*a));
  int64_t x1 = floor((-b - discr) / (2*a));
  int64_t x = x0 > x1 ? x0 : x1;
  assert(x > 0);
  lhsBlock = lhsMult * x;
  rhsBlock = rhsMult * x;
};

class GEMMContext {
public:
  BitSerialMatrix lhs, rhs;
  uint64_t lhsBlock, rhsBlock;
  int32_t * res;

  void printSummary() {
    std::cout << "GEMMContext" << std::endl;
    std::cout << "LHS: ";
    lhs.printSummary();
    std::cout << "Block size: " << lhsBlock << std::endl;
    std::cout << "RHS: ";
    rhs.printSummary();
    std::cout << "Block size: " << rhsBlock << std::endl;
  }
};

void deallocGEMMContext(GEMMContext ctx) {
  delete [] ctx.res;
  BitSerialMatrix::dealloc(ctx.lhs);
  BitSerialMatrix::dealloc(ctx.rhs);
};

// generic implementations using regular & and __builtin_popcountll
#include "arch-generic.hpp"

// select the implementations to be used based on architecture
#if defined(__ARM_NEON) || defined(__aarch64__)
#warning "Compiling with ARM NEON"
#include "arch-neon.hpp"
// ARM NEON-specific implementations
#define gemmBitSerial   gemmBitSerial_neon_usingBinary
// TODO context def
#else
#warning "Compiling using generic popcount"
#define gemmBitSerial     gemmBitSerial_generic_usingBinary
#define allocGEMMContext  allocGEMMContext_generic
#endif

}
<|endoftext|>


<commit_before>#ifndef TPLEXPREFIXTREE_HH
#define TPLEXPREFIXTREE_HH

#include <vector>
#include <assert.h>

#include "HashCache.hh"
#include "SimpleHashCache.hh"

#include "Hmm.hh"
#include "Vocabulary.hh"

// Constructs and maintains the lexical prefix tree.

// NOTE!
// - Assumes that the transitions from the hmm states with identical
//   mixture models are the same, although with different destinations.
// - HMMs must have left-to-right topology, skip states are allowed,
//   except from the source state, see the next note.
// - If there are transitions from HMM source state to other states than to
//   the first real state and to the sink state, the tree construction might
//   not work with shared HMM states.

// Node flags
#define NODE_NORMAL               0x00
#define NODE_USE_WORD_END_BEAM    0x01
#define NODE_AFTER_WORD_ID        0x02
#define NODE_FAN_OUT              0x04
#define NODE_FAN_OUT_FIRST        0x08
#define NODE_FAN_IN               0x10
#define NODE_FAN_IN_FIRST         0x20
#define NODE_INSERT_WORD_BOUNDARY 0x40
#define NODE_FAN_IN_CONNECTION    0x80
#define NODE_LINKED               0x0100
#define NODE_SILENCE_FIRST        0x0200
#define NODE_FIRST_STATE_OF_WORD  0x0400

class TPLexPrefixTree {
public:

  class Node;

  class WordHistory {
  public:
    inline WordHistory(int word_id, int lm_id, class WordHistory *prev);
    inline void link() { m_reference_count++; }
    inline static void unlink(class WordHistory *hist);
    inline int get_num_references(void) { return m_reference_count; }

    int word_id;
    int lm_id; // Word ID in LM
    WordHistory *prev_word;
    bool printed; // TokenPassSearch::print* functions may set this true
    int word_start_frame;
  private:
    int m_reference_count;
  };

  class StateHistory {
  public:
    inline StateHistory(int hmm_model, int start_time,
                        class StateHistory *prev);

    inline void link() { m_reference_count++; }
    inline static void unlink(class StateHistory *hist);

    int hmm_model;
    int start_time;
    StateHistory *prev;
  private:
    int m_reference_count;
  };

  class PathHistory {
  public:
    inline PathHistory(float ll, float dll, int depth, class PathHistory *p);

    inline void link() { m_reference_count++; }
    inline static void unlink(class PathHistory *hist);

    float ll;
    float dll;
    int depth;
    PathHistory *prev;
  private:
    int m_reference_count;
  };

  class Token {
  public:
    Node *node;
    Token *next_node_token;
    float am_log_prob;
    float lm_log_prob;
    float cur_am_log_prob; // Used inside nodes
    float cur_lm_log_prob; // Used for LM lookahead
    float total_log_prob;
    WordHistory *prev_word;
    int word_hist_code; // Hash code for word history (up to LM order)
    int word_start_frame;

#ifdef PRUNING_MEASUREMENT
    float meas[6];
#endif

    int word_count;
    StateHistory *state_history;

    //PathHistory *token_path;
    unsigned char depth;
    unsigned char dur;
  };

  class Arc {
  public:
    float log_prob;
    Node *next;
  };

  class Node {
  public:
    inline Node() : word_id(-1), state(NULL), token_list(NULL), flags(NODE_NORMAL) { }
    inline Node(int wid) : word_id(wid), state(NULL), token_list(NULL), flags(NODE_NORMAL) { }
    inline Node(int wid, HmmState *s) : word_id(wid), state(s), token_list(NULL), flags(NODE_NORMAL) { }
    int word_id; // -1 if none
    int node_id;
    HmmState *state;
    Token *token_list;
    std::vector<Arc> arcs;

    unsigned short flags;

    std::vector<int> possible_word_id_list;
    SimpleHashCache<float> lm_lookahead_buffer;
  };

  struct NodeArcId {
    Node *node;
    int arc_index;
  };

  TPLexPrefixTree(std::map<std::string,int> &hmm_map, std::vector<Hmm> &hmms);
  inline TPLexPrefixTree::Node *root() { return m_root_node; }
  inline TPLexPrefixTree::Node *start_node() { return m_start_node; }
  inline int words() const { return m_words; }

  void set_verbose(int verbose) { m_verbose = verbose; }
  void set_lm_lookahead(int lm_lookahead) { m_lm_lookahead = lm_lookahead; }
  void set_cross_word_triphones(bool cw_triphones) { m_cross_word_triphones = cw_triphones; }
  void set_silence_is_word(bool b) { m_silence_is_word = b; }
  void set_ignore_case(bool b) { m_ignore_case = b; }

  void initialize_lex_tree(void);
  void add_word(std::vector<Hmm*> &hmm_list, int word_id);
  void finish_tree(void);

  void prune_lookahead_buffers(int min_delta, int max_depth);
  void set_lm_lookahead_cache_sizes(int cache_size);

  void set_word_boundary_id(int id) { m_word_boundary_id = id; }
  void set_optional_short_silence(bool state) { m_optional_short_silence = state; }
  void set_sentence_boundary(int sentence_end_id);

  void clear_node_token_lists(void);

  void print_node_info(int node);
  void print_lookahead_info(int node, const Vocabulary &voc);

private:
  void expand_lexical_tree(Node *source, Hmm *hmm, HmmTransition &t,
                           float cur_trans_log_prob,
                           int word_end,
                           std::vector<Node*> &hmm_state_nodes,
                           std::vector<Node*> &sink_nodes,
                           std::vector<float> &sink_trans_log_probs,
                           unsigned short flags);
  void post_process_lex_branch(Node *node, std::vector<int> *lm_la_list);
  bool post_process_fan_triphone(Node *node, std::vector<int> *lm_la_list,
                                 bool fan_in);

  void create_cross_word_network(void);
  void add_hmm_to_fan_network(int hmm_id,
                              bool fan_out);
  void link_fan_out_node_to_fan_in(Node *node, const std::string &key);
  void link_node_to_fan_network(const std::string &key,
                                std::vector<Arc> &source_arcs,
                                bool fan_out,
                                bool ignore_length,
                                float out_transition_log_prob);
  void add_single_hmm_word_for_cross_word_modeling(Hmm *hmm, int word_id);
  void link_fan_in_nodes(void);
  void create_lex_tree_links_from_fan_in(Node *fan_in_node,
                                         const std::string &key);

  void analyze_cross_word_network(void);
  void count_fan_size(Node *node, unsigned short flag,
                      int *num_nodes, int *num_arcs);
  void count_prefix_tree_size(Node *node, int *num_nodes,
                              int *num_arcs);

  void free_cross_word_network_connection_points(void);
  Node* get_short_silence_node(void);
  Node* get_fan_out_entry_node(HmmState *state, const std::string &label);
  Node* get_fan_out_last_node(HmmState *state, const std::string &label);
  Node* get_fan_in_entry_node(HmmState *state, const std::string &label);
  Node* get_fan_in_last_node(HmmState *state, const std::string &label);
  Node* get_fan_state_node(HmmState *state, std::vector<Node*> *nodes);
  std::vector<Node*>* get_fan_node_list(
    const std::string &key,
    std::map< std::string, std::vector<Node*>* > &nmap);
  void add_fan_in_connection_node(Node *node, const std::string &prev_label);

  float get_out_transition_log_prob(Node *node);

  void prune_lm_la_buffer(int delta_thr, int depth_thr,
                          Node *node, int last_size, int cur_depth);

private:
  int m_words; // Largest word_id in the nodes plus one
  Node *m_root_node;
  Node *m_end_node;
  Node *m_start_node;
  Node *m_silence_node;
  Node *m_last_silence_node;
  std::vector<Node*> node_list;
  int m_verbose;
  int m_lm_lookahead; // 0=None, 1=Only in first subtree nodes,
                      // 2=Full
  bool m_cross_word_triphones;
  int m_lm_buf_count;

  bool m_silence_is_word;
  bool m_ignore_case;
  bool m_optional_short_silence;
  HmmState *m_short_silence_state;
  int m_word_boundary_id;

  std::map<std::string,int> &m_hmm_map;
  std::vector<Hmm> &m_hmms;

  std::map< std::string, std::vector<Node*>* > m_fan_out_entry_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_out_last_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_entry_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_last_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_connection_nodes;
  std::vector<NodeArcId> m_silence_arcs;
};

//////////////////////////////////////////////////////////////////////

TPLexPrefixTree::WordHistory::WordHistory(int word_id, int lm_id,
                                          class WordHistory *prev)
  : word_id(word_id),
    lm_id(lm_id),
    prev_word(prev),
    printed(false),
    word_start_frame(0),
    m_reference_count(0)
{
  if (prev)
    prev->link();
}

void
TPLexPrefixTree::WordHistory::unlink(class WordHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      WordHistory *prev = hist->prev_word;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

TPLexPrefixTree::PathHistory::PathHistory(float ll, float dll, int depth,
                                          class PathHistory *p)
  : ll(ll),
    dll(dll),
    depth(depth),
    prev(p),
    m_reference_count(0)
{
  if (p)
    p->link();
}

void
TPLexPrefixTree::PathHistory::unlink(class PathHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      PathHistory *prev = hist->prev;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

TPLexPrefixTree::StateHistory::StateHistory(int hmm_model, int start_time,
                                            class StateHistory *prev)
  : hmm_model(hmm_model),
    start_time(start_time),
    prev(prev),
    m_reference_count(0)
{
  if (prev)
    prev->link();
}

void
TPLexPrefixTree::StateHistory::unlink(class StateHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      StateHistory *prev = hist->prev;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

#endif /* TPLEXPREFIXTREE_HH */
<commit_msg>include cmath<commit_after>#ifndef TPLEXPREFIXTREE_HH
#define TPLEXPREFIXTREE_HH

#include <vector>
#include <assert.h>
#include <cmath>

#include "HashCache.hh"
#include "SimpleHashCache.hh"

#include "Hmm.hh"
#include "Vocabulary.hh"

// Constructs and maintains the lexical prefix tree.

// NOTE!
// - Assumes that the transitions from the hmm states with identical
//   mixture models are the same, although with different destinations.
// - HMMs must have left-to-right topology, skip states are allowed,
//   except from the source state, see the next note.
// - If there are transitions from HMM source state to other states than to
//   the first real state and to the sink state, the tree construction might
//   not work with shared HMM states.

// Node flags
#define NODE_NORMAL               0x00
#define NODE_USE_WORD_END_BEAM    0x01
#define NODE_AFTER_WORD_ID        0x02
#define NODE_FAN_OUT              0x04
#define NODE_FAN_OUT_FIRST        0x08
#define NODE_FAN_IN               0x10
#define NODE_FAN_IN_FIRST         0x20
#define NODE_INSERT_WORD_BOUNDARY 0x40
#define NODE_FAN_IN_CONNECTION    0x80
#define NODE_LINKED               0x0100
#define NODE_SILENCE_FIRST        0x0200
#define NODE_FIRST_STATE_OF_WORD  0x0400

class TPLexPrefixTree {
public:

  class Node;

  class WordHistory {
  public:
    inline WordHistory(int word_id, int lm_id, class WordHistory *prev);
    inline void link() { m_reference_count++; }
    inline static void unlink(class WordHistory *hist);
    inline int get_num_references(void) { return m_reference_count; }

    int word_id;
    int lm_id; // Word ID in LM
    WordHistory *prev_word;
    bool printed; // TokenPassSearch::print* functions may set this true
    int word_start_frame;
  private:
    int m_reference_count;
  };

  class StateHistory {
  public:
    inline StateHistory(int hmm_model, int start_time,
                        class StateHistory *prev);

    inline void link() { m_reference_count++; }
    inline static void unlink(class StateHistory *hist);

    int hmm_model;
    int start_time;
    StateHistory *prev;
  private:
    int m_reference_count;
  };

  class PathHistory {
  public:
    inline PathHistory(float ll, float dll, int depth, class PathHistory *p);

    inline void link() { m_reference_count++; }
    inline static void unlink(class PathHistory *hist);

    float ll;
    float dll;
    int depth;
    PathHistory *prev;
  private:
    int m_reference_count;
  };

  class Token {
  public:
    Node *node;
    Token *next_node_token;
    float am_log_prob;
    float lm_log_prob;
    float cur_am_log_prob; // Used inside nodes
    float cur_lm_log_prob; // Used for LM lookahead
    float total_log_prob;
    WordHistory *prev_word;
    int word_hist_code; // Hash code for word history (up to LM order)
    int word_start_frame;

#ifdef PRUNING_MEASUREMENT
    float meas[6];
#endif

    int word_count;
    StateHistory *state_history;

    //PathHistory *token_path;
    unsigned char depth;
    unsigned char dur;
  };

  class Arc {
  public:
    float log_prob;
    Node *next;
  };

  class Node {
  public:
    inline Node() : word_id(-1), state(NULL), token_list(NULL), flags(NODE_NORMAL) { }
    inline Node(int wid) : word_id(wid), state(NULL), token_list(NULL), flags(NODE_NORMAL) { }
    inline Node(int wid, HmmState *s) : word_id(wid), state(s), token_list(NULL), flags(NODE_NORMAL) { }
    int word_id; // -1 if none
    int node_id;
    HmmState *state;
    Token *token_list;
    std::vector<Arc> arcs;

    unsigned short flags;

    std::vector<int> possible_word_id_list;
    SimpleHashCache<float> lm_lookahead_buffer;
  };

  struct NodeArcId {
    Node *node;
    int arc_index;
  };

  TPLexPrefixTree(std::map<std::string,int> &hmm_map, std::vector<Hmm> &hmms);
  inline TPLexPrefixTree::Node *root() { return m_root_node; }
  inline TPLexPrefixTree::Node *start_node() { return m_start_node; }
  inline int words() const { return m_words; }

  void set_verbose(int verbose) { m_verbose = verbose; }
  void set_lm_lookahead(int lm_lookahead) { m_lm_lookahead = lm_lookahead; }
  void set_cross_word_triphones(bool cw_triphones) { m_cross_word_triphones = cw_triphones; }
  void set_silence_is_word(bool b) { m_silence_is_word = b; }
  void set_ignore_case(bool b) { m_ignore_case = b; }

  void initialize_lex_tree(void);
  void add_word(std::vector<Hmm*> &hmm_list, int word_id);
  void finish_tree(void);

  void prune_lookahead_buffers(int min_delta, int max_depth);
  void set_lm_lookahead_cache_sizes(int cache_size);

  void set_word_boundary_id(int id) { m_word_boundary_id = id; }
  void set_optional_short_silence(bool state) { m_optional_short_silence = state; }
  void set_sentence_boundary(int sentence_end_id);

  void clear_node_token_lists(void);

  void print_node_info(int node);
  void print_lookahead_info(int node, const Vocabulary &voc);

private:
  void expand_lexical_tree(Node *source, Hmm *hmm, HmmTransition &t,
                           float cur_trans_log_prob,
                           int word_end,
                           std::vector<Node*> &hmm_state_nodes,
                           std::vector<Node*> &sink_nodes,
                           std::vector<float> &sink_trans_log_probs,
                           unsigned short flags);
  void post_process_lex_branch(Node *node, std::vector<int> *lm_la_list);
  bool post_process_fan_triphone(Node *node, std::vector<int> *lm_la_list,
                                 bool fan_in);

  void create_cross_word_network(void);
  void add_hmm_to_fan_network(int hmm_id,
                              bool fan_out);
  void link_fan_out_node_to_fan_in(Node *node, const std::string &key);
  void link_node_to_fan_network(const std::string &key,
                                std::vector<Arc> &source_arcs,
                                bool fan_out,
                                bool ignore_length,
                                float out_transition_log_prob);
  void add_single_hmm_word_for_cross_word_modeling(Hmm *hmm, int word_id);
  void link_fan_in_nodes(void);
  void create_lex_tree_links_from_fan_in(Node *fan_in_node,
                                         const std::string &key);

  void analyze_cross_word_network(void);
  void count_fan_size(Node *node, unsigned short flag,
                      int *num_nodes, int *num_arcs);
  void count_prefix_tree_size(Node *node, int *num_nodes,
                              int *num_arcs);

  void free_cross_word_network_connection_points(void);
  Node* get_short_silence_node(void);
  Node* get_fan_out_entry_node(HmmState *state, const std::string &label);
  Node* get_fan_out_last_node(HmmState *state, const std::string &label);
  Node* get_fan_in_entry_node(HmmState *state, const std::string &label);
  Node* get_fan_in_last_node(HmmState *state, const std::string &label);
  Node* get_fan_state_node(HmmState *state, std::vector<Node*> *nodes);
  std::vector<Node*>* get_fan_node_list(
    const std::string &key,
    std::map< std::string, std::vector<Node*>* > &nmap);
  void add_fan_in_connection_node(Node *node, const std::string &prev_label);

  float get_out_transition_log_prob(Node *node);

  void prune_lm_la_buffer(int delta_thr, int depth_thr,
                          Node *node, int last_size, int cur_depth);

private:
  int m_words; // Largest word_id in the nodes plus one
  Node *m_root_node;
  Node *m_end_node;
  Node *m_start_node;
  Node *m_silence_node;
  Node *m_last_silence_node;
  std::vector<Node*> node_list;
  int m_verbose;
  int m_lm_lookahead; // 0=None, 1=Only in first subtree nodes,
                      // 2=Full
  bool m_cross_word_triphones;
  int m_lm_buf_count;

  bool m_silence_is_word;
  bool m_ignore_case;
  bool m_optional_short_silence;
  HmmState *m_short_silence_state;
  int m_word_boundary_id;

  std::map<std::string,int> &m_hmm_map;
  std::vector<Hmm> &m_hmms;

  std::map< std::string, std::vector<Node*>* > m_fan_out_entry_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_out_last_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_entry_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_last_nodes;
  std::map< std::string, std::vector<Node*>* > m_fan_in_connection_nodes;
  std::vector<NodeArcId> m_silence_arcs;
};

//////////////////////////////////////////////////////////////////////

TPLexPrefixTree::WordHistory::WordHistory(int word_id, int lm_id,
                                          class WordHistory *prev)
  : word_id(word_id),
    lm_id(lm_id),
    prev_word(prev),
    printed(false),
    word_start_frame(0),
    m_reference_count(0)
{
  if (prev)
    prev->link();
}

void
TPLexPrefixTree::WordHistory::unlink(class WordHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      WordHistory *prev = hist->prev_word;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

TPLexPrefixTree::PathHistory::PathHistory(float ll, float dll, int depth,
                                          class PathHistory *p)
  : ll(ll),
    dll(dll),
    depth(depth),
    prev(p),
    m_reference_count(0)
{
  if (p)
    p->link();
}

void
TPLexPrefixTree::PathHistory::unlink(class PathHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      PathHistory *prev = hist->prev;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

TPLexPrefixTree::StateHistory::StateHistory(int hmm_model, int start_time,
                                            class StateHistory *prev)
  : hmm_model(hmm_model),
    start_time(start_time),
    prev(prev),
    m_reference_count(0)
{
  if (prev)
    prev->link();
}

void
TPLexPrefixTree::StateHistory::unlink(class StateHistory *hist)
{
  if (hist != NULL)
  {
    while (hist->m_reference_count == 1) {
      StateHistory *prev = hist->prev;
      delete hist;
      hist = prev;
      if (hist == NULL)
        return;
    }
    hist->m_reference_count--;
  }
}

#endif /* TPLEXPREFIXTREE_HH */
<|endoftext|>


<commit_before>#include "HedgeLib/IO/HedgehogEngine.h"
#include "HedgeLib/IO/File.h"
#include "HedgeLib/Offsets.h"
#include "../INBlob.h"

HL_IMPL_ENDIAN_SWAP_CPP(hl_DHHHeader);
HL_IMPL_ENDIAN_SWAP(hl_DHHHeader, v)
{
    hl_SwapUInt32(&v->FileSize);
    hl_SwapUInt32(&v->Version);
}

HL_IMPL_ENDIAN_SWAP_CPP(hl_DHHStandardHeader);
HL_IMPL_ENDIAN_SWAP(hl_DHHStandardHeader, v)
{
    HL_ENDIAN_SWAP(hl_DHHHeader, &v->Header);
    hl_SwapUInt32(&v->DataSize);
    hl_SwapUInt32(&v->DataOffset);
    hl_SwapUInt32(&v->OffsetTableOffset);
    hl_SwapUInt32(&v->EOFOffset);
}

void hl_HHFixOffsets(uint32_t* offTable,
    uint32_t offCount, void* data)
{
    hl_DataOff32* offPtr = static_cast<hl_DataOff32*>(data);
    for (uint32_t i = 0; i < offCount; ++i)
    {
        // Endian swap offset table entry
        hl_SwapUInt32(&(offTable[i]));

        // Get position of next offset
        // (Data Pointer + Current entry in offset table)
        offPtr = hl_GetAbs<hl_DataOff32>(data, offTable[i]);

        // Fix offset
        hl_FixOffset(offPtr, data);
    }
}

void* hl_HHMirageGetDataNode(const struct hl_Blob* blob)
{
    // TODO
    return nullptr;
}

void* hl_HHStandardGetData(struct hl_Blob* blob)
{
    hl_DHHStandardHeader* header = blob->GetData<hl_DHHStandardHeader>();
    return HL_GETABSV(header, header->DataOffset);
}

void* hl_HHMirageGetData(struct hl_Blob* blob)
{
    // TODO
    return nullptr;
}

void* hl_HHGetData(struct hl_Blob* blob)
{
    // Mirage Header
    if (hl_HHDetectHeaderType(blob->GetData<const
        hl_DHHHeader>()) == HL_HHHEADER_TYPE_MIRAGE)
    {
        return hl_HHMirageGetData(blob);
    }

    // Standard Header
    return hl_HHStandardGetData(blob);
}

enum HL_RESULT hl_HHRead(struct hl_File* file, struct hl_Blob** blob)
{
    HL_RESULT result;
    file->DoEndianSwap = true;

    // Get file size
    uint32_t fileSize;
    result = file->Read(fileSize);
    if (HL_FAILED(result)) return result;

    // Get header type and go back to beginning of file
    bool isMirage = false; // "Friends are nothing but a fleeting illusion"
    if (fileSize & HL_HHMIRAGE_FLAGS_MASK) // "Your mask can't hide how sad and lonely you are!"
    {
        // Make sure this is really a Mirage header
        uint32_t version;
        result = file->Read(version);
        if (HL_FAILED(result)) return result;

        if (version == HL_HHMIRAGE_SIGNATURE)
        {
            isMirage = true;
            fileSize &= HL_HHMIRAGE_SIZE_MASK;
        }

        file->JumpBehind(8);
    }
    else
    {
        file->JumpBehind(4);
    }

    // Read entire file
    *blob = hl_INCreateBlob(HL_BLOB_TYPE_HEDGEHOG_ENGINE, fileSize);
    if (!(*blob)) return HL_ERROR_OUT_OF_MEMORY;

    void* fileData = (*blob)->GetData<void>();
    result = file->ReadBytes(fileData, fileSize);
    if (HL_FAILED(result)) return result;

    // Header-specific
    uint32_t* offTable;
    uint32_t offCount;
    void* data;

    if (isMirage) // Mirage Header
    {
        // TODO
        offTable = nullptr;
        offCount = 0;
        data = nullptr;
    }
    else // Standard Header
    {
        // Endian-swap standard header
        hl_DHHStandardHeader* header = static_cast
            <hl_DHHStandardHeader*>(fileData);

        HL_ENDIAN_SWAP(hl_DHHStandardHeader, header);

        // Get offset table
        offTable = hl_GetAbs<uint32_t>(
            fileData, header->OffsetTableOffset);

        hl_SwapUInt32(offTable);
        offCount = *offTable++;

        // Get data pointer
        data = hl_GetAbs<void>(fileData, header->DataOffset);
    }

    // Fix offsets
    hl_HHFixOffsets(offTable, offCount, data);
    return HL_SUCCESS;
}

enum HL_RESULT hl_HHLoad(const char* filePath, struct hl_Blob** blob)
{
    // TODO: Do stuff here instead of just calling hl_HHRead so you
    // can optimize-out the need to read the file size and backtrack.
    hl_File file = hl_File::OpenRead(std::filesystem::u8path(filePath), true);
    return hl_HHRead(&file, blob);
}

enum HL_RESULT hl_HHStartWriteStandard(struct hl_File* file, uint32_t version)
{
    // Create "empty" header
    hl_DHHStandardHeader header = {};
    header.Header.Version = version;
    header.DataOffset = sizeof(hl_DHHStandardHeader);

    // Write header
    file->DoEndianSwap = true;
    HL_RESULT result = file->Write(header);

    // Set origin
    if (HL_OK(result))
    {
        file->Origin = file->Tell();
    }

    return result;
}

enum HL_RESULT hl_HHWriteOffsetTableStandard(const struct hl_File* file,
    const hl_OffsetTable* offTable)
{
    // Write offset count
    uint32_t offCount = static_cast<uint32_t>(offTable->size());
    HL_RESULT result = file->Write(offCount);
    if (HL_FAILED(result)) return result;

    // Write offsets
    uint32_t off;
    const long* offsets = offTable->data();

    for (size_t i = 0; i < offTable->size(); ++i)
    {
        off = static_cast<uint32_t>(
            offsets[i] - file->Origin);

        result = file->Write(off);
        if (HL_FAILED(result)) return result;
    }

    return result;
}

enum HL_RESULT hl_HHFinishWriteStandard(const struct hl_File* file, long headerPos,
    bool writeEOFThing, const hl_OffsetTable* offTable)
{
    // Write offset table
    uint32_t offTablePos = static_cast<uint32_t>(file->Tell());
    if (headerPos >= static_cast<long>(offTablePos))
        return HL_ERROR_UNKNOWN; // TODO: Return a better error

    HL_RESULT result = hl_HHWriteOffsetTableStandard(file, offTable);
    if (HL_FAILED(result)) return result;

    // TODO: Write EOF thing if told to

    // Fill-in file size
    uint32_t fileSize = static_cast<uint32_t>(file->Tell());
    file->JumpTo(headerPos);

    result = file->Write(fileSize);
    if (HL_FAILED(result)) return result;

    if (file->JumpAhead(4)) return HL_ERROR_UNKNOWN; // TODO: Return better error

    // Fill-in data size
    uint32_t dataSize = (offTablePos -
        sizeof(hl_DHHStandardHeader));

    result = file->Write(dataSize);
    if (HL_FAILED(result)) return result;

    file->JumpAhead(4);

    // Fill-in offset table position
    result = file->Write(offTablePos);
    if (HL_FAILED(result)) return result;

    // TODO: Fill-in EOF position if told to
    return result;
}

void hl_HHFreeBlob(struct hl_Blob* blob)
{
#ifdef x64
    // Get offset table
    std::uint32_t offCount;
    std::uint32_t* offTable;
    void* data;

    if (!blob) return;

    if (hl_HHDetectHeaderType(blob->GetData<
        hl_DHHHeader>()) == HL_HHHEADER_TYPE_MIRAGE)
    {
        // TODO
        offCount = 0;
        offTable = nullptr;
        data = nullptr;
    }
    else
    {
        // Get offset table pointer
        hl_DHHStandardHeader* header = blob->GetData
            <hl_DHHStandardHeader>();

        offTable = hl_GetAbs<std::uint32_t>(
            header, header->OffsetTableOffset);

        // Get offset table count
        offCount = *offTable++;

        // Get data
        data = hl_GetAbs<void>(header, header->DataOffset);
    }

    // Free all offsets using data in offset table
    for (uint32_t i = 0; i < offCount; ++i)
    {
        hl_x64RemoveAbsPtr32(*hl_GetAbs<hl_DataOff32>(
            data, offTable[i]));
    }
#endif

    // Free data
    std::free(blob);
}
<commit_msg>Minor bug fix for Hedgehog Engine writing We shouldn't assume headerPos is 0; the user might be writing further into the file.<commit_after>#include "HedgeLib/IO/HedgehogEngine.h"
#include "HedgeLib/IO/File.h"
#include "HedgeLib/Offsets.h"
#include "../INBlob.h"

HL_IMPL_ENDIAN_SWAP_CPP(hl_DHHHeader);
HL_IMPL_ENDIAN_SWAP(hl_DHHHeader, v)
{
    hl_SwapUInt32(&v->FileSize);
    hl_SwapUInt32(&v->Version);
}

HL_IMPL_ENDIAN_SWAP_CPP(hl_DHHStandardHeader);
HL_IMPL_ENDIAN_SWAP(hl_DHHStandardHeader, v)
{
    HL_ENDIAN_SWAP(hl_DHHHeader, &v->Header);
    hl_SwapUInt32(&v->DataSize);
    hl_SwapUInt32(&v->DataOffset);
    hl_SwapUInt32(&v->OffsetTableOffset);
    hl_SwapUInt32(&v->EOFOffset);
}

void hl_HHFixOffsets(uint32_t* offTable,
    uint32_t offCount, void* data)
{
    hl_DataOff32* offPtr = static_cast<hl_DataOff32*>(data);
    for (uint32_t i = 0; i < offCount; ++i)
    {
        // Endian swap offset table entry
        hl_SwapUInt32(&(offTable[i]));

        // Get position of next offset
        // (Data Pointer + Current entry in offset table)
        offPtr = hl_GetAbs<hl_DataOff32>(data, offTable[i]);

        // Fix offset
        hl_FixOffset(offPtr, data);
    }
}

void* hl_HHMirageGetDataNode(const struct hl_Blob* blob)
{
    // TODO
    return nullptr;
}

void* hl_HHStandardGetData(struct hl_Blob* blob)
{
    hl_DHHStandardHeader* header = blob->GetData<hl_DHHStandardHeader>();
    return HL_GETABSV(header, header->DataOffset);
}

void* hl_HHMirageGetData(struct hl_Blob* blob)
{
    // TODO
    return nullptr;
}

void* hl_HHGetData(struct hl_Blob* blob)
{
    // Mirage Header
    if (hl_HHDetectHeaderType(blob->GetData<const
        hl_DHHHeader>()) == HL_HHHEADER_TYPE_MIRAGE)
    {
        return hl_HHMirageGetData(blob);
    }

    // Standard Header
    return hl_HHStandardGetData(blob);
}

enum HL_RESULT hl_HHRead(struct hl_File* file, struct hl_Blob** blob)
{
    HL_RESULT result;
    file->DoEndianSwap = true;

    // Get file size
    uint32_t fileSize;
    result = file->Read(fileSize);
    if (HL_FAILED(result)) return result;

    // Get header type and go back to beginning of file
    bool isMirage = false; // "Friends are nothing but a fleeting illusion"
    if (fileSize & HL_HHMIRAGE_FLAGS_MASK) // "Your mask can't hide how sad and lonely you are!"
    {
        // Make sure this is really a Mirage header
        uint32_t version;
        result = file->Read(version);
        if (HL_FAILED(result)) return result;

        if (version == HL_HHMIRAGE_SIGNATURE)
        {
            isMirage = true;
            fileSize &= HL_HHMIRAGE_SIZE_MASK;
        }

        file->JumpBehind(8);
    }
    else
    {
        file->JumpBehind(4);
    }

    // Read entire file
    *blob = hl_INCreateBlob(HL_BLOB_TYPE_HEDGEHOG_ENGINE, fileSize);
    if (!(*blob)) return HL_ERROR_OUT_OF_MEMORY;

    void* fileData = (*blob)->GetData<void>();
    result = file->ReadBytes(fileData, fileSize);
    if (HL_FAILED(result)) return result;

    // Header-specific
    uint32_t* offTable;
    uint32_t offCount;
    void* data;

    if (isMirage) // Mirage Header
    {
        // TODO
        offTable = nullptr;
        offCount = 0;
        data = nullptr;
    }
    else // Standard Header
    {
        // Endian-swap standard header
        hl_DHHStandardHeader* header = static_cast
            <hl_DHHStandardHeader*>(fileData);

        HL_ENDIAN_SWAP(hl_DHHStandardHeader, header);

        // Get offset table
        offTable = hl_GetAbs<uint32_t>(
            fileData, header->OffsetTableOffset);

        hl_SwapUInt32(offTable);
        offCount = *offTable++;

        // Get data pointer
        data = hl_GetAbs<void>(fileData, header->DataOffset);
    }

    // Fix offsets
    hl_HHFixOffsets(offTable, offCount, data);
    return HL_SUCCESS;
}

enum HL_RESULT hl_HHLoad(const char* filePath, struct hl_Blob** blob)
{
    // TODO: Do stuff here instead of just calling hl_HHRead so you
    // can optimize-out the need to read the file size and backtrack.
    hl_File file = hl_File::OpenRead(std::filesystem::u8path(filePath), true);
    return hl_HHRead(&file, blob);
}

enum HL_RESULT hl_HHStartWriteStandard(struct hl_File* file, uint32_t version)
{
    // Create "empty" header
    hl_DHHStandardHeader header = {};
    header.Header.Version = version;
    header.DataOffset = sizeof(hl_DHHStandardHeader);

    // Write header
    file->DoEndianSwap = true;
    HL_RESULT result = file->Write(header);

    // Set origin
    if (HL_OK(result))
    {
        file->Origin = file->Tell();
    }

    return result;
}

enum HL_RESULT hl_HHWriteOffsetTableStandard(const struct hl_File* file,
    const hl_OffsetTable* offTable)
{
    // Write offset count
    uint32_t offCount = static_cast<uint32_t>(offTable->size());
    HL_RESULT result = file->Write(offCount);
    if (HL_FAILED(result)) return result;

    // Write offsets
    uint32_t off;
    const long* offsets = offTable->data();

    for (size_t i = 0; i < offTable->size(); ++i)
    {
        off = static_cast<uint32_t>(
            offsets[i] - file->Origin);

        result = file->Write(off);
        if (HL_FAILED(result)) return result;
    }

    return result;
}

enum HL_RESULT hl_HHFinishWriteStandard(const struct hl_File* file, long headerPos,
    bool writeEOFThing, const hl_OffsetTable* offTable)
{
    // Write offset table
    uint32_t offTablePos = static_cast<uint32_t>(file->Tell());
    if (headerPos >= static_cast<long>(offTablePos))
        return HL_ERROR_UNKNOWN; // TODO: Return a better error

    HL_RESULT result = hl_HHWriteOffsetTableStandard(file, offTable);
    if (HL_FAILED(result)) return result;

    // TODO: Write EOF thing if told to

    // Fill-in file size
    uint32_t fileSize = static_cast<uint32_t>(file->Tell()) - headerPos;
    file->JumpTo(headerPos);

    result = file->Write(fileSize);
    if (HL_FAILED(result)) return result;

    if (file->JumpAhead(4)) return HL_ERROR_UNKNOWN; // TODO: Return better error

    // Fill-in data size
    uint32_t dataSize = (offTablePos -
        sizeof(hl_DHHStandardHeader));

    result = file->Write(dataSize);
    if (HL_FAILED(result)) return result;

    file->JumpAhead(4);

    // Fill-in offset table position
    result = file->Write(offTablePos);
    if (HL_FAILED(result)) return result;

    // TODO: Fill-in EOF position if told to
    return result;
}

void hl_HHFreeBlob(struct hl_Blob* blob)
{
#ifdef x64
    // Get offset table
    std::uint32_t offCount;
    std::uint32_t* offTable;
    void* data;

    if (!blob) return;

    if (hl_HHDetectHeaderType(blob->GetData<
        hl_DHHHeader>()) == HL_HHHEADER_TYPE_MIRAGE)
    {
        // TODO
        offCount = 0;
        offTable = nullptr;
        data = nullptr;
    }
    else
    {
        // Get offset table pointer
        hl_DHHStandardHeader* header = blob->GetData
            <hl_DHHStandardHeader>();

        offTable = hl_GetAbs<std::uint32_t>(
            header, header->OffsetTableOffset);

        // Get offset table count
        offCount = *offTable++;

        // Get data
        data = hl_GetAbs<void>(header, header->DataOffset);
    }

    // Free all offsets using data in offset table
    for (uint32_t i = 0; i < offCount; ++i)
    {
        hl_x64RemoveAbsPtr32(*hl_GetAbs<hl_DataOff32>(
            data, offTable[i]));
    }
#endif

    // Free data
    std::free(blob);
}
<|endoftext|>


<commit_before>// $Id: forceFieldComponent.C,v 1.4 2000/03/12 22:28:47 oliver Exp $

#include <BALL/MOLMEC/COMMON/forceFieldComponent.h>

using namespace std;

namespace BALL
{

	// default constructor
	ForceFieldComponent::ForceFieldComponent()
	{
		name_ = "GenericForceFieldComponent";
		force_field_ = 0;
		energy_ = 0;
	}

	// constructor
	ForceFieldComponent::ForceFieldComponent(ForceField* force_field)
	{
		force_field_ = force_field;
		name_ = "GenericForceFieldComponent";
		energy_ = 0;
	}

	// copy constructor
	ForceFieldComponent::ForceFieldComponent(const ForceFieldComponent& force_field_component, bool /* clone_deep */)
	{
		name_ = force_field_component.name_;
		force_field_ = force_field_component.force_field_;
		energy_ = force_field_component.energy_;
	}

	// destructor
	ForceFieldComponent::~ForceFieldComponent()
	{
	}

	// setup
	bool ForceFieldComponent::setup()
	{
		return true;
	}

	// Set the name of the component
	void ForceFieldComponent::setName(const String& name)
	{
		name_ = name;
	}

	// Return the name of the component
	String ForceFieldComponent::getName( ) const
	{
		return name_;
	}

	// Return a pointer to the force field
	ForceField* ForceFieldComponent::getForceField() const
	{
		return force_field_;
	}

	// Set the force field to force_field
	void ForceFieldComponent::setForceField(ForceField* force_field)
	{
		force_field_ = force_field;
	}

	float ForceFieldComponent::getEnergy() const
	{
		return energy_;
	}

	float ForceFieldComponent::updateEnergy()
	{
		return 0.0;
	}

	void ForceFieldComponent::updateForces()
	{
	}

} // namespace BALL
<commit_msg>changed: energies are double precision<commit_after>// $Id: forceFieldComponent.C,v 1.5 2000/03/25 22:51:56 oliver Exp $

#include <BALL/MOLMEC/COMMON/forceFieldComponent.h>

using namespace std;

namespace BALL
{

	// default constructor
	ForceFieldComponent::ForceFieldComponent()
	{
		name_ = "GenericForceFieldComponent";
		force_field_ = 0;
		energy_ = 0;
	}

	// constructor
	ForceFieldComponent::ForceFieldComponent(ForceField* force_field)
	{
		force_field_ = force_field;
		name_ = "GenericForceFieldComponent";
		energy_ = 0;
	}

	// copy constructor
	ForceFieldComponent::ForceFieldComponent(const ForceFieldComponent& force_field_component, bool /* clone_deep */)
	{
		name_ = force_field_component.name_;
		force_field_ = force_field_component.force_field_;
		energy_ = force_field_component.energy_;
	}

	// destructor
	ForceFieldComponent::~ForceFieldComponent()
	{
	}

	// setup
	bool ForceFieldComponent::setup()
	{
		return true;
	}

	// Set the name of the component
	void ForceFieldComponent::setName(const String& name)
	{
		name_ = name;
	}

	// Return the name of the component
	String ForceFieldComponent::getName( ) const
	{
		return name_;
	}

	// Return a pointer to the force field
	ForceField* ForceFieldComponent::getForceField() const
	{
		return force_field_;
	}

	// Set the force field to force_field
	void ForceFieldComponent::setForceField(ForceField* force_field)
	{
		force_field_ = force_field;
	}

	double ForceFieldComponent::getEnergy() const
	{
		return energy_;
	}

	double ForceFieldComponent::updateEnergy()
	{
		return 0.0;
	}

	void ForceFieldComponent::updateForces()
	{
	}

} // namespace BALL
<|endoftext|>