<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbOGRDataResampler.h"
#include <string>
#include <fstream>

namespace otb
{

unsigned int
OGRDataResampler
::selectN1(double per,unsigned int T)
{
   if (!(T>0))
     itkExceptionMacro(<< "In OGRDataResampler::selectN1 : sizes of the sampling arrays must be non-negative (" << T << ")." << std::endl);

   double dist = itk::NumericTraits<double>::max();
   double d;
   unsigned int N=0;
   double p;

   for(unsigned int n=0;n<=T;n++)
   {
   p=static_cast<double>(n)/static_cast<double>(T)*100.;
       if (p<=per)
       {
          d = fabs(per-p);
          if (d<dist)
          {
              N=n;
              dist=d;
          }
       }
   }
   return N;
}

unsigned int
OGRDataResampler
::selectN2(double per,unsigned int T)
{
   if (!(T>0))
     itkExceptionMacro(<< "In OGRDataResampler::selectN2 : sizes of the sampling arrays must be non-negative (" << T << ")." << std::endl);

   double dist = itk::NumericTraits<double>::max();
   double d;
   unsigned int N=0;
   double p;

   for(unsigned int n=0;n<=T;n++)
   {
   p=static_cast<double>(n)/static_cast<double>(T)*100.;
       if (p>=per)
       {
          d = fabs(per-p);
          if (d<dist)
          {
              N=n;
              dist=d;
          }
       }
   }
   return N;
}

std::vector<bool>
OGRDataResampler::randArray(unsigned int N,unsigned int T)
{

   if (N>T)
     itkExceptionMacro(<< "N must be <= to T (aka m_SamplingTabSize)." << std::endl);

   std::vector<bool> res(T,0);
   for(unsigned int i=0; i<N;i++)
       res[i]=1;

   std::random_shuffle ( res.begin(), res.end() );

   return res;

}

unsigned int
OGRDataResampler::findBestSize(unsigned int tot)
{
   if (tot<m_MaxSamplingVecSize)
      return tot;

   for(unsigned int size=m_MaxSamplingVecSize; size>=2; size--)
   if (tot%size == 0)
      return size;

   otbWarningMacro(<<"prime number > m_MaxSamplingVecSize (" << tot << ">" << m_MaxSamplingVecSize << ")."<< std::endl );
   return tot;
}

void
OGRDataResampler::InputSamplingVectors()
{

  m_ClassToBools.clear();
  m_ClassToCurrentIndices.clear();

  std::ifstream file(m_InputSamplingVectorsPath.c_str(),std::ios::in);
  std::string line;

  if (file)
  {
      std::string className;
      int tabNum=1;
      std::vector<bool> tab1,tab2;

      while(getline(file,line))
      {

        std::size_t found1 = line.find_first_of("#");
        std::size_t found2 = line.find_first_of(">");

        if (found1 != std::string::npos)
        {
          className = line.substr(found1+1);
        }
        if (found2 != std::string::npos)
        {
          if (tabNum==1)
          {
            std::istringstream issBooleans(line.substr(found2+1));
            std::list<std::string> tokens;
	        copy(std::istream_iterator<std::string>(issBooleans),
		    std::istream_iterator<std::string>(),
		    back_inserter(tokens));

            std::list<std::string>::iterator it = tokens.begin();
            for(; it != tokens.end(); ++it)
                tab1.push_back( (it->find("1")!=std::string::npos) );

            tabNum++;
          }
          else //tabNum==2
          {

            std::istringstream issBooleans(line.substr(found2+1));
            std::list<std::string> tokens;
	        copy(std::istream_iterator<std::string>(issBooleans),
		    std::istream_iterator<std::string>(),
		    back_inserter(tokens));

            std::list<std::string>::iterator it = tokens.begin();
            for(; it != tokens.end(); ++it)
                tab2.push_back( (it->find("1")!=std::string::npos) );

            m_ClassToBools[className] = std::make_pair(tab1,tab2);
            m_ClassToCurrentIndices[className] = std::make_pair(0,0);

             tabNum=1;
             className.erase();
             tab1.clear();
             tab2.clear();
          }
        }
      }
  }
  else
      itkExceptionMacro(<< "Could not open file " << m_InputSamplingVectorsPath << "." << std::endl);
}

void
OGRDataResampler::OutputSamplingVectors()
{
  std::ofstream file(m_OutputSamplingVectorsPath.c_str(), std::ios::out | std::ios::trunc);
  if(file)
  {
     ClassToBoolsType::iterator it = m_ClassToBools.begin();
     for(; it!=m_ClassToBools.end(); ++it)
     {
         file << "#" << it->first << std::endl;
         file << ">";
         for(unsigned int i=0; i<it->second.first.size(); i++)
           file << it->second.first[i] << " ";
          file << std::endl;

         file << ">";
         for(unsigned int i=0; i<it->second.second.size(); i++)
           file << it->second.first[i] << " ";
          file << std::endl;
     }
     file.close();
  }
  else
     itkExceptionMacro(<< "Could not open file " << m_OutputSamplingVectorsPath << "." << std::endl);
}

void
OGRDataResampler::Prepare()
{

if (!m_alreadyPrepared)
{

    if (m_RatesbyClass.empty())
    {
         itkGenericExceptionMacro("m_RatesbyClass is empty (otb::OGRDataResampler) ");
    }

   if (!m_InputSamplingVectorsPath.empty())
        this->InputSamplingVectors();
   else
   {
       SamplingRateCalculator::mapRateType::iterator itRates = m_RatesbyClass.begin();
       for(;itRates !=m_RatesbyClass.end(); ++itRates)
       {
          //std::cout << "#" << itRates->first << " " << itRates->second.required << " " << itRates->second.tot << std::endl;

          unsigned int T1 = findBestSize(itRates->second.tot);
          //std::cout << "*** " << itRates->second.tot << " " << T1 << " " << itRates->second.tot/T1 << std::endl;

          double per = static_cast<double>(itRates->second.required) / static_cast<double>(itRates->second.tot)*100.;
          unsigned int N1 = selectN1(per,T1);

          double selected_prct = static_cast<double>(N1)/static_cast<double>(T1)*100.;
          unsigned int taken = static_cast<unsigned int>(selected_prct*static_cast<double>(itRates->second.tot)/100.);
          unsigned int left = itRates->second.required - taken;
          unsigned int newtot = itRates->second.tot - taken;

          //std::cout << "##" << N1 << " " << per << " " << selected_prct << " " << taken << " " << left << " " << newtot << std::endl;
          if ( !(per>=selected_prct) )
            itkExceptionMacro(<< "In otb::OGRDataResampler::Prepare, per < selected_prct !" << std::endl);

          unsigned int T2 = 0;
          unsigned int N2 = 0;
          if (left>0)
          {

              double per2;
              if (newtot>0)
                 per2 = static_cast<double>(left)/static_cast<double>(newtot)*100.;
              else
                 per2 = 0;

              T2 = findBestSize(itRates->second.tot/T1*(T1-N1));

              /*std::cout << "*** " << itRates->second.tot << " " << T1 << " " << itRates->second.tot/T1 << " " << N1 << " "
              << itRates->second.tot/T1*(T1-N1) << " " << T2 << std::endl;*/

              double selected_prct2  = 0.;
              if (T2>0)
              {
                 N2 = selectN2(per2,T2);
                 selected_prct2 = static_cast<double>(N2)/static_cast<double>(T2)*100.;
              }

              //std::cout << "###" << N2 << " " << per2 << " " << selected_prct2 << std::endl;

              if ( !(per2<=selected_prct2) )
                itkExceptionMacro(<< "In otb::OGRDataResampler::Prepare, per2 > selected_prct2 !" << std::endl);
          }

          std::srand ( unsigned ( time(0) ) );
          std::vector<bool> tab1 = randArray(N1,T1);
          std::vector<bool> tab2;
          if (T2>0)
             tab2 = randArray(N2,T2);

          unsigned int N=0;
          for(unsigned int i=0;i<tab1.size();i++)
          if (tab1[i])
            N++;
          if (N!=N1)
            itkExceptionMacro(<< "N != N1." << std::endl);

          N=0;
          for(unsigned int i=0;i<tab2.size();i++)
          if (tab2[i])
            N++;
          if (N!=N2)
            itkExceptionMacro(<< "N != N2." << std::endl);

          //itRates->first <=>  className
          m_ClassToBools[itRates->first] = std::make_pair(tab1,tab2);
          m_ClassToCurrentIndices[itRates->first] = std::make_pair(0,0);

          if (!m_OutputSamplingVectorsPath.empty())
            this->OutputSamplingVectors();

       }
   }

}
m_alreadyPrepared=true;
}

bool
OGRDataResampler::TakeSample(std::string className)
{
   bool res=false;

   if (m_ElmtsInClass[className] >= m_RatesbyClass[className].required)
     return false;

   unsigned int &ind1 = m_ClassToCurrentIndices[className].first; //first counter
   std::vector<bool> &tab1 = m_ClassToBools[className].first; // First array
   res = tab1[ind1];

   ind1++;

   if (ind1>=tab1.size())
     ind1=0;

   if(!res)
   {
     unsigned int &ind2 = m_ClassToCurrentIndices[className].second; // second counter
     std::vector<bool> &tab2 = m_ClassToBools[className].second; // second array

     if (tab2.size()>0)
     {
         res = tab2[ind2];

         ind2++;

         if (ind2>=tab2.size())
           ind2=0;
     }

   }

   return res;
}

void
OGRDataResampler
::Reset()
{
  m_ElmtsInClass.clear();
}

} // end of namespace otb
<commit_msg>BUG: in method otb::OGRDataResampler::OutputSamplingVectors<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbOGRDataResampler.h"
#include <string>
#include <fstream>

namespace otb
{

unsigned int
OGRDataResampler
::selectN1(double per,unsigned int T)
{
   if (!(T>0))
     itkExceptionMacro(<< "In OGRDataResampler::selectN1 : sizes of the sampling arrays must be non-negative (" << T << ")." << std::endl);

   double dist = itk::NumericTraits<double>::max();
   double d;
   unsigned int N=0;
   double p;

   for(unsigned int n=0;n<=T;n++)
   {
   p=static_cast<double>(n)/static_cast<double>(T)*100.;
       if (p<=per)
       {
          d = fabs(per-p);
          if (d<dist)
          {
              N=n;
              dist=d;
          }
       }
   }
   return N;
}

unsigned int
OGRDataResampler
::selectN2(double per,unsigned int T)
{
   if (!(T>0))
     itkExceptionMacro(<< "In OGRDataResampler::selectN2 : sizes of the sampling arrays must be non-negative (" << T << ")." << std::endl);

   double dist = itk::NumericTraits<double>::max();
   double d;
   unsigned int N=0;
   double p;

   for(unsigned int n=0;n<=T;n++)
   {
   p=static_cast<double>(n)/static_cast<double>(T)*100.;
       if (p>=per)
       {
          d = fabs(per-p);
          if (d<dist)
          {
              N=n;
              dist=d;
          }
       }
   }
   return N;
}

std::vector<bool>
OGRDataResampler::randArray(unsigned int N,unsigned int T)
{

   if (N>T)
     itkExceptionMacro(<< "N must be <= to T (aka m_SamplingTabSize)." << std::endl);

   std::vector<bool> res(T,0);
   for(unsigned int i=0; i<N;i++)
       res[i]=1;

   std::random_shuffle ( res.begin(), res.end() );

   return res;

}

unsigned int
OGRDataResampler::findBestSize(unsigned int tot)
{
   if (tot<m_MaxSamplingVecSize)
      return tot;

   for(unsigned int size=m_MaxSamplingVecSize; size>=2; size--)
   if (tot%size == 0)
      return size;

   otbWarningMacro(<<"prime number > m_MaxSamplingVecSize (" << tot << ">" << m_MaxSamplingVecSize << ")."<< std::endl );
   return tot;
}

void
OGRDataResampler::InputSamplingVectors()
{

  m_ClassToBools.clear();
  m_ClassToCurrentIndices.clear();

  std::ifstream file(m_InputSamplingVectorsPath.c_str(),std::ios::in);
  std::string line;

  if (file)
  {
      std::string className;
      int tabNum=1;
      std::vector<bool> tab1,tab2;

      while(getline(file,line))
      {

        std::size_t found1 = line.find_first_of("#");
        std::size_t found2 = line.find_first_of(">");

        if (found1 != std::string::npos)
        {
          className = line.substr(found1+1);
        }
        if (found2 != std::string::npos)
        {
          if (tabNum==1)
          {
            std::istringstream issBooleans(line.substr(found2+1));
            std::list<std::string> tokens;
	        copy(std::istream_iterator<std::string>(issBooleans),
		    std::istream_iterator<std::string>(),
		    back_inserter(tokens));

            std::list<std::string>::iterator it = tokens.begin();
            for(; it != tokens.end(); ++it)
                tab1.push_back( (it->find("1")!=std::string::npos) );

            tabNum++;
          }
          else //tabNum==2
          {

            std::istringstream issBooleans(line.substr(found2+1));
            std::list<std::string> tokens;
	        copy(std::istream_iterator<std::string>(issBooleans),
		    std::istream_iterator<std::string>(),
		    back_inserter(tokens));

            std::list<std::string>::iterator it = tokens.begin();
            for(; it != tokens.end(); ++it)
                tab2.push_back( (it->find("1")!=std::string::npos) );

            m_ClassToBools[className] = std::make_pair(tab1,tab2);
            m_ClassToCurrentIndices[className] = std::make_pair(0,0);

             tabNum=1;
             className.erase();
             tab1.clear();
             tab2.clear();
          }
        }
      }
  }
  else
      itkExceptionMacro(<< "Could not open file " << m_InputSamplingVectorsPath << "." << std::endl);
}

void
OGRDataResampler::OutputSamplingVectors()
{
  std::ofstream file(m_OutputSamplingVectorsPath.c_str(), std::ios::out | std::ios::trunc);
  if(file)
  {
     ClassToBoolsType::iterator it = m_ClassToBools.begin();
     for(; it!=m_ClassToBools.end(); ++it)
     {
         file << "#" << it->first << std::endl;
         file << ">";
         for(unsigned int i=0; i<it->second.first.size(); i++)
           file << it->second.first[i] << " ";
          file << std::endl;

         file << ">";
         for(unsigned int i=0; i<it->second.second.size(); i++)
           file << it->second.second[i] << " ";
          file << std::endl;
     }
     file.close();
  }
  else
     itkExceptionMacro(<< "Could not open file " << m_OutputSamplingVectorsPath << "." << std::endl);
}

void
OGRDataResampler::Prepare()
{

if (!m_alreadyPrepared)
{

    if (m_RatesbyClass.empty())
    {
         itkGenericExceptionMacro("m_RatesbyClass is empty (otb::OGRDataResampler) ");
    }

   if (!m_InputSamplingVectorsPath.empty())
        this->InputSamplingVectors();
   else
   {
       SamplingRateCalculator::mapRateType::iterator itRates = m_RatesbyClass.begin();
       for(;itRates !=m_RatesbyClass.end(); ++itRates)
       {
          //std::cout << "#" << itRates->first << " " << itRates->second.required << " " << itRates->second.tot << std::endl;

          unsigned int T1 = findBestSize(itRates->second.tot);
          //std::cout << "*** " << itRates->second.tot << " " << T1 << " " << itRates->second.tot/T1 << std::endl;

          double per = static_cast<double>(itRates->second.required) / static_cast<double>(itRates->second.tot)*100.;
          unsigned int N1 = selectN1(per,T1);

          double selected_prct = static_cast<double>(N1)/static_cast<double>(T1)*100.;
          unsigned int taken = static_cast<unsigned int>(selected_prct*static_cast<double>(itRates->second.tot)/100.);
          unsigned int left = itRates->second.required - taken;
          unsigned int newtot = itRates->second.tot - taken;

          //std::cout << "##" << N1 << " " << per << " " << selected_prct << " " << taken << " " << left << " " << newtot << std::endl;
          if ( !(per>=selected_prct) )
            itkExceptionMacro(<< "In otb::OGRDataResampler::Prepare, per < selected_prct !" << std::endl);

          unsigned int T2 = 0;
          unsigned int N2 = 0;
          if (left>0)
          {

              double per2;
              if (newtot>0)
                 per2 = static_cast<double>(left)/static_cast<double>(newtot)*100.;
              else
                 per2 = 0;

              T2 = findBestSize(itRates->second.tot/T1*(T1-N1));

              /*std::cout << "*** " << itRates->second.tot << " " << T1 << " " << itRates->second.tot/T1 << " " << N1 << " "
              << itRates->second.tot/T1*(T1-N1) << " " << T2 << std::endl;*/

              double selected_prct2  = 0.;
              if (T2>0)
              {
                 N2 = selectN2(per2,T2);
                 selected_prct2 = static_cast<double>(N2)/static_cast<double>(T2)*100.;
              }

              //std::cout << "###" << N2 << " " << per2 << " " << selected_prct2 << std::endl;

              if ( !(per2<=selected_prct2) )
                itkExceptionMacro(<< "In otb::OGRDataResampler::Prepare, per2 > selected_prct2 !" << std::endl);
          }

          std::srand ( unsigned ( time(0) ) );
          std::vector<bool> tab1 = randArray(N1,T1);
          std::vector<bool> tab2;
          if (T2>0)
             tab2 = randArray(N2,T2);

          unsigned int N=0;
          for(unsigned int i=0;i<tab1.size();i++)
          if (tab1[i])
            N++;
          if (N!=N1)
            itkExceptionMacro(<< "N != N1." << std::endl);

          N=0;
          for(unsigned int i=0;i<tab2.size();i++)
          if (tab2[i])
            N++;
          if (N!=N2)
            itkExceptionMacro(<< "N != N2." << std::endl);

          //itRates->first <=>  className
          m_ClassToBools[itRates->first] = std::make_pair(tab1,tab2);
          m_ClassToCurrentIndices[itRates->first] = std::make_pair(0,0);

          if (!m_OutputSamplingVectorsPath.empty())
            this->OutputSamplingVectors();

       }
   }

}
m_alreadyPrepared=true;
}

bool
OGRDataResampler::TakeSample(std::string className)
{
   bool res=false;

   if (m_ElmtsInClass[className] >= m_RatesbyClass[className].required)
     return false;

   unsigned int &ind1 = m_ClassToCurrentIndices[className].first; //first counter
   std::vector<bool> &tab1 = m_ClassToBools[className].first; // First array
   res = tab1[ind1];

   ind1++;

   if (ind1>=tab1.size())
     ind1=0;

   if(!res)
   {
     unsigned int &ind2 = m_ClassToCurrentIndices[className].second; // second counter
     std::vector<bool> &tab2 = m_ClassToBools[className].second; // second array

     if (tab2.size()>0)
     {
         res = tab2[ind2];

         ind2++;

         if (ind2>=tab2.size())
           ind2=0;
     }

   }

   return res;
}

void
OGRDataResampler
::Reset()
{
  m_ElmtsInClass.clear();
}

} // end of namespace otb
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015,2016 deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "io_utils.h"

namespace io {

#if defined HAVE_FDATASYNC
#define __FSYNC fdatasync
#elif defined HAVE_FSYNC
#define __FSYNC fsync
#else
inline int __fsync(int fd) { return 0; }
#define __FSYNC __fsync
#endif

ssize_t write(int fd, const void* buf, size_t nbyte) {
	const char* p = static_cast<const char*>(buf);
	while (nbyte) {
		ssize_t c = ::write(fd, p, nbyte);
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		p += c;
		if likely(c == static_cast<ssize_t>(nbyte)) {
			break;
		}
		nbyte -= c;
	}
	return p - static_cast<const char*>(buf);
}

ssize_t pwrite(int fd, const void* buf, size_t nbyte, off_t offset) {
	const char* p = static_cast<const char*>(buf);
#ifndef HAVE_PWRITE
	if unlikely(io::lseek(fd, offset, SEEK_SET) == -1) {
		return -1;
	}
#endif
	while (nbyte) {
#ifndef HAVE_PWRITE
		ssize_t c = ::write(fd, p, nbyte);
#else
		ssize_t c = ::pwrite(fd, p, nbyte, offset);
#endif
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		p += c;
		if likely(c == static_cast<ssize_t>(nbyte)) {
			break;
		}
		nbyte -= c;
		offset += c;
	}
	return p - static_cast<const char*>(buf);
}

ssize_t read(int fd, void* buf, size_t nbyte) {
	char* p = static_cast<char*>(buf);
	while (true) {
		ssize_t c = ::read(fd, p, nbyte);
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return c;
	}
}

ssize_t pread(int fd, void* buf, size_t nbyte, off_t offset) {
	char* p = static_cast<char*>(buf);
#ifndef HAVE_PWRITE
	if unlikely(io::lseek(fd, offset, SEEK_SET) == -1) {
		return -1;
	}
#endif
	while (true) {
#ifndef HAVE_PWRITE
		ssize_t c = ::read(fd, p, nbyte);
#else
		ssize_t c = ::pread(fd, p, nbyte, offset);
#endif
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return c;
	}
}

int fsync(int fd) {
	while (true) {
		int r = __FSYNC(fd);
		if unlikely(r < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return r;
	}
}

int full_fsync(int fd) {
#ifdef F_FULLFSYNC
	while (true) {
		int r = fcntl(fd, F_FULLFSYNC, 0);
		if unlikely(r < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return r;
	}
#else
	return fsync(fd);
#endif
}

#ifndef HAVE_FALLOCATE
int fallocate(int fd, int /* mode */, off_t offset, off_t len) {
#if defined(HAVE_POSIX_FALLOCATE)
	return posix_fallocate(fd, offset, len);
#elif defined(F_PREALLOCATE)
	// Try to get a continous chunk of disk space
	fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, offset, len, 0};
	int ret = fcntl(fd, F_PREALLOCATE, &store);
	if (ret < 0) {
		// OK, perhaps we are too fragmented, allocate non-continuous
		store.fst_flags = F_ALLOCATEALL;
		ret = fcntl(fd, F_PREALLOCATE, &store);
	}

	ftruncate(fd, len);
	return ret;
#else
	// The following is copied from fcntlSizeHint in sqlite
	/* If the OS does not have posix_fallocate(), fake it. First use
	 ** ftruncate() to set the file size, then write a single byte to
	 ** the last byte in each block within the extended region. This
	 ** is the same technique used by glibc to implement posix_fallocate()
	 ** on systems that do not have a real fallocate() system call.
	 */

	struct stat buf;
	if (fstat(fd, &buf))
		return -1;

	if (buf.st_size >= len)
		return -1;

	const int nBlk = buf.st_blksize;

	if (!nBlk)
		return -1;

	if (ftruncate(fd, len))
		return -1;

	int nWrite;
	off_t iWrite = ((buf.st_size + 2 * nBlk - 1) / nBlk) * nBlk - 1; // Next offset to write to
	do {
		nWrite = 0;
		if (io::lseek(fd, iWrite, SEEK_SET) == iWrite) {
			nWrite = ::write(fd, "", 1);
		}
		iWrite += nBlk;
	} while (nWrite == 1 && iWrite < len);
	return 0;
#endif
}
#endif

} /* namespace io */
<commit_msg>Fixing bug in __FSYNC macro<commit_after>/*
 * Copyright (C) 2015,2016 deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "io_utils.h"

namespace io {

#if defined HAVE_FDATASYNC
#define __FSYNC ::fdatasync
#elif defined HAVE_FSYNC
#define __FSYNC ::fsync
#else
inline int __fsync(int fd) { return 0; }
#define __FSYNC __fsync
#endif

ssize_t write(int fd, const void* buf, size_t nbyte) {
	const char* p = static_cast<const char*>(buf);
	while (nbyte) {
		ssize_t c = ::write(fd, p, nbyte);
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		p += c;
		if likely(c == static_cast<ssize_t>(nbyte)) {
			break;
		}
		nbyte -= c;
	}
	return p - static_cast<const char*>(buf);
}

ssize_t pwrite(int fd, const void* buf, size_t nbyte, off_t offset) {
	const char* p = static_cast<const char*>(buf);
#ifndef HAVE_PWRITE
	if unlikely(io::lseek(fd, offset, SEEK_SET) == -1) {
		return -1;
	}
#endif
	while (nbyte) {
#ifndef HAVE_PWRITE
		ssize_t c = ::write(fd, p, nbyte);
#else
		ssize_t c = ::pwrite(fd, p, nbyte, offset);
#endif
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		p += c;
		if likely(c == static_cast<ssize_t>(nbyte)) {
			break;
		}
		nbyte -= c;
		offset += c;
	}
	return p - static_cast<const char*>(buf);
}

ssize_t read(int fd, void* buf, size_t nbyte) {
	char* p = static_cast<char*>(buf);
	while (true) {
		ssize_t c = ::read(fd, p, nbyte);
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return c;
	}
}

ssize_t pread(int fd, void* buf, size_t nbyte, off_t offset) {
	char* p = static_cast<char*>(buf);
#ifndef HAVE_PWRITE
	if unlikely(io::lseek(fd, offset, SEEK_SET) == -1) {
		return -1;
	}
#endif
	while (true) {
#ifndef HAVE_PWRITE
		ssize_t c = ::read(fd, p, nbyte);
#else
		ssize_t c = ::pread(fd, p, nbyte, offset);
#endif
		if unlikely(c < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return c;
	}
}

int fsync(int fd) {
	while (true) {
		int r = __FSYNC(fd);
		if unlikely(r < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return r;
	}
}

int full_fsync(int fd) {
#ifdef F_FULLFSYNC
	while (true) {
		int r = fcntl(fd, F_FULLFSYNC, 0);
		if unlikely(r < 0) {
			if (errno == EINTR) continue;
			return -1;
		}
		return r;
	}
#else
	return fsync(fd);
#endif
}

#ifndef HAVE_FALLOCATE
int fallocate(int fd, int /* mode */, off_t offset, off_t len) {
#if defined(HAVE_POSIX_FALLOCATE)
	return posix_fallocate(fd, offset, len);
#elif defined(F_PREALLOCATE)
	// Try to get a continous chunk of disk space
	fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, offset, len, 0};
	int ret = fcntl(fd, F_PREALLOCATE, &store);
	if (ret < 0) {
		// OK, perhaps we are too fragmented, allocate non-continuous
		store.fst_flags = F_ALLOCATEALL;
		ret = fcntl(fd, F_PREALLOCATE, &store);
	}

	ftruncate(fd, len);
	return ret;
#else
	// The following is copied from fcntlSizeHint in sqlite
	/* If the OS does not have posix_fallocate(), fake it. First use
	 ** ftruncate() to set the file size, then write a single byte to
	 ** the last byte in each block within the extended region. This
	 ** is the same technique used by glibc to implement posix_fallocate()
	 ** on systems that do not have a real fallocate() system call.
	 */

	struct stat buf;
	if (fstat(fd, &buf))
		return -1;

	if (buf.st_size >= len)
		return -1;

	const int nBlk = buf.st_blksize;

	if (!nBlk)
		return -1;

	if (ftruncate(fd, len))
		return -1;

	int nWrite;
	off_t iWrite = ((buf.st_size + 2 * nBlk - 1) / nBlk) * nBlk - 1; // Next offset to write to
	do {
		nWrite = 0;
		if (io::lseek(fd, iWrite, SEEK_SET) == iWrite) {
			nWrite = ::write(fd, "", 1);
		}
		iWrite += nBlk;
	} while (nWrite == 1 && iWrite < len);
	return 0;
#endif
}
#endif

} /* namespace io */
<|endoftext|>


<commit_before>#include "master.hpp"

namespace factor
{

extern int main();

const char *vm_executable_path(void)
{
	static Dl_info info = {0};
	if (!info.dli_fname)
		dladdr(main, &info);
	return info.dli_fname;
}

}
<commit_msg>NetBSD build fix<commit_after>#include "master.hpp"

namespace factor
{

extern "C" int main();

const char *vm_executable_path(void)
{
	static Dl_info info = {0};
	if (!info.dli_fname)
		dladdr((void *)main, &info);
	return info.dli_fname;
}

}
<|endoftext|>


<commit_before><commit_msg>GTK: delete text when cutting from the omnibox.<commit_after><|endoftext|>


<commit_before>#ifndef RPT_XMLREPORTELEMENTBASE_HXX
#define RPT_XMLREPORTELEMENTBASE_HXX
/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: xmlReportElementBase.hxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: rt $ $Date: 2007-07-09 11:56:18 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _XMLOFF_XMLICTXT_HXX
#include <xmloff/xmlictxt.hxx>
#endif
#ifndef _COM_SUN_STAR_REPORT_XREPORTCOMPONENT_HPP_
#include <com/sun/star/report/XReportComponent.hpp>
#endif

namespace rptxml
{
    class ORptFilter;
    class OXMLTable;
    class OXMLReportElementBase : public SvXMLImportContext
    {
        OXMLReportElementBase(const OXMLReportElementBase&);
        void operator =(const OXMLReportElementBase&);
    protected:
        ORptFilter&   m_rImport;
        OXMLTable*    m_pContainer;
        ::com::sun::star::uno::Reference< ::com::sun::star::report::XReportComponent >      m_xComponent;

        virtual SvXMLImportContext* _CreateChildContext( sal_uInt16 nPrefix,
                    const ::rtl::OUString& rLocalName,
                    const ::com::sun::star::uno::Reference< ::com::sun::star::xml::sax::XAttributeList > & xAttrList );
    public:

        OXMLReportElementBase( ORptFilter& rImport
                    ,sal_uInt16 nPrfx
                    ,const ::rtl::OUString& rLName
                    ,const ::com::sun::star::uno::Reference< ::com::sun::star::report::XReportComponent >& _xComponent
                    ,OXMLTable* _pContainer);
        virtual ~OXMLReportElementBase();

        virtual SvXMLImportContext *CreateChildContext( sal_uInt16 nPrefix,
                    const ::rtl::OUString& rLocalName,
                    const ::com::sun::star::uno::Reference< ::com::sun::star::xml::sax::XAttributeList > & xAttrList );

        virtual void EndElement();
    };
// -----------------------------------------------------------------------------
} // namespace rptxml
// -----------------------------------------------------------------------------

#endif // RPT_XMLREPORTELEMENTBASE_HXX
<commit_msg>INTEGRATION: CWS rptchart01_DEV300 (1.2.70); FILE MERGED 2008/02/13 07:11:53 oj 1.2.70.1: #i85225# impl chart readhandler and oleproducer<commit_after>#ifndef RPT_XMLREPORTELEMENTBASE_HXX
#define RPT_XMLREPORTELEMENTBASE_HXX
/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: xmlReportElementBase.hxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: kz $ $Date: 2008-03-05 18:05:31 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _XMLOFF_XMLICTXT_HXX
#include <xmloff/xmlictxt.hxx>
#endif
#ifndef _COM_SUN_STAR_REPORT_XREPORTCOMPONENT_HPP_
#include <com/sun/star/report/XReportComponent.hpp>
#endif
#include <vector>

namespace rptxml
{
    class ORptFilter;
    class OXMLTable;

    class SAL_NO_VTABLE IMasterDetailFieds
    {
    public:
        virtual void addMasterDetailPair(const ::std::pair< ::rtl::OUString,::rtl::OUString >& _aPair) = 0;
    };

    class OXMLReportElementBase : public SvXMLImportContext
    {
        OXMLReportElementBase(const OXMLReportElementBase&);
        void operator =(const OXMLReportElementBase&);
    protected:
        ORptFilter&   m_rImport;
        OXMLTable*    m_pContainer;
        ::com::sun::star::uno::Reference< ::com::sun::star::report::XReportComponent >      m_xComponent;

        virtual SvXMLImportContext* _CreateChildContext( sal_uInt16 nPrefix,
                    const ::rtl::OUString& rLocalName,
                    const ::com::sun::star::uno::Reference< ::com::sun::star::xml::sax::XAttributeList > & xAttrList );
    public:

        OXMLReportElementBase( ORptFilter& rImport
                    ,sal_uInt16 nPrfx
                    ,const ::rtl::OUString& rLName
                    ,const ::com::sun::star::uno::Reference< ::com::sun::star::report::XReportComponent >& _xComponent
                    ,OXMLTable* _pContainer);
        virtual ~OXMLReportElementBase();

        virtual SvXMLImportContext *CreateChildContext( sal_uInt16 nPrefix,
                    const ::rtl::OUString& rLocalName,
                    const ::com::sun::star::uno::Reference< ::com::sun::star::xml::sax::XAttributeList > & xAttrList );

        virtual void EndElement();
    };
// -----------------------------------------------------------------------------
} // namespace rptxml
// -----------------------------------------------------------------------------

#endif // RPT_XMLREPORTELEMENTBASE_HXX
<|endoftext|>


<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/service/service_process_control_manager.h"

#include "base/singleton.h"
#include "base/stl_util-inl.h"
#include "chrome/browser/chrome_thread.h"
#include "chrome/browser/profile.h"
#include "chrome/browser/service/service_process_control.h"

ServiceProcessControlManager::ServiceProcessControlManager() {
}

ServiceProcessControlManager::~ServiceProcessControlManager() {
  DCHECK(process_control_list_.empty());
}

ServiceProcessControl* ServiceProcessControlManager::GetProcessControl(
    Profile* profile, ServiceProcessType type) {
  DCHECK(ChromeThread::CurrentlyOn(ChromeThread::UI));

  // TODO(hclam): We will have different service process for different types of
  // service, but now we only create a new process for a different profile.
  for (ServiceProcessControlList::iterator i = process_control_list_.begin();
       i != process_control_list_.end(); ++i) {
    if ((*i)->profile() == profile)
      return *i;
  }

  // Couldn't find a ServiceProcess so construct a new one.
  ServiceProcessControl* process  = new ServiceProcessControl(profile, type);
  process_control_list_.push_back(process);
  return process;
}

void ServiceProcessControlManager::Shutdown() {
  // TODO(hclam): Normally we should just delete the list but for simplicity
  // we also shutdown the service processes.
  for (ServiceProcessControlList::iterator i = process_control_list_.begin();
       i != process_control_list_.end(); ++i) {
    (*i)->Shutdown();
  }
  STLDeleteElements(&process_control_list_);
}

// static
ServiceProcessControlManager* ServiceProcessControlManager::instance() {
  return Singleton<ServiceProcessControlManager>::get();
}
<commit_msg>Don't shutdown service process when browser shuts down<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/service/service_process_control_manager.h"

#include "base/singleton.h"
#include "base/stl_util-inl.h"
#include "chrome/browser/chrome_thread.h"
#include "chrome/browser/profile.h"
#include "chrome/browser/service/service_process_control.h"

ServiceProcessControlManager::ServiceProcessControlManager() {
}

ServiceProcessControlManager::~ServiceProcessControlManager() {
  DCHECK(process_control_list_.empty());
}

ServiceProcessControl* ServiceProcessControlManager::GetProcessControl(
    Profile* profile, ServiceProcessType type) {
  DCHECK(ChromeThread::CurrentlyOn(ChromeThread::UI));

  // TODO(hclam): We will have different service process for different types of
  // service, but now we only create a new process for a different profile.
  for (ServiceProcessControlList::iterator i = process_control_list_.begin();
       i != process_control_list_.end(); ++i) {
    if ((*i)->profile() == profile)
      return *i;
  }

  // Couldn't find a ServiceProcess so construct a new one.
  ServiceProcessControl* process  = new ServiceProcessControl(profile, type);
  process_control_list_.push_back(process);
  return process;
}

void ServiceProcessControlManager::Shutdown() {
  // When we shutdown the manager we just clear the list and don't actually
  // shutdown the service process.
  STLDeleteElements(&process_control_list_);
}

// static
ServiceProcessControlManager* ServiceProcessControlManager::instance() {
  return Singleton<ServiceProcessControlManager>::get();
}
<|endoftext|>


<commit_before>/*
 * Copyright 2022 The CFU-Playground Authors
 * Copyright 2019 The TensorFlow Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected_accel.h"

#include <cstdio>

#include "tensorflow/lite/kernels/internal/common.h"

namespace tflite {
namespace reference_integer_ops {

// Determine whether operation is suitable for acceleration
bool CanAccelerateFullyConnected(const FullyConnectedParams& params,
                                 const RuntimeShape& filter_shape,
                                 const int32_t* bias_data,
                                 const RuntimeShape& output_shape) {
  if (params.input_offset != 128) return false;
  if (params.weights_offset != 0) return false;
  if (bias_data == NULL) return false;
  if (filter_shape.Dims(1) % 16 != 0) return false;
  if (output_shape.Dims(0) != 1) return false;
  return true;
}

// Accelerated implementation
void AccelerateFullyConnected(
    const FullyConnectedParams& params, const RuntimeShape& input_shape,
    const int8_t* input_data, const RuntimeShape& filter_shape,
    const int8_t* filter_data, const RuntimeShape& bias_shape,
    const int32_t* bias_data, const RuntimeShape& output_shape,
    int8_t* output_data) {
  const int32_t input_offset = 128;
  const int32_t output_offset = params.output_offset;
  const int32_t output_multiplier = params.output_multiplier;
  const int output_shift = params.output_shift;
  const int32_t output_activation_min = params.quantized_activation_min;
  const int32_t output_activation_max = params.quantized_activation_max;
  TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2);
  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 2);

  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
  const int filter_dim_count = filter_shape.DimensionsCount();
  const int output_depth = output_shape.Dims(1);
  TFLITE_DCHECK_LE(output_depth, filter_shape.Dims(filter_dim_count - 2));
  const int accum_depth = filter_shape.Dims(filter_dim_count - 1);
  for (int out_c = 0; out_c < output_depth; ++out_c) {
    int32_t acc = 0;
    for (int d = 0; d < accum_depth; ++d) {
      int32_t input_val = input_data[d];
      int32_t filter_val = filter_data[out_c * accum_depth + d];
      acc += filter_val * (input_val + input_offset);
    }
    acc += bias_data[out_c];
    acc = MultiplyByQuantizedMultiplier(acc, output_multiplier, output_shift);
    acc += output_offset;
    acc = std::max(acc, output_activation_min);
    acc = std::min(acc, output_activation_max);
    output_data[out_c] = static_cast<int8_t>(acc);
  }
}

}  // namespace reference_integer_ops
}  // namespace tflite
<commit_msg>hps_accel/fully_connected: unroll loop<commit_after>/*
 * Copyright 2022 The CFU-Playground Authors
 * Copyright 2019 The TensorFlow Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected_accel.h"

#include <cstdio>

#include "tensorflow/lite/kernels/internal/common.h"

namespace tflite {
namespace reference_integer_ops {

// Determine whether operation is suitable for acceleration
bool CanAccelerateFullyConnected(const FullyConnectedParams& params,
                                 const RuntimeShape& filter_shape,
                                 const int32_t* bias_data,
                                 const RuntimeShape& output_shape) {
  if (params.input_offset != 128) return false;
  if (params.weights_offset != 0) return false;
  if (bias_data == NULL) return false;
  if (filter_shape.Dims(1) % 16 != 0) return false;
  if (output_shape.Dims(0) != 1) return false;
  return true;
}

// Accelerated implementation
void AccelerateFullyConnected(
    const FullyConnectedParams& params, const RuntimeShape& input_shape,
    const int8_t* input_data, const RuntimeShape& filter_shape,
    const int8_t* filter_data, const RuntimeShape& bias_shape,
    const int32_t* bias_data, const RuntimeShape& output_shape,
    int8_t* output_data) {
  const int32_t input_offset = 128;
  const int32_t output_offset = params.output_offset;
  const int32_t output_multiplier = params.output_multiplier;
  const int output_shift = params.output_shift;
  const int32_t output_activation_min = params.quantized_activation_min;
  const int32_t output_activation_max = params.quantized_activation_max;
  TFLITE_DCHECK_GE(filter_shape.DimensionsCount(), 2);
  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 2);

  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
  const int filter_dim_count = filter_shape.DimensionsCount();
  const int output_depth = output_shape.Dims(1);
  TFLITE_DCHECK_LE(output_depth, filter_shape.Dims(filter_dim_count - 2));
  const int accum_depth = filter_shape.Dims(filter_dim_count - 1);
  for (int out_c = 0; out_c < output_depth; ++out_c) {
    const int8_t* ip = input_data;
    int32_t acc = 0;
    for (int d = 0; d < accum_depth; d += 4) {
      acc += *filter_data++ * (input_offset + *ip++);
      acc += *filter_data++ * (input_offset + *ip++);
      acc += *filter_data++ * (input_offset + *ip++);
      acc += *filter_data++ * (input_offset + *ip++);
    }
    acc += bias_data[out_c];
    acc = MultiplyByQuantizedMultiplier(acc, output_multiplier, output_shift);
    acc += output_offset;
    acc = std::max(acc, output_activation_min);
    acc = std::min(acc, output_activation_max);
    output_data[out_c] = static_cast<int8_t>(acc);
  }
}

}  // namespace reference_integer_ops
}  // namespace tflite
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: cuckoo_index.cc
// -----------------------------------------------------------------------------

#include "cuckoo_index.h"

#include <cmath>
#include <cstddef>
#include <iostream>
#include <string>

#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "common/byte_coding.h"
#include "common/profiling.h"
#include "common/rle_bitmap.h"
#include "cuckoo_kicker.h"
#include "cuckoo_utils.h"
#include "evaluation_utils.h"
#include "fingerprint_store.h"

namespace ci {
namespace {

// When distributing values to buckets fails (see Distribute(..), below),
// increase the number of requested buckets by this factor.
constexpr double kNumBucketsGrowFactor = 1.01;

// Returns a map from values to their stripe-bitmaps.
const absl::flat_hash_map<int, Bitmap64Ptr> ValueToStripeBitmaps(
    const Column& column, size_t num_rows_per_stripe) {
  absl::flat_hash_map<int, Bitmap64Ptr> bitmaps;
  // Round down the number of rows to the next multiple of
  // `num_rows_per_stripe`, i.e., ignore the last stripe as elsewhere.
  const size_t num_stripes = column.num_rows() / num_rows_per_stripe;
  const size_t num_rows = num_stripes * num_rows_per_stripe;
  for (size_t row = 0; row < num_rows; ++row) {
    const int value = column[row];
    if (!bitmaps.contains(value))
      bitmaps[value] = absl::make_unique<Bitmap64>(/*size=*/num_stripes);
    bitmaps[value]->Set(row / num_rows_per_stripe, true);
  }
  return bitmaps;
}

// Distributes the `values` to buckets with the "kicking algorithm". Returns an
// empty vector if this failed, i.e., if there were too few buckets.
std::vector<Bucket> DistributeByKicking(size_t num_buckets,
                                        size_t slots_per_bucket,
                                        const std::vector<CuckooValue>& values,
                                        bool skew_kicking) {
  ScopedProfile kicker_insert_values(Counter::Kicking);
  std::vector<Bucket> buckets;
  buckets.reserve(num_buckets);
  for (size_t i = 0; i < num_buckets; ++i)
    buckets.push_back(Bucket(/*num_slots=*/slots_per_bucket));

  // Try to insert values with kicking.
  CuckooKicker kicker(slots_per_bucket, absl::MakeSpan(buckets), skew_kicking);
  const bool success = kicker.InsertValues(values);
  kicker.PrintStats();
  if (!success) return std::vector<Bucket>();

  // Look up values and set Bucket::kicked accordingly (such that `kicked`
  // contains all values that reside in their secondary bucket).
  for (const CuckooValue& value : values) {
    bool in_primary;
    LookupValueInBuckets(buckets, value, &in_primary);
    if (!in_primary) buckets[value.primary_bucket].kicked_.push_back(value);
  }

  return buckets;
}

// Distributes the `distinct_values` to buckets. Returns an empty vector if this
// failed, i.e., if there were too few buckets.
std::vector<Bucket> Distribute(
    size_t num_buckets, size_t slots_per_bucket, CuckooAlgorithm cuckoo_alg,
    const absl::flat_hash_set<int>& distinct_values) {
  std::vector<CuckooValue> values;
  values.reserve(distinct_values.size());
  for (size_t value : distinct_values)
    values.push_back(CuckooValue(value, num_buckets));
  switch (cuckoo_alg) {
    case CuckooAlgorithm::KICKING:
      return DistributeByKicking(num_buckets, slots_per_bucket, values,
                                 /*skew_kicking=*/false);
    case CuckooAlgorithm::SKEWED_KICKING:
      return DistributeByKicking(num_buckets, slots_per_bucket, values,
                                 /*skew_kicking=*/true);
    case CuckooAlgorithm::MATCHING:
      std::cerr << "Not implemented." << std::endl;
      std::exit(-1);
  }

  std::cerr << "Unknown algorithm: " << static_cast<int32_t>(cuckoo_alg)
            << std::endl;
  std::exit(-1);
}

// Computes the minimum `num_bits` which can be used per bucket and fills
// `slot_fingerprints` accordingly, also moves the `value_to_bitmap` entries to
// the corresponding `slot_bitmaps` entries.
void CreateSlots(double scan_rate, size_t slots_per_bucket,
                 const std::vector<Bucket>& buckets,
                 absl::flat_hash_map<int, Bitmap64Ptr>* value_to_bitmap,
                 std::vector<Fingerprint>* slot_fingerprints,
                 const bool prefix_bits_optimization,
                 Bitmap64Ptr* use_prefix_bits_bitmap,
                 std::vector<Bitmap64Ptr>* slot_bitmaps) {
  const size_t num_buckets = buckets.size();
  const size_t num_slots = num_buckets * slots_per_bucket;
  size_t num_empty_buckets = 0;
  for (const Bucket& b : buckets)
    if (b.slots_.empty()) ++num_empty_buckets;
  const double bucket_density =
      1.0 - static_cast<double>(num_empty_buckets) / num_buckets;

  slot_fingerprints->resize(num_slots);
  if (prefix_bits_optimization)
    *use_prefix_bits_bitmap = absl::make_unique<Bitmap64>(num_buckets);
  slot_bitmaps->resize(num_slots);
  for (size_t bucket_id = 0; bucket_id < num_buckets; ++bucket_id) {
    const Bucket& bucket = buckets[bucket_id];

    // Start by determining the minimum number of bits needed to avoid
    // collisions of values which are contained in the bucket or were kicked
    // from this bucket (which was their primary bucket).
    std::vector<uint64_t> possibly_colliding_fingerprints;
    for (const CuckooValue& value : bucket.slots_)
      possibly_colliding_fingerprints.push_back(value.fingerprint);
    for (const CuckooValue& value : bucket.kicked_)
      possibly_colliding_fingerprints.push_back(value.fingerprint);
    bool use_prefix_bits;
    size_t num_bits;
    if (prefix_bits_optimization) {
      num_bits = GetMinCollisionFreeFingerprintPrefixOrSuffix(
          possibly_colliding_fingerprints, &use_prefix_bits);
      (*use_prefix_bits_bitmap)->Set(bucket_id, use_prefix_bits);
    } else {
      num_bits = GetMinCollisionFreeFingerprintLength(
          possibly_colliding_fingerprints, /*use_prefix_bits=*/false);
    }

    // Now add more bits if needed to ensure the desired `scan_rate`.
    for (; num_bits <= 65; ++num_bits) {
      // Compute `actual_scan_rate` of `bucket` by averaging the
      // local scan rates of all items in `bucket`. The intuition here is that a
      // lookup can only match with a single fingerprint & that for an infinite
      // number of lookups we expect the scan rate to average out.
      const double fp_prob = 1.0 / std::pow(2, num_bits);
      double sum_scan_rate = 0.0;
      for (const CuckooValue& value : bucket.slots_) {
        const Bitmap64Ptr& bitmap = (*value_to_bitmap)[value.orig_value];
        assert(bitmap != nullptr);
        sum_scan_rate += (fp_prob * bitmap->GetOnesCount()) / bitmap->bits();
      }
      double actual_scan_rate = sum_scan_rate / bucket.slots_.size();
      // Adjust the scan rate by: 1) taking the density (aka load-factor) into
      // account and 2) taking into account that for every lookup we may
      // actually check two buckets: the primary and the secondary.
      actual_scan_rate *= bucket_density * 2;
      if (actual_scan_rate <= scan_rate) break;
    }
    // Check if can reach the desired scan rate.
    assert(num_bits != 65);

    // We have successfully determined `num_bits` => set the actual slots.
    for (size_t i = 0; i < slots_per_bucket; ++i) {
      const size_t slot = bucket_id * slots_per_bucket + i;
      Fingerprint& fp = (*slot_fingerprints)[slot];
      if (i >= bucket.slots_.size()) {
        fp.active = false;
        fp.num_bits = 0;
        fp.fingerprint = 0ULL;
      } else {
        fp.active = true;
        fp.num_bits = num_bits;
        fp.fingerprint =
            prefix_bits_optimization && use_prefix_bits
                ? GetFingerprintPrefix(bucket.slots_[i].fingerprint, num_bits)
                : GetFingerprintSuffix(bucket.slots_[i].fingerprint, num_bits);
        (*slot_bitmaps)[slot] =
            std::move((*value_to_bitmap)[bucket.slots_[i].orig_value]);
      }
    }
  }
}

// Returns the fingerprints and bitmaps encoded in a compact manner.
std::string Encode(const FingerprintStore& fingerprint_store,
                   const size_t slots_per_bucket,
                   const bool prefix_bits_optimization,
                   const Bitmap64Ptr& prefix_bits_bitmap,
                   const std::vector<Bitmap64Ptr>& slot_bitmaps) {
  ScopedProfile encode_profile(Counter::Encoding);
  ByteBuffer result;
  PutString(fingerprint_store.Encode(), &result);
  const size_t fp_size = result.pos();
  std::cout << "Encoded fingerprints: " << fp_size << std::endl;

  // Flag that denotes whether we use the prefix bits optimization. If set, the
  // flag is followed by the prefix bits bitmap.
  PutPrimitive(true, &result);
  if (prefix_bits_optimization) {
    // Encode prefix bits bitmap as RleBitmap (util::bitmap::DenseEncode() needs
    // significantly more space in sparse cases).
    const RleBitmap rle_bitmap(*prefix_bits_bitmap);
    PutString(rle_bitmap.data(), &result);
    std::cout << "Encoded prefix bits bitmap: " << result.pos() - fp_size
              << std::endl;
  }

  // Add the global bitmap, encoded as RleBitmap.
  const RleBitmap bitmap(GetGlobalBitmap(slot_bitmaps));
  const size_t before_global_bitmap = result.pos();
  PutString(bitmap.data(), &result);
  std::cout << "Encoded bitmaps: " << result.pos() - before_global_bitmap
            << std::endl;
  return std::string(result.data(), result.pos());
}

}  // namespace

bool CuckooIndex::StripeContains(size_t stripe_id, int value) const {
  const CuckooValue val(value, num_buckets_);
  size_t slot;
  if (!BucketContains(val.primary_bucket, val.fingerprint, &slot)) {
    if (!BucketContains(val.secondary_bucket, val.fingerprint, &slot))
      return false;
  }
  assert(slot_bitmaps_[slot] != nullptr);
  return slot_bitmaps_[slot]->Get(stripe_id);
}

Bitmap64 CuckooIndex::GetQualifyingStripes(int value, int num_stripes) const {
  const CuckooValue val(value, num_buckets_);
  size_t slot;
  if (!BucketContains(val.primary_bucket, val.fingerprint, &slot)) {
    if (!BucketContains(val.secondary_bucket, val.fingerprint, &slot)) {
      // Not found. Return empty bitmap.
      return Bitmap64(/*size=*/0);
    }
  }
  assert(slot_bitmaps_[slot] != nullptr);
  const Bitmap64& bitmap = *slot_bitmaps_[slot];
  Bitmap64 result(bitmap.bits());
  for (const size_t index : bitmap.TrueBitIndices())
    result.Set(index, true);
  return result;
}

bool CuckooIndex::BucketContains(size_t bucket, uint64_t fingerprint,
                                 size_t* slot) const {
  const bool use_prefix_bits = use_prefix_bits_bitmap_ == nullptr
                                   ? false
                                   : use_prefix_bits_bitmap_->Get(bucket);
  for (*slot = bucket * slots_per_bucket_;
       *slot < (bucket + 1) * slots_per_bucket_; ++(*slot)) {
    const Fingerprint& fp = fingerprint_store_->GetFingerprint(*slot);
    if (!fp.active) continue;
    if (use_prefix_bits) {
      if (fp.fingerprint == GetFingerprintPrefix(fingerprint, fp.num_bits))
        return true;
    } else {
      if (fp.fingerprint == GetFingerprintSuffix(fingerprint, fp.num_bits))
        return true;
    }
  }
  return false;
}

std::unique_ptr<IndexStructure> CuckooIndexFactory::Create(
    const Column& column, size_t num_rows_per_stripe) const {
  absl::flat_hash_map<int, Bitmap64Ptr> value_to_bitmap =
      ValueToStripeBitmaps(column, num_rows_per_stripe);
  // Fetch the set of distinct-values in `value_to_bitmap`. Note that this is
  // not necessarily the same as column.distinct_values(), since rows may have
  // been dropped at the end (so each stripe has the same size).
  absl::flat_hash_set<int> distinct_values;
  distinct_values.reserve(value_to_bitmap.size());
  for (const auto& [value, _] : value_to_bitmap) distinct_values.insert(value);

  size_t num_buckets = GetMinNumBuckets(distinct_values.size(),
                                        slots_per_bucket_, max_load_factor_);

  std::vector<Bucket> buckets;
  while (buckets.empty()) {
    std::cout << "Attempting to distribute " << distinct_values.size()
              << " values to " << num_buckets << " buckets with "
              << slots_per_bucket_ << " slots each. I.e., load-factor: "
              << static_cast<double>(distinct_values.size()) /
                     (slots_per_bucket_ * num_buckets)
              << std::endl;
    buckets = Distribute(num_buckets, slots_per_bucket_, cuckoo_alg_,
                         distinct_values);
    num_buckets =
        std::max(static_cast<size_t>(num_buckets * kNumBucketsGrowFactor),
                 num_buckets + 1);
  }

  std::vector<Fingerprint> slot_fingerprints;
  Bitmap64Ptr use_prefix_bits_bitmap;
  std::vector<Bitmap64Ptr> slot_bitmaps;
  CreateSlots(scan_rate_, slots_per_bucket_, buckets, &value_to_bitmap,
              &slot_fingerprints, prefix_bits_optimization_,
              &use_prefix_bits_bitmap, &slot_bitmaps);
  auto fingerprint_store = absl::make_unique<FingerprintStore>(
      slot_fingerprints, slots_per_bucket_,
      /*use_rle_to_encode_block_bitmaps=*/false);
  const std::string data =
      Encode(*fingerprint_store, slots_per_bucket_, prefix_bits_optimization_,
             use_prefix_bits_bitmap, slot_bitmaps);

  // Need to use WrapUnique<>(..) since we're calling a private c'tor.
  return absl::WrapUnique<CuckooIndex>(new CuckooIndex(
      index_name(), slots_per_bucket_, std::move(fingerprint_store),
      std::move(use_prefix_bits_bitmap), std::move(slot_bitmaps), data.size(),
      Compress(data).size()));
}

std::string CuckooIndexFactory::index_name() const {
  return absl::StrCat("CuckooIndex:", cuckoo_alg_, ":", max_load_factor_, ":",
                      scan_rate_);
}

}  // namespace ci
<commit_msg>Return an empty bitmap with the appropriate size.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: cuckoo_index.cc
// -----------------------------------------------------------------------------

#include "cuckoo_index.h"

#include <cmath>
#include <cstddef>
#include <iostream>
#include <string>

#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "common/byte_coding.h"
#include "common/profiling.h"
#include "common/rle_bitmap.h"
#include "cuckoo_kicker.h"
#include "cuckoo_utils.h"
#include "evaluation_utils.h"
#include "fingerprint_store.h"

namespace ci {
namespace {

// When distributing values to buckets fails (see Distribute(..), below),
// increase the number of requested buckets by this factor.
constexpr double kNumBucketsGrowFactor = 1.01;

// Returns a map from values to their stripe-bitmaps.
const absl::flat_hash_map<int, Bitmap64Ptr> ValueToStripeBitmaps(
    const Column& column, size_t num_rows_per_stripe) {
  absl::flat_hash_map<int, Bitmap64Ptr> bitmaps;
  // Round down the number of rows to the next multiple of
  // `num_rows_per_stripe`, i.e., ignore the last stripe as elsewhere.
  const size_t num_stripes = column.num_rows() / num_rows_per_stripe;
  const size_t num_rows = num_stripes * num_rows_per_stripe;
  for (size_t row = 0; row < num_rows; ++row) {
    const int value = column[row];
    if (!bitmaps.contains(value))
      bitmaps[value] = absl::make_unique<Bitmap64>(/*size=*/num_stripes);
    bitmaps[value]->Set(row / num_rows_per_stripe, true);
  }
  return bitmaps;
}

// Distributes the `values` to buckets with the "kicking algorithm". Returns an
// empty vector if this failed, i.e., if there were too few buckets.
std::vector<Bucket> DistributeByKicking(size_t num_buckets,
                                        size_t slots_per_bucket,
                                        const std::vector<CuckooValue>& values,
                                        bool skew_kicking) {
  ScopedProfile kicker_insert_values(Counter::Kicking);
  std::vector<Bucket> buckets;
  buckets.reserve(num_buckets);
  for (size_t i = 0; i < num_buckets; ++i)
    buckets.push_back(Bucket(/*num_slots=*/slots_per_bucket));

  // Try to insert values with kicking.
  CuckooKicker kicker(slots_per_bucket, absl::MakeSpan(buckets), skew_kicking);
  const bool success = kicker.InsertValues(values);
  kicker.PrintStats();
  if (!success) return std::vector<Bucket>();

  // Look up values and set Bucket::kicked accordingly (such that `kicked`
  // contains all values that reside in their secondary bucket).
  for (const CuckooValue& value : values) {
    bool in_primary;
    LookupValueInBuckets(buckets, value, &in_primary);
    if (!in_primary) buckets[value.primary_bucket].kicked_.push_back(value);
  }

  return buckets;
}

// Distributes the `distinct_values` to buckets. Returns an empty vector if this
// failed, i.e., if there were too few buckets.
std::vector<Bucket> Distribute(
    size_t num_buckets, size_t slots_per_bucket, CuckooAlgorithm cuckoo_alg,
    const absl::flat_hash_set<int>& distinct_values) {
  std::vector<CuckooValue> values;
  values.reserve(distinct_values.size());
  for (size_t value : distinct_values)
    values.push_back(CuckooValue(value, num_buckets));
  switch (cuckoo_alg) {
    case CuckooAlgorithm::KICKING:
      return DistributeByKicking(num_buckets, slots_per_bucket, values,
                                 /*skew_kicking=*/false);
    case CuckooAlgorithm::SKEWED_KICKING:
      return DistributeByKicking(num_buckets, slots_per_bucket, values,
                                 /*skew_kicking=*/true);
    case CuckooAlgorithm::MATCHING:
      std::cerr << "Not implemented." << std::endl;
      std::exit(-1);
  }

  std::cerr << "Unknown algorithm: " << static_cast<int32_t>(cuckoo_alg)
            << std::endl;
  std::exit(-1);
}

// Computes the minimum `num_bits` which can be used per bucket and fills
// `slot_fingerprints` accordingly, also moves the `value_to_bitmap` entries to
// the corresponding `slot_bitmaps` entries.
void CreateSlots(double scan_rate, size_t slots_per_bucket,
                 const std::vector<Bucket>& buckets,
                 absl::flat_hash_map<int, Bitmap64Ptr>* value_to_bitmap,
                 std::vector<Fingerprint>* slot_fingerprints,
                 const bool prefix_bits_optimization,
                 Bitmap64Ptr* use_prefix_bits_bitmap,
                 std::vector<Bitmap64Ptr>* slot_bitmaps) {
  const size_t num_buckets = buckets.size();
  const size_t num_slots = num_buckets * slots_per_bucket;
  size_t num_empty_buckets = 0;
  for (const Bucket& b : buckets)
    if (b.slots_.empty()) ++num_empty_buckets;
  const double bucket_density =
      1.0 - static_cast<double>(num_empty_buckets) / num_buckets;

  slot_fingerprints->resize(num_slots);
  if (prefix_bits_optimization)
    *use_prefix_bits_bitmap = absl::make_unique<Bitmap64>(num_buckets);
  slot_bitmaps->resize(num_slots);
  for (size_t bucket_id = 0; bucket_id < num_buckets; ++bucket_id) {
    const Bucket& bucket = buckets[bucket_id];

    // Start by determining the minimum number of bits needed to avoid
    // collisions of values which are contained in the bucket or were kicked
    // from this bucket (which was their primary bucket).
    std::vector<uint64_t> possibly_colliding_fingerprints;
    for (const CuckooValue& value : bucket.slots_)
      possibly_colliding_fingerprints.push_back(value.fingerprint);
    for (const CuckooValue& value : bucket.kicked_)
      possibly_colliding_fingerprints.push_back(value.fingerprint);
    bool use_prefix_bits;
    size_t num_bits;
    if (prefix_bits_optimization) {
      num_bits = GetMinCollisionFreeFingerprintPrefixOrSuffix(
          possibly_colliding_fingerprints, &use_prefix_bits);
      (*use_prefix_bits_bitmap)->Set(bucket_id, use_prefix_bits);
    } else {
      num_bits = GetMinCollisionFreeFingerprintLength(
          possibly_colliding_fingerprints, /*use_prefix_bits=*/false);
    }

    // Now add more bits if needed to ensure the desired `scan_rate`.
    for (; num_bits <= 65; ++num_bits) {
      // Compute `actual_scan_rate` of `bucket` by averaging the
      // local scan rates of all items in `bucket`. The intuition here is that a
      // lookup can only match with a single fingerprint & that for an infinite
      // number of lookups we expect the scan rate to average out.
      const double fp_prob = 1.0 / std::pow(2, num_bits);
      double sum_scan_rate = 0.0;
      for (const CuckooValue& value : bucket.slots_) {
        const Bitmap64Ptr& bitmap = (*value_to_bitmap)[value.orig_value];
        assert(bitmap != nullptr);
        sum_scan_rate += (fp_prob * bitmap->GetOnesCount()) / bitmap->bits();
      }
      double actual_scan_rate = sum_scan_rate / bucket.slots_.size();
      // Adjust the scan rate by: 1) taking the density (aka load-factor) into
      // account and 2) taking into account that for every lookup we may
      // actually check two buckets: the primary and the secondary.
      actual_scan_rate *= bucket_density * 2;
      if (actual_scan_rate <= scan_rate) break;
    }
    // Check if can reach the desired scan rate.
    assert(num_bits != 65);

    // We have successfully determined `num_bits` => set the actual slots.
    for (size_t i = 0; i < slots_per_bucket; ++i) {
      const size_t slot = bucket_id * slots_per_bucket + i;
      Fingerprint& fp = (*slot_fingerprints)[slot];
      if (i >= bucket.slots_.size()) {
        fp.active = false;
        fp.num_bits = 0;
        fp.fingerprint = 0ULL;
      } else {
        fp.active = true;
        fp.num_bits = num_bits;
        fp.fingerprint =
            prefix_bits_optimization && use_prefix_bits
                ? GetFingerprintPrefix(bucket.slots_[i].fingerprint, num_bits)
                : GetFingerprintSuffix(bucket.slots_[i].fingerprint, num_bits);
        (*slot_bitmaps)[slot] =
            std::move((*value_to_bitmap)[bucket.slots_[i].orig_value]);
      }
    }
  }
}

// Returns the fingerprints and bitmaps encoded in a compact manner.
std::string Encode(const FingerprintStore& fingerprint_store,
                   const size_t slots_per_bucket,
                   const bool prefix_bits_optimization,
                   const Bitmap64Ptr& prefix_bits_bitmap,
                   const std::vector<Bitmap64Ptr>& slot_bitmaps) {
  ScopedProfile encode_profile(Counter::Encoding);
  ByteBuffer result;
  PutString(fingerprint_store.Encode(), &result);
  const size_t fp_size = result.pos();
  std::cout << "Encoded fingerprints: " << fp_size << std::endl;

  // Flag that denotes whether we use the prefix bits optimization. If set, the
  // flag is followed by the prefix bits bitmap.
  PutPrimitive(true, &result);
  if (prefix_bits_optimization) {
    // Encode prefix bits bitmap as RleBitmap (util::bitmap::DenseEncode() needs
    // significantly more space in sparse cases).
    const RleBitmap rle_bitmap(*prefix_bits_bitmap);
    PutString(rle_bitmap.data(), &result);
    std::cout << "Encoded prefix bits bitmap: " << result.pos() - fp_size
              << std::endl;
  }

  // Add the global bitmap, encoded as RleBitmap.
  const RleBitmap bitmap(GetGlobalBitmap(slot_bitmaps));
  const size_t before_global_bitmap = result.pos();
  PutString(bitmap.data(), &result);
  std::cout << "Encoded bitmaps: " << result.pos() - before_global_bitmap
            << std::endl;
  return std::string(result.data(), result.pos());
}

}  // namespace

bool CuckooIndex::StripeContains(size_t stripe_id, int value) const {
  const CuckooValue val(value, num_buckets_);
  size_t slot;
  if (!BucketContains(val.primary_bucket, val.fingerprint, &slot)) {
    if (!BucketContains(val.secondary_bucket, val.fingerprint, &slot))
      return false;
  }
  assert(slot_bitmaps_[slot] != nullptr);
  return slot_bitmaps_[slot]->Get(stripe_id);
}

Bitmap64 CuckooIndex::GetQualifyingStripes(int value, int num_stripes) const {
  const CuckooValue val(value, num_buckets_);
  size_t slot;
  if (!BucketContains(val.primary_bucket, val.fingerprint, &slot)) {
    if (!BucketContains(val.secondary_bucket, val.fingerprint, &slot)) {
      // Not found. Return an empty bitmap.
      return Bitmap64(/*size=*/num_stripes);
    }
  }
  assert(slot_bitmaps_[slot] != nullptr);
  const Bitmap64& bitmap = *slot_bitmaps_[slot];
  Bitmap64 result(bitmap.bits());
  for (const size_t index : bitmap.TrueBitIndices())
    result.Set(index, true);
  return result;
}

bool CuckooIndex::BucketContains(size_t bucket, uint64_t fingerprint,
                                 size_t* slot) const {
  const bool use_prefix_bits = use_prefix_bits_bitmap_ == nullptr
                                   ? false
                                   : use_prefix_bits_bitmap_->Get(bucket);
  for (*slot = bucket * slots_per_bucket_;
       *slot < (bucket + 1) * slots_per_bucket_; ++(*slot)) {
    const Fingerprint& fp = fingerprint_store_->GetFingerprint(*slot);
    if (!fp.active) continue;
    if (use_prefix_bits) {
      if (fp.fingerprint == GetFingerprintPrefix(fingerprint, fp.num_bits))
        return true;
    } else {
      if (fp.fingerprint == GetFingerprintSuffix(fingerprint, fp.num_bits))
        return true;
    }
  }
  return false;
}

std::unique_ptr<IndexStructure> CuckooIndexFactory::Create(
    const Column& column, size_t num_rows_per_stripe) const {
  absl::flat_hash_map<int, Bitmap64Ptr> value_to_bitmap =
      ValueToStripeBitmaps(column, num_rows_per_stripe);
  // Fetch the set of distinct-values in `value_to_bitmap`. Note that this is
  // not necessarily the same as column.distinct_values(), since rows may have
  // been dropped at the end (so each stripe has the same size).
  absl::flat_hash_set<int> distinct_values;
  distinct_values.reserve(value_to_bitmap.size());
  for (const auto& [value, _] : value_to_bitmap) distinct_values.insert(value);

  size_t num_buckets = GetMinNumBuckets(distinct_values.size(),
                                        slots_per_bucket_, max_load_factor_);

  std::vector<Bucket> buckets;
  while (buckets.empty()) {
    std::cout << "Attempting to distribute " << distinct_values.size()
              << " values to " << num_buckets << " buckets with "
              << slots_per_bucket_ << " slots each. I.e., load-factor: "
              << static_cast<double>(distinct_values.size()) /
                     (slots_per_bucket_ * num_buckets)
              << std::endl;
    buckets = Distribute(num_buckets, slots_per_bucket_, cuckoo_alg_,
                         distinct_values);
    num_buckets =
        std::max(static_cast<size_t>(num_buckets * kNumBucketsGrowFactor),
                 num_buckets + 1);
  }

  std::vector<Fingerprint> slot_fingerprints;
  Bitmap64Ptr use_prefix_bits_bitmap;
  std::vector<Bitmap64Ptr> slot_bitmaps;
  CreateSlots(scan_rate_, slots_per_bucket_, buckets, &value_to_bitmap,
              &slot_fingerprints, prefix_bits_optimization_,
              &use_prefix_bits_bitmap, &slot_bitmaps);
  auto fingerprint_store = absl::make_unique<FingerprintStore>(
      slot_fingerprints, slots_per_bucket_,
      /*use_rle_to_encode_block_bitmaps=*/false);
  const std::string data =
      Encode(*fingerprint_store, slots_per_bucket_, prefix_bits_optimization_,
             use_prefix_bits_bitmap, slot_bitmaps);

  // Need to use WrapUnique<>(..) since we're calling a private c'tor.
  return absl::WrapUnique<CuckooIndex>(new CuckooIndex(
      index_name(), slots_per_bucket_, std::move(fingerprint_store),
      std::move(use_prefix_bits_bitmap), std::move(slot_bitmaps), data.size(),
      Compress(data).size()));
}

std::string CuckooIndexFactory::index_name() const {
  return absl::StrCat("CuckooIndex:", cuckoo_alg_, ":", max_load_factor_, ":",
                      scan_rate_);
}

}  // namespace ci
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit
  Module:    itkBioGene.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) 2002 Insight Consortium. All rights reserved.
  See ITKCopyright.txt or http://www.itk.org/HTML/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "itkBioGene.h"

namespace itk {

namespace bio {

/**
 *    Constructor
 */
Gene
::Gene()
{

}

/**
 *    Destructor
 */
Gene
::~Gene()
{
}

/**
 *    Copy from another genome
 */
void
Gene
::Copy( const Gene & gene )
{
}

}  // end namespace bio

}  // end namespace itk

<commit_msg>ENH: Implmentation added to the Copy() method.<commit_after>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit
  Module:    itkBioGene.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) 2002 Insight Consortium. All rights reserved.
  See ITKCopyright.txt or http://www.itk.org/HTML/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "itkBioGene.h"
#include <algorithm>

namespace itk {

namespace bio {

/**
 *    Constructor
 */
Gene
::Gene()
{
  m_Name = "Unknown";
}

/**
 *    Destructor
 */
Gene
::~Gene()
{
}

/**
 *    Copy from another genome
 */
void
Gene
::Copy( const Gene & gene )
{

  m_Name = gene.m_Name;

  m_ControlDomains.insert( m_ControlDomains.begin(),
                           gene.m_ControlDomains.begin(),
                           gene.m_ControlDomains.end()    );

  m_ProteinDomains.insert( gene.m_ProteinDomains.begin(),
                           gene.m_ProteinDomains.end()    );

}

}  // end namespace bio

}  // end namespace itk

<|endoftext|>


<commit_before>// Copyright (c) 2013-2015, Matt Godbolt
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

#include "internal/HybiPacketDecoder.h"
#include "internal/LogStream.h"

#include <arpa/inet.h>

namespace seasocks {

HybiPacketDecoder::HybiPacketDecoder(Logger& logger,
                                     const std::vector<uint8_t>& buffer) :
    _logger(logger),
    _buffer(buffer),
    _messageStart(0) {
}

HybiPacketDecoder::MessageState HybiPacketDecoder::decodeNextMessage(
        std::vector<uint8_t>& messageOut) {
    if (_messageStart + 1 >= _buffer.size()) {
        return MessageState::NoMessage;
    }
    if ((_buffer[_messageStart] & 0x80) == 0) {
        // FIN bit is not clear...
        // TODO: support
        LS_WARNING(&_logger, "Received hybi frame without FIN bit set - unsupported");
        return MessageState::Error;
    }
    if ((_buffer[_messageStart] & (7<<4)) != 0) {
        LS_WARNING(&_logger, "Received hybi frame with reserved bits set - error");
        return MessageState::Error;
    }
    auto opcode = static_cast<Opcode>(_buffer[_messageStart] & 0xf);
    size_t payloadLength = _buffer[_messageStart + 1] & 0x7fu;
    auto maskBit = _buffer[_messageStart + 1] & 0x80;
    auto ptr = _messageStart + 2;
    if (payloadLength == 126) {
        if (_buffer.size() < 4) { return MessageState::NoMessage; }
        payloadLength = htons(*reinterpret_cast<const uint16_t*>(&_buffer[ptr]));
        ptr += 2;
    } else if (payloadLength == 127) {
        if (_buffer.size() < 10) { return MessageState::NoMessage; }
        payloadLength = __bswap_64(*reinterpret_cast<const uint64_t*>(&_buffer[ptr]));
        ptr += 8;
    }
    uint32_t mask = 0;
    if (maskBit) {
        // MASK is set.
        if (_buffer.size() < ptr + 4) { return MessageState::NoMessage; }
        mask = htonl(*reinterpret_cast<const uint32_t*>(&_buffer[ptr]));
        ptr += 4;
    }
    auto bytesLeftInBuffer = _buffer.size() - ptr;
    if (payloadLength > bytesLeftInBuffer) { return MessageState::NoMessage; }

    messageOut.clear();
    messageOut.reserve(payloadLength);
    for (auto i = 0u; i < payloadLength; ++i) {
        auto byteShift = (3 - (i & 3)) * 8;
        messageOut.push_back(static_cast<uint8_t>((_buffer[ptr++] ^ (mask >> byteShift)) & 0xff));
    }
    _messageStart = ptr;
    switch (opcode) {
    default:
        LS_WARNING(&_logger, "Received hybi frame with unknown opcode "
                             << static_cast<int>(opcode));
        return MessageState::Error;
    case Opcode::Text:
        return MessageState::TextMessage;
    case Opcode::Binary:
        return MessageState::BinaryMessage;
    case Opcode::Ping:
        return MessageState::Ping;
    case Opcode::Close:
        return MessageState::Close;
    }
}

size_t HybiPacketDecoder::numBytesDecoded() const {
    return _messageStart;
}

}
<commit_msg>Fixed alignment error in HybiPacketDecoder on ARM.<commit_after>// Copyright (c) 2013-2015, Matt Godbolt
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

#include "internal/HybiPacketDecoder.h"
#include "internal/LogStream.h"

#include <arpa/inet.h>
#include <string.h>

namespace seasocks {

HybiPacketDecoder::HybiPacketDecoder(Logger& logger,
                                     const std::vector<uint8_t>& buffer) :
    _logger(logger),
    _buffer(buffer),
    _messageStart(0) {
}

HybiPacketDecoder::MessageState HybiPacketDecoder::decodeNextMessage(
        std::vector<uint8_t>& messageOut) {
    if (_messageStart + 1 >= _buffer.size()) {
        return MessageState::NoMessage;
    }
    if ((_buffer[_messageStart] & 0x80) == 0) {
        // FIN bit is not clear...
        // TODO: support
        LS_WARNING(&_logger, "Received hybi frame without FIN bit set - unsupported");
        return MessageState::Error;
    }
    if ((_buffer[_messageStart] & (7<<4)) != 0) {
        LS_WARNING(&_logger, "Received hybi frame with reserved bits set - error");
        return MessageState::Error;
    }
    auto opcode = static_cast<Opcode>(_buffer[_messageStart] & 0xf);
    size_t payloadLength = _buffer[_messageStart + 1] & 0x7fu;
    auto maskBit = _buffer[_messageStart + 1] & 0x80;
    auto ptr = _messageStart + 2;
    if (payloadLength == 126) {
        if (_buffer.size() < 4) { return MessageState::NoMessage; }
        uint16_t raw_length;
        memcpy(&raw_length, &_buffer[ptr], sizeof(raw_length));
        payloadLength = htons(raw_length);
        ptr += 2;
    } else if (payloadLength == 127) {
        if (_buffer.size() < 10) { return MessageState::NoMessage; }
        uint64_t raw_length;
        memcpy(&raw_length, &_buffer[ptr], sizeof(raw_length));
        payloadLength = __bswap_64(raw_length);
        ptr += 8;
    }
    uint32_t mask = 0;
    if (maskBit) {
        // MASK is set.
        if (_buffer.size() < ptr + 4) { return MessageState::NoMessage; }
        uint32_t raw_length;
        memcpy(&raw_length, &_buffer[ptr], sizeof(raw_length));
        mask = htonl(raw_length);
        ptr += 4;
    }
    auto bytesLeftInBuffer = _buffer.size() - ptr;
    if (payloadLength > bytesLeftInBuffer) { return MessageState::NoMessage; }

    messageOut.clear();
    messageOut.reserve(payloadLength);
    for (auto i = 0u; i < payloadLength; ++i) {
        auto byteShift = (3 - (i & 3)) * 8;
        messageOut.push_back(static_cast<uint8_t>((_buffer[ptr++] ^ (mask >> byteShift)) & 0xff));
    }
    _messageStart = ptr;
    switch (opcode) {
    default:
        LS_WARNING(&_logger, "Received hybi frame with unknown opcode "
                             << static_cast<int>(opcode));
        return MessageState::Error;
    case Opcode::Text:
        return MessageState::TextMessage;
    case Opcode::Binary:
        return MessageState::BinaryMessage;
    case Opcode::Ping:
        return MessageState::Ping;
    case Opcode::Close:
        return MessageState::Close;
    }
}

size_t HybiPacketDecoder::numBytesDecoded() const {
    return _messageStart;
}

}
<|endoftext|>


<commit_before>#include "TestParser.h"

void TestParser::init() {
    QFile logFile("episodes.log");
    if (!logFile.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate))
        QFAIL("Could not create sample Episodes log file.");

    QTextStream out(&logFile);
    out << "218.56.155.59 [Sunday, 14-Nov-2010 06:27:03 +0100] \"?ets=css:203,headerjs:94,footerjs:500,domready:843,tabs:110,ToThePointShowHideChangelog:15,DrupalBehaviors:141,frontend:1547\" 200 \"http://driverpacks.net/driverpacks/windows/xp/x86/chipset/10.09\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\" \"driverpacks.net\"" << "\n"
        << "190.166.203.6 [Sunday, 14-Nov-2010 06:27:06 +0100] \"?ets=css:0,headerjs:588,footerjs:61,domready:680,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:998\" 200 \"http://driverpacks.net/\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.44 Safari/534.7\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:08 +0100] \"?ets=css:0,headerjs:41,footerjs:0,domready:822,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:990\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:11 +0100] \"?ets=backend:439,css:61,headerjs:162,footerjs:0,domready:318,tabs:1,tableHeader:16,ToThePointShowHideChangelog:0,DrupalBehaviors:17,pageready:775,frontend:336,totaltime:775\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07/drivers\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:12 +0100] \"?ets=css:40,headerjs:90,footerjs:0,domready:223,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:382\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n";

    logFile.close();
}

void TestParser::cleanup() {
    QFile logFile("episodes.log");
    if (!logFile.remove())
        QFAIL("Could not delete sample Episodes log file.");
}

void TestParser::parse() {
    Parser parser;

    QVERIFY(parser.parse("episodes.log") == 5);
}

void TestParser::mapLineToEpisodesLogLine_data() {
    QTest::addColumn<QString>("line");
    QTest::addColumn<IPAddress>("ip");
    QTest::addColumn<Time>("time");
    QTest::addColumn<EpisodeList>("episodes");
    QTest::addColumn<HTTPStatus>("status");
    QTest::addColumn<DomainID>("domainID");

    QTest::newRow("sample line 1")
      << "218.56.155.59 [Sunday, 14-Nov-2010 06:27:03 +0100] \"?ets=css:203,headerjs:94,footerjs:500,domready:843,tabs:110,ToThePointShowHideChangelog:15,DrupalBehaviors:141,frontend:1547\" 200 \"http://driverpacks.net/driverpacks/windows/xp/x86/chipset/10.09\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\" \"driverpacks.net\""
      << (IPAddress) 3661142843
      << (Time) 1289712423
      << (
            EpisodeList()
            << Episode(0, 203)
            << Episode(1, 94)
            << Episode(2, 244)
            << Episode(3, 75)
            << Episode(4, 110)
            << Episode(5, 15)
            << Episode(6, 141)
            << Episode(7, 11)
      )
      << (HTTPStatus) 200
      << (DomainID) 0;
}

void TestParser::mapLineToEpisodesLogLine() {
    static Parser p;
    static EpisodesLogLine e;
    QFETCH(QString, line);
    QFETCH(IPAddress, ip);
    QFETCH(Time, time);
    QFETCH(EpisodeList, episodes);
    QFETCH(HTTPStatus, status);
    QFETCH(DomainID, domainID);

    e = p.mapLineToEpisodesLogLine(line);

    QCOMPARE(e.ip, ip);
    QCOMPARE(e.time, time);
    QCOMPARE(e.episodes, episodes);
    QCOMPARE(e.status, status);
    QCOMPARE(e.domain.id, domainID);
}
<commit_msg>Finished off the first unit test.<commit_after>#include "TestParser.h"

void TestParser::init() {
    QFile logFile("episodes.log");
    if (!logFile.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate))
        QFAIL("Could not create sample Episodes log file.");

    QTextStream out(&logFile);
    out << "218.56.155.59 [Sunday, 14-Nov-2010 06:27:03 +0100] \"?ets=css:203,headerjs:94,footerjs:500,domready:843,tabs:110,ToThePointShowHideChangelog:15,DrupalBehaviors:141,frontend:1547\" 200 \"http://driverpacks.net/driverpacks/windows/xp/x86/chipset/10.09\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\" \"driverpacks.net\"" << "\n"
        << "190.166.203.6 [Sunday, 14-Nov-2010 06:27:06 +0100] \"?ets=css:0,headerjs:588,footerjs:61,domready:680,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:998\" 200 \"http://driverpacks.net/\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.44 Safari/534.7\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:08 +0100] \"?ets=css:0,headerjs:41,footerjs:0,domready:822,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:990\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:11 +0100] \"?ets=backend:439,css:61,headerjs:162,footerjs:0,domready:318,tabs:1,tableHeader:16,ToThePointShowHideChangelog:0,DrupalBehaviors:17,pageready:775,frontend:336,totaltime:775\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07/drivers\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n"
        << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:12 +0100] \"?ets=css:40,headerjs:90,footerjs:0,domready:223,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:382\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\"" << "\n";

    logFile.close();
}

void TestParser::cleanup() {
    QFile logFile("episodes.log");
    if (!logFile.remove())
        QFAIL("Could not delete sample Episodes log file.");
}

void TestParser::parse() {
    Parser parser;

    QVERIFY(parser.parse("episodes.log") == 5);
}

void TestParser::mapLineToEpisodesLogLine_data() {
    QTest::addColumn<QString>("line");
    QTest::addColumn<IPAddress>("ip");
    QTest::addColumn<Time>("time");
    QTest::addColumn<EpisodeList>("episodes");
    QTest::addColumn<HTTPStatus>("status");
    QTest::addColumn<DomainID>("domainID");

    QTest::newRow("sample line 1")
      << "218.56.155.59 [Sunday, 14-Nov-2010 06:27:03 +0100] \"?ets=css:203,headerjs:94,footerjs:500,domready:843,tabs:110,ToThePointShowHideChangelog:15,DrupalBehaviors:141,frontend:1547\" 200 \"http://driverpacks.net/driverpacks/windows/xp/x86/chipset/10.09\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\" \"driverpacks.net\""
      << (IPAddress) 3661142843
      << (Time) 1289712423
      << (
            EpisodeList()
            << Episode(0, 203)
            << Episode(1, 94)
            << Episode(2, 244)
            << Episode(3, 75)
            << Episode(4, 110)
            << Episode(5, 15)
            << Episode(6, 141)
            << Episode(7, 11)
      )
      << (HTTPStatus) 200
      << (DomainID) 0;

    QTest::newRow("sample line 2")
      << "190.166.203.6 [Sunday, 14-Nov-2010 06:27:06 +0100] \"?ets=css:0,headerjs:588,footerjs:61,domready:680,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:998\" 200 \"http://driverpacks.net/\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.44 Safari/534.7\" \"driverpacks.net\""
      << (IPAddress) 3198601990
      << (Time) 1289712426
      << (
            EpisodeList()
            << Episode(0, 0)
            << Episode(1, 588)
            << Episode(2, 61)
            << Episode(3, 680)
            << Episode(4, 1)
            << Episode(5, 0)
            << Episode(6, 1)
            << Episode(7, 998)
      )
      << (HTTPStatus) 200
      << (DomainID) 0;

    QTest::newRow("sample line 3")
      << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:08 +0100] \"?ets=css:0,headerjs:41,footerjs:0,domready:822,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:990\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\""
      << (IPAddress) 1286248989
      << (Time) 1289712428
      << (
            EpisodeList()
            << Episode(0, 0)
            << Episode(1, 41)
            << Episode(2, 0)
            << Episode(3, 822)
            << Episode(4, 1)
            << Episode(5, 0)
            << Episode(6, 1)
            << Episode(7, 990)
      )
      << (HTTPStatus) 200
      << (DomainID) 0;

    QTest::newRow("sample line 4")
      << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:11 +0100] \"?ets=backend:439,css:61,headerjs:162,footerjs:0,domready:318,tabs:1,tableHeader:16,ToThePointShowHideChangelog:0,DrupalBehaviors:17,pageready:775,frontend:336,totaltime:775\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07/drivers\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\""
      << (IPAddress) 1286248989
      << (Time) 1289712431
      << (
            EpisodeList()
            << Episode(8, 439) // New episode name: "backend".
            << Episode(0, 61)
            << Episode(1, 162)
            << Episode(2, 0)
            << Episode(3, 318)
            << Episode(4, 1)
            << Episode(9, 16) // New episode name: "tableHeader".
            << Episode(5, 0)
            << Episode(6, 17)
            << Episode(10, 775) // New episode name: "pageReady".
            << Episode(7, 336)
            << Episode(11, 775) // New episode name: "totalTime".
      )
      << (HTTPStatus) 200
      << (DomainID) 0;

    QTest::newRow("sample line 5")
      << "76.170.154.29 [Sunday, 14-Nov-2010 06:27:12 +0100] \"?ets=css:40,headerjs:90,footerjs:0,domready:223,tabs:1,ToThePointShowHideChangelog:0,DrupalBehaviors:1,frontend:382\" 200 \"http://driverpacks.net/driverpacks/windows/7/x86/graphics-b/10.07\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12\" \"driverpacks.net\""
      << (IPAddress) 1286248989
      << (Time) 1289712432
      << (
            EpisodeList()
            << Episode(0, 40)
            << Episode(1, 90)
            << Episode(2, 0)
            << Episode(3, 223)
            << Episode(4, 1)
            << Episode(5, 0)
            << Episode(6, 1)
            << Episode(7, 382)
      )
      << (HTTPStatus) 200
      << (DomainID) 0;
}

void TestParser::mapLineToEpisodesLogLine() {
    static Parser p;
    static EpisodesLogLine e;
    QFETCH(QString, line);
    QFETCH(IPAddress, ip);
    QFETCH(Time, time);
    QFETCH(EpisodeList, episodes);
    QFETCH(HTTPStatus, status);
    QFETCH(DomainID, domainID);

    e = p.mapLineToEpisodesLogLine(line);

    QCOMPARE(e.ip, ip);
    QCOMPARE(e.time, time);
    QCOMPARE(e.episodes, episodes);
    QCOMPARE(e.status, status);
    QCOMPARE(e.domain.id, domainID);
}
<|endoftext|>