<commit_before>/*
 * Copyright (C) 2012
 * Alessio Sclocco <a.sclocco@vu.nl>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include <fstream>
#include <vector>
#include <string>
#include <cmath>
#include <H5Cpp.h>

using std::ios;
using std::string;
using std::ifstream;
using std::vector;
using std::sqrt;
using H5::H5File;
using H5::Group;
using H5::DataSet;
using H5::FloatType;
using H5::IntType;

#include <Observation.hpp>
#include <GPUData.hpp>
#include <utils.hpp>

using isa::OpenCL::GPUData;
using isa::utils::toStringValue;
using isa::utils::changeEndianness;

#ifndef READ_DATA_HPP
#define READ_DATA_HPP

namespace AstroData {

template< typename T > void readSIGPROC(Observation< T > &observation, unsigned int bytestoSkip, unsigned int *paddedSecond, ifstream *inputFile, vector< GPUData< T > * > &data);
template< typename T > void readLOFAR(string headerFilename, string rawFilename, Observation< T > &observation, unsigned int *paddedSecond, vector< GPUData< T > * > &data);

// Implementation

template< typename T > void readSIGPROC(Observation< T > &observation, unsigned int bytesToSkip, unsigned int *paddedSecond, string inputFilename, vector< GPUData< T > * > &data) {
	ifstream inputFile;
	const unsigned int BUFFER_DIM = 4;
	*paddedSecond = observation.getNrSamplesPerSecond() + (observation.getNrSamplesPerSecond() % 4);
	char *buffer = new char [BUFFER_DIM];

	inputFile.open(inputFilename.c_str());
	inputFile.sync_with_stdio(false);
	inputFile.ignore(bytesToSkip);
	for ( unsigned int second = 0; second < observation.getNrSeconds(); second++ ) {
		data.at(second) = new GPUData< T >("second" + toStringValue< unsigned int >(second), true, true);
		(data.at(second))->allocateHostData((*paddedSecond) * observation.getNrChannels());

		for ( unsigned int sample = 0; sample < observation.getNrSamplesPerSecond(); sample++ ) {
			for ( unsigned int channel = observation.getNrChannels(); channel > 0; channel-- ) {
				inputFile.read(buffer, BUFFER_DIM);
				((data.at(second))->getHostData())[((channel - 1) * (*paddedSecond)) + sample] = *(reinterpret_cast< T * >(buffer));
			}
		}
	}
	inputFile.close();

	delete [] buffer;
}

template< typename T > void readLOFAR(string headerFilename, string rawFilename, Observation< T > &observation, unsigned int *paddedSecond, vector< GPUData< T > * > &data) {
	unsigned int totalSamples = 0;
	unsigned int nrSubbands = 0;
	char *word = new char[4];
	double totalIntegrationTime = 0.0;

	// Read the HDF5 file with the metadata
	H5File headerFile = H5File(headerFilename, H5F_ACC_RDONLY);
	FloatType typeDouble = FloatType(H5::PredType::NATIVE_DOUBLE);
	double valueDouble = 0.0;
	IntType typeUInt = IntType(H5::PredType::NATIVE_UINT);
	unsigned int valueUInt = 0;

	Group currentNode = headerFile.openGroup("/");
	currentNode.openAttribute("OBSERVATION_FREQUENCY_MAX").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setMaxFreq(valueDouble);
	currentNode.openAttribute("OBSERVATION_FREQUENCY_MIN").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setMinFreq(valueDouble);
	currentNode = currentNode.openGroup(currentNode.getObjnameByIdx(0));
	currentNode.openAttribute("TOTAL_INTEGRATION_TIME").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	totalIntegrationTime = valueDouble;
	currentNode.openAttribute("NOF_BEAMS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrBeams(valueUInt);
	currentNode = currentNode.openGroup(currentNode.getObjnameByIdx(0));
	currentNode.openAttribute("NOF_SAMPLES").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	totalSamples = valueUInt;
	currentNode.openAttribute("NOF_STATIONS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrStations(valueUInt);
	currentNode.openAttribute("CHANNELS_PER_SUBBAND").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrChannels(valueUInt);
	currentNode.openAttribute("CHANNEL_WIDTH").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setChannelBandwidth(valueDouble / 1000000);
	DataSet currentData = currentNode.openDataSet("STOKES_0");
	currentData.openAttribute("NOF_SUBBANDS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	nrSubbands = valueUInt;
	headerFile.close();

	observation.setNrSamplesPerSecond(static_cast< unsigned int >(totalSamples / totalIntegrationTime));
	observation.setSamplingRate(1.0f / observation.getNrSamplesPerSecond());
	observation.setNrSeconds(static_cast< unsigned int >(totalIntegrationTime));
	*paddedSecond = observation.getNrSamplesPerSecond() + (observation.getNrSamplesPerSecond() % 4);

	// Read the raw file with the actual data
	ifstream rawFile;
	rawFile.open(rawFilename.c_str(), ios::binary);
	rawFile.sync_with_stdio(false);

	data.resize(observation.getNrSeconds());

	unsigned int aOld = 0;
	unsigned int vOld = 0;

	for ( unsigned int second = 0; second < observation.getNrSeconds(); second++ ) {
		data.at(second) = new GPUData< T >("second" + toStringValue< unsigned int >(second), true, true);
		(data.at(second))->allocateHostData(nrSubbands * observation.getNrChannels() * (*paddedSecond));
		for ( unsigned int sample = 0; sample < observation.getNrSamplesPerSecond(); sample++ ) {
			for ( unsigned int subband = 0; subband < nrSubbands; subband++ ) {
				for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
					long long unsigned int element = (((subband * observation.getNrChannels()) + channel) * observation.getNrSamplesPerSecond()) + sample;
					T value = 0;

					rawFile.read(word, 4);
					changeEndianness(word);
					value = *(reinterpret_cast< T * >(word));

					((data.at(second))->getHostData())[(((subband * observation.getNrChannels()) + channel) * (*paddedSecond)) + sample] = value;
					if ( value <  observation.getMinValue() ) {
						observation.setMinValue(value);
					}
					if ( value > observation.getMaxValue() ) {
						observation.setMaxValue(value);
					}

					if ( element == 0 ) {
						firstSample = false;
						observation.setAverage(value);
						observation.setVariance(0.0f);
					}
					else {
						aOld = observation.getAverage();
						vOld = observation.getVariance();

						observation.setAverage(aOld + ((value - aOld) / (element + 1)));
						observation.setVariance(vOld + ((value - aOld) * (value - observation.getAverage())));
					}
				}
			}
		}
	}
	rawFile.close();

	observation.setNrChannels(observation.getNrChannels() * nrSubbands);
	observation.setVariance(observation.getVariance() / (observation.getNrChannels() * observation.getNrSamplesPerSecond()));
	observation.setStdDev(sqrt(observation.getVariance()));

	delete [] word;
}

} // AstroData

#endif // READ_DATA_HPP

<commit_msg>Fixing a typo in readLOFAR.<commit_after>/*
 * Copyright (C) 2012
 * Alessio Sclocco <a.sclocco@vu.nl>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include <fstream>
#include <vector>
#include <string>
#include <cmath>
#include <H5Cpp.h>

using std::ios;
using std::string;
using std::ifstream;
using std::vector;
using std::sqrt;
using H5::H5File;
using H5::Group;
using H5::DataSet;
using H5::FloatType;
using H5::IntType;

#include <Observation.hpp>
#include <GPUData.hpp>
#include <utils.hpp>

using isa::OpenCL::GPUData;
using isa::utils::toStringValue;
using isa::utils::changeEndianness;

#ifndef READ_DATA_HPP
#define READ_DATA_HPP

namespace AstroData {

template< typename T > void readSIGPROC(Observation< T > &observation, unsigned int bytestoSkip, unsigned int *paddedSecond, ifstream *inputFile, vector< GPUData< T > * > &data);
template< typename T > void readLOFAR(string headerFilename, string rawFilename, Observation< T > &observation, unsigned int *paddedSecond, vector< GPUData< T > * > &data);

// Implementation

template< typename T > void readSIGPROC(Observation< T > &observation, unsigned int bytesToSkip, unsigned int *paddedSecond, string inputFilename, vector< GPUData< T > * > &data) {
	ifstream inputFile;
	const unsigned int BUFFER_DIM = 4;
	*paddedSecond = observation.getNrSamplesPerSecond() + (observation.getNrSamplesPerSecond() % 4);
	char *buffer = new char [BUFFER_DIM];

	inputFile.open(inputFilename.c_str());
	inputFile.sync_with_stdio(false);
	inputFile.ignore(bytesToSkip);
	for ( unsigned int second = 0; second < observation.getNrSeconds(); second++ ) {
		data.at(second) = new GPUData< T >("second" + toStringValue< unsigned int >(second), true, true);
		(data.at(second))->allocateHostData((*paddedSecond) * observation.getNrChannels());

		for ( unsigned int sample = 0; sample < observation.getNrSamplesPerSecond(); sample++ ) {
			for ( unsigned int channel = observation.getNrChannels(); channel > 0; channel-- ) {
				inputFile.read(buffer, BUFFER_DIM);
				((data.at(second))->getHostData())[((channel - 1) * (*paddedSecond)) + sample] = *(reinterpret_cast< T * >(buffer));
			}
		}
	}
	inputFile.close();

	delete [] buffer;
}

template< typename T > void readLOFAR(string headerFilename, string rawFilename, Observation< T > &observation, unsigned int *paddedSecond, vector< GPUData< T > * > &data) {
	unsigned int totalSamples = 0;
	unsigned int nrSubbands = 0;
	char *word = new char[4];
	double totalIntegrationTime = 0.0;

	// Read the HDF5 file with the metadata
	H5File headerFile = H5File(headerFilename, H5F_ACC_RDONLY);
	FloatType typeDouble = FloatType(H5::PredType::NATIVE_DOUBLE);
	double valueDouble = 0.0;
	IntType typeUInt = IntType(H5::PredType::NATIVE_UINT);
	unsigned int valueUInt = 0;

	Group currentNode = headerFile.openGroup("/");
	currentNode.openAttribute("OBSERVATION_FREQUENCY_MAX").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setMaxFreq(valueDouble);
	currentNode.openAttribute("OBSERVATION_FREQUENCY_MIN").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setMinFreq(valueDouble);
	currentNode = currentNode.openGroup(currentNode.getObjnameByIdx(0));
	currentNode.openAttribute("TOTAL_INTEGRATION_TIME").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	totalIntegrationTime = valueDouble;
	currentNode.openAttribute("NOF_BEAMS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrBeams(valueUInt);
	currentNode = currentNode.openGroup(currentNode.getObjnameByIdx(0));
	currentNode.openAttribute("NOF_SAMPLES").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	totalSamples = valueUInt;
	currentNode.openAttribute("NOF_STATIONS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrStations(valueUInt);
	currentNode.openAttribute("CHANNELS_PER_SUBBAND").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	observation.setNrChannels(valueUInt);
	currentNode.openAttribute("CHANNEL_WIDTH").read(typeDouble, reinterpret_cast< void * >(&valueDouble));
	observation.setChannelBandwidth(valueDouble / 1000000);
	DataSet currentData = currentNode.openDataSet("STOKES_0");
	currentData.openAttribute("NOF_SUBBANDS").read(typeUInt, reinterpret_cast< void * >(&valueUInt));
	nrSubbands = valueUInt;
	headerFile.close();

	observation.setNrSamplesPerSecond(static_cast< unsigned int >(totalSamples / totalIntegrationTime));
	observation.setSamplingRate(1.0f / observation.getNrSamplesPerSecond());
	observation.setNrSeconds(static_cast< unsigned int >(totalIntegrationTime));
	*paddedSecond = observation.getNrSamplesPerSecond() + (observation.getNrSamplesPerSecond() % 4);

	// Read the raw file with the actual data
	ifstream rawFile;
	rawFile.open(rawFilename.c_str(), ios::binary);
	rawFile.sync_with_stdio(false);

	data.resize(observation.getNrSeconds());

	unsigned int aOld = 0;
	unsigned int vOld = 0;

	for ( unsigned int second = 0; second < observation.getNrSeconds(); second++ ) {
		data.at(second) = new GPUData< T >("second" + toStringValue< unsigned int >(second), true, true);
		(data.at(second))->allocateHostData(nrSubbands * observation.getNrChannels() * (*paddedSecond));
		for ( unsigned int sample = 0; sample < observation.getNrSamplesPerSecond(); sample++ ) {
			for ( unsigned int subband = 0; subband < nrSubbands; subband++ ) {
				for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
					long long unsigned int element = (((subband * observation.getNrChannels()) + channel) * observation.getNrSamplesPerSecond()) + sample;
					T value = 0;

					rawFile.read(word, 4);
					changeEndianness(word);
					value = *(reinterpret_cast< T * >(word));

					((data.at(second))->getHostData())[(((subband * observation.getNrChannels()) + channel) * (*paddedSecond)) + sample] = value;
					if ( value <  observation.getMinValue() ) {
						observation.setMinValue(value);
					}
					if ( value > observation.getMaxValue() ) {
						observation.setMaxValue(value);
					}

					if ( element == 0 ) {
						observation.setAverage(value);
						observation.setVariance(0.0f);
					}
					else {
						aOld = observation.getAverage();
						vOld = observation.getVariance();

						observation.setAverage(aOld + ((value - aOld) / (element + 1)));
						observation.setVariance(vOld + ((value - aOld) * (value - observation.getAverage())));
					}
				}
			}
		}
	}
	rawFile.close();

	observation.setNrChannels(observation.getNrChannels() * nrSubbands);
	observation.setVariance(observation.getVariance() / (observation.getNrChannels() * observation.getNrSamplesPerSecond()));
	observation.setStdDev(sqrt(observation.getVariance()));

	delete [] word;
}

} // AstroData

#endif // READ_DATA_HPP

<|endoftext|>


<commit_before>#include <atomic>
#include "percpu.hh"
#include "ref.hh"
// XXX If we move the filetable implementation to a source file, we
// won't need file.hh
#include "file.hh"

class filetable : public referenced {
private:
  static const int cpushift = 16;
  static const int fdmask = (1 << cpushift) - 1;

public:
  static sref<filetable> alloc() {
    return sref<filetable>::transfer(new filetable());
  }

  sref<filetable> copy(bool close_cloexec = false) {
    filetable* t = new filetable(false);

    fdinfo init(nullptr, false);
    for(int cpu = 0; cpu < NCPU; cpu++) {
      for(int fd = 0; fd < NOFILE; fd++) {
        // XXX Relaxed load?
        fdinfo info;
        // Avoid reading info_ altogether if we're closing cloexec FDs
        // and this is a cloexec FD.
        if (close_cloexec && cloexec_[cpu][fd])
          info = init;
        else
          info = info_[cpu][fd].load();
        file *f = info.get_file();
        if (f && (!close_cloexec || !info.get_cloexec())) {
          // XXX f's refcount could have dropped to zero between the
          // load and here
          file* newf = f->dup();
          fdinfo newinfo(newf, info.get_cloexec());

          t->info_[cpu][fd].store(newinfo, std::memory_order_relaxed);
          t->cloexec_[cpu][fd].store(
            info.get_cloexec(), std::memory_order_relaxed);
        } else {
          t->info_[cpu][fd].store(init, std::memory_order_relaxed);
          t->cloexec_[cpu][fd].store(true, std::memory_order_relaxed);
        }
      }
    }
    std::atomic_thread_fence(std::memory_order_release);
    return sref<filetable>::transfer(t);
  }

  // Return the file referenced by FD fd.  If fd is not open, returns
  // sref<file>().
  sref<file> getfile(int fd) {
    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU)
      return sref<file>();

    if (fd < 0 || fd >= NOFILE)
      return sref<file>();

    // XXX This isn't safe: there could be a concurrent close that
    // drops the reference count to zero.
    file* f = info_[cpu][fd].load().get_file();
    return sref<file>::newref(f);
  }

  // Allocate a FD and point it to f.  This takes over the reference
  // to f from the caller.
  int allocfd(sref<file>&& f, bool percpu = false, bool cloexec = false) {
    int cpu = percpu ? myid() : 0;
    fdinfo none(nullptr, false);
    // Transfer f to manual reference counting since we can't store
    // sref's in the info table.
    file *fptr = f->dup();
    fdinfo newinfo(fptr, cloexec, true);
    for (int fd = 0; fd < NOFILE; fd++) {
      // Note that we skip over locked FDs because that means they're
      // either non-null or about to be.
      if (info_[cpu][fd].load(std::memory_order_relaxed) == none &&
          cmpxch(&info_[cpu][fd], none, newinfo)) {
        // The default state of cloexec_ is 'true', so we only need to
        // write to it if this is a keep-exec FD.
        if (!cloexec)
          cloexec_[cpu][fd] = cloexec;
        // Unlock FD
        info_[cpu][fd].store(newinfo.with_locked(false),
                             std::memory_order_release);
        return (cpu << cpushift) | fd;
      }
    }
    cprintf("filetable::allocfd: failed\n");
    // The "dup" call told f that we're binding it to a FD.  That
    // ultimately failed, but we have to tell it that we're "closing"
    // the FD now.
    fptr->pre_close();
    fptr->dec();
    return -1;
  }

  void close(int fd) {
    // XXX(sbw) if f->ref_ > 1 the kernel will not actually close
    // the file when this function returns (i.e. sys_close can return
    // while the file/pipe/socket is still open).
    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU) {
      cprintf("filetable::close: bad fd cpu %u\n", cpu);
      return;
    }

    if (fd < 0 || fd >= NOFILE) {
      cprintf("filetable::close: bad fd %u\n", fd);
      return;
    }

    // Lock the FD to prevent concurrent modifications
    std::atomic<fdinfo> *infop = &info_[cpu][fd];
    fdinfo info = lock_fdinfo(infop);

    // Clear cloexec_ back to default state of 'true'
    if (!cloexec_[cpu][fd])
      cloexec_[cpu][fd] = true;

    // Update and unlock the FD
    fdinfo newinfo(nullptr, false);
    infop->store(newinfo, std::memory_order_release);

    // Close old file
    if (info.get_file()) {
      info.get_file()->pre_close();
      info.get_file()->dec();
    } else {
      cprintf("filetable::close: bad fd %u\n", fd);
    }
  }

  bool replace(int fd, sref<file>&& newf, bool cloexec = false) {
    assert(newf);

    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU) {
      cprintf("filetable::replace: bad fd cpu %u\n", cpu);
      return false;
    }

    if (fd < 0 || fd >= NOFILE) {
      cprintf("filetable::replace: bad fd %u\n", fd);
      return false;
    }

    // Lock the FD to prevent concurrent modifications
    std::atomic<fdinfo> *infop = &info_[cpu][fd];
    fdinfo oldinfo = lock_fdinfo(infop);

    // Update to new info and unlock.  It's safe to update cloexec_
    // non-atomically with info even with concurrent lock-free readers
    // because any that care will double-check the fdinfo bit.
    file *newfptr = newf->dup();
    fdinfo newinfo(newfptr, cloexec);
    if (cloexec != cloexec_[cpu][fd])
      cloexec_[cpu][fd] = cloexec;
    infop->store(newinfo, std::memory_order_release);

    // Close the old FD
    if (oldinfo.get_file() && oldinfo.get_file() != newfptr) {
      oldinfo.get_file()->pre_close();
      oldinfo.get_file()->dec();
    }
    return true;
  }

private:
  filetable(bool clear = true) {
    if (!clear)
      return;
    fdinfo none(nullptr, false);
    for(int cpu = 0; cpu < NCPU; cpu++) {
      for(int fd = 0; fd < NOFILE; fd++) {
        info_[cpu][fd].store(none, std::memory_order_relaxed);
        cloexec_[cpu][fd].store(true, std::memory_order_relaxed);
      }
    }
    std::atomic_thread_fence(std::memory_order_release);
  }

  ~filetable() {
    // Close all FDs
    for(int cpu = 0; cpu < NCPU; cpu++){
      for(int fd = 0; fd < NOFILE; fd++){
        fdinfo info = info_[cpu][fd].load();
        if (info.get_file()) {
          info.get_file()->pre_close();
          info.get_file()->dec();
        }
      }
    }
  }

  filetable& operator=(const filetable&);
  filetable(const filetable& x);
  NEW_DELETE_OPS(filetable);

  class fdinfo
  {
    uintptr_t data_;

    constexpr fdinfo(uintptr_t data) : data_(data) { }

  public:
    fdinfo() = default;

    fdinfo(file* fp, bool cloexec, bool locked = false)
      : data_((uintptr_t)fp | (uintptr_t)cloexec | ((uintptr_t)locked << 1)) { }

    file* get_file() const
    {
      return (file*)(data_ & ~3);
    }

    bool get_cloexec() const
    {
      return data_ & 1;
    }

    bool get_locked() const
    {
      return data_ & 2;
    }

    fdinfo with_locked(bool locked)
    {
      return fdinfo((data_ & ~2) | ((uintptr_t)locked << 1));
    }

    bool operator==(const fdinfo &o) const
    {
      return data_ == o.data_;
    }

    bool operator!=(const fdinfo &o) const
    {
      return data_ != o.data_;
    }
  };

  fdinfo lock_fdinfo(std::atomic<fdinfo> *infop)
  {
    fdinfo info, newinfo;
    while (true) {
      info = infop->load(std::memory_order_relaxed);
    retry:
      if (info.get_locked())
        nop_pause();
      else
        break;
    }
    if (!infop->compare_exchange_weak(info, info.with_locked(true)))
      goto retry;
    return info;
  }

  percpu<std::atomic<fdinfo>[NOFILE]> info_;
  // In addition to storing O_CLOEXEC with each fdinfo so it can be
  // read atomically with the FD, we store it separately so we can
  // scan for keep-exec FDs without reading from info_, which would
  // cause unnecessary sharing between the scan and creating O_CLOEXEC
  // FDs.  To avoid unnecessary sharing on this array itself, the
  // *default* state of this array for closed FDs must be 'true', so
  // we only have to write to it when opening a keep-exec FD.
  // Modifications to this array are protected by the fdinfo lock.
  // Lock-free readers should double-check the O_CLOEXEC bit in
  // fdinfo.
  percpu<std::atomic<bool>[NOFILE]> cloexec_;
};
<commit_msg>filetable: Disable copy/move<commit_after>#include <atomic>
#include "percpu.hh"
#include "ref.hh"
// XXX If we move the filetable implementation to a source file, we
// won't need file.hh
#include "file.hh"

class filetable : public referenced {
private:
  static const int cpushift = 16;
  static const int fdmask = (1 << cpushift) - 1;

public:
  static sref<filetable> alloc() {
    return sref<filetable>::transfer(new filetable());
  }

  sref<filetable> copy(bool close_cloexec = false) {
    filetable* t = new filetable(false);

    fdinfo init(nullptr, false);
    for(int cpu = 0; cpu < NCPU; cpu++) {
      for(int fd = 0; fd < NOFILE; fd++) {
        // XXX Relaxed load?
        fdinfo info;
        // Avoid reading info_ altogether if we're closing cloexec FDs
        // and this is a cloexec FD.
        if (close_cloexec && cloexec_[cpu][fd])
          info = init;
        else
          info = info_[cpu][fd].load();
        file *f = info.get_file();
        if (f && (!close_cloexec || !info.get_cloexec())) {
          // XXX f's refcount could have dropped to zero between the
          // load and here
          file* newf = f->dup();
          fdinfo newinfo(newf, info.get_cloexec());

          t->info_[cpu][fd].store(newinfo, std::memory_order_relaxed);
          t->cloexec_[cpu][fd].store(
            info.get_cloexec(), std::memory_order_relaxed);
        } else {
          t->info_[cpu][fd].store(init, std::memory_order_relaxed);
          t->cloexec_[cpu][fd].store(true, std::memory_order_relaxed);
        }
      }
    }
    std::atomic_thread_fence(std::memory_order_release);
    return sref<filetable>::transfer(t);
  }

  // Return the file referenced by FD fd.  If fd is not open, returns
  // sref<file>().
  sref<file> getfile(int fd) {
    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU)
      return sref<file>();

    if (fd < 0 || fd >= NOFILE)
      return sref<file>();

    // XXX This isn't safe: there could be a concurrent close that
    // drops the reference count to zero.
    file* f = info_[cpu][fd].load().get_file();
    return sref<file>::newref(f);
  }

  // Allocate a FD and point it to f.  This takes over the reference
  // to f from the caller.
  int allocfd(sref<file>&& f, bool percpu = false, bool cloexec = false) {
    int cpu = percpu ? myid() : 0;
    fdinfo none(nullptr, false);
    // Transfer f to manual reference counting since we can't store
    // sref's in the info table.
    file *fptr = f->dup();
    fdinfo newinfo(fptr, cloexec, true);
    for (int fd = 0; fd < NOFILE; fd++) {
      // Note that we skip over locked FDs because that means they're
      // either non-null or about to be.
      if (info_[cpu][fd].load(std::memory_order_relaxed) == none &&
          cmpxch(&info_[cpu][fd], none, newinfo)) {
        // The default state of cloexec_ is 'true', so we only need to
        // write to it if this is a keep-exec FD.
        if (!cloexec)
          cloexec_[cpu][fd] = cloexec;
        // Unlock FD
        info_[cpu][fd].store(newinfo.with_locked(false),
                             std::memory_order_release);
        return (cpu << cpushift) | fd;
      }
    }
    cprintf("filetable::allocfd: failed\n");
    // The "dup" call told f that we're binding it to a FD.  That
    // ultimately failed, but we have to tell it that we're "closing"
    // the FD now.
    fptr->pre_close();
    fptr->dec();
    return -1;
  }

  void close(int fd) {
    // XXX(sbw) if f->ref_ > 1 the kernel will not actually close
    // the file when this function returns (i.e. sys_close can return
    // while the file/pipe/socket is still open).
    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU) {
      cprintf("filetable::close: bad fd cpu %u\n", cpu);
      return;
    }

    if (fd < 0 || fd >= NOFILE) {
      cprintf("filetable::close: bad fd %u\n", fd);
      return;
    }

    // Lock the FD to prevent concurrent modifications
    std::atomic<fdinfo> *infop = &info_[cpu][fd];
    fdinfo info = lock_fdinfo(infop);

    // Clear cloexec_ back to default state of 'true'
    if (!cloexec_[cpu][fd])
      cloexec_[cpu][fd] = true;

    // Update and unlock the FD
    fdinfo newinfo(nullptr, false);
    infop->store(newinfo, std::memory_order_release);

    // Close old file
    if (info.get_file()) {
      info.get_file()->pre_close();
      info.get_file()->dec();
    } else {
      cprintf("filetable::close: bad fd %u\n", fd);
    }
  }

  bool replace(int fd, sref<file>&& newf, bool cloexec = false) {
    assert(newf);

    int cpu = fd >> cpushift;
    fd = fd & fdmask;

    if (cpu < 0 || cpu >= NCPU) {
      cprintf("filetable::replace: bad fd cpu %u\n", cpu);
      return false;
    }

    if (fd < 0 || fd >= NOFILE) {
      cprintf("filetable::replace: bad fd %u\n", fd);
      return false;
    }

    // Lock the FD to prevent concurrent modifications
    std::atomic<fdinfo> *infop = &info_[cpu][fd];
    fdinfo oldinfo = lock_fdinfo(infop);

    // Update to new info and unlock.  It's safe to update cloexec_
    // non-atomically with info even with concurrent lock-free readers
    // because any that care will double-check the fdinfo bit.
    file *newfptr = newf->dup();
    fdinfo newinfo(newfptr, cloexec);
    if (cloexec != cloexec_[cpu][fd])
      cloexec_[cpu][fd] = cloexec;
    infop->store(newinfo, std::memory_order_release);

    // Close the old FD
    if (oldinfo.get_file() && oldinfo.get_file() != newfptr) {
      oldinfo.get_file()->pre_close();
      oldinfo.get_file()->dec();
    }
    return true;
  }

private:
  filetable(bool clear = true) {
    if (!clear)
      return;
    fdinfo none(nullptr, false);
    for(int cpu = 0; cpu < NCPU; cpu++) {
      for(int fd = 0; fd < NOFILE; fd++) {
        info_[cpu][fd].store(none, std::memory_order_relaxed);
        cloexec_[cpu][fd].store(true, std::memory_order_relaxed);
      }
    }
    std::atomic_thread_fence(std::memory_order_release);
  }

  ~filetable() {
    // Close all FDs
    for(int cpu = 0; cpu < NCPU; cpu++){
      for(int fd = 0; fd < NOFILE; fd++){
        fdinfo info = info_[cpu][fd].load();
        if (info.get_file()) {
          info.get_file()->pre_close();
          info.get_file()->dec();
        }
      }
    }
  }

  filetable& operator=(const filetable&) = delete;
  filetable(const filetable& x) = delete;
  filetable& operator=(filetable &&) = delete;
  filetable(filetable &&) = delete;
  NEW_DELETE_OPS(filetable);

  class fdinfo
  {
    uintptr_t data_;

    constexpr fdinfo(uintptr_t data) : data_(data) { }

  public:
    fdinfo() = default;

    fdinfo(file* fp, bool cloexec, bool locked = false)
      : data_((uintptr_t)fp | (uintptr_t)cloexec | ((uintptr_t)locked << 1)) { }

    file* get_file() const
    {
      return (file*)(data_ & ~3);
    }

    bool get_cloexec() const
    {
      return data_ & 1;
    }

    bool get_locked() const
    {
      return data_ & 2;
    }

    fdinfo with_locked(bool locked)
    {
      return fdinfo((data_ & ~2) | ((uintptr_t)locked << 1));
    }

    bool operator==(const fdinfo &o) const
    {
      return data_ == o.data_;
    }

    bool operator!=(const fdinfo &o) const
    {
      return data_ != o.data_;
    }
  };

  fdinfo lock_fdinfo(std::atomic<fdinfo> *infop)
  {
    fdinfo info, newinfo;
    while (true) {
      info = infop->load(std::memory_order_relaxed);
    retry:
      if (info.get_locked())
        nop_pause();
      else
        break;
    }
    if (!infop->compare_exchange_weak(info, info.with_locked(true)))
      goto retry;
    return info;
  }

  percpu<std::atomic<fdinfo>[NOFILE]> info_;
  // In addition to storing O_CLOEXEC with each fdinfo so it can be
  // read atomically with the FD, we store it separately so we can
  // scan for keep-exec FDs without reading from info_, which would
  // cause unnecessary sharing between the scan and creating O_CLOEXEC
  // FDs.  To avoid unnecessary sharing on this array itself, the
  // *default* state of this array for closed FDs must be 'true', so
  // we only have to write to it when opening a keep-exec FD.
  // Modifications to this array are protected by the fdinfo lock.
  // Lock-free readers should double-check the O_CLOEXEC bit in
  // fdinfo.
  percpu<std::atomic<bool>[NOFILE]> cloexec_;
};
<|endoftext|>


<commit_before>#include <cstdlib>
#include <omp.h>
#include <iterator>
#include <vector>
#include <algorithm>
// #include <seqUtils.h>

template <class T,class StrictWeakOrdering>
void omp_par::merge(T A_,T A_last,T B_,T B_last,T C_,int p,StrictWeakOrdering comp){
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;

  _DiffType N1=A_last-A_;
  _DiffType N2=B_last-B_;
  if(N1==0 && N2==0) return;
  if(N1==0 || N2==0){
    _ValType* A=(N1==0? &B_[0]: &A_[0]);
    _DiffType N=(N1==0?  N2  :  N1   );
    #pragma omp parallel for
    for(int i=0;i<p;i++){
      _DiffType indx1=( i   *N)/p;
      _DiffType indx2=((i+1)*N)/p;
      memcpy(&C_[indx1], &A[indx1], (indx2-indx1)*sizeof(_ValType));
    }
    return;
  }

  //Split both arrays ( A and B ) into n equal parts.
  //Find the position of each split in the final merged array.
  int n=10;
  _ValType* split=new _ValType[p*n*2];
  _DiffType* split_size=new _DiffType[p*n*2];
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    for(int j=0;j<n;j++){
      int indx=i*n+j;
      _DiffType indx1=(indx*N1)/(p*n);
      split   [indx]=A_[indx1];
      split_size[indx]=indx1+(std::lower_bound(B_,B_last,split[indx],comp)-B_);

      indx1=(indx*N2)/(p*n);
      indx+=p*n;
      split   [indx]=B_[indx1];
      split_size[indx]=indx1+(std::lower_bound(A_,A_last,split[indx],comp)-A_);
    }
  }

  //Find the closest split position for each thread that will
  //divide the final array equally between the threads.
  _DiffType* split_indx_A=new _DiffType[p+1];
  _DiffType* split_indx_B=new _DiffType[p+1];
  split_indx_A[0]=0;
  split_indx_B[0]=0;
  split_indx_A[p]=N1;
  split_indx_B[p]=N2;
  #pragma omp parallel for
  for(int i=1;i<p;i++){
    _DiffType req_size=(i*(N1+N2))/p;

    int j=std::lower_bound(&split_size[0],&split_size[p*n],req_size,std::less<_DiffType>())-&split_size[0];
    if(j>=p*n)
      j=p*n-1;
    _ValType  split1     =split     [j];
    _DiffType split_size1=split_size[j];

    j=(std::lower_bound(&split_size[p*n],&split_size[p*n*2],req_size,std::less<_DiffType>())-&split_size[p*n])+p*n;
    if(j>=2*p*n)
      j=2*p*n-1;
    if(abs(split_size[j]-req_size)<abs(split_size1-req_size)){
      split1     =split   [j];
      split_size1=split_size[j];
    }

    split_indx_A[i]=std::lower_bound(A_,A_last,split1,comp)-A_;
    split_indx_B[i]=std::lower_bound(B_,B_last,split1,comp)-B_;
  }
  delete[] split;
  delete[] split_size;

  //Merge for each thread independently.
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    T C=C_+split_indx_A[i]+split_indx_B[i];
    //std::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C,comp);
    //sse<_ValType>::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C);
    std::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C);
  }
  delete[] split_indx_A;
  delete[] split_indx_B;
}

template <class T,class StrictWeakOrdering>
void omp_par::merge_sort(T A,T A_last,StrictWeakOrdering comp){
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;

  int p=omp_get_max_threads();
  _DiffType N=A_last-A;
  if(N<2*p || p==1){
    std::sort(A,A_last,comp);
    return;
  }

  //Split the array A into p equal parts.
  _DiffType* split=new _DiffType[p+1];
  split[p]=N;
  #pragma omp parallel for
  for(int id=0;id<p;id++){
    split[id]=(id*N)/p;
  }

  //Sort each part independently.
  #pragma omp parallel for
  for(int id=0;id<p;id++){
    std::sort(A+split[id],A+split[id+1],comp);
  }

  //Merge two parts at a time.
  _ValType* B=new _ValType[N];
  _ValType* A_=&A[0];
  _ValType* B_=&B[0];
  for(int j=1;j<p;j=j*2){
    for(int i=0;i<p;i=i+2*j){
      if(i+j<p){
	omp_par::merge(A_+split[i],A_+split[i+j],A_+split[i+j],A_+split[(i+2*j<=p?i+2*j:p)],B_+split[i],p,comp);
      }else{
	#pragma omp parallel for
	for(int k=split[i];k<split[p];k++)
	  B_[k]=A_[k];
      }
    }
    _ValType* tmp_swap=A_;
    A_=B_;
    B_=tmp_swap;
  }

  //The final result should be in A.
  if(A_!=&A[0]){
    #pragma omp parallel for
    for(int i=0;i<N;i++)
      A[i]=A_[i];
  }

  //Free memory.
  delete[] split;
  delete[] B;
}

template <class T>
void omp_par::merge_sort(T A,T A_last){
  typedef typename std::iterator_traits<T>::value_type _ValType;
  if(sizeof(_ValType)<=8*sizeof(_ValType*))
    omp_par::merge_sort(A,A_last,std::less<_ValType>());
  else
    omp_par::merge_sort_ptrs(A,A_last);
}

template <class T>
struct DataPtr{
public:
  T* elem;

  inline bool  operator < ( DataPtr<T> const  &other) const {
    return ((*elem)<(*(other.elem)));
  }
};

template <class T>
void omp_par::merge_sort_ptrs(T A,T A_last){
  //std::cout<<"Using Pointer sort.\n";
  int p=omp_get_max_threads();
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;
  _DiffType N=A_last-A;

  // Make copy and init pointer array to be sorted.
  DataPtr<_ValType>* B=new DataPtr<_ValType>[N];
  _ValType* C=new _ValType[N];
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    _DiffType start=(N*i)/p;
    _DiffType end=(N*(i+1))/p;
    memcpy(&C[start], &A[start], (end-start)*sizeof(_ValType));
    for(_DiffType j=start;j<end;j++) B[j].elem=&C[0]+j;
  }

  // Sort pointers.
  omp_par::merge_sort(B,B+N,std::less<DataPtr<_ValType> >());

  // Copy data to its sorted position.
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    _DiffType start=(N*i)/p;
    _DiffType end=(N*(i+1))/p;
    for(size_t j=start;j<end;j++) A[j]=*(B[j].elem);
  }

  delete[] B;
  delete[] C;
}

template <class T, class I>
T omp_par::reduce(T* A, I cnt){
  T sum=0;
  #pragma omp parallel for reduction(+:sum)
  for(I i = 0; i < cnt; i++)
    sum+=A[i];
  return sum;
}

template <class T, class I>
void omp_par::scan(T* A, T* B,I cnt){
  int p=omp_get_max_threads();
  if(cnt<100*p){
    for(I i=1;i<cnt;i++)
      B[i]=B[i-1]+A[i-1];
    return;
  }

  I step_size=cnt/p;

  #pragma omp parallel for
  for(int i=0; i<p; i++){
    int start=i*step_size;
    int end=start+step_size;
    if(i==p-1) end=cnt;
    if(i!=0)B[start]=0;
    for(I j=start+1; j<end; j++)
      B[j]=B[j-1]+A[j-1];
  }

  T* sum=new T[p];
  sum[0]=0;
  for(int i=1;i<p;i++)
    sum[i]=sum[i-1]+B[i*step_size-1]+A[i*step_size-1];

  #pragma omp parallel for
  for(int i=1; i<p; i++){
    int start=i*step_size;
    int end=start+step_size;
    if(i==p-1) end=cnt;
    T sum_=sum[i];
    for(I j=start; j<end; j++)
      B[j]+=sum_;
  }
  delete[] sum;
}

<commit_msg>usort can be compiled using gcc<commit_after>#include <cstdlib>
#include <omp.h>
#include <iterator>
#include <vector>
#include <algorithm>
#include <cstring>
// #include <seqUtils.h>

template <class T,class StrictWeakOrdering>
void omp_par::merge(T A_,T A_last,T B_,T B_last,T C_,int p,StrictWeakOrdering comp){
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;

  _DiffType N1=A_last-A_;
  _DiffType N2=B_last-B_;
  if(N1==0 && N2==0) return;
  if(N1==0 || N2==0){
    _ValType* A=(N1==0? &B_[0]: &A_[0]);
    _DiffType N=(N1==0?  N2  :  N1   );
    #pragma omp parallel for
    for(int i=0;i<p;i++){
      _DiffType indx1=( i   *N)/p;
      _DiffType indx2=((i+1)*N)/p;
      memcpy(&C_[indx1], &A[indx1], (indx2-indx1)*sizeof(_ValType));
    }
    return;
  }

  //Split both arrays ( A and B ) into n equal parts.
  //Find the position of each split in the final merged array.
  int n=10;
  _ValType* split=new _ValType[p*n*2];
  _DiffType* split_size=new _DiffType[p*n*2];
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    for(int j=0;j<n;j++){
      int indx=i*n+j;
      _DiffType indx1=(indx*N1)/(p*n);
      split   [indx]=A_[indx1];
      split_size[indx]=indx1+(std::lower_bound(B_,B_last,split[indx],comp)-B_);

      indx1=(indx*N2)/(p*n);
      indx+=p*n;
      split   [indx]=B_[indx1];
      split_size[indx]=indx1+(std::lower_bound(A_,A_last,split[indx],comp)-A_);
    }
  }

  //Find the closest split position for each thread that will
  //divide the final array equally between the threads.
  _DiffType* split_indx_A=new _DiffType[p+1];
  _DiffType* split_indx_B=new _DiffType[p+1];
  split_indx_A[0]=0;
  split_indx_B[0]=0;
  split_indx_A[p]=N1;
  split_indx_B[p]=N2;
  #pragma omp parallel for
  for(int i=1;i<p;i++){
    _DiffType req_size=(i*(N1+N2))/p;

    int j=std::lower_bound(&split_size[0],&split_size[p*n],req_size,std::less<_DiffType>())-&split_size[0];
    if(j>=p*n)
      j=p*n-1;
    _ValType  split1     =split     [j];
    _DiffType split_size1=split_size[j];

    j=(std::lower_bound(&split_size[p*n],&split_size[p*n*2],req_size,std::less<_DiffType>())-&split_size[p*n])+p*n;
    if(j>=2*p*n)
      j=2*p*n-1;
    if(abs(split_size[j]-req_size)<abs(split_size1-req_size)){
      split1     =split   [j];
      split_size1=split_size[j];
    }

    split_indx_A[i]=std::lower_bound(A_,A_last,split1,comp)-A_;
    split_indx_B[i]=std::lower_bound(B_,B_last,split1,comp)-B_;
  }
  delete[] split;
  delete[] split_size;

  //Merge for each thread independently.
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    T C=C_+split_indx_A[i]+split_indx_B[i];
    //std::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C,comp);
    //sse<_ValType>::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C);
    std::merge(A_+split_indx_A[i],A_+split_indx_A[i+1],B_+split_indx_B[i],B_+split_indx_B[i+1],C);
  }
  delete[] split_indx_A;
  delete[] split_indx_B;
}

template <class T,class StrictWeakOrdering>
void omp_par::merge_sort(T A,T A_last,StrictWeakOrdering comp){
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;

  int p=omp_get_max_threads();
  _DiffType N=A_last-A;
  if(N<2*p || p==1){
    std::sort(A,A_last,comp);
    return;
  }

  //Split the array A into p equal parts.
  _DiffType* split=new _DiffType[p+1];
  split[p]=N;
  #pragma omp parallel for
  for(int id=0;id<p;id++){
    split[id]=(id*N)/p;
  }

  //Sort each part independently.
  #pragma omp parallel for
  for(int id=0;id<p;id++){
    std::sort(A+split[id],A+split[id+1],comp);
  }

  //Merge two parts at a time.
  _ValType* B=new _ValType[N];
  _ValType* A_=&A[0];
  _ValType* B_=&B[0];
  for(int j=1;j<p;j=j*2){
    for(int i=0;i<p;i=i+2*j){
      if(i+j<p){
	omp_par::merge(A_+split[i],A_+split[i+j],A_+split[i+j],A_+split[(i+2*j<=p?i+2*j:p)],B_+split[i],p,comp);
      }else{
	#pragma omp parallel for
	for(int k=split[i];k<split[p];k++)
	  B_[k]=A_[k];
      }
    }
    _ValType* tmp_swap=A_;
    A_=B_;
    B_=tmp_swap;
  }

  //The final result should be in A.
  if(A_!=&A[0]){
    #pragma omp parallel for
    for(int i=0;i<N;i++)
      A[i]=A_[i];
  }

  //Free memory.
  delete[] split;
  delete[] B;
}

template <class T>
void omp_par::merge_sort(T A,T A_last){
  typedef typename std::iterator_traits<T>::value_type _ValType;
  if(sizeof(_ValType)<=8*sizeof(_ValType*))
    omp_par::merge_sort(A,A_last,std::less<_ValType>());
  else
    omp_par::merge_sort_ptrs(A,A_last);
}

template <class T>
struct DataPtr{
public:
  T* elem;

  inline bool  operator < ( DataPtr<T> const  &other) const {
    return ((*elem)<(*(other.elem)));
  }
};

template <class T>
void omp_par::merge_sort_ptrs(T A,T A_last){
  //std::cout<<"Using Pointer sort.\n";
  int p=omp_get_max_threads();
  typedef typename std::iterator_traits<T>::difference_type _DiffType;
  typedef typename std::iterator_traits<T>::value_type _ValType;
  _DiffType N=A_last-A;

  // Make copy and init pointer array to be sorted.
  DataPtr<_ValType>* B=new DataPtr<_ValType>[N];
  _ValType* C=new _ValType[N];
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    _DiffType start=(N*i)/p;
    _DiffType end=(N*(i+1))/p;
    memcpy(&C[start], &A[start], (end-start)*sizeof(_ValType));
    for(_DiffType j=start;j<end;j++) B[j].elem=&C[0]+j;
  }

  // Sort pointers.
  omp_par::merge_sort(B,B+N,std::less<DataPtr<_ValType> >());

  // Copy data to its sorted position.
  #pragma omp parallel for
  for(int i=0;i<p;i++){
    _DiffType start=(N*i)/p;
    _DiffType end=(N*(i+1))/p;
    for(size_t j=start;j<end;j++) A[j]=*(B[j].elem);
  }

  delete[] B;
  delete[] C;
}

template <class T, class I>
T omp_par::reduce(T* A, I cnt){
  T sum=0;
  #pragma omp parallel for reduction(+:sum)
  for(I i = 0; i < cnt; i++)
    sum+=A[i];
  return sum;
}

template <class T, class I>
void omp_par::scan(T* A, T* B,I cnt){
  int p=omp_get_max_threads();
  if(cnt<100*p){
    for(I i=1;i<cnt;i++)
      B[i]=B[i-1]+A[i-1];
    return;
  }

  I step_size=cnt/p;

  #pragma omp parallel for
  for(int i=0; i<p; i++){
    int start=i*step_size;
    int end=start+step_size;
    if(i==p-1) end=cnt;
    if(i!=0)B[start]=0;
    for(I j=start+1; j<end; j++)
      B[j]=B[j-1]+A[j-1];
  }

  T* sum=new T[p];
  sum[0]=0;
  for(int i=1;i<p;i++)
    sum[i]=sum[i-1]+B[i*step_size-1]+A[i*step_size-1];

  #pragma omp parallel for
  for(int i=1; i<p; i++){
    int start=i*step_size;
    int end=start+step_size;
    if(i==p-1) end=cnt;
    T sum_=sum[i];
    for(I j=start; j<end; j++)
      B[j]+=sum_;
  }
  delete[] sum;
}

<|endoftext|>


<commit_before>/* pocl/include/vccompat.h - Compatibility header to provide some functions
   which are not found from VC++.

   All functions should be static inline so that they can be included in many places
   without having problem of symbol collision.

   Copyright (c) 2014 Mikael Lepistö <elhigu@gmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef VCCOMPAT_HPP
#define VCCOMPAT_HPP

#include <Windows.h>
#define __restrict__ __restrict

// ERROR is used as label for goto in some OCL API functions
#undef ERROR

// if this causes linking problems, use inline function below...
#define snprintf _snprintf
/*
static inline int snprintf(char *str, size_t size, const char *format, ...) {
   va_list args;
   va_start(args, format);
   _snprintf(str, size, format, args);
   va_end(args);
}
*/

/**
 * ltdl compatibility functions
 */
typedef HMODULE lt_dlhandle;

static inline lt_dlhandle lt_dlopen(const char* filename) {
  return (lt_dlhandle)LoadLibrary(filename);
}

static inline int lt_dlerror(void) {
   return GetLastError();
}

static inline void *lt_dlsym(lt_dlhandle handle, const char *symbol) {
  return GetProcAddress(handle, symbol);
}

static inline void lt_dlinit(void) {
   // separate init not needed in windows
}

/**
 * Memory allocation functions
 */
static int posix_memalign(void **p, size_t align, size_t size) {
   void *buf = _aligned_malloc(size, align);
   if (buf == NULL) return errno;
   *p = buf;
   return 0;
}

#endif
<commit_msg>Added strtok_r implementation for Visual Studio.<commit_after>/* pocl/include/vccompat.h - Compatibility header to provide some functions
   which are not found from VC++.

   All functions should be static inline so that they can be included in many places
   without having problem of symbol collision.

   Copyright (c) 2014 Mikael Lepistö <elhigu@gmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#ifndef VCCOMPAT_HPP
#define VCCOMPAT_HPP

#include <Windows.h>
#define __restrict__ __restrict

// ERROR is used as label for goto in some OCL API functions
#undef ERROR

// if this causes linking problems, use inline function below...
#define snprintf _snprintf
/*
static inline int snprintf(char *str, size_t size, const char *format, ...) {
   va_list args;
   va_start(args, format);
   _snprintf(str, size, format, args);
   va_end(args);
}
*/

static inline char* strtok_r(char *str, const char *delim, char **saveptr) {
   return strtok_s(str, delim, saveptr);
}

/**
 * ltdl compatibility functions
 */
typedef HMODULE lt_dlhandle;

static inline lt_dlhandle lt_dlopen(const char* filename) {
  return (lt_dlhandle)LoadLibrary(filename);
}

static inline int lt_dlerror(void) {
   return GetLastError();
}

static inline void *lt_dlsym(lt_dlhandle handle, const char *symbol) {
  return GetProcAddress(handle, symbol);
}

static inline void lt_dlinit(void) {
   // separate init not needed in windows
}

/**
 * Memory allocation functions
 */
static int posix_memalign(void **p, size_t align, size_t size) {
   void *buf = _aligned_malloc(size, align);
   if (buf == NULL) return errno;
   *p = buf;
   return 0;
}

#endif
<|endoftext|>


<commit_before>/** kmeans.cc
    Jeremy Barnes, 16 December 2014
    Copyright (c) 2014 Datacratic Inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
    Implementation of an KMEANS algorithm for embedding of a dataset.
*/

#include "mldb/jml/stats/distribution_simd.h"
#include "kmeans.h"
#include "matrix.h"
#include "mldb/server/mldb_server.h"
#include "mldb/core/dataset.h"
#include "mldb/jml/stats/distribution.h"
#include <boost/multi_array.hpp>
#include "mldb/jml/utils/guard.h"
#include "mldb/jml/utils/worker_task.h"
#include "mldb/jml/utils/pair_utils.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/simd_vector.h"
#include "mldb/jml/utils/vector_utils.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/ml/value_descriptions.h"
#include "mldb/ml/kmeans.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/plugins/sql_config_validator.h"
#include "mldb/server/analytics.h"
#include "mldb/types/any_impl.h"
#include "mldb/types/optional_description.h"

using namespace std;

namespace Datacratic {
namespace MLDB {

DEFINE_STRUCTURE_DESCRIPTION(KmeansConfig);

KmeansConfigDescription::
KmeansConfigDescription()
{
    Optional<PolyConfigT<Dataset> > optional;
    optional.emplace(PolyConfigT<Dataset>().
                     withType(KmeansConfig::defaultOutputDatasetType));

    addField("trainingData", &KmeansConfig::trainingData,
             "Specification of the data for input to the k-means procedure.  This should be "
             "organized as an embedding, with each selected row containing the same "
             "set of columns with numeric values to be used as coordinates.  The select statement "
             "does not support groupby and having clauses.");
    addField("outputDataset", &KmeansConfig::output,
             "Dataset for cluster assignment.  This dataset will contain the same "
             "row names as the input dataset, but the coordinates will be replaced "
             "by a single column giving the cluster number that the row was assigned to.",
              optional);
    addField("centroidsDataset", &KmeansConfig::centroids,
             "Dataset in which the centroids will be recorded.  This dataset will "
             "have the same coordinates (columns) as those selected from the input "
             "dataset, but will have one row per cluster, providing the centroid of "
             "the cluster.",
             PolyConfigT<Dataset>().withType("embedding"));
    addField("numInputDimensions", &KmeansConfig::numInputDimensions,
             "Number of dimensions from the input to use (-1 = all).  This limits "
             "the number of columns used.  Columns will be ordered alphabetically "
             "and the lowest ones kept.",
             -1);
    addField("numClusters", &KmeansConfig::numClusters,
             "Number of clusters to create.  This will provide the total number of "
             "centroids created.  There must be at least as many rows selected as "
             "clusters.", 10);
    addField("maxIterations", &KmeansConfig::maxIterations,
             "Maximum number of iterations to perform.  If no convergeance is "
             "reached within this number of iterations, the current clustering "
             "will be returned.", 100);
    addField("metric", &KmeansConfig::metric,
             "Metric space in which the k-means distances will be calculated. "
             "Normally this will be Cosine for an orthonormal basis, and "
             "Euclidian for another basis",
             METRIC_COSINE);
    addField("functionName", &KmeansConfig::functionName,
             "If specified, a kmeans function of this name will be created using "
             "the training result.");
    addParent<ProcedureConfig>();

    onPostValidate = validate<KmeansConfig, InputQuery, NoGroupByHaving>(&KmeansConfig::trainingData, "kmeans");
}

// TODO: see http://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf

namespace {

ML::KMeansMetric * makeMetric(MetricSpace metric)
{
    switch (metric) {
    case METRIC_EUCLIDEAN:
        return new ML::KMeansEuclideanMetric();
        break;
    case METRIC_COSINE:
        return new ML::KMeansCosineMetric();
        break;
    default:
        throw ML::Exception("Unknown kmeans metric type");
    }
}

} // file scope

/*****************************************************************************/
/* KMEANS PROCEDURE                                                           */
/*****************************************************************************/

KmeansProcedure::
KmeansProcedure(MldbServer * owner,
               PolyConfig config,
               const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    kmeansConfig = config.params.convert<KmeansConfig>();
}

Any
KmeansProcedure::
getStatus() const
{
    return Any();
}

RunOutput
KmeansProcedure::
run(const ProcedureRunConfig & run,
      const std::function<bool (const Json::Value &)> & onProgress) const
{
    auto runProcConf = applyRunConfOverProcConf(kmeansConfig, run);

    auto onProgress2 = [&] (const Json::Value & progress)
        {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };

    SqlExpressionMldbContext context(server);

    auto embeddingOutput = getEmbedding(*runProcConf.trainingData.stm,
                                        context,
                                        runProcConf.numInputDimensions,
                                        onProgress2);

    std::vector<std::tuple<RowHash, RowName, std::vector<double>,
                           std::vector<ExpressionValue> > > & rows
        = embeddingOutput.first;
    std::vector<KnownColumn> & vars = embeddingOutput.second;

    std::vector<ColumnName> columnNames;
    for (auto & v: vars) {
        columnNames.push_back(v.columnName);
    }

    std::vector<ML::distribution<float> > vecs;

    for (unsigned i = 0;  i < rows.size();  ++i) {
        vecs.emplace_back(ML::distribution<float>(std::get<2>(rows[i]).begin(),
                                                  std::get<2>(rows[i]).end()));
    }

    if (vecs.size() == 0)
        throw HttpReturnException(400, "Kmeans training requires at least 1 datapoint. "
                                  "Make sure your dataset is not empty and that your WHERE expression "
                                  "does not filter all the rows");

    ML::KMeans kmeans;
    kmeans.metric.reset(makeMetric(runProcConf.metric));

    vector<int> inCluster;

    int numClusters = runProcConf.numClusters;
    int numIterations = runProcConf.maxIterations;

    //doProgress("running k-means");

    kmeans.train(vecs, inCluster, numClusters, numIterations);

    //doProgress("finished k-means");
    if (runProcConf.output.get()) {

        PolyConfigT<Dataset> outputDataset = *runProcConf.output;
        if (outputDataset.type.empty())
            outputDataset.type = KmeansConfig::defaultOutputDatasetType;

        auto output = createDataset(server, outputDataset, onProgress2, true /*overwrite*/);

        Date applyDate = Date::now();

        for (unsigned i = 0;  i < rows.size();  ++i) {
            std::vector<std::tuple<ColumnName, CellValue, Date> > cols;
            cols.emplace_back(ColumnName("cluster"), inCluster[i], applyDate);
            output->recordRow(std::get<1>(rows[i]), cols);
        }

        output->commit();
    }

    if (runProcConf.centroids.type != "" || kmeansConfig.centroids.id != "") {

        auto centroids = createDataset(server, runProcConf.centroids, onProgress2, true /*overwrite*/);

        Date applyDate = Date::now();

        for (unsigned i = 0;  i < kmeans.clusters.size();  ++i) {
            auto & cluster = kmeans.clusters[i];

            std::vector<std::tuple<ColumnName, CellValue, Date> > cols;

            for (unsigned j = 0;  j < cluster.centroid.size();  ++j) {
                cols.emplace_back(columnNames[j], cluster.centroid[j], applyDate);
            }

            centroids->recordRow(RowName(ML::format("%i", i)), cols);
        }

        centroids->commit();
    }

    if(!runProcConf.functionName.empty()) {
        KmeansFunctionConfig funcConf;
        funcConf.metric = runProcConf.metric;
        funcConf.centroids = runProcConf.centroids;

        PolyConfig kmeansFuncPC;
        kmeansFuncPC.type = "kmeans";
        kmeansFuncPC.id = runProcConf.functionName;
        kmeansFuncPC.params = funcConf;

        obtainFunction(server, kmeansFuncPC, onProgress);
    }

    return Any();
}

DEFINE_STRUCTURE_DESCRIPTION(KmeansFunctionConfig);

KmeansFunctionConfigDescription::
KmeansFunctionConfigDescription()
{
    addField("centroids", &KmeansFunctionConfig::centroids,
             "Dataset containing centroids of each cluster: one row per cluster.");
    addField("metric", &KmeansFunctionConfig::metric,
             "Metric to use to calculate distances.  This should match the "
             "metric used in training.");
    addField("inputData", &KmeansFunctionConfig::inputData,
             "Fields to select to calculate k-means over.  Only those fields "
             "that are selected here need to be matched.  Default is to use ");
}

/*****************************************************************************/
/* KMEANS FUNCTION                                                              */
/*****************************************************************************/

KmeansFunction::
KmeansFunction(MldbServer * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner)
{
    functionConfig = config.params.convert<KmeansFunctionConfig>();

    auto dataset = obtainDataset(server, functionConfig.centroids, onProgress);

    // Load up the embeddings
    auto embeddingOutput = getEmbedding(functionConfig.inputData.stm->select,
                                        *dataset, "",
                                        functionConfig.inputData.stm->when,
                                        functionConfig.inputData.stm->where,
                                        { },
                                        -1 /* max dimensions */,
                                        ORDER_BY_NOTHING, 0, -1,
                                        onProgress);

    // Each row is a cluster
    std::vector<std::tuple<RowHash, RowName, std::vector<double>,
                           std::vector<ExpressionValue> > > & rows
        = embeddingOutput.first;
    std::vector<KnownColumn> & vars = embeddingOutput.second;

    for (auto & v: vars) {
        columnNames.push_back(v.columnName);
    }

    for (auto & r: rows) {
        Cluster cluster;
        cluster.clusterName = jsonDecodeStr<CellValue>(std::get<1>(r).toString());
        cluster.centroid.insert(cluster.centroid.end(),
                                std::get<2>(r).begin(),
                                std::get<2>(r).end());
        clusters.emplace_back(std::move(cluster));
    }

    cerr << "got " << clusters.size()
         << " clusters with " << columnNames.size()
         << "values" << endl;
}

Any
KmeansFunction::
getStatus() const
{
    return Any();
}

FunctionOutput
KmeansFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    FunctionOutput result;

    //cerr << "context = " << jsonEncode(context) << endl;

    // Extract an embedding with the given column names
    ExpressionValue storage;
    const ExpressionValue & inputVal = context.get("embedding", storage);
    //cerr << "getting embedding" << endl;
    ML::distribution<float> input = inputVal.getEmbedding(columnNames.size());
    Date ts = inputVal.getEffectiveTimestamp();

    std::unique_ptr<ML::KMeansMetric> metric(makeMetric(functionConfig.metric));

    double bestDist = INFINITY;
    CellValue bestCluster;

    for (unsigned i = 0;  i < clusters.size();  ++i) {
        double dist = metric->distance(input, clusters[i].centroid);
        if (dist < bestDist
            || (dist == bestDist && clusters[i].clusterName < bestCluster)) {
            bestDist = dist;
            bestCluster = clusters[i].clusterName;
        }
    }

    result.set("cluster", ExpressionValue(bestCluster, ts));

    return result;
}

FunctionInfo
KmeansFunction::
getFunctionInfo() const
{
    FunctionInfo result;

    result.input.addEmbeddingValue("embedding", columnNames.size());
    result.output.addAtomValue("cluster");

    return result;
}

namespace {

RegisterProcedureType<KmeansProcedure, KmeansConfig>
regKmeans(builtinPackage(),
          "kmeans.train",
          "Simple clustering algorithm based on cluster centroids in embedding space",
          "procedures/KmeansProcedure.md.html");

RegisterFunctionType<KmeansFunction, KmeansFunctionConfig>
regKmeansFunction(builtinPackage(),
                  "kmeans",
                  "Apply a k-means clustering to new data",
                  "functions/Kmeans.md.html");

} // file scope

} // namespace MLDB
} // namespace Datacratic

<commit_msg>[MLDB-1104] fixed kmeans function to have a default query.<commit_after>/** kmeans.cc
    Jeremy Barnes, 16 December 2014
    Copyright (c) 2014 Datacratic Inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
    Implementation of an KMEANS algorithm for embedding of a dataset.
*/

#include "mldb/jml/stats/distribution_simd.h"
#include "kmeans.h"
#include "matrix.h"
#include "mldb/server/mldb_server.h"
#include "mldb/core/dataset.h"
#include "mldb/jml/stats/distribution.h"
#include <boost/multi_array.hpp>
#include "mldb/jml/utils/guard.h"
#include "mldb/jml/utils/worker_task.h"
#include "mldb/jml/utils/pair_utils.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/simd_vector.h"
#include "mldb/jml/utils/vector_utils.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/ml/value_descriptions.h"
#include "mldb/ml/kmeans.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/plugins/sql_config_validator.h"
#include "mldb/server/analytics.h"
#include "mldb/types/any_impl.h"
#include "mldb/types/optional_description.h"

using namespace std;

namespace Datacratic {
namespace MLDB {

DEFINE_STRUCTURE_DESCRIPTION(KmeansConfig);

KmeansConfigDescription::
KmeansConfigDescription()
{
    Optional<PolyConfigT<Dataset> > optional;
    optional.emplace(PolyConfigT<Dataset>().
                     withType(KmeansConfig::defaultOutputDatasetType));

    addField("trainingData", &KmeansConfig::trainingData,
             "Specification of the data for input to the k-means procedure.  This should be "
             "organized as an embedding, with each selected row containing the same "
             "set of columns with numeric values to be used as coordinates.  The select statement "
             "does not support groupby and having clauses.");
    addField("outputDataset", &KmeansConfig::output,
             "Dataset for cluster assignment.  This dataset will contain the same "
             "row names as the input dataset, but the coordinates will be replaced "
             "by a single column giving the cluster number that the row was assigned to.",
              optional);
    addField("centroidsDataset", &KmeansConfig::centroids,
             "Dataset in which the centroids will be recorded.  This dataset will "
             "have the same coordinates (columns) as those selected from the input "
             "dataset, but will have one row per cluster, providing the centroid of "
             "the cluster.",
             PolyConfigT<Dataset>().withType("embedding"));
    addField("numInputDimensions", &KmeansConfig::numInputDimensions,
             "Number of dimensions from the input to use (-1 = all).  This limits "
             "the number of columns used.  Columns will be ordered alphabetically "
             "and the lowest ones kept.",
             -1);
    addField("numClusters", &KmeansConfig::numClusters,
             "Number of clusters to create.  This will provide the total number of "
             "centroids created.  There must be at least as many rows selected as "
             "clusters.", 10);
    addField("maxIterations", &KmeansConfig::maxIterations,
             "Maximum number of iterations to perform.  If no convergeance is "
             "reached within this number of iterations, the current clustering "
             "will be returned.", 100);
    addField("metric", &KmeansConfig::metric,
             "Metric space in which the k-means distances will be calculated. "
             "Normally this will be Cosine for an orthonormal basis, and "
             "Euclidian for another basis",
             METRIC_COSINE);
    addField("functionName", &KmeansConfig::functionName,
             "If specified, a kmeans function of this name will be created using "
             "the training result.");
    addParent<ProcedureConfig>();

    onPostValidate = validate<KmeansConfig, InputQuery, NoGroupByHaving>(&KmeansConfig::trainingData, "kmeans");
}

// TODO: see http://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf

namespace {

ML::KMeansMetric * makeMetric(MetricSpace metric)
{
    switch (metric) {
    case METRIC_EUCLIDEAN:
        return new ML::KMeansEuclideanMetric();
        break;
    case METRIC_COSINE:
        return new ML::KMeansCosineMetric();
        break;
    default:
        throw ML::Exception("Unknown kmeans metric type");
    }
}

} // file scope

/*****************************************************************************/
/* KMEANS PROCEDURE                                                           */
/*****************************************************************************/

KmeansProcedure::
KmeansProcedure(MldbServer * owner,
               PolyConfig config,
               const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    kmeansConfig = config.params.convert<KmeansConfig>();
}

Any
KmeansProcedure::
getStatus() const
{
    return Any();
}

RunOutput
KmeansProcedure::
run(const ProcedureRunConfig & run,
      const std::function<bool (const Json::Value &)> & onProgress) const
{
    auto runProcConf = applyRunConfOverProcConf(kmeansConfig, run);

    auto onProgress2 = [&] (const Json::Value & progress)
        {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };

    SqlExpressionMldbContext context(server);

    auto embeddingOutput = getEmbedding(*runProcConf.trainingData.stm,
                                        context,
                                        runProcConf.numInputDimensions,
                                        onProgress2);

    std::vector<std::tuple<RowHash, RowName, std::vector<double>,
                           std::vector<ExpressionValue> > > & rows
        = embeddingOutput.first;
    std::vector<KnownColumn> & vars = embeddingOutput.second;

    std::vector<ColumnName> columnNames;
    for (auto & v: vars) {
        columnNames.push_back(v.columnName);
    }

    std::vector<ML::distribution<float> > vecs;

    for (unsigned i = 0;  i < rows.size();  ++i) {
        vecs.emplace_back(ML::distribution<float>(std::get<2>(rows[i]).begin(),
                                                  std::get<2>(rows[i]).end()));
    }

    if (vecs.size() == 0)
        throw HttpReturnException(400, "Kmeans training requires at least 1 datapoint. "
                                  "Make sure your dataset is not empty and that your WHERE expression "
                                  "does not filter all the rows");

    ML::KMeans kmeans;
    kmeans.metric.reset(makeMetric(runProcConf.metric));

    vector<int> inCluster;

    int numClusters = runProcConf.numClusters;
    int numIterations = runProcConf.maxIterations;

    //doProgress("running k-means");

    kmeans.train(vecs, inCluster, numClusters, numIterations);

    //doProgress("finished k-means");
    if (runProcConf.output.get()) {

        PolyConfigT<Dataset> outputDataset = *runProcConf.output;
        if (outputDataset.type.empty())
            outputDataset.type = KmeansConfig::defaultOutputDatasetType;

        auto output = createDataset(server, outputDataset, onProgress2, true /*overwrite*/);

        Date applyDate = Date::now();

        for (unsigned i = 0;  i < rows.size();  ++i) {
            std::vector<std::tuple<ColumnName, CellValue, Date> > cols;
            cols.emplace_back(ColumnName("cluster"), inCluster[i], applyDate);
            output->recordRow(std::get<1>(rows[i]), cols);
        }

        output->commit();
    }

    if (runProcConf.centroids.type != "" || kmeansConfig.centroids.id != "") {

        auto centroids = createDataset(server, runProcConf.centroids, onProgress2, true /*overwrite*/);

        Date applyDate = Date::now();

        for (unsigned i = 0;  i < kmeans.clusters.size();  ++i) {
            auto & cluster = kmeans.clusters[i];

            std::vector<std::tuple<ColumnName, CellValue, Date> > cols;

            for (unsigned j = 0;  j < cluster.centroid.size();  ++j) {
                cols.emplace_back(columnNames[j], cluster.centroid[j], applyDate);
            }

            centroids->recordRow(RowName(ML::format("%i", i)), cols);
        }

        centroids->commit();
    }

    if(!runProcConf.functionName.empty()) {
        KmeansFunctionConfig funcConf;
        funcConf.metric = runProcConf.metric;
        funcConf.centroids = runProcConf.centroids;

        PolyConfig kmeansFuncPC;
        kmeansFuncPC.type = "kmeans";
        kmeansFuncPC.id = runProcConf.functionName;
        kmeansFuncPC.params = funcConf;

        obtainFunction(server, kmeansFuncPC, onProgress);
    }

    return Any();
}

DEFINE_STRUCTURE_DESCRIPTION(KmeansFunctionConfig);

KmeansFunctionConfigDescription::
KmeansFunctionConfigDescription()
{
    addField("centroids", &KmeansFunctionConfig::centroids,
             "Dataset containing centroids of each cluster: one row per cluster.");
    addField("metric", &KmeansFunctionConfig::metric,
             "Metric to use to calculate distances.  This should match the "
             "metric used in training.");
    addField("inputData", &KmeansFunctionConfig::inputData,
             "SQL expression to select fields to calculate k-means over.  Only those fields "
             "that are selected here need to be matched.  Default is to use all the fields.");
}

/*****************************************************************************/
/* KMEANS FUNCTION                                                              */
/*****************************************************************************/

KmeansFunction::
KmeansFunction(MldbServer * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner)
{
    functionConfig = config.params.convert<KmeansFunctionConfig>();

    auto dataset = obtainDataset(server, functionConfig.centroids, onProgress);

    SelectStatement query = functionConfig.inputData.stm ?
        *functionConfig.inputData.stm : SelectStatement();

    // Load up the embeddings
    auto embeddingOutput = getEmbedding(query.select,
                                        *dataset, "",
                                        query.when,
                                        query.where,
                                        { },
                                        -1 /* max dimensions */,
                                        ORDER_BY_NOTHING, 0, -1,
                                        onProgress);

    // Each row is a cluster
    std::vector<std::tuple<RowHash, RowName, std::vector<double>,
                           std::vector<ExpressionValue> > > & rows
        = embeddingOutput.first;
    std::vector<KnownColumn> & vars = embeddingOutput.second;

    for (auto & v: vars) {
        columnNames.push_back(v.columnName);
    }

    for (auto & r: rows) {
        Cluster cluster;
        cluster.clusterName = jsonDecodeStr<CellValue>(std::get<1>(r).toString());
        cluster.centroid.insert(cluster.centroid.end(),
                                std::get<2>(r).begin(),
                                std::get<2>(r).end());
        clusters.emplace_back(std::move(cluster));
    }

    cerr << "got " << clusters.size()
         << " clusters with " << columnNames.size()
         << "values" << endl;
}

Any
KmeansFunction::
getStatus() const
{
    return Any();
}

FunctionOutput
KmeansFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    FunctionOutput result;

    //cerr << "context = " << jsonEncode(context) << endl;

    // Extract an embedding with the given column names
    ExpressionValue storage;
    const ExpressionValue & inputVal = context.get("embedding", storage);
    //cerr << "getting embedding" << endl;
    ML::distribution<float> input = inputVal.getEmbedding(columnNames.size());
    Date ts = inputVal.getEffectiveTimestamp();

    std::unique_ptr<ML::KMeansMetric> metric(makeMetric(functionConfig.metric));

    double bestDist = INFINITY;
    CellValue bestCluster;

    for (unsigned i = 0;  i < clusters.size();  ++i) {
        double dist = metric->distance(input, clusters[i].centroid);
        if (dist < bestDist
            || (dist == bestDist && clusters[i].clusterName < bestCluster)) {
            bestDist = dist;
            bestCluster = clusters[i].clusterName;
        }
    }

    result.set("cluster", ExpressionValue(bestCluster, ts));

    return result;
}

FunctionInfo
KmeansFunction::
getFunctionInfo() const
{
    FunctionInfo result;

    result.input.addEmbeddingValue("embedding", columnNames.size());
    result.output.addAtomValue("cluster");

    return result;
}

namespace {

RegisterProcedureType<KmeansProcedure, KmeansConfig>
regKmeans(builtinPackage(),
          "kmeans.train",
          "Simple clustering algorithm based on cluster centroids in embedding space",
          "procedures/KmeansProcedure.md.html");

RegisterFunctionType<KmeansFunction, KmeansFunctionConfig>
regKmeansFunction(builtinPackage(),
                  "kmeans",
                  "Apply a k-means clustering to new data",
                  "functions/Kmeans.md.html");

} // file scope

} // namespace MLDB
} // namespace Datacratic

<|endoftext|>


<commit_before>
// the syntax for messages is:
//   wd 'mb type buttons title message'
//
// type specifies the icon and default behaviour:
//  info      (default OK button)
//  warn      (default OK button)
//  critical  (default OK button)
//  query     (requires two or three buttons)
//
// if one button, there is no result,
// otherwise the result is the button name (ok, cancel, ...)
//
// buttons are from the set:
//  mb_ok
//  mb_cancel
//  mb_yes
//  mb_no
//  mb_save
//  mb_discard

#include <QApplication>
#include <QColorDialog>
#include <QFileDialog>
#include <QFontDialog>
#include <QMessageBox>
#include <QTextDocument>

#ifndef ANDROID
#ifdef QT50
#include <QtPrintSupport/QPrinter>
#include <QtPrintSupport/QPrintDialog>
#else
#include <QPrinter>
#include <QPrintDialog>
#endif
extern QPrinter *Printer;
#endif

#include "wd.h"
#include "cmd.h"
#include "../base/dialog.h"
#include "../base/state.h"

QString mbcolor();
QString mbdir();
QString mbfont();
// QString mbinfo(QString);
QString mbprint(bool);
QString mbprintx(bool);
QString mbmsg();
QString mbopen();
QString mbopen1();
QString mbsave();

QString fixsep(QString s);

QString type;
QStringList arg;

QMessageBox::StandardButton getdefaultbutton();
QMessageBox::StandardButton getonebutton();
QMessageBox::StandardButtons getotherbuttons();
QString getname(int);

// ---------------------------------------------------------------------
QString mb(string p)
{
  arg=qsplit(p);
  if (arg.size()<1) {
    error("missing mb type");
    return "";
  }

  type=arg.first();
  arg.removeFirst();
  if (type=="color")
    return mbcolor();
  if (type=="dir")
    return mbdir();
  if (type=="font")
    return mbfont();
  if (type=="open")
    return mbopen();
  if (type=="open1")
    return mbopen1();
  if (type=="print") {
    QString s=dlb(s2q(p.substr(5)));
    return mbprint('*'==s.at(0));
  }
  if (type=="printx") {
    QString s=dlb(s2q(p.substr(6)));
    return mbprintx('*'==s.at(0));
  }
  if (type=="save")
    return mbsave();
  if (type=="info"||type=="query"||type=="warn"||type=="critical")
    return mbmsg();
  error("invalid mb type: " + q2s(type));
  return "";
}

// ---------------------------------------------------------------------
QString mbmsg()
{
  int r;
  QString t,m;

  QMessageBox::StandardButton button1=getdefaultbutton();
  QMessageBox::StandardButtons buttons=getotherbuttons();

  if (arg.size()==1) {
    t="Message Box";
    m=arg.at(0);
  } else if (arg.size()==2) {
    t=arg.at(0);
    m=arg.at(1);
  } else {
    error("Need title and message: "+q2s(arg.join(" ")));
    return "";
  }

  if (button1==-1) {
    button1=QMessageBox::Ok;
    if (type=="query")
      buttons=QMessageBox::Cancel;
  }
  buttons|=button1;

  if (type=="query") {
    r=QMessageBox::question(QApplication::focusWidget(),t,m,buttons,button1);
    return getname(r);
  }
  if (type=="critical")
    QMessageBox::critical(QApplication::focusWidget(),t,m,buttons,button1);
  else if (type=="info")
    QMessageBox::information(QApplication::focusWidget(),t,m,buttons,button1);
  else if (type=="warn")
    QMessageBox::warning(QApplication::focusWidget(),t,m,buttons,button1);
  return "";
}

// ---------------------------------------------------------------------
QString mbcolor()
{
  QColor c;
  int r,g,b;

  if (arg.size()==3) {
    r=c_strtoi(q2s(arg.at(0)));
    g=c_strtoi(q2s(arg.at(1)));
    b=c_strtoi(q2s(arg.at(2)));
    c=QColor(r,g,b);
  } else
    c=Qt::white;
  c=QColorDialog::getColor(c,QApplication::focusWidget());
  if (!c.isValid()) return "";
  return s2q(i2s(c.red()) + " " + i2s(c.green()) + " " + i2s(c.blue()));
}

// ---------------------------------------------------------------------
QString mbdir()
{
  QString title,dir;
  if (arg.size()!=2) {
    error("dir needs title and directory");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  return QFileDialog::getExistingDirectory(
           QApplication::focusWidget(),title,dir);
}

// ---------------------------------------------------------------------
QString mbfont()
{
  bool ok;
  QFont font, def;
  QString s;
  def.setStrikeOut(false);
  def.setUnderline(false);
  if (arg.size())
    def.setFamily(arg.at(0));
  if (arg.size()>1)
    def.setPointSize(arg.at(1).toFloat());
  for (int i=2; i<arg.size(); i++) {
    s=arg.at(i);
    if (s=="bold")
      def.setBold(true);
    if (s=="italic")
      def.setItalic(true);
    if (s=="strikeout")
      def.setStrikeOut(true);
    if (s=="underline")
      def.setUnderline(true);
  }
  font=QFontDialog::getFont(&ok,def,QApplication::focusWidget());
  if (!ok) return "";
  QString r;
  r="\"" + font.family() + "\" " + QString::number(font.pointSize());
  if (font.bold()) r+=" bold";
  if (font.italic()) r+=" italic";
  if (font.strikeOut()) r+=" strikeout";
  if (font.underline()) r+=" underline";
  return r;
}

// ---------------------------------------------------------------------
QString mbopen()
{
  QString title,dir,filter;
  QStringList fl;
  if (arg.size()<2) {
    error("open needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  fl = QFileDialog::getOpenFileNames(
         QApplication::focusWidget(),title,dir,filter);
  if (fl.isEmpty())
    return "";
  else return fl.join("\012") + "\012";
}

// ---------------------------------------------------------------------
QString mbopen1()
{
  QString title,dir,filter;
  if (arg.size()<2) {
    error("open1 needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  return QFileDialog::getOpenFileName(
           QApplication::focusWidget(),title,dir,filter);
}

// ---------------------------------------------------------------------
QString mbprint(bool iftext)
{
#ifdef ANDROID
  Q_UNUSED(iftext);
  return "";
#else
  QString r="";
  if (arg.size()) {
    QTextDocument *d=0;
    QString s=arg.at(0);
    if (!iftext) {
      if (!cfexist(s)) {
        error("File not found: " + q2s(s));
        return "";
      }
      s=cfread(s);
    }
    d=new QTextDocument(s);

#ifdef __MACH__
    QSysInfo qsi;
    if (qsi.MacintoshVersion < QSysInfo::MV_10_7) {
      if (Printer==0)
        Printer=new QPrinter(QPrinter::HighResolution);
      if (!Printer->isValid()) {
        error("Invalid printer: " + q2s(Printer->printerName()));
        delete d;
        return "";
      }
      d->print(Printer);
      delete d;
      return "";
    }
#endif
    dialogprint(QApplication::focusWidget(),d);
    delete d;
  } else {
    if (Printer==0)
      Printer=new QPrinter(QPrinter::HighResolution);
#ifdef __MACH__
    QSysInfo qsi;
    if (qsi.MacintoshVersion < QSysInfo::MV_10_7) {
      r=Printer->printerName();
      return r;
    }
#endif
    QPrintDialog *dlg = new QPrintDialog(Printer);
    if (dlg->exec() == QDialog::Accepted) {
      switch (Printer->outputFormat()) {
      case QPrinter::PdfFormat :
        r="_pdf:" + Printer->outputFileName();
        break;
#ifndef QT50
      case QPrinter::PostScriptFormat :
        r="_ps:" + Printer->outputFileName();
        break;
#endif
      default :
        r=Printer->printerName();
        break;
      }
    }
    delete dlg;
  }
//set default printer
  config.Printer->setPrinterName(r);
  return r;
#endif
}

// ---------------------------------------------------------------------
// print with no dialog
QString mbprintx(bool iftext)
{
#ifdef ANDROID
  Q_UNUSED(iftext);
#else
  if (arg.size()==0) {
    error("No text given for printx");
    return "";
  }
  QString s=arg.at(0);
  if (!iftext) {
    if (!cfexist(s)) {
      error("File not found: " + q2s(s));
      return "";
    }
    s=cfread(s);
  }
  QTextDocument *d=new QTextDocument(s);
  if (Printer==0)
    Printer=new QPrinter(QPrinter::HighResolution);
  if (!Printer->isValid()) {
    error("Invalid printer: " + q2s(Printer->printerName()));
    return "";
  }
  d->print(Printer);
#endif
  return "";
}

// ---------------------------------------------------------------------
QString mbsave()
{
  QString title,dir,filter;
  if (arg.size()<2) {
    error("save needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  return QFileDialog::getSaveFileName(
           QApplication::focusWidget(),title,dir,filter);
}

// ---------------------------------------------------------------------
QString getname(int b)
{
  if (b==QMessageBox::Ok)
    return "ok";
  if (b==QMessageBox::Cancel)
    return "cancel";
  if (b==QMessageBox::Yes)
    return "yes";
  if (b==QMessageBox::No)
    return "no";
  if (b==QMessageBox::Save)
    return "save";
  if (b==QMessageBox::Discard)
    return "discard";
  return "unknown button";
}

// ---------------------------------------------------------------------
QMessageBox::StandardButton getonebutton()
{
  if (arg.isEmpty()) return QMessageBox::NoButton;
  QString s=arg.first();
  if (s=="mb_ok")
    return QMessageBox::Ok;
  if (s=="mb_cancel")
    return QMessageBox::Cancel;
  if (s=="mb_yes")
    return QMessageBox::Yes;
  if (s=="mb_no")
    return QMessageBox::No;
  if (s=="mb_save")
    return QMessageBox::Save;
  if (s=="mb_discard")
    return QMessageBox::Discard;
  return QMessageBox::NoButton;
}

// ---------------------------------------------------------------------
QMessageBox::StandardButton getdefaultbutton()
{
  QMessageBox::StandardButton r=getonebutton();
  if (r!=QMessageBox::NoButton)
    arg.removeFirst();
  return r;
}

// ---------------------------------------------------------------------
QMessageBox::StandardButtons getotherbuttons()
{
  QMessageBox::StandardButtons r=QMessageBox::NoButton;
  QMessageBox::StandardButton b;
  while (arg.size()) {
    b=getonebutton();
    if (b==QMessageBox::NoButton)
      return r;
    r|=b;
    arg.removeFirst();
  }
  return r;
}

// ---------------------------------------------------------------------
//QString mbinfo(QString s)
//{
//  info("Message Box",s);
//  return "";
//}
//
// ---------------------------------------------------------------------
QString fixsep(QString s)
{
  return s.replace("|",";;");
}
<commit_msg>fix error in previous mb.cpp commit<commit_after>
// the syntax for messages is:
//   wd 'mb type buttons title message'
//
// type specifies the icon and default behaviour:
//  info      (default OK button)
//  warn      (default OK button)
//  critical  (default OK button)
//  query     (requires two or three buttons)
//
// if one button, there is no result,
// otherwise the result is the button name (ok, cancel, ...)
//
// buttons are from the set:
//  mb_ok
//  mb_cancel
//  mb_yes
//  mb_no
//  mb_save
//  mb_discard

#include <QApplication>
#include <QColorDialog>
#include <QFileDialog>
#include <QFontDialog>
#include <QMessageBox>
#include <QTextDocument>

#ifndef ANDROID
#ifdef QT50
#include <QtPrintSupport/QPrinter>
#include <QtPrintSupport/QPrintDialog>
#else
#include <QPrinter>
#include <QPrintDialog>
#endif
extern QPrinter *Printer;
#endif

#include "wd.h"
#include "cmd.h"
#include "../base/dialog.h"
#include "../base/state.h"

QString mbcolor();
QString mbdir();
QString mbfont();
// QString mbinfo(QString);
QString mbprint(bool);
QString mbprintx(bool);
QString mbmsg();
QString mbopen();
QString mbopen1();
QString mbsave();

QString fixsep(QString s);

QString type;
QStringList arg;

QMessageBox::StandardButton getdefaultbutton();
QMessageBox::StandardButton getonebutton();
QMessageBox::StandardButtons getotherbuttons();
QString getname(int);

// ---------------------------------------------------------------------
QString mb(string p)
{
  arg=qsplit(p);
  if (arg.size()<1) {
    error("missing mb type");
    return "";
  }

  type=arg.first();
  arg.removeFirst();
  if (type=="color")
    return mbcolor();
  if (type=="dir")
    return mbdir();
  if (type=="font")
    return mbfont();
  if (type=="open")
    return mbopen();
  if (type=="open1")
    return mbopen1();
  if (type=="print") {
    QString s=dlb(s2q(p.substr(5)));
    return mbprint('*'==s.at(0));
  }
  if (type=="printx") {
    QString s=dlb(s2q(p.substr(6)));
    return mbprintx('*'==s.at(0));
  }
  if (type=="save")
    return mbsave();
  if (type=="info"||type=="query"||type=="warn"||type=="critical")
    return mbmsg();
  error("invalid mb type: " + q2s(type));
  return "";
}

// ---------------------------------------------------------------------
QString mbmsg()
{
  int r;
  QString t,m;

  QMessageBox::StandardButton button1=getdefaultbutton();
  QMessageBox::StandardButtons buttons=getotherbuttons();

  if (arg.size()==1) {
    t="Message Box";
    m=arg.at(0);
  } else if (arg.size()==2) {
    t=arg.at(0);
    m=arg.at(1);
  } else {
    error("Need title and message: "+q2s(arg.join(" ")));
    return "";
  }

  if (button1==-1) {
    button1=QMessageBox::Ok;
    if (type=="query")
      buttons=QMessageBox::Cancel;
  }
  buttons|=button1;

  if (type=="query") {
    r=QMessageBox::question(QApplication::focusWidget(),t,m,buttons,button1);
    return getname(r);
  }
  if (type=="critical")
    QMessageBox::critical(QApplication::focusWidget(),t,m,buttons,button1);
  else if (type=="info")
    QMessageBox::information(QApplication::focusWidget(),t,m,buttons,button1);
  else if (type=="warn")
    QMessageBox::warning(QApplication::focusWidget(),t,m,buttons,button1);
  return "";
}

// ---------------------------------------------------------------------
QString mbcolor()
{
  QColor c;
  int r,g,b;

  if (arg.size()==3) {
    r=c_strtoi(q2s(arg.at(0)));
    g=c_strtoi(q2s(arg.at(1)));
    b=c_strtoi(q2s(arg.at(2)));
    c=QColor(r,g,b);
  } else
    c=Qt::white;
  c=QColorDialog::getColor(c,QApplication::focusWidget());
  if (!c.isValid()) return "";
  return s2q(i2s(c.red()) + " " + i2s(c.green()) + " " + i2s(c.blue()));
}

// ---------------------------------------------------------------------
QString mbdir()
{
  QString title,dir;
  if (arg.size()!=2) {
    error("dir needs title and directory");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  return QFileDialog::getExistingDirectory(
           QApplication::focusWidget(),title,dir);
}

// ---------------------------------------------------------------------
QString mbfont()
{
  bool ok;
  QFont font, def;
  QString s;
  def.setStrikeOut(false);
  def.setUnderline(false);
  if (arg.size())
    def.setFamily(arg.at(0));
  if (arg.size()>1)
    def.setPointSize(arg.at(1).toFloat());
  for (int i=2; i<arg.size(); i++) {
    s=arg.at(i);
    if (s=="bold")
      def.setBold(true);
    if (s=="italic")
      def.setItalic(true);
    if (s=="strikeout")
      def.setStrikeOut(true);
    if (s=="underline")
      def.setUnderline(true);
  }
  font=QFontDialog::getFont(&ok,def,QApplication::focusWidget());
  if (!ok) return "";
  QString r;
  r="\"" + font.family() + "\" " + QString::number(font.pointSize());
  if (font.bold()) r+=" bold";
  if (font.italic()) r+=" italic";
  if (font.strikeOut()) r+=" strikeout";
  if (font.underline()) r+=" underline";
  return r;
}

// ---------------------------------------------------------------------
QString mbopen()
{
  QString title,dir,filter;
  QStringList fl;
  if (arg.size()<2) {
    error("open needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  fl = QFileDialog::getOpenFileNames(
         QApplication::focusWidget(),title,dir,filter);
  if (fl.isEmpty())
    return "";
  else return fl.join("\012") + "\012";
}

// ---------------------------------------------------------------------
QString mbopen1()
{
  QString title,dir,filter;
  if (arg.size()<2) {
    error("open1 needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  return QFileDialog::getOpenFileName(
           QApplication::focusWidget(),title,dir,filter);
}

// ---------------------------------------------------------------------
QString mbprint(bool iftext)
{
#ifdef ANDROID
  Q_UNUSED(iftext);
  return "";
#else
  QString r="";
  if (arg.size()) {
    QTextDocument *d=0;
    QString s=arg.at(0);
    if (!iftext) {
      if (!cfexist(s)) {
        error("File not found: " + q2s(s));
        return "";
      }
      s=cfread(s);
    }
    d=new QTextDocument(s);

#ifdef __MACH__
    QSysInfo qsi;
    if (qsi.MacintoshVersion < QSysInfo::MV_10_7) {
      if (Printer==0)
        Printer=new QPrinter(QPrinter::HighResolution);
      if (!Printer->isValid()) {
        error("Invalid printer: " + q2s(Printer->printerName()));
        delete d;
        return "";
      }
      d->print(Printer);
      delete d;
      return "";
    }
#endif
    dialogprint(QApplication::focusWidget(),d);
    delete d;
  } else {
    if (Printer==0)
      Printer=new QPrinter(QPrinter::HighResolution);
#ifdef __MACH__
    QSysInfo qsi;
    if (qsi.MacintoshVersion < QSysInfo::MV_10_7) {
      r=Printer->printerName();
      return r;
    }
#endif
    QPrintDialog *dlg = new QPrintDialog(Printer);
    if (dlg->exec() == QDialog::Accepted) {
      switch (Printer->outputFormat()) {
      case QPrinter::PdfFormat :
        r="_pdf:" + Printer->outputFileName();
        break;
#ifndef QT50
      case QPrinter::PostScriptFormat :
        r="_ps:" + Printer->outputFileName();
        break;
#endif
      default :
        r=Printer->printerName();
        break;
      }
    }
    delete dlg;
  }
//set default printer
  if (0<r.size()) config.Printer->setPrinterName(r);
  return r;
#endif
}

// ---------------------------------------------------------------------
// print with no dialog
QString mbprintx(bool iftext)
{
#ifdef ANDROID
  Q_UNUSED(iftext);
#else
  if (arg.size()==0) {
    error("No text given for printx");
    return "";
  }
  QString s=arg.at(0);
  if (!iftext) {
    if (!cfexist(s)) {
      error("File not found: " + q2s(s));
      return "";
    }
    s=cfread(s);
  }
  QTextDocument *d=new QTextDocument(s);
  if (Printer==0)
    Printer=new QPrinter(QPrinter::HighResolution);
  if (!Printer->isValid()) {
    error("Invalid printer: " + q2s(Printer->printerName()));
    return "";
  }
  d->print(Printer);
#endif
  return "";
}

// ---------------------------------------------------------------------
QString mbsave()
{
  QString title,dir,filter;
  if (arg.size()<2) {
    error("save needs title, directory, [filters]");
    return "";
  }
  title=arg.at(0);
  dir=arg.at(1);
  if (arg.size()==3)
    filter=fixsep(arg.at(2));
  return QFileDialog::getSaveFileName(
           QApplication::focusWidget(),title,dir,filter);
}

// ---------------------------------------------------------------------
QString getname(int b)
{
  if (b==QMessageBox::Ok)
    return "ok";
  if (b==QMessageBox::Cancel)
    return "cancel";
  if (b==QMessageBox::Yes)
    return "yes";
  if (b==QMessageBox::No)
    return "no";
  if (b==QMessageBox::Save)
    return "save";
  if (b==QMessageBox::Discard)
    return "discard";
  return "unknown button";
}

// ---------------------------------------------------------------------
QMessageBox::StandardButton getonebutton()
{
  if (arg.isEmpty()) return QMessageBox::NoButton;
  QString s=arg.first();
  if (s=="mb_ok")
    return QMessageBox::Ok;
  if (s=="mb_cancel")
    return QMessageBox::Cancel;
  if (s=="mb_yes")
    return QMessageBox::Yes;
  if (s=="mb_no")
    return QMessageBox::No;
  if (s=="mb_save")
    return QMessageBox::Save;
  if (s=="mb_discard")
    return QMessageBox::Discard;
  return QMessageBox::NoButton;
}

// ---------------------------------------------------------------------
QMessageBox::StandardButton getdefaultbutton()
{
  QMessageBox::StandardButton r=getonebutton();
  if (r!=QMessageBox::NoButton)
    arg.removeFirst();
  return r;
}

// ---------------------------------------------------------------------
QMessageBox::StandardButtons getotherbuttons()
{
  QMessageBox::StandardButtons r=QMessageBox::NoButton;
  QMessageBox::StandardButton b;
  while (arg.size()) {
    b=getonebutton();
    if (b==QMessageBox::NoButton)
      return r;
    r|=b;
    arg.removeFirst();
  }
  return r;
}

// ---------------------------------------------------------------------
//QString mbinfo(QString s)
//{
//  info("Message Box",s);
//  return "";
//}
//
// ---------------------------------------------------------------------
QString fixsep(QString s)
{
  return s.replace("|",";;");
}
<|endoftext|>


<commit_before>#include <ncurses.h>
#include <stdlib.h>
#include <time.h>

#include "lightsout.hpp"
#include "print.hpp"

namespace roadagain
{

board::board(int width, int height) : width(width), height(height), y(0), x(0)
{
    // init seed of rand() by current time
    srand(time(NULL));

    lights = new bool*[height + 2]();
    for (int i = 0; i < height + 2; i++){
        lights[i] = new bool[width + 2]();
    }

    // init and print the board
    print_board(width, height);
    for (int i = 1; i < height + 1; i++){
        for (int j = 1; j < width + 1; j++){
            lights[i][j] = (rand() % 2 == 0);
            print_light(i, j, lights[i][j]);
        }
    }
}

board::~board()
{
    ;
}

bool board::is_perfect()
{
    for (int i = 1; i <= this->height + 1; i++){
        for (int j = 1; j <= this->width + 1; j++){
            if (!this->lights[i][j]){
                return false;
            }
        }
    }
    return true;
}

void board::move_board(int dy, int dx)
{
    this->y = (this->y - 1 + dy + this->height) % this->height + 1;
    this->x = (this->x - 1 + dx + this->width) % this->width + 1;
}

}
<commit_msg>Tweak<commit_after>#include <ncurses.h>
#include <stdlib.h>
#include <time.h>

#include "lightsout.hpp"
#include "print.hpp"

namespace roadagain
{

board::board(int width, int height) : width(width), height(height), y(0), x(0)
{
    // init seed of rand() by current time
    srand(time(NULL));

    lights = new bool*[height + 2]();
    for (int i = 0; i < height + 2; i++){
        lights[i] = new bool[width + 2]();
    }

    // init and print the board
    print_board(width, height);
    for (int i = 1; i < height + 1; i++){
        for (int j = 1; j < width + 1; j++){
            lights[i][j] = (rand() % 2 == 0);
            print_light(i, j, lights[i][j]);
        }
    }
}

board::~board()
{
    ;
}

bool board::is_perfect()
{
    for (int i = 1; i <= this->height + 1; i++){
        for (int j = 1; j <= this->width + 1; j++){
            if (!this->lights[i][j]){
                return false;
            }
        }
    }
    return true;
}

void board::move_board(int dy, int dx)
{
    this->y = (this->y - 1 + dy + this->height) % this->height + 1;
    this->x = (this->x - 1 + dx + this->width) % this->width + 1;
    move(this->y * 2 - 1, this->x * 2 - 1);
}

}
<|endoftext|>