<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "sling/file/file.h"

#include <pthread.h>
#include <string>
#include <unordered_map>

#include "sling/base/init.h"
#include "sling/base/logging.h"
#include "sling/base/registry.h"
#include "sling/base/status.h"
#include "sling/base/types.h"

// Registry for file systems.
REGISTER_SINGLETON_REGISTRY("file system", sling::FileSystem);

namespace sling {
namespace {

// The file systems need to be initialized at most once.
pthread_once_t file_systems_initialized = PTHREAD_ONCE_INIT;

// Registered file systems.
std::unordered_map<string, FileSystem *> file_systems;
FileSystem *default_file_system = nullptr;

// Initialize all registered file systems.
void InitializeFileSystems() {
  auto *registry = FileSystem::registry();
  for (auto *fs = registry->components; fs != nullptr; fs = fs->next()) {
    VLOG(2) << "Initializing " << fs->type() << " file system";
    fs->object()->Init();
    if (fs->object()->IsDefaultFileSystem()) {
      default_file_system = fs->object();
    } else {
      file_systems[fs->type()] = fs->object();
    }
  }
}

// Find file system for file name. If no matching file system is found, the
// default file system is returned.
FileSystem *FindFileSystem(const string &filename, string *rest) {
  // Match the first component in the path.
  if (!filename.empty() && filename[0] == '/') {
    int slash = filename.find('/', 1);
    if (slash != -1) {
      auto f = file_systems.find(filename.substr(1, slash - 1));
      if (f != file_systems.end()) {
        *rest = filename.substr(slash + 1);
        return f->second;
      }
    }
  }

  // Fall back on the default file system.
  *rest = filename;
  return default_file_system;
}

Status NoFileSystem(const string &filename) {
  return Status(1, "No file system", filename);
}

}  // namespace

void File::Init() {
  // Allow this method to be called multiple times.
  pthread_once(&file_systems_initialized, InitializeFileSystems);
}

Status File::Open(const string &name, const char *mode, File **f) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Open new file.
  return fs->Open(rest, mode, f);
}

File *File::Open(const string &name, const char *mode) {
  File *f;
  if (!Open(name, mode, &f).ok()) return nullptr;
  return f;
}

File *File::OpenOrDie(const string &name, const char *mode) {
  File *f;
  CHECK(Open(name, mode, &f));
  return f;
}

File *File::TempFile() {
  CHECK(default_file_system != nullptr) << "No filesystems";
  File *f;
  CHECK(default_file_system->CreateTempFile(&f));
  return f;
}

Status File::Delete(const string &name) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Delete file.
  return fs->DeleteFile(rest);
}

bool File::Exists(const string &name) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return false;

  // Check if file exists.
  return fs->FileExists(rest);
}

Status File::GetSize(const string &name, uint64 *size) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Get file size.
  return fs->GetFileSize(rest, size);
}

Status File::Stat(const string &name, FileStat *stat) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Get file size.
  return fs->Stat(rest, stat);
}

Status File::Rename(const string &source, const string &target) {
  // Find file system.
  string src;
  string tgt;
  FileSystem *srcfs = FindFileSystem(source, &src);
  FileSystem *tgtfs = FindFileSystem(target, &tgt);
  if (srcfs == nullptr) return NoFileSystem(source);
  if (tgtfs == nullptr) return NoFileSystem(target);
  if (srcfs != tgtfs) return Status(1, "Cross file system rename", source);

  // Rename file.
  return srcfs->RenameFile(src, tgt);
}

Status File::Mkdir(const string &dir) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(dir, &rest);
  if (fs == nullptr) return NoFileSystem(dir);

  // Create directory.
  return fs->CreateDir(rest);
}

Status File::Rmdir(const string &dir) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(dir, &rest);
  if (fs == nullptr) return NoFileSystem(dir);

  // Create directory.
  return fs->DeleteDir(rest);
}

Status File::CreateLocalTempDir(string *dir) {
  char tmpldir[] = "/tmp/local.XXXXXX";
  char *tmpname = mkdtemp(tmpldir);
  if (tmpname == nullptr) {
    return Status(errno, "mkdtemp", strerror(errno));
  }
  *dir = tmpname;
  return Status::OK;
}

Status File::CreateGlobalTempDir(string *dir) {
  char tmpldir[] = "/var/data/tmp/global.XXXXXX";
  char *tmpname = mkdtemp(tmpldir);
  if (tmpname == nullptr) {
    return Status(errno, "mkdtemp", strerror(errno));
  }
  *dir = tmpname;
  return Status::OK;
}

Status File::Match(const string &pattern, std::vector<string> *filenames) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(pattern, &rest);
  if (fs == nullptr) return NoFileSystem(pattern);

  // Convert sharded file pattern.
  int at = rest.find('@');
  if (at != string::npos) {
    string shards = rest.substr(at + 1);
    rest.resize(at);
    rest.append("-\?\?\?\?\?-of-");
    rest.append(5 - shards.size(), '0');
    rest.append(shards);
  }

  // Find files.
  return fs->Match(rest, filenames);
}

Status File::ReadContents(const string &filename, string *data) {
  // Open file for reading.
  File *f;
  Status st = Open(filename, "r", &f);
  if (!st.ok()) return st;

  // Read contents.
  st = f->ReadToString(data);
  if (!st.ok()) {
    f->Close();
    return st;
  }

  // Close file.
  return f->Close();
}

Status File::WriteContents(const string &filename,
                           const void *data,
                           size_t size) {
  // Open file for writing.
  File *f;
  Status st = Open(filename, "w", &f);
  if (!st.ok()) return st;

  // Write contents.
  st = f->Write(data, size);
  if (!st.ok()) {
    f->Close();
    return st;
  }

  // Close file.
  return f->Close();
}

size_t File::ReadOrDie(void *buffer, size_t size) {
  uint64 read;
  CHECK(Read(buffer, size, &read));
  return read;
}

void File::WriteOrDie(const void *buffer, size_t size) {
  CHECK(Write(buffer, size));
}

Status File::ReadToString(string *contents) {
  // Get current position and size.
  uint64 pos, size;
  Status st = GetPosition(&pos);
  if (!st.ok()) return st;
  st = GetSize(&size);
  if (!st.ok()) return st;
  size -= pos;

  // Read file data into string buffer.
  contents->resize(size);
  char *data = &(*contents)[0];
  uint64 read;
  st = Read(data, size, &read);
  if (st.ok() && read < contents->size()) contents->resize(read);
  return st;
}

Status File::WriteString(const string &str) {
  return Write(str.data(), str.size());
}

Status File::WriteLine(const string &line) {
  Status st = WriteString(line);
  if (st.ok()) st = Write("\n", 1);
  return st;
}

uint64 File::Tell() {
  uint64 pos;
  if (!GetPosition(&pos).ok()) return -1;
  return pos;
}

uint64 File::Size() {
  uint64 size;
  if (!GetSize(&size).ok()) return -1;
  return size;
}

REGISTER_INITIALIZER(filesystem, {
  File::Init();
});

}  // namespace sling

<commit_msg>Ensure that file systems are initialized (#119)<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "sling/file/file.h"

#include <pthread.h>
#include <string>
#include <unordered_map>

#include "sling/base/init.h"
#include "sling/base/logging.h"
#include "sling/base/registry.h"
#include "sling/base/status.h"
#include "sling/base/types.h"

// Registry for file systems.
REGISTER_SINGLETON_REGISTRY("file system", sling::FileSystem);

namespace sling {
namespace {

// The file systems need to be initialized at most once.
pthread_once_t file_systems_initialized = PTHREAD_ONCE_INIT;

// Registered file systems.
std::unordered_map<string, FileSystem *> file_systems;
FileSystem *default_file_system = nullptr;

// Initialize all registered file systems.
void InitializeFileSystems() {
  auto *registry = FileSystem::registry();
  for (auto *fs = registry->components; fs != nullptr; fs = fs->next()) {
    VLOG(2) << "Initializing " << fs->type() << " file system";
    fs->object()->Init();
    if (fs->object()->IsDefaultFileSystem()) {
      default_file_system = fs->object();
    } else {
      file_systems[fs->type()] = fs->object();
    }
  }
}

// Find file system for file name. If no matching file system is found, the
// default file system is returned.
FileSystem *FindFileSystem(const string &filename, string *rest) {
  // Initialize file systems if not already done.
  if (default_file_system == nullptr) File::Init();

  // Match the first component in the path.
  if (!filename.empty() && filename[0] == '/') {
    int slash = filename.find('/', 1);
    if (slash != -1) {
      auto f = file_systems.find(filename.substr(1, slash - 1));
      if (f != file_systems.end()) {
        *rest = filename.substr(slash + 1);
        return f->second;
      }
    }
  }

  // Fall back on the default file system.
  *rest = filename;
  return default_file_system;
}

Status NoFileSystem(const string &filename) {
  return Status(1, "No file system", filename);
}

}  // namespace

void File::Init() {
  // Allow this method to be called multiple times.
  pthread_once(&file_systems_initialized, InitializeFileSystems);
}

Status File::Open(const string &name, const char *mode, File **f) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Open new file.
  return fs->Open(rest, mode, f);
}

File *File::Open(const string &name, const char *mode) {
  File *f;
  if (!Open(name, mode, &f).ok()) return nullptr;
  return f;
}

File *File::OpenOrDie(const string &name, const char *mode) {
  File *f;
  CHECK(Open(name, mode, &f));
  return f;
}

File *File::TempFile() {
  CHECK(default_file_system != nullptr) << "No filesystems";
  File *f;
  CHECK(default_file_system->CreateTempFile(&f));
  return f;
}

Status File::Delete(const string &name) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Delete file.
  return fs->DeleteFile(rest);
}

bool File::Exists(const string &name) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return false;

  // Check if file exists.
  return fs->FileExists(rest);
}

Status File::GetSize(const string &name, uint64 *size) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Get file size.
  return fs->GetFileSize(rest, size);
}

Status File::Stat(const string &name, FileStat *stat) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(name, &rest);
  if (fs == nullptr) return NoFileSystem(name);

  // Get file size.
  return fs->Stat(rest, stat);
}

Status File::Rename(const string &source, const string &target) {
  // Find file system.
  string src;
  string tgt;
  FileSystem *srcfs = FindFileSystem(source, &src);
  FileSystem *tgtfs = FindFileSystem(target, &tgt);
  if (srcfs == nullptr) return NoFileSystem(source);
  if (tgtfs == nullptr) return NoFileSystem(target);
  if (srcfs != tgtfs) return Status(1, "Cross file system rename", source);

  // Rename file.
  return srcfs->RenameFile(src, tgt);
}

Status File::Mkdir(const string &dir) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(dir, &rest);
  if (fs == nullptr) return NoFileSystem(dir);

  // Create directory.
  return fs->CreateDir(rest);
}

Status File::Rmdir(const string &dir) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(dir, &rest);
  if (fs == nullptr) return NoFileSystem(dir);

  // Create directory.
  return fs->DeleteDir(rest);
}

Status File::CreateLocalTempDir(string *dir) {
  char tmpldir[] = "/tmp/local.XXXXXX";
  char *tmpname = mkdtemp(tmpldir);
  if (tmpname == nullptr) {
    return Status(errno, "mkdtemp", strerror(errno));
  }
  *dir = tmpname;
  return Status::OK;
}

Status File::CreateGlobalTempDir(string *dir) {
  char tmpldir[] = "/var/data/tmp/global.XXXXXX";
  char *tmpname = mkdtemp(tmpldir);
  if (tmpname == nullptr) {
    return Status(errno, "mkdtemp", strerror(errno));
  }
  *dir = tmpname;
  return Status::OK;
}

Status File::Match(const string &pattern, std::vector<string> *filenames) {
  // Find file system.
  string rest;
  FileSystem *fs = FindFileSystem(pattern, &rest);
  if (fs == nullptr) return NoFileSystem(pattern);

  // Convert sharded file pattern.
  int at = rest.find('@');
  if (at != string::npos) {
    string shards = rest.substr(at + 1);
    rest.resize(at);
    rest.append("-\?\?\?\?\?-of-");
    rest.append(5 - shards.size(), '0');
    rest.append(shards);
  }

  // Find files.
  return fs->Match(rest, filenames);
}

Status File::ReadContents(const string &filename, string *data) {
  // Open file for reading.
  File *f;
  Status st = Open(filename, "r", &f);
  if (!st.ok()) return st;

  // Read contents.
  st = f->ReadToString(data);
  if (!st.ok()) {
    f->Close();
    return st;
  }

  // Close file.
  return f->Close();
}

Status File::WriteContents(const string &filename,
                           const void *data,
                           size_t size) {
  // Open file for writing.
  File *f;
  Status st = Open(filename, "w", &f);
  if (!st.ok()) return st;

  // Write contents.
  st = f->Write(data, size);
  if (!st.ok()) {
    f->Close();
    return st;
  }

  // Close file.
  return f->Close();
}

size_t File::ReadOrDie(void *buffer, size_t size) {
  uint64 read;
  CHECK(Read(buffer, size, &read));
  return read;
}

void File::WriteOrDie(const void *buffer, size_t size) {
  CHECK(Write(buffer, size));
}

Status File::ReadToString(string *contents) {
  // Get current position and size.
  uint64 pos, size;
  Status st = GetPosition(&pos);
  if (!st.ok()) return st;
  st = GetSize(&size);
  if (!st.ok()) return st;
  size -= pos;

  // Read file data into string buffer.
  contents->resize(size);
  char *data = &(*contents)[0];
  uint64 read;
  st = Read(data, size, &read);
  if (st.ok() && read < contents->size()) contents->resize(read);
  return st;
}

Status File::WriteString(const string &str) {
  return Write(str.data(), str.size());
}

Status File::WriteLine(const string &line) {
  Status st = WriteString(line);
  if (st.ok()) st = Write("\n", 1);
  return st;
}

uint64 File::Tell() {
  uint64 pos;
  if (!GetPosition(&pos).ok()) return -1;
  return pos;
}

uint64 File::Size() {
  uint64 size;
  if (!GetSize(&size).ok()) return -1;
  return size;
}

REGISTER_INITIALIZER(filesystem, {
  File::Init();
});

}  // namespace sling

<|endoftext|>


<commit_before>/*
Highly Optimized Object-Oriented Molecular Dynamics (HOOMD) Open
Source Software License
Copyright (c) 2008 Ames Laboratory Iowa State University
All rights reserved.

Redistribution and use of HOOMD, in source and binary forms, with or
without modification, are permitted, provided that the following
conditions are met:

* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names HOOMD's
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.

Disclaimer

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND
CONTRIBUTORS ``AS IS''  AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS  BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
*/

// $Id$
// $URL$

/*! \file force_compute_bmark.cc
	\brief Executable for benchmarking all of the various ForceCompute classes
	\details This is intended as a quick test to check the performance of ForceComputes
		on various machines and after performance tweaks have been made. It allows for a
		number of command line options to change settings, but will most likely just
		be run with the default settings most of the time.
	\ingroup benchmarks
*/

#ifdef WIN32
#include <windows.h>
#endif

#include <stdlib.h>
#include <iostream>
#include <iomanip>
#include <string>

#include "BondForceCompute.h"
#include "LJForceCompute.h"

#include "BinnedNeighborList.h"
#include "Initializers.h"
#include "SFCPackUpdater.h"

#ifdef USE_CUDA
#include "LJForceComputeGPU.h"
#include "BondForceComputeGPU.h"
#endif

#include "HOOMDVersion.h"

#include <boost/program_options.hpp>
#include <boost/shared_ptr.hpp>

using namespace boost::program_options;
using namespace boost;
using namespace std;

// options from command line
//! quite mode: only display time per compute and nothing else
bool quiet = false;
//! number of threads (for applicable computes)
unsigned int nthreads = 1;
//! cutoff radius for pair forces (for applicable computes)
Scalar r_cut = 3.0;
//! buffer radius for pair forces (for applicable computes)
Scalar r_buff = 0.8;
//! block size for calculation (for applicable computes)
unsigned int block_size = 128;
//! number of particles to benchmark
unsigned int N = 100000;
//! Should the particles be sorted with SFCPACK?
bool sort_particles = true;
//! number of seconds to average over
unsigned int nsec = 10;
//! Activate profiling?
bool profile_compute = true;
//! Use half neighborlist?
bool half_nlist = true;
//! Specify packing fraction of particles for the benchmark (if applicable)
Scalar phi_p = 0.2;

//! Helper function to initialize bonds
void init_bond_tables(shared_ptr<BondForceCompute> bf_compute)
	{
	const unsigned int nbonds = 2;
	for (unsigned int j = 0; j < N-nbonds/2; j++)
		{
		for (unsigned int k = 1; k <= nbonds/2; k++)
			bf_compute->addBond(j,j+k);
		}
	}

//! Initialize the force compute from a string selecting it
shared_ptr<ForceCompute> init_force_compute(const string& fc_name, shared_ptr<ParticleData> pdata, shared_ptr<NeighborList> nlist)
	{
	shared_ptr<ForceCompute> result;

	// handle creation of the various lennard=jones computes
	if (fc_name == "LJ")
		result = shared_ptr<ForceCompute>(new LJForceCompute(pdata, nlist, r_cut));
	#ifdef USE_CUDA
	if (fc_name == "LJ.GPU")
		{
		shared_ptr<LJForceComputeGPU> tmp = shared_ptr<LJForceComputeGPU>(new LJForceComputeGPU(pdata, nlist, r_cut));
		tmp->setBlockSize(block_size);
		result = tmp;
		}
	#endif

	// handle the various bond force computes
	if (fc_name == "Bond")
		{
		shared_ptr<BondForceCompute> tmp = shared_ptr<BondForceCompute>(new BondForceCompute(pdata, 150, 1.0));
		init_bond_tables(tmp);
		result = tmp;
		}
	#ifdef USE_CUDA
	if (fc_name == "Bond.GPU")
		{
		shared_ptr<BondForceComputeGPU> tmp = shared_ptr<BondForceComputeGPU>(new BondForceComputeGPU(pdata, 150, 1.0));
		init_bond_tables(tmp);
		tmp->setBlockSize(block_size);
		result = tmp;
		}
	#endif

	return result;
	}

//! Initializes the particle data to a random set of particles
shared_ptr<ParticleData> init_pdata()
	{
	RandomInitializer rand_init(N, phi_p, 0.0, "A");
	shared_ptr<ParticleData> pdata(new ParticleData(rand_init));
	if (sort_particles)
		{
		SFCPackUpdater sorter(pdata, Scalar(1.0));
		sorter.update(0);
		}

	return pdata;
	}

//! Actually performs the benchmark on the preconstructed force compute
void benchmark(shared_ptr<ForceCompute> fc)
	{
	// initialize profiling if requested
	shared_ptr<Profiler> prof(new Profiler());
	if (profile_compute && !quiet)
		fc->setProfiler(prof);

	// timer to count how long we spend at each point
	ClockSource clk;
	int count = 2;

	// do a warmup run so memory allocations don't change the benchmark numbers
	fc->compute(count++);

	int64_t tstart = clk.getTime();
	int64_t tend;
	// do at least one test, then repeat until we get at least 5s of data at this point
	int nrepeat = 0;
	do
		{
		fc->compute(count++);
		nrepeat++;
		tend = clk.getTime();
		} while((tend - tstart) < int64_t(nsec) * int64_t(1000000000) || nrepeat < 5);

	// make sure all kernels have been executed when using CUDA
	#ifdef USE_CUDA
	cudaThreadSynchronize();
	#endif
	tend = clk.getTime();

	if (!quiet)
		cout << *prof << endl;

	double avgTime = double(tend - tstart)/1e9/double(nrepeat);;
	cout << setprecision(7) << avgTime << " s/step" << endl;
	}

//! Parses the command line and runs the benchmark
int main(int argc, char **argv)
	{

	// the name of the ForceCompute to benchmark (gotten from user)
	string fc_name;

	options_description desc("Allowed options");
	desc.add_options()
		("help,h", "Produce help message")
		("nparticles,N", value<unsigned int>(&N)->default_value(64000), "Number of particles")
		("phi_p", value<Scalar>(&phi_p)->default_value(0.2), "Volume fraction of particles in test system")
		("r_cut", value<Scalar>(&r_cut)->default_value(3.0), "Cutoff radius for pair force sum")
		("r_buff", value<Scalar>(&r_buff)->default_value(0.8), "Buffer radius for pair force sum")
		("nthreads,t", value<unsigned int>(&nthreads)->default_value(1), "Number of threads to execute (for multithreaded computes)")
		("block_size", value<unsigned int>(&block_size)->default_value(128), "Block size for GPU computes")
		("quiet,q", value<bool>(&quiet)->default_value(false)->zero_tokens(), "Only output time per computation")
		("sort", value<bool>(&sort_particles)->default_value(true), "Sort particles with SFCPACK")
		("profile", value<bool>(&profile_compute)->default_value(true), "Profile GFLOPS and GB/s sustained")
		("half_nlist", value<bool>(&half_nlist)->default_value(true), "Only store 1/2 of the neighbors (optimization for some pair force computes")
		("nsec", value<unsigned int>(&nsec)->default_value(10), "Number of seconds to profile for")
		#ifdef USE_CUDA
		("fc_name,f", value<string>(&fc_name)->default_value("LJ.GPU"), "ForceCompute to benchmark")
		#else
		("fc_name,f", value<string>(&fc_name)->default_value("LJ"), "ForceCompute to benchmark")
		#endif
		;

	// parse the command line
	variables_map vm;
	try
		{
		store(parse_command_line(argc, argv, desc), vm);
		notify(vm);
		}
	catch (std::logic_error e)
		{
		// print help on error
		cerr << "Error parsing command line: " << e.what() << endl;
		cout << desc;
		cout << "Available ForceComputes are: ";
		cout << "LJ, LJ.Threads, Bond ";
		#ifdef USE_CUDA
		cout << "LJ.GPU, and Bond.GPU" << endl;
		#else
		cout << endl;
		#endif

		exit(1);
		}

	if (!quiet)
		output_version_info(false);

	// if help is specified, print it
	if (vm.count("help"))
		{
		cout << desc;
		exit(1);
		}

	// initialize the particle data
	if (!quiet)
		cout << "Building particle data..." << endl;
	shared_ptr<ParticleData> pdata = init_pdata();

	// initialize the neighbor list
	if (!quiet)
		cout << "Building neighbor list data..." << endl;
	shared_ptr<NeighborList> nlist(new BinnedNeighborList(pdata, r_cut, r_buff));
	if (half_nlist)
		nlist->setStorageMode(NeighborList::half);
	else
		nlist->setStorageMode(NeighborList::full);
	nlist->setEvery(1000000000);
	nlist->forceUpdate();
	nlist->compute(1);

	// count the average number of neighbors
	int64_t neigh_count = 0;
	vector< vector< unsigned int> > list = nlist->getList();
	for (unsigned int i = 0; i < N; i++)
		neigh_count += list[i].size();
	double avgNneigh = double(neigh_count) / double(N);

	// initialize the force compute
	shared_ptr<ForceCompute> fc = init_force_compute(fc_name, pdata, nlist);
	if (fc == NULL)
		{
		cerr << "Unrecognized force compute: " << fc_name << endl;
		exit(1);
		}

	if (!quiet)
		{
		cout << "Starting benchmarking of: " << fc_name << endl;
		cout << "nsec = " << nsec << " / N = " << N << " / phi_p = "<< phi_p << endl;
		cout << "sort_particles = " << sort_particles << " / half_nlist = " << half_nlist << endl;
		cout << "r_cut = " << r_cut << " / r_buff = " << r_buff << " / avg_n_neigh = " << avgNneigh << endl;
		cout << "nthreads = " << nthreads << " / block_size = " << block_size << endl;
		}
	benchmark(fc);

	return 0;
	}
<commit_msg>Added dot printing to force compute benchmark to tell if it is running or not.<commit_after>/*
Highly Optimized Object-Oriented Molecular Dynamics (HOOMD) Open
Source Software License
Copyright (c) 2008 Ames Laboratory Iowa State University
All rights reserved.

Redistribution and use of HOOMD, in source and binary forms, with or
without modification, are permitted, provided that the following
conditions are met:

* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names HOOMD's
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.

Disclaimer

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND
CONTRIBUTORS ``AS IS''  AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS  BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
*/

// $Id$
// $URL$

/*! \file force_compute_bmark.cc
	\brief Executable for benchmarking all of the various ForceCompute classes
	\details This is intended as a quick test to check the performance of ForceComputes
		on various machines and after performance tweaks have been made. It allows for a
		number of command line options to change settings, but will most likely just
		be run with the default settings most of the time.
	\ingroup benchmarks
*/

#ifdef WIN32
#include <windows.h>
#endif

#include <stdlib.h>
#include <iostream>
#include <iomanip>
#include <string>

#include "BondForceCompute.h"
#include "LJForceCompute.h"

#include "BinnedNeighborList.h"
#include "Initializers.h"
#include "SFCPackUpdater.h"

#ifdef USE_CUDA
#include "LJForceComputeGPU.h"
#include "BondForceComputeGPU.h"
#endif

#include "HOOMDVersion.h"

#include <boost/program_options.hpp>
#include <boost/shared_ptr.hpp>

using namespace boost::program_options;
using namespace boost;
using namespace std;

// options from command line
//! quite mode: only display time per compute and nothing else
bool quiet = false;
//! number of threads (for applicable computes)
unsigned int nthreads = 1;
//! cutoff radius for pair forces (for applicable computes)
Scalar r_cut = 3.0;
//! buffer radius for pair forces (for applicable computes)
Scalar r_buff = 0.8;
//! block size for calculation (for applicable computes)
unsigned int block_size = 128;
//! number of particles to benchmark
unsigned int N = 100000;
//! Should the particles be sorted with SFCPACK?
bool sort_particles = true;
//! number of seconds to average over
unsigned int nsec = 10;
//! Activate profiling?
bool profile_compute = true;
//! Use half neighborlist?
bool half_nlist = true;
//! Specify packing fraction of particles for the benchmark (if applicable)
Scalar phi_p = 0.2;

//! Helper function to initialize bonds
void init_bond_tables(shared_ptr<BondForceCompute> bf_compute)
	{
	const unsigned int nbonds = 2;
	for (unsigned int j = 0; j < N-nbonds/2; j++)
		{
		for (unsigned int k = 1; k <= nbonds/2; k++)
			bf_compute->addBond(j,j+k);
		}
	}

//! Initialize the force compute from a string selecting it
shared_ptr<ForceCompute> init_force_compute(const string& fc_name, shared_ptr<ParticleData> pdata, shared_ptr<NeighborList> nlist)
	{
	shared_ptr<ForceCompute> result;

	// handle creation of the various lennard=jones computes
	if (fc_name == "LJ")
		result = shared_ptr<ForceCompute>(new LJForceCompute(pdata, nlist, r_cut));
	#ifdef USE_CUDA
	if (fc_name == "LJ.GPU")
		{
		shared_ptr<LJForceComputeGPU> tmp = shared_ptr<LJForceComputeGPU>(new LJForceComputeGPU(pdata, nlist, r_cut));
		tmp->setBlockSize(block_size);
		result = tmp;
		}
	#endif

	// handle the various bond force computes
	if (fc_name == "Bond")
		{
		shared_ptr<BondForceCompute> tmp = shared_ptr<BondForceCompute>(new BondForceCompute(pdata, 150, 1.0));
		init_bond_tables(tmp);
		result = tmp;
		}
	#ifdef USE_CUDA
	if (fc_name == "Bond.GPU")
		{
		shared_ptr<BondForceComputeGPU> tmp = shared_ptr<BondForceComputeGPU>(new BondForceComputeGPU(pdata, 150, 1.0));
		init_bond_tables(tmp);
		tmp->setBlockSize(block_size);
		result = tmp;
		}
	#endif

	return result;
	}

//! Initializes the particle data to a random set of particles
shared_ptr<ParticleData> init_pdata()
	{
	RandomInitializer rand_init(N, phi_p, 0.0, "A");
	shared_ptr<ParticleData> pdata(new ParticleData(rand_init));
	if (sort_particles)
		{
		SFCPackUpdater sorter(pdata, Scalar(1.0));
		sorter.update(0);
		}

	return pdata;
	}

//! Actually performs the benchmark on the preconstructed force compute
void benchmark(shared_ptr<ForceCompute> fc)
	{
	// initialize profiling if requested
	shared_ptr<Profiler> prof(new Profiler());
	if (profile_compute && !quiet)
		fc->setProfiler(prof);

	// timer to count how long we spend at each point
	ClockSource clk;
	int count = 2;

	// do a warmup run so memory allocations don't change the benchmark numbers
	fc->compute(count++);

	int64_t tstart = clk.getTime();
	int64_t tend;
	// do at least one test, then repeat until we get at least 5s of data at this point
	int nrepeat = 0;
	do
		{
		fc->compute(count++);
		nrepeat++;
		tend = clk.getTime();

		if (nrepeat % 500 == 0)
			{
			cout << ".";
			cout.flush();
			}

		} while((tend - tstart) < int64_t(nsec) * int64_t(1000000000) || nrepeat < 5);

	// make sure all kernels have been executed when using CUDA
	#ifdef USE_CUDA
	cudaThreadSynchronize();
	#endif
	tend = clk.getTime();

	if (!quiet)
		cout << *prof << endl;

	double avgTime = double(tend - tstart)/1e9/double(nrepeat);;
	cout << setprecision(7) << avgTime << " s/step" << endl;
	}

//! Parses the command line and runs the benchmark
int main(int argc, char **argv)
	{

	// the name of the ForceCompute to benchmark (gotten from user)
	string fc_name;

	options_description desc("Allowed options");
	desc.add_options()
		("help,h", "Produce help message")
		("nparticles,N", value<unsigned int>(&N)->default_value(64000), "Number of particles")
		("phi_p", value<Scalar>(&phi_p)->default_value(0.2), "Volume fraction of particles in test system")
		("r_cut", value<Scalar>(&r_cut)->default_value(3.0), "Cutoff radius for pair force sum")
		("r_buff", value<Scalar>(&r_buff)->default_value(0.8), "Buffer radius for pair force sum")
		("nthreads,t", value<unsigned int>(&nthreads)->default_value(1), "Number of threads to execute (for multithreaded computes)")
		("block_size", value<unsigned int>(&block_size)->default_value(128), "Block size for GPU computes")
		("quiet,q", value<bool>(&quiet)->default_value(false)->zero_tokens(), "Only output time per computation")
		("sort", value<bool>(&sort_particles)->default_value(true), "Sort particles with SFCPACK")
		("profile", value<bool>(&profile_compute)->default_value(true), "Profile GFLOPS and GB/s sustained")
		("half_nlist", value<bool>(&half_nlist)->default_value(true), "Only store 1/2 of the neighbors (optimization for some pair force computes")
		("nsec", value<unsigned int>(&nsec)->default_value(10), "Number of seconds to profile for")
		#ifdef USE_CUDA
		("fc_name,f", value<string>(&fc_name)->default_value("LJ.GPU"), "ForceCompute to benchmark")
		#else
		("fc_name,f", value<string>(&fc_name)->default_value("LJ"), "ForceCompute to benchmark")
		#endif
		;

	// parse the command line
	variables_map vm;
	try
		{
		store(parse_command_line(argc, argv, desc), vm);
		notify(vm);
		}
	catch (std::logic_error e)
		{
		// print help on error
		cerr << "Error parsing command line: " << e.what() << endl;
		cout << desc;
		cout << "Available ForceComputes are: ";
		cout << "LJ, LJ.Threads, Bond ";
		#ifdef USE_CUDA
		cout << "LJ.GPU, and Bond.GPU" << endl;
		#else
		cout << endl;
		#endif

		exit(1);
		}

	if (!quiet)
		output_version_info(false);

	// if help is specified, print it
	if (vm.count("help"))
		{
		cout << desc;
		exit(1);
		}

	// initialize the particle data
	if (!quiet)
		cout << "Building particle data..." << endl;
	shared_ptr<ParticleData> pdata = init_pdata();

	// initialize the neighbor list
	if (!quiet)
		cout << "Building neighbor list data..." << endl;
	shared_ptr<NeighborList> nlist(new BinnedNeighborList(pdata, r_cut, r_buff));
	if (half_nlist)
		nlist->setStorageMode(NeighborList::half);
	else
		nlist->setStorageMode(NeighborList::full);
	nlist->setEvery(1000000000);
	nlist->forceUpdate();
	nlist->compute(1);

	// count the average number of neighbors
	int64_t neigh_count = 0;
	vector< vector< unsigned int> > list = nlist->getList();
	for (unsigned int i = 0; i < N; i++)
		neigh_count += list[i].size();
	double avgNneigh = double(neigh_count) / double(N);

	// initialize the force compute
	shared_ptr<ForceCompute> fc = init_force_compute(fc_name, pdata, nlist);
	if (fc == NULL)
		{
		cerr << "Unrecognized force compute: " << fc_name << endl;
		exit(1);
		}

	if (!quiet)
		{
		cout << "Starting benchmarking of: " << fc_name << endl;
		cout << "nsec = " << nsec << " / N = " << N << " / phi_p = "<< phi_p << endl;
		cout << "sort_particles = " << sort_particles << " / half_nlist = " << half_nlist << endl;
		cout << "r_cut = " << r_cut << " / r_buff = " << r_buff << " / avg_n_neigh = " << avgNneigh << endl;
		cout << "nthreads = " << nthreads << " / block_size = " << block_size << endl;
		}
	benchmark(fc);

	return 0;
	}
<|endoftext|>


<commit_before>
//
// Cint macro to test I/O of mathcore Lorentz Vectors in a Tree and compare with a
// TLorentzVector. A ROOT tree is written and read in both using either a XYZTVector or /// a TLorentzVector.
//
//  To execute the macro type in:
//
// root[0]: .x  mathcoreVectorIO.C

#include "TRandom3.h"
#include "TStopwatch.h"
#include "TSystem.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1D.h"
#include "TCanvas.h"

#include <iostream>

#include "TLorentzVector.h"

#include "Math/Vector4D.h"
#include "Math/Vector3D.h"
#include "Math/Point3D.h"

#define DEBUG

//#define READONLY

//#define USE_REFLEX
#ifdef USE_REFLEX
#include "Cintex/Cintex.h"
#include "Reflex/Reflex.h"
#endif

#define DEFVECTOR4D(TYPE) \
typedef TYPE Vector4D; \
const std::string vector4d_type = #TYPE ;

#define DEFVECTOR3D(TYPE) \
typedef TYPE Vector3D; \
const std::string vector3d_type = #TYPE ;

#define DEFPOINT3D(TYPE) \
typedef TYPE Point3D; \
const std::string point3d_type = #TYPE ;

//const double tol = 1.0E-16;
const double tol = 1.0E-6; // or doublr 32 or float

DEFVECTOR4D(ROOT::Math::LorentzVector<ROOT::Math::PxPyPzE4D<Double32_t> >);

DEFVECTOR3D(ROOT::Math::DisplacementVector3D<ROOT::Math::Cartesian3D<Double32_t> >);

DEFPOINT3D(ROOT::Math::PositionVector3D<ROOT::Math::Cartesian3D<Double32_t> >);

// DEFVECTOR4D(ROOT::Math::LorentzVector<ROOT::Math::PxPyPzE4D<Double32_t> >)

//using namespace ROOT::Math;

template<class Vector>
inline double getMag2(const Vector & v) {
  return v.mag2();
}

inline double getMag2(const TVector3 & v) {
  return v.Mag2();
}

inline double getMag2(const TLorentzVector & v) {
  return v.Mag2();
}

template<class U>
inline void setValues(ROOT::Math::DisplacementVector3D<U> & v, const double x[] ) {
    v.SetXYZ(x[0],x[1],x[2]);
}

template<class U>
inline void setValues(ROOT::Math::PositionVector3D<U> & v, const double x[]) {
    v.SetXYZ(x[0],x[1],x[2]);
}

template<class U>
inline void setValues(ROOT::Math::LorentzVector<U> & v, const double x[]) {
    v.SetXYZT(x[0],x[1],x[2],x[3]);
}
// specialization for T -classes
inline void setValues(TVector3 & v, const double x[]) {
    v.SetXYZ(x[0],x[1],x[2]);
}
inline void setValues(TLorentzVector & v, const double x[]) {
    v.SetXYZT(x[0],x[1],x[2],x[3]);
}

template <class Vector>
double testDummy(int n) {

  TRandom3 R(111);

  TStopwatch timer;

  Vector v1;
  double s = 0;
  double p[4];

  timer.Start();
  for (int i = 0; i < n; ++i) {
        p[0] = R.Gaus(0,10);
	p[1] = R.Gaus(0,10);
	p[2] = R.Gaus(0,10);
	p[3] = R.Gaus(100,10);
	setValues(v1,p);
	s += getMag2(v1);
  }

  timer.Stop();

  double sav = std::sqrt(s/double(n));

  std::cout << " Time for Random gen " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);

  return sav;
}

//----------------------------------------------------------------
/// writing
//----------------------------------------------------------------

template<class Vector>
double write(int n, const std::string & file_name, const std::string & vector_type, int compress = 0) {

  TStopwatch timer;

  TRandom3 R(111);

  std::cout << "writing a tree with " << vector_type << std::endl;

  std::string fname = file_name + ".root";
  TFile f1(fname.c_str(),"RECREATE","",compress);

  // create tree
  std::string tree_name="Tree with" + vector_type;
  TTree t1("t1",tree_name.c_str());

  Vector *v1 = new Vector();
  std::cout << "typeID written : " << typeid(*v1).name() << std::endl;

  t1.Branch("Vector branch",vector_type.c_str(),&v1);

  timer.Start();
  double p[4];
  double s = 0;
  for (int i = 0; i < n; ++i) {
        p[0] = R.Gaus(0,10);
	p[1] = R.Gaus(0,10);
	p[2] = R.Gaus(0,10);
	p[3] = R.Gaus(100,10);
	//CylindricalEta4D<double> & c = v1->Coordinates();
	//c.SetValues(Px,pY,pZ,E);
	setValues(*v1,p);
	t1.Fill();
	s += getMag2(*v1);
  }

  f1.Write();
  timer.Stop();

  double sav = std::sqrt(s/double(n));

#ifdef DEBUG
  t1.Print();
  std::cout << " Time for Writing " << file_name << "\t: " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);
#endif

  return sav;
}

//----------------------------------------------------------------
/// reading
//----------------------------------------------------------------

template<class Vector>
double read(const std::string & file_name) {

  TStopwatch timer;

  std::string fname = file_name + ".root";

  TFile f1(fname.c_str());
  if (f1.IsZombie() ) {
    std::cout << " Error opening file " << file_name << std::endl;
    return -1;
  }

  //TFile f1("mathcoreVectorIO_D32.root");

  // create tree
  TTree *t1 = (TTree*)f1.Get("t1");

  Vector *v1 = 0;

  std::cout << "reading typeID  : " << typeid(*v1).name() << std::endl;

  t1->SetBranchAddress("Vector branch",&v1);

  timer.Start();
  int n = (int) t1->GetEntries();
  std::cout << " Tree Entries " << n << std::endl;
  double s=0;
  for (int i = 0; i < n; ++i) {
    t1->GetEntry(i);
    s += getMag2(*v1);
  }

  timer.Stop();

  double sav = std::sqrt(s/double(n));

#ifdef DEBUG
  std::cout << " Time for Reading " << file_name << "\t: " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);
#endif

  return sav;
}

int testResult(double w1, double r1, const std::string & type) {

  int iret = 0;

  if ( fabs(w1-r1)  > tol) {
    std::cout << "\nERROR: Differeces found  when reading " << std::endl;
    int pr = std::cout.precision(18);  std::cout << w1 << "   !=    " << r1 << std::endl; std::cout.precision(pr);
    iret = -1;
  }

  std::cout << "\n*********************************************************************************************\n";
  std::cout << "Test :\t " << type << "\t\t";
  if (iret ==0)
    std::cout << "OK" << std::endl;
  else
    std::cout << "FAILED" << std::endl;
  std::cout << "********************************************************************************************\n\n";

  return iret;
}

int testVectorIO() {

  int iret = 0;

#ifdef __CINT__
  gSystem->Load("libMathCore");
  gSystem->Load("libPhysics");
  using namespace ROOT::Math;
#endif

#ifdef USE_REFLEX

#ifdef __CINT__
  gSystem->Load("libReflex");
  gSystem->Load("libCintex");
#endif

  ROOT::Cintex::Cintex::SetDebug(1);
  ROOT::Cintex::Cintex::Enable();

  std::cout << "Use Reflex dictionary " << std::endl;

  //iret |= gSystem->Load("libSmatrixRflx");
  //iret |= gSystem->Load("libMathAddRflx");
  iret |= gSystem->Load("libMathRflx");
  if (iret |= 0) {
    std::cerr <<"Failing to Load Reflex dictionaries " << std::endl;
    return iret;
  }

#endif

  int nEvents = 100000;

  double w1, r1 = 0;

  testDummy<Vector4D>(nEvents);

#ifdef READONLY
  w1 = 98.995527276968474;
#else
  w1 = write<Vector4D>(nEvents,"lorentzvector",vector4d_type);
#endif
  r1 = read<Vector4D>("lorentzvector");
  iret |= testResult(w1,r1,vector4d_type);

#ifdef READONLY
  w1 = 17.3281570633214095;
#else
  w1 = write<Vector3D>(nEvents,"displacementvector",vector3d_type);
#endif
  r1 = read<Vector3D>("displacementvector");
  iret |= testResult(w1,r1,vector3d_type);

#ifndef READONLY
  w1 = write<Point3D>(nEvents,"positionvector",point3d_type);
#endif
  r1 = read<Point3D>("positionvector");
  iret |= testResult(w1,r1,point3d_type);

  return iret;
}

int main() {

  int iret =  testVectorIO();
  if (iret != 0)
    std::cerr << "testVectorIO:\t FAILED ! " << std::endl;
  else
    std::cerr << "testVectorIO:\t OK ! " << std::endl;

  return iret;

}

<commit_msg>add test for reading file written with prev version<commit_after>
//
// Cint macro to test I/O of mathcore Lorentz Vectors in a Tree and compare with a
// TLorentzVector. A ROOT tree is written and read in both using either a XYZTVector or /// a TLorentzVector.
//
//  To execute the macro type in:
//
// root[0]: .x  mathcoreVectorIO.C

#include "TRandom3.h"
#include "TStopwatch.h"
#include "TSystem.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1D.h"
#include "TCanvas.h"

#include <iostream>

#include "TLorentzVector.h"

#include "Math/Vector4D.h"
#include "Math/Vector3D.h"
#include "Math/Point3D.h"

#define DEBUG

//#define READONLY

//#define USE_REFLEX
#ifdef USE_REFLEX
#include "Cintex/Cintex.h"
#include "Reflex/Reflex.h"
#endif

#define DEFVECTOR4D(TYPE) \
typedef TYPE Vector4D; \
const std::string vector4d_type = #TYPE ;

#define DEFVECTOR3D(TYPE) \
typedef TYPE Vector3D; \
const std::string vector3d_type = #TYPE ;

#define DEFPOINT3D(TYPE) \
typedef TYPE Point3D; \
const std::string point3d_type = #TYPE ;

//const double tol = 1.0E-16;
const double tol = 1.0E-6; // or doublr 32 or float

DEFVECTOR4D(ROOT::Math::LorentzVector<ROOT::Math::PxPyPzE4D<Double32_t> >);

DEFVECTOR3D(ROOT::Math::DisplacementVector3D<ROOT::Math::Cartesian3D<Double32_t> >);

DEFPOINT3D(ROOT::Math::PositionVector3D<ROOT::Math::Cartesian3D<Double32_t> >);

// DEFVECTOR4D(ROOT::Math::LorentzVector<ROOT::Math::PxPyPzE4D<Double32_t> >)

//using namespace ROOT::Math;

template<class Vector>
inline double getMag2(const Vector & v) {
  return v.mag2();
}

inline double getMag2(const TVector3 & v) {
  return v.Mag2();
}

inline double getMag2(const TLorentzVector & v) {
  return v.Mag2();
}

template<class U>
inline void setValues(ROOT::Math::DisplacementVector3D<U> & v, const double x[] ) {
    v.SetXYZ(x[0],x[1],x[2]);
}

template<class U>
inline void setValues(ROOT::Math::PositionVector3D<U> & v, const double x[]) {
    v.SetXYZ(x[0],x[1],x[2]);
}

template<class U>
inline void setValues(ROOT::Math::LorentzVector<U> & v, const double x[]) {
    v.SetXYZT(x[0],x[1],x[2],x[3]);
}
// specialization for T -classes
inline void setValues(TVector3 & v, const double x[]) {
    v.SetXYZ(x[0],x[1],x[2]);
}
inline void setValues(TLorentzVector & v, const double x[]) {
    v.SetXYZT(x[0],x[1],x[2],x[3]);
}

template <class Vector>
double testDummy(int n) {

  TRandom3 R(111);

  TStopwatch timer;

  Vector v1;
  double s = 0;
  double p[4];

  timer.Start();
  for (int i = 0; i < n; ++i) {
        p[0] = R.Gaus(0,10);
	p[1] = R.Gaus(0,10);
	p[2] = R.Gaus(0,10);
	p[3] = R.Gaus(100,10);
	setValues(v1,p);
	s += getMag2(v1);
  }

  timer.Stop();

  double sav = std::sqrt(s/double(n));

  std::cout << " Time for Random gen " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);

  return sav;
}

//----------------------------------------------------------------
/// writing
//----------------------------------------------------------------

template<class Vector>
double write(int n, const std::string & file_name, const std::string & vector_type, int compress = 0) {

  TStopwatch timer;

  TRandom3 R(111);

  std::cout << "writing a tree with " << vector_type << std::endl;

  std::string fname = file_name + ".root";
  TFile f1(fname.c_str(),"RECREATE","",compress);

  // create tree
  std::string tree_name="Tree with" + vector_type;
  TTree t1("t1",tree_name.c_str());

  Vector *v1 = new Vector();
  std::cout << "typeID written : " << typeid(*v1).name() << std::endl;

  t1.Branch("Vector branch",vector_type.c_str(),&v1);

  timer.Start();
  double p[4];
  double s = 0;
  for (int i = 0; i < n; ++i) {
        p[0] = R.Gaus(0,10);
	p[1] = R.Gaus(0,10);
	p[2] = R.Gaus(0,10);
	p[3] = R.Gaus(100,10);
	//CylindricalEta4D<double> & c = v1->Coordinates();
	//c.SetValues(Px,pY,pZ,E);
	setValues(*v1,p);
	t1.Fill();
	s += getMag2(*v1);
  }

  f1.Write();
  timer.Stop();

  double sav = std::sqrt(s/double(n));

#ifdef DEBUG
  t1.Print();
  std::cout << " Time for Writing " << file_name << "\t: " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);
#endif

  return sav;
}

//----------------------------------------------------------------
/// reading
//----------------------------------------------------------------

template<class Vector>
double read(const std::string & file_name) {

  TStopwatch timer;

  std::string fname = file_name + ".root";

  TFile f1(fname.c_str());
  if (f1.IsZombie() ) {
    std::cout << " Error opening file " << file_name << std::endl;
    return -1;
  }

  //TFile f1("mathcoreVectorIO_D32.root");

  // create tree
  TTree *t1 = (TTree*)f1.Get("t1");

  Vector *v1 = 0;

  std::cout << "reading typeID  : " << typeid(*v1).name() << std::endl;

  t1->SetBranchAddress("Vector branch",&v1);

  timer.Start();
  int n = (int) t1->GetEntries();
  std::cout << " Tree Entries " << n << std::endl;
  double s=0;
  for (int i = 0; i < n; ++i) {
    t1->GetEntry(i);
    s += getMag2(*v1);
  }

  timer.Stop();

  double sav = std::sqrt(s/double(n));

#ifdef DEBUG
  std::cout << " Time for Reading " << file_name << "\t: " << timer.RealTime() << "  " << timer.CpuTime() << std::endl;
  int pr = std::cout.precision(18);  std::cout << "Average : " << sav << std::endl;   std::cout.precision(pr);
#endif

  return sav;
}

int testResult(double w1, double r1, const std::string & type) {

  int iret = 0;

  if ( fabs(w1-r1)  > tol) {
    std::cout << "\nERROR: Differeces found  when reading " << std::endl;
    int pr = std::cout.precision(18);  std::cout << w1 << "   !=    " << r1 << std::endl; std::cout.precision(pr);
    iret = -1;
  }

  std::cout << "\n*********************************************************************************************\n";
  std::cout << "Test :\t " << type << "\t\t";
  if (iret ==0)
    std::cout << "OK" << std::endl;
  else
    std::cout << "FAILED" << std::endl;
  std::cout << "********************************************************************************************\n\n";

  return iret;
}

int testVectorIO(bool readOnly = false) {

  int iret = 0;

#ifdef __CINT__
  gSystem->Load("libMathCore");
  gSystem->Load("libPhysics");
  using namespace ROOT::Math;
#endif

#ifdef USE_REFLEX

#ifdef __CINT__
  gSystem->Load("libReflex");
  gSystem->Load("libCintex");
#endif

  ROOT::Cintex::Cintex::SetDebug(1);
  ROOT::Cintex::Cintex::Enable();

  std::cout << "Use Reflex dictionary " << std::endl;

  //iret |= gSystem->Load("libSmatrixRflx");
  //iret |= gSystem->Load("libMathAddRflx");
  iret |= gSystem->Load("libMathRflx");
  if (iret |= 0) {
    std::cerr <<"Failing to Load Reflex dictionaries " << std::endl;
    return iret;
  }

#endif   // endif on using reflex

  int nEvents = 100000;

  double w1, r1 = 0;
  std::string fname;

  testDummy<Vector4D>(nEvents);

  fname = "lorentzvector";
  if (readOnly) {
     w1 = 98.995527276968474;
     fname += "_prev";
  }
  else
     w1 = write<Vector4D>(nEvents,fname,vector4d_type);

  r1 = read<Vector4D>(fname);
  iret |= testResult(w1,r1,vector4d_type);

  fname = "displacementvector";
  if (readOnly) {
     w1 = 17.3281570633214095;
     fname += "_prev";
  }
  else
     w1 = write<Vector3D>(nEvents,fname,vector3d_type);

  r1 = read<Vector3D>(fname);
  iret |= testResult(w1,r1,vector3d_type);

  fname = "positionvector";
  if (readOnly)
     fname += "_prev";
  else
     w1 = write<Point3D>(nEvents,fname,point3d_type);

  r1 = read<Point3D>(fname);
  iret |= testResult(w1,r1,point3d_type);

  return iret;
}

int main(int argc, const char * ) {

   bool readOnly = false;
   if (argc > 1) readOnly = true;

   int iret =  testVectorIO(readOnly);
   if (iret != 0)
      std::cerr << "testVectorIO:\t FAILED ! " << std::endl;
   else
      std::cerr << "testVectorIO:\t OK ! " << std::endl;

   return iret;

}

<|endoftext|>


<commit_before>#include <io/Socket.hpp>
#include <io/ServerSocket.hpp>
#include <io/StringWriter.hpp>
#include <test.h>

#include <unistd.h>
#include <sys/wait.h>

#include <cstdlib>

using namespace io;
using namespace std;

int server(uint16_t port) {
    StreamServerSocket* server_sock = StreamServerSocket::createAndBindListen(port);

    TEST_BOOL("createBindAndListen", server_sock != NULL);
    LOG("%s","Server socket opened\n");

    StreamSocket* sock = server_sock->accept();

    StringWriter writer;

    writer.stealBaseIO( (BaseIO*&)sock );
    writer.printf("Test%d", 0);

    return 0;
}

int client(uint16_t port) {
    StreamSocket* sock = new StreamSocket();
    int rc = sock->connect("localhost", port);
    TEST_EQ_INT( "connect", rc, 0 );

    char test[4096];
    int size = sock->read( (byte*)test, 4096 - 1);
    TEST_BOOL( "read", size > 0 );
    test[size]=0;
    TEST_EQ( "strncmp", strncmp(test, "Test0", size), 0 ) ;

    delete sock;
    return 0;
}

int main( int argc, char** argv ) {
    (void) argc;
    (void) argv;
    int rc;

    uint16_t port = rand() % 50000 + 5000 ;

    if( fork() == 0) {
        server(port);
    } else {
        sleep(1);
        TEST_FN( client(port) );
    }

    wait( & rc );
    TEST_EQ( "ServerExit", rc, 0 );

    return rc;
}
<commit_msg>forgot srand<commit_after>#include <io/Socket.hpp>
#include <io/ServerSocket.hpp>
#include <io/StringWriter.hpp>
#include <test.h>

#include <unistd.h>
#include <sys/wait.h>

#include <cstdlib>

using namespace io;
using namespace std;

int server(uint16_t port) {
    StreamServerSocket* server_sock = StreamServerSocket::createAndBindListen(port);

    TEST_BOOL("createBindAndListen", server_sock != NULL);
    LOG("%s","Server socket opened\n");

    StreamSocket* sock = server_sock->accept();

    StringWriter writer;

    writer.stealBaseIO( (BaseIO*&)sock );
    writer.printf("Test%d", 0);

    return 0;
}

int client(uint16_t port) {
    StreamSocket* sock = new StreamSocket();
    int rc = sock->connect("localhost", port);
    TEST_EQ_INT( "connect", rc, 0 );

    char test[4096];
    int size = sock->read( (byte*)test, 4096 - 1);
    TEST_BOOL( "read", size > 0 );
    test[size]=0;
    TEST_EQ( "strncmp", strncmp(test, "Test0", size), 0 ) ;

    delete sock;
    return 0;
}

int main( int argc, char** argv ) {
    (void) argc;
    (void) argv;
    int rc;

    srand(time(NULL));
    uint16_t port = rand() % 50000 + 5000 ;

    if( fork() == 0) {
        server(port);
    } else {
        sleep(1);
        TEST_FN( client(port) );
    }

    wait( & rc );
    TEST_EQ( "ServerExit", rc, 0 );

    return rc;
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/pacing/include/paced_sender.h"

#include <assert.h>

#include "webrtc/modules/interface/module_common_types.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/trace_event.h"

namespace {
// Time limit in milliseconds between packet bursts.
const int kMinPacketLimitMs = 5;

// Upper cap on process interval, in case process has not been called in a long
// time.
const int kMaxIntervalTimeMs = 30;

// Max time that the first packet in the queue can sit in the queue if no
// packets are sent, regardless of buffer state. In practice only in effect at
// low bitrates (less than 320 kbits/s).
const int kMaxQueueTimeWithoutSendingMs = 30;

}  // namespace

namespace webrtc {

namespace paced_sender {
struct Packet {
  Packet(uint32_t ssrc, uint16_t seq_number, int64_t capture_time_ms,
         int length_in_bytes, bool retransmission)
      : ssrc_(ssrc),
        sequence_number_(seq_number),
        capture_time_ms_(capture_time_ms),
        bytes_(length_in_bytes),
        retransmission_(retransmission) {
  }
  uint32_t ssrc_;
  uint16_t sequence_number_;
  int64_t capture_time_ms_;
  int bytes_;
  bool retransmission_;
};

// STL list style class which prevents duplicates in the list.
class PacketList {
 public:
  PacketList() {};

  bool empty() const {
    return packet_list_.empty();
  }

  Packet front() const {
    return packet_list_.front();
  }

  void pop_front() {
    Packet& packet = packet_list_.front();
    uint16_t sequence_number = packet.sequence_number_;
    packet_list_.pop_front();
    sequence_number_set_.erase(sequence_number);
  }

  void push_back(const Packet& packet) {
    if (sequence_number_set_.find(packet.sequence_number_) ==
        sequence_number_set_.end()) {
      // Don't insert duplicates.
      packet_list_.push_back(packet);
      sequence_number_set_.insert(packet.sequence_number_);
    }
  }

 private:
  std::list<Packet> packet_list_;
  std::set<uint16_t> sequence_number_set_;
};

class IntervalBudget {
 public:
  explicit IntervalBudget(int initial_target_rate_kbps)
      : target_rate_kbps_(initial_target_rate_kbps),
        bytes_remaining_(0) {}

  void set_target_rate_kbps(int target_rate_kbps) {
    target_rate_kbps_ = target_rate_kbps;
  }

  void IncreaseBudget(int delta_time_ms) {
    int bytes = target_rate_kbps_ * delta_time_ms / 8;
    if (bytes_remaining_ < 0) {
      // We overused last interval, compensate this interval.
      bytes_remaining_ = bytes_remaining_ + bytes;
    } else {
      // If we underused last interval we can't use it this interval.
      bytes_remaining_ = bytes;
    }
  }

  void UseBudget(int bytes) {
    bytes_remaining_ = std::max(bytes_remaining_ - bytes,
                                -100 * target_rate_kbps_ / 8);
  }

  int bytes_remaining() const { return bytes_remaining_; }

 private:
  int target_rate_kbps_;
  int bytes_remaining_;
};
}  // namespace paced_sender

PacedSender::PacedSender(Callback* callback, int target_bitrate_kbps,
                         float pace_multiplier)
    : callback_(callback),
      pace_multiplier_(pace_multiplier),
      enabled_(false),
      paused_(false),
      critsect_(CriticalSectionWrapper::CreateCriticalSection()),
      media_budget_(new paced_sender::IntervalBudget(
          pace_multiplier_ * target_bitrate_kbps)),
      padding_budget_(new paced_sender::IntervalBudget(0)),
      // No padding until UpdateBitrate is called.
      pad_up_to_bitrate_budget_(new paced_sender::IntervalBudget(0)),
      time_last_update_(TickTime::Now()),
      capture_time_ms_last_queued_(0),
      capture_time_ms_last_sent_(0),
      high_priority_packets_(new paced_sender::PacketList),
      normal_priority_packets_(new paced_sender::PacketList),
      low_priority_packets_(new paced_sender::PacketList) {
  UpdateBytesPerInterval(kMinPacketLimitMs);
}

PacedSender::~PacedSender() {
}

void PacedSender::Pause() {
  CriticalSectionScoped cs(critsect_.get());
  paused_ = true;
}

void PacedSender::Resume() {
  CriticalSectionScoped cs(critsect_.get());
  paused_ = false;
}

void PacedSender::SetStatus(bool enable) {
  CriticalSectionScoped cs(critsect_.get());
  enabled_ = enable;
}

bool PacedSender::Enabled() const {
  CriticalSectionScoped cs(critsect_.get());
  return enabled_;
}

void PacedSender::UpdateBitrate(int target_bitrate_kbps,
                                int max_padding_bitrate_kbps,
                                int pad_up_to_bitrate_kbps) {
  CriticalSectionScoped cs(critsect_.get());
  media_budget_->set_target_rate_kbps(pace_multiplier_ * target_bitrate_kbps);
  padding_budget_->set_target_rate_kbps(max_padding_bitrate_kbps);
  pad_up_to_bitrate_budget_->set_target_rate_kbps(pad_up_to_bitrate_kbps);
}

bool PacedSender::SendPacket(Priority priority, uint32_t ssrc,
    uint16_t sequence_number, int64_t capture_time_ms, int bytes,
    bool retransmission) {
  CriticalSectionScoped cs(critsect_.get());

  if (!enabled_) {
    return true;  // We can send now.
  }
  if (capture_time_ms < 0) {
    capture_time_ms = TickTime::MillisecondTimestamp();
  }
  if (priority != kHighPriority &&
      capture_time_ms > capture_time_ms_last_queued_) {
    capture_time_ms_last_queued_ = capture_time_ms;
    TRACE_EVENT_ASYNC_BEGIN1("webrtc_rtp", "PacedSend", capture_time_ms,
                             "capture_time_ms", capture_time_ms);
  }
  paced_sender::PacketList* packet_list = NULL;
  switch (priority) {
    case kHighPriority:
      packet_list = high_priority_packets_.get();
      break;
    case kNormalPriority:
      packet_list = normal_priority_packets_.get();
      break;
    case kLowPriority:
      packet_list = low_priority_packets_.get();
      break;
  }
  packet_list->push_back(paced_sender::Packet(ssrc, sequence_number,
                                              capture_time_ms, bytes,
                                              retransmission));
  return false;
}

int PacedSender::QueueInMs() const {
  CriticalSectionScoped cs(critsect_.get());
  int64_t now_ms = TickTime::MillisecondTimestamp();
  int64_t oldest_packet_capture_time = now_ms;
  if (!high_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        high_priority_packets_->front().capture_time_ms_);
  }
  if (!normal_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        normal_priority_packets_->front().capture_time_ms_);
  }
  if (!low_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        low_priority_packets_->front().capture_time_ms_);
  }
  return now_ms - oldest_packet_capture_time;
}

int32_t PacedSender::TimeUntilNextProcess() {
  CriticalSectionScoped cs(critsect_.get());
  int64_t elapsed_time_ms =
      (TickTime::Now() - time_last_update_).Milliseconds();
  if (elapsed_time_ms <= 0) {
    return kMinPacketLimitMs;
  }
  if (elapsed_time_ms >= kMinPacketLimitMs) {
    return 0;
  }
  return kMinPacketLimitMs - elapsed_time_ms;
}

int32_t PacedSender::Process() {
  TickTime now = TickTime::Now();
  CriticalSectionScoped cs(critsect_.get());
  int elapsed_time_ms = (now - time_last_update_).Milliseconds();
  time_last_update_ = now;
  if (!enabled_) {
    return 0;
  }
  if (!paused_) {
    if (elapsed_time_ms > 0) {
      uint32_t delta_time_ms = std::min(kMaxIntervalTimeMs, elapsed_time_ms);
      UpdateBytesPerInterval(delta_time_ms);
    }
    uint32_t ssrc;
    uint16_t sequence_number;
    int64_t capture_time_ms;
    bool retransmission;
    paced_sender::PacketList* packet_list;
    while (ShouldSendNextPacket(&packet_list)) {
      GetNextPacketFromList(packet_list, &ssrc, &sequence_number,
                            &capture_time_ms, &retransmission);
      critsect_->Leave();

      const bool success = callback_->TimeToSendPacket(ssrc, sequence_number,
                                                       capture_time_ms,
                                                       retransmission);
      critsect_->Enter();
      // If packet cannot be sent then keep it in packet list and exit early.
      // There's no need to send more packets.
      if (!success) {
        return 0;
      }
      packet_list->pop_front();
      const bool last_packet = packet_list->empty() ||
          packet_list->front().capture_time_ms_ > capture_time_ms;
      if (packet_list != high_priority_packets_.get()) {
        if (capture_time_ms > capture_time_ms_last_sent_) {
          capture_time_ms_last_sent_ = capture_time_ms;
        } else if (capture_time_ms == capture_time_ms_last_sent_ &&
                   last_packet) {
          TRACE_EVENT_ASYNC_END0("webrtc_rtp", "PacedSend", capture_time_ms);
        }
      }
    }
    if (high_priority_packets_->empty() &&
        normal_priority_packets_->empty() &&
        low_priority_packets_->empty() &&
        padding_budget_->bytes_remaining() > 0 &&
        pad_up_to_bitrate_budget_->bytes_remaining() > 0) {
      int padding_needed = std::min(
          padding_budget_->bytes_remaining(),
          pad_up_to_bitrate_budget_->bytes_remaining());
      critsect_->Leave();
      int bytes_sent = callback_->TimeToSendPadding(padding_needed);
      critsect_->Enter();
      media_budget_->UseBudget(bytes_sent);
      padding_budget_->UseBudget(bytes_sent);
      pad_up_to_bitrate_budget_->UseBudget(bytes_sent);
    }
  }
  return 0;
}

// MUST have critsect_ when calling.
void PacedSender::UpdateBytesPerInterval(uint32_t delta_time_ms) {
  media_budget_->IncreaseBudget(delta_time_ms);
  padding_budget_->IncreaseBudget(delta_time_ms);
  pad_up_to_bitrate_budget_->IncreaseBudget(delta_time_ms);
}

// MUST have critsect_ when calling.
bool PacedSender::ShouldSendNextPacket(paced_sender::PacketList** packet_list) {
  if (media_budget_->bytes_remaining() <= 0) {
    // All bytes consumed for this interval.
    // Check if we have not sent in a too long time.
    if ((TickTime::Now() - time_last_send_).Milliseconds() >
        kMaxQueueTimeWithoutSendingMs) {
      if (!high_priority_packets_->empty()) {
        *packet_list = high_priority_packets_.get();
        return true;
      }
      if (!normal_priority_packets_->empty()) {
        *packet_list = normal_priority_packets_.get();
        return true;
      }
    }
    return false;
  }
  if (!high_priority_packets_->empty()) {
    *packet_list = high_priority_packets_.get();
    return true;
  }
  if (!normal_priority_packets_->empty()) {
    *packet_list = normal_priority_packets_.get();
    return true;
  }
  if (!low_priority_packets_->empty()) {
    *packet_list = low_priority_packets_.get();
    return true;
  }
  return false;
}

void PacedSender::GetNextPacketFromList(paced_sender::PacketList* packets,
    uint32_t* ssrc, uint16_t* sequence_number, int64_t* capture_time_ms,
    bool* retransmission) {
  paced_sender::Packet packet = packets->front();
  UpdateMediaBytesSent(packet.bytes_);
  *sequence_number = packet.sequence_number_;
  *ssrc = packet.ssrc_;
  *capture_time_ms = packet.capture_time_ms_;
  *retransmission = packet.retransmission_;
}

// MUST have critsect_ when calling.
void PacedSender::UpdateMediaBytesSent(int num_bytes) {
  time_last_send_ = TickTime::Now();
  media_budget_->UseBudget(num_bytes);
  pad_up_to_bitrate_budget_->UseBudget(num_bytes);
}

}  // namespace webrtc
<commit_msg>Increase size of pacer window to 500 ms as that better matches the encoder.<commit_after>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/pacing/include/paced_sender.h"

#include <assert.h>

#include "webrtc/modules/interface/module_common_types.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/trace_event.h"

namespace {
// Time limit in milliseconds between packet bursts.
const int kMinPacketLimitMs = 5;

// Upper cap on process interval, in case process has not been called in a long
// time.
const int kMaxIntervalTimeMs = 30;

// Max time that the first packet in the queue can sit in the queue if no
// packets are sent, regardless of buffer state. In practice only in effect at
// low bitrates (less than 320 kbits/s).
const int kMaxQueueTimeWithoutSendingMs = 30;

}  // namespace

namespace webrtc {

namespace paced_sender {
struct Packet {
  Packet(uint32_t ssrc, uint16_t seq_number, int64_t capture_time_ms,
         int length_in_bytes, bool retransmission)
      : ssrc_(ssrc),
        sequence_number_(seq_number),
        capture_time_ms_(capture_time_ms),
        bytes_(length_in_bytes),
        retransmission_(retransmission) {
  }
  uint32_t ssrc_;
  uint16_t sequence_number_;
  int64_t capture_time_ms_;
  int bytes_;
  bool retransmission_;
};

// STL list style class which prevents duplicates in the list.
class PacketList {
 public:
  PacketList() {};

  bool empty() const {
    return packet_list_.empty();
  }

  Packet front() const {
    return packet_list_.front();
  }

  void pop_front() {
    Packet& packet = packet_list_.front();
    uint16_t sequence_number = packet.sequence_number_;
    packet_list_.pop_front();
    sequence_number_set_.erase(sequence_number);
  }

  void push_back(const Packet& packet) {
    if (sequence_number_set_.find(packet.sequence_number_) ==
        sequence_number_set_.end()) {
      // Don't insert duplicates.
      packet_list_.push_back(packet);
      sequence_number_set_.insert(packet.sequence_number_);
    }
  }

 private:
  std::list<Packet> packet_list_;
  std::set<uint16_t> sequence_number_set_;
};

class IntervalBudget {
 public:
  explicit IntervalBudget(int initial_target_rate_kbps)
      : target_rate_kbps_(initial_target_rate_kbps),
        bytes_remaining_(0) {}

  void set_target_rate_kbps(int target_rate_kbps) {
    target_rate_kbps_ = target_rate_kbps;
  }

  void IncreaseBudget(int delta_time_ms) {
    int bytes = target_rate_kbps_ * delta_time_ms / 8;
    if (bytes_remaining_ < 0) {
      // We overused last interval, compensate this interval.
      bytes_remaining_ = bytes_remaining_ + bytes;
    } else {
      // If we underused last interval we can't use it this interval.
      bytes_remaining_ = bytes;
    }
  }

  void UseBudget(int bytes) {
    bytes_remaining_ = std::max(bytes_remaining_ - bytes,
                                -500 * target_rate_kbps_ / 8);
  }

  int bytes_remaining() const { return bytes_remaining_; }

 private:
  int target_rate_kbps_;
  int bytes_remaining_;
};
}  // namespace paced_sender

PacedSender::PacedSender(Callback* callback, int target_bitrate_kbps,
                         float pace_multiplier)
    : callback_(callback),
      pace_multiplier_(pace_multiplier),
      enabled_(false),
      paused_(false),
      critsect_(CriticalSectionWrapper::CreateCriticalSection()),
      media_budget_(new paced_sender::IntervalBudget(
          pace_multiplier_ * target_bitrate_kbps)),
      padding_budget_(new paced_sender::IntervalBudget(0)),
      // No padding until UpdateBitrate is called.
      pad_up_to_bitrate_budget_(new paced_sender::IntervalBudget(0)),
      time_last_update_(TickTime::Now()),
      capture_time_ms_last_queued_(0),
      capture_time_ms_last_sent_(0),
      high_priority_packets_(new paced_sender::PacketList),
      normal_priority_packets_(new paced_sender::PacketList),
      low_priority_packets_(new paced_sender::PacketList) {
  UpdateBytesPerInterval(kMinPacketLimitMs);
}

PacedSender::~PacedSender() {
}

void PacedSender::Pause() {
  CriticalSectionScoped cs(critsect_.get());
  paused_ = true;
}

void PacedSender::Resume() {
  CriticalSectionScoped cs(critsect_.get());
  paused_ = false;
}

void PacedSender::SetStatus(bool enable) {
  CriticalSectionScoped cs(critsect_.get());
  enabled_ = enable;
}

bool PacedSender::Enabled() const {
  CriticalSectionScoped cs(critsect_.get());
  return enabled_;
}

void PacedSender::UpdateBitrate(int target_bitrate_kbps,
                                int max_padding_bitrate_kbps,
                                int pad_up_to_bitrate_kbps) {
  CriticalSectionScoped cs(critsect_.get());
  media_budget_->set_target_rate_kbps(pace_multiplier_ * target_bitrate_kbps);
  padding_budget_->set_target_rate_kbps(max_padding_bitrate_kbps);
  pad_up_to_bitrate_budget_->set_target_rate_kbps(pad_up_to_bitrate_kbps);
}

bool PacedSender::SendPacket(Priority priority, uint32_t ssrc,
    uint16_t sequence_number, int64_t capture_time_ms, int bytes,
    bool retransmission) {
  CriticalSectionScoped cs(critsect_.get());

  if (!enabled_) {
    return true;  // We can send now.
  }
  if (capture_time_ms < 0) {
    capture_time_ms = TickTime::MillisecondTimestamp();
  }
  if (priority != kHighPriority &&
      capture_time_ms > capture_time_ms_last_queued_) {
    capture_time_ms_last_queued_ = capture_time_ms;
    TRACE_EVENT_ASYNC_BEGIN1("webrtc_rtp", "PacedSend", capture_time_ms,
                             "capture_time_ms", capture_time_ms);
  }
  paced_sender::PacketList* packet_list = NULL;
  switch (priority) {
    case kHighPriority:
      packet_list = high_priority_packets_.get();
      break;
    case kNormalPriority:
      packet_list = normal_priority_packets_.get();
      break;
    case kLowPriority:
      packet_list = low_priority_packets_.get();
      break;
  }
  packet_list->push_back(paced_sender::Packet(ssrc, sequence_number,
                                              capture_time_ms, bytes,
                                              retransmission));
  return false;
}

int PacedSender::QueueInMs() const {
  CriticalSectionScoped cs(critsect_.get());
  int64_t now_ms = TickTime::MillisecondTimestamp();
  int64_t oldest_packet_capture_time = now_ms;
  if (!high_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        high_priority_packets_->front().capture_time_ms_);
  }
  if (!normal_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        normal_priority_packets_->front().capture_time_ms_);
  }
  if (!low_priority_packets_->empty()) {
    oldest_packet_capture_time = std::min(
        oldest_packet_capture_time,
        low_priority_packets_->front().capture_time_ms_);
  }
  return now_ms - oldest_packet_capture_time;
}

int32_t PacedSender::TimeUntilNextProcess() {
  CriticalSectionScoped cs(critsect_.get());
  int64_t elapsed_time_ms =
      (TickTime::Now() - time_last_update_).Milliseconds();
  if (elapsed_time_ms <= 0) {
    return kMinPacketLimitMs;
  }
  if (elapsed_time_ms >= kMinPacketLimitMs) {
    return 0;
  }
  return kMinPacketLimitMs - elapsed_time_ms;
}

int32_t PacedSender::Process() {
  TickTime now = TickTime::Now();
  CriticalSectionScoped cs(critsect_.get());
  int elapsed_time_ms = (now - time_last_update_).Milliseconds();
  time_last_update_ = now;
  if (!enabled_) {
    return 0;
  }
  if (!paused_) {
    if (elapsed_time_ms > 0) {
      uint32_t delta_time_ms = std::min(kMaxIntervalTimeMs, elapsed_time_ms);
      UpdateBytesPerInterval(delta_time_ms);
    }
    uint32_t ssrc;
    uint16_t sequence_number;
    int64_t capture_time_ms;
    bool retransmission;
    paced_sender::PacketList* packet_list;
    while (ShouldSendNextPacket(&packet_list)) {
      GetNextPacketFromList(packet_list, &ssrc, &sequence_number,
                            &capture_time_ms, &retransmission);
      critsect_->Leave();

      const bool success = callback_->TimeToSendPacket(ssrc, sequence_number,
                                                       capture_time_ms,
                                                       retransmission);
      critsect_->Enter();
      // If packet cannot be sent then keep it in packet list and exit early.
      // There's no need to send more packets.
      if (!success) {
        return 0;
      }
      packet_list->pop_front();
      const bool last_packet = packet_list->empty() ||
          packet_list->front().capture_time_ms_ > capture_time_ms;
      if (packet_list != high_priority_packets_.get()) {
        if (capture_time_ms > capture_time_ms_last_sent_) {
          capture_time_ms_last_sent_ = capture_time_ms;
        } else if (capture_time_ms == capture_time_ms_last_sent_ &&
                   last_packet) {
          TRACE_EVENT_ASYNC_END0("webrtc_rtp", "PacedSend", capture_time_ms);
        }
      }
    }
    if (high_priority_packets_->empty() &&
        normal_priority_packets_->empty() &&
        low_priority_packets_->empty() &&
        padding_budget_->bytes_remaining() > 0 &&
        pad_up_to_bitrate_budget_->bytes_remaining() > 0) {
      int padding_needed = std::min(
          padding_budget_->bytes_remaining(),
          pad_up_to_bitrate_budget_->bytes_remaining());
      critsect_->Leave();
      int bytes_sent = callback_->TimeToSendPadding(padding_needed);
      critsect_->Enter();
      media_budget_->UseBudget(bytes_sent);
      padding_budget_->UseBudget(bytes_sent);
      pad_up_to_bitrate_budget_->UseBudget(bytes_sent);
    }
  }
  return 0;
}

// MUST have critsect_ when calling.
void PacedSender::UpdateBytesPerInterval(uint32_t delta_time_ms) {
  media_budget_->IncreaseBudget(delta_time_ms);
  padding_budget_->IncreaseBudget(delta_time_ms);
  pad_up_to_bitrate_budget_->IncreaseBudget(delta_time_ms);
}

// MUST have critsect_ when calling.
bool PacedSender::ShouldSendNextPacket(paced_sender::PacketList** packet_list) {
  if (media_budget_->bytes_remaining() <= 0) {
    // All bytes consumed for this interval.
    // Check if we have not sent in a too long time.
    if ((TickTime::Now() - time_last_send_).Milliseconds() >
        kMaxQueueTimeWithoutSendingMs) {
      if (!high_priority_packets_->empty()) {
        *packet_list = high_priority_packets_.get();
        return true;
      }
      if (!normal_priority_packets_->empty()) {
        *packet_list = normal_priority_packets_.get();
        return true;
      }
    }
    return false;
  }
  if (!high_priority_packets_->empty()) {
    *packet_list = high_priority_packets_.get();
    return true;
  }
  if (!normal_priority_packets_->empty()) {
    *packet_list = normal_priority_packets_.get();
    return true;
  }
  if (!low_priority_packets_->empty()) {
    *packet_list = low_priority_packets_.get();
    return true;
  }
  return false;
}

void PacedSender::GetNextPacketFromList(paced_sender::PacketList* packets,
    uint32_t* ssrc, uint16_t* sequence_number, int64_t* capture_time_ms,
    bool* retransmission) {
  paced_sender::Packet packet = packets->front();
  UpdateMediaBytesSent(packet.bytes_);
  *sequence_number = packet.sequence_number_;
  *ssrc = packet.ssrc_;
  *capture_time_ms = packet.capture_time_ms_;
  *retransmission = packet.retransmission_;
}

// MUST have critsect_ when calling.
void PacedSender::UpdateMediaBytesSent(int num_bytes) {
  time_last_send_ = TickTime::Now();
  media_budget_->UseBudget(num_bytes);
  pad_up_to_bitrate_budget_->UseBudget(num_bytes);
}

}  // namespace webrtc
<|endoftext|>


<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <thread>
#include <stx/util/Base64.h>
#include <stx/fnv.h>
#include <stx/protobuf/msg.h>
#include <stx/io/fileutil.h>
#include <sstable/sstablereader.h>
#include <zbase/core/PartitionMap.h>
#include <zbase/core/PartitionState.pb.h>
#include <zbase/core/PartitionReplication.h>

using namespace stx;

namespace zbase {

static mdb::MDBOptions tsdb_mdb_opts() {
  mdb::MDBOptions opts;
  opts.data_filename = "index.db";
  opts.lock_filename = "index.db.lck";
  return opts;
};

PartitionMap::PartitionMap(
    const String& db_path,
    RefPtr<ReplicationScheme> repl_scheme) :
    db_path_(db_path),
    db_(mdb::MDB::open(db_path, tsdb_mdb_opts())),
    repl_scheme_(repl_scheme) {}

Option<RefPtr<Table>> PartitionMap::findTable(
    const String& stream_ns,
    const String& table_name) const {
  std::unique_lock<std::mutex> lk(mutex_);
  return findTableWithLock(stream_ns, table_name);
}

Option<RefPtr<Table>> PartitionMap::findTableWithLock(
    const String& stream_ns,
    const String& table_name) const {
  auto stream_ns_key = stream_ns + "~" + table_name;

  const auto& iter = tables_.find(stream_ns_key);
  if (iter == tables_.end()) {
    return None<RefPtr<Table>>();
  } else {
    return Some(iter->second);
  }
}

void PartitionMap::configureTable(const TableDefinition& table) {
  std::unique_lock<std::mutex> lk(mutex_);
  auto tbl_key = table.customer() + "~" + table.table_name();

  auto iter = tables_.find(tbl_key);
  if (iter == tables_.end()) {
    tables_.emplace(tbl_key, new Table(table));
  } else {
    iter->second->updateConfig(table);
  }
}

void PartitionMap::open() {
  auto txn = db_->startTransaction(false);
  auto cursor = txn->getCursor();

  Vector<PartitionKey> partitions;
  for (int i = 0; ; ++i) {
    Buffer key;
    Buffer value;
    if (i == 0) {
      if (!cursor->getFirst(&key, &value)) {
        break;
      }
    } else {
      if (!cursor->getNext(&key, &value)) {
        break;
      }
    }

    auto db_key = key.toString();
    if (db_key.size() == 0) {
      continue;
    }

    if (db_key[0] == 0x1b) {
      continue;
    }

    auto tsdb_namespace_off = StringUtil::find(db_key, '~');
    if (tsdb_namespace_off == String::npos) {
      RAISEF(kRuntimeError, "invalid partition key: $0", db_key);
    }

    auto tsdb_namespace = db_key.substr(0, tsdb_namespace_off);
    SHA1Hash partition_key(
        db_key.data() + tsdb_namespace_off + 1,
        db_key.size() - tsdb_namespace_off - 1);

    auto table_key = value.toString();
    auto table = findTableWithLock(tsdb_namespace, table_key);

    partitions_.emplace(db_key, mkScoped(new LazyPartition()));

    if (table.isEmpty()) {
      logWarning(
          "tsdb",
          "Orphaned partition: $0/$1",
          table_key,
          partition_key.toString());
      continue;
    }

    partitions.emplace_back(
        std::make_tuple(tsdb_namespace, table_key, partition_key));
  }

  cursor->close();
  txn->abort();

  auto background_load_thread = std::thread(
      std::bind(&PartitionMap::loadPartitions, this, partitions));

  background_load_thread.detach();
}

void PartitionMap::loadPartitions(const Vector<PartitionKey>& partitions) {
  for (const auto& p : partitions) {
    try {
      findPartition(std::get<0>(p), std::get<1>(p), std::get<2>(p));
    } catch (const StandardException& e) {
      logError(
          "tsdb",
          e,
          "error while loading partition $0/$1/$2",
          std::get<0>(p),
          std::get<1>(p),
          std::get<2>(p).toString());
    }
  }
}

RefPtr<Partition> PartitionMap::findOrCreatePartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  std::unique_lock<std::mutex> lk(mutex_);
  auto iter = partitions_.find(db_key);
  if (iter != partitions_.end()) {
    if (iter->second->isLoaded()) {
      return iter->second->getPartition();
    }
  }

  auto table = findTableWithLock(tsdb_namespace, table_name);
  if (table.isEmpty()) {
    RAISEF(kNotFoundError, "table not found: $0", table_name);
  }

  if (iter != partitions_.end()) {
    auto partition = iter->second.get();
    lk.unlock();

    return partition->getPartition(
        tsdb_namespace,
        table.get(),
        partition_key,
        db_path_,
        this);
  }

  auto partition = Partition::create(
      tsdb_namespace,
      table.get(),
      partition_key,
      db_path_);

  partitions_.emplace(db_key, mkScoped(new LazyPartition(partition)));

  auto txn = db_->startTransaction(false);
  txn->update(
      db_key.data(),
      db_key.size(),
      table_name.data(),
      table_name.size());

  txn->commit();
  lk.unlock();

  auto change = mkRef(new PartitionChangeNotification());
  change->partition = partition;
  publishPartitionChange(change);

  return partition;
}

Option<RefPtr<Partition>> PartitionMap::findPartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  std::unique_lock<std::mutex> lk(mutex_);
  auto iter = partitions_.find(db_key);
  if (iter == partitions_.end()) {
    return None<RefPtr<Partition>>();
  } else {
    if (iter->second->isLoaded()) {
      return Some(iter->second->getPartition());
    }

    auto table = findTableWithLock(tsdb_namespace, table_name);
    if (table.isEmpty()) {
      RAISEF(kNotFoundError, "table not found: $0", table_name);
    }

    auto partition = iter->second.get();
    lk.unlock();

    return Some(
        partition->getPartition(
            tsdb_namespace,
            table.get(),
            partition_key,
            db_path_,
            this));
  }
}

void PartitionMap::listTables(
    const String& tsdb_namespace,
    Function<void (const TSDBTableInfo& table)> fn) const {
  for (const auto& tbl : tables_) {
    if (tbl.second->tsdbNamespace() != tsdb_namespace) {
      continue;
    }

    TSDBTableInfo ti;
    ti.table_name = tbl.second->name();
    ti.config = tbl.second->config();
    ti.schema = tbl.second->schema();
    fn(ti);
  }
}

bool PartitionMap::dropLocalPartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto partition_opt = findPartition(tsdb_namespace, table_name, partition_key);
  if (partition_opt.isEmpty()) {
    RAISE(kNotFoundError, "partition not found");
  }

  auto partition = partition_opt.get();
  auto partition_writer = partition->getWriter();

  /* lock partition */
  partition_writer->lock();

  /* check preconditions */
  size_t full_copies = 0;
  try {
    full_copies = partition
        ->getReplicationStrategy(repl_scheme_, nullptr)
        ->numFullRemoteCopies();

    if (repl_scheme_->hasLocalReplica(partition_key) ||
        full_copies < repl_scheme_->minNumCopies()) {
      RAISE(kIllegalStateError, "can't delete partition");
    }
  } catch (const StandardException& e) {
    /* unlock partition and bail */
    partition_writer->unlock();
    return false;
  }

  /* start deletion */
  logInfo(
      "z1.core",
      "Partition $0/$1/$2 is not owned by this node and has $3 other " \
      "full copies, trying to unload and drop",
      tsdb_namespace,
      table_name,
      partition_key.toString(),
      full_copies);

  /* freeze partition and unlock waiting writers (they will fail) */
  partition_writer->freeze();
  partition_writer->unlock();

  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  /* grab the main lock */
  std::unique_lock<std::mutex> lk(mutex_);

  /* delete from in memory partition map */
  auto iter = partitions_.find(db_key);
  if (iter == partitions_.end()) {
    /* somebody else already deleted this partition */
    return true;
  } else {
    partitions_.erase(iter);
  }

  /* delete from on disk partition map */
  auto txn = db_->startTransaction(false);
  txn->del(db_key);
  txn->commit();

  /* delete partition data from disk (move to trash) */
  {
    auto src_path = FileUtil::joinPaths(
        db_path_,
        StringUtil::format(
            "$0/$1/$2",
            tsdb_namespace,
            SHA1::compute(table_name).toString(),
            partition_key.toString()));

    auto dst_path = FileUtil::joinPaths(
        db_path_,
        StringUtil::format(
            "../trash/$0~$1~$2~$3",
            tsdb_namespace,
            SHA1::compute(table_name).toString(),
            partition_key.toString(),
            Random::singleton()->hex64()));

    FileUtil::mv(src_path, dst_path);
  }

  return true;
}

Option<TSDBTableInfo> PartitionMap::tableInfo(
      const String& tsdb_namespace,
      const String& table_key) const {
  auto table = findTable(tsdb_namespace, table_key);
  if (table.isEmpty()) {
    return None<TSDBTableInfo>();
  }

  TSDBTableInfo ti;
  ti.table_name = table.get()->name();
  ti.config = table.get()->config();
  ti.schema = table.get()->schema();
  return Some(ti);
}

void PartitionMap::subscribeToPartitionChanges(PartitionChangeCallbackFn fn) {
  callbacks_.emplace_back(fn);
}

void PartitionMap::publishPartitionChange(
    RefPtr<PartitionChangeNotification> change) {
  for (const auto& cb : callbacks_) {
    cb(change);
  }
}

} // namespace tdsb

<commit_msg>dry run...<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <thread>
#include <stx/util/Base64.h>
#include <stx/fnv.h>
#include <stx/protobuf/msg.h>
#include <stx/io/fileutil.h>
#include <sstable/sstablereader.h>
#include <zbase/core/PartitionMap.h>
#include <zbase/core/PartitionState.pb.h>
#include <zbase/core/PartitionReplication.h>

using namespace stx;

namespace zbase {

static mdb::MDBOptions tsdb_mdb_opts() {
  mdb::MDBOptions opts;
  opts.data_filename = "index.db";
  opts.lock_filename = "index.db.lck";
  return opts;
};

PartitionMap::PartitionMap(
    const String& db_path,
    RefPtr<ReplicationScheme> repl_scheme) :
    db_path_(db_path),
    db_(mdb::MDB::open(db_path, tsdb_mdb_opts())),
    repl_scheme_(repl_scheme) {}

Option<RefPtr<Table>> PartitionMap::findTable(
    const String& stream_ns,
    const String& table_name) const {
  std::unique_lock<std::mutex> lk(mutex_);
  return findTableWithLock(stream_ns, table_name);
}

Option<RefPtr<Table>> PartitionMap::findTableWithLock(
    const String& stream_ns,
    const String& table_name) const {
  auto stream_ns_key = stream_ns + "~" + table_name;

  const auto& iter = tables_.find(stream_ns_key);
  if (iter == tables_.end()) {
    return None<RefPtr<Table>>();
  } else {
    return Some(iter->second);
  }
}

void PartitionMap::configureTable(const TableDefinition& table) {
  std::unique_lock<std::mutex> lk(mutex_);
  auto tbl_key = table.customer() + "~" + table.table_name();

  auto iter = tables_.find(tbl_key);
  if (iter == tables_.end()) {
    tables_.emplace(tbl_key, new Table(table));
  } else {
    iter->second->updateConfig(table);
  }
}

void PartitionMap::open() {
  auto txn = db_->startTransaction(false);
  auto cursor = txn->getCursor();

  Vector<PartitionKey> partitions;
  for (int i = 0; ; ++i) {
    Buffer key;
    Buffer value;
    if (i == 0) {
      if (!cursor->getFirst(&key, &value)) {
        break;
      }
    } else {
      if (!cursor->getNext(&key, &value)) {
        break;
      }
    }

    auto db_key = key.toString();
    if (db_key.size() == 0) {
      continue;
    }

    if (db_key[0] == 0x1b) {
      continue;
    }

    auto tsdb_namespace_off = StringUtil::find(db_key, '~');
    if (tsdb_namespace_off == String::npos) {
      RAISEF(kRuntimeError, "invalid partition key: $0", db_key);
    }

    auto tsdb_namespace = db_key.substr(0, tsdb_namespace_off);
    SHA1Hash partition_key(
        db_key.data() + tsdb_namespace_off + 1,
        db_key.size() - tsdb_namespace_off - 1);

    auto table_key = value.toString();
    auto table = findTableWithLock(tsdb_namespace, table_key);

    partitions_.emplace(db_key, mkScoped(new LazyPartition()));

    if (table.isEmpty()) {
      logWarning(
          "tsdb",
          "Orphaned partition: $0/$1",
          table_key,
          partition_key.toString());
      continue;
    }

    partitions.emplace_back(
        std::make_tuple(tsdb_namespace, table_key, partition_key));
  }

  cursor->close();
  txn->abort();

  auto background_load_thread = std::thread(
      std::bind(&PartitionMap::loadPartitions, this, partitions));

  background_load_thread.detach();
}

void PartitionMap::loadPartitions(const Vector<PartitionKey>& partitions) {
  for (const auto& p : partitions) {
    try {
      findPartition(std::get<0>(p), std::get<1>(p), std::get<2>(p));
    } catch (const StandardException& e) {
      logError(
          "tsdb",
          e,
          "error while loading partition $0/$1/$2",
          std::get<0>(p),
          std::get<1>(p),
          std::get<2>(p).toString());
    }
  }
}

RefPtr<Partition> PartitionMap::findOrCreatePartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  std::unique_lock<std::mutex> lk(mutex_);
  auto iter = partitions_.find(db_key);
  if (iter != partitions_.end()) {
    if (iter->second->isLoaded()) {
      return iter->second->getPartition();
    }
  }

  auto table = findTableWithLock(tsdb_namespace, table_name);
  if (table.isEmpty()) {
    RAISEF(kNotFoundError, "table not found: $0", table_name);
  }

  if (iter != partitions_.end()) {
    auto partition = iter->second.get();
    lk.unlock();

    return partition->getPartition(
        tsdb_namespace,
        table.get(),
        partition_key,
        db_path_,
        this);
  }

  auto partition = Partition::create(
      tsdb_namespace,
      table.get(),
      partition_key,
      db_path_);

  partitions_.emplace(db_key, mkScoped(new LazyPartition(partition)));

  auto txn = db_->startTransaction(false);
  txn->update(
      db_key.data(),
      db_key.size(),
      table_name.data(),
      table_name.size());

  txn->commit();
  lk.unlock();

  auto change = mkRef(new PartitionChangeNotification());
  change->partition = partition;
  publishPartitionChange(change);

  return partition;
}

Option<RefPtr<Partition>> PartitionMap::findPartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  std::unique_lock<std::mutex> lk(mutex_);
  auto iter = partitions_.find(db_key);
  if (iter == partitions_.end()) {
    return None<RefPtr<Partition>>();
  } else {
    if (iter->second->isLoaded()) {
      return Some(iter->second->getPartition());
    }

    auto table = findTableWithLock(tsdb_namespace, table_name);
    if (table.isEmpty()) {
      RAISEF(kNotFoundError, "table not found: $0", table_name);
    }

    auto partition = iter->second.get();
    lk.unlock();

    return Some(
        partition->getPartition(
            tsdb_namespace,
            table.get(),
            partition_key,
            db_path_,
            this));
  }
}

void PartitionMap::listTables(
    const String& tsdb_namespace,
    Function<void (const TSDBTableInfo& table)> fn) const {
  for (const auto& tbl : tables_) {
    if (tbl.second->tsdbNamespace() != tsdb_namespace) {
      continue;
    }

    TSDBTableInfo ti;
    ti.table_name = tbl.second->name();
    ti.config = tbl.second->config();
    ti.schema = tbl.second->schema();
    fn(ti);
  }
}

bool PartitionMap::dropLocalPartition(
    const String& tsdb_namespace,
    const String& table_name,
    const SHA1Hash& partition_key) {
  auto partition_opt = findPartition(tsdb_namespace, table_name, partition_key);
  if (partition_opt.isEmpty()) {
    RAISE(kNotFoundError, "partition not found");
  }

  auto partition = partition_opt.get();
  auto partition_writer = partition->getWriter();

  /* lock partition */
  partition_writer->lock();

  /* check preconditions */
  size_t full_copies = 0;
  try {
    full_copies = partition
        ->getReplicationStrategy(repl_scheme_, nullptr)
        ->numFullRemoteCopies();

    if (repl_scheme_->hasLocalReplica(partition_key) ||
        full_copies < repl_scheme_->minNumCopies()) {
      RAISE(kIllegalStateError, "can't delete partition");
    }
  } catch (const StandardException& e) {
    /* unlock partition and bail */
    partition_writer->unlock();
    return false;
  }

  /* start deletion */
  logInfo(
      "z1.core",
      "Partition $0/$1/$2 is not owned by this node and has $3 other " \
      "full copies, trying to unload and drop",
      tsdb_namespace,
      table_name,
      partition_key.toString(),
      full_copies);

  /* freeze partition and unlock waiting writers (they will fail) */
  partition_writer->freeze();
  partition_writer->unlock();

  /// BAIL FOR DRY RUN
  return true;
  /// EOF DRY RUN

  auto db_key = tsdb_namespace + "~";
  db_key.append((char*) partition_key.data(), partition_key.size());

  /* grab the main lock */
  std::unique_lock<std::mutex> lk(mutex_);

  /* delete from in memory partition map */
  auto iter = partitions_.find(db_key);
  if (iter == partitions_.end()) {
    /* somebody else already deleted this partition */
    return true;
  } else {
    partitions_.erase(iter);
  }

  /* delete from on disk partition map */
  auto txn = db_->startTransaction(false);
  txn->del(db_key);
  txn->commit();

  /* delete partition data from disk (move to trash) */
  {
    auto src_path = FileUtil::joinPaths(
        db_path_,
        StringUtil::format(
            "$0/$1/$2",
            tsdb_namespace,
            SHA1::compute(table_name).toString(),
            partition_key.toString()));

    auto dst_path = FileUtil::joinPaths(
        db_path_,
        StringUtil::format(
            "../trash/$0~$1~$2~$3",
            tsdb_namespace,
            SHA1::compute(table_name).toString(),
            partition_key.toString(),
            Random::singleton()->hex64()));

    FileUtil::mv(src_path, dst_path);
  }

  return true;
}

Option<TSDBTableInfo> PartitionMap::tableInfo(
      const String& tsdb_namespace,
      const String& table_key) const {
  auto table = findTable(tsdb_namespace, table_key);
  if (table.isEmpty()) {
    return None<TSDBTableInfo>();
  }

  TSDBTableInfo ti;
  ti.table_name = table.get()->name();
  ti.config = table.get()->config();
  ti.schema = table.get()->schema();
  return Some(ti);
}

void PartitionMap::subscribeToPartitionChanges(PartitionChangeCallbackFn fn) {
  callbacks_.emplace_back(fn);
}

void PartitionMap::publishPartitionChange(
    RefPtr<PartitionChangeNotification> change) {
  for (const auto& cb : callbacks_) {
    cb(change);
  }
}

} // namespace tdsb

<|endoftext|>