<commit_before>// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2005 OPeNDAP, Inc.
// Author: Nathan David Potter <ndp@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
#include <cppunit/TextTestRunner.h>
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/extensions/HelperMacros.h>

#include <cstdio>

#define DODS_DEBUG

#include "debug.h"

#include "gridfields/grid.h"
#include "gridfields/gridfield.h"
#include "gridfields/bind.h"
#include "gridfields/array.h"
#include "gridfields/restrict.h"
#include "gridfields/refrestrict.h"
#include "gridfields/arrayreader.h"
#include "gridfields/accumulate.h"

using namespace GF;

namespace ugrid {

class BindTest : public CppUnit::TestFixture {

private:

    Grid *makeGrid(int scale, string name) {
      CellArray *twocells;
      CellArray *onecells;
      CellArray *zerocells;
      Grid *grid;
      Node triangle[3];
      Node segment[2];
      Node node;

      bool wf;
      int i;
      twocells = new CellArray();
      for (i=0; i<scale/2; i++) {
        triangle[0] = i;
        triangle[1] = i+1;
        triangle[2] = i+2;
        twocells->addCellNodes(triangle, 3);
      }
      //twocells->print();
      //getchar();
      onecells = new CellArray();
      for (i=0; i<scale-1; i++) {
        segment[0] = i;
        segment[1] = i+1;
        onecells->addCellNodes(segment, 2);
      }
      //onecells->print();

      //getchar();
      grid = new Grid(name, 2);
      grid->setImplicit0Cells(scale);
      grid->setKCells(onecells, 1);
      grid->setKCells(twocells, 2);
      //grid->print(0);
      //getchar();
      return grid;
    }

    Array *makeFloatArray(int size,const char *name) {
      Array *arr;
      arr = new Array(name, FLOAT, size);
      float *data;
      arr->getData(data);
      int i;

      for (i=0; i<size; i++) {
          data[i] = 2*i-10;
      }
      return arr;
    }

    /**
     *
     */
    GridField *makeGridField(int size, string gridname,const char *datname, int k) {

      Grid *G;
      GridField *GF;
      Array *data;

      G = makeGrid(12, "A");
      k = 0;
      data = makeFloatArray(12, "x");

      GF = new GridField(G, k, data);
      //printf("Valid? %i\n", !notValid(GF));
      //GF->print();

      return GF;
    }

public:

    // Called once before everything gets tested
    BindTest() {
    //    DBG(cerr << " BindTest - Constructor" << endl);

    }

    // Called at the end of the test
    ~BindTest() {
    //    DBG(cerr << " BindTest - Destructor" << endl);
    }

    // Called before each test
    void setup() {
    //    DBG(cerr << " BindTest - setup()" << endl);
    }

    // Called after each test
    void tearDown() {
    //    DBG(cerr << " tearDown()" << endl);
    }

    CPPUNIT_TEST_SUITE( BindTest );

    CPPUNIT_TEST(bind_test);

    CPPUNIT_TEST_SUITE_END();

    void  bind_test() {
      DBG(cerr << " bind_test - BEGIN" << endl);

      try {
        GridField *GF;
        GridField *Result;

        GF = makeGridField(12, "A", "x", 0);
        Array *arr = new Array("io", FLOAT, 12);
        GF->Bind(0, arr);

        GridField *aGF = AccumulateOp::Accumulate(GF, 0, "result", "result+1", "0", 0);
        DBG(aGF->PrintTo(cerr,9));

        DBG( cerr << "restricting..." << endl);
        Result = RefRestrictOp::Restrict("x<4",0,GF);
        DBG(Result->PrintTo(cerr,0));

        Result = RefRestrictOp::Restrict("x>-4",0,Result);
        DBG(Result->PrintTo(cerr,10));

        FileArrayReader *ar = new FileArrayReader("bindtest.dat", 0);
        ar->setPatternAttribute("result");
        GridField *G = BindOp::Bind("io", FLOAT, ar, 0, Result);
        DBG(G->PrintTo(cerr,0));

        CPPUNIT_ASSERT(true);
      }
      catch (std::string &e) {
        cerr << "Error: " << e << endl;
        CPPUNIT_ASSERT(false);
      }
      catch (...) {
        cerr << "Unknown Error." << endl;
        CPPUNIT_ASSERT(false);
      }
    }

}; // BindTest

CPPUNIT_TEST_SUITE_REGISTRATION(BindTest);

} // namespace ugrid

int
main( int, char** )
{
    CppUnit::TextTestRunner runner;
    runner.addTest( CppUnit::TestFactoryRegistry::getRegistry().makeTest() );

    bool wasSuccessful = runner.run( "", false ) ;

    return wasSuccessful ? 0 : 1;
}

<commit_msg>ugrid_functions: Invrmental test changes...<commit_after>// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2005 OPeNDAP, Inc.
// Author: Nathan David Potter <ndp@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
#include <cppunit/TextTestRunner.h>
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/extensions/HelperMacros.h>

#include <cstdio>

#define DODS_DEBUG

#include "debug.h"
#include "util.h"

#include "gridfields/grid.h"
#include "gridfields/gridfield.h"
#include "gridfields/bind.h"
#include "gridfields/array.h"
#include "gridfields/restrict.h"
#include "gridfields/refrestrict.h"
#include "gridfields/arrayreader.h"
#include "gridfields/accumulate.h"

using namespace GF;

namespace ugrid {

static ArrayReader *new_makeArrayReader(double *array, int size) {
 stringstream *ss = new stringstream();
 stringbuf *pbuf;
 pbuf=ss->rdbuf();
 pbuf->sputn((char *) array, sizeof(double)*size);
 return new ArrayReader(ss);
}

class BindTest: public CppUnit::TestFixture {

private:

    Grid *makeGrid(int scale, string name) {
        CellArray *twocells;
        CellArray *onecells;
        CellArray *zerocells;
        Grid *grid;
        Node triangle[3];
        Node segment[2];
        Node node;

        bool wf;
        int i;
        twocells = new CellArray();
        for (i = 0; i < scale / 2; i++) {
            triangle[0] = i;
            triangle[1] = i + 1;
            triangle[2] = i + 2;
            twocells->addCellNodes(triangle, 3);
        }
        //twocells->print();
        //getchar();
        onecells = new CellArray();
        for (i = 0; i < scale - 1; i++) {
            segment[0] = i;
            segment[1] = i + 1;
            onecells->addCellNodes(segment, 2);
        }
        //onecells->print();

        //getchar();
        grid = new Grid(name, 2);
        grid->setImplicit0Cells(scale);
        grid->setKCells(onecells, 1);
        grid->setKCells(twocells, 2);
        //grid->print(0);
        //getchar();
        return grid;
    }

    Array *makeFloatArray(int size, const char *name) {
        Array *arr;
        arr = new Array(name, FLOAT, size);
        float *data;
        arr->getData(data);
        int i;

        for (i = 0; i < size; i++) {
            data[i] = 2 * i - 10;
        }
        return arr;
    }

    /**
     *
     */
    GridField *makeGridField(int size, string gridname, const char *datname,
            int k) {

        Grid *G;
        GridField *GF;
        Array *data;

        G = makeGrid(size, "A");
        k = 0;
        data = makeFloatArray(size, "x");

        GF = new GridField(G, k, data);
        //printf("Valid? %i\n", !notValid(GF));
        //GF->print();

        return GF;
    }

public:

    // Called once before everything gets tested
    BindTest() {
        //    DBG(cerr << " BindTest - Constructor" << endl);

    }

    // Called at the end of the test
    ~BindTest() {
        //    DBG(cerr << " BindTest - Destructor" << endl);
    }

    // Called before each test
    void setup() {
        //    DBG(cerr << " BindTest - setup()" << endl);
    }

    // Called after each test
    void tearDown() {
        //    DBG(cerr << " tearDown()" << endl);
    }

CPPUNIT_TEST_SUITE( BindTest );

    CPPUNIT_TEST(bind_test);

    CPPUNIT_TEST_SUITE_END()
    ;

    void bind_test() {
        DBG(cerr << " bind_test - BEGIN" << endl);

        try {
            GridField *gfResult;
            GridField *GF;
            GridField *Result;
            GF::Array* gfa;
            string msg;
            int size = 12;
            double dbls[size];
            int ints[size];
            ArrayReader arf;
            ArrayReader *memArrayReader;

            GF = makeGridField(size, "A", "x", 0);
            Array *arr_floats = new Array("io_floats", FLOAT, size);
            GF->Bind(0, arr_floats);

            Array *arr_ints = new Array("io_ints", INT, size);
            GF->Bind(0, arr_ints);

            GridField *aGF = AccumulateOp::Accumulate(GF, 0, "result", "result+1", "0", 0);
            DBG(GF->PrintTo(cerr,9));

            DBG( cerr << "restricting..." << endl;)
            Result = RefRestrictOp::Restrict("x<4", 0, GF);
            DBG(Result->PrintTo(cerr,0));

            Result = RefRestrictOp::Restrict("x>-4", 0, Result);
            DBG(Result->PrintTo(cerr,10));

#if 0
            FileArrayReader *far = new FileArrayReader("bindtest.dat", 0);

            far->setPatternAttribute("result");
            gfResult = BindOp::Bind("io_floats", FLOAT, far, 0, Result);
            DBG(gfResult->PrintTo(cerr,0);)

            gfa = gfResult->GetAttribute(0, "result");
            vector<double> file_dbls = gfa->makeArrayf();
            msg.clear();
            for(int i=0; i<file_dbls.size(); i++)
            msg += libdap::double_to_string(file_dbls[i]) + "   ";

            DBG(cerr << endl << "File array reader result: "<< msg << endl << endl << endl);

#endif
#if 0

            for (int i = 0; i < size; i++) {
                ints[i] = i - size/2.0;
                msg += libdap::long_to_string(ints[i]) + "   ";
            } DBG(cerr << endl << "Int array input: "<< msg << endl);

            DBG(cerr << endl <<"Memory Backed ArrayReader - integers"<< endl);
            memArrayReader = arf.makeArrayReader(ints, size);
            memArrayReader->setPatternAttribute("result");
            gfResult = BindOp::Bind("io_ints", INT, memArrayReader, 0, Result);
            DBG(cerr << endl << "Result "; gfResult->PrintTo(cerr,0));

            gfa = gfResult->GetAttribute(0, "result");
            DBG(cerr << endl << "Retrieving Int Result Values... " << endl);
            vector<int> gf_ints = gfa->makeArray();
            msg.clear();
            for (int i = 0; i < gf_ints.size(); i++)
                msg += libdap::long_to_string(gf_ints[i]) + "   ";

            DBG(cerr << endl << "Int array result: "<< msg << endl << endl << endl);
#endif

#if 1
            for(int i=0; i<size;i++) {
                dbls[i] = (i - size/2.0) * 11.01;
                msg += libdap::double_to_string(dbls[i]) + "   ";
            }
            DBG(cerr << endl << "Doubles array input: "<< msg << endl);

            DBG(cerr << endl << "Memory Backed ArrayReader - doubles" << endl);
            memArrayReader = new_makeArrayReader(dbls,size);
            memArrayReader->setPatternAttribute("result");
            gfResult = BindOp::Bind("io_floats", FLOAT, memArrayReader, 0, Result);
            DBG(cerr << endl << "Array of doubles 'Bound' to restricted Grid result: "<< endl; gfResult->PrintTo(cerr,8));

            gfa = gfResult->GetAttribute(0, "result");
            vector<double> gf_dbls = gfa->makeArrayf();
            msg.clear();
            for(int i=0; i<gf_dbls.size(); i++)
            msg += libdap::double_to_string(gf_dbls[i]) + "   ";

            DBG(cerr << endl << "Doubles array result: "<< msg << endl);

#endif

            CPPUNIT_ASSERT(true);
        } catch (std::string &e) {
            cerr << "Error: " << e << endl;
            CPPUNIT_ASSERT(false);
        } catch (...) {
            cerr << "Unknown Error." << endl;
            CPPUNIT_ASSERT(false);
        }
    }

};
// BindTest

CPPUNIT_TEST_SUITE_REGISTRATION(BindTest);

} // namespace ugrid

int main(int, char**) {
    CppUnit::TextTestRunner runner;
    runner.addTest(CppUnit::TestFactoryRegistry::getRegistry().makeTest());

    bool wasSuccessful = runner.run("", false);

    return wasSuccessful ? 0 : 1;
}

<|endoftext|>


<commit_before>/* **************************************************************************

   Copyright (c): 2008 - 2013 Hericus Software, LLC

   License: The MIT License (MIT)

   Authors: Steven M. Cherry

************************************************************************** */

#include <AnException.h>
#include <EnEx.h>
#include <Log.h>
#include <XmlHelpers.h>
#include <Timer.h>
using namespace SLib;

#include "HelixFS.h"
#include "HelixConfig.h"
using namespace Helix::Build;

// Our global instance of the filesystem
static HelixFS* m_helix_fs = nullptr;

HelixFS& HelixFS::getInstance()
{
	EnEx ee(FL, "HelixFS::getInstance()", true);

	if(m_helix_fs == nullptr){
		m_helix_fs = new HelixFS();
		m_helix_fs->Load();
	}
	return *m_helix_fs;
}

HelixFS::HelixFS()
{
	EnEx ee(FL, "HelixFS::HelixFS()");

}

HelixFS::~HelixFS()
{
	EnEx ee(FL, "HelixFS::~HelixFS()");

}

void HelixFS::Load()
{
	EnEx ee(FL, "HelixFS::Load()");

	// Clear out everything we might have loaded and reload from scratch
	m_folders.clear();

	// Hard code our top level folders
	auto root = std::make_shared<HelixFSFolder_Bare>( "root" ); root->Load(); m_folders.push_back( root );
	if(HelixConfig::getInstance().UseCore() == false){
		// Load these folders only when not using a core project helper
		auto build = std::make_shared<HelixFSFolder_Bare>( "build" );
		build->Load(); m_folders.push_back( build );
		auto client = std::make_shared<HelixFSFolder_Bare>( "client" );
		client->Load(); m_folders.push_back( client );
		auto glob = std::make_shared<HelixFSFolder_Bare>( "glob" );
		glob->Load(); m_folders.push_back( glob );
		auto server = std::make_shared<HelixFSFolder_Bare>( "server" );
		server->Load(); m_folders.push_back( server );
	}
	auto logic = std::make_shared<HelixFSFolder_Bare>( "logic" ); logic->Load(); m_folders.push_back( logic );
}

HelixFSFile HelixFS::FindFile(const twine& fileName)
{
	twine searchName;
	if(fileName.startsWith( "../" )){
		searchName = fileName.substr(3);
	} else if(fileName.startsWith( "./" )){
		searchName = fileName.substr(2);
	} else {
		searchName = fileName;
	}
	if(searchName.find( "/" ) != TWINE_NOT_FOUND){
		// Folder + filename.  Find the folder first, and then find the file in the target folder
		size_t lastSlash = searchName.rfind( "/" );
		twine searchPath( searchName.substr(0, lastSlash) ); // Everything but the last part
		twine searchFile( searchName.substr( lastSlash + 1 ) ); // Only the last part
		auto targetFolder = FindPath( searchPath );
		if(targetFolder){
			auto ret = targetFolder->FindFile( searchFile );
			if(ret){
				return ret;
			} else {
				return nullptr;
			}
		} else {
			return nullptr;
		}
	}

	// Search our list of folders one by one for the first match of the file name
	for(auto folder : m_folders){
		HelixFSFile ret = folder->FindFile( fileName );
		if(ret){
			return ret;
		}
	}

	// Didn't find anything - return the null pointer
	return nullptr;
}

vector<HelixFSFile> HelixFS::FindFilesByType( const twine& fileType )
{
	vector<HelixFSFile> ret;
	for(auto folder : m_folders){
		folder->FindFilesByType( fileType, ret );
	}

	return ret;
}

HelixFSFolder HelixFS::FindFolder(const twine& folderName)
{
	// Search our list of sub-folders
	for(auto folder : m_folders){
		if(folder->FolderName() == folderName){
			return folder;
		}
	}

	// Didn't find anything - return the null pointer
	return nullptr;
}

HelixFSFolder HelixFS::FindPath(const twine& folderPath)
{
	twine path( folderPath );
	vector<twine> pathSplits = path.split("/");
	if(pathSplits.size() == 0){
		return nullptr;
	}

	// Find the top level folder first:
	HelixFSFolder currentFolder = FindFolder( pathSplits[0] );
	if(currentFolder == nullptr) return nullptr;

	for(size_t i = 1; i < pathSplits.size(); i++){
		HelixFSFolder subFolder = currentFolder->FindFolder( pathSplits[i] );
		if(!subFolder) return nullptr;
		currentFolder = subFolder;
	}

	return currentFolder;
}

twine HelixFS::FixPhysical(const twine& path)
{
	twine ret(path);
#ifdef _WIN32
	ret.replace( '/', '\\' );
#endif
	return ret;
}

xmlDocPtr HelixFS::GetCache()
{
	if(m_hbuild_cache == nullptr){
		if(File::Exists( "hbuild.cache" )){
			m_hbuild_cache = xmlParseFile( "hbuild.cache" );
			if(m_hbuild_cache == nullptr){
				throw AnException(0, FL, "Error parsing hbuild.cache as xml.");
			}
		} else {
			m_hbuild_cache = xmlNewDoc((const xmlChar*)"1.0");
			m_hbuild_cache->children = xmlNewDocNode(m_hbuild_cache, NULL, (const xmlChar*)"HBuild", NULL);
		}
	}
	return m_hbuild_cache;
}

void HelixFS::SaveCache()
{
	xmlSaveFile( "hbuild.cache", GetCache() );
}

<commit_msg>Don't include build/ when loading the filesystem.<commit_after>/* **************************************************************************

   Copyright (c): 2008 - 2013 Hericus Software, LLC

   License: The MIT License (MIT)

   Authors: Steven M. Cherry

************************************************************************** */

#include <AnException.h>
#include <EnEx.h>
#include <Log.h>
#include <XmlHelpers.h>
#include <Timer.h>
using namespace SLib;

#include "HelixFS.h"
#include "HelixConfig.h"
using namespace Helix::Build;

// Our global instance of the filesystem
static HelixFS* m_helix_fs = nullptr;

HelixFS& HelixFS::getInstance()
{
	EnEx ee(FL, "HelixFS::getInstance()", true);

	if(m_helix_fs == nullptr){
		m_helix_fs = new HelixFS();
		m_helix_fs->Load();
	}
	return *m_helix_fs;
}

HelixFS::HelixFS()
{
	EnEx ee(FL, "HelixFS::HelixFS()");

}

HelixFS::~HelixFS()
{
	EnEx ee(FL, "HelixFS::~HelixFS()");

}

void HelixFS::Load()
{
	EnEx ee(FL, "HelixFS::Load()");

	// Clear out everything we might have loaded and reload from scratch
	m_folders.clear();

	// Hard code our top level folders
	auto root = std::make_shared<HelixFSFolder_Bare>( "root" ); root->Load(); m_folders.push_back( root );
	if(HelixConfig::getInstance().UseCore() == false){
		// Load these folders only when not using a core project helper
		//auto build = std::make_shared<HelixFSFolder_Bare>( "build" );
		//build->Load(); m_folders.push_back( build );
		auto client = std::make_shared<HelixFSFolder_Bare>( "client" );
		client->Load(); m_folders.push_back( client );
		auto glob = std::make_shared<HelixFSFolder_Bare>( "glob" );
		glob->Load(); m_folders.push_back( glob );
		auto server = std::make_shared<HelixFSFolder_Bare>( "server" );
		server->Load(); m_folders.push_back( server );
	}
	auto logic = std::make_shared<HelixFSFolder_Bare>( "logic" ); logic->Load(); m_folders.push_back( logic );
}

HelixFSFile HelixFS::FindFile(const twine& fileName)
{
	twine searchName;
	if(fileName.startsWith( "../" )){
		searchName = fileName.substr(3);
	} else if(fileName.startsWith( "./" )){
		searchName = fileName.substr(2);
	} else {
		searchName = fileName;
	}
	if(searchName.find( "/" ) != TWINE_NOT_FOUND){
		// Folder + filename.  Find the folder first, and then find the file in the target folder
		size_t lastSlash = searchName.rfind( "/" );
		twine searchPath( searchName.substr(0, lastSlash) ); // Everything but the last part
		twine searchFile( searchName.substr( lastSlash + 1 ) ); // Only the last part
		auto targetFolder = FindPath( searchPath );
		if(targetFolder){
			auto ret = targetFolder->FindFile( searchFile );
			if(ret){
				return ret;
			} else {
				return nullptr;
			}
		} else {
			return nullptr;
		}
	}

	// Search our list of folders one by one for the first match of the file name
	for(auto folder : m_folders){
		HelixFSFile ret = folder->FindFile( fileName );
		if(ret){
			return ret;
		}
	}

	// Didn't find anything - return the null pointer
	return nullptr;
}

vector<HelixFSFile> HelixFS::FindFilesByType( const twine& fileType )
{
	vector<HelixFSFile> ret;
	for(auto folder : m_folders){
		folder->FindFilesByType( fileType, ret );
	}

	return ret;
}

HelixFSFolder HelixFS::FindFolder(const twine& folderName)
{
	// Search our list of sub-folders
	for(auto folder : m_folders){
		if(folder->FolderName() == folderName){
			return folder;
		}
	}

	// Didn't find anything - return the null pointer
	return nullptr;
}

HelixFSFolder HelixFS::FindPath(const twine& folderPath)
{
	twine path( folderPath );
	vector<twine> pathSplits = path.split("/");
	if(pathSplits.size() == 0){
		return nullptr;
	}

	// Find the top level folder first:
	HelixFSFolder currentFolder = FindFolder( pathSplits[0] );
	if(currentFolder == nullptr) return nullptr;

	for(size_t i = 1; i < pathSplits.size(); i++){
		HelixFSFolder subFolder = currentFolder->FindFolder( pathSplits[i] );
		if(!subFolder) return nullptr;
		currentFolder = subFolder;
	}

	return currentFolder;
}

twine HelixFS::FixPhysical(const twine& path)
{
	twine ret(path);
#ifdef _WIN32
	ret.replace( '/', '\\' );
#endif
	return ret;
}

xmlDocPtr HelixFS::GetCache()
{
	if(m_hbuild_cache == nullptr){
		if(File::Exists( "hbuild.cache" )){
			m_hbuild_cache = xmlParseFile( "hbuild.cache" );
			if(m_hbuild_cache == nullptr){
				throw AnException(0, FL, "Error parsing hbuild.cache as xml.");
			}
		} else {
			m_hbuild_cache = xmlNewDoc((const xmlChar*)"1.0");
			m_hbuild_cache->children = xmlNewDocNode(m_hbuild_cache, NULL, (const xmlChar*)"HBuild", NULL);
		}
	}
	return m_hbuild_cache;
}

void HelixFS::SaveCache()
{
	xmlSaveFile( "hbuild.cache", GetCache() );
}

<|endoftext|>


<commit_before>//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//

#include <map>
#include <string>
#include <vector>

#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/table.h"
#include "table/block.h"
#include "table/block_builder.h"
#include "table/format.h"
#include "util/ldb_cmd.h"
#include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h"

namespace rocksdb {

class SstFileReader {
 public:
  explicit SstFileReader(const std::string& file_name,
                         bool verify_checksum,
                         bool output_hex);

  Status ReadSequential(bool print_kv,
                        uint64_t read_num,
                        bool has_from,
                        const std::string& from_key,
                        bool has_to,
                        const std::string& to_key);

  uint64_t GetReadNumber() { return read_num_; }

private:
  std::string file_name_;
  uint64_t read_num_;
  bool verify_checksum_;
  bool output_hex_;
  EnvOptions soptions_;
};

SstFileReader::SstFileReader(const std::string& file_path,
                             bool verify_checksum,
                             bool output_hex)
 :file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
  output_hex_(output_hex) {
  std::cout << "Process " << file_path << "\n";
}

Status SstFileReader::ReadSequential(bool print_kv,
                                     uint64_t read_num,
                                     bool has_from,
                                     const std::string& from_key,
                                     bool has_to,
                                     const std::string& to_key)
{
  unique_ptr<TableReader> table_reader;
  InternalKeyComparator internal_comparator_(BytewiseComparator());
  Options table_options;
  table_options.comparator = &internal_comparator_;
  unique_ptr<RandomAccessFile> file;
  Status s = table_options.env->NewRandomAccessFile(file_name_, &file,
                                                    soptions_);
  if(!s.ok()) {
   return s;
  }
  uint64_t file_size;
  table_options.env->GetFileSize(file_name_, &file_size);
  unique_ptr<TableFactory> table_factory;
  s = table_options.table_factory->GetTableReader(table_options, soptions_,
                                                  std::move(file), file_size,
                                                  &table_reader);
  if(!s.ok()) {
   return s;
  }

  Iterator* iter = table_reader->NewIterator(ReadOptions(verify_checksum_,
                                                         false));
  uint64_t i = 0;
  if (has_from) {
    InternalKey ikey(from_key, kMaxSequenceNumber, kValueTypeForSeek);
    iter->Seek(ikey.Encode());
  } else {
    iter->SeekToFirst();
  }
  for (; iter->Valid(); iter->Next()) {
    Slice key = iter->key();
    Slice value = iter->value();
    ++i;
    if (read_num > 0 && i > read_num)
      break;

    ParsedInternalKey ikey;
    if (!ParseInternalKey(key, &ikey)) {
      std::cerr << "Internal Key ["
                << key.ToString(true /* in hex*/)
                << "] parse error!\n";
      continue;
    }

    // If end marker was specified, we stop before it
    if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) {
      break;
    }

    if (print_kv) {
      std::cout << ikey.DebugString(output_hex_)
                << " => "
                << value.ToString(output_hex_) << "\n";
    }

   }

   read_num_ += i;

   Status ret = iter->status();
   delete iter;
   return ret;
}

} // namespace rocksdb

static void print_help() {
  fprintf(stderr,
      "sst_dump [--command=check|scan] [--verify_checksum] "
      "--file=data_dir_OR_sst_file"
      " [--output_hex]"
      " [--input_key_hex]"
      " [--from=<user_key>]"
      " [--to=<user_key>]"
      " [--read_num=NUM]\n");
}

string HexToString(const string& str) {
  string parsed;
  if (str[0] != '0' || str[1] != 'x') {
    fprintf(stderr, "Invalid hex input %s.  Must start with 0x\n",
            str.c_str());
    throw "Invalid hex input";
  }

  for (unsigned int i = 2; i < str.length();) {
    int c;
    sscanf(str.c_str() + i, "%2X", &c);
    parsed.push_back(c);
    i += 2;
  }
  return parsed;
}

int main(int argc, char** argv) {

  const char* dir_or_file = nullptr;
  uint64_t read_num = -1;
  std::string command;

  char junk;
  uint64_t n;
  bool verify_checksum = false;
  bool output_hex = false;
  bool input_key_hex = false;
  bool has_from = false;
  bool has_to = false;
  std::string from_key;
  std::string to_key;
  for (int i = 1; i < argc; i++)
  {
    if (strncmp(argv[i], "--file=", 7) == 0) {
      dir_or_file = argv[i] + 7;
    } else if (strcmp(argv[i], "--output_hex") == 0) {
      output_hex = true;
    } else if (strcmp(argv[i], "--input_key_hex") == 0) {
      input_key_hex = true;
    } else if (sscanf(argv[i],
               "--read_num=%lu%c",
               (unsigned long*)&n, &junk) == 1) {
      read_num = n;
    } else if (strcmp(argv[i], "--verify_checksum") == 0) {
      verify_checksum = true;
    } else if (strncmp(argv[i], "--command=", 10) == 0) {
      command = argv[i] + 10;
    } else if (strncmp(argv[i], "--from=", 7) == 0) {
      from_key = argv[i] + 7;
      has_from = true;
    } else if (strncmp(argv[i], "--to=", 5) == 0) {
      to_key = argv[i] + 5;
      has_to = true;
    }else {
      print_help();
      exit(1);
    }
  }

  if (input_key_hex) {
    if (has_from) {
      from_key = HexToString(from_key);
    }
    if (has_to) {
      to_key = HexToString(to_key);
    }
  }

  if(dir_or_file == nullptr) {
    print_help();
    exit(1);
  }

  std::vector<std::string> filenames;
  rocksdb::Env* env = rocksdb::Env::Default();
  rocksdb::Status st = env->GetChildren(dir_or_file, &filenames);
  bool dir = true;
  if (!st.ok()) {
    filenames.clear();
    filenames.push_back(dir_or_file);
    dir = false;
  }

  std::cout << "from [" << rocksdb::Slice(from_key).ToString(true)
            << "] to [" << rocksdb::Slice(to_key).ToString(true) << "]\n";

  uint64_t total_read = 0;
  for (size_t i = 0; i < filenames.size(); i++) {
    std::string filename = filenames.at(i);
    if (filename.length() <= 4 ||
        filename.rfind(".sst") != filename.length() - 4) {
      //ignore
      continue;
    }
    if(dir) {
      filename = std::string(dir_or_file) + "/" + filename;
    }
    rocksdb::SstFileReader reader(filename, verify_checksum,
                                  output_hex);
    rocksdb::Status st;
    // scan all files in give file path.
    if (command == "" || command == "scan" || command == "check") {
      st = reader.ReadSequential(command != "check",
                                 read_num > 0 ? (read_num - total_read) :
                                                read_num,
                                 has_from, from_key, has_to, to_key);
      if (!st.ok()) {
        fprintf(stderr, "%s: %s\n", filename.c_str(),
            st.ToString().c_str());
      }
      total_read += reader.GetReadNumber();
      if (read_num > 0 && total_read > read_num) {
        break;
      }
    }
  }
}
<commit_msg>Allow command line tool sst-dump to display table properties.<commit_after>//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//

#include <map>
#include <string>
#include <vector>

#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "table/block.h"
#include "table/block_builder.h"
#include "table/format.h"
#include "util/ldb_cmd.h"
#include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h"

namespace rocksdb {

class SstFileReader {
 public:
  explicit SstFileReader(const std::string& file_name,
                         bool verify_checksum,
                         bool output_hex);

  Status ReadSequential(bool print_kv,
                        uint64_t read_num,
                        bool has_from,
                        const std::string& from_key,
                        bool has_to,
                        const std::string& to_key);

  Status ReadTableProperties(TableProperties* table_properties);
  uint64_t GetReadNumber() { return read_num_; }

 private:
  Status NewTableReader(const std::string& file_path);

  std::string file_name_;
  uint64_t read_num_;
  bool verify_checksum_;
  bool output_hex_;
  EnvOptions soptions_;

  Status init_result_;
  unique_ptr<TableReader> table_reader_;
  unique_ptr<RandomAccessFile> file_;
  // table_options_ and internal_comparator_ will also be used in
  // ReadSequential internally (specifically, seek-related operations)
  Options table_options_;
  InternalKeyComparator internal_comparator_;
};

SstFileReader::SstFileReader(const std::string& file_path,
                             bool verify_checksum,
                             bool output_hex)
    :file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
    output_hex_(output_hex), internal_comparator_(BytewiseComparator()) {
  fprintf(stdout, "Process %s\n", file_path.c_str());

  init_result_ = NewTableReader(file_name_);
}

Status SstFileReader::NewTableReader(const std::string& file_path) {
  table_options_.comparator = &internal_comparator_;
  Status s = table_options_.env->NewRandomAccessFile(file_path, &file_,
                                                    soptions_);
  if (!s.ok()) {
    return s;
  }
  uint64_t file_size;
  table_options_.env->GetFileSize(file_path, &file_size);
  unique_ptr<TableFactory> table_factory;
  s = table_options_.table_factory->GetTableReader(table_options_, soptions_,
                                                  std::move(file_), file_size,
                                                  &table_reader_);
  return s;
}

Status SstFileReader::ReadSequential(bool print_kv,
                                     uint64_t read_num,
                                     bool has_from,
                                     const std::string& from_key,
                                     bool has_to,
                                     const std::string& to_key) {
  if (!table_reader_) {
    return init_result_;
  }

  Iterator* iter = table_reader_->NewIterator(ReadOptions(verify_checksum_,
                                                         false));
  uint64_t i = 0;
  if (has_from) {
    InternalKey ikey(from_key, kMaxSequenceNumber, kValueTypeForSeek);
    iter->Seek(ikey.Encode());
  } else {
    iter->SeekToFirst();
  }
  for (; iter->Valid(); iter->Next()) {
    Slice key = iter->key();
    Slice value = iter->value();
    ++i;
    if (read_num > 0 && i > read_num)
      break;

    ParsedInternalKey ikey;
    if (!ParseInternalKey(key, &ikey)) {
      std::cerr << "Internal Key ["
                << key.ToString(true /* in hex*/)
                << "] parse error!\n";
      continue;
    }

    // If end marker was specified, we stop before it
    if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) {
      break;
    }

    if (print_kv) {
      fprintf(stdout, "%s => %s\n",
          ikey.DebugString(output_hex_).c_str(),
          value.ToString(output_hex_).c_str());
    }
  }

  read_num_ += i;

  Status ret = iter->status();
  delete iter;
  return ret;
}

Status SstFileReader::ReadTableProperties(TableProperties* table_properties) {
  if (!table_reader_) {
    return init_result_;
  }

  *table_properties = table_reader_->GetTableProperties();
  return init_result_;
}

}  // namespace rocksdb

static void print_help() {
  fprintf(stderr,
      "sst_dump [--command=check|scan] [--verify_checksum] "
      "--file=data_dir_OR_sst_file"
      " [--output_hex]"
      " [--input_key_hex]"
      " [--from=<user_key>]"
      " [--to=<user_key>]"
      " [--read_num=NUM]"
      " [--show_properties]\n");
}

string HexToString(const string& str) {
  string parsed;
  if (str[0] != '0' || str[1] != 'x') {
    fprintf(stderr, "Invalid hex input %s.  Must start with 0x\n",
            str.c_str());
    throw "Invalid hex input";
  }

  for (unsigned int i = 2; i < str.length();) {
    int c;
    sscanf(str.c_str() + i, "%2X", &c);
    parsed.push_back(c);
    i += 2;
  }
  return parsed;
}

int main(int argc, char** argv) {
  const char* dir_or_file = nullptr;
  uint64_t read_num = -1;
  std::string command;

  char junk;
  uint64_t n;
  bool verify_checksum = false;
  bool output_hex = false;
  bool input_key_hex = false;
  bool has_from = false;
  bool has_to = false;
  bool show_properties = false;
  std::string from_key;
  std::string to_key;
  for (int i = 1; i < argc; i++) {
    if (strncmp(argv[i], "--file=", 7) == 0) {
      dir_or_file = argv[i] + 7;
    } else if (strcmp(argv[i], "--output_hex") == 0) {
      output_hex = true;
    } else if (strcmp(argv[i], "--input_key_hex") == 0) {
      input_key_hex = true;
    } else if (sscanf(argv[i],
               "--read_num=%lu%c",
               (unsigned long*)&n, &junk) == 1) {
      read_num = n;
    } else if (strcmp(argv[i], "--verify_checksum") == 0) {
      verify_checksum = true;
    } else if (strncmp(argv[i], "--command=", 10) == 0) {
      command = argv[i] + 10;
    } else if (strncmp(argv[i], "--from=", 7) == 0) {
      from_key = argv[i] + 7;
      has_from = true;
    } else if (strncmp(argv[i], "--to=", 5) == 0) {
      to_key = argv[i] + 5;
      has_to = true;
    } else if (strcmp(argv[i], "--show_properties") == 0) {
      show_properties = true;
    } else {
      print_help();
      exit(1);
    }
  }

  if (input_key_hex) {
    if (has_from) {
      from_key = HexToString(from_key);
    }
    if (has_to) {
      to_key = HexToString(to_key);
    }
  }

  if (dir_or_file == nullptr) {
    print_help();
    exit(1);
  }

  std::vector<std::string> filenames;
  rocksdb::Env* env = rocksdb::Env::Default();
  rocksdb::Status st = env->GetChildren(dir_or_file, &filenames);
  bool dir = true;
  if (!st.ok()) {
    filenames.clear();
    filenames.push_back(dir_or_file);
    dir = false;
  }

  fprintf(stdout, "from [%s] to [%s]\n",
      rocksdb::Slice(from_key).ToString(true).c_str(),
      rocksdb::Slice(to_key).ToString(true).c_str());

  uint64_t total_read = 0;
  for (size_t i = 0; i < filenames.size(); i++) {
    std::string filename = filenames.at(i);
    if (filename.length() <= 4 ||
        filename.rfind(".sst") != filename.length() - 4) {
      // ignore
      continue;
    }
    if (dir) {
      filename = std::string(dir_or_file) + "/" + filename;
    }
    rocksdb::SstFileReader reader(filename, verify_checksum,
                                  output_hex);
    rocksdb::Status st;
    // scan all files in give file path.
    if (command == "" || command == "scan" || command == "check") {
      st = reader.ReadSequential(command != "check",
                                 read_num > 0 ? (read_num - total_read) :
                                                read_num,
                                 has_from, from_key, has_to, to_key);
      if (!st.ok()) {
        fprintf(stderr, "%s: %s\n", filename.c_str(),
            st.ToString().c_str());
      }
      total_read += reader.GetReadNumber();
      if (read_num > 0 && total_read > read_num) {
        break;
      }
    }
    if (show_properties) {
      rocksdb::TableProperties table_properties;
      st = reader.ReadTableProperties(&table_properties);
      if (!st.ok()) {
        fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str());
      } else {
        fprintf(stdout,
            "Table Properties:\n"
            "------------------------------\n"
            "  %s", table_properties.ToString("\n  ", ": ").c_str());
        fprintf(stdout, "# deleted keys: %lu\n",
            rocksdb::GetDeletedKeys(
                table_properties.user_collected_properties));
      }
    }
  }
}
<|endoftext|>


<commit_before>/*
 * nt loader
 *
 * Copyright 2009 Mike McCormack
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
 */

#include "alloc_bitmap.h"

#ifdef HAVE_VALGRIND_VALGRIND_H
#include <valgrind/valgrind.h>
#else
#define VALGRIND_MALLOCLIKE_BLOCK( addr, sizeB, rzB, is_zeroed )
#define VALGRIND_FREELIKE_BLOCK( start, rzB )
#endif

allocation_bitmap_t::allocation_bitmap_t() :
	size(0),
	array_size(0),
	max_bits(0),
	bitmap(0),
	ptr(0)
{
}

void allocation_bitmap_t::set_area( void *_ptr, size_t _size )
{
	ptr = reinterpret_cast<unsigned char*>( _ptr );
	size = _size;
	array_size = size / (8 * allocation_granularity);
	max_bits = array_size * 8;
	bitmap = new unsigned char[ array_size ];
	memset( bitmap, 0, array_size );
}

size_t allocation_bitmap_t::count_zero_bits( size_t start, size_t max )
{
	if (max > (max_bits - start))
		max = max_bits - start;
	size_t i = 0;
	while ( i<max && !bit_value(start + i) )
		i++;
	return i;
}

size_t allocation_bitmap_t::count_one_bits( size_t start, size_t max )
{
	if (max > (max_bits - start))
		max = max_bits - start;
	size_t i = 0;
	while ( i<max && bit_value(start + i) )
		i++;
	return i;
}

void allocation_bitmap_t::set_bits( size_t start, size_t count )
{
	assert( start + count < max_bits );
	for (size_t i = 0; i<count; i++ )
		set_bit( start + i );
}

void allocation_bitmap_t::clear_bits( size_t start, size_t count )
{
	assert( start + count < max_bits );
	for (size_t i = 0; i<count; i++ )
		clear_bit( start + i );
}

size_t allocation_bitmap_t::bits_required( size_t len )
{
	return (len + allocation_granularity - 1) / allocation_granularity;
}

unsigned char* allocation_bitmap_t::alloc( size_t len )
{
	assert( ptr != 0 );
	size_t i = 0;

	size_t required = bits_required( len );

	while (i < max_bits )
	{
		size_t free, used;
		free = count_zero_bits( i, required );
		assert( free <= required );
		if (free == required)
		{
			// mark as allocated
			set_bits( i, required );

			// check that we allocated the bits correctly
			assert( required == count_one_bits( i, required ) );
			unsigned char *ret = &ptr[ i * allocation_granularity ];
			VALGRIND_MALLOCLIKE_BLOCK( ret, len, 0, 0 );
			return ret;
		}

		i += free;
		if (i == max_bits)
			break;

		used = count_one_bits( i, max_bits - i );
		assert( used > 0 );
		i += used;
	}
	return NULL;
}

void allocation_bitmap_t::free( unsigned char *start, size_t len )
{
	assert( ptr != 0 );

	// check the pointer is within bounds
	size_t ofs = (start - ptr);
	assert( ofs %allocation_granularity == 0 );
	ofs /= allocation_granularity;
	assert( ofs < max_bits );

	// assert the memory is allocated
	size_t required = bits_required( len );
	size_t n = count_one_bits( ofs, required );
	assert( required == n );

	// mark the memory as being clear
	clear_bits( ofs, required );

	VALGRIND_FREELIKE_BLOCK( start, 0 );
}

void allocation_bitmap_t::get_info( size_t& total, size_t& used, size_t& free )
{
	size_t n;
	used = 0;
	free = 0;
	size_t i = 0;
	bool ones = false;
	while ( i<max_bits )
	{
		if (!ones)
		{
			n = count_zero_bits( i, max_bits );
			free += n;
		}
		else
		{
			n = count_one_bits( i, max_bits );
			used += n;
		}
		i += n;
		ones = !ones;
	}
	assert (free + used == max_bits );
	used *= allocation_granularity;
	free *= allocation_granularity;
	total = max_bits * allocation_granularity;
}

void allocation_bitmap_t::test()
{
	size_t test_size = 0x1000;
	size_t used, free, total;
	unsigned char *test_buffer;
	static const int num_pointers = 0x10;
	unsigned char *ptr[num_pointers];
	allocation_bitmap_t *abm;
	int i;
	size_t test_alloc_sz = allocation_granularity * 0x10;

	// create a new buffer to manage
	test_buffer = new unsigned char[test_size];
	abm = new allocation_bitmap_t;

	abm->set_area( test_buffer, test_size );

	// check everything is free
	abm->get_info( total, used, free );
	assert( used == 0 );
	assert( free == total );

	// allocate a number of blocks
	for (i=0; i<num_pointers; i++)
		ptr[i] = abm->alloc( test_alloc_sz );

	// check the blocks are as big as we asked for
	for (i=0; i<(num_pointers-1); i++)
		assert( (size_t)(ptr[i+1] - ptr[i]) >= test_alloc_sz );

	// check there's some used memory
	abm->get_info( total, used, free );
	assert( used != 0 );

	for (i=0; i<num_pointers; i++)
		abm->free( ptr[i], test_alloc_sz );

	// check everything is free again
	abm->get_info( total, used, free );
	assert( used == 0 );
	assert( free == total );

	// free the memory
	delete abm;
	delete test_buffer;
}
<commit_msg>Use the correct delete operator<commit_after>/*
 * nt loader
 *
 * Copyright 2009 Mike McCormack
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
 */

#include "alloc_bitmap.h"

#ifdef HAVE_VALGRIND_VALGRIND_H
#include <valgrind/valgrind.h>
#else
#define VALGRIND_MALLOCLIKE_BLOCK( addr, sizeB, rzB, is_zeroed )
#define VALGRIND_FREELIKE_BLOCK( start, rzB )
#endif

allocation_bitmap_t::allocation_bitmap_t() :
	size(0),
	array_size(0),
	max_bits(0),
	bitmap(0),
	ptr(0)
{
}

void allocation_bitmap_t::set_area( void *_ptr, size_t _size )
{
	ptr = reinterpret_cast<unsigned char*>( _ptr );
	size = _size;
	array_size = size / (8 * allocation_granularity);
	max_bits = array_size * 8;
	bitmap = new unsigned char[ array_size ];
	memset( bitmap, 0, array_size );
}

size_t allocation_bitmap_t::count_zero_bits( size_t start, size_t max )
{
	if (max > (max_bits - start))
		max = max_bits - start;
	size_t i = 0;
	while ( i<max && !bit_value(start + i) )
		i++;
	return i;
}

size_t allocation_bitmap_t::count_one_bits( size_t start, size_t max )
{
	if (max > (max_bits - start))
		max = max_bits - start;
	size_t i = 0;
	while ( i<max && bit_value(start + i) )
		i++;
	return i;
}

void allocation_bitmap_t::set_bits( size_t start, size_t count )
{
	assert( start + count < max_bits );
	for (size_t i = 0; i<count; i++ )
		set_bit( start + i );
}

void allocation_bitmap_t::clear_bits( size_t start, size_t count )
{
	assert( start + count < max_bits );
	for (size_t i = 0; i<count; i++ )
		clear_bit( start + i );
}

size_t allocation_bitmap_t::bits_required( size_t len )
{
	return (len + allocation_granularity - 1) / allocation_granularity;
}

unsigned char* allocation_bitmap_t::alloc( size_t len )
{
	assert( ptr != 0 );
	size_t i = 0;

	size_t required = bits_required( len );

	while (i < max_bits )
	{
		size_t free, used;
		free = count_zero_bits( i, required );
		assert( free <= required );
		if (free == required)
		{
			// mark as allocated
			set_bits( i, required );

			// check that we allocated the bits correctly
			assert( required == count_one_bits( i, required ) );
			unsigned char *ret = &ptr[ i * allocation_granularity ];
			VALGRIND_MALLOCLIKE_BLOCK( ret, len, 0, 0 );
			return ret;
		}

		i += free;
		if (i == max_bits)
			break;

		used = count_one_bits( i, max_bits - i );
		assert( used > 0 );
		i += used;
	}
	return NULL;
}

void allocation_bitmap_t::free( unsigned char *start, size_t len )
{
	assert( ptr != 0 );

	// check the pointer is within bounds
	size_t ofs = (start - ptr);
	assert( ofs %allocation_granularity == 0 );
	ofs /= allocation_granularity;
	assert( ofs < max_bits );

	// assert the memory is allocated
	size_t required = bits_required( len );
	size_t n = count_one_bits( ofs, required );
	assert( required == n );

	// mark the memory as being clear
	clear_bits( ofs, required );

	VALGRIND_FREELIKE_BLOCK( start, 0 );
}

void allocation_bitmap_t::get_info( size_t& total, size_t& used, size_t& free )
{
	size_t n;
	used = 0;
	free = 0;
	size_t i = 0;
	bool ones = false;
	while ( i<max_bits )
	{
		if (!ones)
		{
			n = count_zero_bits( i, max_bits );
			free += n;
		}
		else
		{
			n = count_one_bits( i, max_bits );
			used += n;
		}
		i += n;
		ones = !ones;
	}
	assert (free + used == max_bits );
	used *= allocation_granularity;
	free *= allocation_granularity;
	total = max_bits * allocation_granularity;
}

void allocation_bitmap_t::test()
{
	size_t test_size = 0x1000;
	size_t used, free, total;
	unsigned char *test_buffer;
	static const int num_pointers = 0x10;
	unsigned char *ptr[num_pointers];
	allocation_bitmap_t *abm;
	int i;
	size_t test_alloc_sz = allocation_granularity * 0x10;

	// create a new buffer to manage
	test_buffer = new unsigned char[test_size];
	abm = new allocation_bitmap_t;

	abm->set_area( test_buffer, test_size );

	// check everything is free
	abm->get_info( total, used, free );
	assert( used == 0 );
	assert( free == total );

	// allocate a number of blocks
	for (i=0; i<num_pointers; i++)
		ptr[i] = abm->alloc( test_alloc_sz );

	// check the blocks are as big as we asked for
	for (i=0; i<(num_pointers-1); i++)
		assert( (size_t)(ptr[i+1] - ptr[i]) >= test_alloc_sz );

	// check there's some used memory
	abm->get_info( total, used, free );
	assert( used != 0 );

	for (i=0; i<num_pointers; i++)
		abm->free( ptr[i], test_alloc_sz );

	// check everything is free again
	abm->get_info( total, used, free );
	assert( used == 0 );
	assert( free == total );

	// free the memory
	delete abm;
	delete[] test_buffer;
}
<|endoftext|>


<commit_before>/*
 *  AvanceDB - an in-memory database similar to Apache CouchDB
 *  Copyright (C) 2015 Ripcord Software
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "map_reduce.h"

#include <boost/make_shared.hpp>
#include <boost/scope_exit.hpp>

#include <memory>
#include <algorithm>

#include "script_array_jsapi_key_value_source.h"
#include "map_reduce_result.h"
#include "script_object_jsapi_source.h"
#include "script_array_jsapi_source.h"
#include "map_reduce_script_object_state.h"
#include "documents.h"
#include "config.h"
#include "set_thread_name.h"
#include "map_reduce_thread_pool.h"

#include "script_object_factory.h"
#include "script_array_factory.h"

MapReduce::MapReduce() : mapReduceThreadPool_(MapReduceThreadPool::Get()) {

}

map_reduce_results_ptr MapReduce::Execute(const GetViewOptions& options, const MapReduceTask& task, document_collections_ptr colls) {
    std::mutex m;
    auto collsSize = colls->size();
    std::vector<map_reduce_result_array_ptr> resultStack;
    resultStack.reserve(collsSize);
    std::atomic<unsigned> threads(collsSize);

    for (auto& docs : *colls) {
        mapReduceThreadPool_->Post([&]() {
            BOOST_SCOPE_EXIT(&threads) { --threads; } BOOST_SCOPE_EXIT_END
            auto& rt = mapReduceThreadPool_->GetThreadRuntime();

            auto result = Execute(rt, task, docs);

            std::unique_lock<std::mutex> l(m);
            resultStack.emplace_back(result);
        });
    }

    auto results = boost::make_shared<map_reduce_result_array_ptr::element_type>();

    auto mergeResultsWorker = [&](map_reduce_result_array_ptr result) {
        auto oldSize = results->size();
        if (oldSize == 0) {
            // the first shard result is a guess for the amount of memory we will
            // need to hold all the results
            results->reserve(result->size() * collsSize);
        }
        results->insert(results->end(), result->cbegin(), result->cend());
        if (oldSize > 0) {
            std::inplace_merge(results->begin(), results->begin() + oldSize, results->end(),
                [](const map_reduce_result_ptr& a, const map_reduce_result_ptr& b) {
                    return MapReduceResult::Less(a, b);
            });
        }
    };

    while (threads.load() > 0) {
        while (resultStack.size() > 0) {
            std::unique_lock<std::mutex> l(m);
            if (resultStack.size() > 0) {
                auto result = resultStack.back();
                resultStack.pop_back();
                l.unlock();

                mergeResultsWorker(result);
            }
        }

        std::this_thread::sleep_for(std::chrono::milliseconds(1));
    }

    for (auto result : resultStack) {
        mergeResultsWorker(result);
    }

    return boost::make_shared<map_reduce_results_ptr::element_type>(options, results);
}

// TODO: this is still very basic
map_reduce_result_array_ptr MapReduce::Execute(rs::jsapi::Runtime& rt, const MapReduceTask& task, const document_array& docs) {
    map_reduce_result_array_ptr results = boost::make_shared<map_reduce_result_array_ptr::element_type>();

    // create the function script
    std::string mapScript = "(function() { return ";
    mapScript += task.Map();
    mapScript += "; })();";

    document_ptr doc = nullptr;

    // define a function in global scope implemented by a C++ lambda
    rs::jsapi::Global::DefineFunction(rt, "emit",
        [&](const std::vector<rs::jsapi::Value>& args, rs::jsapi::Value&) {
            auto source = ScriptArrayJsapiKeyValueSource::Create(args[0], args[1]);

            auto resultArr = rs::scriptobject::ScriptArrayFactory::CreateArray(source);
            auto result = MapReduceResult::Create(resultArr, doc);
            results->emplace_back(result);
    });

    // TODO: elegantly handle JS syntax errors
    // execute the script in the context of the runtime, getting the resulting function
    rs::jsapi::Value func(rt);
    rt.Evaluate(mapScript.c_str(), func);

    auto state = new MapReduceScriptObjectState{nullptr};

    rs::scriptobject::ScriptObjectPtr scriptObj = nullptr;
    rs::jsapi::Value object(rt);
    rs::jsapi::DynamicObject::Create(rt,
        [&](const char* name, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(scriptObj, name, value);
        },
        nullptr,
        [&](std::vector<std::string>& props, std::vector<std::pair<std::string, JSNative>>&) {
            // TODO: only enumerate when really needed
            for (decltype(scriptObj->getCount()) i = 0, count = scriptObj->getCount(); i < count; ++i) {
                props.emplace_back(scriptObj->getName(i));
            }
            return true;
        },
        [state]() { delete state; },
        object);

    rs::jsapi::DynamicObject::SetPrivate(object, 0, state);

    rs::jsapi::FunctionArguments args(rt);
    args.Append(object);

    for (DocumentsCollection::size_type i = 0, size = docs.size(); i < size; ++i) {
        doc = docs[i];
        scriptObj = doc->getObject();

        state->scriptObj_ = scriptObj;

        // TODO: handle exception cases here
        func.CallFunction(args);
    }

    SortResultArray(results);

    return results;
}

void MapReduce::GetFieldValue(script_object_ptr scriptObj, const char* name, rs::jsapi::Value& value) {
    int index = 0;
    switch (scriptObj->getType(name, index)) {
        case rs::scriptobject::ScriptObjectType::Boolean:
            value = scriptObj->getBoolean(index);
            return;
        case rs::scriptobject::ScriptObjectType::Int32:
            value = scriptObj->getInt32(index);
            return;
        case rs::scriptobject::ScriptObjectType::String:
            value = scriptObj->getString(index);
            return;
        case rs::scriptobject::ScriptObjectType::Double:
            value = scriptObj->getDouble(index);
            return;
        case rs::scriptobject::ScriptObjectType::Object: {
            auto childObj = scriptObj->getObject(index);
            MapReduce::CreateValueObject(childObj, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Array: {
            auto childArr = scriptObj->getArray(index);
            MapReduce::CreateValueArray(childArr, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Null:
            value = JS::NullHandleValue;
            return;
        default:
            value = JS::UndefinedHandleValue;
            return;
    }
}

void MapReduce::GetFieldValue(script_array_ptr scriptArr, int index, rs::jsapi::Value& value) {
    switch (scriptArr->getType(index)) {
        case rs::scriptobject::ScriptObjectType::Boolean:
            value = scriptArr->getBoolean(index);
            return;
        case rs::scriptobject::ScriptObjectType::Int32:
            value = scriptArr->getInt32(index);
            return;
        case rs::scriptobject::ScriptObjectType::String:
            value = scriptArr->getString(index);
            return;
        case rs::scriptobject::ScriptObjectType::Double:
            value = scriptArr->getDouble(index);
            return;
        case rs::scriptobject::ScriptObjectType::Object: {
            auto childObj = scriptArr->getObject(index);
            MapReduce::CreateValueObject(childObj, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Array: {
            auto childArr = scriptArr->getArray(index);
            MapReduce::CreateValueArray(childArr, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Null:
            value = JS::NullHandleValue;
            return;
        default:
            value = JS::UndefinedHandleValue;
    }
}

void MapReduce::CreateValueObject(script_object_ptr obj, rs::jsapi::Value& value) {
    auto state = new MapReduceScriptObjectState{obj};

    auto cx = value.getContext();
    rs::jsapi::DynamicObject::Create(cx,
        [state](const char* name, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(state->scriptObj_, name, value);
        },
        nullptr,
        [state](std::vector<std::string>& props, std::vector<std::pair<std::string, JSNative>>&) {
            // TODO: it would be useful to cache this for performance
            for (decltype(state->scriptObj_->getCount()) i = 0, count = state->scriptObj_->getCount(); i < count; ++i) {
                props.emplace_back(state->scriptObj_->getName(i));
            }
            return true;
        },
        [state]() { delete state; },
        value);

    rs::jsapi::DynamicObject::SetPrivate(value, 0, state);
}

void MapReduce::CreateValueArray(script_array_ptr arr, rs::jsapi::Value& value) {
    auto state = new MapReduceScriptArrayState{arr};

    auto cx = value.getContext();
    rs::jsapi::DynamicArray::Create(cx,
        [state](int index, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(state->scriptArray_, index, value);
        },
        nullptr,
        [state]() { return state->scriptArray_->getCount(); },
        [state]() { delete state; },
        value);

    rs::jsapi::DynamicArray::SetPrivate(value, 0, state);
}

script_object_ptr MapReduce::GetValueScriptObject(const rs::jsapi::Value& obj) {
    if (rs::jsapi::DynamicObject::IsDynamicObject(obj)) {
        uint64_t value = 0;
        void* ptr = nullptr;
        rs::jsapi::DynamicObject::GetPrivate(obj, value, ptr);
        return reinterpret_cast<MapReduceScriptObjectState*>(ptr)->scriptObj_;
    }
    else {
        auto source = ScriptObjectJsapiSource::Create(obj);
        return rs::scriptobject::ScriptObjectFactory::CreateObject(source);
    }
}

script_array_ptr MapReduce::GetValueScriptArray(const rs::jsapi::Value& arr) {
    if (rs::jsapi::DynamicArray::IsDynamicArray(arr)) {
        uint64_t value = 0;
        void* ptr = nullptr;
        rs::jsapi::DynamicArray::GetPrivate(arr, value, ptr);
        return reinterpret_cast<MapReduceScriptArrayState*>(ptr)->scriptArray_;
    } else {
        auto source = ScriptArrayJsapiSource::Create(arr);
        return rs::scriptobject::ScriptArrayFactory::CreateArray(source);
    }
}

void MapReduce::SortResultArray(map_reduce_result_array_ptr results) {
    std::sort(results->begin(), results->end(), [](const map_reduce_result_ptr& a, const map_reduce_result_ptr& b) {
        return MapReduceResult::Less(a, b);
    });
}<commit_msg>Use the map/reduce thread pool to merge the m/r results<commit_after>/*
 *  AvanceDB - an in-memory database similar to Apache CouchDB
 *  Copyright (C) 2015 Ripcord Software
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "map_reduce.h"

#include <boost/make_shared.hpp>
#include <boost/scope_exit.hpp>

#include <memory>
#include <algorithm>

#include "script_array_jsapi_key_value_source.h"
#include "map_reduce_result.h"
#include "script_object_jsapi_source.h"
#include "script_array_jsapi_source.h"
#include "map_reduce_script_object_state.h"
#include "documents.h"
#include "config.h"
#include "set_thread_name.h"
#include "map_reduce_thread_pool.h"

#include "script_object_factory.h"
#include "script_array_factory.h"

MapReduce::MapReduce() : mapReduceThreadPool_(MapReduceThreadPool::Get()) {

}

map_reduce_results_ptr MapReduce::Execute(const GetViewOptions& options, const MapReduceTask& task, document_collections_ptr colls) {
    std::mutex m;
    auto collsSize = colls->size();
    std::vector<map_reduce_result_array_ptr> resultArray;
    std::atomic<int> threads(collsSize);

    // run the map
    for (auto& docs : *colls) {
        mapReduceThreadPool_->Post([&]() {
            BOOST_SCOPE_EXIT(&threads) { --threads; } BOOST_SCOPE_EXIT_END
            auto& rt = mapReduceThreadPool_->GetThreadRuntime();

            auto result = Execute(rt, task, docs);

            std::unique_lock<std::mutex> l(m);
            resultArray.emplace_back(result);
        });
    }

    // wait for the map threads to finish
    while (threads.load() > 0) {
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
    }

    // calculate the number of map rows and offsets
    decltype(resultArray.size()) totalRows = 0;
    std::vector<decltype(totalRows)> rowOffsets;
    for (const auto& result : resultArray) {
        rowOffsets.emplace_back(totalRows);
        totalRows += result->size();
    }
    rowOffsets.emplace_back(totalRows);

    // allocate the results collection
    auto results = boost::make_shared<map_reduce_result_array_ptr::element_type>();
    results->reserve(totalRows);

    // copy the shard results into the main results collection
    for (const auto& result : resultArray) {
        results->insert(results->end(), result->cbegin(), result->cend());
    }

    auto less = [](const map_reduce_result_ptr& a, const map_reduce_result_ptr& b) {
        return MapReduceResult::Less(a, b);
    };

    auto mergeResultsWorker = [&](decltype(collsSize) startIndex, decltype(collsSize) step) {
        BOOST_SCOPE_EXIT(&threads) { --threads; } BOOST_SCOPE_EXIT_END

        auto midStep = step / 2;
        auto midIndex = startIndex + midStep;
        auto endIndex = std::min(startIndex + step, collsSize);

        auto startOffset = rowOffsets[startIndex];
        auto midOffset = rowOffsets[midIndex];
        auto endOffset = rowOffsets[endIndex];

        const auto& begin = results->begin();
        std::inplace_merge(begin + startOffset, begin + midOffset, begin + endOffset, less);
    };

    // merge the result shards on the main results collection
    decltype(collsSize) step = 2;
    while ((threads = collsSize / step) > 0) {

        for (decltype(collsSize) i = 0; i <= collsSize - step; i += step) {
            mapReduceThreadPool_->Post([=]() { mergeResultsWorker(i, step); });
        }

        // wait for the merge threads to finish
        while (threads.load() > 0) {
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
        }

        step *= 2;
    }

    // if we had an odd number of collections then there is one left to merge
    if ((collsSize % step) > 0) {
        mergeResultsWorker(0, step);
    }

    return boost::make_shared<map_reduce_results_ptr::element_type>(options, results);
}

map_reduce_result_array_ptr MapReduce::Execute(rs::jsapi::Runtime& rt, const MapReduceTask& task, const document_array& docs) {
    map_reduce_result_array_ptr results = boost::make_shared<map_reduce_result_array_ptr::element_type>();

    // create the function script
    std::string mapScript = "(function() { return ";
    mapScript += task.Map();
    mapScript += "; })();";

    document_ptr doc = nullptr;

    // define a function in global scope implemented by a C++ lambda
    rs::jsapi::Global::DefineFunction(rt, "emit",
        [&](const std::vector<rs::jsapi::Value>& args, rs::jsapi::Value&) {
            auto source = ScriptArrayJsapiKeyValueSource::Create(args[0], args[1]);

            auto resultArr = rs::scriptobject::ScriptArrayFactory::CreateArray(source);
            auto result = MapReduceResult::Create(resultArr, doc);
            results->emplace_back(result);
    });

    // TODO: elegantly handle JS syntax errors
    // execute the script in the context of the runtime, getting the resulting function
    rs::jsapi::Value func(rt);
    rt.Evaluate(mapScript.c_str(), func);

    auto state = new MapReduceScriptObjectState{nullptr};

    rs::scriptobject::ScriptObjectPtr scriptObj = nullptr;
    rs::jsapi::Value object(rt);
    rs::jsapi::DynamicObject::Create(rt,
        [&](const char* name, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(scriptObj, name, value);
        },
        nullptr,
        [&](std::vector<std::string>& props, std::vector<std::pair<std::string, JSNative>>&) {
            // TODO: only enumerate when really needed
            for (decltype(scriptObj->getCount()) i = 0, count = scriptObj->getCount(); i < count; ++i) {
                props.emplace_back(scriptObj->getName(i));
            }
            return true;
        },
        [state]() { delete state; },
        object);

    rs::jsapi::DynamicObject::SetPrivate(object, 0, state);

    rs::jsapi::FunctionArguments args(rt);
    args.Append(object);

    for (DocumentsCollection::size_type i = 0, size = docs.size(); i < size; ++i) {
        doc = docs[i];
        scriptObj = doc->getObject();

        state->scriptObj_ = scriptObj;

        // TODO: handle exception cases here
        func.CallFunction(args);
    }

    SortResultArray(results);

    return results;
}

void MapReduce::GetFieldValue(script_object_ptr scriptObj, const char* name, rs::jsapi::Value& value) {
    int index = 0;
    switch (scriptObj->getType(name, index)) {
        case rs::scriptobject::ScriptObjectType::Boolean:
            value = scriptObj->getBoolean(index);
            return;
        case rs::scriptobject::ScriptObjectType::Int32:
            value = scriptObj->getInt32(index);
            return;
        case rs::scriptobject::ScriptObjectType::String:
            value = scriptObj->getString(index);
            return;
        case rs::scriptobject::ScriptObjectType::Double:
            value = scriptObj->getDouble(index);
            return;
        case rs::scriptobject::ScriptObjectType::Object: {
            auto childObj = scriptObj->getObject(index);
            MapReduce::CreateValueObject(childObj, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Array: {
            auto childArr = scriptObj->getArray(index);
            MapReduce::CreateValueArray(childArr, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Null:
            value = JS::NullHandleValue;
            return;
        default:
            value = JS::UndefinedHandleValue;
            return;
    }
}

void MapReduce::GetFieldValue(script_array_ptr scriptArr, int index, rs::jsapi::Value& value) {
    switch (scriptArr->getType(index)) {
        case rs::scriptobject::ScriptObjectType::Boolean:
            value = scriptArr->getBoolean(index);
            return;
        case rs::scriptobject::ScriptObjectType::Int32:
            value = scriptArr->getInt32(index);
            return;
        case rs::scriptobject::ScriptObjectType::String:
            value = scriptArr->getString(index);
            return;
        case rs::scriptobject::ScriptObjectType::Double:
            value = scriptArr->getDouble(index);
            return;
        case rs::scriptobject::ScriptObjectType::Object: {
            auto childObj = scriptArr->getObject(index);
            MapReduce::CreateValueObject(childObj, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Array: {
            auto childArr = scriptArr->getArray(index);
            MapReduce::CreateValueArray(childArr, value);
            return;
        }
        case rs::scriptobject::ScriptObjectType::Null:
            value = JS::NullHandleValue;
            return;
        default:
            value = JS::UndefinedHandleValue;
    }
}

void MapReduce::CreateValueObject(script_object_ptr obj, rs::jsapi::Value& value) {
    auto state = new MapReduceScriptObjectState{obj};

    auto cx = value.getContext();
    rs::jsapi::DynamicObject::Create(cx,
        [state](const char* name, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(state->scriptObj_, name, value);
        },
        nullptr,
        [state](std::vector<std::string>& props, std::vector<std::pair<std::string, JSNative>>&) {
            // TODO: it would be useful to cache this for performance
            for (decltype(state->scriptObj_->getCount()) i = 0, count = state->scriptObj_->getCount(); i < count; ++i) {
                props.emplace_back(state->scriptObj_->getName(i));
            }
            return true;
        },
        [state]() { delete state; },
        value);

    rs::jsapi::DynamicObject::SetPrivate(value, 0, state);
}

void MapReduce::CreateValueArray(script_array_ptr arr, rs::jsapi::Value& value) {
    auto state = new MapReduceScriptArrayState{arr};

    auto cx = value.getContext();
    rs::jsapi::DynamicArray::Create(cx,
        [state](int index, rs::jsapi::Value& value) {
            return MapReduce::GetFieldValue(state->scriptArray_, index, value);
        },
        nullptr,
        [state]() { return state->scriptArray_->getCount(); },
        [state]() { delete state; },
        value);

    rs::jsapi::DynamicArray::SetPrivate(value, 0, state);
}

script_object_ptr MapReduce::GetValueScriptObject(const rs::jsapi::Value& obj) {
    if (rs::jsapi::DynamicObject::IsDynamicObject(obj)) {
        uint64_t value = 0;
        void* ptr = nullptr;
        rs::jsapi::DynamicObject::GetPrivate(obj, value, ptr);
        return reinterpret_cast<MapReduceScriptObjectState*>(ptr)->scriptObj_;
    }
    else {
        auto source = ScriptObjectJsapiSource::Create(obj);
        return rs::scriptobject::ScriptObjectFactory::CreateObject(source);
    }
}

script_array_ptr MapReduce::GetValueScriptArray(const rs::jsapi::Value& arr) {
    if (rs::jsapi::DynamicArray::IsDynamicArray(arr)) {
        uint64_t value = 0;
        void* ptr = nullptr;
        rs::jsapi::DynamicArray::GetPrivate(arr, value, ptr);
        return reinterpret_cast<MapReduceScriptArrayState*>(ptr)->scriptArray_;
    } else {
        auto source = ScriptArrayJsapiSource::Create(arr);
        return rs::scriptobject::ScriptArrayFactory::CreateArray(source);
    }
}

void MapReduce::SortResultArray(map_reduce_result_array_ptr results) {
    std::sort(results->begin(), results->end(), [](const map_reduce_result_ptr& a, const map_reduce_result_ptr& b) {
        return MapReduceResult::Less(a, b);
    });
}<|endoftext|>


<commit_before>/**************************************************************************
 * Copyright(c) 1998-2004, ALICE Experiment at CERN, All rights reserved. *
 *                                                                        *
 * Author: The ALICE Off-line Project.                                    *
 * Contributors are mentioned in the code where appropriate.              *
 *                                                                        *
 * Permission to use, copy, modify and distribute this software and its   *
 * documentation strictly for non-commercial purposes is hereby granted   *
 * without fee, provided that the above copyright notice appears in all   *
 * copies and that both the copyright notice and this permission notice   *
 * appear in the supporting documentation. The authors make no claims     *
 * about the suitability of this software for any purpose. It is          *
 * provided "as is" without express or implied warranty.                  *
 **************************************************************************/
#include <TMath.h>
#include "AliLog.h"
#include "AliITSUSDigit.h"
ClassImp(AliITSUSDigit)

using std::istream;
using std::swap;
using namespace TMath;

// Addapted from ITS/AliITSpListItem, ruben.shahoyan@cern.ch

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit()
: fModule(0)
  ,fNTracks(0)
  ,fTsignal(0.0)
  ,fNoise(0.0)
  ,fSignalAfterElect(0.0)
{
  // Default constructor
  for (int i=kBuffSize;i--;) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(UInt_t module,UInt_t index,Double_t noise)
  :fModule(module)
  ,fNTracks(0)
  ,fTsignal(0.0)
  ,fNoise(noise)
  ,fSignalAfterElect(0.0)
{
  // Standard noise constructor
  SetUniqueID(index);
  for (int i=kBuffSize;i--;) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(Int_t track,Int_t hit,UInt_t module,UInt_t index,Double_t signal)
  :fModule(module)
  ,fNTracks(1)
  ,fTsignal(signal)
  ,fNoise(0.0)
  ,fSignalAfterElect(0.0)
{
  // Standard signal constructor
  // Inputs:
  //    Int_t track     The track number which produced this signal
  //    Int_t hit       The hit number which produced this signal
  //    Int_t module    The module where this signal occurred
  //    Int_t index     The cell index where this signal occurred
  //    Double_t signal The value of the signal (ionization)
  SetUniqueID(index);
  fTrack[0]  = track;
  fHits[0]   = hit;
  fSignal[0] = signal;
  for (int i=1;i<kBuffSize;i++) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit& AliITSUSDigit::operator=(const AliITSUSDigit &source)
{
  // = operator
  if (&source!=this) {
    this->~AliITSUSDigit();
    new(this) AliITSUSDigit(source);
  }
  return *this;
  //
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(const AliITSUSDigit &source)
  :TObject(source)
  ,fModule(source.fModule)
  ,fNTracks(source.fNTracks)
  ,fTsignal(source.fTsignal)
  ,fNoise(source.fNoise)
  ,fSignalAfterElect(source.fSignalAfterElect)
{
  // Copy operator
  for(Int_t i=kBuffSize;i--;) {
    fTrack[i]  = source.fTrack[i];
    fSignal[i] = source.fSignal[i];
    fHits[i]   = source.fHits[i];
  } // end if i
  //
}

//______________________________________________________________________
void AliITSUSDigit::AddSignal(Int_t track,Int_t hit,Double_t signal)
{
  // Adds this track number and signal to the pList and orders them
  // Inputs:
  //    Int_t track     The track number which produced this signal
  //    Int_t hit       The hit number which produced this signal
  //    Int_t module    The module where this signal occurred
  //    Int_t index     The cell index where this signal occurred
  //    Double_t signal The value of the signal (ionization)
  Int_t    i,j;
  Bool_t   flg=kFALSE;
  //
  if (Abs(signal)>2147483647.0) {
    //PH 2147483647 is the max. integer
    //PH This apparently is a problem which needs investigation
    AliWarning(Form("Too big or too small signal value %f",signal));
    signal = Sign((Double_t)2147483647,signal);
  }
  //
  fTsignal += signal; // Keep track of sum signal.
  for (i=fNTracks;i--;) {
    if ( track==fTrack[i]  ) {
      fSignal[i] += signal;
      flg = kTRUE;
      break;
    } // end for i & if.
  }
  //
  if (flg) {
    if (fNTracks>1) { // resort arrays.
      for (i=1;i<fNTracks;i++) {
	j = i;
	while(j>0 && fSignal[j]>fSignal[j-1]) {
	  std::swap(fTrack[j-1],fTrack[j]);
	  std::swap(fHits[j-1] ,fHits[j]);
	  std::swap(fSignal[j-1],fSignal[j]);
	  j--;
	} // end while
      } // end if i
    } // end if added to existing and resorted array
    return;
  }
  //
  // new entry add it in order.
  // if this signal is <= smallest then don't add it.
  if (fNTracks==(kBuffSize-1) && signal <= fSignal[kBuffSize-1]) return;
  //
  for (i=fNTracks;i--;) {
    if (signal > fSignal[i]) {
      fSignal[i+1] = fSignal[i];
      fTrack[i+1]  = fTrack[i];
      fHits[i+1]   = fHits[i];
    } else {
      fSignal[i+1] = signal;
      fTrack[i+1]  = track;
      fHits[i+1]   = hit;
      fNTracks++;
      return; // put it in the right place, now exit.
    } //  end if
  } // end if; end for i
  //
  // Still haven't found the right place. Must be at top of list.
  fSignal[0] = signal;
  fTrack[0]  = track;
  fHits[0]   = hit;
  fNTracks++;
  return;
}

//______________________________________________________________________
void AliITSUSDigit::Add(const AliITSUSDigit *pl)
{
  // Adds the contents of pl to this
  // pl could come from different module and index
  Double_t sigT = 0.0;
  for(int i=pl->GetNTracks();i--;) {
    double sig = pl->GetSignal(i);
    AddSignal(pl->GetTrack(i),pl->GetHit(i),sig);
    sigT += sig;
  } // end for i
  fTsignal += (pl->fTsignal - sigT);
  fNoise   += pl->fNoise;
  return;
  //
}

//______________________________________________________________________
void AliITSUSDigit::AddTo(Int_t fileIndex,const AliITSUSDigit *pl)
{
  // Adds the contents of pl to this with track number off set given by
  // fileIndex.
  // Inputs:
  //    Int_t fileIndex      track number offset value
  //    AliITSUSDigit *pl  an AliITSUSDigit to be added to this class.
  //
  for (int i=pl->GetNTracks();i--;) AddSignal(pl->GetTrack(i)+fileIndex,pl->GetHit(i),pl->GetSignal(i));
  fSignalAfterElect += (pl->fSignalAfterElect + pl->fNoise - fNoise);
  fNoise = pl->fNoise;
}

//______________________________________________________________________
void AliITSUSDigit::ShiftIndices(Int_t fileIndex)
{
  // Shift track numbers
  //
  for (int i=GetNTracks();i--;) fTrack[i] += fileIndex;
}

//______________________________________________________________________
Int_t AliITSUSDigit::Compare(const TObject* obj) const
{
  // compare objects
  if (GetUniqueID()<obj->GetUniqueID()) return -1;
  if (GetUniqueID()>obj->GetUniqueID()) return  1;
  return 0;
}

//______________________________________________________________________
void AliITSUSDigit::Print(Option_t*) const
{
  // print itself
  printf("Mod: %4d Index:%7d Ntr:%2d | TotSignal:%.2e Noise:%.2e |",
	 fModule,GetUniqueID(),fNTracks,fTsignal,fNoise);
  for (int i=0;i<fNTracks;i++) printf("%d(%.2e) |",fTrack[i],fSignal[i]); printf("\n");
}
<commit_msg>Update missed in previous commit<commit_after>/**************************************************************************
 * Copyright(c) 1998-2004, ALICE Experiment at CERN, All rights reserved. *
 *                                                                        *
 * Author: The ALICE Off-line Project.                                    *
 * Contributors are mentioned in the code where appropriate.              *
 *                                                                        *
 * Permission to use, copy, modify and distribute this software and its   *
 * documentation strictly for non-commercial purposes is hereby granted   *
 * without fee, provided that the above copyright notice appears in all   *
 * copies and that both the copyright notice and this permission notice   *
 * appear in the supporting documentation. The authors make no claims     *
 * about the suitability of this software for any purpose. It is          *
 * provided "as is" without express or implied warranty.                  *
 **************************************************************************/
#include <TMath.h>
#include "AliLog.h"
#include "AliITSUSDigit.h"
ClassImp(AliITSUSDigit)

using std::istream;
using std::swap;
using namespace TMath;

// Addapted from ITS/AliITSpListItem, ruben.shahoyan@cern.ch

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit()
: fModule(0)
  ,fNTracks(0)
  ,fROCycle(0)
  ,fTsignal(0.0)
  ,fNoise(0.0)
  ,fSignalAfterElect(0.0)
{
  // Default constructor
  for (int i=kBuffSize;i--;) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(UInt_t module,UInt_t index,Double_t noise,Int_t roCycle)
  :fModule(module)
  ,fNTracks(0)
  ,fROCycle(roCycle)
  ,fTsignal(0.0)
  ,fNoise(noise)
  ,fSignalAfterElect(0.0)
{
  // Standard noise constructor
  SetUniqueID(index);
  for (int i=kBuffSize;i--;) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(Int_t track,Int_t hit,UInt_t module,UInt_t index,Double_t signal,Int_t roCycle)
  :fModule(module)
  ,fNTracks(1)
  ,fROCycle(roCycle)
  ,fTsignal(signal)
  ,fNoise(0.0)
  ,fSignalAfterElect(0.0)
{
  // Standard signal constructor
  // Inputs:
  //    Int_t track     The track number which produced this signal
  //    Int_t hit       The hit number which produced this signal
  //    Int_t module    The module where this signal occurred
  //    Int_t index     The cell index where this signal occurred
  //    Double_t signal The value of the signal (ionization)
  //    Int_t roCycle   Read-Out cycle
  SetUniqueID(index);
  fTrack[0]  = track;
  fHits[0]   = hit;
  fSignal[0] = signal;
  for (int i=1;i<kBuffSize;i++) {
    fTrack[i] = -2;
    fHits[i] = -1;
    fSignal[i] = 0;
  }
}

//______________________________________________________________________
AliITSUSDigit& AliITSUSDigit::operator=(const AliITSUSDigit &source)
{
  // = operator
  if (&source!=this) {
    this->~AliITSUSDigit();
    new(this) AliITSUSDigit(source);
  }
  return *this;
  //
}

//______________________________________________________________________
AliITSUSDigit::AliITSUSDigit(const AliITSUSDigit &source)
  :TObject(source)
  ,fModule(source.fModule)
  ,fNTracks(source.fNTracks)
  ,fROCycle(source.fROCycle)
  ,fTsignal(source.fTsignal)
  ,fNoise(source.fNoise)
  ,fSignalAfterElect(source.fSignalAfterElect)
{
  // Copy operator
  for(Int_t i=kBuffSize;i--;) {
    fTrack[i]  = source.fTrack[i];
    fSignal[i] = source.fSignal[i];
    fHits[i]   = source.fHits[i];
  } // end if i
  //
}

//______________________________________________________________________
void AliITSUSDigit::AddSignal(Int_t track,Int_t hit,Double_t signal)
{
  // Adds this track number and signal to the pList and orders them
  // Inputs:
  //    Int_t track     The track number which produced this signal
  //    Int_t hit       The hit number which produced this signal
  //    Int_t module    The module where this signal occurred
  //    Int_t index     The cell index where this signal occurred
  //    Double_t signal The value of the signal (ionization)
  Int_t    i,j;
  Bool_t   flg=kFALSE;
  //
  if (Abs(signal)>2147483647.0) {
    //PH 2147483647 is the max. integer
    //PH This apparently is a problem which needs investigation
    AliWarning(Form("Too big or too small signal value %f",signal));
    signal = Sign((Double_t)2147483647,signal);
  }
  //
  fTsignal += signal; // Keep track of sum signal.
  for (i=fNTracks;i--;) {
    if ( track==fTrack[i]  ) {
      fSignal[i] += signal;
      flg = kTRUE;
      break;
    } // end for i & if.
  }
  //
  if (flg) {
    if (fNTracks>1) { // resort arrays.
      for (i=1;i<fNTracks;i++) {
	j = i;
	while(j>0 && fSignal[j]>fSignal[j-1]) {
	  std::swap(fTrack[j-1],fTrack[j]);
	  std::swap(fHits[j-1] ,fHits[j]);
	  std::swap(fSignal[j-1],fSignal[j]);
	  j--;
	} // end while
      } // end if i
    } // end if added to existing and resorted array
    return;
  }
  //
  // new entry add it in order.
  // if this signal is <= smallest then don't add it.
  if (fNTracks==(kBuffSize-1) && signal <= fSignal[kBuffSize-1]) return;
  //
  for (i=fNTracks;i--;) {
    if (signal > fSignal[i]) {
      fSignal[i+1] = fSignal[i];
      fTrack[i+1]  = fTrack[i];
      fHits[i+1]   = fHits[i];
    } else {
      fSignal[i+1] = signal;
      fTrack[i+1]  = track;
      fHits[i+1]   = hit;
      fNTracks++;
      return; // put it in the right place, now exit.
    } //  end if
  } // end if; end for i
  //
  // Still haven't found the right place. Must be at top of list.
  fSignal[0] = signal;
  fTrack[0]  = track;
  fHits[0]   = hit;
  fNTracks++;
  return;
}

//______________________________________________________________________
void AliITSUSDigit::Add(const AliITSUSDigit *pl)
{
  // Adds the contents of pl to this
  // pl could come from different module and index
  Double_t sigT = 0.0;
  for(int i=pl->GetNTracks();i--;) {
    double sig = pl->GetSignal(i);
    AddSignal(pl->GetTrack(i),pl->GetHit(i),sig);
    sigT += sig;
  } // end for i
  fTsignal += (pl->fTsignal - sigT);
  fNoise   += pl->fNoise;
  return;
  //
}

//______________________________________________________________________
void AliITSUSDigit::AddTo(Int_t fileIndex,const AliITSUSDigit *pl)
{
  // Adds the contents of pl to this with track number off set given by
  // fileIndex.
  // Inputs:
  //    Int_t fileIndex      track number offset value
  //    AliITSUSDigit *pl  an AliITSUSDigit to be added to this class.
  //
  for (int i=pl->GetNTracks();i--;) AddSignal(pl->GetTrack(i)+fileIndex,pl->GetHit(i),pl->GetSignal(i));
  fSignalAfterElect += (pl->fSignalAfterElect + pl->fNoise - fNoise);
  fNoise = pl->fNoise;
}

//______________________________________________________________________
void AliITSUSDigit::ShiftIndices(Int_t fileIndex)
{
  // Shift track numbers
  //
  for (int i=GetNTracks();i--;) fTrack[i] += fileIndex;
}

//______________________________________________________________________
Int_t AliITSUSDigit::Compare(const TObject* obj) const
{
  // compare objects
  if (GetUniqueID()<obj->GetUniqueID()) return -1;
  if (GetUniqueID()>obj->GetUniqueID()) return  1;
  return 0;
}

//______________________________________________________________________
void AliITSUSDigit::Print(Option_t*) const
{
  // print itself
  printf("Mod: %4d Index:%7d Ntr:%2d | TotSignal:%.2e Noise:%.2e ROCycle: %d|",
	 fModule,GetUniqueID(),fNTracks,fTsignal,fNoise,fROCycle);
  for (int i=0;i<fNTracks;i++) printf("%d(%.2e) |",fTrack[i],fSignal[i]); printf("\n");
}
<|endoftext|>