<commit_before>// Copyright (c) 2002-2010, Boyce Griffith
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//    * Redistributions of source code must retain the above copyright notice,
//      this list of conditions and the following disclaimer.
//
//    * Redistributions in binary form must reproduce the above copyright
//      notice, this list of conditions and the following disclaimer in the
//      documentation and/or other materials provided with the distribution.
//
//    * Neither the name of New York University nor the names of its
//      contributors may be used to endorse or promote products derived from
//      this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

// Config files
#include <IBTK_prefix_config.h>
#include <SAMRAI_config.h>

// Headers for basic PETSc objects
#include <petscsys.h>

// Headers for major SAMRAI objects
#include <BergerRigoutsos.h>
#include <CartesianGridGeometry.h>
#include <GriddingAlgorithm.h>
#include <LoadBalancer.h>
#include <StandardTagAndInitialize.h>

// Headers for application-specific algorithm/data structure objects
#include <ibtk/AppInitializer.h>
#include <ibtk/CCLaplaceOperator.h>
#include <ibtk/muParserCartGridFunction.h>
#include <ibtk/app_namespaces.h>

/*******************************************************************************
 * For each run, the input filename must be given on the command line.  In all *
 * cases, the command line is:                                                 *
 *                                                                             *
 *    executable <input file name>                                             *
 *                                                                             *
 *******************************************************************************/
int
main(
    int argc,
    char *argv[])
{
    // Initialize PETSc, MPI, and SAMRAI.
    PetscInitialize(&argc,&argv,PETSC_NULL,PETSC_NULL);
    SAMRAI_MPI::setCommunicator(PETSC_COMM_WORLD);
    SAMRAI_MPI::setCallAbortInSerialInsteadOfExit();
    SAMRAIManager::startup();

    {// cleanup dynamically allocated objects prior to shutdown

        // Parse command line options, set some standard options from the input
        // file, and enable file logging.
        Pointer<AppInitializer> app_initializer = new AppInitializer(argc, argv, "cc_laplace.log");
        Pointer<Database> input_db = app_initializer->getInputDatabase();

        // Create major algorithm and data objects that comprise the
        // application.  These objects are configured from the input database.
        Pointer<CartesianGridGeometry<NDIM> > grid_geometry = new CartesianGridGeometry<NDIM>(
            "CartesianGeometry", app_initializer->getComponentDatabase("CartesianGeometry"));
        Pointer<PatchHierarchy<NDIM> > patch_hierarchy = new PatchHierarchy<NDIM>(
            "PatchHierarchy",grid_geometry);
        Pointer<StandardTagAndInitialize<NDIM> > error_detector = new StandardTagAndInitialize<NDIM>(
            "StandardTagAndInitialize", NULL, app_initializer->getComponentDatabase("StandardTagAndInitialize"));
        Pointer<BergerRigoutsos<NDIM> > box_generator = new BergerRigoutsos<NDIM>();
        Pointer<LoadBalancer<NDIM> > load_balancer = new LoadBalancer<NDIM>(
            "LoadBalancer", app_initializer->getComponentDatabase("LoadBalancer"));
        Pointer<GriddingAlgorithm<NDIM> > gridding_algorithm = new GriddingAlgorithm<NDIM>(
            "GriddingAlgorithm", app_initializer->getComponentDatabase("GriddingAlgorithm"), error_detector, box_generator, load_balancer);

        // Create variables and register them with the variable database.
        VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
        Pointer<VariableContext> ctx = var_db->getContext("context");

        Pointer<CellVariable<NDIM,double> > u_cc_var = new CellVariable<NDIM,double>("u_cc");
        Pointer<CellVariable<NDIM,double> > f_cc_var = new CellVariable<NDIM,double>("f_cc");
        Pointer<CellVariable<NDIM,double> > e_cc_var = new CellVariable<NDIM,double>("e_cc");

        const int u_cc_idx = var_db->registerVariableAndContext(u_cc_var, ctx, IntVector<NDIM>(1));
        const int f_cc_idx = var_db->registerVariableAndContext(f_cc_var, ctx, IntVector<NDIM>(1));
        const int e_cc_idx = var_db->registerVariableAndContext(e_cc_var, ctx, IntVector<NDIM>(1));

        // Register variables for plotting.
        Pointer<VisItDataWriter<NDIM> > visit_data_writer = app_initializer->getVisItDataWriter();
        TBOX_ASSERT(!visit_data_writer.isNull());

        visit_data_writer->registerPlotQuantity(u_cc_var->getName(), "SCALAR", u_cc_idx);
        visit_data_writer->registerPlotQuantity(f_cc_var->getName(), "SCALAR", f_cc_idx);
        visit_data_writer->registerPlotQuantity(e_cc_var->getName(), "SCALAR", e_cc_idx);

        // Initialize the AMR patch hierarchy.
        gridding_algorithm->makeCoarsestLevel(patch_hierarchy, 0.0);
        int tag_buffer = 1;
        int level_number = 0;
        bool done = false;
        while (!done && (gridding_algorithm->levelCanBeRefined(level_number)))
        {
            gridding_algorithm->makeFinerLevel(patch_hierarchy, 0.0, 0.0, tag_buffer);
            done = !patch_hierarchy->finerLevelExists(level_number);
            ++level_number;
        }

        // Allocate data on each level of the patch hierarchy.
        for (int ln = 0; ln <= patch_hierarchy->getFinestLevelNumber(); ++ln)
        {
            Pointer<PatchLevel<NDIM> > level = patch_hierarchy->getPatchLevel(ln);
            level->allocatePatchData(u_cc_idx, 0.0);
            level->allocatePatchData(f_cc_idx, 0.0);
            level->allocatePatchData(e_cc_idx, 0.0);
        }

        // Setup vector objects.
        HierarchyMathOps hier_math_ops("hier_math_ops", patch_hierarchy);
        const int h_cc_idx = hier_math_ops.getCellWeightPatchDescriptorIndex();

        SAMRAIVectorReal<NDIM,double> u_vec("u", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());
        SAMRAIVectorReal<NDIM,double> f_vec("f", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());
        SAMRAIVectorReal<NDIM,double> e_vec("e", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());

        u_vec.addComponent(u_cc_var, u_cc_idx, h_cc_idx);
        f_vec.addComponent(f_cc_var, f_cc_idx, h_cc_idx);
        e_vec.addComponent(e_cc_var, e_cc_idx, h_cc_idx);

        u_vec.setToScalar(0.0);
        f_vec.setToScalar(0.0);
        e_vec.setToScalar(0.0);

        // Setup exact solutions.
        muParserCartGridFunction u_fcn("u", app_initializer->getComponentDatabase("u"), grid_geometry);
        muParserCartGridFunction f_fcn("f", app_initializer->getComponentDatabase("f"), grid_geometry);

        u_fcn.setDataOnPatchHierarchy(u_cc_idx, u_cc_var, patch_hierarchy, 0.0);
        f_fcn.setDataOnPatchHierarchy(e_cc_idx, e_cc_var, patch_hierarchy, 0.0);

        // Compute -L*u = f.
        PoissonSpecifications poisson_spec("poisson_spec");
        poisson_spec.setCConstant( 0.0);
        poisson_spec.setDConstant(-1.0);
        RobinBcCoefStrategy<NDIM>* bc_coef = NULL;
        CCLaplaceOperator laplace_op("laplace op", poisson_spec, bc_coef);
        laplace_op.initializeOperatorState(u_vec,f_vec);
        laplace_op.apply(u_vec,f_vec);

        // Compute error and print error norms.
        e_vec.subtract(Pointer<SAMRAIVectorReal<NDIM,double> >(&e_vec,false),
                       Pointer<SAMRAIVectorReal<NDIM,double> >(&f_vec,false));
        pout << "|e|_oo = " << e_vec.maxNorm() << "\n";
        pout << "|e|_2  = " << e_vec. L2Norm() << "\n";
        pout << "|e|_1  = " << e_vec. L1Norm() << "\n";

        // Set invalid values on coarse levels (i.e., coarse-grid values that
        // are covered by finer grid patches) to equal zero.
        for (int ln = 0; ln <= patch_hierarchy->getFinestLevelNumber()-1; ++ln)
        {
            Pointer<PatchLevel<NDIM> > level = patch_hierarchy->getPatchLevel(ln);
            BoxArray<NDIM> refined_region_boxes;
            Pointer<PatchLevel<NDIM> > next_finer_level = patch_hierarchy->getPatchLevel(ln+1);
            refined_region_boxes = next_finer_level->getBoxes();
            refined_region_boxes.coarsen(next_finer_level->getRatioToCoarserLevel());
            for (PatchLevel<NDIM>::Iterator p(level); p; p++)
            {
                Pointer<Patch<NDIM> > patch = level->getPatch(p());
                const Box<NDIM>& patch_box = patch->getBox();
                Pointer<CellData<NDIM,double> > e_cc_data = patch->getPatchData(e_cc_idx);
                for (int i = 0; i < refined_region_boxes.getNumberOfBoxes(); ++i)
                {
                    const Box<NDIM> refined_box = refined_region_boxes[i];
                    const Box<NDIM> intersection = Box<NDIM>::grow(patch_box,1)*refined_box;
                    if (!intersection.empty())
                    {
                        e_cc_data->fillAll(0.0, intersection);
                    }
                }
            }
        }

        // Output data for plotting.
        visit_data_writer->writePlotData(patch_hierarchy, 0, 0.0);

    }// cleanup dynamically allocated objects prior to shutdown

    SAMRAIManager::shutdown();
    PetscFinalize();
    return 0;
}// main
<commit_msg>branch boyceg: updating CCLaplace example main.C<commit_after>// Copyright (c) 2002-2010, Boyce Griffith
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//    * Redistributions of source code must retain the above copyright notice,
//      this list of conditions and the following disclaimer.
//
//    * Redistributions in binary form must reproduce the above copyright
//      notice, this list of conditions and the following disclaimer in the
//      documentation and/or other materials provided with the distribution.
//
//    * Neither the name of New York University nor the names of its
//      contributors may be used to endorse or promote products derived from
//      this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

// Config files
#include <IBTK_prefix_config.h>
#include <SAMRAI_config.h>

// Headers for basic PETSc objects
#include <petscsys.h>

// Headers for major SAMRAI objects
#include <BergerRigoutsos.h>
#include <CartesianGridGeometry.h>
#include <GriddingAlgorithm.h>
#include <LoadBalancer.h>
#include <StandardTagAndInitialize.h>

// Headers for application-specific algorithm/data structure objects
#include <ibtk/AppInitializer.h>
#include <ibtk/CCLaplaceOperator.h>
#include <ibtk/muParserCartGridFunction.h>
#include <ibtk/app_namespaces.h>

/*******************************************************************************
 * For each run, the input filename must be given on the command line.  In all *
 * cases, the command line is:                                                 *
 *                                                                             *
 *    executable <input file name>                                             *
 *                                                                             *
 *******************************************************************************/
int
main(
    int argc,
    char *argv[])
{
    // Initialize PETSc, MPI, and SAMRAI.
    PetscInitialize(&argc,&argv,PETSC_NULL,PETSC_NULL);
    SAMRAI_MPI::setCommunicator(PETSC_COMM_WORLD);
    SAMRAI_MPI::setCallAbortInSerialInsteadOfExit();
    SAMRAIManager::startup();

    {// cleanup dynamically allocated objects prior to shutdown

        // Parse command line options, set some standard options from the input
        // file, and enable file logging.
        Pointer<AppInitializer> app_initializer = new AppInitializer(argc, argv, "cc_laplace.log");
        Pointer<Database> input_db = app_initializer->getInputDatabase();

        // Create major algorithm and data objects that comprise the
        // application.  These objects are configured from the input database.
        Pointer<CartesianGridGeometry<NDIM> > grid_geometry = new CartesianGridGeometry<NDIM>("CartesianGeometry", app_initializer->getComponentDatabase("CartesianGeometry"));
        Pointer<PatchHierarchy<NDIM> > patch_hierarchy = new PatchHierarchy<NDIM>("PatchHierarchy",grid_geometry);
        Pointer<StandardTagAndInitialize<NDIM> > error_detector = new StandardTagAndInitialize<NDIM>("StandardTagAndInitialize", NULL, app_initializer->getComponentDatabase("StandardTagAndInitialize"));
        Pointer<BergerRigoutsos<NDIM> > box_generator = new BergerRigoutsos<NDIM>();
        Pointer<LoadBalancer<NDIM> > load_balancer = new LoadBalancer<NDIM>("LoadBalancer", app_initializer->getComponentDatabase("LoadBalancer"));
        Pointer<GriddingAlgorithm<NDIM> > gridding_algorithm = new GriddingAlgorithm<NDIM>("GriddingAlgorithm", app_initializer->getComponentDatabase("GriddingAlgorithm"), error_detector, box_generator, load_balancer);

        // Create variables and register them with the variable database.
        VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
        Pointer<VariableContext> ctx = var_db->getContext("context");

        Pointer<CellVariable<NDIM,double> > u_cc_var = new CellVariable<NDIM,double>("u_cc");
        Pointer<CellVariable<NDIM,double> > f_cc_var = new CellVariable<NDIM,double>("f_cc");
        Pointer<CellVariable<NDIM,double> > e_cc_var = new CellVariable<NDIM,double>("e_cc");

        const int u_cc_idx = var_db->registerVariableAndContext(u_cc_var, ctx, IntVector<NDIM>(1));
        const int f_cc_idx = var_db->registerVariableAndContext(f_cc_var, ctx, IntVector<NDIM>(1));
        const int e_cc_idx = var_db->registerVariableAndContext(e_cc_var, ctx, IntVector<NDIM>(1));

        // Register variables for plotting.
        Pointer<VisItDataWriter<NDIM> > visit_data_writer = app_initializer->getVisItDataWriter();
        TBOX_ASSERT(!visit_data_writer.isNull());

        visit_data_writer->registerPlotQuantity(u_cc_var->getName(), "SCALAR", u_cc_idx);
        visit_data_writer->registerPlotQuantity(f_cc_var->getName(), "SCALAR", f_cc_idx);
        visit_data_writer->registerPlotQuantity(e_cc_var->getName(), "SCALAR", e_cc_idx);

        // Initialize the AMR patch hierarchy.
        gridding_algorithm->makeCoarsestLevel(patch_hierarchy, 0.0);
        int tag_buffer = 1;
        int level_number = 0;
        bool done = false;
        while (!done && (gridding_algorithm->levelCanBeRefined(level_number)))
        {
            gridding_algorithm->makeFinerLevel(patch_hierarchy, 0.0, 0.0, tag_buffer);
            done = !patch_hierarchy->finerLevelExists(level_number);
            ++level_number;
        }

        // Allocate data on each level of the patch hierarchy.
        for (int ln = 0; ln <= patch_hierarchy->getFinestLevelNumber(); ++ln)
        {
            Pointer<PatchLevel<NDIM> > level = patch_hierarchy->getPatchLevel(ln);
            level->allocatePatchData(u_cc_idx, 0.0);
            level->allocatePatchData(f_cc_idx, 0.0);
            level->allocatePatchData(e_cc_idx, 0.0);
        }

        // Setup vector objects.
        HierarchyMathOps hier_math_ops("hier_math_ops", patch_hierarchy);
        const int h_cc_idx = hier_math_ops.getCellWeightPatchDescriptorIndex();

        SAMRAIVectorReal<NDIM,double> u_vec("u", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());
        SAMRAIVectorReal<NDIM,double> f_vec("f", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());
        SAMRAIVectorReal<NDIM,double> e_vec("e", patch_hierarchy, 0, patch_hierarchy->getFinestLevelNumber());

        u_vec.addComponent(u_cc_var, u_cc_idx, h_cc_idx);
        f_vec.addComponent(f_cc_var, f_cc_idx, h_cc_idx);
        e_vec.addComponent(e_cc_var, e_cc_idx, h_cc_idx);

        u_vec.setToScalar(0.0);
        f_vec.setToScalar(0.0);
        e_vec.setToScalar(0.0);

        // Setup exact solutions.
        muParserCartGridFunction u_fcn("u", app_initializer->getComponentDatabase("u"), grid_geometry);
        muParserCartGridFunction f_fcn("f", app_initializer->getComponentDatabase("f"), grid_geometry);

        u_fcn.setDataOnPatchHierarchy(u_cc_idx, u_cc_var, patch_hierarchy, 0.0);
        f_fcn.setDataOnPatchHierarchy(e_cc_idx, e_cc_var, patch_hierarchy, 0.0);

        // Compute -L*u = f.
        PoissonSpecifications poisson_spec("poisson_spec");
        poisson_spec.setCConstant( 0.0);
        poisson_spec.setDConstant(-1.0);
        RobinBcCoefStrategy<NDIM>* bc_coef = NULL;
        CCLaplaceOperator laplace_op("laplace op", poisson_spec, bc_coef);
        laplace_op.initializeOperatorState(u_vec,f_vec);
        laplace_op.apply(u_vec,f_vec);

        // Compute error and print error norms.
        e_vec.subtract(Pointer<SAMRAIVectorReal<NDIM,double> >(&e_vec,false), Pointer<SAMRAIVectorReal<NDIM,double> >(&f_vec,false));
        pout << "|e|_oo = " << e_vec.maxNorm() << "\n";
        pout << "|e|_2  = " << e_vec. L2Norm() << "\n";
        pout << "|e|_1  = " << e_vec. L1Norm() << "\n";

        // Set invalid values on coarse levels (i.e., coarse-grid values that
        // are covered by finer grid patches) to equal zero.
        for (int ln = 0; ln <= patch_hierarchy->getFinestLevelNumber()-1; ++ln)
        {
            Pointer<PatchLevel<NDIM> > level = patch_hierarchy->getPatchLevel(ln);
            BoxArray<NDIM> refined_region_boxes;
            Pointer<PatchLevel<NDIM> > next_finer_level = patch_hierarchy->getPatchLevel(ln+1);
            refined_region_boxes = next_finer_level->getBoxes();
            refined_region_boxes.coarsen(next_finer_level->getRatioToCoarserLevel());
            for (PatchLevel<NDIM>::Iterator p(level); p; p++)
            {
                Pointer<Patch<NDIM> > patch = level->getPatch(p());
                const Box<NDIM>& patch_box = patch->getBox();
                Pointer<CellData<NDIM,double> > e_cc_data = patch->getPatchData(e_cc_idx);
                for (int i = 0; i < refined_region_boxes.getNumberOfBoxes(); ++i)
                {
                    const Box<NDIM> refined_box = refined_region_boxes[i];
                    const Box<NDIM> intersection = Box<NDIM>::grow(patch_box,1)*refined_box;
                    if (!intersection.empty())
                    {
                        e_cc_data->fillAll(0.0, intersection);
                    }
                }
            }
        }

        // Output data for plotting.
        visit_data_writer->writePlotData(patch_hierarchy, 0, 0.0);

    }// cleanup dynamically allocated objects prior to shutdown

    SAMRAIManager::shutdown();
    PetscFinalize();
    return 0;
}// main
<|endoftext|>


<commit_before>#include "stdafx.h"

void*	NULLC::defaultAlloc(int size)
{
	return ::new(std::nothrow) char[size];
}
void	NULLC::defaultDealloc(void* ptr)
{
	::delete[] (char*)ptr;
}

void*	(*NULLC::alloc)(int) = NULLC::defaultAlloc;
void	(*NULLC::dealloc)(void*) = NULLC::defaultDealloc;

void* NULLC::alignedAlloc(int size)
{
	void *unaligned = alloc((size + 16 - 1) + sizeof(void*));
	if(!unaligned)
		return NULL;
	void *ptr = (void*)(((intptr_t)unaligned + sizeof(void*) + 16 - 1) & ~(16 - 1));
	*((void**)ptr - 1) = unaligned;
	return ptr;
}

void* NULLC::alignedAlloc(int size, int extraSize)
{
	void *unaligned = alloc((size + 16 - 1) + sizeof(void*) + extraSize);
	if(!unaligned)
		return NULL;
	void *ptr = (void*)((((intptr_t)unaligned + sizeof(void*) + extraSize + 16 - 1) & ~(16 - 1)) - extraSize);
	*((void**)ptr - 1) = unaligned;
	return ptr;
}

void NULLC::alignedDealloc(void* ptr)
{
	dealloc(*((void **)ptr - 1));
}

const void*	NULLC::defaultFileLoad(const char* name, unsigned int* size, int* nullcShouldFreePtr)
{
	assert(name);
	assert(size);
	assert(nullcShouldFreePtr);

	FILE *file = fopen(name, "rb");
	if(file)
	{
		fseek(file, 0, SEEK_END);
		*size = ftell(file);
		fseek(file, 0, SEEK_SET);
		char *fileContent = (char*)NULLC::alloc(*size + 1);
		fread(fileContent, 1, *size, file);
		fileContent[*size] = 0;
		fclose(file);
		*nullcShouldFreePtr = 1;
		return fileContent;
	}
	*size = 0;
	*nullcShouldFreePtr = false;
	return NULL;
}

const void*	(*NULLC::fileLoad)(const char*, unsigned int*, int*) = NULLC::defaultFileLoad;
<commit_msg>Unaligned memory access fix<commit_after>#include "stdafx.h"

void*	NULLC::defaultAlloc(int size)
{
	return ::new(std::nothrow) char[size];
}
void	NULLC::defaultDealloc(void* ptr)
{
	::delete[] (char*)ptr;
}

void*	(*NULLC::alloc)(int) = NULLC::defaultAlloc;
void	(*NULLC::dealloc)(void*) = NULLC::defaultDealloc;

void* NULLC::alignedAlloc(int size)
{
	void *unaligned = alloc((size + 16 - 1) + sizeof(void*));
	if(!unaligned)
		return NULL;
	void *ptr = (void*)(((intptr_t)unaligned + sizeof(void*) + 16 - 1) & ~(16 - 1));
	memcpy((void**)ptr - 1, &unaligned, sizeof(unaligned));
	return ptr;
}

void* NULLC::alignedAlloc(int size, int extraSize)
{
	void *unaligned = alloc((size + 16 - 1) + sizeof(void*) + extraSize);
	if(!unaligned)
		return NULL;
	void *ptr = (void*)((((intptr_t)unaligned + sizeof(void*) + extraSize + 16 - 1) & ~(16 - 1)) - extraSize);
	memcpy((void**)ptr - 1, &unaligned, sizeof(unaligned));
	return ptr;
}

void NULLC::alignedDealloc(void* ptr)
{
	void* unaligned = NULL;
	memcpy(&unaligned, (void**)ptr - 1, sizeof(unaligned));
	dealloc(unaligned);
}

const void*	NULLC::defaultFileLoad(const char* name, unsigned int* size, int* nullcShouldFreePtr)
{
	assert(name);
	assert(size);
	assert(nullcShouldFreePtr);

	FILE *file = fopen(name, "rb");
	if(file)
	{
		fseek(file, 0, SEEK_END);
		*size = ftell(file);
		fseek(file, 0, SEEK_SET);
		char *fileContent = (char*)NULLC::alloc(*size + 1);
		fread(fileContent, 1, *size, file);
		fileContent[*size] = 0;
		fclose(file);
		*nullcShouldFreePtr = 1;
		return fileContent;
	}
	*size = 0;
	*nullcShouldFreePtr = false;
	return NULL;
}

const void*	(*NULLC::fileLoad)(const char*, unsigned int*, int*) = NULLC::defaultFileLoad;
<|endoftext|>


<commit_before>/* Copyright (C) 2000 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex,select_result *result)
{
  SELECT_LEX *sl, *last_sl, *lex_sl;
  ORDER *order;
  List<Item> item_list;
  TABLE *table;
  int describe=(lex->select_lex.options & SELECT_DESCRIBE) ? 1 : 0;
  int res;
  bool fr=false;
  TABLE_LIST result_table_list;
  TABLE_LIST *first_table=(TABLE_LIST *)lex->select_lex.table_list.first;
  TMP_TABLE_PARAM tmp_table_param;
  select_union *union_result;
  DBUG_ENTER("mysql_union");

  /* Fix tables 'to-be-unioned-from' list to point at opened tables */
  last_sl= &lex->select_lex;
  for (sl= last_sl;
       sl && sl->linkage != NOT_A_SELECT;
       last_sl=sl, sl=sl->next)
  {
    for (TABLE_LIST *cursor= (TABLE_LIST *)sl->table_list.first;
	 cursor;
	 cursor=cursor->next)
      cursor->table= ((TABLE_LIST*) cursor->table)->table;
  }

  /* last_sel now points at the last select where the ORDER BY is stored */
  if (sl)
  {
    /*
      The found SL is an extra SELECT_LEX argument that contains
      the ORDER BY and LIMIT parameter for the whole UNION
    */
    lex_sl= sl;
    order=  (ORDER *) lex_sl->order_list.first;
    fr = lex->select_lex.options & OPTION_FOUND_ROWS && !describe && sl->select_limit && sl->select_limit != HA_POS_ERROR;
    if (!order || !describe)
      last_sl->next=0;				// Remove this extra element
  }
  else if (!last_sl->braces)
  {
    lex_sl= last_sl;				// ORDER BY is here
    order=  (ORDER *) lex_sl->order_list.first;
  }
  else
  {
    lex_sl=0;
    order=0;
  }

  if (describe)
  {
    Item *item;
    item_list.push_back(new Item_empty_string("table",NAME_LEN));
    item_list.push_back(new Item_empty_string("type",10));
    item_list.push_back(item=new Item_empty_string("possible_keys",
						  NAME_LEN*MAX_KEY));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("key",NAME_LEN));
    item->maybe_null=1;
    item_list.push_back(item=new Item_int("key_len",0,3));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("ref",
						    NAME_LEN*MAX_REF_PARTS));
    item->maybe_null=1;
    item_list.push_back(new Item_real("rows",0.0,0,10));
    item_list.push_back(new Item_empty_string("Extra",255));
  }
  else
  {
    Item *item;
    List_iterator<Item> it(lex->select_lex.item_list);
    TABLE_LIST *first_table= (TABLE_LIST*) lex->select_lex.table_list.first;

    /* Create a list of items that will be in the result set */
    while ((item= it++))
      if (item_list.push_back(item))
	DBUG_RETURN(-1);
    if (setup_fields(thd,first_table,item_list,0,0,1))
      DBUG_RETURN(-1);
  }

  bzero((char*) &tmp_table_param,sizeof(tmp_table_param));
  tmp_table_param.field_count=item_list.elements;
  if (!(table=create_tmp_table(thd, &tmp_table_param, item_list,
			       (ORDER*) 0, !describe & !lex->union_option,
			       1, 0,
			       (lex->select_lex.options | thd->options |
				TMP_TABLE_ALL_COLUMNS))))
    DBUG_RETURN(-1);
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.name=(char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
  {
    res= -1;
    goto exit;
  }
  union_result->save_time_stamp=!describe;

  for (sl= &lex->select_lex; sl; sl=sl->next)
  {
    lex->select=sl;
    thd->offset_limit=sl->offset_limit;
    thd->select_limit=sl->select_limit+sl->offset_limit;
    if (thd->select_limit < sl->select_limit)
      thd->select_limit= HA_POS_ERROR;		// no limit
    if (thd->select_limit == HA_POS_ERROR)
      sl->options&= ~OPTION_FOUND_ROWS;

    res=mysql_select(thd, (describe && sl->linkage==NOT_A_SELECT) ? first_table :  (TABLE_LIST*) sl->table_list.first,
		     sl->item_list,
		     sl->where,
		     (sl->braces) ? (ORDER *)sl->order_list.first : (ORDER *) 0,
		     (ORDER*) sl->group_list.first,
		     sl->having,
		     (ORDER*) NULL,
		     sl->options | thd->options | SELECT_NO_UNLOCK | ((describe) ? SELECT_DESCRIBE : 0),
		     union_result);
    if (res)
      goto exit;
  }
  if (union_result->flush())
  {
    res= 1;					// Error is already sent
    goto exit;
  }
  delete union_result;

  /* Send result to 'result' */
  lex->select = &lex->select_lex;
  res =-1;
  {
    /* Create a list of fields in the temporary table */
    List_iterator<Item> it(item_list);
    Field **field;
#if 0
    List<Item_func_match> ftfunc_list;
    ftfunc_list.empty();
#else
    thd->lex.select_lex.ftfunc_list.empty();
#endif

    for (field=table->field ; *field ; field++)
    {
      (void) it++;
      (void) it.replace(new Item_field(*field));
    }
    if (!thd->fatal_error)			// Check if EOM
    {
      if (lex_sl)
      {
	thd->offset_limit=lex_sl->offset_limit;
	thd->select_limit=lex_sl->select_limit+lex_sl->offset_limit;
	if (thd->select_limit < lex_sl->select_limit)
	  thd->select_limit= HA_POS_ERROR;		// no limit
	if (thd->select_limit == HA_POS_ERROR)
	  thd->options&= ~OPTION_FOUND_ROWS;
      }
      else
      {
	thd->offset_limit= 0;
	thd->select_limit= thd->default_select_limit;
      }
      if (describe)
	thd->select_limit= HA_POS_ERROR;		// no limit
      res=mysql_select(thd,&result_table_list,
		       item_list, NULL, (describe) ? 0 : order,
		       (ORDER*) NULL, NULL, (ORDER*) NULL,
		       thd->options, result);
      if (fr && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
    }
  }

exit:
  free_tmp_table(thd,table);
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
    :table(table_par)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates=DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list)
{
  if (save_time_stamp && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (thd->offset_limit)
  {						// using limit offset,count
    thd->offset_limit--;
    return 0;
  }
  fill_record(table->field,values);
  return write_record(table,&info) ? 1 : 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(&thd->net);
    return 1;
  }
  return 0;
}
<commit_msg>Adding some things according to Monty's correct observations<commit_after>/* Copyright (C) 2000 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex,select_result *result)
{
  SELECT_LEX *sl, *last_sl, *lex_sl;
  ORDER *order;
  List<Item> item_list;
  TABLE *table;
  int describe=(lex->select_lex.options & SELECT_DESCRIBE) ? 1 : 0;
  int res;
  bool found_rows_for_union=false;
  TABLE_LIST result_table_list;
  TABLE_LIST *first_table=(TABLE_LIST *)lex->select_lex.table_list.first;
  TMP_TABLE_PARAM tmp_table_param;
  select_union *union_result;
  DBUG_ENTER("mysql_union");

  /* Fix tables 'to-be-unioned-from' list to point at opened tables */
  last_sl= &lex->select_lex;
  for (sl= last_sl;
       sl && sl->linkage != NOT_A_SELECT;
       last_sl=sl, sl=sl->next)
  {
    for (TABLE_LIST *cursor= (TABLE_LIST *)sl->table_list.first;
	 cursor;
	 cursor=cursor->next)
      cursor->table= ((TABLE_LIST*) cursor->table)->table;
  }

  /* last_sel now points at the last select where the ORDER BY is stored */
  if (sl)
  {
    /*
      The found SL is an extra SELECT_LEX argument that contains
      the ORDER BY and LIMIT parameter for the whole UNION
    */
    lex_sl= sl;
    order=  (ORDER *) lex_sl->order_list.first;
    found_rows_for_union = lex->select_lex.options & OPTION_FOUND_ROWS && !describe && sl->select_limit;
    if (found_rows_for_union)
      lex->select_lex.options ^=  OPTION_FOUND_ROWS;
// This is done to eliminate unnecessary slowing down of the first query
    if (!order || !describe)
      last_sl->next=0;				// Remove this extra element
  }
  else if (!last_sl->braces)
  {
    lex_sl= last_sl;				// ORDER BY is here
    order=  (ORDER *) lex_sl->order_list.first;
  }
  else
  {
    lex_sl=0;
    order=0;
  }

  if (describe)
  {
    Item *item;
    item_list.push_back(new Item_empty_string("table",NAME_LEN));
    item_list.push_back(new Item_empty_string("type",10));
    item_list.push_back(item=new Item_empty_string("possible_keys",
						  NAME_LEN*MAX_KEY));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("key",NAME_LEN));
    item->maybe_null=1;
    item_list.push_back(item=new Item_int("key_len",0,3));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("ref",
						    NAME_LEN*MAX_REF_PARTS));
    item->maybe_null=1;
    item_list.push_back(new Item_real("rows",0.0,0,10));
    item_list.push_back(new Item_empty_string("Extra",255));
  }
  else
  {
    Item *item;
    List_iterator<Item> it(lex->select_lex.item_list);
    TABLE_LIST *first_table= (TABLE_LIST*) lex->select_lex.table_list.first;

    /* Create a list of items that will be in the result set */
    while ((item= it++))
      if (item_list.push_back(item))
	DBUG_RETURN(-1);
    if (setup_fields(thd,first_table,item_list,0,0,1))
      DBUG_RETURN(-1);
  }

  bzero((char*) &tmp_table_param,sizeof(tmp_table_param));
  tmp_table_param.field_count=item_list.elements;
  if (!(table=create_tmp_table(thd, &tmp_table_param, item_list,
			       (ORDER*) 0, !describe & !lex->union_option,
			       1, 0,
			       (lex->select_lex.options | thd->options |
				TMP_TABLE_ALL_COLUMNS))))
    DBUG_RETURN(-1);
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.name=(char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
  {
    res= -1;
    goto exit;
  }
  union_result->save_time_stamp=!describe;

  for (sl= &lex->select_lex; sl; sl=sl->next)
  {
    lex->select=sl;
    thd->offset_limit=sl->offset_limit;
    thd->select_limit=sl->select_limit+sl->offset_limit;
    if (thd->select_limit < sl->select_limit)
      thd->select_limit= HA_POS_ERROR;		// no limit
    if (thd->select_limit == HA_POS_ERROR)
      sl->options&= ~OPTION_FOUND_ROWS;

    res=mysql_select(thd, (describe && sl->linkage==NOT_A_SELECT) ? first_table :  (TABLE_LIST*) sl->table_list.first,
		     sl->item_list,
		     sl->where,
		     (sl->braces) ? (ORDER *)sl->order_list.first : (ORDER *) 0,
		     (ORDER*) sl->group_list.first,
		     sl->having,
		     (ORDER*) NULL,
		     sl->options | thd->options | SELECT_NO_UNLOCK | ((describe) ? SELECT_DESCRIBE : 0),
		     union_result);
    if (res)
      goto exit;
  }
  if (union_result->flush())
  {
    res= 1;					// Error is already sent
    goto exit;
  }
  delete union_result;

  /* Send result to 'result' */
  lex->select = &lex->select_lex;
  res =-1;
  {
    /* Create a list of fields in the temporary table */
    List_iterator<Item> it(item_list);
    Field **field;
#if 0
    List<Item_func_match> ftfunc_list;
    ftfunc_list.empty();
#else
    thd->lex.select_lex.ftfunc_list.empty();
#endif

    for (field=table->field ; *field ; field++)
    {
      (void) it++;
      (void) it.replace(new Item_field(*field));
    }
    if (!thd->fatal_error)			// Check if EOM
    {
      if (lex_sl)
      {
	thd->offset_limit=lex_sl->offset_limit;
	thd->select_limit=lex_sl->select_limit+lex_sl->offset_limit;
	if (thd->select_limit < lex_sl->select_limit)
	  thd->select_limit= HA_POS_ERROR;		// no limit
	if (thd->select_limit == HA_POS_ERROR)
	  thd->options&= ~OPTION_FOUND_ROWS;
      }
      else
      {
	thd->offset_limit= 0;
	thd->select_limit= thd->default_select_limit;
      }
      if (describe)
	thd->select_limit= HA_POS_ERROR;		// no limit
      res=mysql_select(thd,&result_table_list,
		       item_list, NULL, (describe) ? 0 : order,
		       (ORDER*) NULL, NULL, (ORDER*) NULL,
		       thd->options, result);
      if (found_rows_for_union && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
    }
  }

exit:
  free_tmp_table(thd,table);
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
    :table(table_par)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates=DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list)
{
  if (save_time_stamp && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (thd->offset_limit)
  {						// using limit offset,count
    thd->offset_limit--;
    return 0;
  }
  fill_record(table->field,values);
  return write_record(table,&info) ? 1 : 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(&thd->net);
    return 1;
  }
  return 0;
}
<|endoftext|>


<commit_before>/*
 * File:   Wheel.hpp
 * Author: Barath Kannan
 *
 * Created on 13 June 2016, 12:50 AM
 */

#ifndef WHEEL_HPP
#define WHEEL_HPP

#include <atomic>

namespace BSignals{ namespace details{

//T must be default constructable
template <class T, uint32_t N>
class Wheel{
public:

    T& getSpoke(){
        return wheelymajig[fetchWrapIncrement()];
    }

    T& getSpoke(uint32_t index){
        return wheelymajig[index];
    }

    uint32_t getIndex(){
        return fetchWrapIncrement();
    }

    const uint32_t size(){
        return N;
    }

private:
    uint32_t fetchWrapIncrement(){
        uint32_t oldValue = currentElement.load();
        uint32_t newValue;
        do {
            newValue = (oldValue+1 == N) ? 0 : oldValue+1;
        } while (!currentElement.compare_exchange_weak(oldValue, newValue));
        return oldValue;
    }
    std::atomic<uint32_t> currentElement{0};
    std::array<T, N> wheelymajig;
};
}}

#endif /* WHEEL_HPP */

<commit_msg>noexcept and padding<commit_after>/*
 * File:   Wheel.hpp
 * Author: Barath Kannan
 *
 * Created on 13 June 2016, 12:50 AM
 */

#ifndef WHEEL_HPP
#define WHEEL_HPP

#include <atomic>

namespace BSignals{ namespace details{

//T must be default constructable
template <class T, uint32_t N>
class Wheel{
public:

    T& getSpoke() noexcept{
        return wheelymajig[fetchWrapIncrement()];
    }

    T& getSpokeRandom() noexcept{
        return wheelymajig[std::rand()%N];
    }

    T& getSpoke(uint32_t index) noexcept{
        return wheelymajig[index];
    }

    uint32_t getIndex() noexcept{
        return fetchWrapIncrement();
    }

    const uint32_t size() noexcept{
        return N;
    }

private:
    inline uint32_t fetchWrapIncrement() noexcept{
        uint32_t oldValue = currentElement.load();
        while (!currentElement.compare_exchange_weak(oldValue, (oldValue+1 == N) ? 0 : oldValue+1));
        return oldValue;
    }
    char padding0[64];
    std::atomic<uint32_t> currentElement{0};
    char padding1[64];
    std::array<T, N> wheelymajig;
    char padding2[64];
};
}}

#endif /* WHEEL_HPP */

<|endoftext|>


<commit_before>/**
 * @brief A high-performance and multi-threaded trie implementation based on Hdb.
 *        Incremental update is also supported.
 *        See TR "A high-performance and multi-threaded trie based on HdbTrie" for details.
 * @author Wei Cao
 * @date 2009-12-1
 */

#ifndef _MT_TRIE_H_
#define _MT_TRIE_H_

#include <iostream>
#include <fstream>

#include <boost/archive/xml_oarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/serialization/split_member.hpp>
#include <boost/serialization/nvp.hpp>

#include <util/ThreadModel.h>

#include "sampler.h"
#include "partition_trie.hpp"

NS_IZENELIB_AM_BEGIN

template <typename StringType>
class MtTrie
{
public:
    typedef uint64_t TrieNodeIDType;
    typedef PartitionTrie<StringType, TrieNodeIDType> PartitionTrieType;
    typedef FinalTrie<StringType, TrieNodeIDType> FinalTrieType;

public:
    /**
     *  @param partitionNum - how many partitions do we divide all input terms into.
     */
    MtTrie(const std::string& name, const int partitionNum)
    :   name_(name), configPath_(name_ + ".config.xml"),
        writeCachePath_(name_+".writecache"),
        sampler_(name_ + ".sampler"),
        trie_(name_ + ".trie")
    {
        if( partitionNum <= 0 || partitionNum > 256 ) {
            throw std::runtime_error( logHead() + "wrong partitionNum,\
                should between 0 and 256");
        }

        if(load()) {
            if(partitionNum_ != partitionNum)
                throw std::runtime_error( logHead() + "inconsistent partitionNum");
        } else {
            partitionNum_ = partitionNum;
            boundariesInitialized_ = false;
            boundaries_.clear();
            partitionsInitialized_.resize(partitionNum_, false);
            startNodeID_.resize(partitionNum_, NodeIDTraits<TrieNodeIDType>::MinValue);
            trieInitialized_ = false;
        }

        sync();
    }

    void open()
    {
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary | std::ofstream::app );
        if(writeCache_.fail())
            std::cerr << logHead() << " fail to open write cache" << std::endl;

        trie_.open();
    }

    void close()
    {
        writeCache_.close();
        trie_.close();
    }

    void flush()
    {
        sync();
        writeCache_.flush();
        trie_.flush();
    }

    /**
     * @brief Append term into write cache of MtTrie.
     */
	void insert(const StringType & term)
	{
        try {
             int len = (int) term.size();
             writeCache_.write((char*) &len, sizeof(int));
             writeCache_.write((const char*)term.c_str(), term.size());
             sampler_.collect(term);
        } catch(const std::exception & e) {
            std::cerr << logHead() << " fail to update writecache" << std::endl;
        }
    }

    /**
     * @brief Main control flow, invoking multiple threads processing string collection.
     *        Called after all insert() operations.
     * @param   filename - input file that contains all terms.
     *          threadNum - how many threads do we start, each thread could process
     *                      one or more partitions.
     */
    void executeTask(const int threadNum) {
        threadNum_ = threadNum;
        if( threadNum_<= 0)
            throw std::runtime_error(logHead() + "wrong threadNum, should be larger than 0");
        if( partitionNum_ < threadNum_ ) {
            std::cout << logHead() << " threadNum(" << threadNum << ") is less than partitionNum("
                << partitionNum_ << ", shrink it" << std::endl;
            threadNum_ = partitionNum_;
        }

        if( !boundariesInitialized_ ) {
            std::set<StringType> samples;
            sampler_.getSamples(samples);

            std::cout << logHead() << "start computing " << (partitionNum_-1) <<
                " boundaries from " << samples.size() << " smaples" << std::endl;

            if( samples.size() > (size_t)(partitionNum_-1) ) {
                int count = 0;
                size_t interval = samples.size()/partitionNum_;

                typename std::set<StringType>::iterator it = samples.begin();
                while(count < partitionNum_-1) {
                    for(size_t i=0; i<interval; i++)
                        it++;
                    boundaries_.push_back(*it);
                    count ++;
                }
            } else {
                std::cout << logHead() << "number of terms isn't enough to be partitioned to "
                    << partitionNum_ << " partitions" << std::endl;
                sync();
                return;
            }

            boundariesInitialized_ = true;

            sync();

        } else {
            std::cout << logHead() << "skip computing boundareis, already exist"
                << std::endl;
        }

        writeCache_.flush();
        writeCache_.close();
        splitInput();
        remove( writeCachePath_.c_str() );
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out | std::ofstream::binary );

        // multi-threaded building tries on partitions
        std::cout << logHead() << "Build " << partitionNum_ << " partitions in "
            << threadNum_ << " threads" << std::endl;

        processedPartitions_ = 0;
        for( int i=0; i < threadNum_; i++ )
          workerThreads_.create_thread(boost::bind(&MtTrie::taskBody, this, i) );
        workerThreads_.join_all();
        std::cout << std::endl;

        // merging tries on partitions into one
        mergeTries();

        flush();
    }

    /**
     * @brief Get a list of values whose keys match the wildcard query. Only "*" and
     * "?" are supported currently, legal input looks like "ea?th", "her*", or "*ear?h".
     * @return true at leaset one result found.
     *         false nothing found.
     */
    bool findRegExp(const StringType& regexp,
        std::vector<StringType>& keyList,
        int maximumResultNumber = 100)
    {
        return trie_.findRegExp(regexp, keyList, maximumResultNumber);
    }

protected:

    std::string logHead() {
        return "MtTrie[" + name_ + "] ";
    }

    /**
     * @return starting point of NodeID in a specific partition.
     */
    TrieNodeIDType getPartitionStartNodeID(int partitionId) {
        TrieNodeIDType ret = 1;
        ret <<= (sizeof(TrieNodeIDType)*8 - 8);
        ret *= partitionId;
        return ret;
    }

    /**
     * @return prefix of files associated with a specific partition.
     */
    std::string getPartitionName(int partitionId) {
        return name_ + ".partition" + boost::lexical_cast<std::string>(partitionId);
    }

    /**
     * @brief Split orignal input files into multiple partitions.
     */
    void splitInput() {

        std::ifstream input;
        input.open(writeCachePath_.c_str(), std::ifstream::in|std::ifstream::binary );
        if(input.fail())
            throw std::runtime_error("failed open write cache " + writeCachePath_);

        long sizeofInput;
        input.seekg(0, ios_base::end);
        sizeofInput = input.tellg();
        input.seekg(0, ios_base::beg);

        std::ofstream* output = new std::ofstream[partitionNum_];
        for(int i = 0; i<partitionNum_; i++ ) {
            std::string outputName = getPartitionName(i) + ".input";
            output[i].open(outputName.c_str(), std::ofstream::out|std::ofstream::binary|std::ofstream::app);
            if(output[i].fail())
                throw std::runtime_error("failed prepare input for " + getPartitionName(i) );
        }

        StringType term;
        int buffersize = 0;
        char charBuffer[256];
        int progress = 0;
        long sizeofNextProgress = 0;
        long sizeofProcessed = 0;

        while(!input.eof())
        {
            input.read((char*)&buffersize, sizeof(int));
            // skip too long term
            if(buffersize > 256) {
                input.seekg(buffersize, std::ifstream::cur);
                continue;
            }
            input.read( charBuffer, buffersize );
            term.assign( std::string(charBuffer,buffersize) );

            if(term.size() > 0) {
                int pos = std::lower_bound(boundaries_.begin(), boundaries_.end(),
                        term) - boundaries_.begin();
                output[pos].write((char*)&buffersize, sizeof(int));
                output[pos].write(charBuffer, buffersize);
            }

            sizeofProcessed += (buffersize + sizeof(int));
            if(sizeofProcessed >= sizeofNextProgress ) {
                std::cout << "\r" << logHead() << "Split input, progress ["
                    << progress << "%]" << std::flush;
                progress ++;
                sizeofNextProgress = (long)( ((double)(progress)/(double)100)
                    *(double)sizeofInput );
            }
        }
        input.close();
        for(int i = 0; i<partitionNum_; i++ ) {
            output[i].close();
        }
        delete[] output;
        std::cout << std::endl;
    }

    void taskBody(int threadId) {
        int partitionId = threadId;
        while(partitionId < partitionNum_) {

            std::string inputName = getPartitionName(partitionId) + ".input";
            std::ifstream input(inputName.c_str(), std::ifstream::in|std::ifstream::binary);

            std::string trieName = getPartitionName(partitionId) + ".trie";
            PartitionTrieType trie(trieName);
            trie.open();

            // Initialize trie at the first time.
            if( !partitionsInitialized_[partitionId] ) {
                // Insert all boundaries into trie, to ensure NodeIDs are consistent
                // in all tries, the details of reasoning and proof see MtTrie TR.
                for(size_t i =0; i<boundaries_.size(); i++) {
                    trie.insert(boundaries_[i]);
                }
                trie.setBaseNID(getPartitionStartNodeID(partitionId));
                startNodeID_[partitionId] = trie.getNextNID();
                partitionsInitialized_[partitionId] = true;
            }

            if(input) {
                // Insert all terms in this partition
                StringType term;
                int buffersize = 0;
                char charBuffer[256];
                while(!input.eof())
                {
                    input.read((char*)&buffersize, sizeof(int));
                    input.read( charBuffer, buffersize );
                    term.assign( std::string(charBuffer,buffersize) );
                    trie.insert(term);
                }
                trie.optimize();

                input.close();
                remove( inputName.c_str() );
            }

            trie.close();

            printLock_.acquire_write_lock();
            processedPartitions_ ++;
            std::cout << "\r" << logHead() << "Build partitions, progress ["
                << (100*processedPartitions_)/partitionNum_ << "%]" << std::flush;
            printLock_.release_write_lock();

            partitionId += threadNum_;
        }
    }

    void mergeTries() {

        // Initialize trie at the first time.
        if(!trieInitialized_) {
            for(size_t i =0; i<boundaries_.size(); i++) {
                trie_.insert(boundaries_[i]);
            }
            trieInitialized_ = true;
        }

        int mergedPartitions = 0;
        for(int i=0; i<partitionNum_; i++) {
            PartitionTrieType pt(getPartitionName(i) + ".trie");
            pt.open();

            mergeToFinal(trie_, pt, startNodeID_[i]);
            startNodeID_[i] = pt.getNextNID();
            sync();

            pt.close();
            mergedPartitions ++;
            std::cout << "\r" << logHead() << "Merge partitions, progress ["
                << (100*mergedPartitions)/(partitionNum_+1) << "%]" << std::flush;
        }

        trie_.optimize();
        trie_.flush();

        std::cout << "\r" << logHead() << "Merge partitions, progress [100%]"
            << std::endl << std::flush;
    }

protected:

    friend class boost::serialization::access;

    template<class Archive>
    void save(Archive & ar, const unsigned int version) const
    {
        ar & boost::serialization::make_nvp("PartitionNumber", partitionNum_);

        ar & boost::serialization::make_nvp("BoundariesInitialized", boundariesInitialized_);
        std::vector<std::string> buffer;
        for(size_t i =0; i< boundaries_.size(); i++ ) {
            buffer.push_back( std::string((char*)boundaries_[i].c_str(),
                    boundaries_[i].size()) );
        }
        ar & boost::serialization::make_nvp("Boundaries", buffer);

        ar & boost::serialization::make_nvp("PartitionsInitialized", partitionsInitialized_);
        ar & boost::serialization::make_nvp("StartNodeID", startNodeID_);

        ar & boost::serialization::make_nvp("TrieInitialized", trieInitialized_);
    }

    template<class Archive>
    void load(Archive & ar, const unsigned int version)
    {
        ar & boost::serialization::make_nvp("PartitionNumber", partitionNum_);

        ar & boost::serialization::make_nvp("BoundariesInitialized", boundariesInitialized_);
        std::vector<std::string> buffer;
        ar & boost::serialization::make_nvp("Boundaries", buffer);
        for(size_t i =0; i< buffer.size(); i++ ) {
            boundaries_.push_back( StringType(buffer[i]) );
        }

        ar & boost::serialization::make_nvp("PartitionsInitialized", partitionsInitialized_);
        ar & boost::serialization::make_nvp("StartNodeID", startNodeID_);

        ar & boost::serialization::make_nvp("TrieInitialized", trieInitialized_);
    }

    BOOST_SERIALIZATION_SPLIT_MEMBER()

    bool load()
    {
        std::ifstream config(configPath_.c_str(), std::ifstream::in);
        if( config ) {
            boost::archive::xml_iarchive xml(config);
            try {
                xml >> boost::serialization::make_nvp("MtTrie", *this);
            } catch (...) {
                throw std::runtime_error( logHead() + "config file corrputed");
            }
            config.close();
            return true;
        }
        return false;
    }

    void sync()
    {
        std::ofstream config(configPath_.c_str(), std::ifstream::out);
        boost::archive::xml_oarchive xml(config);
        xml << boost::serialization::make_nvp("MtTrie", *this);
        config.flush();
    }

private:

    /// @brief prefix of assoicated files' name.
    std::string name_;

    /// @brief load configuration from this file.
    std::string configPath_;

    /// @brief path of write cache file.
    std::string writeCachePath_;

    /// @brief file acts as write cache.
    ///        all inputs are cached in this file first,
    ///        then inserted into trie in executeTask().
    std::ofstream writeCache_;

    /// @brief collect given numbers of samples from input term stream.
    StreamSampler<StringType> sampler_;

    /// @brief how many partitions do we divide all input terms into.
    int partitionNum_;

    /// @brief Indicate whether boundaries have been determined.
    bool boundariesInitialized_;

    /// @brief boundaries between partitions, there are partitionNum-1 elements.
    std::vector<StringType> boundaries_;

    /// @brief Indicate whether partitions has been initialized.
    std::vector<bool> partitionsInitialized_;

    /// @brief next available NodeID of tries on each partition before insertions,
    ///        skip all previous NodeID when merging a partition trie into the final trie.
    ///        designed for supporting incremental updates. there are partitionNum_ elements.
    std::vector<TrieNodeIDType> startNodeID_;

    /// @brief Indicate whether the final trie has been initialized.
    bool trieInitialized_;

    /// @brief the final(major) trie, findRegExp are operating this trie.
    FinalTrieType trie_;

    /// @brief number of partitions that finishes building.
    int processedPartitions_;

    /// @brief Lock for correct printing between work threads
    ReadWriteLock printLock_;

    /// @brief number of work threads.
    int threadNum_;

    /// @brief work thread pool.
    boost::thread_group workerThreads_;
};

NS_IZENELIB_AM_END

#endif
<commit_msg>fix a spelling error<commit_after>/**
 * @brief A high-performance and multi-threaded trie implementation based on Hdb.
 *        Incremental update is also supported.
 *        See TR "A high-performance and multi-threaded trie based on HdbTrie" for details.
 * @author Wei Cao
 * @date 2009-12-1
 */

#ifndef _MT_TRIE_H_
#define _MT_TRIE_H_

#include <iostream>
#include <fstream>

#include <boost/archive/xml_oarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/serialization/split_member.hpp>
#include <boost/serialization/nvp.hpp>

#include <util/ThreadModel.h>

#include "sampler.h"
#include "partition_trie.hpp"

NS_IZENELIB_AM_BEGIN

template <typename StringType>
class MtTrie
{
public:
    typedef uint64_t TrieNodeIDType;
    typedef PartitionTrie<StringType, TrieNodeIDType> PartitionTrieType;
    typedef FinalTrie<StringType, TrieNodeIDType> FinalTrieType;

public:
    /**
     *  @param partitionNum - how many partitions do we divide all input terms into.
     */
    MtTrie(const std::string& name, const int partitionNum)
    :   name_(name), configPath_(name_ + ".config.xml"),
        writeCachePath_(name_+".writecache"),
        sampler_(name_ + ".sampler"),
        trie_(name_ + ".trie")
    {
        if( partitionNum <= 0 || partitionNum > 256 ) {
            throw std::runtime_error( logHead() + "wrong partitionNum,\
                should between 0 and 256");
        }

        if(load()) {
            if(partitionNum_ != partitionNum)
                throw std::runtime_error( logHead() + "inconsistent partitionNum");
        } else {
            partitionNum_ = partitionNum;
            boundariesInitialized_ = false;
            boundaries_.clear();
            partitionsInitialized_.resize(partitionNum_, false);
            startNodeID_.resize(partitionNum_, NodeIDTraits<TrieNodeIDType>::MinValue);
            trieInitialized_ = false;
        }

        sync();
    }

    void open()
    {
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary | std::ofstream::app );
        if(writeCache_.fail())
            std::cerr << logHead() << " fail to open write cache" << std::endl;

        trie_.open();
    }

    void close()
    {
        writeCache_.close();
        trie_.close();
    }

    void flush()
    {
        sync();
        writeCache_.flush();
        trie_.flush();
    }

    /**
     * @brief Append term into write cache of MtTrie.
     */
	void insert(const StringType & term)
	{
        try {
             int len = (int) term.size();
             writeCache_.write((char*) &len, sizeof(int));
             writeCache_.write((const char*)term.c_str(), term.size());
             sampler_.collect(term);
        } catch(const std::exception & e) {
            std::cerr << logHead() << " fail to update writecache" << std::endl;
        }
    }

    /**
     * @brief Main control flow, invoking multiple threads processing string collection.
     *        Called after all insert() operations.
     * @param   filename - input file that contains all terms.
     *          threadNum - how many threads do we start, each thread could process
     *                      one or more partitions.
     */
    void executeTask(const int threadNum) {
        threadNum_ = threadNum;
        if( threadNum_<= 0)
            throw std::runtime_error(logHead() + "wrong threadNum, should be larger than 0");
        if( partitionNum_ < threadNum_ ) {
            std::cout << logHead() << " threadNum(" << threadNum << ") is less than partitionNum("
                << partitionNum_ << ", shrink it" << std::endl;
            threadNum_ = partitionNum_;
        }

        if( !boundariesInitialized_ ) {
            std::set<StringType> samples;
            sampler_.getSamples(samples);

            std::cout << logHead() << "start computing " << (partitionNum_-1) <<
                " boundaries from " << samples.size() << " samples" << std::endl;

            if( samples.size() > (size_t)(partitionNum_-1) ) {
                int count = 0;
                size_t interval = samples.size()/partitionNum_;

                typename std::set<StringType>::iterator it = samples.begin();
                while(count < partitionNum_-1) {
                    for(size_t i=0; i<interval; i++)
                        it++;
                    boundaries_.push_back(*it);
                    count ++;
                }
            } else {
                std::cout << logHead() << "number of terms isn't enough to be partitioned to "
                    << partitionNum_ << " partitions" << std::endl;
                sync();
                return;
            }

            boundariesInitialized_ = true;

            sync();

        } else {
            std::cout << logHead() << "skip computing boundareis, already exist"
                << std::endl;
        }

        writeCache_.flush();
        writeCache_.close();
        splitInput();
        remove( writeCachePath_.c_str() );
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out | std::ofstream::binary );

        // multi-threaded building tries on partitions
        std::cout << logHead() << "Build " << partitionNum_ << " partitions in "
            << threadNum_ << " threads" << std::endl;

        processedPartitions_ = 0;
        for( int i=0; i < threadNum_; i++ )
          workerThreads_.create_thread(boost::bind(&MtTrie::taskBody, this, i) );
        workerThreads_.join_all();
        std::cout << std::endl;

        // merging tries on partitions into one
        mergeTries();

        flush();
    }

    /**
     * @brief Get a list of values whose keys match the wildcard query. Only "*" and
     * "?" are supported currently, legal input looks like "ea?th", "her*", or "*ear?h".
     * @return true at leaset one result found.
     *         false nothing found.
     */
    bool findRegExp(const StringType& regexp,
        std::vector<StringType>& keyList,
        int maximumResultNumber = 100)
    {
        return trie_.findRegExp(regexp, keyList, maximumResultNumber);
    }

protected:

    std::string logHead() {
        return "MtTrie[" + name_ + "] ";
    }

    /**
     * @return starting point of NodeID in a specific partition.
     */
    TrieNodeIDType getPartitionStartNodeID(int partitionId) {
        TrieNodeIDType ret = 1;
        ret <<= (sizeof(TrieNodeIDType)*8 - 8);
        ret *= partitionId;
        return ret;
    }

    /**
     * @return prefix of files associated with a specific partition.
     */
    std::string getPartitionName(int partitionId) {
        return name_ + ".partition" + boost::lexical_cast<std::string>(partitionId);
    }

    /**
     * @brief Split orignal input files into multiple partitions.
     */
    void splitInput() {

        std::ifstream input;
        input.open(writeCachePath_.c_str(), std::ifstream::in|std::ifstream::binary );
        if(input.fail())
            throw std::runtime_error("failed open write cache " + writeCachePath_);

        long sizeofInput;
        input.seekg(0, ios_base::end);
        sizeofInput = input.tellg();
        input.seekg(0, ios_base::beg);

        std::ofstream* output = new std::ofstream[partitionNum_];
        for(int i = 0; i<partitionNum_; i++ ) {
            std::string outputName = getPartitionName(i) + ".input";
            output[i].open(outputName.c_str(), std::ofstream::out|std::ofstream::binary|std::ofstream::app);
            if(output[i].fail())
                throw std::runtime_error("failed prepare input for " + getPartitionName(i) );
        }

        StringType term;
        int buffersize = 0;
        char charBuffer[256];
        int progress = 0;
        long sizeofNextProgress = 0;
        long sizeofProcessed = 0;

        while(!input.eof())
        {
            input.read((char*)&buffersize, sizeof(int));
            // skip too long term
            if(buffersize > 256) {
                input.seekg(buffersize, std::ifstream::cur);
                continue;
            }
            input.read( charBuffer, buffersize );
            term.assign( std::string(charBuffer,buffersize) );

            if(term.size() > 0) {
                int pos = std::lower_bound(boundaries_.begin(), boundaries_.end(),
                        term) - boundaries_.begin();
                output[pos].write((char*)&buffersize, sizeof(int));
                output[pos].write(charBuffer, buffersize);
            }

            sizeofProcessed += (buffersize + sizeof(int));
            if(sizeofProcessed >= sizeofNextProgress ) {
                std::cout << "\r" << logHead() << "Split input, progress ["
                    << progress << "%]" << std::flush;
                progress ++;
                sizeofNextProgress = (long)( ((double)(progress)/(double)100)
                    *(double)sizeofInput );
            }
        }
        input.close();
        for(int i = 0; i<partitionNum_; i++ ) {
            output[i].close();
        }
        delete[] output;
        std::cout << std::endl;
    }

    void taskBody(int threadId) {
        int partitionId = threadId;
        while(partitionId < partitionNum_) {

            std::string inputName = getPartitionName(partitionId) + ".input";
            std::ifstream input(inputName.c_str(), std::ifstream::in|std::ifstream::binary);

            std::string trieName = getPartitionName(partitionId) + ".trie";
            PartitionTrieType trie(trieName);
            trie.open();

            // Initialize trie at the first time.
            if( !partitionsInitialized_[partitionId] ) {
                // Insert all boundaries into trie, to ensure NodeIDs are consistent
                // in all tries, the details of reasoning and proof see MtTrie TR.
                for(size_t i =0; i<boundaries_.size(); i++) {
                    trie.insert(boundaries_[i]);
                }
                trie.setBaseNID(getPartitionStartNodeID(partitionId));
                startNodeID_[partitionId] = trie.getNextNID();
                partitionsInitialized_[partitionId] = true;
            }

            if(input) {
                // Insert all terms in this partition
                StringType term;
                int buffersize = 0;
                char charBuffer[256];
                while(!input.eof())
                {
                    input.read((char*)&buffersize, sizeof(int));
                    input.read( charBuffer, buffersize );
                    term.assign( std::string(charBuffer,buffersize) );
                    trie.insert(term);
                }
                trie.optimize();

                input.close();
                remove( inputName.c_str() );
            }

            trie.close();

            printLock_.acquire_write_lock();
            processedPartitions_ ++;
            std::cout << "\r" << logHead() << "Build partitions, progress ["
                << (100*processedPartitions_)/partitionNum_ << "%]" << std::flush;
            printLock_.release_write_lock();

            partitionId += threadNum_;
        }
    }

    void mergeTries() {

        // Initialize trie at the first time.
        if(!trieInitialized_) {
            for(size_t i =0; i<boundaries_.size(); i++) {
                trie_.insert(boundaries_[i]);
            }
            trieInitialized_ = true;
        }

        int mergedPartitions = 0;
        for(int i=0; i<partitionNum_; i++) {
            PartitionTrieType pt(getPartitionName(i) + ".trie");
            pt.open();

            mergeToFinal(trie_, pt, startNodeID_[i]);
            startNodeID_[i] = pt.getNextNID();
            sync();

            pt.close();
            mergedPartitions ++;
            std::cout << "\r" << logHead() << "Merge partitions, progress ["
                << (100*mergedPartitions)/(partitionNum_+1) << "%]" << std::flush;
        }

        trie_.optimize();
        trie_.flush();

        std::cout << "\r" << logHead() << "Merge partitions, progress [100%]"
            << std::endl << std::flush;
    }

protected:

    friend class boost::serialization::access;

    template<class Archive>
    void save(Archive & ar, const unsigned int version) const
    {
        ar & boost::serialization::make_nvp("PartitionNumber", partitionNum_);

        ar & boost::serialization::make_nvp("BoundariesInitialized", boundariesInitialized_);
        std::vector<std::string> buffer;
        for(size_t i =0; i< boundaries_.size(); i++ ) {
            buffer.push_back( std::string((char*)boundaries_[i].c_str(),
                    boundaries_[i].size()) );
        }
        ar & boost::serialization::make_nvp("Boundaries", buffer);

        ar & boost::serialization::make_nvp("PartitionsInitialized", partitionsInitialized_);
        ar & boost::serialization::make_nvp("StartNodeID", startNodeID_);

        ar & boost::serialization::make_nvp("TrieInitialized", trieInitialized_);
    }

    template<class Archive>
    void load(Archive & ar, const unsigned int version)
    {
        ar & boost::serialization::make_nvp("PartitionNumber", partitionNum_);

        ar & boost::serialization::make_nvp("BoundariesInitialized", boundariesInitialized_);
        std::vector<std::string> buffer;
        ar & boost::serialization::make_nvp("Boundaries", buffer);
        for(size_t i =0; i< buffer.size(); i++ ) {
            boundaries_.push_back( StringType(buffer[i]) );
        }

        ar & boost::serialization::make_nvp("PartitionsInitialized", partitionsInitialized_);
        ar & boost::serialization::make_nvp("StartNodeID", startNodeID_);

        ar & boost::serialization::make_nvp("TrieInitialized", trieInitialized_);
    }

    BOOST_SERIALIZATION_SPLIT_MEMBER()

    bool load()
    {
        std::ifstream config(configPath_.c_str(), std::ifstream::in);
        if( config ) {
            boost::archive::xml_iarchive xml(config);
            try {
                xml >> boost::serialization::make_nvp("MtTrie", *this);
            } catch (...) {
                throw std::runtime_error( logHead() + "config file corrputed");
            }
            config.close();
            return true;
        }
        return false;
    }

    void sync()
    {
        std::ofstream config(configPath_.c_str(), std::ifstream::out);
        boost::archive::xml_oarchive xml(config);
        xml << boost::serialization::make_nvp("MtTrie", *this);
        config.flush();
    }

private:

    /// @brief prefix of assoicated files' name.
    std::string name_;

    /// @brief load configuration from this file.
    std::string configPath_;

    /// @brief path of write cache file.
    std::string writeCachePath_;

    /// @brief file acts as write cache.
    ///        all inputs are cached in this file first,
    ///        then inserted into trie in executeTask().
    std::ofstream writeCache_;

    /// @brief collect given numbers of samples from input term stream.
    StreamSampler<StringType> sampler_;

    /// @brief how many partitions do we divide all input terms into.
    int partitionNum_;

    /// @brief Indicate whether boundaries have been determined.
    bool boundariesInitialized_;

    /// @brief boundaries between partitions, there are partitionNum-1 elements.
    std::vector<StringType> boundaries_;

    /// @brief Indicate whether partitions has been initialized.
    std::vector<bool> partitionsInitialized_;

    /// @brief next available NodeID of tries on each partition before insertions,
    ///        skip all previous NodeID when merging a partition trie into the final trie.
    ///        designed for supporting incremental updates. there are partitionNum_ elements.
    std::vector<TrieNodeIDType> startNodeID_;

    /// @brief Indicate whether the final trie has been initialized.
    bool trieInitialized_;

    /// @brief the final(major) trie, findRegExp are operating this trie.
    FinalTrieType trie_;

    /// @brief number of partitions that finishes building.
    int processedPartitions_;

    /// @brief Lock for correct printing between work threads
    ReadWriteLock printLock_;

    /// @brief number of work threads.
    int threadNum_;

    /// @brief work thread pool.
    boost::thread_group workerThreads_;
};

NS_IZENELIB_AM_END

#endif
<|endoftext|>


<commit_before>/*! \file tuple.hpp
    \brief Support for types found in \<tuple\>
    \ingroup STLSupport */
/*
  Copyright (c) 2014, Randolph Voorhies, Shane Grant
  All rights reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:
      * Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.
      * Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
      * Neither the name of cereal nor the
        names of its contributors may be used to endorse or promote products
        derived from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  DISCLAIMED. IN NO EVENT SHALL RANDOLPH VOORHIES OR SHANE GRANT BE LIABLE FOR ANY
  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CEREAL_TYPES_TUPLE_HPP_
#define CEREAL_TYPES_TUPLE_HPP_

#include "cereal/cereal.hpp"
#include <tuple>

namespace cereal
{
  namespace tuple_detail
  {
    //! Creates a c string from a sequence of characters
    /*! The c string created will alwas be prefixed by "tuple_element"
        Based on code from: http://stackoverflow/a/20973438/710791
        @internal */
    template<char...Cs>
    struct char_seq_to_c_str
    {
      static const int size = 14;// Size of array for the word: tuple_element
      typedef const char (&arr_type)[sizeof...(Cs) + size];
      static const char str[sizeof...(Cs) + size];
    };

    // the word tuple_element plus a number
    //! @internal
    template<char...Cs>
    const char char_seq_to_c_str<Cs...>::str[sizeof...(Cs) + size] =
      {'t','u','p','l','e','_','e','l','e','m','e','n','t', Cs..., '\0'};

    //! Converts a number into a sequence of characters
    /*! @tparam Q The quotient of dividing the original number by 10
        @tparam R The remainder of dividing the original number by 10
        @tparam C The sequence built so far
        @internal */
    template <size_t Q, size_t R, char ... C>
    struct to_string_impl
    {
      using type = typename to_string_impl<Q/10, Q%10, R+'0', C...>::type;
    };

    //! Base case with no quotient
    /*! @internal */
    template <size_t R, char ... C>
    struct to_string_impl<0, R, C...>
    {
      using type = char_seq_to_c_str<R+'0', C...>;
    };

    //! Generates a c string for a given index of a tuple
    /*! Example use:
        @code{cpp}
        tuple_element_name<3>::c_str();// returns "tuple_element3"
        @endcode
        @internal */
    template<size_t T>
    struct tuple_element_name
    {
      using type = typename to_string_impl<T/10, T%10>::type;
      static const typename type::arr_type c_str(){ return type::str; }
    };

    // unwinds a tuple to save it
    //! @internal
    template <size_t Height>
    struct serialize
    {
      template <class Archive, class ... Types> inline
      static void apply( Archive & ar, std::tuple<Types...> & tuple )
      {
        serialize<Height - 1>::template apply( ar, tuple );
        ar( CEREAL_NVP_(tuple_element_name<Height - 1>::c_str(),
            std::get<Height - 1>( tuple )) );
      }
    };

    // Zero height specialization - nothing to do here
    //! @internal
    template <>
    struct serialize<0>
    {
      template <class Archive, class ... Types> inline
      static void apply( Archive &, std::tuple<Types...> & )
      { }
    };
  }

  //! Serializing for std::tuple
  template <class Archive, class ... Types> inline
  void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, std::tuple<Types...> & tuple )
  {
    tuple_detail::serialize<std::tuple_size<std::tuple<Types...>>::value>::template apply( ar, tuple );
  }
} // namespace cereal

#endif // CEREAL_TYPES_TUPLE_HPP_
<commit_msg>fix -Wconversion warning in gcc7<commit_after>/*! \file tuple.hpp
    \brief Support for types found in \<tuple\>
    \ingroup STLSupport */
/*
  Copyright (c) 2014, Randolph Voorhies, Shane Grant
  All rights reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:
      * Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.
      * Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
      * Neither the name of cereal nor the
        names of its contributors may be used to endorse or promote products
        derived from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  DISCLAIMED. IN NO EVENT SHALL RANDOLPH VOORHIES OR SHANE GRANT BE LIABLE FOR ANY
  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CEREAL_TYPES_TUPLE_HPP_
#define CEREAL_TYPES_TUPLE_HPP_

#include "cereal/cereal.hpp"
#include <tuple>

namespace cereal
{
  namespace tuple_detail
  {
    //! Creates a c string from a sequence of characters
    /*! The c string created will always be prefixed by "tuple_element"
        Based on code from: http://stackoverflow/a/20973438/710791
        @internal */
    template<char...Cs>
    struct char_seq_to_c_str
    {
      static const int size = 14;// Size of array for the word: tuple_element
      typedef const char (&arr_type)[sizeof...(Cs) + size];
      static const char str[sizeof...(Cs) + size];
    };

    // the word tuple_element plus a number
    //! @internal
    template<char...Cs>
    const char char_seq_to_c_str<Cs...>::str[sizeof...(Cs) + size] =
      {'t','u','p','l','e','_','e','l','e','m','e','n','t', Cs..., '\0'};

    //! Converts a number into a sequence of characters
    /*! @tparam Q The quotient of dividing the original number by 10
        @tparam R The remainder of dividing the original number by 10
        @tparam C The sequence built so far
        @internal */
    template <size_t Q, size_t R, char ... C>
    struct to_string_impl
    {
      using type = typename to_string_impl<Q/10, Q%10, static_cast<char>(R+std::size_t{'0'}), C...>::type;
    };

    //! Base case with no quotient
    /*! @internal */
    template <size_t R, char ... C>
    struct to_string_impl<0, R, C...>
    {
      using type = char_seq_to_c_str<static_cast<char>(R+std::size_t{'0'}), C...>;
    };

    //! Generates a c string for a given index of a tuple
    /*! Example use:
        @code{cpp}
        tuple_element_name<3>::c_str();// returns "tuple_element3"
        @endcode
        @internal */
    template<size_t T>
    struct tuple_element_name
    {
      using type = typename to_string_impl<T/10, T%10>::type;
      static const typename type::arr_type c_str(){ return type::str; }
    };

    // unwinds a tuple to save it
    //! @internal
    template <size_t Height>
    struct serialize
    {
      template <class Archive, class ... Types> inline
      static void apply( Archive & ar, std::tuple<Types...> & tuple )
      {
        serialize<Height - 1>::template apply( ar, tuple );
        ar( CEREAL_NVP_(tuple_element_name<Height - 1>::c_str(),
            std::get<Height - 1>( tuple )) );
      }
    };

    // Zero height specialization - nothing to do here
    //! @internal
    template <>
    struct serialize<0>
    {
      template <class Archive, class ... Types> inline
      static void apply( Archive &, std::tuple<Types...> & )
      { }
    };
  }

  //! Serializing for std::tuple
  template <class Archive, class ... Types> inline
  void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, std::tuple<Types...> & tuple )
  {
    tuple_detail::serialize<std::tuple_size<std::tuple<Types...>>::value>::template apply( ar, tuple );
  }
} // namespace cereal

#endif // CEREAL_TYPES_TUPLE_HPP_
<|endoftext|>


<commit_before>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
Copyright (c) 2008, 2009 Aristid Breitkreuz, Ash Berlin, Ruediger Sonderfeld

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#ifndef FLUSSPFERD_IMPORTER_HPP
#define FLUSSPFERD_IMPORTER_HPP

#include "init.hpp"
#include "object.hpp"

namespace flusspferd {

/**
 * Load the 'require()' function into @p container.
 *
 * Creates a new instance of the require function.
 *
 * @param container The object to load the function into.
 *
 * @ingroup loadable_modules
 * @ingroup jsext
 */
void load_require_function(object container);

/**
 * The prototype for module loader functions.
 *
 * Modules should define a function @c flusspferd_load (with
 * extern "C") with this signature.
 *
 * Example:
 * @dontinclude help/examples/dummy_module.cpp
 * @skip extern "C"
 * @until }
 *
 * @ingroup loadable_modules
 */
typedef void (*flusspferd_load_t)(object &exports, object &context);

/**
 * Define a module loader.
 *
 * @param exports The object containing the module exports.
 * @param context The root object for the module's scope.
 */
#define FLUSSPFERD_LOADER(exports, context) \
  extern "C" \
  void flusspferd_load( \
    ::flusspferd::object &exports, \
    ::flusspferd::object &context)

/**
 * Define a module loader ("simple": without context parameter).
 *
 * @param exports The object containing the module exports.
 *
 * @ingroup loadable_modules
 */
#define FLUSSPFERD_LOADER_SIMPLE(exports) \
  extern "C" \
  void flusspferd_load( \
    ::flusspferd::object &exports, \
    ::flusspferd::object &)
}

#endif
<commit_msg>doc: improve documentation for modules functions and macros<commit_after>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
Copyright (c) 2008, 2009 Aristid Breitkreuz, Ash Berlin, Ruediger Sonderfeld

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#ifndef FLUSSPFERD_IMPORTER_HPP
#define FLUSSPFERD_IMPORTER_HPP

#include "init.hpp"
#include "object.hpp"

namespace flusspferd {

/**
 * Load the 'require()' function into @p container.
 *
 * Creates a new instance of the require function.
 *
 * @param container The object to load the function into.
 *
 * @ingroup loadable_modules
 */
void load_require_function(object container);

/**
 * The prototype for module loader functions.
 *
 * Modules should define a %function @c flusspferd_load (with
 * extern "C") with this signature.
 *
 * However, they should not define this %function directly but rather
 * use either #FLUSSPFERD_LOADER or #FLUSSPFERD_LOADER_SIMPLE, which will adapt
 * to changes in the signature.
 *
 * @ingroup loadable_modules
 */
typedef void (*flusspferd_load_t)(object &exports, object &context);

/**
 * Define a module loader.
 *
 * The parameter @p context contains the module's scope, which includes
 * the @c require function. You can access it with
 *
 * @code
 * context.get_property_object("require")
 * @endcode
 *
 * or
 *
 * @code
 * context.call("require", "module-name")
 * @endcode
 *
 * @param exports The object containing the module exports.
 * @param context The root object for the module's scope.
 *
 * @ingroup loadable_modules
 */
#define FLUSSPFERD_LOADER(exports, context) \
  extern "C" \
  void flusspferd_load( \
    ::flusspferd::object &exports, \
    ::flusspferd::object &context)

/**
 * Define a module loader ("simple": without context parameter).
 *
 * @param exports The object containing the module exports.
 *
 * @ingroup loadable_modules
 */
#define FLUSSPFERD_LOADER_SIMPLE(exports) \
  extern "C" \
  void flusspferd_load( \
    ::flusspferd::object &exports, \
    ::flusspferd::object &)
}

#endif
<|endoftext|>


<commit_before>#ifndef LIBPORT_REFCOUNTED_HH
# define LIBPORT_REFCOUNTED_HH

namespace libport
{
  class RefCounted
  {
    public:
    RefCounted():count_(0) {}
    void counter_inc() {++count_;}
    bool counter_dec() {return !--count_;}
    private:
    int count_;
  };
}
#endif
<commit_msg>Enable libport::refcounted objects to be const.<commit_after>#ifndef LIBPORT_REFCOUNTED_HH
# define LIBPORT_REFCOUNTED_HH

namespace libport
{
  class RefCounted
  {
  public:
    RefCounted():count_(0) {}
    void counter_inc() const {++count_;}
    bool counter_dec() const {return !--count_;}
  private:
    mutable int count_;
  };
}
#endif
<|endoftext|>