<commit_before>/*
 * Copyright 2001,2004 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * $Id$
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/schema/identity/XPathMatcher.hpp>
#include <xercesc/validators/schema/identity/XercesXPath.hpp>
#include <xercesc/validators/schema/SchemaElementDecl.hpp>
#include <xercesc/validators/schema/SchemaAttDef.hpp>
#include <xercesc/validators/schema/SchemaSymbols.hpp>
#include <xercesc/util/RuntimeException.hpp>
#include <xercesc/util/OutOfMemoryException.hpp>

XERCES_CPP_NAMESPACE_BEGIN

typedef JanitorMemFunCall<XPathMatcher>     CleanupType;

// ---------------------------------------------------------------------------
//  XPathMatcher: Constructors and Destructor
// ---------------------------------------------------------------------------
XPathMatcher::XPathMatcher( XercesXPath* const xpath
                          , MemoryManager* const manager)
    : fLocationPathSize(0)
    , fMatched(0)
    , fNoMatchDepth(0)
    , fCurrentStep(0)
    , fStepIndexes(0)
    , fLocationPaths(0)
    , fIdentityConstraint(0)
    , fMemoryManager(manager)
{
    CleanupType cleanup(this, &XPathMatcher::cleanUp);

    try {
        init(xpath);
    }
    catch(const OutOfMemoryException&)
    {
        cleanup.release();

        throw;
    }

    cleanup.release();
}

XPathMatcher::XPathMatcher(XercesXPath* const xpath,
                           IdentityConstraint* const ic,
						   MemoryManager* const manager)
    : fLocationPathSize(0)
    , fMatched(0)
    , fNoMatchDepth(0)
    , fCurrentStep(0)
    , fStepIndexes(0)
    , fLocationPaths(0)
    , fIdentityConstraint(ic)
    , fMemoryManager(manager)
{
    CleanupType cleanup(this, &XPathMatcher::cleanUp);

    try {
        init(xpath);
    }
    catch(const OutOfMemoryException&)
    {
        cleanup.release();

        throw;
    }

    cleanup.release();
}

XPathMatcher::~XPathMatcher()
{
    cleanUp();
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Helper methods
// ---------------------------------------------------------------------------
void XPathMatcher::init(XercesXPath* const xpath) {

    if (xpath) {

        fLocationPaths = xpath->getLocationPaths();
        fLocationPathSize = (fLocationPaths ? fLocationPaths->size() : 0);

        if (fLocationPathSize) {

            fStepIndexes = new (fMemoryManager) RefVectorOf<ValueStackOf<int> >(fLocationPathSize, true, fMemoryManager);
            fCurrentStep = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];
            fNoMatchDepth = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];
            fMatched = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];

            for(unsigned int i=0; i < fLocationPathSize; i++) {
                fStepIndexes->addElement(new (fMemoryManager) ValueStackOf<int>(8, fMemoryManager));
            }
        }
    }
}

// ---------------------------------------------------------------------------
//  XPathMatcher: XMLDocumentHandler methods
// ---------------------------------------------------------------------------
void XPathMatcher::startDocumentFragment() {

    for(unsigned int i = 0; i < fLocationPathSize; i++) {

        fStepIndexes->elementAt(i)->removeAllElements();
        fCurrentStep[i] = 0;
        fNoMatchDepth[i] = 0;
        fMatched[i] = 0;
    }
}

void XPathMatcher::startElement(const XMLElementDecl& elemDecl,
                                const unsigned int urlId,
                                const XMLCh* const elemPrefix,
								const RefVectorOf<XMLAttr>& attrList,
                                const unsigned int attrCount) {

    for (int i = 0; i < (int) fLocationPathSize; i++) {

        // push context
        int startStep = fCurrentStep[i];
        fStepIndexes->elementAt(i)->push(startStep);

        // try next xpath, if not matching
        if ((fMatched[i] & XP_MATCHED_D) == XP_MATCHED || fNoMatchDepth[i] > 0) {
            fNoMatchDepth[i]++;
            continue;
        }

        if((fMatched[i] & XP_MATCHED_D) == XP_MATCHED_D) {
            fMatched[i] = XP_MATCHED_DP;
        }

        // consume self::node() steps
        XercesLocationPath* locPath = fLocationPaths->elementAt(i);
        int stepSize = locPath->getStepSize();

        while (fCurrentStep[i] < stepSize &&
               locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::SELF) {
            fCurrentStep[i]++;
        }

        if (fCurrentStep[i] == stepSize) {

            fMatched[i] = XP_MATCHED;
            continue;
        }

        // now if the current step is a descendant step, we let the next
        // step do its thing; if it fails, we reset ourselves
        // to look at this step for next time we're called.
        // so first consume all descendants:
        int descendantStep = fCurrentStep[i];

        while (fCurrentStep[i] < stepSize &&
               locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::DESCENDANT) {
            fCurrentStep[i]++;
        }

        bool sawDescendant = fCurrentStep[i] > descendantStep;
        if (fCurrentStep[i] == stepSize) {

            fNoMatchDepth[i]++;
            continue;
        }

        // match child::... step, if haven't consumed any self::node()
        if ((fCurrentStep[i] == startStep || fCurrentStep[i] > descendantStep) &&
            locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::CHILD) {

            XercesStep* step = locPath->getStep(fCurrentStep[i]);
            XercesNodeTest* nodeTest = step->getNodeTest();

            if (nodeTest->getType() == XercesNodeTest::QNAME) {

                QName elemQName(elemPrefix, elemDecl.getElementName()->getLocalPart(), urlId, fMemoryManager);

//                if (!(*(nodeTest->getName()) == *(elemDecl.getElementName()))) {
                if (!(*(nodeTest->getName()) == elemQName)) {

                    if(fCurrentStep[i] > descendantStep) {
                        fCurrentStep[i] = descendantStep;
                        continue;
                    }

                    fNoMatchDepth[i]++;
                    continue;
                }
            }

            fCurrentStep[i]++;
        }

        if (fCurrentStep[i] == stepSize) {

            if (sawDescendant) {

                fCurrentStep[i] = descendantStep;
                fMatched[i] = XP_MATCHED_D;
            }
            else {
                fMatched[i] = XP_MATCHED;
            }

            continue;
        }

        // match attribute::... step
        if (fCurrentStep[i] < stepSize &&
            locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::ATTRIBUTE) {

            if (attrCount) {

                XercesNodeTest* nodeTest = locPath->getStep(fCurrentStep[i])->getNodeTest();

                for (unsigned int attrIndex = 0; attrIndex < attrCount; attrIndex++) {

                    const XMLAttr* curDef = attrList.elementAt(attrIndex);

                    if (nodeTest->getType() != XercesNodeTest::QNAME ||
                        (*(nodeTest->getName()) == *(curDef->getAttName()))) {

                        fCurrentStep[i]++;

                        if (fCurrentStep[i] == stepSize) {

                            fMatched[i] = XP_MATCHED_A;
                            int j=0;

                            for(; j<i && ((fMatched[j] & XP_MATCHED) != XP_MATCHED); j++) ;

                            if(j == i) {

                                SchemaAttDef* attDef = ((SchemaElementDecl&) elemDecl).getAttDef(curDef->getName(), curDef->getURIId());
                                DatatypeValidator* dv = (attDef) ? attDef->getDatatypeValidator() : 0;
                                matched(curDef->getValue(), dv, false);
                            }
                        }
                        break;
                    }
                }
            }

            if ((fMatched[i] & XP_MATCHED) != XP_MATCHED) {

                if(fCurrentStep[i] > descendantStep) {

                    fCurrentStep[i] = descendantStep;
                    continue;
                }

                fNoMatchDepth[i]++;
            }
        }
    }
}

void XPathMatcher::endElement(const XMLElementDecl& elemDecl,
                              const XMLCh* const elemContent) {

    for(int i = 0; i < (int) fLocationPathSize; i++) {

        // go back a step
        fCurrentStep[i] = fStepIndexes->elementAt(i)->pop();

        // don't do anything, if not matching
        if (fNoMatchDepth[i] > 0) {
            fNoMatchDepth[i]--;
        }
        // signal match, if appropriate
        else {

            int j=0;
            for(; j<i && ((fMatched[j] & XP_MATCHED) != XP_MATCHED); j++) ;

            if (j < i || (fMatched[j] == 0)
                || ((fMatched[j] & XP_MATCHED_A) == XP_MATCHED_A))
				continue;

            DatatypeValidator* dv = ((SchemaElementDecl*) &elemDecl)->getDatatypeValidator();
            bool isNillable = (((SchemaElementDecl *) &elemDecl)->getMiscFlags() & SchemaSymbols::XSD_NILLABLE) != 0;

            matched(elemContent, dv, isNillable);
            fMatched[i] = 0;
        }
    }
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Match methods
// ---------------------------------------------------------------------------
int XPathMatcher::isMatched() {

    // xpath has been matched if any one of the members of the union have matched.
    for (int i=0; i < (int) fLocationPathSize; i++) {
        if (((fMatched[i] & XP_MATCHED) == XP_MATCHED)
            && ((fMatched[i] & XP_MATCHED_DP) != XP_MATCHED_DP))
            return fMatched[i];
    }

    return 0;
}

void XPathMatcher::matched(const XMLCh* const,
                           DatatypeValidator* const,
                           const bool) {
    return;
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Match methods
// ---------------------------------------------------------------------------
int XPathMatcher::getInitialDepth() const
{
    ThrowXMLwithMemMgr(RuntimeException, XMLExcepts::Regex_NotSupported, fMemoryManager);
    return 0; // to make some compilers happy
}

XERCES_CPP_NAMESPACE_END

/**
  * End of file XPathMatcher.cpp
  */

<commit_msg>Change uppercase enums that were conflicting with other products.<commit_after>/*
 * Copyright 2001,2004 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * $Id$
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/schema/identity/XPathMatcher.hpp>
#include <xercesc/validators/schema/identity/XercesXPath.hpp>
#include <xercesc/validators/schema/SchemaElementDecl.hpp>
#include <xercesc/validators/schema/SchemaAttDef.hpp>
#include <xercesc/validators/schema/SchemaSymbols.hpp>
#include <xercesc/util/RuntimeException.hpp>
#include <xercesc/util/OutOfMemoryException.hpp>

XERCES_CPP_NAMESPACE_BEGIN

typedef JanitorMemFunCall<XPathMatcher>     CleanupType;

// ---------------------------------------------------------------------------
//  XPathMatcher: Constructors and Destructor
// ---------------------------------------------------------------------------
XPathMatcher::XPathMatcher( XercesXPath* const xpath
                          , MemoryManager* const manager)
    : fLocationPathSize(0)
    , fMatched(0)
    , fNoMatchDepth(0)
    , fCurrentStep(0)
    , fStepIndexes(0)
    , fLocationPaths(0)
    , fIdentityConstraint(0)
    , fMemoryManager(manager)
{
    CleanupType cleanup(this, &XPathMatcher::cleanUp);

    try {
        init(xpath);
    }
    catch(const OutOfMemoryException&)
    {
        cleanup.release();

        throw;
    }

    cleanup.release();
}

XPathMatcher::XPathMatcher(XercesXPath* const xpath,
                           IdentityConstraint* const ic,
						   MemoryManager* const manager)
    : fLocationPathSize(0)
    , fMatched(0)
    , fNoMatchDepth(0)
    , fCurrentStep(0)
    , fStepIndexes(0)
    , fLocationPaths(0)
    , fIdentityConstraint(ic)
    , fMemoryManager(manager)
{
    CleanupType cleanup(this, &XPathMatcher::cleanUp);

    try {
        init(xpath);
    }
    catch(const OutOfMemoryException&)
    {
        cleanup.release();

        throw;
    }

    cleanup.release();
}

XPathMatcher::~XPathMatcher()
{
    cleanUp();
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Helper methods
// ---------------------------------------------------------------------------
void XPathMatcher::init(XercesXPath* const xpath) {

    if (xpath) {

        fLocationPaths = xpath->getLocationPaths();
        fLocationPathSize = (fLocationPaths ? fLocationPaths->size() : 0);

        if (fLocationPathSize) {

            fStepIndexes = new (fMemoryManager) RefVectorOf<ValueStackOf<int> >(fLocationPathSize, true, fMemoryManager);
            fCurrentStep = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];
            fNoMatchDepth = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];
            fMatched = (int*) fMemoryManager->allocate
            (
                fLocationPathSize * sizeof(int)
            );//new int[fLocationPathSize];

            for(unsigned int i=0; i < fLocationPathSize; i++) {
                fStepIndexes->addElement(new (fMemoryManager) ValueStackOf<int>(8, fMemoryManager));
            }
        }
    }
}

// ---------------------------------------------------------------------------
//  XPathMatcher: XMLDocumentHandler methods
// ---------------------------------------------------------------------------
void XPathMatcher::startDocumentFragment() {

    for(unsigned int i = 0; i < fLocationPathSize; i++) {

        fStepIndexes->elementAt(i)->removeAllElements();
        fCurrentStep[i] = 0;
        fNoMatchDepth[i] = 0;
        fMatched[i] = 0;
    }
}

void XPathMatcher::startElement(const XMLElementDecl& elemDecl,
                                const unsigned int urlId,
                                const XMLCh* const elemPrefix,
								const RefVectorOf<XMLAttr>& attrList,
                                const unsigned int attrCount) {

    for (int i = 0; i < (int) fLocationPathSize; i++) {

        // push context
        int startStep = fCurrentStep[i];
        fStepIndexes->elementAt(i)->push(startStep);

        // try next xpath, if not matching
        if ((fMatched[i] & XP_MATCHED_D) == XP_MATCHED || fNoMatchDepth[i] > 0) {
            fNoMatchDepth[i]++;
            continue;
        }

        if((fMatched[i] & XP_MATCHED_D) == XP_MATCHED_D) {
            fMatched[i] = XP_MATCHED_DP;
        }

        // consume self::node() steps
        XercesLocationPath* locPath = fLocationPaths->elementAt(i);
        int stepSize = locPath->getStepSize();

        while (fCurrentStep[i] < stepSize &&
               locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::AxisType_SELF) {
            fCurrentStep[i]++;
        }

        if (fCurrentStep[i] == stepSize) {

            fMatched[i] = XP_MATCHED;
            continue;
        }

        // now if the current step is a descendant step, we let the next
        // step do its thing; if it fails, we reset ourselves
        // to look at this step for next time we're called.
        // so first consume all descendants:
        int descendantStep = fCurrentStep[i];

        while (fCurrentStep[i] < stepSize &&
               locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::AxisType_DESCENDANT) {
            fCurrentStep[i]++;
        }

        bool sawDescendant = fCurrentStep[i] > descendantStep;
        if (fCurrentStep[i] == stepSize) {

            fNoMatchDepth[i]++;
            continue;
        }

        // match child::... step, if haven't consumed any self::node()
        if ((fCurrentStep[i] == startStep || fCurrentStep[i] > descendantStep) &&
            locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::AxisType_CHILD) {

            XercesStep* step = locPath->getStep(fCurrentStep[i]);
            XercesNodeTest* nodeTest = step->getNodeTest();

            if (nodeTest->getType() == XercesNodeTest::NodeType_QNAME) {

                QName elemQName(elemPrefix, elemDecl.getElementName()->getLocalPart(), urlId, fMemoryManager);

//                if (!(*(nodeTest->getName()) == *(elemDecl.getElementName()))) {
                if (!(*(nodeTest->getName()) == elemQName)) {

                    if(fCurrentStep[i] > descendantStep) {
                        fCurrentStep[i] = descendantStep;
                        continue;
                    }

                    fNoMatchDepth[i]++;
                    continue;
                }
            }

            fCurrentStep[i]++;
        }

        if (fCurrentStep[i] == stepSize) {

            if (sawDescendant) {

                fCurrentStep[i] = descendantStep;
                fMatched[i] = XP_MATCHED_D;
            }
            else {
                fMatched[i] = XP_MATCHED;
            }

            continue;
        }

        // match attribute::... step
        if (fCurrentStep[i] < stepSize &&
            locPath->getStep(fCurrentStep[i])->getAxisType() == XercesStep::AxisType_ATTRIBUTE) {

            if (attrCount) {

                XercesNodeTest* nodeTest = locPath->getStep(fCurrentStep[i])->getNodeTest();

                for (unsigned int attrIndex = 0; attrIndex < attrCount; attrIndex++) {

                    const XMLAttr* curDef = attrList.elementAt(attrIndex);

                    if (nodeTest->getType() != XercesNodeTest::NodeType_QNAME ||
                        (*(nodeTest->getName()) == *(curDef->getAttName()))) {

                        fCurrentStep[i]++;

                        if (fCurrentStep[i] == stepSize) {

                            fMatched[i] = XP_MATCHED_A;
                            int j=0;

                            for(; j<i && ((fMatched[j] & XP_MATCHED) != XP_MATCHED); j++) ;

                            if(j == i) {

                                SchemaAttDef* attDef = ((SchemaElementDecl&) elemDecl).getAttDef(curDef->getName(), curDef->getURIId());
                                DatatypeValidator* dv = (attDef) ? attDef->getDatatypeValidator() : 0;
                                matched(curDef->getValue(), dv, false);
                            }
                        }
                        break;
                    }
                }
            }

            if ((fMatched[i] & XP_MATCHED) != XP_MATCHED) {

                if(fCurrentStep[i] > descendantStep) {

                    fCurrentStep[i] = descendantStep;
                    continue;
                }

                fNoMatchDepth[i]++;
            }
        }
    }
}

void XPathMatcher::endElement(const XMLElementDecl& elemDecl,
                              const XMLCh* const elemContent) {

    for(int i = 0; i < (int) fLocationPathSize; i++) {

        // go back a step
        fCurrentStep[i] = fStepIndexes->elementAt(i)->pop();

        // don't do anything, if not matching
        if (fNoMatchDepth[i] > 0) {
            fNoMatchDepth[i]--;
        }
        // signal match, if appropriate
        else {

            int j=0;
            for(; j<i && ((fMatched[j] & XP_MATCHED) != XP_MATCHED); j++) ;

            if (j < i || (fMatched[j] == 0)
                || ((fMatched[j] & XP_MATCHED_A) == XP_MATCHED_A))
				continue;

            DatatypeValidator* dv = ((SchemaElementDecl*) &elemDecl)->getDatatypeValidator();
            bool isNillable = (((SchemaElementDecl *) &elemDecl)->getMiscFlags() & SchemaSymbols::XSD_NILLABLE) != 0;

            matched(elemContent, dv, isNillable);
            fMatched[i] = 0;
        }
    }
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Match methods
// ---------------------------------------------------------------------------
int XPathMatcher::isMatched() {

    // xpath has been matched if any one of the members of the union have matched.
    for (int i=0; i < (int) fLocationPathSize; i++) {
        if (((fMatched[i] & XP_MATCHED) == XP_MATCHED)
            && ((fMatched[i] & XP_MATCHED_DP) != XP_MATCHED_DP))
            return fMatched[i];
    }

    return 0;
}

void XPathMatcher::matched(const XMLCh* const,
                           DatatypeValidator* const,
                           const bool) {
    return;
}

// ---------------------------------------------------------------------------
//  XPathMatcher: Match methods
// ---------------------------------------------------------------------------
int XPathMatcher::getInitialDepth() const
{
    ThrowXMLwithMemMgr(RuntimeException, XMLExcepts::Regex_NotSupported, fMemoryManager);
    return 0; // to make some compilers happy
}

XERCES_CPP_NAMESPACE_END

/**
  * End of file XPathMatcher.cpp
  */

<|endoftext|>


<commit_before>/*ckwg +5
 * Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
 * KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
 * Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.
 */

#include "registration.h"

#include "sync_schedule.h"
#include "thread_per_process_schedule.h"
#include "thread_pool_schedule.h"

#include <vistk/pipeline/schedule_registry.h>

#include <boost/make_shared.hpp>

#include <cstddef>

/**
 * \file examples/registration.cxx
 *
 * \brief Register schedules for use.
 */

using namespace vistk;

static schedule_t create_sync_schedule(config_t const& config, pipeline_t const& pipe);
static schedule_t create_thread_per_process_schedule(config_t const& config, pipeline_t const& pipe);
static schedule_t create_thread_pool_schedule(config_t const& config, pipeline_t const& pipe);

void
register_schedules()
{
  static schedule_registry::module_t const module_name = schedule_registry::module_t("example_schedules");

  schedule_registry_t const registry = schedule_registry::self();

  if (registry->is_module_loaded(module_name))
  {
    return;
  }

  registry->register_schedule("sync", "Runs the pipeline synchronously", create_sync_schedule);
  registry->register_schedule("thread_per_process", "Runs each process in its own thread", create_thread_per_process_schedule);
  registry->register_schedule("thread_pool", "Uses a pool of threads to step processes", create_thread_pool_schedule);

  registry->mark_module_as_loaded(module_name);
}

schedule_t
create_sync_schedule(config_t const& config, pipeline_t const& pipe)
{
  return boost::make_shared<sync_schedule>(config, pipe);
}

schedule_t
create_thread_per_process_schedule(config_t const& config, pipeline_t const& pipe)
{
  return boost::make_shared<thread_per_process_schedule>(config, pipe);
}

schedule_t
create_thread_pool_schedule(config_t const& config, pipeline_t const& pipe)
{
  return boost::make_shared<thread_pool_schedule>(config, pipe);
}
<commit_msg>Use the schedule registration macro<commit_after>/*ckwg +5
 * Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
 * KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
 * Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.
 */

#include "registration.h"

#include "sync_schedule.h"
#include "thread_per_process_schedule.h"
#include "thread_pool_schedule.h"

#include <vistk/pipeline/schedule_registry.h>

#include <boost/make_shared.hpp>

#include <cstddef>

/**
 * \file examples/registration.cxx
 *
 * \brief Register schedules for use.
 */

using namespace vistk;

void
register_schedules()
{
  static schedule_registry::module_t const module_name = schedule_registry::module_t("example_schedules");

  schedule_registry_t const registry = schedule_registry::self();

  if (registry->is_module_loaded(module_name))
  {
    return;
  }

  registry->register_schedule("sync", "Runs the pipeline synchronously", CREATE_SCHEDULE(sync_schedule));
  registry->register_schedule("thread_per_process", "Runs each process in its own thread", CREATE_SCHEDULE(thread_per_process_schedule));
  registry->register_schedule("thread_pool", "Uses a pool of threads to step processes", CREATE_SCHEDULE(thread_pool_schedule));

  registry->mark_module_as_loaded(module_name);
}
<|endoftext|>


<commit_before>#include "twister.h"

const unsigned CYCLE = 0x100;

//Suggest we wrap these arrays inside an fsm or test base class
const unsigned C_TRANS_ = 0xff;
int block_units[][0x10] = {
	0x74, 0x63, 0x69, 0x67, 0x72, 0x68, 0x6e, 0x6c, 0x61, 0x67, 0x74, 0x6c, 0x69, 0x73, 0x61, 0x61, 0x6f, 0x6d, 0x6f, 0x72, 0x6e, 0x72, 0x69, 0x6d, 0x6e, 0x6e, 0x65, 0x74, 0x72, 0x6e, 0x6b, 0x73, 0x61, 0x6f, 0x65, 0x69, 0x73, 0x64, 0x6c, 0x65, 0x69, 0x6b, 0x68, 0x68, 0x68, 0x65, 0x72, 0x64, 0x66, 0x74, 0x61, 0x73, 0x6f, 0x69, 0x65, 0x74, 0x69, 0x68, 0x6e, 0x65, 0x62, 0x74, 0x6b, 0x65
  };
int triad_units[][0x03] = {
	0x31, 0xfe, 0x45, 0x74, 0x68, 0x65, 0x74, 0x68, 0x61, 0x65, 0x74, 0x68, 0x66, 0x6f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x65, 0x68, 0x61, 0x74, 0x68, 0x69, 0x73, 0x1a, 0x36, 0x49, 0x91, 0x43, 0x53
};
const int transition_seq[] = { 0xce, 0xab, 0xdf, 0xcf, 0xee,
                               0xcf, 0xad, 0xdf, 0xff, 0xce,
                               0x32, 0x40, 0xd3, 0x27, 0x82,
                               0xda, 0xee, 0xff, 0xfc, 0xbf,
                               0x1c, 0x90, 0x13, 0x4a, 0xa5,
                               0xe0, 0x21, 0x9f, 0xe1, 0xc6,
                               0xaf, 0x05, 0x81, 0xf0, 0xee,
                               0xe4, 0x38, 0x1f, 0x60, 0x24,
                               0x0c, 0x35, 0x51, 0x32, 0xcf,
                               0x12, 0x9a, 0x30, 0x44, 0x72,
                               0x51, 0x3c, 0x61, 0x2d, 0x5f,
                               0x04, 0x1c, 0x52, 0xca, 0xdf,
                               0x12, 0x0b, 0x30, 0xa0, 0x1e,
                               0x03, 0x14, 0x09, 0x73, 0x23,
                               0xf2, 0xca, 0xa2, 0x51, 0xc6,
                               0x01, 0xdf, 0x41, 0x96, 0xa0,
                               0x51, 0x19, 0x71, 0x23, 0x47,
                               0xcb, 0xbd, 0xba, 0xac, 0xdf,
			       0xdf, 0xde, 0xcd, 0xfd, 0xca };

int ENTRY_LINK__ = 0x05;
int ENTRY_LINK__TEST = 0x09;
int ENTRY_C_REF_ECM = 0x10;
int ENTRY_C_REF_ECM_TEST = 0x03;
int ENTRY_C_OUTER = 0x51;
int ENTRY_C_OUTER_PROD = 0x41;
int ENTRY_C_OUTER_PROD_TEST = 0x33;
int ENTRY_C_OUTER_PROD_TOR_TEST = 0x57;
int ENTRY_A_OUTER_PROD_TOR_TEST = 0xc9;
int ENTRY_B_OUTER_PROD_EUC_TEST = 0xaf;
int ENTRY_C_OUTER_PROD_EUC_TEST = 0xa1;
int ENTRY_C_INNER_PROD_EUC_TEST = 0xc5;
int ENTRY_C_INNER_PROD_ELIP_TEST = 0xa2;
int ENTRY_C_INNER_PROD_RELAY_TEST = 0xf5;
int ENTRY_C_BLOCK_REL_FLAG_TEST = 0x0fffffff;
int ENTRY_C_BLOCK_REL_FLAG_VECTOR = 0x0effffff;

unsigned reflect(unsigned center, unsigned (*r)(unsigned))
{
  return (*r)(center)^center;
}
unsigned char base(unsigned char a, unsigned char (*s)(unsigned char), int pos)
{
	return (*s)(pos)^a;
}

void transHomExt(vector<unsigned char>& data, unsigned char (*f)(unsigned char), unsigned char (*g)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d)^base(d, g, d);
    }
    data[i] = d;
  }
}
void transHom(vector<unsigned char>& data, unsigned char (*f)(unsigned char), unsigned char (*g)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d)^(*g)(d);
    }
    data[i] = d;
  }
}

void trans(vector<unsigned char>& data, unsigned char (*f)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d);
    }
    data[i] = d;
  }
}

vector<double> f_dist(vector<unsigned char>& in)
{
  vector<double> fdist;
  fdist.resize(256);
  for(unsigned i = 0; i<in.size(); i++)
  {
    fdist[in[i]]++;
  }

  for(unsigned i=0; i<fdist.size(); i++)
  {
    fdist[i] = (fdist[i] / in.size()) * 100;
  }

  return fdist;
}

double s_entropy(vector<double>& v)
{
  double entropy = 0;
  double p;

  for(unsigned i = 0; i < v.size(); i++)
  {
    p = v[i] / 100;
    if (p == 0) continue;
    entropy += p * std::log(p) / std::log(2);
  }
  return -entropy;
}
void rms(const string& s, string& r)
{
  for(unsigned int i=0; i<s.size(); i++)
  {
    if(::isspace(s[i])) continue;
    r+= s[i];
  }
}
//add extended euc
double sw(double weight, int i, int j, int (*inv)(int, int))
{
  return weight*(*inv)(i, j);
}
void hPerm(int s, int n, void (*p)(int), void (*inv)(int, int), void (*center)(int))
{
  if(transition_seq[ENTRY_C_REF_ECM ] == s)
  {
	  (*center)(s);
	  return;
  }
  if(s == 1)
  {
    (*p)(n);
    return;
  }
  for(int i=0; i< s; i++)
  {
    hPerm(s-1, n, p, inv, p);
    if(s%2 == 1)
      (*inv)(0, s-1);
    else
      (*inv)(i, s-1);
  }
}

double ic(const string& t)
{
  string text; rms(t, text);
  vector<double> freq(256,0);
  for(unsigned int i=0; i<text.size(); i++)
  {
    if(text[i] == ' ') continue;
    freq[text[i]] ++;
  }
  double sum=0;
  for(unsigned int i=0; i<freq.size(); i++)
  {
    if(freq[i] != 0)
    {
      double c = freq[i];
      if(c != 0)
        sum += c * (c - 1);
    }
  }
  double ic = 26 * sum / (text.size() * (text.size() - 1));
  return ic;
}

int outer_sect(int (*s)(int), int (*t)(int), int r, int q)
{
  return (*s)(r) * (*t)(q);
}

void switchIO(unsigned char (*p)(unsigned char, unsigned char), unsigned char m)
{
  //test seq
  (*p)(transition_seq[ENTRY_LINK__], m);
  (*p)(transition_seq[ENTRY_LINK__TEST], m);
  (*p)(transition_seq[ENTRY_C_REF_ECM], m);
  (*p)(transition_seq[ENTRY_C_INNER_PROD_ELIP_TEST  ], m);
  (*p)(transition_seq[ENTRY_LINK__TEST], m ^ transition_seq[ENTRY_LINK__TEST]);
  (*p)(transition_seq[ENTRY_LINK__TEST], m ^ transition_seq[ENTRY_C_REF_ECM]);
}

//Suggest we wrap these in util base class, abstract col cont
std::tuple<int, int, int> extended_gcd(int __alpha, int __beta, int (*col)(int x, int y))
{
  if(__alpha == 0) return make_tuple(__beta,0,1);
  int __com=0;
  int x=0;
  int y=0;
  tie(__com, x, y) = extended_gcd(__beta%__alpha, __alpha, col);
  return make_tuple(__com, y-(__beta/__alpha)*x, x);
}

std::tuple<int, int, int> extended_gcd(int __alpha, int __beta)
{
  if(__alpha == 0) return make_tuple(__beta,0,1);
  int __com=0;
  int x=0;
  int y=0;
  tie(__com, x, y) = extended_gcd(__beta%__alpha, __alpha);
  return make_tuple(__com, y-(__beta/__alpha)*x, x);
}
<commit_msg>block rel,  flag test, ext<commit_after>#include "twister.h"

const unsigned CYCLE = 0x100;

//Suggest we wrap these arrays inside an fsm or test base class
const unsigned C_TRANS_ = 0xff;
int block_units[][0x10] = {
	0x74, 0x63, 0x69, 0x67, 0x72, 0x68, 0x6e, 0x6c, 0x61, 0x67, 0x74, 0x6c, 0x69, 0x73, 0x61, 0x61, 0x6f, 0x6d, 0x6f, 0x72, 0x6e, 0x72, 0x69, 0x6d, 0x6e, 0x6e, 0x65, 0x74, 0x72, 0x6e, 0x6b, 0x73, 0x61, 0x6f, 0x65, 0x69, 0x73, 0x64, 0x6c, 0x65, 0x69, 0x6b, 0x68, 0x68, 0x68, 0x65, 0x72, 0x64, 0x66, 0x74, 0x61, 0x73, 0x6f, 0x69, 0x65, 0x74, 0x69, 0x68, 0x6e, 0x65, 0x62, 0x74, 0x6b, 0x65
  };
int triad_units[][0x03] = {
	0x31, 0xfe, 0x45, 0x74, 0x68, 0x65, 0x74, 0x68, 0x61, 0x65, 0x74, 0x68, 0x66, 0x6f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x65, 0x68, 0x61, 0x74, 0x68, 0x69, 0x73, 0x1a, 0x36, 0x49, 0x91, 0x43, 0x53
};
const int transition_seq[] = { 0xce, 0xab, 0xdf, 0xcf, 0xee,
                               0xcf, 0xad, 0xdf, 0xff, 0xce,
                               0x32, 0x40, 0xd3, 0x27, 0x82,
                               0xda, 0xee, 0xff, 0xfc, 0xbf,
                               0x1c, 0x90, 0x13, 0x4a, 0xa5,
                               0xe0, 0x21, 0x9f, 0xe1, 0xc6,
                               0xaf, 0x05, 0x81, 0xf0, 0xee,
                               0xe4, 0x38, 0x1f, 0x60, 0x24,
                               0x0c, 0x35, 0x51, 0x32, 0xcf,
                               0x12, 0x9a, 0x30, 0x44, 0x72,
                               0x51, 0x3c, 0x61, 0x2d, 0x5f,
                               0x04, 0x1c, 0x52, 0xca, 0xdf,
                               0x12, 0x0b, 0x30, 0xa0, 0x1e,
                               0x03, 0x14, 0x09, 0x73, 0x23,
                               0xf2, 0xca, 0xa2, 0x51, 0xc6,
                               0x01, 0xdf, 0x41, 0x96, 0xa0,
                               0x51, 0x19, 0x71, 0x23, 0x47,
                               0xcb, 0xbd, 0xba, 0xac, 0xdf,
			       0xdf, 0xde, 0xcd, 0xfd, 0xca };

int ENTRY_LINK__ = 0x05;
int ENTRY_LINK__TEST = 0x09;
int ENTRY_C_REF_ECM = 0x10;
int ENTRY_C_REF_ECM_TEST = 0x03;
int ENTRY_C_OUTER = 0x51;
int ENTRY_C_OUTER_PROD = 0x41;
int ENTRY_C_OUTER_PROD_TEST = 0x33;
int ENTRY_C_OUTER_PROD_TOR_TEST = 0x57;
int ENTRY_A_OUTER_PROD_TOR_TEST = 0xc9;
int ENTRY_B_OUTER_PROD_EUC_TEST = 0xaf;
int ENTRY_C_OUTER_PROD_EUC_TEST = 0xa1;
int ENTRY_C_INNER_PROD_EUC_TEST = 0xc5;
int ENTRY_C_INNER_PROD_ELIP_TEST = 0xa2;
int ENTRY_C_INNER_PROD_RELAY_TEST = 0xf5;
int ENTRY_C_BLOCK_REL_FLAG_TEST = 0x0fffffff;
int ENTRY_C_BLOCK_REL_FLAG_VECTOR = 0x0effffff;

unsigned reflect(unsigned center, unsigned (*r)(unsigned))
{
  return (*r)(center)^center;
}
unsigned char base(unsigned char a, unsigned char (*s)(unsigned char), int pos)
{
	return (*s)(pos)^a;
}

void transHomExt(vector<unsigned char>& data, unsigned char (*f)(unsigned char), unsigned char (*g)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d)^base(d, g, d);
    }
    data[i] = d;
  }
}
void transHom(vector<unsigned char>& data, unsigned char (*f)(unsigned char), unsigned char (*g)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d)^(*g)(d);
    }
    data[i] = d;
  }
}

void trans(vector<unsigned char>& data, unsigned char (*f)(unsigned char))
{
  for(unsigned i = 0; i<data.size(); i++)
  {
    unsigned char d = data[i];
    for(unsigned j=0; j<CYCLE; j++)
    {
      d = (*f)(d);
    }
    data[i] = d;
  }
}

vector<double> f_dist(vector<unsigned char>& in)
{
  vector<double> fdist;
  fdist.resize(256);
  for(unsigned i = 0; i<in.size(); i++)
  {
    fdist[in[i]]++;
  }

  for(unsigned i=0; i<fdist.size(); i++)
  {
    fdist[i] = (fdist[i] / in.size()) * 100;
  }

  return fdist;
}

double s_entropy(vector<double>& v)
{
  double entropy = 0;
  double p;

  for(unsigned i = 0; i < v.size(); i++)
  {
    p = v[i] / 100;
    if (p == 0) continue;
    entropy += p * std::log(p) / std::log(2);
  }
  return -entropy;
}
void rms(const string& s, string& r)
{
  for(unsigned int i=0; i<s.size(); i++)
  {
    if(::isspace(s[i])) continue;
    r+= s[i];
  }
}
//add extended euc
double sw(double weight, int i, int j, int (*inv)(int, int))
{
  return weight*(*inv)(i, j);
}
//Suggest we wrap these routines in a base class in particular for testing
//and expose test api
void hPerm(int s, int n, void (*p)(int), void (*inv)(int, int), void (*center)(int))
{
  if(transition_seq[ENTRY_C_REF_ECM ] == s)
  {
	  (*center)(s);
	  return;
  }
  if(transition_seq[ENTRY_C_INNER_PROD_ELIP_TEST ] == s)
  {
	  (*p)(s);
	  return;
  }
  if(s == 1)
  {
    (*p)(n);
    return;
  }
  for(int i=0; i< s; i++)
  {
    hPerm(s-1, n, p, inv, p);
    if(s%2 == 1)
      (*inv)(0, s-1);
    else
      (*inv)(i, s-1);
  }
}

double ic(const string& t)
{
  string text; rms(t, text);
  vector<double> freq(256,0);
  for(unsigned int i=0; i<text.size(); i++)
  {
    if(text[i] == ' ') continue;
    freq[text[i]] ++;
  }
  double sum=0;
  for(unsigned int i=0; i<freq.size(); i++)
  {
    if(freq[i] != 0)
    {
      double c = freq[i];
      if(c != 0)
        sum += c * (c - 1);
    }
  }
  double ic = 26 * sum / (text.size() * (text.size() - 1));
  return ic;
}

int outer_sect(int (*s)(int), int (*t)(int), int r, int q)
{
  return (*s)(r) * (*t)(q);
}

void switchIO(unsigned char (*p)(unsigned char, unsigned char), unsigned char m)
{
  //test seq
  (*p)(transition_seq[ENTRY_LINK__], m);
  (*p)(transition_seq[ENTRY_LINK__TEST], m);
  (*p)(transition_seq[ENTRY_C_REF_ECM], m);
  (*p)(transition_seq[ENTRY_C_INNER_PROD_ELIP_TEST  ], m);
  (*p)(transition_seq[ENTRY_LINK__TEST], m ^ transition_seq[ENTRY_LINK__TEST]);
  (*p)(transition_seq[ENTRY_LINK__TEST], m ^ transition_seq[ENTRY_C_REF_ECM]);
}

//Suggest we wrap these in util base class, abstract col cont
std::tuple<int, int, int> extended_gcd(int __alpha, int __beta, int (*col)(int x, int y))
{
  if(__alpha == 0) return make_tuple(__beta,0,1);
  int __com=0;
  int x=0;
  int y=0;
  tie(__com, x, y) = extended_gcd(__beta%__alpha, __alpha, col);
  return make_tuple(__com, y-(__beta/__alpha)*x, x);
}

std::tuple<int, int, int> extended_gcd(int __alpha, int __beta)
{
  if(__alpha == 0) return make_tuple(__beta,0,1);
  int __com=0;
  int x=0;
  int y=0;
  tie(__com, x, y) = extended_gcd(__beta%__alpha, __alpha);
  return make_tuple(__com, y-(__beta/__alpha)*x, x);
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <string>

#include "version.h"

// Name of client reported in the 'version' message. Report the same name
// for both bitcoind and bitcoin-qt, to make it harder for attackers to
// target servers or GUI users specifically.
const std::string CLIENT_NAME("Dopecoin");

// Client version number
#define CLIENT_VERSION_SUFFIX   "-supr"

// The following part of the code determines the CLIENT_BUILD variable.
// Several mechanisms are used for this:
// * first, if HAVE_BUILD_INFO is defined, include build.h, a file that is
//   generated by the build environment, possibly containing the output
//   of git-describe in a macro called BUILD_DESC
// * secondly, if this is an exported version of the code, GIT_ARCHIVE will
//   be defined (automatically using the export-subst git attribute), and
//   GIT_COMMIT will contain the commit id.
// * then, three options exist for determining CLIENT_BUILD:
//   * if BUILD_DESC is defined, use that literally (output of git-describe)
//   * if not, but GIT_COMMIT is defined, use v[maj].[min].[rev].[build]-g[commit]
//   * otherwise, use v[maj].[min].[rev].[build]-unk
// finally CLIENT_VERSION_SUFFIX is added

// First, include build.h if requested
#ifdef HAVE_BUILD_INFO
#    include "build.h"
#endif

// git will put "#define GIT_ARCHIVE 1" on the next line inside archives.
#define GIT_ARCHIVE 1
#ifdef GIT_ARCHIVE
#    define GIT_COMMIT_ID ""
#    define GIT_COMMIT_DATE "2014/07/18"
#endif

#define BUILD_DESC_FROM_COMMIT(maj,min,rev,build,commit) \
    "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "" commit

#define BUILD_DESC_FROM_UNKNOWN(maj,min,rev,build) \
    "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) ""

#ifndef BUILD_DESC
#    ifdef GIT_COMMIT_ID
#        define BUILD_DESC BUILD_DESC_FROM_COMMIT(DISPLAY_VERSION_MAJOR, DISPLAY_VERSION_MINOR, DISPLAY_VERSION_REVISION, DISPLAY_VERSION_BUILD, GIT_COMMIT_ID)
#    else
#        define BUILD_DESC BUILD_DESC_FROM_UNKNOWN(DISPLAY_VERSION_MAJOR, DISPLAY_VERSION_MINOR, DISPLAY_VERSION_REVISION, DISPLAY_VERSION_BUILD)
#    endif
#endif

#ifndef BUILD_DATE
#    ifdef GIT_COMMIT_DATE
#        define BUILD_DATE GIT_COMMIT_DATE
#    else
#        define BUILD_DATE __DATE__ ", " __TIME__
#    endif
#endif

const std::string CLIENT_BUILD(BUILD_DESC CLIENT_VERSION_SUFFIX);
const std::string CLIENT_DATE(BUILD_DATE);
<commit_msg>Fix build date<commit_after>// Copyright (c) 2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <string>

#include "version.h"

// Name of client reported in the 'version' message. Report the same name
// for both bitcoind and bitcoin-qt, to make it harder for attackers to
// target servers or GUI users specifically.
const std::string CLIENT_NAME("Dopecoin");

// Client version number
#define CLIENT_VERSION_SUFFIX   "-supr"

// The following part of the code determines the CLIENT_BUILD variable.
// Several mechanisms are used for this:
// * first, if HAVE_BUILD_INFO is defined, include build.h, a file that is
//   generated by the build environment, possibly containing the output
//   of git-describe in a macro called BUILD_DESC
// * secondly, if this is an exported version of the code, GIT_ARCHIVE will
//   be defined (automatically using the export-subst git attribute), and
//   GIT_COMMIT will contain the commit id.
// * then, three options exist for determining CLIENT_BUILD:
//   * if BUILD_DESC is defined, use that literally (output of git-describe)
//   * if not, but GIT_COMMIT is defined, use v[maj].[min].[rev].[build]-g[commit]
//   * otherwise, use v[maj].[min].[rev].[build]-unk
// finally CLIENT_VERSION_SUFFIX is added

// First, include build.h if requested
#ifdef HAVE_BUILD_INFO
#    include "build.h"
#endif

// git will put "#define GIT_ARCHIVE 1" on the next line inside archives.
// #define GIT_ARCHIVE 1
#ifdef GIT_ARCHIVE
#    define GIT_COMMIT_ID ""
#    define GIT_COMMIT_DATE "2014/07/18"
#endif

#define BUILD_DESC_FROM_COMMIT(maj,min,rev,build,commit) \
    "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "" commit

#define BUILD_DESC_FROM_UNKNOWN(maj,min,rev,build) \
    "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) ""

#ifndef BUILD_DESC
#    ifdef GIT_COMMIT_ID
#        define BUILD_DESC BUILD_DESC_FROM_COMMIT(DISPLAY_VERSION_MAJOR, DISPLAY_VERSION_MINOR, DISPLAY_VERSION_REVISION, DISPLAY_VERSION_BUILD, GIT_COMMIT_ID)
#    else
#        define BUILD_DESC BUILD_DESC_FROM_UNKNOWN(DISPLAY_VERSION_MAJOR, DISPLAY_VERSION_MINOR, DISPLAY_VERSION_REVISION, DISPLAY_VERSION_BUILD)
#    endif
#endif

#ifndef BUILD_DATE
#    ifdef GIT_COMMIT_DATE
#        define BUILD_DATE GIT_COMMIT_DATE
#    else
#        define BUILD_DATE __DATE__ ", " __TIME__
#    endif
#endif

const std::string CLIENT_BUILD(BUILD_DESC CLIENT_VERSION_SUFFIX);
const std::string CLIENT_DATE(BUILD_DATE);
<|endoftext|>


<commit_before>//
//  Copyright Silvin Lubecki 2010
//
//  Distributed under the Boost Software License, Version 1.0. (See
//  accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//

#include "test.h"
#include "openclam/cl.hpp"

BOOST_AUTO_TEST_CASE( kernel_keyword_substitution_test )
{
    const std::string expected =
        "kernel void MyKernel() "
        "{ "
        "}";
    const std::string actual = KERNEL( MyKernel,
        kernel void MyKernel()
        {
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( __kernel_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel() "
        "{ "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel()
        {
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( global_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel( global const float* a ) "
        "{ "
        "a; "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel( global const float* a )
        {
            a;
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( __global_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel( __global const float* a ) "
        "{ "
        "a; "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel( __global const float* a )
        {
            a;
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}
<commit_msg>add a commented test related to issue 1<commit_after>//
//  Copyright Silvin Lubecki 2010
//
//  Distributed under the Boost Software License, Version 1.0. (See
//  accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//

#include "test.h"
#include "openclam/cl.hpp"

//BOOST_AUTO_TEST_CASE( kernel_keyword_is_a_valid_identifier ) // $$$$ 27-02-2010 SILVIN: uncomment this test related to issue 1 (http://code.google.com/p/openclam/issues/detail?id=1)
//{
//    const std::string kernel   = "";
//    const std::string __kernel = "";
//    const std::string global   = "";
//    const std::string __global = "";
//}

BOOST_AUTO_TEST_CASE( kernel_keyword_substitution_test )
{
    const std::string expected =
        "kernel void MyKernel() "
        "{ "
        "}";
    const std::string actual = KERNEL( MyKernel,
        kernel void MyKernel()
        {
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( __kernel_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel() "
        "{ "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel()
        {
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( global_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel( global const float* a ) "
        "{ "
        "a; "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel( global const float* a )
        {
            a;
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}

BOOST_AUTO_TEST_CASE( __global_keyword_substitution_test )
{
    const std::string expected =
        "__kernel void MyKernel( __global const float* a ) "
        "{ "
        "a; "
        "}";
    const std::string actual = KERNEL( MyKernel,
        __kernel void MyKernel( __global const float* a )
        {
            a;
        } );
    BOOST_CHECK_EQUAL( actual, expected );
}
<|endoftext|>


<commit_before>#include <chrono>
#include <cstdint>
#include <exception>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

#include <boost/algorithm/string.hpp>
#include <boost/exception/exception.hpp>

#include <gflags/gflags.h>
#include <glog/logging.h>

#include <tudocomp/Compressor.hpp>
#include <tudocomp/io.h>
#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp_driver/registry.h>

/*namespace validate {
  static bool algorithm(const char* flagname, int32 value) {
    if (value > 0 && value < 32768)   // value is ok
      return true;
    printf("Invalid value for --%s: %d\n", flagname, (int)value);
    return false;
  }
  static const bool algorithm_dummy = RegisterFlagValidator(&FLAGS_algorithm, &algorithm);
}*/

DEFINE_string(algorithm, "", "Algorithm to use for (de)compression. See --list for a complete list of them.");
DEFINE_bool(decompress, false, "Decompress input instead of compressing it.");
DEFINE_string(output, "", "Choose output filename instead the the default generated one of <input>.tdc or stdout.");
DEFINE_bool(stats, false, "Print statistics to stdout.");
DEFINE_bool(force, false, "Overwrite output even if it exists.");
DEFINE_bool(list, false, "List all compression algorithms supported by this tool.");
DEFINE_bool(raw, false, "Do not emit an header into the output file when compressing.");
DEFINE_bool(stdin, false, "Read from stdin instead of trying to open a file.");
DEFINE_bool(stdout, false, "Output to stdout instead of writing to a file");

namespace tudocomp_driver {

using namespace tudocomp;

const std::string COMPRESSED_FILE_ENDING = "tdc";

static void exit(std::string msg) {
    // TODO: Replace with more specific logic-error-exception
    throw std::runtime_error(msg);
}

static void validate_flag_combination() {
    auto err = [](std::string s) {
        exit(s);
    };

    if (!FLAGS_list) {
        if (!FLAGS_decompress && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for compression");
        }
        if (FLAGS_decompress && FLAGS_raw && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for raw decompression");
        }
    }
}

static bool fexists(std::string filename)
{
  std::ifstream ifile(filename);
  return bool(ifile);
}

static bool check_for_file_already_exist(std::string& ofile,
                                         bool allow_overwrite) {
    // Don't accidentially overwrite files
    if (!allow_overwrite && fexists(ofile)) {
        std::cerr << "Outputfile already exists\n";
        return false;
    }
    return true;
}

} // namespace tudocomp_driver

int main(int argc, char** argv)
{
    using namespace tudocomp_driver;

    google::InitGoogleLogging(argv[0]);
    int first_cmd_arg = google::ParseCommandLineFlags(&argc, &argv, true);

    try {
        validate_flag_combination();

        std::map<std::string, const std::string> algorithm_options;

        /*for (auto& os : string_args("--option")) {
            std::vector<std::string> options;
            boost::split(options, os, boost::is_any_of(","));
            for (auto& o : options) {
                std::vector<std::string> key_value;
                boost::split(key_value, o, boost::is_any_of("="));
                CHECK(key_value.size() == 2);
                algorithm_options.emplace(key_value[0], key_value[1]);
            }
        }*/

        /*if (arg_exists("--help")) {
            show_help();
            return 0;
        }*/

        Env algorithm_env(algorithm_options, {});

        // Set up algorithms
        AlgorithmDb root;
        Registry registry {&root};
        register_algos(registry);

        if (FLAGS_list) {
            std::cout << "This build supports the following algorithms:\n";
            std::cout << std::endl;

            for (auto& e: registry.get_sub_algos()) {
                e.print_to(std::cout, 0);
            }

            return 0;
        }

        bool print_stats = FLAGS_stats;
        int alphabet_size = 0;

        bool do_compress = !FLAGS_decompress;

        /////////////////////////////////////////////////////////////////////////
        // Select algorithm

        bool do_raw = FLAGS_raw;

        std::string algorithm_id;
        std::unique_ptr<Compressor> algo;

        if (do_raw || do_compress) {
            algorithm_id = FLAGS_algorithm;
            algo = select_algo_or_exit(registry, algorithm_env, algorithm_id);
        }

        /////////////////////////////////////////////////////////////////////////
        // Select where the input comes from

        std::string file;

        if ((!FLAGS_stdin) && (first_cmd_arg < argc)) {
            file = argv[first_cmd_arg];
            if (!fexists(file)) {
                std::cerr << "input file " << file << " does not exist\n";
                return 1;
            }
        } else if (!FLAGS_stdin) {
            std::cerr << "No input file given\n";
            return 1;
        }

        bool use_stdin = FLAGS_stdin;

        /////////////////////////////////////////////////////////////////////////
        // Select where the output goes to

        std::string ofile;
        bool use_stdout = FLAGS_stdout;
        bool use_explict_output = std::string(FLAGS_output) != "" ;

        if (use_explict_output) {
            // Output to manually specifed file
            ofile = FLAGS_output;
        } else if (!use_stdin && do_compress) {
            // Output to a automatically determined file
            ofile = file + "." + COMPRESSED_FILE_ENDING;
        } else if (!use_stdin && !do_compress && !use_stdout) {
            std::cerr << "Need to specify a output filename\n";
            return 1;
        }

        /////////////////////////////////////////////////////////////////////////
        // open streams

        using clk = std::chrono::high_resolution_clock;

        clk::time_point start_time = clk::now();
        clk::time_point setup_time;
        clk::time_point comp_time;
        clk::time_point end_time;

        {
            std::vector<uint8_t> stream_buffer;
            Input inp;

            if (use_stdin) {
                // Input from stdin
                stream_buffer = read_stream_to_stl_byte_container<
                    std::vector<uint8_t>
                >(std::cin);
                inp = Input::from_memory(stream_buffer);
            } else {
                // Input from specified file
                inp = Input::from_path(file);
            }

            Output out;
            if (use_stdout) {
                // Output to stdout
                out = Output::from_stream(std::cout);
            } else {
                // Output to specified file
                bool force = FLAGS_force;
                if (!check_for_file_already_exist(ofile, force)) {
                    return 1;
                } else {
                    out = Output::from_path(ofile, true);
                }
            }

            /////////////////////////////////////////////////////////////////////////
            // call into actual library

            if (do_compress) {
                if (print_stats) {
                    alphabet_size = 0; // count_alphabet_size(inp_vec);
                }

                if (!do_raw) {
                    CHECK(algorithm_id.find('%') == std::string::npos);

                    auto o_stream = out.as_stream();
                    (*o_stream) << algorithm_id << '%';
                }

                setup_time = clk::now();

                algo->compress(inp, out);

                comp_time = clk::now();
            } else {
                if (!do_raw) {
                    auto i_stream = inp.as_stream();
                    std::string algorithm_header;

                    char c;
                    size_t sanity_size_check = 0;
                    bool err = false;
                    while ((*i_stream).get(c)) {
                        err = false;
                        if (sanity_size_check > 1023) {
                            err = true;
                            break;
                        } else if (c == '%') {
                            break;
                        } else {
                            algorithm_header.push_back(c);
                        }
                        sanity_size_check++;
                        err = true;
                    }
                    if (err) {
                        exit("Input did not have an algorithm header!");
                    }
                    algorithm_id = std::move(algorithm_header);
                    algo = select_algo_or_exit(registry, algorithm_env, algorithm_id);
                }

                setup_time = clk::now();

                algo->decompress(inp, out);

                comp_time = clk::now();
            }
        }

        end_time = clk::now();

        if (print_stats) {
            auto setup_duration = setup_time - start_time;
            auto comp_duration = comp_time - setup_time;
            auto end_duration = end_time - comp_time;
            std::cout << "---------------\n";
            std::cout << "Config: " << algorithm_id << std::endl;
            std::cout << "---------------\n";
            auto inp_size = 0;
            if (use_stdin) {
                std::cout << "input: <stdin>\n";
                std::cout << "input size: ? B\n";
            } else {
                inp_size = read_file_size(file);
                std::cout << "input: "<<file<<"\n";
                std::cout << "input size: "<<inp_size<<" B\n";
            }
            std::cout << "alphabet size: " << alphabet_size << "\n";

            auto out_size = 0;
            if (use_stdout) {
                std::cout << "output: <stdout>\n";
                std::cout << "output size: ? B\n";
            } else {
                out_size = read_file_size(ofile);
                std::cout << "output: "<<ofile<<"\n";
                std::cout << "output size: "<<out_size<<" B\n";
            }
            std::cout << "---------------\n";
            if (inp_size != 0) {
                std::cout << "compress ratio: "<<(double(out_size)/double(inp_size) * 100)<<"%\n";
            }

            auto print_time = [] (std::string s, decltype(setup_duration)& t) {
                std::chrono::seconds sec(1);
                std::chrono::milliseconds milsec(1);
                std::chrono::microseconds micsec(1);
                using dur = std::chrono::duration<float>;
                using mildur = std::chrono::duration<float, std::milli>;
                using micdur = std::chrono::duration<float, std::micro>;

                if (t < milsec) {
                    auto ct = std::chrono::duration_cast<micdur>(t).count();
                    std::cout << s <<" time: "<< ct << " µs\n";
                } else if (t < sec) {
                    auto ct = std::chrono::duration_cast<mildur>(t).count();
                    std::cout << s <<" time: "<< ct << " ms\n";
                } else {
                    auto ct = std::chrono::duration_cast<dur>(t).count();
                    std::cout << s <<" time: "<< ct << " s\n";
                }
            };

            std::cout << "---------------\n";
            std::cout << "Algorithm Known Options:\n";
            for (auto& pair : algorithm_env.get_known_options()) {
                std::cout << "  " << pair << "\n";
            }
            std::cout << "Algorithm Given Options:\n";
            for (auto& pair : algorithm_env.get_options()) {
                std::cout << "  " << pair.first << " = " << pair.second << "\n";
            }

            std::cout << "---------------\n";
            std::cout << "Algorithm Stats:\n";
            for (auto& pair : algorithm_env.get_stats()) {
                std::cout << "  " << pair.first << ": " << pair.second << "\n";
            }

            std::cout << "---------------\n";
            print_time("startup", setup_duration);
            print_time("compression", comp_duration);
            print_time("teardown", end_duration);
            std::cout << "---------------\n";
        }
    } catch (std::exception& e) {
        std::cout << "Error: " << e.what() << '\n';
        return 1;
    }

    return 0;
}
<commit_msg>Purge unused boost includes from tudocomp_driver<commit_after>#include <chrono>
#include <cstdint>
#include <exception>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

#include <gflags/gflags.h>
#include <glog/logging.h>

#include <tudocomp/Compressor.hpp>
#include <tudocomp/io.h>
#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp_driver/registry.h>

/*namespace validate {
  static bool algorithm(const char* flagname, int32 value) {
    if (value > 0 && value < 32768)   // value is ok
      return true;
    printf("Invalid value for --%s: %d\n", flagname, (int)value);
    return false;
  }
  static const bool algorithm_dummy = RegisterFlagValidator(&FLAGS_algorithm, &algorithm);
}*/

DEFINE_string(algorithm, "", "Algorithm to use for (de)compression. See --list for a complete list of them.");
DEFINE_bool(decompress, false, "Decompress input instead of compressing it.");
DEFINE_string(output, "", "Choose output filename instead the the default generated one of <input>.tdc or stdout.");
DEFINE_bool(stats, false, "Print statistics to stdout.");
DEFINE_bool(force, false, "Overwrite output even if it exists.");
DEFINE_bool(list, false, "List all compression algorithms supported by this tool.");
DEFINE_bool(raw, false, "Do not emit an header into the output file when compressing.");
DEFINE_bool(stdin, false, "Read from stdin instead of trying to open a file.");
DEFINE_bool(stdout, false, "Output to stdout instead of writing to a file");

namespace tudocomp_driver {

using namespace tudocomp;

const std::string COMPRESSED_FILE_ENDING = "tdc";

static void exit(std::string msg) {
    // TODO: Replace with more specific logic-error-exception
    throw std::runtime_error(msg);
}

static void validate_flag_combination() {
    auto err = [](std::string s) {
        exit(s);
    };

    if (!FLAGS_list) {
        if (!FLAGS_decompress && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for compression");
        }
        if (FLAGS_decompress && FLAGS_raw && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for raw decompression");
        }
    }
}

static bool fexists(std::string filename)
{
  std::ifstream ifile(filename);
  return bool(ifile);
}

static bool check_for_file_already_exist(std::string& ofile,
                                         bool allow_overwrite) {
    // Don't accidentially overwrite files
    if (!allow_overwrite && fexists(ofile)) {
        std::cerr << "Outputfile already exists\n";
        return false;
    }
    return true;
}

} // namespace tudocomp_driver

int main(int argc, char** argv)
{
    using namespace tudocomp_driver;

    google::InitGoogleLogging(argv[0]);
    int first_cmd_arg = google::ParseCommandLineFlags(&argc, &argv, true);

    try {
        validate_flag_combination();

        std::map<std::string, const std::string> algorithm_options;

        /*for (auto& os : string_args("--option")) {
            std::vector<std::string> options;
            boost::split(options, os, boost::is_any_of(","));
            for (auto& o : options) {
                std::vector<std::string> key_value;
                boost::split(key_value, o, boost::is_any_of("="));
                CHECK(key_value.size() == 2);
                algorithm_options.emplace(key_value[0], key_value[1]);
            }
        }*/

        /*if (arg_exists("--help")) {
            show_help();
            return 0;
        }*/

        Env algorithm_env(algorithm_options, {});

        // Set up algorithms
        AlgorithmDb root;
        Registry registry {&root};
        register_algos(registry);

        if (FLAGS_list) {
            std::cout << "This build supports the following algorithms:\n";
            std::cout << std::endl;

            for (auto& e: registry.get_sub_algos()) {
                e.print_to(std::cout, 0);
            }

            return 0;
        }

        bool print_stats = FLAGS_stats;
        int alphabet_size = 0;

        bool do_compress = !FLAGS_decompress;

        /////////////////////////////////////////////////////////////////////////
        // Select algorithm

        bool do_raw = FLAGS_raw;

        std::string algorithm_id;
        std::unique_ptr<Compressor> algo;

        if (do_raw || do_compress) {
            algorithm_id = FLAGS_algorithm;
            algo = select_algo_or_exit(registry, algorithm_env, algorithm_id);
        }

        /////////////////////////////////////////////////////////////////////////
        // Select where the input comes from

        std::string file;

        if ((!FLAGS_stdin) && (first_cmd_arg < argc)) {
            file = argv[first_cmd_arg];
            if (!fexists(file)) {
                std::cerr << "input file " << file << " does not exist\n";
                return 1;
            }
        } else if (!FLAGS_stdin) {
            std::cerr << "No input file given\n";
            return 1;
        }

        bool use_stdin = FLAGS_stdin;

        /////////////////////////////////////////////////////////////////////////
        // Select where the output goes to

        std::string ofile;
        bool use_stdout = FLAGS_stdout;
        bool use_explict_output = std::string(FLAGS_output) != "" ;

        if (use_explict_output) {
            // Output to manually specifed file
            ofile = FLAGS_output;
        } else if (!use_stdin && do_compress) {
            // Output to a automatically determined file
            ofile = file + "." + COMPRESSED_FILE_ENDING;
        } else if (!use_stdin && !do_compress && !use_stdout) {
            std::cerr << "Need to specify a output filename\n";
            return 1;
        }

        /////////////////////////////////////////////////////////////////////////
        // open streams

        using clk = std::chrono::high_resolution_clock;

        clk::time_point start_time = clk::now();
        clk::time_point setup_time;
        clk::time_point comp_time;
        clk::time_point end_time;

        {
            std::vector<uint8_t> stream_buffer;
            Input inp;

            if (use_stdin) {
                // Input from stdin
                stream_buffer = read_stream_to_stl_byte_container<
                    std::vector<uint8_t>
                >(std::cin);
                inp = Input::from_memory(stream_buffer);
            } else {
                // Input from specified file
                inp = Input::from_path(file);
            }

            Output out;
            if (use_stdout) {
                // Output to stdout
                out = Output::from_stream(std::cout);
            } else {
                // Output to specified file
                bool force = FLAGS_force;
                if (!check_for_file_already_exist(ofile, force)) {
                    return 1;
                } else {
                    out = Output::from_path(ofile, true);
                }
            }

            /////////////////////////////////////////////////////////////////////////
            // call into actual library

            if (do_compress) {
                if (print_stats) {
                    alphabet_size = 0; // count_alphabet_size(inp_vec);
                }

                if (!do_raw) {
                    CHECK(algorithm_id.find('%') == std::string::npos);

                    auto o_stream = out.as_stream();
                    (*o_stream) << algorithm_id << '%';
                }

                setup_time = clk::now();

                algo->compress(inp, out);

                comp_time = clk::now();
            } else {
                if (!do_raw) {
                    auto i_stream = inp.as_stream();
                    std::string algorithm_header;

                    char c;
                    size_t sanity_size_check = 0;
                    bool err = false;
                    while ((*i_stream).get(c)) {
                        err = false;
                        if (sanity_size_check > 1023) {
                            err = true;
                            break;
                        } else if (c == '%') {
                            break;
                        } else {
                            algorithm_header.push_back(c);
                        }
                        sanity_size_check++;
                        err = true;
                    }
                    if (err) {
                        exit("Input did not have an algorithm header!");
                    }
                    algorithm_id = std::move(algorithm_header);
                    algo = select_algo_or_exit(registry, algorithm_env, algorithm_id);
                }

                setup_time = clk::now();

                algo->decompress(inp, out);

                comp_time = clk::now();
            }
        }

        end_time = clk::now();

        if (print_stats) {
            auto setup_duration = setup_time - start_time;
            auto comp_duration = comp_time - setup_time;
            auto end_duration = end_time - comp_time;
            std::cout << "---------------\n";
            std::cout << "Config: " << algorithm_id << std::endl;
            std::cout << "---------------\n";
            auto inp_size = 0;
            if (use_stdin) {
                std::cout << "input: <stdin>\n";
                std::cout << "input size: ? B\n";
            } else {
                inp_size = read_file_size(file);
                std::cout << "input: "<<file<<"\n";
                std::cout << "input size: "<<inp_size<<" B\n";
            }
            std::cout << "alphabet size: " << alphabet_size << "\n";

            auto out_size = 0;
            if (use_stdout) {
                std::cout << "output: <stdout>\n";
                std::cout << "output size: ? B\n";
            } else {
                out_size = read_file_size(ofile);
                std::cout << "output: "<<ofile<<"\n";
                std::cout << "output size: "<<out_size<<" B\n";
            }
            std::cout << "---------------\n";
            if (inp_size != 0) {
                std::cout << "compress ratio: "<<(double(out_size)/double(inp_size) * 100)<<"%\n";
            }

            auto print_time = [] (std::string s, decltype(setup_duration)& t) {
                std::chrono::seconds sec(1);
                std::chrono::milliseconds milsec(1);
                std::chrono::microseconds micsec(1);
                using dur = std::chrono::duration<float>;
                using mildur = std::chrono::duration<float, std::milli>;
                using micdur = std::chrono::duration<float, std::micro>;

                if (t < milsec) {
                    auto ct = std::chrono::duration_cast<micdur>(t).count();
                    std::cout << s <<" time: "<< ct << " µs\n";
                } else if (t < sec) {
                    auto ct = std::chrono::duration_cast<mildur>(t).count();
                    std::cout << s <<" time: "<< ct << " ms\n";
                } else {
                    auto ct = std::chrono::duration_cast<dur>(t).count();
                    std::cout << s <<" time: "<< ct << " s\n";
                }
            };

            std::cout << "---------------\n";
            std::cout << "Algorithm Known Options:\n";
            for (auto& pair : algorithm_env.get_known_options()) {
                std::cout << "  " << pair << "\n";
            }
            std::cout << "Algorithm Given Options:\n";
            for (auto& pair : algorithm_env.get_options()) {
                std::cout << "  " << pair.first << " = " << pair.second << "\n";
            }

            std::cout << "---------------\n";
            std::cout << "Algorithm Stats:\n";
            for (auto& pair : algorithm_env.get_stats()) {
                std::cout << "  " << pair.first << ": " << pair.second << "\n";
            }

            std::cout << "---------------\n";
            print_time("startup", setup_duration);
            print_time("compression", comp_duration);
            print_time("teardown", end_duration);
            std::cout << "---------------\n";
        }
    } catch (std::exception& e) {
        std::cout << "Error: " << e.what() << '\n';
        return 1;
    }

    return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "leveled_manifest.hh"

namespace sstables {

class leveled_compaction_strategy : public compaction_strategy_impl {
    static constexpr int32_t DEFAULT_MAX_SSTABLE_SIZE_IN_MB = 160;
    const sstring SSTABLE_SIZE_OPTION = "sstable_size_in_mb";

    int32_t _max_sstable_size_in_mb = DEFAULT_MAX_SSTABLE_SIZE_IN_MB;
    stdx::optional<std::vector<stdx::optional<dht::decorated_key>>> _last_compacted_keys;
    std::vector<int> _compaction_counter;
public:
    leveled_compaction_strategy(const std::map<sstring, sstring>& options)
        : compaction_strategy_impl(options)
    {
        using namespace cql3::statements;

        auto tmp_value = compaction_strategy_impl::get_value(options, SSTABLE_SIZE_OPTION);
        _max_sstable_size_in_mb = property_definitions::to_int(SSTABLE_SIZE_OPTION, tmp_value, DEFAULT_MAX_SSTABLE_SIZE_IN_MB);
        if (_max_sstable_size_in_mb >= 1000) {
            leveled_manifest::logger.warn("Max sstable size of {}MB is configured; having a unit of compaction this large is probably a bad idea",
                _max_sstable_size_in_mb);
        } else if (_max_sstable_size_in_mb < 50) {
            leveled_manifest::logger.warn("Max sstable size of {}MB is configured. Testing done for CASSANDRA-5727 indicates that performance" \
                "improves up to 160MB", _max_sstable_size_in_mb);
        }
        _compaction_counter.resize(leveled_manifest::MAX_LEVELS);
    }

    virtual compaction_descriptor get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) override;

    virtual std::vector<resharding_descriptor> get_resharding_jobs(column_family& cf, std::vector<shared_sstable> candidates) override;

    virtual void notify_completion(const std::vector<lw_shared_ptr<sstable>>& removed, const std::vector<lw_shared_ptr<sstable>>& added) override;

    // for each level > 0, get newest sstable and use its last key as last
    // compacted key for the previous level.
    void generate_last_compacted_keys(leveled_manifest& manifest);

    virtual int64_t estimated_pending_compactions(column_family& cf) const override;

    virtual bool parallel_compaction() const override {
        return false;
    }

    virtual compaction_strategy_type type() const {
        return compaction_strategy_type::leveled;
    }
    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const override;
};

compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) {
    // NOTE: leveled_manifest creation may be slightly expensive, so later on,
    // we may want to store it in the strategy itself. However, the sstable
    // lists managed by the manifest may become outdated. For example, one
    // sstable in it may be marked for deletion after compacted.
    // Currently, we create a new manifest whenever it's time for compaction.
    leveled_manifest manifest = leveled_manifest::create(cfs, candidates, _max_sstable_size_in_mb);
    if (!_last_compacted_keys) {
        generate_last_compacted_keys(manifest);
    }
    auto candidate = manifest.get_compaction_candidates(*_last_compacted_keys, _compaction_counter);

    if (!candidate.sstables.empty()) {
        leveled_manifest::logger.debug("leveled: Compacting {} out of {} sstables", candidate.sstables.size(), cfs.get_sstables()->size());
        return std::move(candidate);
    }

    // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
    // unlike stcs, lcs can look for sstable with highest droppable tombstone ratio, so as not to choose
    // a sstable which droppable data shadow data in older sstable, by starting from highest levels which
    // theoretically contain oldest non-overlapping data.
    auto gc_before = gc_clock::now() - cfs.schema()->gc_grace_seconds();
    for (auto level = manifest.get_level_count(); level >= 0; level--) {
        auto& sstables = manifest.get_level(level);
        // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
        auto e = boost::range::remove_if(sstables, [this, &gc_before] (const sstables::shared_sstable& sst) -> bool {
            return !worth_dropping_tombstones(sst, gc_before);
        });
        sstables.erase(e, sstables.end());
        if (sstables.empty()) {
            continue;
        }
        auto& sst = *std::min_element(sstables.begin(), sstables.end(), [&] (auto& i, auto& j) {
            return i->estimate_droppable_tombstone_ratio(gc_before) < j->estimate_droppable_tombstone_ratio(gc_before);
        });
        return sstables::compaction_descriptor({ sst }, sst->get_sstable_level());
    }
}

std::vector<resharding_descriptor> leveled_compaction_strategy::get_resharding_jobs(column_family& cf, std::vector<shared_sstable> candidates) {
    leveled_manifest manifest = leveled_manifest::create(cf, candidates, _max_sstable_size_in_mb);

    std::vector<resharding_descriptor> descriptors;
    shard_id target_shard = 0;
    auto get_shard = [&target_shard] { return target_shard++ % smp::count; };

    // Basically, we'll iterate through all levels, and for each, we'll sort the
    // sstables by first key because there's a need to reshard together adjacent
    // sstables.
    // The shard at which the job will run is chosen in a round-robin fashion.
    for (auto level = 0U; level <= manifest.get_level_count(); level++) {
        uint64_t max_sstable_size = !level ? std::numeric_limits<uint64_t>::max() : (_max_sstable_size_in_mb*1024*1024);
        auto& sstables = manifest.get_level(level);
        boost::sort(sstables, [] (auto& i, auto& j) {
            return i->compare_by_first_key(*j) < 0;
        });

        resharding_descriptor current_descriptor = resharding_descriptor{{}, max_sstable_size, get_shard(), level};

        for (auto it = sstables.begin(); it != sstables.end(); it++) {
            current_descriptor.sstables.push_back(*it);

            auto next = std::next(it);
            if (current_descriptor.sstables.size() == smp::count || next == sstables.end()) {
                descriptors.push_back(std::move(current_descriptor));
                current_descriptor = resharding_descriptor{{}, max_sstable_size, get_shard(), level};
            }
        }
    }
    return descriptors;
}

void leveled_compaction_strategy::notify_completion(const std::vector<lw_shared_ptr<sstable>>& removed, const std::vector<lw_shared_ptr<sstable>>& added) {
    if (removed.empty() || added.empty()) {
        return;
    }
    auto min_level = std::numeric_limits<uint32_t>::max();
    for (auto& sstable : removed) {
        min_level = std::min(min_level, sstable->get_sstable_level());
    }

    const sstables::sstable *last = nullptr;
    for (auto& candidate : added) {
        if (!last || last->compare_by_first_key(*candidate) < 0) {
            last = &*candidate;
        }
    }
    _last_compacted_keys.value().at(min_level) = last->get_last_decorated_key();
}

void leveled_compaction_strategy::generate_last_compacted_keys(leveled_manifest& manifest) {
    std::vector<stdx::optional<dht::decorated_key>> last_compacted_keys(leveled_manifest::MAX_LEVELS);
    for (auto i = 0; i < leveled_manifest::MAX_LEVELS - 1; i++) {
        if (manifest.get_level(i + 1).empty()) {
            continue;
        }

        const sstables::sstable* sstable_with_last_compacted_key = nullptr;
        stdx::optional<db_clock::time_point> max_creation_time;
        for (auto& sst : manifest.get_level(i + 1)) {
            auto wtime = sst->data_file_write_time();
            if (!max_creation_time || wtime >= *max_creation_time) {
                sstable_with_last_compacted_key = &*sst;
                max_creation_time = wtime;
            }
        }
        last_compacted_keys[i] = sstable_with_last_compacted_key->get_last_decorated_key();
    }
    _last_compacted_keys = std::move(last_compacted_keys);
}

int64_t leveled_compaction_strategy::estimated_pending_compactions(column_family& cf) const {
    std::vector<sstables::shared_sstable> sstables;
    sstables.reserve(cf.sstables_count());
    for (auto& entry : *cf.get_sstables()) {
        sstables.push_back(entry);
    }
    leveled_manifest manifest = leveled_manifest::create(cf, sstables, _max_sstable_size_in_mb);
    return manifest.get_estimated_tasks();
}

}
<commit_msg>lcs: prevent leveled_compaction_strategy.hh from being included more than once<commit_after>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "leveled_manifest.hh"

namespace sstables {

class leveled_compaction_strategy : public compaction_strategy_impl {
    static constexpr int32_t DEFAULT_MAX_SSTABLE_SIZE_IN_MB = 160;
    const sstring SSTABLE_SIZE_OPTION = "sstable_size_in_mb";

    int32_t _max_sstable_size_in_mb = DEFAULT_MAX_SSTABLE_SIZE_IN_MB;
    stdx::optional<std::vector<stdx::optional<dht::decorated_key>>> _last_compacted_keys;
    std::vector<int> _compaction_counter;
public:
    leveled_compaction_strategy(const std::map<sstring, sstring>& options)
        : compaction_strategy_impl(options)
    {
        using namespace cql3::statements;

        auto tmp_value = compaction_strategy_impl::get_value(options, SSTABLE_SIZE_OPTION);
        _max_sstable_size_in_mb = property_definitions::to_int(SSTABLE_SIZE_OPTION, tmp_value, DEFAULT_MAX_SSTABLE_SIZE_IN_MB);
        if (_max_sstable_size_in_mb >= 1000) {
            leveled_manifest::logger.warn("Max sstable size of {}MB is configured; having a unit of compaction this large is probably a bad idea",
                _max_sstable_size_in_mb);
        } else if (_max_sstable_size_in_mb < 50) {
            leveled_manifest::logger.warn("Max sstable size of {}MB is configured. Testing done for CASSANDRA-5727 indicates that performance" \
                "improves up to 160MB", _max_sstable_size_in_mb);
        }
        _compaction_counter.resize(leveled_manifest::MAX_LEVELS);
    }

    virtual compaction_descriptor get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) override;

    virtual std::vector<resharding_descriptor> get_resharding_jobs(column_family& cf, std::vector<shared_sstable> candidates) override;

    virtual void notify_completion(const std::vector<lw_shared_ptr<sstable>>& removed, const std::vector<lw_shared_ptr<sstable>>& added) override;

    // for each level > 0, get newest sstable and use its last key as last
    // compacted key for the previous level.
    void generate_last_compacted_keys(leveled_manifest& manifest);

    virtual int64_t estimated_pending_compactions(column_family& cf) const override;

    virtual bool parallel_compaction() const override {
        return false;
    }

    virtual compaction_strategy_type type() const {
        return compaction_strategy_type::leveled;
    }
    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const override;
};

compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(column_family& cfs, std::vector<sstables::shared_sstable> candidates) {
    // NOTE: leveled_manifest creation may be slightly expensive, so later on,
    // we may want to store it in the strategy itself. However, the sstable
    // lists managed by the manifest may become outdated. For example, one
    // sstable in it may be marked for deletion after compacted.
    // Currently, we create a new manifest whenever it's time for compaction.
    leveled_manifest manifest = leveled_manifest::create(cfs, candidates, _max_sstable_size_in_mb);
    if (!_last_compacted_keys) {
        generate_last_compacted_keys(manifest);
    }
    auto candidate = manifest.get_compaction_candidates(*_last_compacted_keys, _compaction_counter);

    if (!candidate.sstables.empty()) {
        leveled_manifest::logger.debug("leveled: Compacting {} out of {} sstables", candidate.sstables.size(), cfs.get_sstables()->size());
        return std::move(candidate);
    }

    // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
    // unlike stcs, lcs can look for sstable with highest droppable tombstone ratio, so as not to choose
    // a sstable which droppable data shadow data in older sstable, by starting from highest levels which
    // theoretically contain oldest non-overlapping data.
    auto gc_before = gc_clock::now() - cfs.schema()->gc_grace_seconds();
    for (auto level = manifest.get_level_count(); level >= 0; level--) {
        auto& sstables = manifest.get_level(level);
        // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
        auto e = boost::range::remove_if(sstables, [this, &gc_before] (const sstables::shared_sstable& sst) -> bool {
            return !worth_dropping_tombstones(sst, gc_before);
        });
        sstables.erase(e, sstables.end());
        if (sstables.empty()) {
            continue;
        }
        auto& sst = *std::min_element(sstables.begin(), sstables.end(), [&] (auto& i, auto& j) {
            return i->estimate_droppable_tombstone_ratio(gc_before) < j->estimate_droppable_tombstone_ratio(gc_before);
        });
        return sstables::compaction_descriptor({ sst }, sst->get_sstable_level());
    }
}

std::vector<resharding_descriptor> leveled_compaction_strategy::get_resharding_jobs(column_family& cf, std::vector<shared_sstable> candidates) {
    leveled_manifest manifest = leveled_manifest::create(cf, candidates, _max_sstable_size_in_mb);

    std::vector<resharding_descriptor> descriptors;
    shard_id target_shard = 0;
    auto get_shard = [&target_shard] { return target_shard++ % smp::count; };

    // Basically, we'll iterate through all levels, and for each, we'll sort the
    // sstables by first key because there's a need to reshard together adjacent
    // sstables.
    // The shard at which the job will run is chosen in a round-robin fashion.
    for (auto level = 0U; level <= manifest.get_level_count(); level++) {
        uint64_t max_sstable_size = !level ? std::numeric_limits<uint64_t>::max() : (_max_sstable_size_in_mb*1024*1024);
        auto& sstables = manifest.get_level(level);
        boost::sort(sstables, [] (auto& i, auto& j) {
            return i->compare_by_first_key(*j) < 0;
        });

        resharding_descriptor current_descriptor = resharding_descriptor{{}, max_sstable_size, get_shard(), level};

        for (auto it = sstables.begin(); it != sstables.end(); it++) {
            current_descriptor.sstables.push_back(*it);

            auto next = std::next(it);
            if (current_descriptor.sstables.size() == smp::count || next == sstables.end()) {
                descriptors.push_back(std::move(current_descriptor));
                current_descriptor = resharding_descriptor{{}, max_sstable_size, get_shard(), level};
            }
        }
    }
    return descriptors;
}

void leveled_compaction_strategy::notify_completion(const std::vector<lw_shared_ptr<sstable>>& removed, const std::vector<lw_shared_ptr<sstable>>& added) {
    if (removed.empty() || added.empty()) {
        return;
    }
    auto min_level = std::numeric_limits<uint32_t>::max();
    for (auto& sstable : removed) {
        min_level = std::min(min_level, sstable->get_sstable_level());
    }

    const sstables::sstable *last = nullptr;
    for (auto& candidate : added) {
        if (!last || last->compare_by_first_key(*candidate) < 0) {
            last = &*candidate;
        }
    }
    _last_compacted_keys.value().at(min_level) = last->get_last_decorated_key();
}

void leveled_compaction_strategy::generate_last_compacted_keys(leveled_manifest& manifest) {
    std::vector<stdx::optional<dht::decorated_key>> last_compacted_keys(leveled_manifest::MAX_LEVELS);
    for (auto i = 0; i < leveled_manifest::MAX_LEVELS - 1; i++) {
        if (manifest.get_level(i + 1).empty()) {
            continue;
        }

        const sstables::sstable* sstable_with_last_compacted_key = nullptr;
        stdx::optional<db_clock::time_point> max_creation_time;
        for (auto& sst : manifest.get_level(i + 1)) {
            auto wtime = sst->data_file_write_time();
            if (!max_creation_time || wtime >= *max_creation_time) {
                sstable_with_last_compacted_key = &*sst;
                max_creation_time = wtime;
            }
        }
        last_compacted_keys[i] = sstable_with_last_compacted_key->get_last_decorated_key();
    }
    _last_compacted_keys = std::move(last_compacted_keys);
}

int64_t leveled_compaction_strategy::estimated_pending_compactions(column_family& cf) const {
    std::vector<sstables::shared_sstable> sstables;
    sstables.reserve(cf.sstables_count());
    for (auto& entry : *cf.get_sstables()) {
        sstables.push_back(entry);
    }
    leveled_manifest manifest = leveled_manifest::create(cf, sstables, _max_sstable_size_in_mb);
    return manifest.get_estimated_tasks();
}

}
<|endoftext|>