<commit_before>/*
 * Copyright 2001-2002,2004 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * $Id$
 */

#include <xercesc/util/XercesDefs.hpp>
#include <xercesc/dom/DOMException.hpp>
#include <xercesc/dom/DOMNode.hpp>

#include "DOMDocumentImpl.hpp"
#include "DOMNodeListImpl.hpp"
#include "DOMRangeImpl.hpp"
#include "DOMNodeIteratorImpl.hpp"
#include "DOMParentNode.hpp"
#include "DOMCasts.hpp"

XERCES_CPP_NAMESPACE_BEGIN

DOMParentNode::DOMParentNode(DOMDocument *ownerDoc)
    : fOwnerDocument(ownerDoc), fFirstChild(0), fChildNodeList(castToNode(this))
{
}

// This only makes a shallow copy, cloneChildren must also be called for a
// deep clone
DOMParentNode::DOMParentNode(const DOMParentNode &other)  :
    fChildNodeList(castToNode(this))
{
    this->fOwnerDocument = other.fOwnerDocument;

    // Need to break the association w/ original kids
    this->fFirstChild = 0;
}

void DOMParentNode::changed()
{
    DOMDocumentImpl *doc = (DOMDocumentImpl *)this->getOwnerDocument();
    doc->changed();
}

int DOMParentNode::changes() const
{
    DOMDocumentImpl *doc = (DOMDocumentImpl *)this->getOwnerDocument();
    return doc->changes();
}

DOMNode * DOMParentNode::appendChild(DOMNode *newChild)
{
    return insertBefore(newChild, 0);
}

void DOMParentNode::cloneChildren(const DOMNode *other) {
  //    for (DOMNode *mykid = other.getFirstChild();
    for (DOMNode *mykid = other->getFirstChild();
         mykid != 0;
         mykid = mykid->getNextSibling())
    {
        appendChild(mykid->cloneNode(true));
    }
}

DOMDocument * DOMParentNode::getOwnerDocument() const {
    return fOwnerDocument;
}

// unlike getOwnerDocument this is not overriden by DocumentImpl to return 0
DOMDocument * DOMParentNode::getDocument() const {
    return fOwnerDocument;
}

void DOMParentNode::setOwnerDocument(DOMDocument* doc) {
    fOwnerDocument = doc;
}

DOMNodeList *DOMParentNode::getChildNodes() const {
    const DOMNodeList *ret = &fChildNodeList;
    return (DOMNodeList *)ret;   // cast off const.
}

DOMNode * DOMParentNode::getFirstChild() const {
    return fFirstChild;
}

DOMNode * DOMParentNode::getLastChild() const
{
    return lastChild();
}

DOMNode * DOMParentNode::lastChild() const
{
    // last child is stored as the previous sibling of first child
    if (fFirstChild == 0) {
        return 0;
    }

    DOMChildNode *firstChild = castToChildImpl(fFirstChild);
    DOMNode *ret = firstChild->previousSibling;
    return ret;
}

//
//  revisit.  Is this function used anywhere?  I don't see it.
//
void DOMParentNode::lastChild(DOMNode *node) {
    // store lastChild as previous sibling of first child
    if (fFirstChild != 0) {
        DOMChildNode *firstChild = castToChildImpl(fFirstChild);
        firstChild->previousSibling = node;
    }
}

bool DOMParentNode::hasChildNodes() const
{
    return fFirstChild!=0;
}

DOMNode *DOMParentNode::insertBefore(DOMNode *newChild, DOMNode *refChild) {
    DOMNodeImpl *thisNodeImpl = castToNodeImpl(this);
    if (thisNodeImpl->isReadOnly())
        throw DOMException(DOMException::NO_MODIFICATION_ALLOWED_ERR, 0, GetDOMParentNodeMemoryManager);

    if (newChild->getOwnerDocument() != fOwnerDocument)
        throw DOMException(DOMException::WRONG_DOCUMENT_ERR, 0, GetDOMParentNodeMemoryManager);

    // Prevent cycles in the tree
    //only need to do this if the node has children
    if(newChild->hasChildNodes()) {
        bool treeSafe=true;
        for(DOMNode *a=castToNode(this)->getParentNode();
            treeSafe && a!=0;
            a=a->getParentNode())
            treeSafe=(newChild!=a);
        if(!treeSafe)
            throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);
    }

    // refChild must in fact be a child of this node (or 0)
    if (refChild!=0 && refChild->getParentNode() != castToNode(this))
        throw DOMException(DOMException::NOT_FOUND_ERR,0, GetDOMParentNodeMemoryManager);

    // if the new node has to be placed before itself, we don't have to do anything
    // (even worse, we would crash if we continue, as we assume they are two distinct nodes)
    if (refChild!=0 && newChild->isSameNode(refChild))
        return newChild;

    if (newChild->getNodeType() == DOMNode::DOCUMENT_FRAGMENT_NODE)
    {
        // SLOW BUT SAFE: We could insert the whole subtree without
        // juggling so many next/previous pointers. (Wipe out the
        // parent's child-list, patch the parent pointers, set the
        // ends of the list.) But we know some subclasses have special-
        // case behavior they add to insertBefore(), so we don't risk it.
        // This approch also takes fewer bytecodes.

        // NOTE: If one of the children is not a legal child of this
        // node, throw HIERARCHY_REQUEST_ERR before _any_ of the children
        // have been transferred. (Alternative behaviors would be to
        // reparent up to the first failure point or reparent all those
        // which are acceptable to the target node, neither of which is
        // as robust. PR-DOM-0818 isn't entirely clear on which it
        // recommends?????

        // No need to check kids for right-document; if they weren't,
        // they wouldn't be kids of that DocFrag.
        for(DOMNode *kid=newChild->getFirstChild(); // Prescan
              kid!=0;
              kid=kid->getNextSibling())
        {
            if (!DOMDocumentImpl::isKidOK(castToNode(this), kid))
              throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);
        }
        while(newChild->hasChildNodes())     // Move
            insertBefore(newChild->getFirstChild(),refChild);
    }

    else if (!DOMDocumentImpl::isKidOK(castToNode(this), newChild))
        throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);

    else
    {
        DOMNode *oldparent=newChild->getParentNode();
        if(oldparent!=0)
            oldparent->removeChild(newChild);

        // Attach up
        castToNodeImpl(newChild)->fOwnerNode = castToNode(this);
        castToNodeImpl(newChild)->isOwned(true);

        // Attach before and after
        // Note: fFirstChild.previousSibling == lastChild!!
        if (fFirstChild == 0) {
            // this our first and only child
            fFirstChild = newChild;
            castToNodeImpl(newChild)->isFirstChild(true);
            // castToChildImpl(newChild)->previousSibling = newChild;
            DOMChildNode *newChild_ci = castToChildImpl(newChild);
            newChild_ci->previousSibling = newChild;
        } else {
            if (refChild == 0) {
                // this is an append
                DOMNode *lastChild = castToChildImpl(fFirstChild)->previousSibling;
                castToChildImpl(lastChild)->nextSibling = newChild;
                castToChildImpl(newChild)->previousSibling = lastChild;
                castToChildImpl(fFirstChild)->previousSibling = newChild;
            } else {
                // this is an insert
                if (refChild == fFirstChild) {
                    // at the head of the list
                    castToNodeImpl(fFirstChild)->isFirstChild(false);
                    castToChildImpl(newChild)->nextSibling = fFirstChild;
                    castToChildImpl(newChild)->previousSibling = castToChildImpl(fFirstChild)->previousSibling;
                    castToChildImpl(fFirstChild)->previousSibling = newChild;
                    fFirstChild = newChild;
                    castToNodeImpl(newChild)->isFirstChild(true);
                } else {
                    // somewhere in the middle
                    DOMNode *prev = castToChildImpl(refChild)->previousSibling;
                    castToChildImpl(newChild)->nextSibling = refChild;
                    castToChildImpl(prev)->nextSibling = newChild;
                    castToChildImpl(refChild)->previousSibling = newChild;
                    castToChildImpl(newChild)->previousSibling = prev;
                }
            }
        }
    }

    changed();

    if (this->getOwnerDocument() != 0) {
        Ranges* ranges = ((DOMDocumentImpl *)this->getOwnerDocument())->getRanges();
        if ( ranges != 0) {
            XMLSize_t sz = ranges->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    ranges->elementAt(i)->updateRangeForInsertedNode(newChild);
                }
            }
        }
    }

    return newChild;
}

DOMNode *DOMParentNode::removeChild(DOMNode *oldChild)
{
    if (castToNodeImpl(this)->isReadOnly())
        throw DOMException(
        DOMException::NO_MODIFICATION_ALLOWED_ERR, 0, GetDOMParentNodeMemoryManager);

    if (oldChild == 0 || oldChild->getParentNode() != castToNode(this))
        throw DOMException(DOMException::NOT_FOUND_ERR, 0, GetDOMParentNodeMemoryManager);

    if (this->getOwnerDocument() !=  0  ) {
        //notify iterators
        NodeIterators* nodeIterators = ((DOMDocumentImpl *)this->getOwnerDocument())->getNodeIterators();
        if (nodeIterators != 0) {
            XMLSize_t sz = nodeIterators->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    if (nodeIterators->elementAt(i) != 0)
                        nodeIterators->elementAt(i)->removeNode(oldChild);
                }
            }
        }

        //fix other ranges for change before deleting the node
        Ranges* ranges = ((DOMDocumentImpl *)this->getOwnerDocument())->getRanges();
        if (ranges != 0) {
            XMLSize_t sz = ranges->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    if (ranges->elementAt(i) != 0)
                        ranges->elementAt(i)->updateRangeForDeletedNode(oldChild);
                }
            }
        }
    }

    // Patch linked list around oldChild
    // Note: lastChild == fFirstChild->previousSibling
    if (oldChild == fFirstChild) {
        // removing first child
        castToNodeImpl(oldChild)->isFirstChild(false);
        fFirstChild = castToChildImpl(oldChild)->nextSibling;
        if (fFirstChild != 0) {
            castToNodeImpl(fFirstChild)->isFirstChild(true);
            castToChildImpl(fFirstChild)->previousSibling = castToChildImpl(oldChild)->previousSibling;
        }
    } else {
        DOMNode *prev = castToChildImpl(oldChild)->previousSibling;
        DOMNode *next = castToChildImpl(oldChild)->nextSibling;
        castToChildImpl(prev)->nextSibling = next;
        if (next == 0) {
            // removing last child
            castToChildImpl(fFirstChild)->previousSibling = prev;
        } else {
            // removing some other child in the middle
            castToChildImpl(next)->previousSibling = prev;
        }
    }

    // Remove oldChild's references to tree
    castToNodeImpl(oldChild)->fOwnerNode = fOwnerDocument;
    castToNodeImpl(oldChild)->isOwned(false);
    castToChildImpl(oldChild)->nextSibling = 0;
    castToChildImpl(oldChild)->previousSibling = 0;

    changed();

    return oldChild;
}

DOMNode *DOMParentNode::replaceChild(DOMNode *newChild, DOMNode *oldChild)
{
    insertBefore(newChild, oldChild);
    // changed() already done.
    return removeChild(oldChild);
}

//Introduced in DOM Level 2

void DOMParentNode::normalize()
{
    DOMNode *kid, *next;
    for (kid = fFirstChild; kid != 0; kid = next)
    {
        next = castToChildImpl(kid)->nextSibling;

        // If kid and next are both Text nodes (but _not_ CDATASection,
        // which is a subclass of Text), they can be merged.
        if (next != 0 &&
            kid->getNodeType() == DOMNode::TEXT_NODE   &&
            next->getNodeType() == DOMNode::TEXT_NODE )
        {
            ((DOMTextImpl *) kid)->appendData(((DOMTextImpl *) next)->getData());
            // revisit:
            //   should I release the removed node?
            //   not released in case user still referencing it externally
            removeChild(next);
            next = kid; // Don't advance; there might be another.
        }

        // Otherwise it might be an Element, which is handled recursively
        else
            if (kid->getNodeType() == DOMNode::ELEMENT_NODE)
                kid->normalize();
    }

    // changed() will have occurred when the removeChild() was done,
    // so does not have to be reissued.
}

//Introduced in DOM Level 3

bool DOMParentNode::isEqualNode(const DOMNode* arg) const
{
    if (arg && castToNodeImpl(this)->isSameNode(arg))
        return true;

    if (arg && castToNodeImpl(this)->isEqualNode(arg))
    {
        DOMNode *kid, *argKid;
        for (kid = fFirstChild, argKid = arg->getFirstChild();
             kid != 0 && argKid != 0;
             kid = kid->getNextSibling(), argKid = argKid->getNextSibling())
        {
            if (!kid->isEqualNode(argKid))
                return false;
        }
        return (kid || argKid) ? false : true;
    }
    return false;
}

//Non-standard extension
void DOMParentNode::release()
{
    DOMNode *kid, *next;
    for (kid = fFirstChild; kid != 0; kid = next)
    {
        next = castToChildImpl(kid)->nextSibling;

        // set is Owned false before releasing its child
        castToNodeImpl(kid)->isToBeReleased(true);
        kid->release();
    }
}

XERCES_CPP_NAMESPACE_END

<commit_msg>appendChild now throws an exception when a NULL pointer is given (jira# 1401)<commit_after>/*
 * Copyright 2001-2002,2004 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * $Id$
 */

#include <xercesc/util/XercesDefs.hpp>
#include <xercesc/dom/DOMException.hpp>
#include <xercesc/dom/DOMNode.hpp>

#include "DOMDocumentImpl.hpp"
#include "DOMNodeListImpl.hpp"
#include "DOMRangeImpl.hpp"
#include "DOMNodeIteratorImpl.hpp"
#include "DOMParentNode.hpp"
#include "DOMCasts.hpp"

XERCES_CPP_NAMESPACE_BEGIN

DOMParentNode::DOMParentNode(DOMDocument *ownerDoc)
    : fOwnerDocument(ownerDoc), fFirstChild(0), fChildNodeList(castToNode(this))
{
}

// This only makes a shallow copy, cloneChildren must also be called for a
// deep clone
DOMParentNode::DOMParentNode(const DOMParentNode &other)  :
    fChildNodeList(castToNode(this))
{
    this->fOwnerDocument = other.fOwnerDocument;

    // Need to break the association w/ original kids
    this->fFirstChild = 0;
}

void DOMParentNode::changed()
{
    DOMDocumentImpl *doc = (DOMDocumentImpl *)this->getOwnerDocument();
    doc->changed();
}

int DOMParentNode::changes() const
{
    DOMDocumentImpl *doc = (DOMDocumentImpl *)this->getOwnerDocument();
    return doc->changes();
}

DOMNode * DOMParentNode::appendChild(DOMNode *newChild)
{
    return insertBefore(newChild, 0);
}

void DOMParentNode::cloneChildren(const DOMNode *other) {
  //    for (DOMNode *mykid = other.getFirstChild();
    for (DOMNode *mykid = other->getFirstChild();
         mykid != 0;
         mykid = mykid->getNextSibling())
    {
        appendChild(mykid->cloneNode(true));
    }
}

DOMDocument * DOMParentNode::getOwnerDocument() const {
    return fOwnerDocument;
}

// unlike getOwnerDocument this is not overriden by DocumentImpl to return 0
DOMDocument * DOMParentNode::getDocument() const {
    return fOwnerDocument;
}

void DOMParentNode::setOwnerDocument(DOMDocument* doc) {
    fOwnerDocument = doc;
}

DOMNodeList *DOMParentNode::getChildNodes() const {
    const DOMNodeList *ret = &fChildNodeList;
    return (DOMNodeList *)ret;   // cast off const.
}

DOMNode * DOMParentNode::getFirstChild() const {
    return fFirstChild;
}

DOMNode * DOMParentNode::getLastChild() const
{
    return lastChild();
}

DOMNode * DOMParentNode::lastChild() const
{
    // last child is stored as the previous sibling of first child
    if (fFirstChild == 0) {
        return 0;
    }

    DOMChildNode *firstChild = castToChildImpl(fFirstChild);
    DOMNode *ret = firstChild->previousSibling;
    return ret;
}

//
//  revisit.  Is this function used anywhere?  I don't see it.
//
void DOMParentNode::lastChild(DOMNode *node) {
    // store lastChild as previous sibling of first child
    if (fFirstChild != 0) {
        DOMChildNode *firstChild = castToChildImpl(fFirstChild);
        firstChild->previousSibling = node;
    }
}

bool DOMParentNode::hasChildNodes() const
{
    return fFirstChild!=0;
}

DOMNode *DOMParentNode::insertBefore(DOMNode *newChild, DOMNode *refChild) {
    //not really in the specs, but better than nothing
    if(newChild==NULL)
        throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);

    DOMNodeImpl *thisNodeImpl = castToNodeImpl(this);
    if (thisNodeImpl->isReadOnly())
        throw DOMException(DOMException::NO_MODIFICATION_ALLOWED_ERR, 0, GetDOMParentNodeMemoryManager);

    if (newChild->getOwnerDocument() != fOwnerDocument)
        throw DOMException(DOMException::WRONG_DOCUMENT_ERR, 0, GetDOMParentNodeMemoryManager);

    // Prevent cycles in the tree
    //only need to do this if the node has children
    if(newChild->hasChildNodes()) {
        bool treeSafe=true;
        for(DOMNode *a=castToNode(this)->getParentNode();
            treeSafe && a!=0;
            a=a->getParentNode())
            treeSafe=(newChild!=a);
        if(!treeSafe)
            throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);
    }

    // refChild must in fact be a child of this node (or 0)
    if (refChild!=0 && refChild->getParentNode() != castToNode(this))
        throw DOMException(DOMException::NOT_FOUND_ERR,0, GetDOMParentNodeMemoryManager);

    // if the new node has to be placed before itself, we don't have to do anything
    // (even worse, we would crash if we continue, as we assume they are two distinct nodes)
    if (refChild!=0 && newChild->isSameNode(refChild))
        return newChild;

    if (newChild->getNodeType() == DOMNode::DOCUMENT_FRAGMENT_NODE)
    {
        // SLOW BUT SAFE: We could insert the whole subtree without
        // juggling so many next/previous pointers. (Wipe out the
        // parent's child-list, patch the parent pointers, set the
        // ends of the list.) But we know some subclasses have special-
        // case behavior they add to insertBefore(), so we don't risk it.
        // This approch also takes fewer bytecodes.

        // NOTE: If one of the children is not a legal child of this
        // node, throw HIERARCHY_REQUEST_ERR before _any_ of the children
        // have been transferred. (Alternative behaviors would be to
        // reparent up to the first failure point or reparent all those
        // which are acceptable to the target node, neither of which is
        // as robust. PR-DOM-0818 isn't entirely clear on which it
        // recommends?????

        // No need to check kids for right-document; if they weren't,
        // they wouldn't be kids of that DocFrag.
        for(DOMNode *kid=newChild->getFirstChild(); // Prescan
              kid!=0;
              kid=kid->getNextSibling())
        {
            if (!DOMDocumentImpl::isKidOK(castToNode(this), kid))
              throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);
        }
        while(newChild->hasChildNodes())     // Move
            insertBefore(newChild->getFirstChild(),refChild);
    }

    else if (!DOMDocumentImpl::isKidOK(castToNode(this), newChild))
        throw DOMException(DOMException::HIERARCHY_REQUEST_ERR,0, GetDOMParentNodeMemoryManager);

    else
    {
        DOMNode *oldparent=newChild->getParentNode();
        if(oldparent!=0)
            oldparent->removeChild(newChild);

        // Attach up
        castToNodeImpl(newChild)->fOwnerNode = castToNode(this);
        castToNodeImpl(newChild)->isOwned(true);

        // Attach before and after
        // Note: fFirstChild.previousSibling == lastChild!!
        if (fFirstChild == 0) {
            // this our first and only child
            fFirstChild = newChild;
            castToNodeImpl(newChild)->isFirstChild(true);
            // castToChildImpl(newChild)->previousSibling = newChild;
            DOMChildNode *newChild_ci = castToChildImpl(newChild);
            newChild_ci->previousSibling = newChild;
        } else {
            if (refChild == 0) {
                // this is an append
                DOMNode *lastChild = castToChildImpl(fFirstChild)->previousSibling;
                castToChildImpl(lastChild)->nextSibling = newChild;
                castToChildImpl(newChild)->previousSibling = lastChild;
                castToChildImpl(fFirstChild)->previousSibling = newChild;
            } else {
                // this is an insert
                if (refChild == fFirstChild) {
                    // at the head of the list
                    castToNodeImpl(fFirstChild)->isFirstChild(false);
                    castToChildImpl(newChild)->nextSibling = fFirstChild;
                    castToChildImpl(newChild)->previousSibling = castToChildImpl(fFirstChild)->previousSibling;
                    castToChildImpl(fFirstChild)->previousSibling = newChild;
                    fFirstChild = newChild;
                    castToNodeImpl(newChild)->isFirstChild(true);
                } else {
                    // somewhere in the middle
                    DOMNode *prev = castToChildImpl(refChild)->previousSibling;
                    castToChildImpl(newChild)->nextSibling = refChild;
                    castToChildImpl(prev)->nextSibling = newChild;
                    castToChildImpl(refChild)->previousSibling = newChild;
                    castToChildImpl(newChild)->previousSibling = prev;
                }
            }
        }
    }

    changed();

    if (this->getOwnerDocument() != 0) {
        Ranges* ranges = ((DOMDocumentImpl *)this->getOwnerDocument())->getRanges();
        if ( ranges != 0) {
            XMLSize_t sz = ranges->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    ranges->elementAt(i)->updateRangeForInsertedNode(newChild);
                }
            }
        }
    }

    return newChild;
}

DOMNode *DOMParentNode::removeChild(DOMNode *oldChild)
{
    if (castToNodeImpl(this)->isReadOnly())
        throw DOMException(
        DOMException::NO_MODIFICATION_ALLOWED_ERR, 0, GetDOMParentNodeMemoryManager);

    if (oldChild == 0 || oldChild->getParentNode() != castToNode(this))
        throw DOMException(DOMException::NOT_FOUND_ERR, 0, GetDOMParentNodeMemoryManager);

    if (this->getOwnerDocument() !=  0  ) {
        //notify iterators
        NodeIterators* nodeIterators = ((DOMDocumentImpl *)this->getOwnerDocument())->getNodeIterators();
        if (nodeIterators != 0) {
            XMLSize_t sz = nodeIterators->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    if (nodeIterators->elementAt(i) != 0)
                        nodeIterators->elementAt(i)->removeNode(oldChild);
                }
            }
        }

        //fix other ranges for change before deleting the node
        Ranges* ranges = ((DOMDocumentImpl *)this->getOwnerDocument())->getRanges();
        if (ranges != 0) {
            XMLSize_t sz = ranges->size();
            if (sz != 0) {
                for (XMLSize_t i =0; i<sz; i++) {
                    if (ranges->elementAt(i) != 0)
                        ranges->elementAt(i)->updateRangeForDeletedNode(oldChild);
                }
            }
        }
    }

    // Patch linked list around oldChild
    // Note: lastChild == fFirstChild->previousSibling
    if (oldChild == fFirstChild) {
        // removing first child
        castToNodeImpl(oldChild)->isFirstChild(false);
        fFirstChild = castToChildImpl(oldChild)->nextSibling;
        if (fFirstChild != 0) {
            castToNodeImpl(fFirstChild)->isFirstChild(true);
            castToChildImpl(fFirstChild)->previousSibling = castToChildImpl(oldChild)->previousSibling;
        }
    } else {
        DOMNode *prev = castToChildImpl(oldChild)->previousSibling;
        DOMNode *next = castToChildImpl(oldChild)->nextSibling;
        castToChildImpl(prev)->nextSibling = next;
        if (next == 0) {
            // removing last child
            castToChildImpl(fFirstChild)->previousSibling = prev;
        } else {
            // removing some other child in the middle
            castToChildImpl(next)->previousSibling = prev;
        }
    }

    // Remove oldChild's references to tree
    castToNodeImpl(oldChild)->fOwnerNode = fOwnerDocument;
    castToNodeImpl(oldChild)->isOwned(false);
    castToChildImpl(oldChild)->nextSibling = 0;
    castToChildImpl(oldChild)->previousSibling = 0;

    changed();

    return oldChild;
}

DOMNode *DOMParentNode::replaceChild(DOMNode *newChild, DOMNode *oldChild)
{
    insertBefore(newChild, oldChild);
    // changed() already done.
    return removeChild(oldChild);
}

//Introduced in DOM Level 2

void DOMParentNode::normalize()
{
    DOMNode *kid, *next;
    for (kid = fFirstChild; kid != 0; kid = next)
    {
        next = castToChildImpl(kid)->nextSibling;

        // If kid and next are both Text nodes (but _not_ CDATASection,
        // which is a subclass of Text), they can be merged.
        if (next != 0 &&
            kid->getNodeType() == DOMNode::TEXT_NODE   &&
            next->getNodeType() == DOMNode::TEXT_NODE )
        {
            ((DOMTextImpl *) kid)->appendData(((DOMTextImpl *) next)->getData());
            // revisit:
            //   should I release the removed node?
            //   not released in case user still referencing it externally
            removeChild(next);
            next = kid; // Don't advance; there might be another.
        }

        // Otherwise it might be an Element, which is handled recursively
        else
            if (kid->getNodeType() == DOMNode::ELEMENT_NODE)
                kid->normalize();
    }

    // changed() will have occurred when the removeChild() was done,
    // so does not have to be reissued.
}

//Introduced in DOM Level 3

bool DOMParentNode::isEqualNode(const DOMNode* arg) const
{
    if (arg && castToNodeImpl(this)->isSameNode(arg))
        return true;

    if (arg && castToNodeImpl(this)->isEqualNode(arg))
    {
        DOMNode *kid, *argKid;
        for (kid = fFirstChild, argKid = arg->getFirstChild();
             kid != 0 && argKid != 0;
             kid = kid->getNextSibling(), argKid = argKid->getNextSibling())
        {
            if (!kid->isEqualNode(argKid))
                return false;
        }
        return (kid || argKid) ? false : true;
    }
    return false;
}

//Non-standard extension
void DOMParentNode::release()
{
    DOMNode *kid, *next;
    for (kid = fFirstChild; kid != 0; kid = next)
    {
        next = castToChildImpl(kid)->nextSibling;

        // set is Owned false before releasing its child
        castToNodeImpl(kid)->isToBeReleased(true);
        kid->release();
    }
}

XERCES_CPP_NAMESPACE_END

<|endoftext|>


<commit_before>#include "clstm.h"
#include <assert.h>
#include <math.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <memory>
#include <regex>
#include <set>
#include <sstream>
#include <vector>
#include "clstmhl.h"
#include "extras.h"
#include "pstring.h"
#include "utils.h"

using namespace Eigen;
using namespace ocropus;
using std::vector;
using std::map;
using std::make_pair;
using std::shared_ptr;
using std::unique_ptr;
using std::cout;
using std::ifstream;
using std::set;
using std::to_string;
using std_string = std::string;
using std_wstring = std::wstring;
using std::regex;
using std::regex_replace;
#define string std_string
#define wstring std_wstring

#ifndef NODISPLAY
void show(PyServer &py, Sequence &s, int subplot = 0, int batch = 0) {
  Tensor<float, 2> temp;
  temp.resize(s.size(), s.rows());
  for (int i = 0; i < s.size(); i++)
    for (int j = 0; j < s.rows(); j++) temp(i, j) = s[i].v(j, batch);
  if (subplot > 0) py.evalf("subplot(%d)", subplot);
  py.imshowT(temp, "cmap=cm.hot");
}
#endif

wstring separate_chars(const wstring &s, const wstring &charsep) {
  if (charsep == L"") return s;
  wstring result;
  for (int i = 0; i < s.size(); i++) {
    if (i > 0) result.push_back(charsep[0]);
    result.push_back(s[i]);
  }
  return result;
}

struct Dataset {
  vector<string> fnames;
  wstring charsep = utf8_to_utf32(getsenv("charsep", ""));
  int size() { return fnames.size(); }
  Dataset() {}
  Dataset(string file_list) { readFileList(file_list); }
  void readFileList(string file_list) { read_lines(fnames, file_list); }
  void getCodec(Codec &codec) {
    vector<string> gtnames;
    for (auto s : fnames) gtnames.push_back(basename(s) + ".gt.txt");
    codec.build(gtnames, charsep);
  }
  void readSample(Tensor2 &raw, wstring &gt, int index) {
    string fname = fnames[index];
    string base = basename(fname);
    gt = separate_chars(read_text32(base + ".gt.txt"), charsep);
    read_png(raw, fname.c_str());
    raw() = -raw() + Float(1);
  }
};

pair<double, double> test_set_error(CLSTMOCR &clstm, Dataset &testset) {
  double count = 0.0;
  double errors = 0.0;
  for (int test = 0; test < testset.size(); test++) {
    Tensor2 raw;
    wstring gt;
    testset.readSample(raw, gt, test);
    wstring pred = clstm.predict(raw());
    count += gt.size();
    errors += levenshtein(pred, gt);
  }
  return make_pair(count, errors);
}

int main1(int argc, char **argv) {
  int ntrain = getienv("ntrain", 10000000);
  string save_name = getsenv("save_name", "_ocr");
  int report_time = getienv("report_time", 0);

  if (argc < 2 || argc > 3) THROW("... training [testing]");
  Dataset trainingset(argv[1]);
  assert(trainingset.size() > 0);
  Dataset testset;
  if (argc > 2) testset.readFileList(argv[2]);
  print("got", trainingset.size(), "files,", testset.size(), "tests");

  string load_name = getsenv("load", "");

  CLSTMOCR clstm;

  if (load_name != "") {
    clstm.load(load_name);
  } else {
    Codec codec;
    trainingset.getCodec(codec);
    print("got", codec.size(), "classes");

    clstm.target_height = int(getrenv("target_height", 48));
    clstm.createBidi(codec.codec, getienv("nhidden", 100));
    clstm.setLearningRate(getdenv("lrate", 1e-4), getdenv("momentum", 0.9));
  }
  network_info(clstm.net);

  double test_error = 9999.0;
  double best_error = 1e38;

#ifndef NODISPLAY
  PyServer py;
  if (display_every > 0) py.open();
#endif
  double start_time = now();
  int start = clstm.net->attr.get("trial", getienv("start", -1)) + 1;
  if (start > 0) print("start", start);

  Trigger test_trigger(getienv("test_every", 10000), -1, start);
  test_trigger.skip0();
  Trigger save_trigger(getienv("save_every", 10000), ntrain, start);
  save_trigger.enable(save_name != "").skip0();
  Trigger report_trigger(getienv("report_every", 100), ntrain, start);
  Trigger display_trigger(getienv("display_every", 0), ntrain, start);

  for (int trial = start; trial < ntrain; trial++) {
    int sample = lrand48() % trainingset.size();
    Tensor2 raw;
    wstring gt;
    trainingset.readSample(raw, gt, sample);
    wstring pred = clstm.train(raw(), gt);

    if (report_trigger(trial)) {
      print(trial);
      print("TRU", gt);
      print("ALN", clstm.aligned_utf8());
      print("OUT", utf32_to_utf8(pred));
      if (trial > 0 && report_time)
        print("steptime", (now() - start_time) / report_trigger.since());
      start_time = now();
    }

#ifndef NODISPLAY
    if (display_trigger(trial)) {
      py.evalf("clf");
      show(py, clstm.net->inputs, 411);
      show(py, clstm.net->outputs, 412);
      show(py, clstm.targets, 413);
      show(py, clstm.aligned, 414);
    }
#endif

    if (test_trigger(trial)) {
      auto tse = test_set_error(clstm, testset);
      double errors = tse.first;
      double count = tse.second;
      test_error = errors / count;
      print("ERROR", trial, test_error, "   ", errors, count);
      if (test_error < best_error) {
        best_error = test_error;
        string fname = save_name + ".clstm";
        print("saving best performing network so far", fname, "error rate: ",
              best_error);
        clstm.net->attr.set("trial", trial);
        clstm.save(fname);
      }
    }

    if (save_trigger(trial)) {
      string fname = save_name + "-" + to_string(trial) + ".clstm";
      print("saving", fname);
      clstm.net->attr.set("trial", trial);
      clstm.save(fname);
    }
  }

  return 0;
}

int main(int argc, char **argv) {
  TRY { main1(argc, argv); }
  CATCH(const char *message) { cerr << "FATAL: " << message << endl; }
}
<commit_msg>Fix test error reporting<commit_after>#include "clstm.h"
#include <assert.h>
#include <math.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <memory>
#include <regex>
#include <set>
#include <sstream>
#include <vector>
#include "clstmhl.h"
#include "extras.h"
#include "pstring.h"
#include "utils.h"

using namespace Eigen;
using namespace ocropus;
using std::vector;
using std::map;
using std::make_pair;
using std::shared_ptr;
using std::unique_ptr;
using std::cout;
using std::ifstream;
using std::set;
using std::to_string;
using std_string = std::string;
using std_wstring = std::wstring;
using std::regex;
using std::regex_replace;
#define string std_string
#define wstring std_wstring

#ifndef NODISPLAY
void show(PyServer &py, Sequence &s, int subplot = 0, int batch = 0) {
  Tensor<float, 2> temp;
  temp.resize(s.size(), s.rows());
  for (int i = 0; i < s.size(); i++)
    for (int j = 0; j < s.rows(); j++) temp(i, j) = s[i].v(j, batch);
  if (subplot > 0) py.evalf("subplot(%d)", subplot);
  py.imshowT(temp, "cmap=cm.hot");
}
#endif

wstring separate_chars(const wstring &s, const wstring &charsep) {
  if (charsep == L"") return s;
  wstring result;
  for (int i = 0; i < s.size(); i++) {
    if (i > 0) result.push_back(charsep[0]);
    result.push_back(s[i]);
  }
  return result;
}

struct Dataset {
  vector<string> fnames;
  wstring charsep = utf8_to_utf32(getsenv("charsep", ""));
  int size() { return fnames.size(); }
  Dataset() {}
  Dataset(string file_list) { readFileList(file_list); }
  void readFileList(string file_list) { read_lines(fnames, file_list); }
  void getCodec(Codec &codec) {
    vector<string> gtnames;
    for (auto s : fnames) gtnames.push_back(basename(s) + ".gt.txt");
    codec.build(gtnames, charsep);
  }
  void readSample(Tensor2 &raw, wstring &gt, int index) {
    string fname = fnames[index];
    string base = basename(fname);
    gt = separate_chars(read_text32(base + ".gt.txt"), charsep);
    read_png(raw, fname.c_str());
    raw() = -raw() + Float(1);
  }
};

pair<double, double> test_set_error(CLSTMOCR &clstm, Dataset &testset) {
  double count = 0.0;
  double errors = 0.0;
  for (int test = 0; test < testset.size(); test++) {
    Tensor2 raw;
    wstring gt;
    testset.readSample(raw, gt, test);
    wstring pred = clstm.predict(raw());
    count += gt.size();
    errors += levenshtein(pred, gt);
  }
  return make_pair(count, errors);
}

int main1(int argc, char **argv) {
  int ntrain = getienv("ntrain", 10000000);
  string save_name = getsenv("save_name", "_ocr");
  int report_time = getienv("report_time", 0);

  if (argc < 2 || argc > 3) THROW("... training [testing]");
  Dataset trainingset(argv[1]);
  assert(trainingset.size() > 0);
  Dataset testset;
  if (argc > 2) testset.readFileList(argv[2]);
  print("got", trainingset.size(), "files,", testset.size(), "tests");

  string load_name = getsenv("load", "");

  CLSTMOCR clstm;

  if (load_name != "") {
    clstm.load(load_name);
  } else {
    Codec codec;
    trainingset.getCodec(codec);
    print("got", codec.size(), "classes");

    clstm.target_height = int(getrenv("target_height", 48));
    clstm.createBidi(codec.codec, getienv("nhidden", 100));
    clstm.setLearningRate(getdenv("lrate", 1e-4), getdenv("momentum", 0.9));
  }
  network_info(clstm.net);

  double test_error = 9999.0;
  double best_error = 1e38;

#ifndef NODISPLAY
  PyServer py;
  if (display_every > 0) py.open();
#endif
  double start_time = now();
  int start = clstm.net->attr.get("trial", getienv("start", -1)) + 1;
  if (start > 0) print("start", start);

  Trigger test_trigger(getienv("test_every", 10000), -1, start);
  test_trigger.skip0();
  Trigger save_trigger(getienv("save_every", 10000), ntrain, start);
  save_trigger.enable(save_name != "").skip0();
  Trigger report_trigger(getienv("report_every", 100), ntrain, start);
  Trigger display_trigger(getienv("display_every", 0), ntrain, start);

  for (int trial = start; trial < ntrain; trial++) {
    int sample = lrand48() % trainingset.size();
    Tensor2 raw;
    wstring gt;
    trainingset.readSample(raw, gt, sample);
    wstring pred = clstm.train(raw(), gt);

    if (report_trigger(trial)) {
      print(trial);
      print("TRU", gt);
      print("ALN", clstm.aligned_utf8());
      print("OUT", utf32_to_utf8(pred));
      if (trial > 0 && report_time)
        print("steptime", (now() - start_time) / report_trigger.since());
      start_time = now();
    }

#ifndef NODISPLAY
    if (display_trigger(trial)) {
      py.evalf("clf");
      show(py, clstm.net->inputs, 411);
      show(py, clstm.net->outputs, 412);
      show(py, clstm.targets, 413);
      show(py, clstm.aligned, 414);
    }
#endif

    if (test_trigger(trial)) {
      auto tse = test_set_error(clstm, testset);
      double count = tse.first;
      double errors = tse.second;
      test_error = errors / count;
      print("ERROR", trial, test_error, "   ", errors, count);
      if (test_error < best_error) {
        best_error = test_error;
        string fname = save_name + ".clstm";
        print("saving best performing network so far", fname, "error rate: ",
              best_error);
        clstm.net->attr.set("trial", trial);
        clstm.save(fname);
      }
    }

    if (save_trigger(trial)) {
      string fname = save_name + "-" + to_string(trial) + ".clstm";
      print("saving", fname);
      clstm.net->attr.set("trial", trial);
      clstm.save(fname);
    }
  }

  return 0;
}

int main(int argc, char **argv) {
  TRY { main1(argc, argv); }
  CATCH(const char *message) { cerr << "FATAL: " << message << endl; }
}
<|endoftext|>


<commit_before>/*

Copyright (c) 2013, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "test.hpp"
#include "libtorrent/file.hpp"
#include "libtorrent/torrent_info.hpp"

#if TORRENT_USE_IOSTREAM
#include <sstream>
#endif

struct test_torrent_t
{
	char const* file;
};

using namespace libtorrent;

test_torrent_t test_torrents[] =
{
	{ "base.torrent" },
	{ "empty_path.torrent" },
	{ "parent_path.torrent" },
	{ "hidden_parent_path.torrent" },
	{ "single_multi_file.torrent" },
	{ "slash_path.torrent" },
	{ "backslash_path.torrent" },
	{ "url_list.torrent" },
	{ "url_list2.torrent" },
	{ "url_list3.torrent" },
	{ "httpseed.torrent" },
	{ "empty_httpseed.torrent" },
	{ "long_name.torrent" },
	{ "whitespace_url.torrent" },
	{ "duplicate_files.torrent" },
	{ "pad_file.torrent" },
	{ "creation_date.torrent" },
	{ "no_creation_date.torrent" },
	{ "url_seed.torrent" },
	{ "url_seed_multi.torrent" },
	{ "url_seed_multi_space.torrent" },
	{ "url_seed_multi_space_nolist.torrent" },
	{ "root_hash.torrent" },
};

struct test_failing_torrent_t
{
	char const* file;
	error_code error; // the expected error
};

test_failing_torrent_t test_error_torrents[] =
{
	{ "missing_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "invalid_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "negative_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "no_name.torrent", errors::torrent_missing_name },
	{ "invalid_name.torrent", errors::torrent_missing_name },
	{ "invalid_name2.torrent", errors::torrent_invalid_name },
	{ "invalid_info.torrent", errors::torrent_missing_info },
	{ "string.torrent", errors::torrent_is_no_dict },
	{ "negative_size.torrent", errors::torrent_invalid_length },
	{ "negative_file_size.torrent", errors::torrent_file_parse_failed },
	{ "invalid_path_list.torrent", errors::torrent_file_parse_failed },
	{ "missing_path_list.torrent", errors::torrent_file_parse_failed },
	{ "invalid_pieces.torrent", errors::torrent_missing_pieces },
	{ "unaligned_pieces.torrent", errors::torrent_invalid_hashes },
	{ "invalid_root_hash.torrent", errors::torrent_invalid_hashes },
	{ "invalid_root_hash2.torrent", errors::torrent_missing_pieces},
	{ "invalid_file_size.torrent", errors::torrent_file_parse_failed },
};

// TODO: test remap_files
// TODO: merkle torrents. specifically torrent_info::add_merkle_nodes and torrent with "root hash"
// TODO: torrent with 'p' (padfile) attribute
// TODO: torrent with 'h' (hidden) attribute
// TODO: torrent with 'x' (executable) attribute
// TODO: torrent with 'l' (symlink) attribute
// TODO: creating a merkle torrent (torrent_info::build_merkle_list)
// TODO: torrent with multiple trackers in multiple tiers, making sure we shuffle them (how do you test shuffling?, load it multiple times and make sure it's in different order at least once)

int test_main()
{
	for (int i = 0; i < sizeof(test_torrents)/sizeof(test_torrents[0]); ++i)
	{
		error_code ec;
		fprintf(stderr, "loading %s\n", test_torrents[i].file);
		boost::intrusive_ptr<torrent_info> ti(new torrent_info(combine_path(combine_path("..", "test_torrents"), test_torrents[i].file), ec));
		TEST_CHECK(!ec);
		if (ec) fprintf(stderr, "  -> failed %s\n", ec.message().c_str());

		if (std::string(test_torrents[i].file) == "whitespace_url.torrent")
		{
			// make sure we trimmed the url
			TEST_CHECK(ti->trackers().size() > 0);
			if (ti->trackers().size() > 0)
				TEST_CHECK(ti->trackers()[0].url == "udp://test.com/announce");
		}
		else if (std::string(test_torrents[i].file) == "duplicate_files.torrent")
		{
			// make sure we disambiguated the files
			TEST_EQUAL(ti->num_files(), 2);
			TEST_CHECK(ti->file_at(0).path == combine_path(combine_path("temp", "foo"), "bar.txt"));
			TEST_CHECK(ti->file_at(1).path == combine_path(combine_path("temp", "foo"), "bar.1.txt"));
		}
		else if (std::string(test_torrents[i].file) == "pad_file.torrent")
		{
			TEST_EQUAL(ti->num_files(), 2);
			TEST_CHECK(ti->file_at(0).pad_file == false);
			TEST_CHECK(ti->file_at(1).pad_file == true);
		}
		else if (std::string(test_torrents[i].file) == "creation_date.torrent")
		{
			TEST_CHECK(*ti->creation_date() == 1234567);
		}
		else if (std::string(test_torrents[i].file) == "no_creation_date.torrent")
		{
			TEST_CHECK(!ti->creation_date());
		}
		else if (std::string(test_torrents[i].file) == "url_seed.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/file");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/file");
#endif
		}
		else if (std::string(test_torrents[i].file) == "url_seed_multi.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/file/");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/file/");
#endif
		}
		else if (std::string(test_torrents[i].file) == "url_seed_multi_space.torrent"
			|| std::string(test_torrents[i].file) == "url_seed_multi_space_nolist.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/test%20file/foo%20bar/");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/test%20file/foo%20bar/");
#endif
		}

		int index = 0;
		for (torrent_info::file_iterator i = ti->begin_files();
			i != ti->end_files(); ++i, ++index)
		{
			int first = ti->map_file(index, 0, 0).piece;
			int last = ti->map_file(index, (std::max)(size_type(i->size)-1, size_type(0)), 0).piece;
			fprintf(stderr, "  %11"PRId64" %c%c%c%c [ %4d, %4d ] %7u %s %s %s%s\n"
				, i->size
				, (i->pad_file?'p':'-')
				, (i->executable_attribute?'x':'-')
				, (i->hidden_attribute?'h':'-')
				, (i->symlink_attribute?'l':'-')
				, first, last
				, boost::uint32_t(ti->files().mtime(*i))
				, ti->files().hash(*i) != sha1_hash(0) ? to_hex(ti->files().hash(*i).to_string()).c_str() : ""
				, ti->files().file_path(*i).c_str()
				, i->symlink_attribute ? "-> ": ""
				, i->symlink_attribute && i->symlink_index != -1 ? ti->files().symlink(*i).c_str() : "");
		}

		// test swap
#if !defined TORRENT_NO_DEPRECATE && TORRENT_USE_IOSTREAM
		std::stringstream str1;
		ti->print(str1);

		torrent_info temp("temp", ec);
		temp.swap(*ti);

		std::stringstream str2;
		temp.print(str2);
		TEST_EQUAL(str1.str(), str2.str());
#endif

	}

	for (int i = 0; i < sizeof(test_error_torrents)/sizeof(test_error_torrents[0]); ++i)
	{
		error_code ec;
		fprintf(stderr, "loading %s\n", test_error_torrents[i].file);
		boost::intrusive_ptr<torrent_info> ti(new torrent_info(combine_path(combine_path("..", "test_torrents"), test_error_torrents[i].file), ec));
		fprintf(stderr, "E: %s\nexpected: %s\n", ec.message().c_str(), test_error_torrents[i].error.message().c_str());
		TEST_EQUAL(ec, test_error_torrents[i].error);
	}

	return 0;
}

<commit_msg>fix test_torrent_parse for windows<commit_after>/*

Copyright (c) 2013, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "test.hpp"
#include "libtorrent/file.hpp"
#include "libtorrent/torrent_info.hpp"

#if TORRENT_USE_IOSTREAM
#include <sstream>
#endif

struct test_torrent_t
{
	char const* file;
};

using namespace libtorrent;

test_torrent_t test_torrents[] =
{
	{ "base.torrent" },
	{ "empty_path.torrent" },
	{ "parent_path.torrent" },
	{ "hidden_parent_path.torrent" },
	{ "single_multi_file.torrent" },
	{ "slash_path.torrent" },
	{ "backslash_path.torrent" },
	{ "url_list.torrent" },
	{ "url_list2.torrent" },
	{ "url_list3.torrent" },
	{ "httpseed.torrent" },
	{ "empty_httpseed.torrent" },
	{ "long_name.torrent" },
	{ "whitespace_url.torrent" },
	{ "duplicate_files.torrent" },
	{ "pad_file.torrent" },
	{ "creation_date.torrent" },
	{ "no_creation_date.torrent" },
	{ "url_seed.torrent" },
	{ "url_seed_multi.torrent" },
	{ "url_seed_multi_space.torrent" },
	{ "url_seed_multi_space_nolist.torrent" },
	{ "root_hash.torrent" },
};

struct test_failing_torrent_t
{
	char const* file;
	error_code error; // the expected error
};

test_failing_torrent_t test_error_torrents[] =
{
	{ "missing_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "invalid_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "negative_piece_len.torrent", errors::torrent_missing_piece_length },
	{ "no_name.torrent", errors::torrent_missing_name },
	{ "invalid_name.torrent", errors::torrent_missing_name },
	{ "invalid_name2.torrent", errors::torrent_invalid_name },
	{ "invalid_info.torrent", errors::torrent_missing_info },
	{ "string.torrent", errors::torrent_is_no_dict },
	{ "negative_size.torrent", errors::torrent_invalid_length },
	{ "negative_file_size.torrent", errors::torrent_file_parse_failed },
	{ "invalid_path_list.torrent", errors::torrent_file_parse_failed },
	{ "missing_path_list.torrent", errors::torrent_file_parse_failed },
	{ "invalid_pieces.torrent", errors::torrent_missing_pieces },
	{ "unaligned_pieces.torrent", errors::torrent_invalid_hashes },
	{ "invalid_root_hash.torrent", errors::torrent_invalid_hashes },
	{ "invalid_root_hash2.torrent", errors::torrent_missing_pieces},
	{ "invalid_file_size.torrent", errors::torrent_file_parse_failed },
};

// TODO: test remap_files
// TODO: merkle torrents. specifically torrent_info::add_merkle_nodes and torrent with "root hash"
// TODO: torrent with 'p' (padfile) attribute
// TODO: torrent with 'h' (hidden) attribute
// TODO: torrent with 'x' (executable) attribute
// TODO: torrent with 'l' (symlink) attribute
// TODO: creating a merkle torrent (torrent_info::build_merkle_list)
// TODO: torrent with multiple trackers in multiple tiers, making sure we shuffle them (how do you test shuffling?, load it multiple times and make sure it's in different order at least once)

int test_main()
{
	std::string root_dir = parent_path(current_working_directory());
	for (int i = 0; i < sizeof(test_torrents)/sizeof(test_torrents[0]); ++i)
	{
		error_code ec;
		fprintf(stderr, "loading %s\n", test_torrents[i].file);
		boost::intrusive_ptr<torrent_info> ti(new torrent_info(combine_path(combine_path(root_dir, "test_torrents"), test_torrents[i].file), ec));
		TEST_CHECK(!ec);
		if (ec) fprintf(stderr, "  -> failed %s\n", ec.message().c_str());

		if (std::string(test_torrents[i].file) == "whitespace_url.torrent")
		{
			// make sure we trimmed the url
			TEST_CHECK(ti->trackers().size() > 0);
			if (ti->trackers().size() > 0)
				TEST_CHECK(ti->trackers()[0].url == "udp://test.com/announce");
		}
		else if (std::string(test_torrents[i].file) == "duplicate_files.torrent")
		{
			// make sure we disambiguated the files
			TEST_EQUAL(ti->num_files(), 2);
			TEST_CHECK(ti->file_at(0).path == combine_path(combine_path("temp", "foo"), "bar.txt"));
			TEST_CHECK(ti->file_at(1).path == combine_path(combine_path("temp", "foo"), "bar.1.txt"));
		}
		else if (std::string(test_torrents[i].file) == "pad_file.torrent")
		{
			TEST_EQUAL(ti->num_files(), 2);
			TEST_CHECK(ti->file_at(0).pad_file == false);
			TEST_CHECK(ti->file_at(1).pad_file == true);
		}
		else if (std::string(test_torrents[i].file) == "creation_date.torrent")
		{
			TEST_CHECK(*ti->creation_date() == 1234567);
		}
		else if (std::string(test_torrents[i].file) == "no_creation_date.torrent")
		{
			TEST_CHECK(!ti->creation_date());
		}
		else if (std::string(test_torrents[i].file) == "url_seed.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/file");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/file");
#endif
		}
		else if (std::string(test_torrents[i].file) == "url_seed_multi.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/file/");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/file/");
#endif
		}
		else if (std::string(test_torrents[i].file) == "url_seed_multi_space.torrent"
			|| std::string(test_torrents[i].file) == "url_seed_multi_space_nolist.torrent")
		{
			TEST_EQUAL(ti->web_seeds().size(), 1);
			TEST_EQUAL(ti->web_seeds()[0].url, "http://test.com/test%20file/foo%20bar/");
#ifndef TORRENT_NO_DEPRECATE
			TEST_EQUAL(ti->http_seeds().size(), 0);
			TEST_EQUAL(ti->url_seeds().size(), 1);
			TEST_EQUAL(ti->url_seeds()[0], "http://test.com/test%20file/foo%20bar/");
#endif
		}

		int index = 0;
		for (torrent_info::file_iterator i = ti->begin_files();
			i != ti->end_files(); ++i, ++index)
		{
			int first = ti->map_file(index, 0, 0).piece;
			int last = ti->map_file(index, (std::max)(size_type(i->size)-1, size_type(0)), 0).piece;
			fprintf(stderr, "  %11"PRId64" %c%c%c%c [ %4d, %4d ] %7u %s %s %s%s\n"
				, i->size
				, (i->pad_file?'p':'-')
				, (i->executable_attribute?'x':'-')
				, (i->hidden_attribute?'h':'-')
				, (i->symlink_attribute?'l':'-')
				, first, last
				, boost::uint32_t(ti->files().mtime(*i))
				, ti->files().hash(*i) != sha1_hash(0) ? to_hex(ti->files().hash(*i).to_string()).c_str() : ""
				, ti->files().file_path(*i).c_str()
				, i->symlink_attribute ? "-> ": ""
				, i->symlink_attribute && i->symlink_index != -1 ? ti->files().symlink(*i).c_str() : "");
		}

		// test swap
#if !defined TORRENT_NO_DEPRECATE && TORRENT_USE_IOSTREAM
		std::stringstream str1;
		ti->print(str1);

		torrent_info temp("temp", ec);
		temp.swap(*ti);

		std::stringstream str2;
		temp.print(str2);
		TEST_EQUAL(str1.str(), str2.str());
#endif

	}

	for (int i = 0; i < sizeof(test_error_torrents)/sizeof(test_error_torrents[0]); ++i)
	{
		error_code ec;
		fprintf(stderr, "loading %s\n", test_error_torrents[i].file);
		boost::intrusive_ptr<torrent_info> ti(new torrent_info(combine_path(combine_path(root_dir, "test_torrents"), test_error_torrents[i].file), ec));
		fprintf(stderr, "E: %s\nexpected: %s\n", ec.message().c_str(), test_error_torrents[i].error.message().c_str());
		TEST_EQUAL(ec, test_error_torrents[i].error);
	}

	return 0;
}

<|endoftext|>


<commit_before>#pragma once
#include <bts/blockchain/types.hpp>

namespace bts { namespace blockchain {

static std::unordered_map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {       1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    {  274001, bts::blockchain::block_id_type( "f46c109cfb1bac323122ae59b08edc23328d880c" ) },
    {  316002, bts::blockchain::block_id_type( "206b7e6574019f4352515bd3d96162fb40a1b18a" ) },
    {  340001, bts::blockchain::block_id_type( "d3858436100abe3ad7ee9f026e5f7f4781732e06" ) },
    {  357001, bts::blockchain::block_id_type( "d240848ca9bf4be30204eded02fb35ce2e215d41" ) },
    {  408751, bts::blockchain::block_id_type( "12481e818abe9bc86069e45586df11d2cff7dbb8" ) },
    {  451501, bts::blockchain::block_id_type( "42e519db48f09570fcc02f38288648a92789dcb3" ) },
    {  494001, bts::blockchain::block_id_type( "59836f51f13e07be1de734a02360742c7b5f0dd6" ) },
    {  554801, bts::blockchain::block_id_type( "4a992cc282a0024d393c6389cf84cbb8df1fc839" ) },
    {  578901, bts::blockchain::block_id_type( "add6131c7bc6f1c9470de209b0e9c257d664f1c9" ) },
    {  613201, bts::blockchain::block_id_type( "3317ea0a272976a10ef6593e840f9ff2032affad" ) },
    {  640001, bts::blockchain::block_id_type( "11c748cdbfc445396deb6d24908521701aaa80aa" ) },
    {  820201, bts::blockchain::block_id_type( "904278b4bd585e3055c092aada1e030fbdf69a0e" ) },
    {  871001, bts::blockchain::block_id_type( "137ee1e9171a7ed1fe9cc58536d8e20749c3f199" ) },
    {  991701, bts::blockchain::block_id_type( "2e48f0a16d2107b9ab3d1c87e803ee70d3c91ce0" ) },
    { 1315315, bts::blockchain::block_id_type( "baede744122d4b8d296ab3497bf5c849d1131466" ) },
    { 1575501, bts::blockchain::block_id_type( "52c5e5764fba4c876cde8c80e598c89ee5c35d9f" ) },
    { 1772201, bts::blockchain::block_id_type( "5832516564d86e86edbbb501cabec98c77504307" ) },
    { 1809000, bts::blockchain::block_id_type( "0846babe3b88da36350f287b69c6440f59168beb" ) }
};

// Initialized in load_checkpoints()
static uint32_t LAST_CHECKPOINT_BLOCK_NUM = 0;

} } // bts::blockchain
<commit_msg>Update checkpoint<commit_after>#pragma once
#include <bts/blockchain/types.hpp>

namespace bts { namespace blockchain {

static std::unordered_map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {       1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    {  274001, bts::blockchain::block_id_type( "f46c109cfb1bac323122ae59b08edc23328d880c" ) },
    {  316002, bts::blockchain::block_id_type( "206b7e6574019f4352515bd3d96162fb40a1b18a" ) },
    {  340001, bts::blockchain::block_id_type( "d3858436100abe3ad7ee9f026e5f7f4781732e06" ) },
    {  357001, bts::blockchain::block_id_type( "d240848ca9bf4be30204eded02fb35ce2e215d41" ) },
    {  408751, bts::blockchain::block_id_type( "12481e818abe9bc86069e45586df11d2cff7dbb8" ) },
    {  451501, bts::blockchain::block_id_type( "42e519db48f09570fcc02f38288648a92789dcb3" ) },
    {  494001, bts::blockchain::block_id_type( "59836f51f13e07be1de734a02360742c7b5f0dd6" ) },
    {  554801, bts::blockchain::block_id_type( "4a992cc282a0024d393c6389cf84cbb8df1fc839" ) },
    {  578901, bts::blockchain::block_id_type( "add6131c7bc6f1c9470de209b0e9c257d664f1c9" ) },
    {  613201, bts::blockchain::block_id_type( "3317ea0a272976a10ef6593e840f9ff2032affad" ) },
    {  640001, bts::blockchain::block_id_type( "11c748cdbfc445396deb6d24908521701aaa80aa" ) },
    {  820201, bts::blockchain::block_id_type( "904278b4bd585e3055c092aada1e030fbdf69a0e" ) },
    {  871001, bts::blockchain::block_id_type( "137ee1e9171a7ed1fe9cc58536d8e20749c3f199" ) },
    {  991701, bts::blockchain::block_id_type( "2e48f0a16d2107b9ab3d1c87e803ee70d3c91ce0" ) },
    { 1315315, bts::blockchain::block_id_type( "baede744122d4b8d296ab3497bf5c849d1131466" ) },
    { 1575501, bts::blockchain::block_id_type( "52c5e5764fba4c876cde8c80e598c89ee5c35d9f" ) },
    { 1772201, bts::blockchain::block_id_type( "5832516564d86e86edbbb501cabec98c77504307" ) },
    { 1876000, bts::blockchain::block_id_type( "1e5bbf285af23d52f89532b213774ecf294565bf" ) }
};

// Initialized in load_checkpoints()
static uint32_t LAST_CHECKPOINT_BLOCK_NUM = 0;

} } // bts::blockchain
<|endoftext|>


<commit_before>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/***************************************************************************
 *   Copyright (C) 2004-2012 by CEA LIST                              *
 *                                                                         *
 ***************************************************************************/

#include "common/LimaCommon.h"
#include "common/MediaticData/mediaticData.h"
#include "common/XMLConfigurationFiles/xmlConfigurationFileParser.h"
#include "common/Data/strwstrtools.h"
#include "common/time/traceUtils.h"
#include "linguisticProcessing/common/BagOfWords/bowText.h"
#include "linguisticProcessing/common/BagOfWords/bowToken.h"
#include "linguisticProcessing/common/BagOfWords/bowTerm.h"
#include "linguisticProcessing/common/linguisticData/languageData.h"
#include "linguisticProcessing/common/PropertyCode/PropertyCodeManager.h"
#include "linguisticProcessing/common/PropertyCode/PropertyManager.h"

#include "linguisticProcessing/client/AnalysisHandlers/BowTextHandler.h"
#include "linguisticProcessing/client/AnalysisHandlers/BowTextWriter.h"
#include "linguisticProcessing/client/LinguisticProcessingClientFactory.h"
#include "linguisticProcessing/client/AnalysisHandlers/SimpleStreamHandler.h"
#include "common/MediaProcessors/MediaProcessUnit.h"
#include "linguisticProcessing/core/LinguisticResources/AbstractResource.h"

#include <string>
#include <vector>
#include <iostream>
#include <fstream>

#include <QtCore/QCoreApplication>

using namespace Lima::LinguisticProcessing;
using namespace Lima::Common::MediaticData;
using namespace Lima::Common::Misc;
using namespace Lima::Common::BagOfWords;
using namespace Lima;
using namespace std;

void usage(int argc, char* argv[]);
void listunits();
int dowork(int argc,char* argv[]);
multimap<LimaString,string> extractNormalization(const LimaString& source,const BoWText& bowText,MediaId lang);
pair<int,int> getStartEnd(const BoWToken* tok);

int main(int argc,char* argv[])
{
  QCoreApplication a(argc, argv);
  QsLogging::initQsLog();
  bool docatch = false;
  if (argc>1)
  {
    for (int i = 1 ; i < argc; i++)
    {
      std::string arg(argv[i]);
      if (arg== "--catch")
        docatch = true;
    }
  }
  if (docatch)
  {
    try
    {
      std::cerr << "Doing work in try block." << std::endl;
      return dowork(argc, argv);
    }
    catch (const std::exception& e)
    {
      std::cerr << "Catched an exception: " << e.what() << std::endl;
    }
    catch (...)
    {
      std::cerr << "Catched an unknown exception " << std::endl;
    }
  }
  else
    return dowork(argc,argv);
}

int dowork(int argc,char* argv[])
{

  string resourcesPath=getenv("LIMA_RESOURCES")==0?"/usr/share/apps/lima/resources":string(getenv("LIMA_RESOURCES"));
  string configDir=getenv("LIMA_CONF")==0?"/usr/share/config/lima":string(getenv("LIMA_CONF"));
  string lpConfigFile=string("lima-analysis.xml");
  string commonConfigFile=string("lima-common.xml");
  string pipeline=string("normalization");
  string clientId=string("lima-coreclient");

  bool printCategs=false;

  deque<string> langs;
  deque<string> files;

  if (argc>1)
  {
    for (int i = 1 ; i < argc; i++)
    {
      std::string arg(argv[i]);
      std::string::size_type pos = std::string::npos;
      if ( arg[0] == '-' )
      {
        if (arg == "--help")
          usage(argc, argv);
        else if (arg== "--printCategs")
          printCategs=true;
        else if (arg== "--availableUnits")
          listunits();
        else if (arg== "--catch") ;
        else if ( (pos = arg.find("--lp-config-file=")) != std::string::npos )
          lpConfigFile = arg.substr(pos+14);
        else if ( (pos = arg.find("--common-config-file=")) != std::string::npos )
          commonConfigFile = arg.substr(pos+20);
        else if ( (pos = arg.find("--config-dir=")) != std::string::npos )
          configDir = arg.substr(pos+13);
        else if ( (pos = arg.find("--resources-dir=")) != std::string::npos )
          resourcesPath = arg.substr(pos+16);
        else if ( (pos = arg.find("--language=")) != std::string::npos )
          langs.push_back(arg.substr(pos+11));
//         else if ( (pos = arg.find("--pipeline=")) != std::string::npos )
//           pipeline = arg.substr(pos+11);
        else if ( (pos = arg.find("--client=")) != std::string::npos )
          clientId=arg.substr(pos+9);
        else usage(argc, argv);
      }
      else
      {
        files.push_back(arg);
      }
    }
  }

  if (langs.size()<1)
  {
    std::cerr << "no language defined !" << std::endl;
    return -1;
  }

  AbstractLinguisticProcessingClient* client(0);

  try
  {

    // initialize common
    MediaticData::changeable().init(
      resourcesPath,
      configDir,
      commonConfigFile,
      langs);

    // initialize linguistic processing
    deque<string> pipelines;
    pipelines.push_back(pipeline);
    Lima::Common::XMLConfigurationFiles::XMLConfigurationFileParser lpconfig(configDir + "/" + lpConfigFile);
    LinguisticProcessingClientFactory::changeable().configureClientFactory(
      clientId,
      lpconfig,
      langs,
      pipelines);

    client=dynamic_cast<AbstractLinguisticProcessingClient*>(LinguisticProcessingClientFactory::single().createClient(clientId));

    // Set the handlers
    std::map<std::string, AbstractAnalysisHandler*> handlers;
    BowTextHandler bowTextHandler;
    handlers.insert(std::make_pair("bowTextHandler", &bowTextHandler));

    map<string,string> metaData;
    metaData["Lang"]=langs[0];
    MediaId lang=MediaticData::single().getMediaId(langs[0]);

    for (deque<string>::iterator fileItr=files.begin();
         fileItr!=files.end();
         fileItr++)
    {
      // open the output file
      ostringstream os;
      os << *fileItr << ".norm";
      ofstream fout(os.str().c_str(), std::ofstream::binary);

      // loading of the input file
      TimeUtils::updateCurrentTime();
      std::ifstream file(fileItr->c_str(), std::ifstream::binary);
      char buf[256];
      file.getline(buf,256);
      std::string line(buf);
      while (!file.eof())
      {
        if (line.size()==0)
        {
          file.getline(buf,256);
          line=string(buf);
          continue;
        }
        //        cout << "normalize " << line << endl;
        LimaString contentText;
        // The input text MUST be UTF-8 encoded !!!
        contentText = utf8stdstring2limastring(line);

        // analyze it
        metaData["FileName"]=*fileItr;

        client->analyze(contentText,metaData,pipeline,handlers);

        // analyze resulting bowText to extract normalization
        multimap<LimaString,string> norms=extractNormalization(contentText,bowTextHandler.getBowText(),lang);
        if (norms.empty())
        {
          norms.insert(make_pair(contentText,"NONE_1"));
        }
        for (multimap<LimaString,string>::iterator it=norms.begin();it!=norms.end();it++)
        {
          fout << limastring2utf8stdstring(it->first);
          if (printCategs)
          {
            fout << "#" << it->second;
          }
          fout << ";";
        }
        fout << endl;
        // read next line
        file.getline(buf,256);
        line=string(buf);
      }
    }
  }
  catch (InvalidConfiguration& e)
  {
    throw e;
  }

  delete client;
  return SUCCESS_ID;
}

multimap<LimaString,string> extractNormalization(const LimaString& source,const BoWText& bowText,MediaId lang)
{
  const Common::PropertyCode::PropertyManager& macroManager = static_cast<const Common::MediaticData::LanguageData&>(MediaticData::single().mediaData(lang)).getPropertyCodeManager().getPropertyManager("MACRO");
  multimap<LimaString,string> result;
  // si un seul bowtoken on le prend
  //  if (bowText.size()==1)
  //  {
  //    cerr << "- found only one norm : " << bowText.front()->getLemma() << endl;
  //    result.push_back(bowText.front()->getLemma());
  //  }
  // sinon on prend tous les bowtoken qui vont du d�ut �la fin
  //  else
  //  {
  //    cerr << "extractNormalisation : " << source << endl;
  for (BoWText::const_iterator bowItr=bowText.begin();
       bowItr!=bowText.end();
       bowItr++)
  {
    pair<int,int> posLen=getStartEnd(*bowItr);
    //      cerr << "  - " << (*bowItr)->getLemma() << " at " << posLen.first << "," << posLen.second;
    if ((posLen.first==1) && (posLen.second==int(source.size()+1)))
    {
      result.insert(make_pair(
                      (*bowItr)->getLemma(),
                      macroManager.getPropertySymbolicValue((*bowItr)->getCategory())));
      //        cerr << " keep it !";
    }
    //      cerr << endl;
  }
  //   }
  return result;
}

pair<int,int> getStartEnd(const BoWToken* tok)
{
  pair<int,int> res;
  if (tok->getType()==BOW_TOKEN)
  {
    res.first=tok->getPosition();
    res.second=tok->getPosition()+tok->getLength();
  }
  else
  {
    const BoWComplexToken* complextok=dynamic_cast<const BoWComplexToken*>(tok);
    if (complextok==0)
    {
      cerr << "ERROR ! complextok==0 ! should not happen !" << endl;
      exit(0);
    }
    const std::deque< BoWComplexToken::Part >& parts=complextok->getParts();
    if (parts.size()==0)
    {
      cerr << "ERROR ! complex token should have at least one part ! " << endl;
      exit(0);
    }
    std::deque< BoWComplexToken::Part >::const_iterator partItr=parts.begin();
    res=getStartEnd(partItr->get<1>());
    partItr++;
    for (;partItr!=parts.end();partItr++)
    {
      pair<int,int> tmp=getStartEnd(partItr->get<1>());
      if (tmp.first<res.first) res.first=tmp.first;
      if (tmp.second>res.second) res.second=tmp.second;
    }
  }
  return res;
}

void usage(int argc, char *argv[])
{
  LIMA_UNUSED(argc);
  std::cout << "usage: " << argv[0] << " [OPTIONS] [file1 [file2 [...]]] " << std::endl;
  std::cout << "\t--printCategs\t\tprint categories with normalizations" << std::endl;
  std::cout << "\t--resources-dir=</path/to/the/resources> Optional. Default is $LIMA_RESOURCES" << std::endl;
  std::cout << "\t--config-dir=</path/to/the/configuration/directory> Optional. Default is $LIMA_CONF" << std::endl;
  std::cout << "\t--lp-config-file=<configuration/file/name>\tOptional. Default is lima-analysis.xml" << std::endl;
  std::cout << "\t--common-config-file=<configuration/file/name>\tOptional. Default is lima-common.xml" << std::endl;
  std::cout << "\t--client=<clientId>\tOptional. Default is 'lima-coreclient'" << std::endl;
  std::cout << "\t--language=<language trigram>\tOptional. Language of document to analyze." << std::endl;
//   std::cout << "\t--pipeline=<pipelineId>\tOptional. Default is 'main'" << std::endl;
  std::cout << "\t--availableUnits\tshow all available resources, processUnits and dumpers" << std::endl;
  std::cout << "\twhere files are files to analyze." << std::endl;
  std::cout << endl;
  std::cout << "Available client factories are : " << std::endl;
  {
    deque<string> ids=LinguisticProcessingClientFactory::single().getRegisteredFactories();
    for (deque<string>::iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  exit(0);
}

void listunits()
{
  {
    cout << "Available resources factories : " << endl;
    deque<string> ids=AbstractResource::Factory::getRegisteredFactories();
    for (deque<string>::const_iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  {
    cout << "Available process units factories : " << endl;
    deque<string> ids=MediaProcessUnit::Factory::getRegisteredFactories();
    for (deque<string>::const_iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  std::cout << "Available client factories are : " << std::endl;
  {
    deque<string> ids=LinguisticProcessingClientFactory::single().getRegisteredFactories();
    for (deque<string>::iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  exit(0);
}
<commit_msg>add AmosePluginsManager to normalizeTerm<commit_after>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/***************************************************************************
 *   Copyright (C) 2004-2012 by CEA LIST                              *
 *                                                                         *
 ***************************************************************************/

#include "common/LimaCommon.h"
#include "common/MediaticData/mediaticData.h"
#include "common/XMLConfigurationFiles/xmlConfigurationFileParser.h"
#include "common/Data/strwstrtools.h"
#include "common/time/traceUtils.h"
#include "common/AbstractFactoryPattern/AmosePluginsManager.h"
#include "linguisticProcessing/common/BagOfWords/bowText.h"
#include "linguisticProcessing/common/BagOfWords/bowToken.h"
#include "linguisticProcessing/common/BagOfWords/bowTerm.h"
#include "linguisticProcessing/common/linguisticData/languageData.h"
#include "linguisticProcessing/common/PropertyCode/PropertyCodeManager.h"
#include "linguisticProcessing/common/PropertyCode/PropertyManager.h"

#include "linguisticProcessing/client/AnalysisHandlers/BowTextHandler.h"
#include "linguisticProcessing/client/AnalysisHandlers/BowTextWriter.h"
#include "linguisticProcessing/client/LinguisticProcessingClientFactory.h"
#include "linguisticProcessing/client/AnalysisHandlers/SimpleStreamHandler.h"
#include "common/MediaProcessors/MediaProcessUnit.h"
#include "linguisticProcessing/core/LinguisticResources/AbstractResource.h"

#include <string>
#include <vector>
#include <iostream>
#include <fstream>

#include <QtCore/QCoreApplication>

using namespace Lima::LinguisticProcessing;
using namespace Lima::Common::MediaticData;
using namespace Lima::Common::Misc;
using namespace Lima::Common::BagOfWords;
using namespace Lima;
using namespace std;

void usage(int argc, char* argv[]);
void listunits();
int dowork(int argc,char* argv[]);
multimap<LimaString,string> extractNormalization(const LimaString& source,const BoWText& bowText,MediaId lang);
pair<int,int> getStartEnd(const BoWToken* tok);

int main(int argc,char* argv[])
{
  QCoreApplication a(argc, argv);
  // Necessary to initialize factories
  Lima::AmosePluginsManager::single();

  bool docatch = false;
  if (argc>1)
  {
    for (int i = 1 ; i < argc; i++)
    {
      std::string arg(argv[i]);
      if (arg== "--catch")
        docatch = true;
    }
  }
  if (docatch)
  {
    try
    {
      std::cerr << "Doing work in try block." << std::endl;
      return dowork(argc, argv);
    }
    catch (const std::exception& e)
    {
      std::cerr << "Catched an exception: " << e.what() << std::endl;
    }
    catch (...)
    {
      std::cerr << "Catched an unknown exception " << std::endl;
    }
  }
  else
    return dowork(argc,argv);
}

int dowork(int argc,char* argv[])
{

  string resourcesPath=getenv("LIMA_RESOURCES")==0?"/usr/share/apps/lima/resources":string(getenv("LIMA_RESOURCES"));
  string configDir=getenv("LIMA_CONF")==0?"/usr/share/config/lima":string(getenv("LIMA_CONF"));
  string lpConfigFile=string("lima-analysis.xml");
  string commonConfigFile=string("lima-common.xml");
  string pipeline=string("normalization");
  string clientId=string("lima-coreclient");

  bool printCategs=false;

  deque<string> langs;
  deque<string> files;

  if (argc>1)
  {
    for (int i = 1 ; i < argc; i++)
    {
      std::string arg(argv[i]);
      std::string::size_type pos = std::string::npos;
      if ( arg[0] == '-' )
      {
        if (arg == "--help")
          usage(argc, argv);
        else if (arg== "--printCategs")
          printCategs=true;
        else if (arg== "--availableUnits")
          listunits();
        else if (arg== "--catch") ;
        else if ( (pos = arg.find("--lp-config-file=")) != std::string::npos )
          lpConfigFile = arg.substr(pos+14);
        else if ( (pos = arg.find("--common-config-file=")) != std::string::npos )
          commonConfigFile = arg.substr(pos+20);
        else if ( (pos = arg.find("--config-dir=")) != std::string::npos )
          configDir = arg.substr(pos+13);
        else if ( (pos = arg.find("--resources-dir=")) != std::string::npos )
          resourcesPath = arg.substr(pos+16);
        else if ( (pos = arg.find("--language=")) != std::string::npos )
          langs.push_back(arg.substr(pos+11));
//         else if ( (pos = arg.find("--pipeline=")) != std::string::npos )
//           pipeline = arg.substr(pos+11);
        else if ( (pos = arg.find("--client=")) != std::string::npos )
          clientId=arg.substr(pos+9);
        else usage(argc, argv);
      }
      else
      {
        files.push_back(arg);
      }
    }
  }

  if (langs.size()<1)
  {
    std::cerr << "no language defined !" << std::endl;
    return -1;
  }

  AbstractLinguisticProcessingClient* client(0);

  try
  {

    // initialize common
    MediaticData::changeable().init(
      resourcesPath,
      configDir,
      commonConfigFile,
      langs);

    // initialize linguistic processing
    deque<string> pipelines;
    pipelines.push_back(pipeline);
    Lima::Common::XMLConfigurationFiles::XMLConfigurationFileParser lpconfig(configDir + "/" + lpConfigFile);
    LinguisticProcessingClientFactory::changeable().configureClientFactory(
      clientId,
      lpconfig,
      langs,
      pipelines);

    client=dynamic_cast<AbstractLinguisticProcessingClient*>(LinguisticProcessingClientFactory::single().createClient(clientId));

    // Set the handlers
    std::map<std::string, AbstractAnalysisHandler*> handlers;
    BowTextHandler bowTextHandler;
    handlers.insert(std::make_pair("bowTextHandler", &bowTextHandler));

    map<string,string> metaData;
    metaData["Lang"]=langs[0];
    MediaId lang=MediaticData::single().getMediaId(langs[0]);

    for (deque<string>::iterator fileItr=files.begin();
         fileItr!=files.end();
         fileItr++)
    {
      // open the output file
      ostringstream os;
      os << *fileItr << ".norm";
      ofstream fout(os.str().c_str(), std::ofstream::binary);

      // loading of the input file
      TimeUtils::updateCurrentTime();
      std::ifstream file(fileItr->c_str(), std::ifstream::binary);
      char buf[256];
      file.getline(buf,256);
      std::string line(buf);
      while (!file.eof())
      {
        if (line.size()==0)
        {
          file.getline(buf,256);
          line=string(buf);
          continue;
        }
        //        cout << "normalize " << line << endl;
        LimaString contentText;
        // The input text MUST be UTF-8 encoded !!!
        contentText = utf8stdstring2limastring(line);

        // analyze it
        metaData["FileName"]=*fileItr;

        client->analyze(contentText,metaData,pipeline,handlers);

        // analyze resulting bowText to extract normalization
        multimap<LimaString,string> norms=extractNormalization(contentText,bowTextHandler.getBowText(),lang);
        if (norms.empty())
        {
          norms.insert(make_pair(contentText,"NONE_1"));
        }
        for (multimap<LimaString,string>::iterator it=norms.begin();it!=norms.end();it++)
        {
          fout << limastring2utf8stdstring(it->first);
          if (printCategs)
          {
            fout << "#" << it->second;
          }
          fout << ";";
        }
        fout << endl;
        // read next line
        file.getline(buf,256);
        line=string(buf);
      }
    }
  }
  catch (InvalidConfiguration& e)
  {
    throw e;
  }

  delete client;
  return SUCCESS_ID;
}

multimap<LimaString,string> extractNormalization(const LimaString& source,const BoWText& bowText,MediaId lang)
{
  const Common::PropertyCode::PropertyManager& macroManager = static_cast<const Common::MediaticData::LanguageData&>(MediaticData::single().mediaData(lang)).getPropertyCodeManager().getPropertyManager("MACRO");
  multimap<LimaString,string> result;
  // si un seul bowtoken on le prend
  //  if (bowText.size()==1)
  //  {
  //    cerr << "- found only one norm : " << bowText.front()->getLemma() << endl;
  //    result.push_back(bowText.front()->getLemma());
  //  }
  // sinon on prend tous les bowtoken qui vont du d�ut �la fin
  //  else
  //  {
  //    cerr << "extractNormalisation : " << source << endl;
  for (BoWText::const_iterator bowItr=bowText.begin();
       bowItr!=bowText.end();
       bowItr++)
  {
    pair<int,int> posLen=getStartEnd(*bowItr);
    //      cerr << "  - " << (*bowItr)->getLemma() << " at " << posLen.first << "," << posLen.second;
    if ((posLen.first==1) && (posLen.second==int(source.size()+1)))
    {
      result.insert(make_pair(
                      (*bowItr)->getLemma(),
                      macroManager.getPropertySymbolicValue((*bowItr)->getCategory())));
      //        cerr << " keep it !";
    }
    //      cerr << endl;
  }
  //   }
  return result;
}

pair<int,int> getStartEnd(const BoWToken* tok)
{
  pair<int,int> res;
  if (tok->getType()==BOW_TOKEN)
  {
    res.first=tok->getPosition();
    res.second=tok->getPosition()+tok->getLength();
  }
  else
  {
    const BoWComplexToken* complextok=dynamic_cast<const BoWComplexToken*>(tok);
    if (complextok==0)
    {
      cerr << "ERROR ! complextok==0 ! should not happen !" << endl;
      exit(0);
    }
    const std::deque< BoWComplexToken::Part >& parts=complextok->getParts();
    if (parts.size()==0)
    {
      cerr << "ERROR ! complex token should have at least one part ! " << endl;
      exit(0);
    }
    std::deque< BoWComplexToken::Part >::const_iterator partItr=parts.begin();
    res=getStartEnd(partItr->get<1>());
    partItr++;
    for (;partItr!=parts.end();partItr++)
    {
      pair<int,int> tmp=getStartEnd(partItr->get<1>());
      if (tmp.first<res.first) res.first=tmp.first;
      if (tmp.second>res.second) res.second=tmp.second;
    }
  }
  return res;
}

void usage(int argc, char *argv[])
{
  LIMA_UNUSED(argc);
  std::cout << "usage: " << argv[0] << " [OPTIONS] [file1 [file2 [...]]] " << std::endl;
  std::cout << "\t--printCategs\t\tprint categories with normalizations" << std::endl;
  std::cout << "\t--resources-dir=</path/to/the/resources> Optional. Default is $LIMA_RESOURCES" << std::endl;
  std::cout << "\t--config-dir=</path/to/the/configuration/directory> Optional. Default is $LIMA_CONF" << std::endl;
  std::cout << "\t--lp-config-file=<configuration/file/name>\tOptional. Default is lima-analysis.xml" << std::endl;
  std::cout << "\t--common-config-file=<configuration/file/name>\tOptional. Default is lima-common.xml" << std::endl;
  std::cout << "\t--client=<clientId>\tOptional. Default is 'lima-coreclient'" << std::endl;
  std::cout << "\t--language=<language trigram>\tOptional. Language of document to analyze." << std::endl;
//   std::cout << "\t--pipeline=<pipelineId>\tOptional. Default is 'main'" << std::endl;
  std::cout << "\t--availableUnits\tshow all available resources, processUnits and dumpers" << std::endl;
  std::cout << "\twhere files are files to analyze." << std::endl;
  std::cout << endl;
  std::cout << "Available client factories are : " << std::endl;
  {
    deque<string> ids=LinguisticProcessingClientFactory::single().getRegisteredFactories();
    for (deque<string>::iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  exit(0);
}

void listunits()
{
  {
    cout << "Available resources factories : " << endl;
    deque<string> ids=AbstractResource::Factory::getRegisteredFactories();
    for (deque<string>::const_iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  {
    cout << "Available process units factories : " << endl;
    deque<string> ids=MediaProcessUnit::Factory::getRegisteredFactories();
    for (deque<string>::const_iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  std::cout << "Available client factories are : " << std::endl;
  {
    deque<string> ids=LinguisticProcessingClientFactory::single().getRegisteredFactories();
    for (deque<string>::iterator it=ids.begin();
         it!=ids.end();
         it++)
    {
      cout << "- " << *it << endl;
    }
    cout << endl;
  }
  exit(0);
}
<|endoftext|>