<commit_before>//
// This file is part of the Marble Desktop Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2009      Bastian Holst <bastianholst@gmx.de>
//

// Self
#include "AbstractDataPluginModel.h"

// Qt
#include <QtCore/QUrl>
#include <QtCore/QTimer>
#include <QtCore/QPointF>
#include <QtCore/QtAlgorithms>
#include <QtCore/QVariant>

// Marble
#include "MarbleDebug.h"
#include "AbstractDataPluginItem.h"
#include "CacheStoragePolicy.h"
#include "GeoDataCoordinates.h"
#include "GeoDataLatLonAltBox.h"
#include "HttpDownloadManager.h"
#include "MarbleDataFacade.h"
#include "MarbleDirs.h"
#include "ViewportParams.h"

#include <cmath>

namespace Marble
{

const QString descriptionPrefix( "description_" );

// Time between two tried description file downloads (we decided not to download anything) in ms
const int timeBetweenTriedDownloads = 500;
// Time between two real description file downloads in ms
const int timeBetweenDownloads = 1500;

// The factor describing how much the box has to be changed to download a new description file.
// A higher factor means more downloads.
const qreal boxComparisonFactor = 16.0;

// Separator to separate the id of the item from the file type
const char fileIdSeparator = '_';

class AbstractDataPluginModelPrivate
{
 public:
    AbstractDataPluginModelPrivate( const QString& name, AbstractDataPluginModel * parent )
        : m_parent( parent ),
          m_name( name ),
          m_lastBox(),
          m_downloadedBox(),
          m_lastNumber( 0 ),
          m_downloadedNumber( 0 ),
          m_lastDataFacade( 0 ),
          m_downloadTimer( new QTimer( m_parent ) ),
          m_descriptionFileNumber( 0 ),
          m_itemSettings(),
          m_downloadManager( 0 )
    {
    }

    ~AbstractDataPluginModelPrivate() {
        QList<AbstractDataPluginItem*>::iterator lIt = m_itemSet.begin();
        QList<AbstractDataPluginItem*>::iterator const lItEnd = m_itemSet.end();
        for (; lIt != lItEnd; ++lIt ) {
            (*lIt)->deleteLater();
        }

        QHash<QString,AbstractDataPluginItem*>::iterator hIt = m_downloadingItems.begin();
        QHash<QString,AbstractDataPluginItem*>::iterator const hItEnd = m_downloadingItems.end();
        for (; hIt != hItEnd; ++hIt ) {
            (*hIt)->deleteLater();
        }

        m_downloadManager->storagePolicy()->clearCache();
        delete m_downloadManager;
    }

    AbstractDataPluginModel *m_parent;
    QString m_name;
    GeoDataLatLonAltBox m_lastBox;
    GeoDataLatLonAltBox m_downloadedBox;
    qint32 m_lastNumber;
    qint32 m_downloadedNumber;
    MarbleDataFacade *m_lastDataFacade;
    QString m_downloadedTarget;
    QList<AbstractDataPluginItem*> m_itemSet;
    QHash<QString, AbstractDataPluginItem*> m_downloadingItems;
    QList<AbstractDataPluginItem*> m_displayedItems;
    QTimer *m_downloadTimer;
    quint32 m_descriptionFileNumber;
    QHash<QString, QVariant> m_itemSettings;

    HttpDownloadManager *m_downloadManager;
};

AbstractDataPluginModel::AbstractDataPluginModel( const QString& name, QObject *parent )
    : QObject(  parent ),
      d( new AbstractDataPluginModelPrivate( name, this ) )
{
    // Initializing file and download System
    CacheStoragePolicy *storagePolicy = new CacheStoragePolicy( MarbleDirs::localPath()
                                                                + "/cache/" + d->m_name + '/' );
    d->m_downloadManager = new HttpDownloadManager( storagePolicy );
    connect( d->m_downloadManager, SIGNAL( downloadComplete( QString, QString ) ),
             this,                 SLOT( processFinishedJob( QString , QString ) ) );

    // We want to download a new description file every timeBetweenDownloads ms
    connect( d->m_downloadTimer, SIGNAL( timeout() ),
             this,               SLOT( handleChangedViewport() ),
             Qt::QueuedConnection );
    d->m_downloadTimer->start( timeBetweenDownloads );
}

AbstractDataPluginModel::~AbstractDataPluginModel()
{
    delete d;
}

QList<AbstractDataPluginItem*> AbstractDataPluginModel::items( ViewportParams *viewport,
                                                               MarbleDataFacade *facade,
                                                               qint32 number )
{
    GeoDataLatLonAltBox currentBox = viewport->viewLatLonAltBox();
    QString target = facade->target();
    QList<AbstractDataPluginItem*> list;

    d->m_displayedItems.removeAll( 0 );

    QList<AbstractDataPluginItem*>::iterator i = d->m_displayedItems.begin();
    QList<AbstractDataPluginItem*>::iterator end = d->m_displayedItems.end();

    // Items that are already shown have the highest priority
    for (; i != end && list.size() < number; ++i ) {
        // Don't try to access an object that doesn't exist
        if( !*i ) {
            continue;
        }

        // Only show items that are initialized
        if( !(*i)->initialized() ) {
            continue;
        }

        // Only show items that are on the current planet
        if( (*i)->target() != target ) {
            continue;
        }

        if( !currentBox.contains( (*i)->coordinate() ) ) {
            continue;
        }

        // If the item was added initially at a nearer position, they don't have priority,
        // because we zoomed out since then.
        if( (*i)->addedAngularResolution() >= viewport->angularResolution() ) {
            list.append( *i );
            (*i)->setSettings( d->m_itemSettings );
        }
    }

    d->m_itemSet.removeAll( 0 );

    for ( i = d->m_itemSet.begin(), end = d->m_itemSet.end();
          i != end && list.size() < number; ++i )
    {
        // Don't try to access an object that doesn't exist
        if( !*i ) {
            mDebug() << "Warning: Null pointer in m_itemSet";
            continue;
        }

        // Only show items that are initialized
        if( !(*i)->initialized() ) {
            continue;
        }

        // Only show items that are on the current planet
        if( (*i)->target() != target ) {
            continue;
        }

        // If the item is on the viewport, we want to return it
        if( currentBox.contains( (*i)->coordinate() )
            && !list.contains( *i ) )
        {
            list.append( *i );
            (*i)->setSettings( d->m_itemSettings );

            // We want to save the angular resolution of the first time the item got added.
            // If it is in the list of displayedItems, it was added before
            if( !d->m_displayedItems.contains( *i ) ) {
                (*i)->setAddedAngularResolution( viewport->angularResolution() );
            }
        }
        // FIXME: We have to do something if the item that is not on the viewport.
    }

    d->m_lastDataFacade = facade;

    if( (!(currentBox == d->m_lastBox)
          || number != d->m_lastNumber ) )
    {
        d->m_lastBox = currentBox;
        d->m_lastNumber = number;
        d->m_lastDataFacade = facade;
    }
    else {
    }

    d->m_displayedItems = list;
    return list;
}

QList<AbstractDataPluginItem *> AbstractDataPluginModel::whichItemAt( const QPoint& curpos )
{
    QList<AbstractDataPluginItem *> itemsAt;

    foreach( AbstractDataPluginItem* item, d->m_displayedItems ) {
        if( item && item->contains( QPointF( curpos ) ) )
            itemsAt.append( item );
    }

    return itemsAt;
}

void AbstractDataPluginModel::parseFile( const QByteArray& file )
{
    Q_UNUSED( file );
}

void AbstractDataPluginModel::downloadItemData( const QUrl& url,
                                                const QString& type,
                                                AbstractDataPluginItem *item )
{
    if( !item ) {
        return;
    }

    QString id = generateFilename( item->id(), type );

    d->m_downloadManager->addJob( url, id, id, DownloadBrowse );
    d->m_downloadingItems.insert( id, item );

    connect( item, SIGNAL( destroyed( QObject* ) ), this, SLOT( removeItem( QObject* ) ) );

    addItemToList( item );
}

void AbstractDataPluginModel::downloadDescriptionFile( const QUrl& url )
{
    if( !url.isEmpty() ) {
        QString name( descriptionPrefix );
        name += QString::number( d->m_descriptionFileNumber );

        d->m_downloadManager->addJob( url, name, name, DownloadBrowse );
        d->m_descriptionFileNumber++;
    }
}

static bool lessThanByPointer( const AbstractDataPluginItem *item1,
                               const AbstractDataPluginItem *item2 )
{
    if( item1 != 0 && item2 != 0 ) {
        return item1->operator<( item2 );
    }
    else {
        return false;
    }
}

void AbstractDataPluginModel::addItemToList( AbstractDataPluginItem *item )
{
    if( !item ) {
        return;
    }

    // If the item is already in our list, don't add it.
    if( AbstractDataPluginItem *oldItem = findItem( item->id() ) ) {
        if ( oldItem == item ) {
            return;
        }
        else {
            item->deleteLater();
            return;
        }
    }

    mDebug() << "New item " << item->id();

    // This find the right position in the sorted to insert the new item
    QList<AbstractDataPluginItem*>::iterator i = qLowerBound( d->m_itemSet.begin(),
                                                                d->m_itemSet.end(),
                                                                item,
                                                                lessThanByPointer );
    // Insert the item on the right position in the list
    d->m_itemSet.insert( i, item );

    connect( item, SIGNAL( destroyed( QObject* ) ), this, SLOT( removeItem( QObject* ) ) );
    connect( item, SIGNAL( updated() ), this, SIGNAL( itemsUpdated() ) );

    if ( item->initialized() ) {
        emit itemsUpdated();
    }
}

QString AbstractDataPluginModel::name() const
{
    return d->m_name;
}

void AbstractDataPluginModel::setName( const QString& name )
{
    d->m_name = name;
}

QString AbstractDataPluginModel::generateFilename( const QString& id, const QString& type ) const
{
    QString name;
    name += id;
    name += fileIdSeparator;
    name += type;

    return name;
}

QString AbstractDataPluginModel::generateFilepath( const QString& id, const QString& type ) const
{
    return MarbleDirs::localPath() + "/cache/" + d->m_name + '/' + generateFilename( id, type );
}

bool AbstractDataPluginModel::fileExists( const QString& fileName ) const
{
    return d->m_downloadManager->storagePolicy()->fileExists( fileName );
}

bool AbstractDataPluginModel::fileExists( const QString& id, const QString& type ) const
{
    return fileExists( generateFilename( id, type ) );
}

AbstractDataPluginItem *AbstractDataPluginModel::findItem( const QString& id ) const
{
    QList<AbstractDataPluginItem*>::iterator listIt = d->m_itemSet.begin();
    QList<AbstractDataPluginItem*>::iterator const end = d->m_itemSet.end();

    for (; listIt != end; ++listIt ) {
        if( (*listIt)->id() == id ) {
            return (*listIt);
        }
    }

    return 0;
}

bool AbstractDataPluginModel::itemExists( const QString& id ) const
{
    if ( findItem( id ) ) {
        return true;
    }
    else {
        return false;
    }
}

void AbstractDataPluginModel::setItemSettings( QHash<QString,QVariant> itemSettings )
{
    d->m_itemSettings = itemSettings;
}

void AbstractDataPluginModel::handleChangedViewport()
{
    if( !d->m_lastDataFacade ) {
        return;
    }

    // All this is to prevent to often downloads
    if( d->m_lastNumber != 0
        // We don't need to download if nothing changed
        && ( !( d->m_downloadedBox == d->m_lastBox )
             || d->m_downloadedNumber != d->m_lastNumber
             || d->m_downloadedTarget != d->m_lastDataFacade->target() )
        // We try to filter little changes of the bounding box
        && ( fabs( d->m_downloadedBox.east() - d->m_lastBox.east() ) * boxComparisonFactor
                                > d->m_lastBox.width()
             || fabs( d->m_downloadedBox.south() - d->m_lastBox.south() ) * boxComparisonFactor
                                > d->m_lastBox.height()
             || fabs( d->m_downloadedBox.north() - d->m_lastBox.north() ) * boxComparisonFactor
                                > d->m_lastBox.height()
             || fabs( d->m_downloadedBox.west() - d->m_lastBox.west() ) * boxComparisonFactor
                                > d->m_lastBox.width() ) )
    {
        // We will wait a little bit longer to start the the
        // next download as we will really download something now.
        d->m_downloadTimer->setInterval( timeBetweenDownloads );

        // Save the download parameter
        d->m_downloadedBox = d->m_lastBox;
        d->m_downloadedNumber = d->m_lastNumber;
        d->m_downloadedTarget = d->m_lastDataFacade->target();

        // Get items
        getAdditionalItems( d->m_lastBox, d->m_lastDataFacade, d->m_lastNumber );
    }
    else {
        // Don't wait to long to start the next download as we decided not to download anything.
        // This will enhance response.
        d->m_downloadTimer->setInterval( timeBetweenTriedDownloads );
    }
}

void AbstractDataPluginModel::processFinishedJob( const QString& relativeUrlString,
                                                  const QString& id )
{
    Q_UNUSED( relativeUrlString );

    if( id.startsWith( descriptionPrefix ) ) {
        CacheStoragePolicy *storagePolicy
                = qobject_cast<CacheStoragePolicy*>( d->m_downloadManager->storagePolicy() );
        if ( storagePolicy ) {
            parseFile( storagePolicy->data( id ) );
        }
    }
    else {
        // The downloaded file contains item data.

        // Splitting the id in itemId and fileType
        QStringList fileInformation = id.split( fileIdSeparator );

        if( fileInformation.size() < 2) {
            mDebug() << "Strange file information " << id;
            return;
        }
        QString itemId = fileInformation.at( 0 );
        fileInformation.removeAt( 0 );
        QString fileType = fileInformation.join( QString( fileIdSeparator ) );

        // Searching for the right item in m_downloadingItems
        QHash<QString, AbstractDataPluginItem *>::iterator i = d->m_downloadingItems.find( id );
        if( i != d->m_downloadingItems.end() ) {
            if( itemId != (*i)->id() ) {
                return;
            }

            (*i)->addDownloadedFile( generateFilepath( itemId, fileType ),
                                     fileType );

            d->m_downloadingItems.erase( i );
        }
    }
}

void AbstractDataPluginModel::removeItem( QObject *item )
{
    d->m_itemSet.removeAll( (AbstractDataPluginItem *) item );
    QHash<QString, AbstractDataPluginItem *>::iterator i;
    for( i = d->m_downloadingItems.begin(); i != d->m_downloadingItems.end(); ++i ) {
        if( (*i) == (AbstractDataPluginItem *) item ) {
            d->m_downloadingItems.erase( i );
        }
    }
}

} // namespace Marble

#include "AbstractDataPluginModel.moc"
<commit_msg>AbstractDataPluginModel::itemExists() simplified.<commit_after>//
// This file is part of the Marble Desktop Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2009      Bastian Holst <bastianholst@gmx.de>
//

// Self
#include "AbstractDataPluginModel.h"

// Qt
#include <QtCore/QUrl>
#include <QtCore/QTimer>
#include <QtCore/QPointF>
#include <QtCore/QtAlgorithms>
#include <QtCore/QVariant>

// Marble
#include "MarbleDebug.h"
#include "AbstractDataPluginItem.h"
#include "CacheStoragePolicy.h"
#include "GeoDataCoordinates.h"
#include "GeoDataLatLonAltBox.h"
#include "HttpDownloadManager.h"
#include "MarbleDataFacade.h"
#include "MarbleDirs.h"
#include "ViewportParams.h"

#include <cmath>

namespace Marble
{

const QString descriptionPrefix( "description_" );

// Time between two tried description file downloads (we decided not to download anything) in ms
const int timeBetweenTriedDownloads = 500;
// Time between two real description file downloads in ms
const int timeBetweenDownloads = 1500;

// The factor describing how much the box has to be changed to download a new description file.
// A higher factor means more downloads.
const qreal boxComparisonFactor = 16.0;

// Separator to separate the id of the item from the file type
const char fileIdSeparator = '_';

class AbstractDataPluginModelPrivate
{
 public:
    AbstractDataPluginModelPrivate( const QString& name, AbstractDataPluginModel * parent )
        : m_parent( parent ),
          m_name( name ),
          m_lastBox(),
          m_downloadedBox(),
          m_lastNumber( 0 ),
          m_downloadedNumber( 0 ),
          m_lastDataFacade( 0 ),
          m_downloadTimer( new QTimer( m_parent ) ),
          m_descriptionFileNumber( 0 ),
          m_itemSettings(),
          m_downloadManager( 0 )
    {
    }

    ~AbstractDataPluginModelPrivate() {
        QList<AbstractDataPluginItem*>::iterator lIt = m_itemSet.begin();
        QList<AbstractDataPluginItem*>::iterator const lItEnd = m_itemSet.end();
        for (; lIt != lItEnd; ++lIt ) {
            (*lIt)->deleteLater();
        }

        QHash<QString,AbstractDataPluginItem*>::iterator hIt = m_downloadingItems.begin();
        QHash<QString,AbstractDataPluginItem*>::iterator const hItEnd = m_downloadingItems.end();
        for (; hIt != hItEnd; ++hIt ) {
            (*hIt)->deleteLater();
        }

        m_downloadManager->storagePolicy()->clearCache();
        delete m_downloadManager;
    }

    AbstractDataPluginModel *m_parent;
    QString m_name;
    GeoDataLatLonAltBox m_lastBox;
    GeoDataLatLonAltBox m_downloadedBox;
    qint32 m_lastNumber;
    qint32 m_downloadedNumber;
    MarbleDataFacade *m_lastDataFacade;
    QString m_downloadedTarget;
    QList<AbstractDataPluginItem*> m_itemSet;
    QHash<QString, AbstractDataPluginItem*> m_downloadingItems;
    QList<AbstractDataPluginItem*> m_displayedItems;
    QTimer *m_downloadTimer;
    quint32 m_descriptionFileNumber;
    QHash<QString, QVariant> m_itemSettings;

    HttpDownloadManager *m_downloadManager;
};

AbstractDataPluginModel::AbstractDataPluginModel( const QString& name, QObject *parent )
    : QObject(  parent ),
      d( new AbstractDataPluginModelPrivate( name, this ) )
{
    // Initializing file and download System
    CacheStoragePolicy *storagePolicy = new CacheStoragePolicy( MarbleDirs::localPath()
                                                                + "/cache/" + d->m_name + '/' );
    d->m_downloadManager = new HttpDownloadManager( storagePolicy );
    connect( d->m_downloadManager, SIGNAL( downloadComplete( QString, QString ) ),
             this,                 SLOT( processFinishedJob( QString , QString ) ) );

    // We want to download a new description file every timeBetweenDownloads ms
    connect( d->m_downloadTimer, SIGNAL( timeout() ),
             this,               SLOT( handleChangedViewport() ),
             Qt::QueuedConnection );
    d->m_downloadTimer->start( timeBetweenDownloads );
}

AbstractDataPluginModel::~AbstractDataPluginModel()
{
    delete d;
}

QList<AbstractDataPluginItem*> AbstractDataPluginModel::items( ViewportParams *viewport,
                                                               MarbleDataFacade *facade,
                                                               qint32 number )
{
    GeoDataLatLonAltBox currentBox = viewport->viewLatLonAltBox();
    QString target = facade->target();
    QList<AbstractDataPluginItem*> list;

    d->m_displayedItems.removeAll( 0 );

    QList<AbstractDataPluginItem*>::iterator i = d->m_displayedItems.begin();
    QList<AbstractDataPluginItem*>::iterator end = d->m_displayedItems.end();

    // Items that are already shown have the highest priority
    for (; i != end && list.size() < number; ++i ) {
        // Don't try to access an object that doesn't exist
        if( !*i ) {
            continue;
        }

        // Only show items that are initialized
        if( !(*i)->initialized() ) {
            continue;
        }

        // Only show items that are on the current planet
        if( (*i)->target() != target ) {
            continue;
        }

        if( !currentBox.contains( (*i)->coordinate() ) ) {
            continue;
        }

        // If the item was added initially at a nearer position, they don't have priority,
        // because we zoomed out since then.
        if( (*i)->addedAngularResolution() >= viewport->angularResolution() ) {
            list.append( *i );
            (*i)->setSettings( d->m_itemSettings );
        }
    }

    d->m_itemSet.removeAll( 0 );

    for ( i = d->m_itemSet.begin(), end = d->m_itemSet.end();
          i != end && list.size() < number; ++i )
    {
        // Don't try to access an object that doesn't exist
        if( !*i ) {
            mDebug() << "Warning: Null pointer in m_itemSet";
            continue;
        }

        // Only show items that are initialized
        if( !(*i)->initialized() ) {
            continue;
        }

        // Only show items that are on the current planet
        if( (*i)->target() != target ) {
            continue;
        }

        // If the item is on the viewport, we want to return it
        if( currentBox.contains( (*i)->coordinate() )
            && !list.contains( *i ) )
        {
            list.append( *i );
            (*i)->setSettings( d->m_itemSettings );

            // We want to save the angular resolution of the first time the item got added.
            // If it is in the list of displayedItems, it was added before
            if( !d->m_displayedItems.contains( *i ) ) {
                (*i)->setAddedAngularResolution( viewport->angularResolution() );
            }
        }
        // FIXME: We have to do something if the item that is not on the viewport.
    }

    d->m_lastDataFacade = facade;

    if( (!(currentBox == d->m_lastBox)
          || number != d->m_lastNumber ) )
    {
        d->m_lastBox = currentBox;
        d->m_lastNumber = number;
        d->m_lastDataFacade = facade;
    }
    else {
    }

    d->m_displayedItems = list;
    return list;
}

QList<AbstractDataPluginItem *> AbstractDataPluginModel::whichItemAt( const QPoint& curpos )
{
    QList<AbstractDataPluginItem *> itemsAt;

    foreach( AbstractDataPluginItem* item, d->m_displayedItems ) {
        if( item && item->contains( QPointF( curpos ) ) )
            itemsAt.append( item );
    }

    return itemsAt;
}

void AbstractDataPluginModel::parseFile( const QByteArray& file )
{
    Q_UNUSED( file );
}

void AbstractDataPluginModel::downloadItemData( const QUrl& url,
                                                const QString& type,
                                                AbstractDataPluginItem *item )
{
    if( !item ) {
        return;
    }

    QString id = generateFilename( item->id(), type );

    d->m_downloadManager->addJob( url, id, id, DownloadBrowse );
    d->m_downloadingItems.insert( id, item );

    connect( item, SIGNAL( destroyed( QObject* ) ), this, SLOT( removeItem( QObject* ) ) );

    addItemToList( item );
}

void AbstractDataPluginModel::downloadDescriptionFile( const QUrl& url )
{
    if( !url.isEmpty() ) {
        QString name( descriptionPrefix );
        name += QString::number( d->m_descriptionFileNumber );

        d->m_downloadManager->addJob( url, name, name, DownloadBrowse );
        d->m_descriptionFileNumber++;
    }
}

static bool lessThanByPointer( const AbstractDataPluginItem *item1,
                               const AbstractDataPluginItem *item2 )
{
    if( item1 != 0 && item2 != 0 ) {
        return item1->operator<( item2 );
    }
    else {
        return false;
    }
}

void AbstractDataPluginModel::addItemToList( AbstractDataPluginItem *item )
{
    if( !item ) {
        return;
    }

    // If the item is already in our list, don't add it.
    if( AbstractDataPluginItem *oldItem = findItem( item->id() ) ) {
        if ( oldItem == item ) {
            return;
        }
        else {
            item->deleteLater();
            return;
        }
    }

    mDebug() << "New item " << item->id();

    // This find the right position in the sorted to insert the new item
    QList<AbstractDataPluginItem*>::iterator i = qLowerBound( d->m_itemSet.begin(),
                                                                d->m_itemSet.end(),
                                                                item,
                                                                lessThanByPointer );
    // Insert the item on the right position in the list
    d->m_itemSet.insert( i, item );

    connect( item, SIGNAL( destroyed( QObject* ) ), this, SLOT( removeItem( QObject* ) ) );
    connect( item, SIGNAL( updated() ), this, SIGNAL( itemsUpdated() ) );

    if ( item->initialized() ) {
        emit itemsUpdated();
    }
}

QString AbstractDataPluginModel::name() const
{
    return d->m_name;
}

void AbstractDataPluginModel::setName( const QString& name )
{
    d->m_name = name;
}

QString AbstractDataPluginModel::generateFilename( const QString& id, const QString& type ) const
{
    QString name;
    name += id;
    name += fileIdSeparator;
    name += type;

    return name;
}

QString AbstractDataPluginModel::generateFilepath( const QString& id, const QString& type ) const
{
    return MarbleDirs::localPath() + "/cache/" + d->m_name + '/' + generateFilename( id, type );
}

bool AbstractDataPluginModel::fileExists( const QString& fileName ) const
{
    return d->m_downloadManager->storagePolicy()->fileExists( fileName );
}

bool AbstractDataPluginModel::fileExists( const QString& id, const QString& type ) const
{
    return fileExists( generateFilename( id, type ) );
}

AbstractDataPluginItem *AbstractDataPluginModel::findItem( const QString& id ) const
{
    QList<AbstractDataPluginItem*>::iterator listIt = d->m_itemSet.begin();
    QList<AbstractDataPluginItem*>::iterator const end = d->m_itemSet.end();

    for (; listIt != end; ++listIt ) {
        if( (*listIt)->id() == id ) {
            return (*listIt);
        }
    }

    return 0;
}

bool AbstractDataPluginModel::itemExists( const QString& id ) const
{
    return findItem( id );
}

void AbstractDataPluginModel::setItemSettings( QHash<QString,QVariant> itemSettings )
{
    d->m_itemSettings = itemSettings;
}

void AbstractDataPluginModel::handleChangedViewport()
{
    if( !d->m_lastDataFacade ) {
        return;
    }

    // All this is to prevent to often downloads
    if( d->m_lastNumber != 0
        // We don't need to download if nothing changed
        && ( !( d->m_downloadedBox == d->m_lastBox )
             || d->m_downloadedNumber != d->m_lastNumber
             || d->m_downloadedTarget != d->m_lastDataFacade->target() )
        // We try to filter little changes of the bounding box
        && ( fabs( d->m_downloadedBox.east() - d->m_lastBox.east() ) * boxComparisonFactor
                                > d->m_lastBox.width()
             || fabs( d->m_downloadedBox.south() - d->m_lastBox.south() ) * boxComparisonFactor
                                > d->m_lastBox.height()
             || fabs( d->m_downloadedBox.north() - d->m_lastBox.north() ) * boxComparisonFactor
                                > d->m_lastBox.height()
             || fabs( d->m_downloadedBox.west() - d->m_lastBox.west() ) * boxComparisonFactor
                                > d->m_lastBox.width() ) )
    {
        // We will wait a little bit longer to start the the
        // next download as we will really download something now.
        d->m_downloadTimer->setInterval( timeBetweenDownloads );

        // Save the download parameter
        d->m_downloadedBox = d->m_lastBox;
        d->m_downloadedNumber = d->m_lastNumber;
        d->m_downloadedTarget = d->m_lastDataFacade->target();

        // Get items
        getAdditionalItems( d->m_lastBox, d->m_lastDataFacade, d->m_lastNumber );
    }
    else {
        // Don't wait to long to start the next download as we decided not to download anything.
        // This will enhance response.
        d->m_downloadTimer->setInterval( timeBetweenTriedDownloads );
    }
}

void AbstractDataPluginModel::processFinishedJob( const QString& relativeUrlString,
                                                  const QString& id )
{
    Q_UNUSED( relativeUrlString );

    if( id.startsWith( descriptionPrefix ) ) {
        CacheStoragePolicy *storagePolicy
                = qobject_cast<CacheStoragePolicy*>( d->m_downloadManager->storagePolicy() );
        if ( storagePolicy ) {
            parseFile( storagePolicy->data( id ) );
        }
    }
    else {
        // The downloaded file contains item data.

        // Splitting the id in itemId and fileType
        QStringList fileInformation = id.split( fileIdSeparator );

        if( fileInformation.size() < 2) {
            mDebug() << "Strange file information " << id;
            return;
        }
        QString itemId = fileInformation.at( 0 );
        fileInformation.removeAt( 0 );
        QString fileType = fileInformation.join( QString( fileIdSeparator ) );

        // Searching for the right item in m_downloadingItems
        QHash<QString, AbstractDataPluginItem *>::iterator i = d->m_downloadingItems.find( id );
        if( i != d->m_downloadingItems.end() ) {
            if( itemId != (*i)->id() ) {
                return;
            }

            (*i)->addDownloadedFile( generateFilepath( itemId, fileType ),
                                     fileType );

            d->m_downloadingItems.erase( i );
        }
    }
}

void AbstractDataPluginModel::removeItem( QObject *item )
{
    d->m_itemSet.removeAll( (AbstractDataPluginItem *) item );
    QHash<QString, AbstractDataPluginItem *>::iterator i;
    for( i = d->m_downloadingItems.begin(); i != d->m_downloadingItems.end(); ++i ) {
        if( (*i) == (AbstractDataPluginItem *) item ) {
            d->m_downloadingItems.erase( i );
        }
    }
}

} // namespace Marble

#include "AbstractDataPluginModel.moc"
<|endoftext|>


<commit_before>//
//  main.cpp
//  ProcessDispatch
//
//  Created by Nero Zuo on 16/11/9.
//  Copyright © 2016年 Nero. All rights reserved.
//

#include <iostream>
#include <vector>
#include <algorithm>
#include <string>
#include <fstream>
#define PROCESS_NAME_LEN 32

#include <chrono>
#include <thread>

using namespace std;

class Process {
public:
    string name;
    int arrive;
    int duration;
    int priority;
    int waitingTime;
    int turnoverTime;
};

bool operator==(Process const &p1 ,Process const & p2) {
    return p1.name == p2.name;
}

bool sjfCpmpare(const Process &p1, const Process &p2) {
    return p1.duration < p2.duration;
}

bool sjfCpmparePoint(const Process *p1, const Process *p2) {
    return p1->duration < p2->duration;
}

bool roundRobinComparePoint(const Process *p1, const Process *p2) {
    return p1->arrive < p2->arrive;
}

bool priorityComparePoint(const Process *p1, const Process *p2) {
    return p1->priority > p2->priority;
}

class ProcessesManager {
private:

    int timeNow;
public:
    vector<Process> processes;

    vector<Process> finishedProcesses;
    ProcessesManager(vector<Process> processes) {
        this->processes = processes;
        this->finishedProcesses = {};
        timeNow = 0;
    }

    void run(Process *process, int duration) {
        if (process->duration - duration < 0) {
            duration = process->duration;
        }
        process->duration = process->duration - duration;
        process->waitingTime += (timeNow-process->arrive);
        process->turnoverTime += (timeNow-process->arrive + duration);
        timeNow += duration;
        process->arrive = timeNow;
        Process p = *process;
        if(process->duration <= 0) {
            finishedProcesses.push_back(p);
            remove(processes.begin(), processes.end(), *process);
            processes.pop_back();
        }
        cout << "run " + p.name + " form " + to_string(timeNow-duration) + " to " + to_string(timeNow) + " for " + to_string(duration) << endl;
    }

    vector<Process *> rightNowProcesses() {
        vector<Process *> rightNowProcess;
        for (std::vector<Process>::size_type iter = 0; iter!=processes.size(); iter++) {

            if (processes[iter].arrive <= timeNow) {
                rightNowProcess.push_back(&processes[iter]);
            }
        }

        return rightNowProcess;
    }

    void sjf() {
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), sjfCpmparePoint);
            run(toRunProcess[0], toRunProcess[0]->duration);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

    void roundRobin() {
        vector<int> timeSlice = {1,2,3};
        int timeSliceIter = 0;
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            int timeToRun = timeSlice[timeSliceIter];
//            this_thread::sleep_for(std::chrono::milliseconds(timeToRun*1000));
            timeSliceIter = (timeSliceIter + 1)%timeSlice.size();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), roundRobinComparePoint);
            run(toRunProcess[0], timeToRun);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

    void highPriorityFirst() {
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), priorityComparePoint);
            run(toRunProcess[0], toRunProcess[0]->duration);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

};

void splitString(const std::string& s, std::vector<std::string>& v, const std::string& c)
{
    std::string::size_type pos1, pos2;
    pos2 = s.find(c);
    pos1 = 0;
    while(std::string::npos != pos2)
    {
        v.push_back(s.substr(pos1, pos2-pos1));

        pos1 = pos2 + c.size();
        pos2 = s.find(c, pos1);
    }
    if(pos1 != s.length())
        v.push_back(s.substr(pos1));
}

int main(int argc, const char * argv[]) {

    vector<Process> processes = {{"p1", 0, 7, 5, 0, 0}, {"p2", 1, 1, 1, 0, 0}, {"p3", 1, 3, 4, 0 ,0}, {"p4", 2, 5, 3, 0, 0}, {"p5", 4, 4, 2, 0, 0}};
    ProcessesManager manager = *new ProcessesManager(processes);
//    manager.run(&manager.processes[0], 10);
//    cout << manager.processes[4].name << endl;
//    manager.rightNowProcesses();

    vector<std::string> str;
    splitString("h\ne\nl\nl\no", str, "\n");

    manager.highPriorityFirst();
    return 0;
}
<commit_msg>Add read file;<commit_after>//
//  main.cpp
//  ProcessDispatch
//
//  Created by Nero Zuo on 16/11/9.
//  Copyright © 2016年 Nero. All rights reserved.
//

#include <iostream>
#include <vector>
#include <algorithm>
#include <string>
#include <fstream>
#define PROCESS_NAME_LEN 32

#include <chrono>
#include <thread>

using namespace std;

class Process {
public:
    string name;
    int arrive;
    int duration;
    int priority;
    int waitingTime;
    int turnoverTime;

    Process(vector<string> processInfo) {
        name = processInfo[0];
        arrive = stoi(processInfo[1]);
        duration = stoi(processInfo[2]);
        priority = stoi(processInfo[3]);
        waitingTime = 0;
        turnoverTime = 0;
    }
};

bool operator==(Process const &p1 ,Process const & p2) {
    return p1.name == p2.name;
}

bool sjfCpmpare(const Process &p1, const Process &p2) {
    return p1.duration < p2.duration;
}

bool sjfCpmparePoint(const Process *p1, const Process *p2) {
    return p1->duration < p2->duration;
}

bool roundRobinComparePoint(const Process *p1, const Process *p2) {
    return p1->arrive < p2->arrive;
}

bool priorityComparePoint(const Process *p1, const Process *p2) {
    return p1->priority > p2->priority;
}

class ProcessesManager {
private:

    int timeNow;
public:
    vector<Process> processes;

    vector<Process> finishedProcesses;
    ProcessesManager(vector<Process> processes) {
        this->processes = processes;
        this->finishedProcesses = {};
        timeNow = 0;
    }

    void run(Process *process, int duration) {
        if (process->duration - duration < 0) {
            duration = process->duration;
        }
        process->duration = process->duration - duration;
        process->waitingTime += (timeNow-process->arrive);
        process->turnoverTime += (timeNow-process->arrive + duration);
        timeNow += duration;
        process->arrive = timeNow;
        Process p = *process;
        if(process->duration <= 0) {
            finishedProcesses.push_back(p);
            remove(processes.begin(), processes.end(), *process);
            processes.pop_back();
        }
        cout << "run " + p.name + " form " + to_string(timeNow-duration) + " to " + to_string(timeNow) + " for " + to_string(duration) << endl;
    }

    vector<Process *> rightNowProcesses() {
        vector<Process *> rightNowProcess;
        for (std::vector<Process>::size_type iter = 0; iter!=processes.size(); iter++) {

            if (processes[iter].arrive <= timeNow) {
                rightNowProcess.push_back(&processes[iter]);
            }
        }

        return rightNowProcess;
    }

    void sjf() {
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), sjfCpmparePoint);
            run(toRunProcess[0], toRunProcess[0]->duration);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

    void roundRobin() {
        vector<int> timeSlice = {1,2,3};
        int timeSliceIter = 0;
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            int timeToRun = timeSlice[timeSliceIter];
//            this_thread::sleep_for(std::chrono::milliseconds(timeToRun*1000));
            timeSliceIter = (timeSliceIter + 1)%timeSlice.size();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), roundRobinComparePoint);
            run(toRunProcess[0], timeToRun);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

    void highPriorityFirst() {
        while (processes.size() != 0) {
            vector<Process *> toRunProcess = rightNowProcesses();
            if (toRunProcess.size() == 0) {
                timeNow += 1;
                continue;
            }
            sort(toRunProcess.begin(), toRunProcess.end(), priorityComparePoint);
            run(toRunProcess[0], toRunProcess[0]->duration);
        }
        for (std::vector<Process>::size_type iter = 0; iter!=finishedProcesses.size(); iter++) {
            Process process = finishedProcesses[iter];
            cout << process.name + " waiting time " + to_string(process.waitingTime) + "| turnover time " + to_string(process.turnoverTime) << endl;
        }
    }

};

void splitString(const std::string& s, std::vector<std::string>& v, const std::string& c)
{
    std::string::size_type pos1, pos2;
    pos2 = s.find(c);
    pos1 = 0;
    while(std::string::npos != pos2)
    {
        v.push_back(s.substr(pos1, pos2-pos1));

        pos1 = pos2 + c.size();
        pos2 = s.find(c, pos1);
    }
    if(pos1 != s.length())
        v.push_back(s.substr(pos1));
}

vector<Process> readFile() {
    ifstream file;
    string line;
    vector<Process> processes = {};
    file.open("/Users/Nero/Desktop/process.txt");
    if (file.is_open()) {
        while (getline(file, line)) {
            vector<string> processInfo;
            splitString(line, processInfo, ",");
            Process process = Process(processInfo);
            processes.push_back(process);
        }
    }
    file.close();
    return processes;
}

int main(int argc, const char * argv[]) {

//    vector<Process> processes = {{"p1", 0, 7, 5, 0, 0}, {"p2", 1, 1, 1, 0, 0}, {"p3", 1, 3, 4, 0 ,0}, {"p4", 2, 5, 3, 0, 0}, {"p5", 4, 4, 2, 0, 0}};

    vector<std::string> str;
    splitString("h\ne\nl\nl\no", str, "\n");

    vector<Process> processes = readFile();
    ProcessesManager manager = *new ProcessesManager(processes);
    manager.highPriorityFirst();
    return 0;
}
<|endoftext|>


<commit_before>#include "caffe2/operators/spatial_batch_norm_op.h"

namespace caffe2 {

template <>
bool SpatialBNOp<CPUContext>::RunOnDevice() {
  const auto& X = Input(INPUT);
  const auto& scale = Input(SCALE);
  const auto& bias = Input(BIAS);

  CAFFE_ENFORCE(X.ndim() >= 3 && X.ndim() <= 5);
  const int N = X.dim32(0);
  const int C =
      (order_ == StorageOrder::NCHW ? X.dim32(1) : X.dim32(X.ndim() - 1));
  const int H = (order_ == StorageOrder::NCHW ? X.dim32(2) : X.dim32(1));
  const int W = X.ndim() > 3
      ? (order_ == StorageOrder::NCHW ? X.dim32(3) : X.dim32(2))
      : 1;
  const int D = X.ndim() > 4
      ? (order_ == StorageOrder::NCHW ? X.dim32(4) : X.dim32(3))
      : 1;

  const int sample_size = H * W * D;
  CAFFE_ENFORCE_EQ(scale.ndim(), 1);
  CAFFE_ENFORCE_EQ(bias.ndim(), 1);
  CAFFE_ENFORCE_EQ(scale.dim32(0), C);
  CAFFE_ENFORCE_EQ(bias.dim32(0), C);

  ConstEigenVectorArrayMap<float> scale_arr(scale.data<float>(), C);
  ConstEigenVectorArrayMap<float> bias_arr(bias.data<float>(), C);

  auto* Y = Output(OUTPUT);
  Y->ResizeLike(X);

  if (!is_test_) {
    // training mode
    // Get the mean and variance.
    // Note that, to be consistent with cudnn, we will output saved inverse
    // std as output 5, but we will still use the same storage place to
    // compute var as well. The inverse is going to be carried out at the end
    // of the op.
    Output(SAVED_MEAN)->Resize(C);
    Output(SAVED_INV_VAR)->Resize(C);
    EigenVectorArrayMap<float> mean(
        Output(SAVED_MEAN)->mutable_data<float>(), C);
    EigenVectorArrayMap<float> var(
        Output(SAVED_INV_VAR)->mutable_data<float>(), C);

    mean.setZero();
    var.setZero();
    switch (order_) {
      case StorageOrder::NCHW: {
        ConstEigenArrayMap<float> X_arr(X.data<float>(), sample_size, N * C);
        for (int nc = 0; nc < N * C; ++nc) {
          mean(nc % C) += X_arr.col(nc).sum();
        }
        mean /= N * sample_size;
        for (int nc = 0; nc < N * C; ++nc) {
          var(nc % C) += (X_arr.col(nc) - mean(nc % C)).matrix().squaredNorm();
        }
        var /= N * sample_size;
        break;
      }
      case StorageOrder::NHWC: {
        ConstEigenArrayMap<float> X_arr(X.data<float>(), C, N * sample_size);
        for (int i = 0; i < N * sample_size; ++i) {
          mean += X_arr.col(i);
        }
        mean /= N * sample_size;
        for (int i = 0; i < N * sample_size; ++i) {
          var += (X_arr.col(i) - mean) * (X_arr.col(i) - mean);
        }
        var /= N * sample_size;
        break;
      }
      default:
        CAFFE_THROW("Unknown storage order: ", order_);
    }

    // Compute the running mean and running inv variance.
    auto* running_mean = Output(RUNNING_MEAN);
    auto* running_var = Output(RUNNING_VAR);
    // Check if they are initialized
    if (!running_mean->size()) {
      running_mean->Resize(C);
      EigenVectorArrayMap<float> running_mean_map(
          running_mean->mutable_data<float>(), C);
      running_mean_map.setZero();
    }
    if (!running_var->size()) {
      running_var->Resize(C);
      EigenVectorArrayMap<float> running_var_map(
          running_var->mutable_data<float>(), C);
      running_var_map.setZero();
    }
    EigenVectorArrayMap<float> running_mean_arr(
        running_mean->mutable_data<float>(), C);
    EigenVectorArrayMap<float> running_var_arr(
        running_var->mutable_data<float>(), C);
    running_mean_arr = running_mean_arr * momentum_ + mean * (1. - momentum_);
    running_var_arr = running_var_arr * momentum_ + var * (1. - momentum_);
  }

  // Regardless of training or testing, we will apply the estimated mean
  // and standard deviation to the input. For testing, they are
  // specified directly by the input, and for training, they are computed
  // by the op.
  Eigen::Array<float, Eigen::Dynamic, 1> inv_std(C);
  if (is_test_) {
    ConstEigenVectorArrayMap<float> var_arr(Input(EST_VAR).data<float>(), C);
    inv_std = (var_arr + epsilon_).sqrt().inverse();
  } else {
    EigenVectorArrayMap<float> saved_inv_std(
        Output(SAVED_INV_VAR)->mutable_data<float>(), C);
    saved_inv_std = (saved_inv_std + epsilon_).inverse().sqrt();
    inv_std = saved_inv_std;
  }
  ConstEigenVectorArrayMap<float> mean_arr(
      is_test_ ? Input(EST_MEAN).data<float>()
               : Output(SAVED_MEAN)->data<float>(),
      C);
  // We can fuse the output computation as follows:
  //   ((x - est_mean) * (inv_var) * scale + bias
  // to
  //   (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
  Eigen::Array<float, Eigen::Dynamic, 1> new_scale = inv_std * scale_arr;
  Eigen::Array<float, Eigen::Dynamic, 1> new_bias =
      bias_arr - mean_arr * inv_std * scale_arr;
  switch (order_) {
    case StorageOrder::NHWC: {
      EigenArrayMap<float>(Y->mutable_data<float>(), C, N * sample_size) =
          (ConstEigenArrayMap<float>(X.data<float>(), C, N * sample_size)
               .colwise() *
           new_scale)
              .colwise() +
          new_bias;
      break;
    }
    case StorageOrder::NCHW: {
      EigenArrayMap<float> Y_arr(Y->mutable_data<float>(), sample_size, N * C);
      ConstEigenArrayMap<float> X_arr(X.data<float>(), sample_size, N * C);
      for (int nc = 0; nc < N * C; ++nc) {
        Y_arr.col(nc) = X_arr.col(nc) * new_scale(nc % C) + new_bias(nc % C);
      }
      break;
    }
    default:
      CAFFE_THROW("Unknown storage order: ", order_);
  }
  return true;
}

REGISTER_CPU_OPERATOR(SpatialBN, SpatialBNOp<CPUContext>);

OPERATOR_SCHEMA(SpatialBN)
    .NumInputs(5)
    .NumOutputs({1, 5})
    .AllowInplace({{0, 0}})
    .EnforceInplace({{3, 1}, {4, 2}})
    .TensorInferenceFunction(
        [](const OperatorDef& def, const vector<TensorShape>& in) {
          ArgumentHelper helper(def);
          bool is_test = helper.GetSingleArgument<int>("is_test", 0);

          if (!is_test) {
            vector<TensorShape> out;
            StorageOrder order = StringToStorageOrder(
                helper.GetSingleArgument<string>("order", "NCHW"));
            const TensorShape& X = in[0];
            const int C =
                (order == StorageOrder::NCHW ? X.dims(1)
                                             : X.dims(X.dims_size() - 1));

            out.push_back(in[0]);
            TensorShape meanvar_tp =
                CreateTensorShape(vector<int>{C}, TensorProto::FLOAT);
            out.push_back(meanvar_tp); // RUNNING_MEAN
            out.push_back(meanvar_tp); // RUNNING_MEAN
            out.push_back(meanvar_tp); // SAVED_MEAN
            out.push_back(meanvar_tp); // SAVED_VAR
            return out;
          } else {
            return vector<TensorShape>{in[0]};
          }
        })
    .SetDoc(R"DOC(
Carries out spatial batch normalization as described in the paper
https://arxiv.org/abs/1502.03167. Depending on the mode it is being run,
there are multiple cases for the number of outputs, which we list below:

Output case #1: Y, mean, var, saved_mean, saved_var
                (training mode)
Output case #2: Y (test mode)
)DOC")
    .Arg(
        "is_test",
        "If set to nonzero, run spatial batch normalization in test mode.")
    .Arg("epsilon", "The epsilon value to use to avoid division by zero.")
    .Arg("order", "A StorageOrder string.")
    .Arg(
        "momentum",
        "Factor used in computing the running mean and variance."
        "e.g., running_mean = running_mean * momentum + mean * (1 - momentum)")
    .Input(
        0,
        "X",
        "The input 4-dimensional tensor of shape NCHW or NHWC depending "
        "on the order parameter.")
    .Input(
        1,
        "scale",
        "The scale as a 1-dimensional tensor of size C to be applied to the "
        "output.")
    .Input(
        2,
        "bias",
        "The bias as a 1-dimensional tensor of size C to be applied to the "
        "output.")
    .Input(
        3,
        "mean",
        "The running mean (training) or the estimated mean (testing) "
        "as a 1-dimensional tensor of size C.")
    .Input(
        4,
        "var",
        "The running variance (training) or the estimated "
        "variance (testing) as a 1-dimensional tensor of size C.")
    .Output(0, "Y", "The output 4-dimensional tensor of the same shape as X.")
    .Output(
        1,
        "mean",
        "The running mean after the spatial BN operator. Must be in-place "
        "with the input mean. Should not be used for testing.")
    .Output(
        2,
        "var",
        "The running variance after the spatial BN operator. Must be "
        "in-place with the input var. Should not be used for testing.")
    .Output(
        3,
        "saved_mean",
        "Saved mean used during training to speed up gradient "
        "computation. Should not be used for testing.")
    .Output(
        4,
        "saved_var",
        "Saved variance used during training to speed up "
        "gradient computation. Should not be used for testing.");

} // namespace caffe2
<commit_msg>fix arxiv link to batch-norm paper<commit_after>#include "caffe2/operators/spatial_batch_norm_op.h"

namespace caffe2 {

template <>
bool SpatialBNOp<CPUContext>::RunOnDevice() {
  const auto& X = Input(INPUT);
  const auto& scale = Input(SCALE);
  const auto& bias = Input(BIAS);

  CAFFE_ENFORCE(X.ndim() >= 3 && X.ndim() <= 5);
  const int N = X.dim32(0);
  const int C =
      (order_ == StorageOrder::NCHW ? X.dim32(1) : X.dim32(X.ndim() - 1));
  const int H = (order_ == StorageOrder::NCHW ? X.dim32(2) : X.dim32(1));
  const int W = X.ndim() > 3
      ? (order_ == StorageOrder::NCHW ? X.dim32(3) : X.dim32(2))
      : 1;
  const int D = X.ndim() > 4
      ? (order_ == StorageOrder::NCHW ? X.dim32(4) : X.dim32(3))
      : 1;

  const int sample_size = H * W * D;
  CAFFE_ENFORCE_EQ(scale.ndim(), 1);
  CAFFE_ENFORCE_EQ(bias.ndim(), 1);
  CAFFE_ENFORCE_EQ(scale.dim32(0), C);
  CAFFE_ENFORCE_EQ(bias.dim32(0), C);

  ConstEigenVectorArrayMap<float> scale_arr(scale.data<float>(), C);
  ConstEigenVectorArrayMap<float> bias_arr(bias.data<float>(), C);

  auto* Y = Output(OUTPUT);
  Y->ResizeLike(X);

  if (!is_test_) {
    // training mode
    // Get the mean and variance.
    // Note that, to be consistent with cudnn, we will output saved inverse
    // std as output 5, but we will still use the same storage place to
    // compute var as well. The inverse is going to be carried out at the end
    // of the op.
    Output(SAVED_MEAN)->Resize(C);
    Output(SAVED_INV_VAR)->Resize(C);
    EigenVectorArrayMap<float> mean(
        Output(SAVED_MEAN)->mutable_data<float>(), C);
    EigenVectorArrayMap<float> var(
        Output(SAVED_INV_VAR)->mutable_data<float>(), C);

    mean.setZero();
    var.setZero();
    switch (order_) {
      case StorageOrder::NCHW: {
        ConstEigenArrayMap<float> X_arr(X.data<float>(), sample_size, N * C);
        for (int nc = 0; nc < N * C; ++nc) {
          mean(nc % C) += X_arr.col(nc).sum();
        }
        mean /= N * sample_size;
        for (int nc = 0; nc < N * C; ++nc) {
          var(nc % C) += (X_arr.col(nc) - mean(nc % C)).matrix().squaredNorm();
        }
        var /= N * sample_size;
        break;
      }
      case StorageOrder::NHWC: {
        ConstEigenArrayMap<float> X_arr(X.data<float>(), C, N * sample_size);
        for (int i = 0; i < N * sample_size; ++i) {
          mean += X_arr.col(i);
        }
        mean /= N * sample_size;
        for (int i = 0; i < N * sample_size; ++i) {
          var += (X_arr.col(i) - mean) * (X_arr.col(i) - mean);
        }
        var /= N * sample_size;
        break;
      }
      default:
        CAFFE_THROW("Unknown storage order: ", order_);
    }

    // Compute the running mean and running inv variance.
    auto* running_mean = Output(RUNNING_MEAN);
    auto* running_var = Output(RUNNING_VAR);
    // Check if they are initialized
    if (!running_mean->size()) {
      running_mean->Resize(C);
      EigenVectorArrayMap<float> running_mean_map(
          running_mean->mutable_data<float>(), C);
      running_mean_map.setZero();
    }
    if (!running_var->size()) {
      running_var->Resize(C);
      EigenVectorArrayMap<float> running_var_map(
          running_var->mutable_data<float>(), C);
      running_var_map.setZero();
    }
    EigenVectorArrayMap<float> running_mean_arr(
        running_mean->mutable_data<float>(), C);
    EigenVectorArrayMap<float> running_var_arr(
        running_var->mutable_data<float>(), C);
    running_mean_arr = running_mean_arr * momentum_ + mean * (1. - momentum_);
    running_var_arr = running_var_arr * momentum_ + var * (1. - momentum_);
  }

  // Regardless of training or testing, we will apply the estimated mean
  // and standard deviation to the input. For testing, they are
  // specified directly by the input, and for training, they are computed
  // by the op.
  Eigen::Array<float, Eigen::Dynamic, 1> inv_std(C);
  if (is_test_) {
    ConstEigenVectorArrayMap<float> var_arr(Input(EST_VAR).data<float>(), C);
    inv_std = (var_arr + epsilon_).sqrt().inverse();
  } else {
    EigenVectorArrayMap<float> saved_inv_std(
        Output(SAVED_INV_VAR)->mutable_data<float>(), C);
    saved_inv_std = (saved_inv_std + epsilon_).inverse().sqrt();
    inv_std = saved_inv_std;
  }
  ConstEigenVectorArrayMap<float> mean_arr(
      is_test_ ? Input(EST_MEAN).data<float>()
               : Output(SAVED_MEAN)->data<float>(),
      C);
  // We can fuse the output computation as follows:
  //   ((x - est_mean) * (inv_var) * scale + bias
  // to
  //   (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
  Eigen::Array<float, Eigen::Dynamic, 1> new_scale = inv_std * scale_arr;
  Eigen::Array<float, Eigen::Dynamic, 1> new_bias =
      bias_arr - mean_arr * inv_std * scale_arr;
  switch (order_) {
    case StorageOrder::NHWC: {
      EigenArrayMap<float>(Y->mutable_data<float>(), C, N * sample_size) =
          (ConstEigenArrayMap<float>(X.data<float>(), C, N * sample_size)
               .colwise() *
           new_scale)
              .colwise() +
          new_bias;
      break;
    }
    case StorageOrder::NCHW: {
      EigenArrayMap<float> Y_arr(Y->mutable_data<float>(), sample_size, N * C);
      ConstEigenArrayMap<float> X_arr(X.data<float>(), sample_size, N * C);
      for (int nc = 0; nc < N * C; ++nc) {
        Y_arr.col(nc) = X_arr.col(nc) * new_scale(nc % C) + new_bias(nc % C);
      }
      break;
    }
    default:
      CAFFE_THROW("Unknown storage order: ", order_);
  }
  return true;
}

REGISTER_CPU_OPERATOR(SpatialBN, SpatialBNOp<CPUContext>);

OPERATOR_SCHEMA(SpatialBN)
    .NumInputs(5)
    .NumOutputs({1, 5})
    .AllowInplace({{0, 0}})
    .EnforceInplace({{3, 1}, {4, 2}})
    .TensorInferenceFunction(
        [](const OperatorDef& def, const vector<TensorShape>& in) {
          ArgumentHelper helper(def);
          bool is_test = helper.GetSingleArgument<int>("is_test", 0);

          if (!is_test) {
            vector<TensorShape> out;
            StorageOrder order = StringToStorageOrder(
                helper.GetSingleArgument<string>("order", "NCHW"));
            const TensorShape& X = in[0];
            const int C =
                (order == StorageOrder::NCHW ? X.dims(1)
                                             : X.dims(X.dims_size() - 1));

            out.push_back(in[0]);
            TensorShape meanvar_tp =
                CreateTensorShape(vector<int>{C}, TensorProto::FLOAT);
            out.push_back(meanvar_tp); // RUNNING_MEAN
            out.push_back(meanvar_tp); // RUNNING_MEAN
            out.push_back(meanvar_tp); // SAVED_MEAN
            out.push_back(meanvar_tp); // SAVED_VAR
            return out;
          } else {
            return vector<TensorShape>{in[0]};
          }
        })
    .SetDoc(R"DOC(
Carries out spatial batch normalization as described in the paper
https://arxiv.org/abs/1502.03167 . Depending on the mode it is being run,
there are multiple cases for the number of outputs, which we list below:

Output case #1: Y, mean, var, saved_mean, saved_var
                (training mode)
Output case #2: Y (test mode)
)DOC")
    .Arg(
        "is_test",
        "If set to nonzero, run spatial batch normalization in test mode.")
    .Arg("epsilon", "The epsilon value to use to avoid division by zero.")
    .Arg("order", "A StorageOrder string.")
    .Arg(
        "momentum",
        "Factor used in computing the running mean and variance."
        "e.g., running_mean = running_mean * momentum + mean * (1 - momentum)")
    .Input(
        0,
        "X",
        "The input 4-dimensional tensor of shape NCHW or NHWC depending "
        "on the order parameter.")
    .Input(
        1,
        "scale",
        "The scale as a 1-dimensional tensor of size C to be applied to the "
        "output.")
    .Input(
        2,
        "bias",
        "The bias as a 1-dimensional tensor of size C to be applied to the "
        "output.")
    .Input(
        3,
        "mean",
        "The running mean (training) or the estimated mean (testing) "
        "as a 1-dimensional tensor of size C.")
    .Input(
        4,
        "var",
        "The running variance (training) or the estimated "
        "variance (testing) as a 1-dimensional tensor of size C.")
    .Output(0, "Y", "The output 4-dimensional tensor of the same shape as X.")
    .Output(
        1,
        "mean",
        "The running mean after the spatial BN operator. Must be in-place "
        "with the input mean. Should not be used for testing.")
    .Output(
        2,
        "var",
        "The running variance after the spatial BN operator. Must be "
        "in-place with the input var. Should not be used for testing.")
    .Output(
        3,
        "saved_mean",
        "Saved mean used during training to speed up gradient "
        "computation. Should not be used for testing.")
    .Output(
        4,
        "saved_var",
        "Saved variance used during training to speed up "
        "gradient computation. Should not be used for testing.");

} // namespace caffe2
<|endoftext|>


<commit_before>//===- lib/ReaderWriter/ELF/Mips/MipsRelocationHandler.cpp ----------------===//
//
//                             The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "MipsTargetHandler.h"
#include "MipsLinkingContext.h"
#include "MipsRelocationHandler.h"

#include "lld/ReaderWriter/RelocationHelperFunctions.h"

using namespace lld;
using namespace elf;
using namespace llvm::ELF;

namespace {

inline void applyReloc(uint8_t *location, uint32_t result) {
  auto target = reinterpret_cast<llvm::support::ulittle32_t *>(location);
  *target = result | uint32_t(*target);
}

/// \brief Calculate AHL value combines addends from 'hi' and 'lo' relocations.
inline int64_t calcAHL(int64_t AHI, int64_t ALO) {
  AHI &= 0xffff;
  ALO &= 0xffff;
  return (AHI << 16) + (int16_t)ALO;
}

template <size_t BITS, class T> inline T signExtend(T val) {
  if (val & (T(1) << (BITS - 1)))
    val |= T(-1) << BITS;
  return val;
}

/// \brief R_MIPS_32
/// local/external: word32 S + A (truncate)
void reloc32(uint8_t *location, uint64_t P, uint64_t S, int64_t A) {
  uint32_t result = (uint32_t)(S + A);
  applyReloc(location, result);
}

/// \brief R_MIPS_26
/// local   : ((A | ((P + 4) & 0x3F000000)) + S) >> 2
/// external: (sign–extend(A) + S) >> 2
void reloc26(uint8_t *location, uint64_t P, uint64_t S, bool isLocal) {
  int32_t A = (*(uint32_t*)location & 0x03FFFFFFU) << 2;
  uint32_t result = isLocal ? (A | ((P + 4) & 0x3F000000)) : signExtend<28>(A);
  result = (result + S) >> 2;
  applyReloc(location, result);
}

/// \brief R_MIPS_HI16
/// local/external: hi16 (AHL + S) - (short)(AHL + S) (truncate)
/// _gp_disp      : hi16 (AHL + GP - P) - (short)(AHL + GP - P) (verify)
void relocHi16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
               uint64_t GP, bool isGPDisp) {
  int32_t result = 0;

  if (isGPDisp)
    result = (AHL + GP - P) - (int16_t)(AHL + GP - P);
  else
    result = (AHL + S) - (int16_t)(AHL + S);

  result = lld::scatterBits<uint32_t>(result >> 16, 0xffff);
  applyReloc(location, result);
}

/// \brief R_MIPS_LO16
/// local/external: lo16 AHL + S (truncate)
/// _gp_disp      : lo16 AHL + GP - P + 4 (verify)
void relocLo16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
               uint64_t GP, bool isGPDisp) {
  int32_t result = 0;

  if (isGPDisp)
    result = AHL + GP - P + 4;
  else
    result = AHL + S;

  result = lld::scatterBits<uint32_t>(result, 0xffff);
  applyReloc(location, result);
}

/// \brief R_MIPS_GOT16
/// local/external: rel16 G (verify)
void relocGOT16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
                uint64_t GP) {
  // FIXME (simon): for local sym put high 16 bit of AHL to the GOT
  int32_t G = (int32_t)(S - GP);
  int32_t result = lld::scatterBits<uint32_t>(G, 0xffff);
  applyReloc(location, result);
}

/// \brief R_MIPS_CALL16
/// external: rel16 G (verify)
void relocCall16(uint8_t *location, uint64_t P, uint64_t S, int64_t A,
                 uint64_t GP) {
  int32_t G = (int32_t)(S - GP);
  int32_t result = lld::scatterBits<uint32_t>(G, 0xffff);
  applyReloc(location, result);
}

/// \brief LLD_R_MIPS_HI16
void relocLldHi16(uint8_t *location, uint64_t S) {
  int32_t result = lld::scatterBits<uint32_t>((S + 0x8000) >> 16, 0xffff);
  applyReloc(location, result);
}

/// \brief LLD_R_MIPS_LO16
void relocLldLo16(uint8_t *location, uint64_t S) {
  int32_t result = lld::scatterBits<uint32_t>(S, 0xffff);
  applyReloc(location, result);
}

} // end anon namespace

MipsTargetRelocationHandler::MipsTargetRelocationHandler(
    const MipsLinkingContext &context, const MipsTargetHandler &handler)
    : _targetHandler(handler) {}

MipsTargetRelocationHandler::~MipsTargetRelocationHandler() {
  assert(_pairedRelocations.empty());
}

void
MipsTargetRelocationHandler::savePairedRelocation(const lld::AtomLayout &atom,
                                                  const Reference &ref) const {
  auto pi = _pairedRelocations.find(&atom);
  if (pi == _pairedRelocations.end())
    pi = _pairedRelocations.emplace(&atom, PairedRelocationsT()).first;

  pi->second.push_back(&ref);
}

void MipsTargetRelocationHandler::applyPairedRelocations(
    ELFWriter &writer, llvm::FileOutputBuffer &buf, const lld::AtomLayout &atom,
    int64_t loAddend) const {
  auto pi = _pairedRelocations.find(&atom);
  if (pi == _pairedRelocations.end())
    return;

  for (auto ri : pi->second) {
    uint8_t *atomContent = buf.getBufferStart() + atom._fileOffset;
    uint8_t *location = atomContent + ri->offsetInAtom();
    uint64_t targetVAddress = writer.addressOfAtom(ri->target());
    uint64_t relocVAddress = atom._virtualAddr + ri->offsetInAtom();

    int64_t ahl = calcAHL(ri->addend(), loAddend);

    if (ri->kindNamespace() != lld::Reference::KindNamespace::ELF)
      continue;
    assert(ri->kindArch() == Reference::KindArch::Mips);
    switch (ri->kindValue()) {
    case R_MIPS_HI16:
      relocHi16(location, relocVAddress, targetVAddress, ahl,
                _targetHandler.getGPDispSymAddr(),
                ri->target()->name() == "_gp_disp");
      break;
    case R_MIPS_GOT16:
      relocGOT16(location, relocVAddress, targetVAddress, ahl,
                 _targetHandler.getGPDispSymAddr());
      break;
    default:
      llvm_unreachable("Unknown type of paired relocation.");
    }
  }

  _pairedRelocations.erase(pi);
}

error_code MipsTargetRelocationHandler::applyRelocation(
    ELFWriter &writer, llvm::FileOutputBuffer &buf, const lld::AtomLayout &atom,
    const Reference &ref) const {
  uint8_t *atomContent = buf.getBufferStart() + atom._fileOffset;
  uint8_t *location = atomContent + ref.offsetInAtom();
  uint64_t targetVAddress = writer.addressOfAtom(ref.target());
  uint64_t relocVAddress = atom._virtualAddr + ref.offsetInAtom();

  if (ref.kindNamespace() != lld::Reference::KindNamespace::ELF)
    return error_code::success();
  assert(ref.kindArch() == Reference::KindArch::Mips);
  switch (ref.kindValue()) {
  case R_MIPS_NONE:
    break;
  case R_MIPS_32:
    reloc32(location, relocVAddress, targetVAddress, ref.addend());
    break;
  case R_MIPS_26:
    reloc26(location, relocVAddress, targetVAddress, true);
    break;
  case R_MIPS_HI16:
    savePairedRelocation(atom, ref);
    break;
  case R_MIPS_LO16:
    relocLo16(location, relocVAddress, targetVAddress, calcAHL(0, ref.addend()),
              _targetHandler.getGPDispSymAddr(),
              ref.target()->name() == "_gp_disp");
    applyPairedRelocations(writer, buf, atom, ref.addend());
    break;
  case R_MIPS_GOT16:
    savePairedRelocation(atom, ref);
    break;
  case R_MIPS_CALL16:
    relocCall16(location, relocVAddress, targetVAddress, ref.addend(),
                _targetHandler.getGPDispSymAddr());
    break;
  case R_MIPS_JALR:
    // We do not do JALR optimization now.
    break;
  case R_MIPS_JUMP_SLOT:
    // Ignore runtime relocations.
    break;
  case LLD_R_MIPS_GLOBAL_GOT:
    // Do nothing.
    break;
  case LLD_R_MIPS_GLOBAL_GOT16:
    relocGOT16(location, relocVAddress, targetVAddress, ref.addend(),
               _targetHandler.getGPDispSymAddr());
    break;
  case LLD_R_MIPS_GLOBAL_26:
    reloc26(location, relocVAddress, targetVAddress, false);
    break;
  case LLD_R_MIPS_HI16:
    relocLldHi16(location, targetVAddress);
    break;
  case LLD_R_MIPS_LO16:
    relocLldLo16(location, targetVAddress);
    break;
  default: {
    std::string str;
    llvm::raw_string_ostream s(str);
    s << "Unhandled Mips relocation: " << ref.kindValue();
    llvm_unreachable(s.str().c_str());
  }
  }

  return error_code::success();
}
<commit_msg>[Mips] Simplify calculations of relocations - replace scatterBits calls by bitwise AND operator and remove redundant local variables.<commit_after>//===- lib/ReaderWriter/ELF/Mips/MipsRelocationHandler.cpp ----------------===//
//
//                             The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#include "MipsTargetHandler.h"
#include "MipsLinkingContext.h"
#include "MipsRelocationHandler.h"

#include "lld/ReaderWriter/RelocationHelperFunctions.h"

using namespace lld;
using namespace elf;
using namespace llvm::ELF;

namespace {

inline void applyReloc(uint8_t *location, uint32_t result) {
  auto target = reinterpret_cast<llvm::support::ulittle32_t *>(location);
  *target = result | uint32_t(*target);
}

/// \brief Calculate AHL value combines addends from 'hi' and 'lo' relocations.
inline int64_t calcAHL(int64_t AHI, int64_t ALO) {
  AHI &= 0xffff;
  ALO &= 0xffff;
  return (AHI << 16) + (int16_t)ALO;
}

template <size_t BITS, class T> inline T signExtend(T val) {
  if (val & (T(1) << (BITS - 1)))
    val |= T(-1) << BITS;
  return val;
}

/// \brief R_MIPS_32
/// local/external: word32 S + A (truncate)
void reloc32(uint8_t *location, uint64_t P, uint64_t S, int64_t A) {
  applyReloc(location, (S + A) & 0xffffffff);
}

/// \brief R_MIPS_26
/// local   : ((A | ((P + 4) & 0x3F000000)) + S) >> 2
/// external: (sign–extend(A) + S) >> 2
void reloc26(uint8_t *location, uint64_t P, uint64_t S, bool isLocal) {
  int32_t A = (*(uint32_t*)location & 0x03FFFFFFU) << 2;
  uint32_t result = isLocal ? (A | ((P + 4) & 0x3F000000)) : signExtend<28>(A);
  result = (result + S) >> 2;
  applyReloc(location, result);
}

/// \brief R_MIPS_HI16
/// local/external: hi16 (AHL + S) - (short)(AHL + S) (truncate)
/// _gp_disp      : hi16 (AHL + GP - P) - (short)(AHL + GP - P) (verify)
void relocHi16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
               uint64_t GP, bool isGPDisp) {
  int32_t result = 0;

  if (isGPDisp)
    result = (AHL + GP - P) - (int16_t)(AHL + GP - P);
  else
    result = (AHL + S) - (int16_t)(AHL + S);

  applyReloc(location, (result >> 16) & 0xffff);
}

/// \brief R_MIPS_LO16
/// local/external: lo16 AHL + S (truncate)
/// _gp_disp      : lo16 AHL + GP - P + 4 (verify)
void relocLo16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
               uint64_t GP, bool isGPDisp) {
  int32_t result = 0;

  if (isGPDisp)
    result = AHL + GP - P + 4;
  else
    result = AHL + S;

  applyReloc(location, result & 0xffff);
}

/// \brief R_MIPS_GOT16
/// local/external: rel16 G (verify)
void relocGOT16(uint8_t *location, uint64_t P, uint64_t S, int64_t AHL,
                uint64_t GP) {
  // FIXME (simon): for local sym put high 16 bit of AHL to the GOT
  int32_t G = (int32_t)(S - GP);
  applyReloc(location, G & 0xffff);
}

/// \brief R_MIPS_CALL16
/// external: rel16 G (verify)
void relocCall16(uint8_t *location, uint64_t P, uint64_t S, int64_t A,
                 uint64_t GP) {
  int32_t G = (int32_t)(S - GP);
  applyReloc(location, G & 0xffff);
}

/// \brief LLD_R_MIPS_HI16
void relocLldHi16(uint8_t *location, uint64_t S) {
  applyReloc(location, ((S + 0x8000) >> 16) & 0xffff);
}

/// \brief LLD_R_MIPS_LO16
void relocLldLo16(uint8_t *location, uint64_t S) {
  applyReloc(location, S & 0xffff);
}

} // end anon namespace

MipsTargetRelocationHandler::MipsTargetRelocationHandler(
    const MipsLinkingContext &context, const MipsTargetHandler &handler)
    : _targetHandler(handler) {}

MipsTargetRelocationHandler::~MipsTargetRelocationHandler() {
  assert(_pairedRelocations.empty());
}

void
MipsTargetRelocationHandler::savePairedRelocation(const lld::AtomLayout &atom,
                                                  const Reference &ref) const {
  auto pi = _pairedRelocations.find(&atom);
  if (pi == _pairedRelocations.end())
    pi = _pairedRelocations.emplace(&atom, PairedRelocationsT()).first;

  pi->second.push_back(&ref);
}

void MipsTargetRelocationHandler::applyPairedRelocations(
    ELFWriter &writer, llvm::FileOutputBuffer &buf, const lld::AtomLayout &atom,
    int64_t loAddend) const {
  auto pi = _pairedRelocations.find(&atom);
  if (pi == _pairedRelocations.end())
    return;

  for (auto ri : pi->second) {
    uint8_t *atomContent = buf.getBufferStart() + atom._fileOffset;
    uint8_t *location = atomContent + ri->offsetInAtom();
    uint64_t targetVAddress = writer.addressOfAtom(ri->target());
    uint64_t relocVAddress = atom._virtualAddr + ri->offsetInAtom();

    int64_t ahl = calcAHL(ri->addend(), loAddend);

    if (ri->kindNamespace() != lld::Reference::KindNamespace::ELF)
      continue;
    assert(ri->kindArch() == Reference::KindArch::Mips);
    switch (ri->kindValue()) {
    case R_MIPS_HI16:
      relocHi16(location, relocVAddress, targetVAddress, ahl,
                _targetHandler.getGPDispSymAddr(),
                ri->target()->name() == "_gp_disp");
      break;
    case R_MIPS_GOT16:
      relocGOT16(location, relocVAddress, targetVAddress, ahl,
                 _targetHandler.getGPDispSymAddr());
      break;
    default:
      llvm_unreachable("Unknown type of paired relocation.");
    }
  }

  _pairedRelocations.erase(pi);
}

error_code MipsTargetRelocationHandler::applyRelocation(
    ELFWriter &writer, llvm::FileOutputBuffer &buf, const lld::AtomLayout &atom,
    const Reference &ref) const {
  uint8_t *atomContent = buf.getBufferStart() + atom._fileOffset;
  uint8_t *location = atomContent + ref.offsetInAtom();
  uint64_t targetVAddress = writer.addressOfAtom(ref.target());
  uint64_t relocVAddress = atom._virtualAddr + ref.offsetInAtom();

  if (ref.kindNamespace() != lld::Reference::KindNamespace::ELF)
    return error_code::success();
  assert(ref.kindArch() == Reference::KindArch::Mips);
  switch (ref.kindValue()) {
  case R_MIPS_NONE:
    break;
  case R_MIPS_32:
    reloc32(location, relocVAddress, targetVAddress, ref.addend());
    break;
  case R_MIPS_26:
    reloc26(location, relocVAddress, targetVAddress, true);
    break;
  case R_MIPS_HI16:
    savePairedRelocation(atom, ref);
    break;
  case R_MIPS_LO16:
    relocLo16(location, relocVAddress, targetVAddress, calcAHL(0, ref.addend()),
              _targetHandler.getGPDispSymAddr(),
              ref.target()->name() == "_gp_disp");
    applyPairedRelocations(writer, buf, atom, ref.addend());
    break;
  case R_MIPS_GOT16:
    savePairedRelocation(atom, ref);
    break;
  case R_MIPS_CALL16:
    relocCall16(location, relocVAddress, targetVAddress, ref.addend(),
                _targetHandler.getGPDispSymAddr());
    break;
  case R_MIPS_JALR:
    // We do not do JALR optimization now.
    break;
  case R_MIPS_JUMP_SLOT:
    // Ignore runtime relocations.
    break;
  case LLD_R_MIPS_GLOBAL_GOT:
    // Do nothing.
    break;
  case LLD_R_MIPS_GLOBAL_GOT16:
    relocGOT16(location, relocVAddress, targetVAddress, ref.addend(),
               _targetHandler.getGPDispSymAddr());
    break;
  case LLD_R_MIPS_GLOBAL_26:
    reloc26(location, relocVAddress, targetVAddress, false);
    break;
  case LLD_R_MIPS_HI16:
    relocLldHi16(location, targetVAddress);
    break;
  case LLD_R_MIPS_LO16:
    relocLldLo16(location, targetVAddress);
    break;
  default: {
    std::string str;
    llvm::raw_string_ostream s(str);
    s << "Unhandled Mips relocation: " << ref.kindValue();
    llvm_unreachable(s.str().c_str());
  }
  }

  return error_code::success();
}
<|endoftext|>


<commit_before>/*******************************************************************************
 The MIT License (MIT)

 Copyright (c) 2015 Alexander Zolotarev <me@alex.bio> from Minsk, Belarus

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 *******************************************************************************/

// Small demo which prints raw (ungzipped) cereal stream from stdin.

// This define is needed to preserve client's timestamps in events.
#define ALOHALYTICS_SERVER
#include "../../src/event_base.h"

#include "../../src/Bricks/rtti/dispatcher.h"

#include <iostream>
#include <iomanip>
#include <typeinfo>

// If you use only virtual functions in your events, you don't need any Processor at all.
// Here we process some events with Processor for example only.
struct Processor {
  void operator()(const AlohalyticsBaseEvent & event) {
    std::cout << "Unhandled event of type " << typeid(event).name() << " with timestamp " << event.ToString()
              << std::endl;
  }
  void operator()(const AlohalyticsIdEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyValueEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyPairsEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyPairsLocationEvent & event) { std::cout << event.ToString() << std::endl; }
};

int main(int, char **) {
  cereal::BinaryInputArchive ar(std::cin);
  Processor processor;
  try {
    while (std::cin.good()) {
      std::unique_ptr<AlohalyticsBaseEvent> ptr;
      ar(ptr);
      bricks::rtti::RuntimeDispatcher<AlohalyticsBaseEvent, AlohalyticsKeyPairsLocationEvent, AlohalyticsKeyPairsEvent,
                                      AlohalyticsIdEvent, AlohalyticsKeyValueEvent,
                                      AlohalyticsKeyEvent>::DispatchCall(*ptr, processor);
    }
  } catch (const cereal::Exception &) {
  }
  return 0;
}
<commit_msg>Updated demo to process all events in the event_base.h<commit_after>/*******************************************************************************
 The MIT License (MIT)

 Copyright (c) 2015 Alexander Zolotarev <me@alex.bio> from Minsk, Belarus

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 *******************************************************************************/

// Small demo which prints raw (ungzipped) cereal stream from stdin.

// This define is needed to preserve client's timestamps in events.
#define ALOHALYTICS_SERVER
#include "../../src/event_base.h"

#include "../../src/Bricks/rtti/dispatcher.h"

#include <iostream>
#include <iomanip>
#include <typeinfo>

// If you use only virtual functions in your events, you don't need any Processor at all.
// Here we process some events with Processor for example only.
struct Processor {
  void operator()(const AlohalyticsBaseEvent & event) {
    std::cout << "Unhandled event of type " << typeid(event).name() << " with timestamp " << event.ToString()
              << std::endl;
  }
  void operator()(const AlohalyticsIdEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsIdServerEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyLocationEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyValueEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyValueLocationEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyPairsEvent & event) { std::cout << event.ToString() << std::endl; }
  void operator()(const AlohalyticsKeyPairsLocationEvent & event) { std::cout << event.ToString() << std::endl; }
};

int main(int, char **) {
  cereal::BinaryInputArchive ar(std::cin);
  Processor processor;
  try {
    while (std::cin.good()) {
      std::unique_ptr<AlohalyticsBaseEvent> ptr;
      ar(ptr);
      bricks::rtti::RuntimeDispatcher<AlohalyticsBaseEvent, AlohalyticsKeyPairsLocationEvent, AlohalyticsKeyPairsEvent,
                                      AlohalyticsIdServerEvent, AlohalyticsIdEvent, AlohalyticsKeyValueLocationEvent,
                                      AlohalyticsKeyValueEvent, AlohalyticsKeyLocationEvent,
                                      AlohalyticsKeyEvent>::DispatchCall(*ptr, processor);
    }
  } catch (const cereal::Exception &) {
  }
  return 0;
}
<|endoftext|>