<commit_before>#include "contour.hpp"

ContourTransition::ContourTransition( cv::Mat& image )
{
	copy_data( image );
	set_transition_to_no();
}

cv::Mat ContourTransition::show_matDataTrans()
{
	cv::Mat blackImage(matDataTrans.rows, matDataTrans.cols * channels, CV_8UC3, cv::Scalar(0,0,0));

	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col+=3 )
		{
			switch (matDataTrans(row, col).transition )
			{
				case lToR : *blackImage.ptr(row, col + 1) = 255; break;
				case upToDw : *blackImage.ptr(row, col + 2) = 255; break;
				default : *blackImage.ptr(row, col + 1) = 64;
			}
		}

	return blackImage;
}

void ContourTransition::bw_push_transition(std::vector<IndexTransition> const & indexTransition)
{
	std::for_each(indexTransition.begin(), indexTransition.end(), [ this ](auto& el){
		matDataTrans( el.row, el.col ).transition = el.transition;
    });
}

void ContourTransition::copy_data( cv::Mat& image )
{
	set_transition_to_no();
	matDataTrans.create( image.rows, image.cols );
	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col++ )
		{
			matDataTrans(row, col).pixel = *image.ptr(row, col);
		}
}

void ContourTransition::set_transition_to_no()
{
	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col++ )
		{
			matDataTrans(row, col).transition = Transition::no;
		}
}

Preprocess::Preprocess( cv::Mat_<double> filterKernel_, cv::Mat const & image ) : srcImgSize(image.rows, image.cols)
{
	if(filterKernel_.rows & 1 == 0 | filterKernel_.cols & 1 == 0 )
		std::cout << "\nnot symmetric filterKernel_ Preprocess::get_correction_edge\n";

	filterKernel = filterKernel_.clone();
}

cv::Mat Preprocess::get_thick_kernel( cv::Mat const & image, uint dilationSize )//tested visually
{
	cv::Mat thickKernel = image.clone();

    cv::Mat gray, edge, draw;
    cvtColor(image, gray, CV_BGR2GRAY);

    blur( gray, gray, cv::Size(7, 7) );

    int apertureSize = 5;
    Canny( gray, edge, 80, 500, apertureSize, true);//

    edge.convertTo(thickKernel, CV_8U);

    //dilate here
    cv::Mat element = cv::getStructuringElement( cv::MORPH_ELLIPSE,
                                       cv::Size( 2*dilationSize + 1, 2*dilationSize+1 ),
                                       cv::Point( dilationSize, dilationSize ) );
  /// Apply the dilation operation
  	dilate( thickKernel, thickKernel, element );

  	cv::Mat kernelSilentBorder;
	cv::copyMakeBorder( thickKernel, kernelSilentBorder, filterKernel.rows / 2, filterKernel.rows / 2, filterKernel.cols / 2, filterKernel.cols / 2, cv::BORDER_REFLECT);//to be safe

	return kernelSilentBorder;
}

ContourTransition Preprocess::get_correction_edge( cv::Mat const & image, std::vector<IndexTransition> const & indexTransition, uint dilationSize )
{
	cv::Mat kernelSilentBorder = get_thick_kernel( image, dilationSize );//thick canny

	// cv::Mat kernelSilentBorderROI = kernelSilentBorder(rct);
	cv::Mat_<Transition> matTransSilentBorder = this->cvt_it_to_matT( indexTransition );//silent border inside

	cv::Rect rct( filterKernel.cols / 2 + 1, filterKernel.rows / 2 + 1, image.cols - filterKernel.cols / 2 + 1, image.rows - filterKernel.rows / 2 + 1 );
	ContourTransition contourTransition( kernelSilentBorder );//full , not roi //::no //v<0;255>
	contourTransition.matDataTrans( rct );//silent border
	for( int row = 0; row < contourTransition.matDataTrans.rows; ++row )
	{
		for( int col = 0; col < contourTransition.matDataTrans.cols; ++col )
		{
			if( contourTransition.matDataTrans( row, col ).pixel == (unsigned char)(255) )
			{
				contourTransition.matDataTrans( row, col ).transition = get_direction( row, col, matTransSilentBorder);
			}
		}
	}
	return contourTransition;
}

Transition Preprocess::get_direction( int const row, int const col, cv::Mat_<Transition> matTransSilentBorder )//core method
{
	Transition result{ empty };
	std::pair<double, uint> histo[ transDirCombi ];
	for( int i = 0; i < transDirCombi; i++ )
	{
		histo[i] = std::pair<double, uint>( 0.0, i );
	}

	for(int _row = row - filterKernel.rows / 2; _row <= row + filterKernel.rows / 2; _row++ )
		for(int _col = col - filterKernel.cols/2; _col <= col + filterKernel.cols / 2; _col++ )
		{
			auto debugValue = (matTransSilentBorder(_row, _col) & all) >> shiftToDistinct;
			if( (debugValue) >= transDirCombi )
				std::cout << "\n\n\nERRR\n\n\n";

			histo[ (matTransSilentBorder(_row, _col) & all) >> shiftToDistinct ].second += filterKernel(_row - row + filterKernel.rows / 2, _col - col + filterKernel.cols / 2 );
		}

	std::pair<double, uint> orgHisto[ transDirCombi ];
	std::copy( histo, histo + transDirCombi, orgHisto );
	for( int i = 0; i < transDirCombi; ++i )//concatenate similar transition probability
	{
		for( int j = 0; j < transDirCombi; ++j )
		{
			if( i & j == i )
			{
				histo[i].first += orgHisto[j].first;
			}
		}
	}

	std::sort(histo, histo + transDirCombi, std::greater<std::pair<double, uint>>());

	if( histo[0].first / histo[1].first > 2 )
	{

		result = static_cast<Transition>( histo[0].second << shiftToDistinct );
		if( DataProcess::is_noise_detection( result ))
		{
			return Transition::unknown;
		}
		else
		{
			return result;
		}
	}
	else
	{
		result = static_cast<Transition>( ( histo[0].second | histo[1].second )<< shiftToDistinct );
		if( DataProcess::is_noise_detection( result ))
		{
			return Transition::unknown;
		}
		else
		{
			return result;
		}
	}
}

cv::Mat_<Transition> Preprocess::cvt_it_to_matT( std::vector<IndexTransition> const & indexTransition )
{
	cv::Mat_<Transition> result;
	cv::Mat_<Transition> tmpResult( srcImgSize);//, Transition::no );
	for( int i = 0; i < tmpResult.total(); ++i){ ((Transition*)(tmpResult.data))[i] = Transition::no; }
	// for( int i = 0; i < tmpResult.total(); ++i)//debug
	// {
	// 	//((double*)(tmpResult.data))[i] = 1.0/( filterSize * filterSize );
	// 	std::cout << tmpResult(0 , i);
	// }
	// std::cout<<std::endl;

	std::for_each( indexTransition.begin(), indexTransition.end(), [&tmpResult]( auto& el){
		tmpResult( el.row, el.col) = el.transition;
	});
	cv::copyMakeBorder( tmpResult, result, filterKernel.rows / 2, filterKernel.rows / 2, filterKernel.cols / 2, filterKernel.cols / 2, cv::BORDER_REFLECT);//to be safe

	return result;//return with silent border
}

cv::Mat_<double> MakeFilter::get_square_filter( int filterSize )
{
	if( filterSize & 1 == 0)
		filterSize -= 1;

	cv::Mat_<double> result( cv::Size2i( filterSize, filterSize ), 1.0/( filterSize * filterSize ) );

	return result;
}<commit_msg>test2<commit_after>#include "contour.hpp"

ContourTransition::ContourTransition( cv::Mat& image )
{
	copy_data( image );
	set_transition_to_no();
}

cv::Mat ContourTransition::show_matDataTrans()
{
	cv::Mat blackImage(matDataTrans.rows, matDataTrans.cols, CV_8UC3, cv::Scalar(0,0,0));

	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col+=3 )
		{
			Transition trans = matDataTrans(row, col).transition;
			if(trans & lToR)
			{
				blackImage.data[ row * blackImage.cols * channels + col * channels + 0 ] = 255;
			}
			if(trans & rToL)
			{
				blackImage.data[ row * blackImage.cols * channels + col * channels + 2 ] = 255;
			}
			if(trans & all)
			{
				blackImage.data[ row * blackImage.cols * channels + col * channels + 1 ] = 255;
			}
		}

	return blackImage;
}

void ContourTransition::bw_push_transition(std::vector<IndexTransition> const & indexTransition)
{
	std::for_each(indexTransition.begin(), indexTransition.end(), [ this ](auto& el){
		matDataTrans( el.row, el.col ).transition = el.transition;
    });
}

void ContourTransition::copy_data( cv::Mat& image )
{
	set_transition_to_no();
	matDataTrans.create( image.rows, image.cols );
	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col++ )
		{
			matDataTrans(row, col).pixel = *image.ptr(row, col);
		}
}

void ContourTransition::set_transition_to_no()
{
	for(int row = 0; row < matDataTrans.rows; row++ )
		for(int col = 0; col < matDataTrans.cols; col++ )
		{
			matDataTrans(row, col).transition = Transition::no;
		}
}

Preprocess::Preprocess( cv::Mat_<double> filterKernel_, cv::Mat const & image ) : srcImgSize(image.rows, image.cols)
{
	if(filterKernel_.rows & 1 == 0 | filterKernel_.cols & 1 == 0 )
		std::cout << "\nnot symmetric filterKernel_ Preprocess::get_correction_edge\n";

	filterKernel = filterKernel_.clone();
}

cv::Mat Preprocess::get_thick_kernel( cv::Mat const & image, uint dilationSize )//tested visually
{
	cv::Mat thickKernel = image.clone();

    cv::Mat gray, edge, draw;
    cvtColor(image, gray, CV_BGR2GRAY);

    blur( gray, gray, cv::Size(7, 7) );

    int apertureSize = 5;
    Canny( gray, edge, 80, 500, apertureSize, true);//

    edge.convertTo(thickKernel, CV_8U);

    //dilate here
    cv::Mat element = cv::getStructuringElement( cv::MORPH_ELLIPSE,
                                       cv::Size( 2*dilationSize + 1, 2*dilationSize+1 ),
                                       cv::Point( dilationSize, dilationSize ) );
  /// Apply the dilation operation
  	dilate( thickKernel, thickKernel, element );

  	cv::Mat kernelSilentBorder;
	cv::copyMakeBorder( thickKernel, kernelSilentBorder, filterKernel.rows / 2, filterKernel.rows / 2, filterKernel.cols / 2, filterKernel.cols / 2, cv::BORDER_REFLECT);//to be safe

	return kernelSilentBorder;
}

ContourTransition Preprocess::get_correction_edge( cv::Mat const & image, std::vector<IndexTransition> const & indexTransition, uint dilationSize )
{
	cv::Mat kernelSilentBorder = get_thick_kernel( image, dilationSize );//thick canny

	// cv::Mat kernelSilentBorderROI = kernelSilentBorder(rct);
	cv::Mat_<Transition> matTransSilentBorder = this->cvt_it_to_matT( indexTransition );//silent border inside

	cv::Rect rct( filterKernel.cols / 2 + 1, filterKernel.rows / 2 + 1, image.cols - filterKernel.cols / 2 + 1, image.rows - filterKernel.rows / 2 + 1 );
	ContourTransition contourTransition( kernelSilentBorder );//full , not roi //::no //v<0;255>
	contourTransition.matDataTrans( rct );//silent border
	for( int row = 0; row < contourTransition.matDataTrans.rows; ++row )
	{
		for( int col = 0; col < contourTransition.matDataTrans.cols; ++col )
		{
			if( contourTransition.matDataTrans( row, col ).pixel == (unsigned char)(255) )
			{
				contourTransition.matDataTrans( row, col ).transition = get_direction( row, col, matTransSilentBorder);
			}
		}
	}
	return contourTransition;
}

Transition Preprocess::get_direction( int const row, int const col, cv::Mat_<Transition> matTransSilentBorder )//core method
{
	Transition result{ empty };
	std::pair<double, uint> histo[ transDirCombi ];
	for( int i = 0; i < transDirCombi; i++ )
	{
		histo[i] = std::pair<double, uint>( 0.0, i );
	}

	for(int _row = row - filterKernel.rows / 2; _row <= row + filterKernel.rows / 2; _row++ )
		for(int _col = col - filterKernel.cols/2; _col <= col + filterKernel.cols / 2; _col++ )
		{
			auto debugValue = (matTransSilentBorder(_row, _col) & all) >> shiftToDistinct;
			if( (debugValue) >= transDirCombi )
				std::cout << "\n\n\nERRR\n\n\n";

			histo[ (matTransSilentBorder(_row, _col) & all) >> shiftToDistinct ].first += filterKernel(_row - row + filterKernel.rows / 2, _col - col + filterKernel.cols / 2 );
		}
	histo[0].first = 0;

	std::pair<double, uint> orgHisto[ transDirCombi ];
	std::copy( histo, histo + transDirCombi, orgHisto );
	for( int i = 0; i < transDirCombi; ++i )//concatenate similar transition probability
	{
		for( int j = 0; j < transDirCombi; ++j )
		{
			if( i & j == i )// shouldd i!=j
			{
				histo[i].first += orgHisto[j].first;
			}
		}
	}

	std::sort(histo, histo + transDirCombi, std::greater<std::pair<double, uint>>());

	if( histo[0].first / histo[1].first > 2 )
	{

		result = static_cast<Transition>( histo[0].second << shiftToDistinct );
		if( DataProcess::is_noise_detection( result ))
		{
			return Transition::unknown;
		}
		else
		{
			return result;
		}
	}
	else
	{
		result = static_cast<Transition>( ( histo[0].second | histo[1].second )<< shiftToDistinct );
		if( DataProcess::is_noise_detection( result ))
		{
			return Transition::unknown;
		}
		else
		{
			return result;
		}
	}
}

cv::Mat_<Transition> Preprocess::cvt_it_to_matT( std::vector<IndexTransition> const & indexTransition )
{
	cv::Mat_<Transition> result;
	cv::Mat_<Transition> tmpResult( srcImgSize);//, Transition::no );
	for( int i = 0; i < tmpResult.total(); ++i){ ((Transition*)(tmpResult.data))[i] = Transition::no; }
	// for( int i = 0; i < tmpResult.total(); ++i)//debug
	// {
	// 	//((double*)(tmpResult.data))[i] = 1.0/( filterSize * filterSize );
	// 	std::cout << tmpResult(0 , i);
	// }
	// std::cout<<std::endl;

	std::for_each( indexTransition.begin(), indexTransition.end(), [&tmpResult]( auto& el){
		tmpResult( el.row, el.col) = el.transition;
	});
	cv::copyMakeBorder( tmpResult, result, filterKernel.rows / 2, filterKernel.rows / 2, filterKernel.cols / 2, filterKernel.cols / 2, cv::BORDER_REFLECT);//to be safe

	return result;//return with silent border
}

cv::Mat_<double> MakeFilter::get_square_filter( int filterSize )
{
	if( filterSize & 1 == 0)
		filterSize -= 1;

	cv::Mat_<double> result( cv::Size2i( filterSize, filterSize ), 1.0/( filterSize * filterSize ) );

	return result;
}<|endoftext|>


<commit_before>#include "UASQuickView.h"
#include <QMetaMethod>
#include <QDebug>
#include "UASQuickViewItemSelect.h"
#include "UASQuickViewTextItem.h"
#include <QSettings>
#include <QInputDialog>
UASQuickView::UASQuickView(QWidget *parent) : QWidget(parent)
{
    quickViewSelectDialog=0;
    m_columnCount=2;
    m_currentColumn=0;
    ui.setupUi(this);

    m_verticalLayoutList.append(new QVBoxLayout());
    ui.horizontalLayout->addItem(m_verticalLayoutList[0]);

    connect(UASManager::instance(),SIGNAL(activeUASSet(UASInterface*)),this,SLOT(setActiveUAS(UASInterface*)));
    connect(UASManager::instance(),SIGNAL(UASCreated(UASInterface*)),this,SLOT(addUAS(UASInterface*)));
    if (UASManager::instance()->getActiveUAS())
    {
        addUAS(UASManager::instance()->getActiveUAS());
    }
    this->setContextMenuPolicy(Qt::ActionsContextMenu);

    loadSettings();

    //If we don't have any predefined settings, set some defaults.
    if (uasPropertyValueMap.size() == 0)
    {
        valueEnabled("altitude");
        valueEnabled("groundSpeed");
        valueEnabled("distToWP");
        valueEnabled("yaw");
        valueEnabled("roll");
    }

    QAction *action = new QAction("Add Item",this);
    action->setCheckable(false);
    connect(action,SIGNAL(triggered()),this,SLOT(actionTriggered()));
    this->addAction(action);

    QAction *columnaction = new QAction("Set Column Count",this);
    columnaction->setCheckable(false);
    connect(columnaction,SIGNAL(triggered()),this,SLOT(columnActionTriggered()));
    this->addAction(columnaction);

    updateTimer = new QTimer(this);
    connect(updateTimer,SIGNAL(timeout()),this,SLOT(updateTimerTick()));
    updateTimer->start(1000);

}
UASQuickView::~UASQuickView()
{
    if (quickViewSelectDialog)
    {
        delete quickViewSelectDialog;
    }
}
void UASQuickView::columnActionTriggered()
{
    bool ok = false;
    int newcolumns = QInputDialog::getInt(this,"Columns","Enter number of columns",1,0,100,1,&ok);
    if (!ok)
    {
        return;
    }
    m_columnCount = newcolumns;
    sortItems(newcolumns);
    saveSettings();
}

void UASQuickView::actionTriggered()
{
    if (quickViewSelectDialog)
    {
        quickViewSelectDialog->show();
        return;
    }
    quickViewSelectDialog = new UASQuickViewItemSelect();
    connect(quickViewSelectDialog,SIGNAL(destroyed()),this,SLOT(selectDialogClosed()));
    connect(quickViewSelectDialog,SIGNAL(valueDisabled(QString)),this,SLOT(valueDisabled(QString)));
    connect(quickViewSelectDialog,SIGNAL(valueEnabled(QString)),this,SLOT(valueEnabled(QString)));
    quickViewSelectDialog->setAttribute(Qt::WA_DeleteOnClose,true);
    for (QMap<QString,double>::const_iterator i = uasPropertyValueMap.constBegin();i!=uasPropertyValueMap.constEnd();i++)
    {
        quickViewSelectDialog->addItem(i.key(),uasEnabledPropertyList.contains(i.key()));
    }
    quickViewSelectDialog->show();
}
void UASQuickView::saveSettings()
{
    QSettings settings;
    settings.beginWriteArray("UAS_QUICK_VIEW_ITEMS");
    int count = 0;
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin();i!=uasPropertyToLabelMap.constEnd();i++)
    {
        settings.setArrayIndex(count++);
        settings.setValue("name",i.key());
        settings.setValue("type","text");
    }
    settings.endArray();
    settings.setValue("UAS_QUICK_VIEW_COLUMNS",m_columnCount);
    settings.sync();
}
void UASQuickView::loadSettings()
{
    QSettings settings;
    int size = settings.beginReadArray("UAS_QUICK_VIEW_ITEMS");
    for (int i=0;i<size;i++)
    {
        settings.setArrayIndex(i);
        QString nameval = settings.value("name").toString();
        QString typeval = settings.value("type").toString();
        if (typeval == "text" && !uasPropertyToLabelMap.contains(nameval))
        {
            valueEnabled(nameval);
        }
    }
    settings.endArray();
    m_columnCount = settings.value("UAS_QUICK_VIEW_COLUMNS",1).toInt();
    sortItems(m_columnCount);
}

void UASQuickView::valueEnabled(QString value)
{
    UASQuickViewItem *item = new UASQuickViewTextItem(this);
    item->setTitle(value);
    //ui.verticalLayout->addWidget(item);
    //m_currentColumn
    m_verticalLayoutList[m_currentColumn]->addWidget(item);
    m_PropertyToLayoutIndexMap[value] = m_currentColumn;
    m_currentColumn++;
    if (m_currentColumn >= m_columnCount-1)
    {
        m_currentColumn = 0;
    }
    uasPropertyToLabelMap[value] = item;
    uasEnabledPropertyList.append(value);

    if (!uasPropertyValueMap.contains(value))
    {
        uasPropertyValueMap[value] = 0;
    }
    saveSettings();
    item->show();
    sortItems(m_columnCount);

}
void UASQuickView::sortItems(int columncount)
{
    QList<QWidget*> itemlist;
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin();i!=uasPropertyToLabelMap.constEnd();i++)
    {
        m_verticalLayoutList[m_PropertyToLayoutIndexMap[i.key()]]->removeWidget(i.value());
        m_PropertyToLayoutIndexMap.remove(i.key());
        itemlist.append(i.value());
    }
    //Item list has all the widgets availble, now re-add them to the layouts.
    for (int i=0;i<m_verticalLayoutList.size();i++)
    {
        ui.horizontalLayout->removeItem(m_verticalLayoutList[i]);
        m_verticalLayoutList[i]->deleteLater(); //removeItem de-parents the item.
    }
    m_verticalLayoutList.clear();

    //Create a vertical layout for every intended column
    for (int i=0;i<columncount;i++)
    {
        QVBoxLayout *layout = new QVBoxLayout();
        ui.horizontalLayout->addItem(layout);
        m_verticalLayoutList.append(layout);
    }

    //Cycle through all items and add them to the layout
    int currcol = 0;
    for (int i=0;i<itemlist.size();i++)
    {
        m_verticalLayoutList[currcol]->addWidget(itemlist[i]);
        currcol++;
        if (currcol >= columncount)
        {
            currcol = 0;
        }
    }
    m_currentColumn = currcol;
}

void UASQuickView::valueDisabled(QString value)
{
    if (uasPropertyToLabelMap.contains(value))
    {
        UASQuickViewItem *item = uasPropertyToLabelMap[value];
        uasPropertyToLabelMap.remove(value);
        item->hide();
        //ui.verticalLayout->removeWidget(item);
        //layout->removeWidget(item);
        m_verticalLayoutList[m_PropertyToLayoutIndexMap[value]]->removeWidget(item);
        sortItems(m_columnCount);
        item->deleteLater();
        uasEnabledPropertyList.removeOne(value);
        saveSettings();
    }
}

void UASQuickView::selectDialogClosed()
{
    quickViewSelectDialog = 0;
}

void UASQuickView::updateTimerTick()
{
    //uasPropertyValueMap
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin(); i != uasPropertyToLabelMap.constEnd();i++)
    {
        if (uasPropertyValueMap.contains(i.key()))
        {
            i.value()->setValue(uasPropertyValueMap[i.key()]);
        }
    }
}

void UASQuickView::addUAS(UASInterface* uas)
{
    if (uas)
    {
        if (!this->uas)
        {
            setActiveUAS(uas);
        }
    }
}

void UASQuickView::setActiveUAS(UASInterface* uas)
{
    if (!uas)
    {
        return;
    }
    this->uas = uas;
    connect(uas,SIGNAL(valueChanged(int,QString,QString,QVariant,quint64)),this,SLOT(valueChanged(int,QString,QString,QVariant,quint64)));
    //connect(uas,SIGNAL())
}
void UASQuickView::addSource(MAVLinkDecoder *decoder)
{
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,double,quint64)),this,SLOT(valueChanged(int,QString,QString,double,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint8,quint64)),this,SLOT(valueChanged(int,QString,QString,qint8,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint16,quint64)),this,SLOT(valueChanged(int,QString,QString,qint16,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint32,quint64)),this,SLOT(valueChanged(int,QString,QString,qint32,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint64,quint64)),this,SLOT(valueChanged(int,QString,QString,qint64,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint8,quint64)),this,SLOT(valueChanged(int,QString,QString,quint8,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint16,quint64)),this,SLOT(valueChanged(int,QString,QString,quint16,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint32,quint64)),this,SLOT(valueChanged(int,QString,QString,quint32,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint64,quint64)),this,SLOT(valueChanged(int,QString,QString,quint64,quint64)));
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint8 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}

void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint8 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint16 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint16 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint32 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint32 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint64 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint64 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const double value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}

void UASQuickView::actionTriggered(bool checked)
{
    QAction *senderlabel = qobject_cast<QAction*>(sender());
    if (!senderlabel)
    {
        return;
    }
    if (checked)
    {
        valueEnabled(senderlabel->text());
        /*UASQuickViewItem *item = new UASQuickViewTextItem(this);
        item->setTitle(senderlabel->text());
        layout->addWidget(item);
        //ui.verticalLayout->addWidget(item);
        m_currentColumn++;
        if (m_currentColumn >= m_verticalLayoutList.size())
        {
            m_currentColumn = 0;
        }
        uasPropertyToLabelMap[senderlabel->text()] = item;*/

    }
    else
    {
        valueDisabled(senderlabel->text());
        /*layout->removeWidget(uasPropertyToLabelMap[senderlabel->text()]);
        uasPropertyToLabelMap[senderlabel->text()]->deleteLater();
        uasPropertyToLabelMap.remove(senderlabel->text());*/

    }
}
void UASQuickView::valueChanged(const int uasid, const QString& name, const QString& unit, const QVariant value,const quint64 msecs)
{
    uasPropertyValueMap[name] = value.toDouble();
}

void UASQuickView::valChanged(double val,QString type)
{
    //qDebug() << "Value changed:" << type << val;
   // uasPropertyValueMap[type] = val;
}
<commit_msg>Fix for saving and reloading QuickView column count<commit_after>#include "UASQuickView.h"
#include <QMetaMethod>
#include <QDebug>
#include "UASQuickViewItemSelect.h"
#include "UASQuickViewTextItem.h"
#include <QSettings>
#include <QInputDialog>
UASQuickView::UASQuickView(QWidget *parent) : QWidget(parent)
{
    quickViewSelectDialog=0;
    m_columnCount=2;
    m_currentColumn=0;
    ui.setupUi(this);

    m_verticalLayoutList.append(new QVBoxLayout());
    ui.horizontalLayout->addItem(m_verticalLayoutList[0]);

    connect(UASManager::instance(),SIGNAL(activeUASSet(UASInterface*)),this,SLOT(setActiveUAS(UASInterface*)));
    connect(UASManager::instance(),SIGNAL(UASCreated(UASInterface*)),this,SLOT(addUAS(UASInterface*)));
    if (UASManager::instance()->getActiveUAS())
    {
        addUAS(UASManager::instance()->getActiveUAS());
    }
    this->setContextMenuPolicy(Qt::ActionsContextMenu);

    loadSettings();

    //If we don't have any predefined settings, set some defaults.
    if (uasPropertyValueMap.size() == 0)
    {
        valueEnabled("altitude");
        valueEnabled("groundSpeed");
        valueEnabled("distToWP");
        valueEnabled("yaw");
        valueEnabled("roll");
    }

    QAction *action = new QAction("Add Item",this);
    action->setCheckable(false);
    connect(action,SIGNAL(triggered()),this,SLOT(actionTriggered()));
    this->addAction(action);

    QAction *columnaction = new QAction("Set Column Count",this);
    columnaction->setCheckable(false);
    connect(columnaction,SIGNAL(triggered()),this,SLOT(columnActionTriggered()));
    this->addAction(columnaction);

    updateTimer = new QTimer(this);
    connect(updateTimer,SIGNAL(timeout()),this,SLOT(updateTimerTick()));
    updateTimer->start(1000);

}
UASQuickView::~UASQuickView()
{
    if (quickViewSelectDialog)
    {
        delete quickViewSelectDialog;
    }
}
void UASQuickView::columnActionTriggered()
{
    bool ok = false;
    int newcolumns = QInputDialog::getInt(this,"Columns","Enter number of columns",1,0,100,1,&ok);
    if (!ok)
    {
        return;
    }
    m_columnCount = newcolumns;
    sortItems(newcolumns);
    saveSettings();
}

void UASQuickView::actionTriggered()
{
    if (quickViewSelectDialog)
    {
        quickViewSelectDialog->show();
        return;
    }
    quickViewSelectDialog = new UASQuickViewItemSelect();
    connect(quickViewSelectDialog,SIGNAL(destroyed()),this,SLOT(selectDialogClosed()));
    connect(quickViewSelectDialog,SIGNAL(valueDisabled(QString)),this,SLOT(valueDisabled(QString)));
    connect(quickViewSelectDialog,SIGNAL(valueEnabled(QString)),this,SLOT(valueEnabled(QString)));
    quickViewSelectDialog->setAttribute(Qt::WA_DeleteOnClose,true);
    for (QMap<QString,double>::const_iterator i = uasPropertyValueMap.constBegin();i!=uasPropertyValueMap.constEnd();i++)
    {
        quickViewSelectDialog->addItem(i.key(),uasEnabledPropertyList.contains(i.key()));
    }
    quickViewSelectDialog->show();
}
void UASQuickView::saveSettings()
{
    QSettings settings;
    settings.beginWriteArray("UAS_QUICK_VIEW_ITEMS");
    int count = 0;
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin();i!=uasPropertyToLabelMap.constEnd();i++)
    {
        settings.setArrayIndex(count++);
        settings.setValue("name",i.key());
        settings.setValue("type","text");
    }
    settings.endArray();
    settings.setValue("UAS_QUICK_VIEW_COLUMNS",m_columnCount);
    settings.sync();
}
void UASQuickView::loadSettings()
{
    QSettings settings;
    m_columnCount = settings.value("UAS_QUICK_VIEW_COLUMNS",1).toInt();
    int size = settings.beginReadArray("UAS_QUICK_VIEW_ITEMS");
    for (int i=0;i<size;i++)
    {
        settings.setArrayIndex(i);
        QString nameval = settings.value("name").toString();
        QString typeval = settings.value("type").toString();
        if (typeval == "text" && !uasPropertyToLabelMap.contains(nameval))
        {
            valueEnabled(nameval);
        }
    }
    settings.endArray();
    sortItems(m_columnCount);
}

void UASQuickView::valueEnabled(QString value)
{
    UASQuickViewItem *item = new UASQuickViewTextItem(this);
    item->setTitle(value);
    //ui.verticalLayout->addWidget(item);
    //m_currentColumn
    m_verticalLayoutList[m_currentColumn]->addWidget(item);
    m_PropertyToLayoutIndexMap[value] = m_currentColumn;
    m_currentColumn++;
    if (m_currentColumn >= m_columnCount-1)
    {
        m_currentColumn = 0;
    }
    uasPropertyToLabelMap[value] = item;
    uasEnabledPropertyList.append(value);

    if (!uasPropertyValueMap.contains(value))
    {
        uasPropertyValueMap[value] = 0;
    }
    saveSettings();
    item->show();
    sortItems(m_columnCount);

}
void UASQuickView::sortItems(int columncount)
{
    QList<QWidget*> itemlist;
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin();i!=uasPropertyToLabelMap.constEnd();i++)
    {
        m_verticalLayoutList[m_PropertyToLayoutIndexMap[i.key()]]->removeWidget(i.value());
        m_PropertyToLayoutIndexMap.remove(i.key());
        itemlist.append(i.value());
    }
    //Item list has all the widgets availble, now re-add them to the layouts.
    for (int i=0;i<m_verticalLayoutList.size();i++)
    {
        ui.horizontalLayout->removeItem(m_verticalLayoutList[i]);
        m_verticalLayoutList[i]->deleteLater(); //removeItem de-parents the item.
    }
    m_verticalLayoutList.clear();

    //Create a vertical layout for every intended column
    for (int i=0;i<columncount;i++)
    {
        QVBoxLayout *layout = new QVBoxLayout();
        ui.horizontalLayout->addItem(layout);
        m_verticalLayoutList.append(layout);
    }

    //Cycle through all items and add them to the layout
    int currcol = 0;
    for (int i=0;i<itemlist.size();i++)
    {
        m_verticalLayoutList[currcol]->addWidget(itemlist[i]);
        currcol++;
        if (currcol >= columncount)
        {
            currcol = 0;
        }
    }
    m_currentColumn = currcol;
}

void UASQuickView::valueDisabled(QString value)
{
    if (uasPropertyToLabelMap.contains(value))
    {
        UASQuickViewItem *item = uasPropertyToLabelMap[value];
        uasPropertyToLabelMap.remove(value);
        item->hide();
        //ui.verticalLayout->removeWidget(item);
        //layout->removeWidget(item);
        m_verticalLayoutList[m_PropertyToLayoutIndexMap[value]]->removeWidget(item);
        sortItems(m_columnCount);
        item->deleteLater();
        uasEnabledPropertyList.removeOne(value);
        saveSettings();
    }
}

void UASQuickView::selectDialogClosed()
{
    quickViewSelectDialog = 0;
}

void UASQuickView::updateTimerTick()
{
    //uasPropertyValueMap
    for (QMap<QString,UASQuickViewItem*>::const_iterator i = uasPropertyToLabelMap.constBegin(); i != uasPropertyToLabelMap.constEnd();i++)
    {
        if (uasPropertyValueMap.contains(i.key()))
        {
            i.value()->setValue(uasPropertyValueMap[i.key()]);
        }
    }
}

void UASQuickView::addUAS(UASInterface* uas)
{
    if (uas)
    {
        if (!this->uas)
        {
            setActiveUAS(uas);
        }
    }
}

void UASQuickView::setActiveUAS(UASInterface* uas)
{
    if (!uas)
    {
        return;
    }
    this->uas = uas;
    connect(uas,SIGNAL(valueChanged(int,QString,QString,QVariant,quint64)),this,SLOT(valueChanged(int,QString,QString,QVariant,quint64)));
    //connect(uas,SIGNAL())
}
void UASQuickView::addSource(MAVLinkDecoder *decoder)
{
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,double,quint64)),this,SLOT(valueChanged(int,QString,QString,double,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint8,quint64)),this,SLOT(valueChanged(int,QString,QString,qint8,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint16,quint64)),this,SLOT(valueChanged(int,QString,QString,qint16,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint32,quint64)),this,SLOT(valueChanged(int,QString,QString,qint32,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,qint64,quint64)),this,SLOT(valueChanged(int,QString,QString,qint64,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint8,quint64)),this,SLOT(valueChanged(int,QString,QString,quint8,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint16,quint64)),this,SLOT(valueChanged(int,QString,QString,quint16,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint32,quint64)),this,SLOT(valueChanged(int,QString,QString,quint32,quint64)));
    connect(decoder,SIGNAL(valueChanged(int,QString,QString,quint64,quint64)),this,SLOT(valueChanged(int,QString,QString,quint64,quint64)));
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint8 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}

void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint8 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint16 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint16 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint32 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint32 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const quint64 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const qint64 value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}
void UASQuickView::valueChanged(const int uasId, const QString& name, const QString& unit, const double value, const quint64 msec)
{
    if (!uasPropertyValueMap.contains(name))
    {
        if (quickViewSelectDialog)
        {
            quickViewSelectDialog->addItem(name);
        }
    }
    uasPropertyValueMap[name] = value;
}

void UASQuickView::actionTriggered(bool checked)
{
    QAction *senderlabel = qobject_cast<QAction*>(sender());
    if (!senderlabel)
    {
        return;
    }
    if (checked)
    {
        valueEnabled(senderlabel->text());
        /*UASQuickViewItem *item = new UASQuickViewTextItem(this);
        item->setTitle(senderlabel->text());
        layout->addWidget(item);
        //ui.verticalLayout->addWidget(item);
        m_currentColumn++;
        if (m_currentColumn >= m_verticalLayoutList.size())
        {
            m_currentColumn = 0;
        }
        uasPropertyToLabelMap[senderlabel->text()] = item;*/

    }
    else
    {
        valueDisabled(senderlabel->text());
        /*layout->removeWidget(uasPropertyToLabelMap[senderlabel->text()]);
        uasPropertyToLabelMap[senderlabel->text()]->deleteLater();
        uasPropertyToLabelMap.remove(senderlabel->text());*/

    }
}
void UASQuickView::valueChanged(const int uasid, const QString& name, const QString& unit, const QVariant value,const quint64 msecs)
{
    uasPropertyValueMap[name] = value.toDouble();
}

void UASQuickView::valChanged(double val,QString type)
{
    //qDebug() << "Value changed:" << type << val;
   // uasPropertyValueMap[type] = val;
}
<|endoftext|>


<commit_before>#include "groups.h"

#include <cassert>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#include "data.h"

namespace redsea {

GroupType::GroupType(uint16_t type_code) : num((type_code >> 1) & 0xF), ab(type_code & 0x1) {}
GroupType::GroupType(const GroupType& obj) : num(obj.num), ab(obj.ab) {}

std::string GroupType::toString() {
  return std::string(std::to_string(num) + (ab == TYPE_A ? "A" : "B"));
}

bool operator==(const GroupType& obj1, const GroupType& obj2) {
  return (obj1.num == obj2.num && obj1.ab == obj2.ab);
}

bool operator<(const GroupType& obj1, const GroupType& obj2) { return ((obj1.num < obj2.num) || (obj1.ab < obj2.ab)); }

// extract len bits from bitstring, starting at starting_at from the right
uint16_t bits (uint16_t bitstring, int starting_at, int len) {
  return ((bitstring >> starting_at) & ((1<<len) - 1));
}

RDSString::RDSString(int len) : chars_(len), is_char_sequential_(len), prev_pos_(-1) {
  last_complete_string_ = getString();
}

void RDSString::setAt(int pos, int chr) {
  if (pos < 0 || pos >= (int)chars_.size())
    return;

  chars_[pos] = chr;

  if (pos != prev_pos_ + 1) {
    for (size_t i=0; i<is_char_sequential_.size(); i++)
      is_char_sequential_[i] = false;
  }

  is_char_sequential_[pos] = true;

  if (isComplete())
    last_complete_string_ = getString();

  prev_pos_ = pos;

}

size_t RDSString::lengthReceived() const {

  size_t result = 0;
  for (size_t i=0; i<is_char_sequential_.size(); i++) {
    if (!is_char_sequential_[i])
      break;
    result = i+1;
  }

  return result;
}

size_t RDSString::lengthExpected() const {

  size_t result = chars_.size();

  for (size_t i=0; i<chars_.size(); i++) {
    if (chars_[i] == 0x0D) {
      result = i;
      break;
    }
  }

  return result;
}

std::string RDSString::getString() const {
  std::string result;
  size_t len = lengthExpected();
  for (size_t i=0; i<len; i++) {
    result += (is_char_sequential_[i] ? getLCDchar(chars_[i]) : " ");
  }

  return result;

}

std::string RDSString::getLastCompleteString() const {
  return last_complete_string_;
}

bool RDSString::isComplete() const {
  return lengthReceived() >= lengthExpected();
}

void RDSString::clear() {
  for (size_t i=0; i<chars_.size(); i++) {
    is_char_sequential_[i] = false;
  }
  last_complete_string_ = getString();
}

Station::Station() : Station(0x0000) {

}

Station::Station(uint16_t _pi) : pi_(_pi), ps_(8), rt_(64) {

}

void Station::update(Group group) {

  is_tp_   = bits(group.block2, 10, 1);
  pty_     = bits(group.block2,  5, 5);

  printf("{ pi: \"0x%04x\", group: \"%s\", ", pi_, group.type.toString().c_str());

  printf("tp: %s, ", is_tp_ ? "true" : "false");
  printf("prog_type: \"%s\", ", getPTYname(pty_).c_str());

  if      (group.type.num == 0)  { decodeType0(group); }
  else if (group.type.num == 1)  { decodeType1(group); }
  else if (group.type.num == 2)  { decodeType2(group); }
  else if (group.type.num == 3)  { decodeType3(group); }
  else if (group.type.num == 4)  { decodeType4(group); }
  else if (group.type.num == 8)  { decodeType8(group); }
  else if (group.type.num == 14) { decodeType14(group); }
  else                           { printf("/* TODO */ "); }

  printf("}\n");
}

void Station::addAltFreq(uint8_t af_code) {
  if (af_code >= 1 && af_code <= 204) {
    alt_freqs_.insert(87.5 + af_code / 10.0);
  } else if (af_code == 205) {
    // filler
  } else if (af_code == 224) {
    // no AF exists
  } else if (af_code >= 225 && af_code <= 249) {
    num_alt_freqs_ = af_code - 224;
  } else if (af_code == 250) {
    // AM/LF freq follows
  }
}

bool Station::hasPS() const {
  return ps_.isComplete();
}

std::string Station::getPS() const {
  return ps_.getLastCompleteString();
}

std::string Station::getRT() const {
  return rt_.getLastCompleteString();
}

uint16_t Station::getPI() const {
  return pi_;
}

std::string Station::getCountryCode() const {
  return getCountryString(pi_, ecc_);
}

void Station::updatePS(int pos, std::vector<int> chars) {

  for (int i=pos; i<pos+(int)chars.size(); i++)
    ps_.setAt(i, chars[i-pos]);

  if (ps_.isComplete())
    printf("ps: \"%s\", ",ps_.getLastCompleteString().c_str());

}

void Station::updateRadioText(int pos, std::vector<int> chars) {

  for (int i=pos; i<pos+(int)chars.size(); i++)
    rt_.setAt(i, chars[i-pos]);

  if (rt_.isComplete())
    printf("radiotext: \"%s\", ",rt_.getLastCompleteString().c_str());

}

void Station::decodeType0 (Group group) {

  // not implemented: Decoder Identification

  is_ta_    = bits(group.block2, 4, 1);
  is_music_ = bits(group.block2, 3, 1);

  printf("ta: %s, ", is_ta_ ? "true" : "false");

  if (group.num_blocks < 3)
    return;

  if (group.type.ab == TYPE_A) {
    for (int i=0; i<2; i++) {
      addAltFreq(bits(group.block3, 8-i*8, 8));
    }

    if ((int)alt_freqs_.size() == num_alt_freqs_) {
      printf("alt_freqs: [ ");
      for (auto f : alt_freqs_)
        printf("%.1f, ", f);
      printf("], ");
      alt_freqs_.clear();
    }
  }

  if (group.num_blocks < 4)
    return;

  updatePS(bits(group.block2, 0, 2) * 2,
      { bits(group.block4, 8, 8), bits(group.block4, 0, 8) });

}

void Station::decodeType1 (Group group) {

  if (group.num_blocks < 4)
    return;

  pin_ = group.block4;

  if (group.type.ab == TYPE_A) {
    pager_tng_ = bits(group.block2, 2, 3);
    if (pager_tng_ != 0) {
      pager_interval_ = bits(group.block2, 0, 2);
    }
    linkage_la_ = bits(group.block3, 15, 1);

    int slc_variant = bits(group.block3, 12, 3);

    if (slc_variant == 0) {
      if (pager_tng_ != 0) {
        pager_opc_ = bits(group.block3, 8, 4);
      }

      // No PIN, section M.3.2.4.3
      if (group.num_blocks == 4 && (group.block4 >> 11) == 0) {
        int subtype = bits(group.block4, 10, 1);
        if (subtype == 0) {
          if (pager_tng_ != 0) {
            pager_pac_ = bits(group.block4, 4, 6);
            pager_opc_ = bits(group.block4, 0, 4);
          }
        } else if (subtype == 1) {
          if (pager_tng_ != 0) {
            int b = bits(group.block4, 8, 2);
            if (b == 0) {
              pager_ecc_ = bits(group.block4, 0, 6);
            } else if (b == 3) {
              pager_ccf_ = bits(group.block4, 0, 4);
            }
          }
        }
      }

      ecc_ = bits(group.block3,  0, 8);
      cc_  = bits(group.block1, 12, 4);

      if (ecc_ != 0x00) {
        has_country_ = true;

        printf("country: \"%s\", ", getCountryString(pi_, ecc_).c_str());
      }

    } else if (slc_variant == 1) {
      tmc_id_ = bits(group.block3, 0, 12);
      printf("tmc_id: \"0x%03x\", ", tmc_id_);

    } else if (slc_variant == 2) {
      if (pager_tng_ != 0) {
        pager_pac_ = bits(group.block3, 0, 6);
        pager_opc_ = bits(group.block3, 8, 4);
      }

      // No PIN, section M.3.2.4.3
      if (group.num_blocks == 4 && (group.block4 >> 11) == 0) {
        int subtype = bits(group.block4, 10, 1);
        if (subtype == 0) {
          if (pager_tng_ != 0) {
            pager_pac_ = bits(group.block4, 4, 6);
            pager_opc_ = bits(group.block4, 0, 4);
          }
        } else if (subtype == 1) {
          if (pager_tng_ != 0) {
            int b = bits(group.block4, 8, 2);
            if (b == 0) {
              pager_ecc_ = bits(group.block4, 0, 6);
            } else if (b == 3) {
              pager_ccf_ = bits(group.block4, 0, 4);
            }
          }
        }
      }

    } else if (slc_variant == 3) {
      lang_ = bits(group.block3, 0, 8);
      printf("language: \"%s\", ", getLanguageString(lang_).c_str());

    } else if (slc_variant == 6) {
      // TODO:
      // broadcaster data

    } else if (slc_variant == 7) {
      ews_channel_ = bits(group.block3, 0, 12);
      printf("ews: \"0x%03x\", ", ews_channel_);
    }

  }

}

void Station::decodeType2 (Group group) {

  if (group.num_blocks < 3)
    return;

  int rt_position = bits(group.block2, 0, 4) * (group.type.ab == TYPE_A ? 4 : 2);
  int prev_textAB = rt_ab_;
  rt_ab_          = bits(group.block2, 4, 1);

  if (prev_textAB != rt_ab_)
    rt_.clear();

  std::string chars;

  if (group.type.ab == TYPE_A) {
    updateRadioText(rt_position, {bits(group.block3, 8, 8), bits(group.block3, 0, 8)});
  }

  if (group.num_blocks == 4) {
    updateRadioText(rt_position+2, {bits(group.block4, 8, 8), bits(group.block4, 0, 8)});
  }

}

void Station::decodeType3 (Group group) {

  if (group.num_blocks < 4)
    return;

  if (group.type.ab != TYPE_A)
    return;

  GroupType oda_group(bits(group.block2, 0, 5));
  uint16_t oda_msg = group.block3;
  uint16_t oda_aid = group.block4;

  oda_app_for_group_[oda_group] = oda_aid;

  printf("open_data_app: { group: \"%s\", app_name: \"%s\", message: \"0x%02x\" }, ",
      oda_group.toString().c_str(), getAppName(oda_aid).c_str(), group.block3);

}

void Station::decodeType4 (Group group) {

  if (group.num_blocks < 3 || group.type.ab == TYPE_B)
    return;

  int mjd = (bits(group.block2, 0, 2) << 15) + bits(group.block3, 1, 15);
  double lto;

  if (group.num_blocks == 4) {
    lto = (bits(group.block4, 5, 1) ? -1 : 1) * bits(group.block4, 0, 5) / 2.0;
    mjd = int(mjd + lto / 24.0);
  }

  int yr = int((mjd - 15078.2) / 365.25);
  int mo = int((mjd - 14956.1 - int(yr * 365.25)) / 30.6001);
  int dy = mjd - 14956 - int(yr * 365.25) - int(mo * 30.6001);
  if (mo == 14 || mo == 15) {
    yr += 1;
    mo -= 12;
  }
  yr += 1900;
  mo -= 1;

  if (group.num_blocks == 4) {
    int ltom = (lto - int(lto)) * 60;

    int hr = int((bits(group.block3, 0, 1) << 4) +
        bits(group.block4, 12, 14) + lto) % 24;
    int mn = bits(group.block4, 6, 6) + ltom;

    char buff[100];
    snprintf(buff, sizeof(buff),
        "%04d-%02d-%02dT%02d:%02d:00%+03d:%02d",yr,mo,dy,hr,mn,int(lto),ltom);
    clock_time_ = buff;
    printf("clock_time: \"%s\", ", clock_time_.c_str());

  }
}

void Station::decodeType8 (Group group) {
  if (oda_app_for_group_.find(group.type) == oda_app_for_group_.end())
    return;

  uint16_t aid = oda_app_for_group_[group.type];

  if (aid == 0xCD46 || aid == 0xCD47) {
    printf("tmc_message: \"0x%02x%04x%04x\", ", bits(group.block2, 0, 5), group.block3, group.block4);
  }

}

void Station::decodeType14 (Group group) {

}

} // namespace redsea
<commit_msg>formatting: commas in json<commit_after>#include "groups.h"

#include <cassert>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#include "data.h"

namespace redsea {

GroupType::GroupType(uint16_t type_code) : num((type_code >> 1) & 0xF), ab(type_code & 0x1) {}
GroupType::GroupType(const GroupType& obj) : num(obj.num), ab(obj.ab) {}

std::string GroupType::toString() {
  return std::string(std::to_string(num) + (ab == TYPE_A ? "A" : "B"));
}

bool operator==(const GroupType& obj1, const GroupType& obj2) {
  return (obj1.num == obj2.num && obj1.ab == obj2.ab);
}

bool operator<(const GroupType& obj1, const GroupType& obj2) { return ((obj1.num < obj2.num) || (obj1.ab < obj2.ab)); }

// extract len bits from bitstring, starting at starting_at from the right
uint16_t bits (uint16_t bitstring, int starting_at, int len) {
  return ((bitstring >> starting_at) & ((1<<len) - 1));
}

RDSString::RDSString(int len) : chars_(len), is_char_sequential_(len), prev_pos_(-1) {
  last_complete_string_ = getString();
}

void RDSString::setAt(int pos, int chr) {
  if (pos < 0 || pos >= (int)chars_.size())
    return;

  chars_[pos] = chr;

  if (pos != prev_pos_ + 1) {
    for (size_t i=0; i<is_char_sequential_.size(); i++)
      is_char_sequential_[i] = false;
  }

  is_char_sequential_[pos] = true;

  if (isComplete())
    last_complete_string_ = getString();

  prev_pos_ = pos;

}

size_t RDSString::lengthReceived() const {

  size_t result = 0;
  for (size_t i=0; i<is_char_sequential_.size(); i++) {
    if (!is_char_sequential_[i])
      break;
    result = i+1;
  }

  return result;
}

size_t RDSString::lengthExpected() const {

  size_t result = chars_.size();

  for (size_t i=0; i<chars_.size(); i++) {
    if (chars_[i] == 0x0D) {
      result = i;
      break;
    }
  }

  return result;
}

std::string RDSString::getString() const {
  std::string result;
  size_t len = lengthExpected();
  for (size_t i=0; i<len; i++) {
    result += (is_char_sequential_[i] ? getLCDchar(chars_[i]) : " ");
  }

  return result;

}

std::string RDSString::getLastCompleteString() const {
  return last_complete_string_;
}

bool RDSString::isComplete() const {
  return lengthReceived() >= lengthExpected();
}

void RDSString::clear() {
  for (size_t i=0; i<chars_.size(); i++) {
    is_char_sequential_[i] = false;
  }
  last_complete_string_ = getString();
}

Station::Station() : Station(0x0000) {

}

Station::Station(uint16_t _pi) : pi_(_pi), ps_(8), rt_(64) {

}

void Station::update(Group group) {

  is_tp_   = bits(group.block2, 10, 1);
  pty_     = bits(group.block2,  5, 5);

  printf("{ pi: \"0x%04x\", group: \"%s\"", pi_, group.type.toString().c_str());

  printf(", tp: %s", is_tp_ ? "true" : "false");
  printf(", prog_type: \"%s\"", getPTYname(pty_).c_str());

  if      (group.type.num == 0)  { decodeType0(group); }
  else if (group.type.num == 1)  { decodeType1(group); }
  else if (group.type.num == 2)  { decodeType2(group); }
  else if (group.type.num == 3)  { decodeType3(group); }
  else if (group.type.num == 4)  { decodeType4(group); }
  else if (group.type.num == 8)  { decodeType8(group); }
  else if (group.type.num == 14) { decodeType14(group); }
  else                           { printf("/* TODO */ "); }

  printf(" }\n");
}

void Station::addAltFreq(uint8_t af_code) {
  if (af_code >= 1 && af_code <= 204) {
    alt_freqs_.insert(87.5 + af_code / 10.0);
  } else if (af_code == 205) {
    // filler
  } else if (af_code == 224) {
    // no AF exists
  } else if (af_code >= 225 && af_code <= 249) {
    num_alt_freqs_ = af_code - 224;
  } else if (af_code == 250) {
    // AM/LF freq follows
  }
}

bool Station::hasPS() const {
  return ps_.isComplete();
}

std::string Station::getPS() const {
  return ps_.getLastCompleteString();
}

std::string Station::getRT() const {
  return rt_.getLastCompleteString();
}

uint16_t Station::getPI() const {
  return pi_;
}

std::string Station::getCountryCode() const {
  return getCountryString(pi_, ecc_);
}

void Station::updatePS(int pos, std::vector<int> chars) {

  for (int i=pos; i<pos+(int)chars.size(); i++)
    ps_.setAt(i, chars[i-pos]);

  if (ps_.isComplete())
    printf(", ps: \"%s\"",ps_.getLastCompleteString().c_str());

}

void Station::updateRadioText(int pos, std::vector<int> chars) {

  for (int i=pos; i<pos+(int)chars.size(); i++)
    rt_.setAt(i, chars[i-pos]);

  if (rt_.isComplete())
    printf(", radiotext: \"%s\"",rt_.getLastCompleteString().c_str());

}

void Station::decodeType0 (Group group) {

  // not implemented: Decoder Identification

  is_ta_    = bits(group.block2, 4, 1);
  is_music_ = bits(group.block2, 3, 1);

  printf(", ta: %s", is_ta_ ? "true" : "false");

  if (group.num_blocks < 3)
    return;

  if (group.type.ab == TYPE_A) {
    for (int i=0; i<2; i++) {
      addAltFreq(bits(group.block3, 8-i*8, 8));
    }

    if ((int)alt_freqs_.size() == num_alt_freqs_) {
      printf(", alt_freqs: [ ");
      int i = 0;
      for (auto f : alt_freqs_) {
        printf("%.1f", f);
        if (i < alt_freqs_.size() - 1)
          printf(", ");
        i++;
      }
      printf(" ]");
      alt_freqs_.clear();
    }
  }

  if (group.num_blocks < 4)
    return;

  updatePS(bits(group.block2, 0, 2) * 2,
      { bits(group.block4, 8, 8), bits(group.block4, 0, 8) });

}

void Station::decodeType1 (Group group) {

  if (group.num_blocks < 4)
    return;

  pin_ = group.block4;

  if (group.type.ab == TYPE_A) {
    pager_tng_ = bits(group.block2, 2, 3);
    if (pager_tng_ != 0) {
      pager_interval_ = bits(group.block2, 0, 2);
    }
    linkage_la_ = bits(group.block3, 15, 1);

    int slc_variant = bits(group.block3, 12, 3);

    if (slc_variant == 0) {
      if (pager_tng_ != 0) {
        pager_opc_ = bits(group.block3, 8, 4);
      }

      // No PIN, section M.3.2.4.3
      if (group.num_blocks == 4 && (group.block4 >> 11) == 0) {
        int subtype = bits(group.block4, 10, 1);
        if (subtype == 0) {
          if (pager_tng_ != 0) {
            pager_pac_ = bits(group.block4, 4, 6);
            pager_opc_ = bits(group.block4, 0, 4);
          }
        } else if (subtype == 1) {
          if (pager_tng_ != 0) {
            int b = bits(group.block4, 8, 2);
            if (b == 0) {
              pager_ecc_ = bits(group.block4, 0, 6);
            } else if (b == 3) {
              pager_ccf_ = bits(group.block4, 0, 4);
            }
          }
        }
      }

      ecc_ = bits(group.block3,  0, 8);
      cc_  = bits(group.block1, 12, 4);

      if (ecc_ != 0x00) {
        has_country_ = true;

        printf(", country: \"%s\"", getCountryString(pi_, ecc_).c_str());
      }

    } else if (slc_variant == 1) {
      tmc_id_ = bits(group.block3, 0, 12);
      printf(", tmc_id: \"0x%03x\"", tmc_id_);

    } else if (slc_variant == 2) {
      if (pager_tng_ != 0) {
        pager_pac_ = bits(group.block3, 0, 6);
        pager_opc_ = bits(group.block3, 8, 4);
      }

      // No PIN, section M.3.2.4.3
      if (group.num_blocks == 4 && (group.block4 >> 11) == 0) {
        int subtype = bits(group.block4, 10, 1);
        if (subtype == 0) {
          if (pager_tng_ != 0) {
            pager_pac_ = bits(group.block4, 4, 6);
            pager_opc_ = bits(group.block4, 0, 4);
          }
        } else if (subtype == 1) {
          if (pager_tng_ != 0) {
            int b = bits(group.block4, 8, 2);
            if (b == 0) {
              pager_ecc_ = bits(group.block4, 0, 6);
            } else if (b == 3) {
              pager_ccf_ = bits(group.block4, 0, 4);
            }
          }
        }
      }

    } else if (slc_variant == 3) {
      lang_ = bits(group.block3, 0, 8);
      printf(", language: \"%s\"", getLanguageString(lang_).c_str());

    } else if (slc_variant == 6) {
      // TODO:
      // broadcaster data

    } else if (slc_variant == 7) {
      ews_channel_ = bits(group.block3, 0, 12);
      printf(", ews: \"0x%03x\"", ews_channel_);
    }

  }

}

void Station::decodeType2 (Group group) {

  if (group.num_blocks < 3)
    return;

  int rt_position = bits(group.block2, 0, 4) * (group.type.ab == TYPE_A ? 4 : 2);
  int prev_textAB = rt_ab_;
  rt_ab_          = bits(group.block2, 4, 1);

  if (prev_textAB != rt_ab_)
    rt_.clear();

  std::string chars;

  if (group.type.ab == TYPE_A) {
    updateRadioText(rt_position, {bits(group.block3, 8, 8), bits(group.block3, 0, 8)});
  }

  if (group.num_blocks == 4) {
    updateRadioText(rt_position+2, {bits(group.block4, 8, 8), bits(group.block4, 0, 8)});
  }

}

void Station::decodeType3 (Group group) {

  if (group.num_blocks < 4)
    return;

  if (group.type.ab != TYPE_A)
    return;

  GroupType oda_group(bits(group.block2, 0, 5));
  uint16_t oda_msg = group.block3;
  uint16_t oda_aid = group.block4;

  oda_app_for_group_[oda_group] = oda_aid;

  printf(", open_data_app: { group: \"%s\", app_name: \"%s\", message: \"0x%02x\" }",
      oda_group.toString().c_str(), getAppName(oda_aid).c_str(), group.block3);

}

void Station::decodeType4 (Group group) {

  if (group.num_blocks < 3 || group.type.ab == TYPE_B)
    return;

  int mjd = (bits(group.block2, 0, 2) << 15) + bits(group.block3, 1, 15);
  double lto;

  if (group.num_blocks == 4) {
    lto = (bits(group.block4, 5, 1) ? -1 : 1) * bits(group.block4, 0, 5) / 2.0;
    mjd = int(mjd + lto / 24.0);
  }

  int yr = int((mjd - 15078.2) / 365.25);
  int mo = int((mjd - 14956.1 - int(yr * 365.25)) / 30.6001);
  int dy = mjd - 14956 - int(yr * 365.25) - int(mo * 30.6001);
  if (mo == 14 || mo == 15) {
    yr += 1;
    mo -= 12;
  }
  yr += 1900;
  mo -= 1;

  if (group.num_blocks == 4) {
    int ltom = (lto - int(lto)) * 60;

    int hr = int((bits(group.block3, 0, 1) << 4) +
        bits(group.block4, 12, 14) + lto) % 24;
    int mn = bits(group.block4, 6, 6) + ltom;

    char buff[100];
    snprintf(buff, sizeof(buff),
        "%04d-%02d-%02dT%02d:%02d:00%+03d:%02d",yr,mo,dy,hr,mn,int(lto),ltom);
    clock_time_ = buff;
    printf(", clock_time: \"%s\"", clock_time_.c_str());

  }
}

void Station::decodeType8 (Group group) {
  if (oda_app_for_group_.find(group.type) == oda_app_for_group_.end())
    return;

  uint16_t aid = oda_app_for_group_[group.type];

  if (aid == 0xCD46 || aid == 0xCD47) {
    printf(", tmc_message: \"0x%02x%04x%04x\"", bits(group.block2, 0, 5), group.block3, group.block4);
  }

}

void Station::decodeType14 (Group group) {

}

} // namespace redsea
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2009 Daniël de Kok
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA
 */

#include "maxent.ih"

// Active features in at least one context.
FeatureSet fsqueeze::activeFeatures(vector<FeatureSet> const &contextActiveFeatures)
{
	FeatureSet active;

	for (vector<FeatureSet>::const_iterator ctxIter = contextActiveFeatures.begin();
			ctxIter != contextActiveFeatures.end(); ++ctxIter)
		active.insert(ctxIter->begin(), ctxIter->end());

	return active;
}

void fsqueeze::adjustModel(DataSet const &dataSet, size_t feature,
	double alpha, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double fVal = featureVals.coeff(j, feature);
			if (fVal != 0.0)
			{
				(*zs)[i] -= (*sums)[i][j];
				(*sums)[i][j] *= exp(alpha * fVal);
				(*zs)[i] += (*sums)[i][j];
			}
		}

		++ctxIter; ++i;
	}
}

double fsqueeze::calcGain(DataSet const &dataSet,
	ExpectedValues const &expFeatureValues,
	Sums const &sums,
	Zs const &zs,
	size_t feature,
	double alpha
)
{
	double gainSum = 0.0;
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while(ctxIter != dataSet.contexts().end())
	{
		double newZ = zf(ctxIter->featureValues(), sums[i], zs[i], feature, alpha);
		double lg = ctxIter->prob() * log(newZ / zs[i]);
		gainSum -= lg;

		++ctxIter; ++i;
	}

	return gainSum + alpha * expFeatureValues[feature];
}

// Calculate the gain of adding each feature.
OrderedGains fsqueeze::calcGains(DataSet const &dataSet,
	vector<FeatureSet> const &contextActiveFeatures,
	ExpectedValues const &expFeatureValues,
	Sums const &sums,
	Zs const &zs,
	FeatureWeights const &alphas
)
{
	GainMap gainSum;

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	vector<FeatureSet>::const_iterator ctxFsIter = contextActiveFeatures.begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		for (FeatureSet::const_iterator fsIter = ctxFsIter->begin();
			fsIter != ctxFsIter->end(); ++fsIter)
		{
			int f = *fsIter;

			double newZ = zf(ctxIter->featureValues(), sums[i], zs[i], f,
				alphas[f]);

			double lg = ctxIter->prob() * log(newZ / zs[i]);

			gainSum[f] -= lg;
		}

		++ctxIter; ++ctxFsIter; ++i;
	}

	OrderedGains gains;
	for (int f = 0; f < alphas.rows(); ++f)
		gains.insert(make_pair(f, gainSum[f] + alphas[f] *
			expFeatureValues[f]));

	return gains;
}

vector<FeatureSet> fsqueeze::contextActiveFeatures(DataSet const &dataSet,
	FeatureSet const &excludedFeatures, Sums const &sums, Zs const &zs)
{
	vector<FeatureSet> ctxActive;

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureSet active;

		// This context can not have active features if its probability is zero.
		if (ctxIter->prob() == 0.0)
		{
			ctxActive.push_back(active);
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// This event can not have active features if its probability is zero.
			if (p_yx(sums[i][j], zs[i]) == 0.0)
				continue;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (excludedFeatures.find(fIter.index()) == excludedFeatures.end() &&
						fIter.value() != 0.0)
					active.insert(fIter.index());
		}

		ctxActive.push_back(active);

		++ctxIter; ++i;
	}

	return ctxActive;
}

ExpectedValues fsqueeze::expFeatureValues(DataSet const &dataSet)
{
	ExpectedValues expVals(dataSet.nFeatures());

	for (DsFeatureMap::const_iterator fIter = dataSet.features().begin();
		fIter != dataSet.features().end(); ++fIter)
	{
		double expVal = 0.0;
		for (std::vector<std::pair<double, double> >::const_iterator occIter =
				fIter->second.begin(); occIter != fIter->second.end();
				++occIter)
			expVal += occIter->first * occIter->second;
		expVals[fIter->first] = expVal;
	}

	return expVals;
}

ExpectedValues fsqueeze::expModelFeatureValues(
	DataSet const &dataSet,
	Sums const &sums, Zs const &zs)
{
	ExpectedValues expVals(dataSet.nFeatures());

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double pyx = p_yx(sums[i][j], zs[i]);

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				expVals[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	return expVals;
}

Zs fsqueeze::initialZs(DataSet const &ds)
{
	size_t nContexts = ds.contexts().size();
	ContextVector const &contexts = ds.contexts();

	Zs zs(nContexts);
	for (size_t i = 0; i < nContexts; ++i)
		zs[i] = contexts[i].eventProbs().size();

	return zs;
}

Sums fsqueeze::initialSums(DataSet const &ds)
{
	Sums sums;

	transform(ds.contexts().begin(), ds.contexts().end(), back_inserter(sums),
		makeSumVector());

	return sums;
}

double fsqueeze::zf(FeatureValues const &featureValues, Sum const &ctxSums,
	double z, size_t feature, double alpha)
{
	for (int i = 0; i < featureValues.outerSize(); ++i)
	{
		double fVal = featureValues.coeff(i, feature);
		if (fVal != 0.0)
			z = z - ctxSums[i] + ctxSums[i] * exp(alpha * fVal);
	}

	return z;
}
<commit_msg>Parallize calcGain.<commit_after>/*
 * Copyright (c) 2009 Daniël de Kok
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA
 */

#include "maxent.ih"

// NOTE
//
// Currently, only one loop here has an OpenMP annotation. Most other
// cases would require a critical section in an inner loop. So, let's
// not do it without proper benchmarking.

// Active features in at least one context.
FeatureSet fsqueeze::activeFeatures(vector<FeatureSet> const &contextActiveFeatures)
{
	FeatureSet active;

	for (vector<FeatureSet>::const_iterator ctxIter = contextActiveFeatures.begin();
			ctxIter != contextActiveFeatures.end(); ++ctxIter)
		active.insert(ctxIter->begin(), ctxIter->end());

	return active;
}

void fsqueeze::adjustModel(DataSet const &dataSet, size_t feature,
	double alpha, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double fVal = featureVals.coeff(j, feature);
			if (fVal != 0.0)
			{
				(*zs)[i] -= (*sums)[i][j];
				(*sums)[i][j] *= exp(alpha * fVal);
				(*zs)[i] += (*sums)[i][j];
			}
		}

		++ctxIter; ++i;
	}
}

double fsqueeze::calcGain(DataSet const &dataSet,
	ExpectedValues const &expFeatureValues,
	Sums const &sums,
	Zs const &zs,
	size_t feature,
	double alpha
)
{
	double gainSum = 0.0;
	ContextVector const &contexts = dataSet.contexts();

	#pragma omp parallel for
	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], feature, alpha);
		double lg = contexts[i].prob() * log(newZ / zs[i]);

		#pragma omp atomic
		gainSum -= lg;
	}

	return gainSum + alpha * expFeatureValues[feature];
}

// Calculate the gain of adding each feature.
OrderedGains fsqueeze::calcGains(DataSet const &dataSet,
	vector<FeatureSet> const &contextActiveFeatures,
	ExpectedValues const &expFeatureValues,
	Sums const &sums,
	Zs const &zs,
	FeatureWeights const &alphas
)
{
	GainMap gainSum;

	ContextVector const &contexts = dataSet.contexts();

	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		for (FeatureSet::const_iterator fsIter = contextActiveFeatures[i].begin();
			fsIter != contextActiveFeatures[i].end(); ++fsIter)
		{
			int f = *fsIter;

			double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], f,
				alphas[f]);

			double lg = contexts[i].prob() * log(newZ / zs[i]);

			gainSum[f] -= lg;
		}
	}

	OrderedGains gains;
	for (int f = 0; f < alphas.rows(); ++f)
		gains.insert(make_pair(f, gainSum[f] + alphas[f] *
			expFeatureValues[f]));

	return gains;
}

vector<FeatureSet> fsqueeze::contextActiveFeatures(DataSet const &dataSet,
	FeatureSet const &excludedFeatures, Sums const &sums, Zs const &zs)
{
	vector<FeatureSet> ctxActive;

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureSet active;

		// This context can not have active features if its probability is zero.
		if (ctxIter->prob() == 0.0)
		{
			ctxActive.push_back(active);
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// This event can not have active features if its probability is zero.
			if (p_yx(sums[i][j], zs[i]) == 0.0)
				continue;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (excludedFeatures.find(fIter.index()) == excludedFeatures.end() &&
						fIter.value() != 0.0)
					active.insert(fIter.index());
		}

		ctxActive.push_back(active);

		++ctxIter; ++i;
	}

	return ctxActive;
}

ExpectedValues fsqueeze::expFeatureValues(DataSet const &dataSet)
{
	ExpectedValues expVals(dataSet.nFeatures());

	for (DsFeatureMap::const_iterator fIter = dataSet.features().begin();
		fIter != dataSet.features().end(); ++fIter)
	{
		double expVal = 0.0;
		for (std::vector<std::pair<double, double> >::const_iterator occIter =
				fIter->second.begin(); occIter != fIter->second.end();
				++occIter)
			expVal += occIter->first * occIter->second;
		expVals[fIter->first] = expVal;
	}

	return expVals;
}

ExpectedValues fsqueeze::expModelFeatureValues(
	DataSet const &dataSet,
	Sums const &sums, Zs const &zs)
{
	ExpectedValues expVals(dataSet.nFeatures());

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double pyx = p_yx(sums[i][j], zs[i]);

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				expVals[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	return expVals;
}

Zs fsqueeze::initialZs(DataSet const &ds)
{
	size_t nContexts = ds.contexts().size();
	ContextVector const &contexts = ds.contexts();

	Zs zs(nContexts);
	for (size_t i = 0; i < nContexts; ++i)
		zs[i] = contexts[i].eventProbs().size();

	return zs;
}

Sums fsqueeze::initialSums(DataSet const &ds)
{
	Sums sums;

	transform(ds.contexts().begin(), ds.contexts().end(), back_inserter(sums),
		makeSumVector());

	return sums;
}

double fsqueeze::zf(FeatureValues const &featureValues, Sum const &ctxSums,
	double z, size_t feature, double alpha)
{
	for (int i = 0; i < featureValues.outerSize(); ++i)
	{
		double fVal = featureValues.coeff(i, feature);
		if (fVal != 0.0)
			z = z - ctxSums[i] + ctxSums[i] * exp(alpha * fVal);
	}

	return z;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2009 Daniël de Kok
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA
 */

#include "maxent.ih"

// NOTE
//
// Currently, only one loop here has an OpenMP annotation. Most other
// cases would require a critical section in an inner loop. So, let's
// not do it without proper benchmarking.

// Active features in at least one context.
FeatureSet fsqueeze::activeFeatures(vector<FeatureSet> const &contextActiveFeatures)
{
	FeatureSet active;

	for (vector<FeatureSet>::const_iterator ctxIter = contextActiveFeatures.begin();
			ctxIter != contextActiveFeatures.end(); ++ctxIter)
		active.insert(ctxIter->begin(), ctxIter->end());

	return active;
}

void fsqueeze::adjustModel(DataSet const &dataSet, size_t feature,
	double alpha, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double fVal = featureVals.coeff(j, feature);
			if (fVal != 0.0)
			{
				(*zs)[i] -= (*sums)[i][j];
				(*sums)[i][j] *= exp(alpha * fVal);
				(*zs)[i] += (*sums)[i][j];
			}
		}

		++ctxIter; ++i;
	}
}

void fsqueeze::adjustModelFull(DataSet const &dataSet, FeatureSet const &featureSet,
	Eigen::VectorXd const &lambdas, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		(*zs)[i] = 0.0;

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double sum = 0.0;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet.find(fIter.index()) != featureSet.end())
					sum += fIter.value() * lambdas[fIter.index()];

			sum = exp(sum);
			(*sums)[i][j] = sum;
			(*zs)[i] += sum;
		}

		++ctxIter; ++i;
	}
}

double fsqueeze::calcGain(DataSet const &dataSet,
	Sums const &sums,
	Zs const &zs,
	size_t feature,
	double alpha
)
{
	double gainSum = 0.0;
	ContextVector const &contexts = dataSet.contexts();

	#pragma omp parallel for
	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], feature, alpha);
		double lg = contexts[i].prob() * log(newZ / zs[i]);

		#pragma omp atomic
		gainSum -= lg;
	}

	return gainSum + alpha * dataSet.expFeatureValues()[feature];
}

// Calculate the gain of adding each feature.
OrderedGains fsqueeze::calcGains(DataSet const &dataSet,
	vector<FeatureSet> const &contextActiveFeatures,
	Sums const &sums,
	Zs const &zs,
	FeatureWeights const &alphas
)
{
	GainMap gainSum;

	ContextVector const &contexts = dataSet.contexts();

	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		for (FeatureSet::const_iterator fsIter = contextActiveFeatures[i].begin();
			fsIter != contextActiveFeatures[i].end(); ++fsIter)
		{
			int f = *fsIter;

			double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], f,
				alphas[f]);

			double lg = contexts[i].prob() * log(newZ / zs[i]);

			gainSum[f] -= lg;
		}
	}

	OrderedGains gains;
	for (int f = 0; f < alphas.rows(); ++f)
		gains.insert(make_pair(f, gainSum[f] + alphas[f] *
			dataSet.expFeatureValues()[f]));

	return gains;
}

vector<FeatureSet> fsqueeze::contextActiveFeatures(DataSet const &dataSet,
	FeatureSet const &excludedFeatures, Sums const &sums, Zs const &zs)
{
	vector<FeatureSet> ctxActive;

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureSet active;

		// This context can not have active features if its probability is zero.
		if (ctxIter->prob() == 0.0)
		{
			ctxActive.push_back(active);
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// This event can not have active features if its probability is zero.
			if (p_yx(sums[i][j], zs[i]) == 0.0)
				continue;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (excludedFeatures.find(fIter.index()) == excludedFeatures.end() &&
						fIter.value() != 0.0)
					active.insert(fIter.index());
		}

		ctxActive.push_back(active);

		++ctxIter; ++i;
	}

	return ctxActive;
}

ExpectedValues fsqueeze::expFeatureValues(DsFeatureMap const &features, int nFeatures)
{
	ExpectedValues expVals(nFeatures);

	for (DsFeatureMap::const_iterator fIter = features.begin();
		fIter != features.end(); ++fIter)
	{
		double expVal = 0.0;
		for (std::vector<std::pair<double, double> >::const_iterator occIter =
				fIter->second.begin(); occIter != fIter->second.end();
				++occIter)
			expVal += occIter->first * occIter->second;
		expVals[fIter->first] = expVal;
	}

	return expVals;
}

ExpectedValues fsqueeze::expModelFeatureValues(
	DataSet const &dataSet,
	Sums const &sums, Zs const &zs)
{
	ExpectedValues expVals(dataSet.nFeatures());

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double pyx = p_yx(sums[i][j], zs[i]);

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				expVals[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	return expVals;
}

Zs fsqueeze::initialZs(DataSet const &ds)
{
	size_t nContexts = ds.contexts().size();
	ContextVector const &contexts = ds.contexts();

	Zs zs(nContexts);
	for (size_t i = 0; i < nContexts; ++i)
		zs[i] = contexts[i].eventProbs().size();

	return zs;
}

Sums fsqueeze::initialSums(DataSet const &ds)
{
	Sums sums;

	transform(ds.contexts().begin(), ds.contexts().end(), back_inserter(sums),
		makeSumVector());

	return sums;
}

lbfgsfloatval_t lbfgs_maxent_evaluate(void *instance, lbfgsfloatval_t const *x, lbfgsfloatval_t *g,
	int const n, lbfgsfloatval_t const step);

int lbfgs_maxent_progress(void *instance, const lbfgsfloatval_t *x, const lbfgsfloatval_t *g,
	const lbfgsfloatval_t fx, const lbfgsfloatval_t xnorm, const lbfgsfloatval_t gnorm,
	const lbfgsfloatval_t step, int n, int k, int ls);

struct EvaluateData
{
	DataSet const *dataSet;
	FeatureSet const *featureSet;
};

Eigen::VectorXd fsqueeze::lbfgs_maxent(DataSet const &dataSet, FeatureSet const &featureSet)
{
	lbfgsfloatval_t *x = lbfgs_malloc(dataSet.nFeatures());

	lbfgs_parameter_t param;
	lbfgs_parameter_init(&param);

	EvaluateData evalData = {&dataSet, &featureSet};
	int r = lbfgs(dataSet.nFeatures(), x, 0, lbfgs_maxent_evaluate, lbfgs_maxent_progress,
		const_cast<void *>(reinterpret_cast<void const *>(&evalData)), &param);

	cerr << "r: " << r << endl;
//	cerr << LBFGSERR_ROUNDING_ERROR << endl;

	Eigen::VectorXd weights(dataSet.nFeatures());

	for (int i = 0; i < weights.size(); ++i)
		weights[i] = x[i];

	cerr << "ngram: " << x[3833] << endl;

	lbfgs_free(x);

	return weights;
}

lbfgsfloatval_t lbfgs_maxent_evaluate(void *instance, lbfgsfloatval_t const *x, lbfgsfloatval_t *g,
	int const n, lbfgsfloatval_t const step)
{
	/*
	for (int i = 0; i < n; ++i)
		if (x[i] != 0.0)
			cerr << "x[" << i << "] = " << x[i] << endl;
	*/
	EvaluateData const *evalData = reinterpret_cast<EvaluateData const *>(instance);
	DataSet const *dataSet = evalData->dataSet;
	FeatureSet const *featureSet = evalData->featureSet;

	Eigen::VectorXd const &expVals = dataSet->expFeatureValues();
	for (int i = 0; i < n; ++ i)
		if (featureSet->find(i) != featureSet->end())
			g[i] = -expVals[i];

	//Eigen::VectorXd expModelVals = Eigen::VectorXd::Zero(dataSet->nFeatures());
	lbfgsfloatval_t ll = 0.0;

	ContextVector::const_iterator ctxIter = dataSet->contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet->contexts().end())
	{
		// Skip contexts that have a probability of zero. If we allow such
		// contexts, we can not calculate empirical p(y|x).
		if (ctxIter->prob() == 0.0) {
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		int nEvents = ctxIter->eventProbs().size();

		Eigen::VectorXd sums = Eigen::VectorXd::Zero(nEvents);
		double z = 0.0;

		// Calculate unnormalized probabilities, and the normalizer (Z(x)).
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet->find(fIter.index()) != featureSet->end())
					sums[j] += x[fIter.index()] * fIter.value();

			sums[j] = exp(sums[j]);
			z += sums[j];
		}

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// Conditional probability of the event y, given the context x.
			double pyx = p_yx(sums[j], z);

			ll += ctxIter->eventProbs()[j] * log(pyx);

			// Contribution of this context to p(f).
			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet->find(fIter.index()) != featureSet->end())
					g[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	// Calculate gradients of the log-likelihood functions for a feature,
	// given the parameter x[i].
	/*
	Eigen::VectorXd const &expVals = dataSet->expFeatureValues();
	for (int i = 0; i < expModelVals.size(); ++i)
		if (featureSet->find(i) != featureSet->end()) {
			cerr << "expVals[i] = " << expVals[i] << ", expModelVals[i] = " << expModelVals[i] << endl;
			g[i] = expVals[i] - expModelVals[i];
			cerr << "g[" << i << "] = " << g[i] << endl;
		}
    */

	//cerr << "-log_likelihood: " << -ll << endl;

	return -ll;
}

int lbfgs_maxent_progress(void *instance, const lbfgsfloatval_t *x, const lbfgsfloatval_t *g,
	const lbfgsfloatval_t fx, const lbfgsfloatval_t xnorm, const lbfgsfloatval_t gnorm,
	const lbfgsfloatval_t step, int n, int k, int ls)
{
	cerr << "lbfgs iteration: " << k << endl;
	cerr << "fx = " << fx << endl;

	return 0;
}

double fsqueeze::zf(FeatureValues const &featureValues, Sum const &ctxSums,
	double z, size_t feature, double alpha)
{
	for (int i = 0; i < featureValues.outerSize(); ++i)
	{
		double fVal = featureValues.coeff(i, feature);
		if (fVal != 0.0)
			z = z - ctxSums[i] + ctxSums[i] * exp(alpha * fVal);
	}

	return z;
}
<commit_msg>Cleanup maxent evaluation function.<commit_after>/*
 * Copyright (c) 2009 Daniël de Kok
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA
 */

#include "maxent.ih"

// NOTE
//
// Currently, only one loop here has an OpenMP annotation. Most other
// cases would require a critical section in an inner loop. So, let's
// not do it without proper benchmarking.

// Active features in at least one context.
FeatureSet fsqueeze::activeFeatures(vector<FeatureSet> const &contextActiveFeatures)
{
	FeatureSet active;

	for (vector<FeatureSet>::const_iterator ctxIter = contextActiveFeatures.begin();
			ctxIter != contextActiveFeatures.end(); ++ctxIter)
		active.insert(ctxIter->begin(), ctxIter->end());

	return active;
}

void fsqueeze::adjustModel(DataSet const &dataSet, size_t feature,
	double alpha, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double fVal = featureVals.coeff(j, feature);
			if (fVal != 0.0)
			{
				(*zs)[i] -= (*sums)[i][j];
				(*sums)[i][j] *= exp(alpha * fVal);
				(*zs)[i] += (*sums)[i][j];
			}
		}

		++ctxIter; ++i;
	}
}

void fsqueeze::adjustModelFull(DataSet const &dataSet, FeatureSet const &featureSet,
	Eigen::VectorXd const &lambdas, Sums *sums, Zs *zs)
{
	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		(*zs)[i] = 0.0;

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double sum = 0.0;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet.find(fIter.index()) != featureSet.end())
					sum += fIter.value() * lambdas[fIter.index()];

			sum = exp(sum);
			(*sums)[i][j] = sum;
			(*zs)[i] += sum;
		}

		++ctxIter; ++i;
	}
}

double fsqueeze::calcGain(DataSet const &dataSet,
	Sums const &sums,
	Zs const &zs,
	size_t feature,
	double alpha
)
{
	double gainSum = 0.0;
	ContextVector const &contexts = dataSet.contexts();

	#pragma omp parallel for
	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], feature, alpha);
		double lg = contexts[i].prob() * log(newZ / zs[i]);

		#pragma omp atomic
		gainSum -= lg;
	}

	return gainSum + alpha * dataSet.expFeatureValues()[feature];
}

// Calculate the gain of adding each feature.
OrderedGains fsqueeze::calcGains(DataSet const &dataSet,
	vector<FeatureSet> const &contextActiveFeatures,
	Sums const &sums,
	Zs const &zs,
	FeatureWeights const &alphas
)
{
	GainMap gainSum;

	ContextVector const &contexts = dataSet.contexts();

	for (int i = 0; i < static_cast<int>(dataSet.contexts().size()); ++i)
	{
		for (FeatureSet::const_iterator fsIter = contextActiveFeatures[i].begin();
			fsIter != contextActiveFeatures[i].end(); ++fsIter)
		{
			int f = *fsIter;

			double newZ = zf(contexts[i].featureValues(), sums[i], zs[i], f,
				alphas[f]);

			double lg = contexts[i].prob() * log(newZ / zs[i]);

			gainSum[f] -= lg;
		}
	}

	OrderedGains gains;
	for (int f = 0; f < alphas.rows(); ++f)
		gains.insert(make_pair(f, gainSum[f] + alphas[f] *
			dataSet.expFeatureValues()[f]));

	return gains;
}

vector<FeatureSet> fsqueeze::contextActiveFeatures(DataSet const &dataSet,
	FeatureSet const &excludedFeatures, Sums const &sums, Zs const &zs)
{
	vector<FeatureSet> ctxActive;

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureSet active;

		// This context can not have active features if its probability is zero.
		if (ctxIter->prob() == 0.0)
		{
			ctxActive.push_back(active);
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// This event can not have active features if its probability is zero.
			if (p_yx(sums[i][j], zs[i]) == 0.0)
				continue;

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (excludedFeatures.find(fIter.index()) == excludedFeatures.end() &&
						fIter.value() != 0.0)
					active.insert(fIter.index());
		}

		ctxActive.push_back(active);

		++ctxIter; ++i;
	}

	return ctxActive;
}

ExpectedValues fsqueeze::expFeatureValues(DsFeatureMap const &features, int nFeatures)
{
	ExpectedValues expVals(nFeatures);

	for (DsFeatureMap::const_iterator fIter = features.begin();
		fIter != features.end(); ++fIter)
	{
		double expVal = 0.0;
		for (std::vector<std::pair<double, double> >::const_iterator occIter =
				fIter->second.begin(); occIter != fIter->second.end();
				++occIter)
			expVal += occIter->first * occIter->second;
		expVals[fIter->first] = expVal;
	}

	return expVals;
}

ExpectedValues fsqueeze::expModelFeatureValues(
	DataSet const &dataSet,
	Sums const &sums, Zs const &zs)
{
	ExpectedValues expVals(dataSet.nFeatures());

	ContextVector::const_iterator ctxIter = dataSet.contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet.contexts().end())
	{
		FeatureValues const &featureVals = ctxIter->featureValues();

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			double pyx = p_yx(sums[i][j], zs[i]);

			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				expVals[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	return expVals;
}

Zs fsqueeze::initialZs(DataSet const &ds)
{
	size_t nContexts = ds.contexts().size();
	ContextVector const &contexts = ds.contexts();

	Zs zs(nContexts);
	for (size_t i = 0; i < nContexts; ++i)
		zs[i] = contexts[i].eventProbs().size();

	return zs;
}

Sums fsqueeze::initialSums(DataSet const &ds)
{
	Sums sums;

	transform(ds.contexts().begin(), ds.contexts().end(), back_inserter(sums),
		makeSumVector());

	return sums;
}

lbfgsfloatval_t lbfgs_maxent_evaluate(void *instance, lbfgsfloatval_t const *x, lbfgsfloatval_t *g,
	int const n, lbfgsfloatval_t const step);

int lbfgs_maxent_progress(void *instance, const lbfgsfloatval_t *x, const lbfgsfloatval_t *g,
	const lbfgsfloatval_t fx, const lbfgsfloatval_t xnorm, const lbfgsfloatval_t gnorm,
	const lbfgsfloatval_t step, int n, int k, int ls);

struct EvaluateData
{
	DataSet const *dataSet;
	FeatureSet const *featureSet;
};

Eigen::VectorXd fsqueeze::lbfgs_maxent(DataSet const &dataSet, FeatureSet const &featureSet)
{
	lbfgsfloatval_t *x = lbfgs_malloc(dataSet.nFeatures());

	lbfgs_parameter_t param;
	lbfgs_parameter_init(&param);

	EvaluateData evalData = {&dataSet, &featureSet};
	int r = lbfgs(dataSet.nFeatures(), x, 0, lbfgs_maxent_evaluate, lbfgs_maxent_progress,
		const_cast<void *>(reinterpret_cast<void const *>(&evalData)), &param);

	cerr << "r: " << r << endl;
//	cerr << LBFGSERR_ROUNDING_ERROR << endl;

	Eigen::VectorXd weights(dataSet.nFeatures());

	for (int i = 0; i < weights.size(); ++i)
		weights[i] = x[i];

	cerr << "ngram: " << x[3833] << endl;

	lbfgs_free(x);

	return weights;
}

lbfgsfloatval_t lbfgs_maxent_evaluate(void *instance, lbfgsfloatval_t const *x, lbfgsfloatval_t *g,
	int const n, lbfgsfloatval_t const step)
{
	EvaluateData const *evalData = reinterpret_cast<EvaluateData const *>(instance);
	DataSet const *dataSet = evalData->dataSet;
	FeatureSet const *featureSet = evalData->featureSet;

	Eigen::VectorXd const &expVals = dataSet->expFeatureValues();
	for (int i = 0; i < n; ++ i)
		if (featureSet->find(i) != featureSet->end())
			g[i] = -expVals[i];

	lbfgsfloatval_t ll = 0.0;

	ContextVector::const_iterator ctxIter = dataSet->contexts().begin();
	size_t i = 0;
	while (ctxIter != dataSet->contexts().end())
	{
		// Skip contexts that have a probability of zero. If we allow such
		// contexts, we can not calculate empirical p(y|x).
		if (ctxIter->prob() == 0.0) {
			++ctxIter; ++i;
			continue;
		}

		FeatureValues const &featureVals = ctxIter->featureValues();
		int nEvents = ctxIter->eventProbs().size();

		Eigen::VectorXd sums = Eigen::VectorXd::Zero(nEvents);
		double z = 0.0;

		// Calculate unnormalized probabilities, and the normalizer (Z(x)).
		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet->find(fIter.index()) != featureSet->end())
					sums[j] += x[fIter.index()] * fIter.value();

			sums[j] = exp(sums[j]);
			z += sums[j];
		}

		for (int j = 0; j < featureVals.outerSize(); ++j)
		{
			// Conditional probability of the event y, given the context x.
			double pyx = p_yx(sums[j], z);

			// Update log-likelihood of the model.
			ll += ctxIter->eventProbs()[j] * log(pyx);

			// Contribution of this context to p(f).
			for (FeatureValues::InnerIterator fIter(featureVals, j);
					fIter; ++fIter)
				if (featureSet->find(fIter.index()) != featureSet->end())
					g[fIter.index()] += ctxIter->prob() * pyx * fIter.value();
		}

		++ctxIter; ++i;
	}

	return -ll;
}

int lbfgs_maxent_progress(void *instance, const lbfgsfloatval_t *x, const lbfgsfloatval_t *g,
	const lbfgsfloatval_t fx, const lbfgsfloatval_t xnorm, const lbfgsfloatval_t gnorm,
	const lbfgsfloatval_t step, int n, int k, int ls)
{
	cerr << "lbfgs iteration: " << k << endl;
	cerr << "fx = " << fx << endl;

	return 0;
}

double fsqueeze::zf(FeatureValues const &featureValues, Sum const &ctxSums,
	double z, size_t feature, double alpha)
{
	for (int i = 0; i < featureValues.outerSize(); ++i)
	{
		double fVal = featureValues.coeff(i, feature);
		if (fVal != 0.0)
			z = z - ctxSums[i] + ctxSums[i] * exp(alpha * fVal);
	}

	return z;
}
<|endoftext|>