<commit_before>/*******************************************************************************
* ALMA - Atacama Large Millimiter Array
* (c) European Southern Observatory, 2011
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*
* "@(#) $Id: bulkDataNTReceiverFlow.cpp,v 1.27 2013/02/07 13:30:58 bjeram Exp $"
*
* who       when      what
* --------  --------  ----------------------------------------------
* bjeram  2011-04-19  created
*/

#include "bulkDataNTReceiverFlow.h"
#include <iostream>
#include "bulkDataNTReceiverStream.h"
#include <AV/FlowSpec_Entry.h>  // we need it for TAO_Tokenizer ??

static char *rcsId="@(#) $Id: bulkDataNTReceiverFlow.cpp,v 1.27 2013/02/07 13:30:58 bjeram Exp $";
static void *use_rcsId = ((void)&use_rcsId,(void *) &rcsId);

using namespace AcsBulkdata;
using namespace std;

BulkDataNTReceiverFlow::BulkDataNTReceiverFlow(BulkDataNTReceiverStreamBase *receiverStream,
    const char* flowName,
    const ReceiverFlowConfiguration &rcvCfg,
    BulkDataNTCallback *cb,
    bool releaseCB) :
    BulkDataNTFlow(flowName),
    receiverStream_m(receiverStream),
    callback_m(cb), releaseCB_m(releaseCB),
    rcvCfg_m(rcvCfg)
{
  AUTO_TRACE(__PRETTY_FUNCTION__);
  std::string streamName, topicName;
  streamName = receiverStream_m->getName();
  ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Going to create Receiver Flow: %s @ Stream: %s ...", flowName_m.c_str(), streamName.c_str()));

  callback_m->setStreamName(receiverStream_m->getName().c_str());
  callback_m->setFlowName(flowName);
  callback_m->setReceiverName(receiverStream_m->getReceiverName());
  callback_m->setCBReceiveProcessTimeout(rcvCfg.getCbReceiveProcessTimeout());
  callback_m->setCBReceiveAvgProcessTimeout(rcvCfg.getCbReceiveAvgProcessTimeout());

  receiverStream->addDDSQoSProfile(rcvCfg_m);

  if (!rcvCfg_m.isEnableMulticast() &&  rcvCfg_m.getUnicastPort()==ReceiverFlowConfiguration::DEFAULT_UNICAST_PORT/*=0*/) //unicast && no unicast port was defined
  {
	  rcvCfg_m.setUnicastPort(receiverStream_m->getNextFlowUnicastPort()); //re-set unicast port on local(!) copy
  }
  // should be refactor to have just one object for comunication !! DDSDataWriter or similar
  ddsSubscriber_m = new BulkDataNTDDSSubscriber(receiverStream_m->getDDSParticipant(), rcvCfg_m);

  topicName =  streamName + "#" + flowName_m;
  ddsTopic_m = ddsSubscriber_m->createDDSTopic(topicName.c_str());

  dataReaderListener_m = new BulkDataNTReaderListener(topicName.c_str(), callback_m);

  ddsDataReader_m= ddsSubscriber_m->createDDSReader(ddsTopic_m, dataReaderListener_m);
  ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Receiver Flow: %s @ Stream: %s has been created.", flowName_m.c_str(), streamName.c_str()));
}//BulkDataNTReceiverFlow

BulkDataNTReceiverFlow::~BulkDataNTReceiverFlow()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	std::string streamName = receiverStream_m->getName();

	receiverStream_m->removeFlowFromMap(flowName_m.c_str());

	// this part can go to BulkDataNTDDSPublisher, anyway we need to refactor
	DDS::DomainParticipant *participant = receiverStream_m->getDDSParticipant();
	if (participant!=0)
	{
		ddsSubscriber_m->destroyDDSReader(ddsDataReader_m);
		ddsDataReader_m = 0;
		delete dataReaderListener_m;

		ddsSubscriber_m->destroyDDSTopic(ddsTopic_m);
		ddsTopic_m = 0;
	}
	else
	{
		ACS_SHORT_LOG((LM_ERROR, "Problem deleting data reader and topic because participant is NULL"));
	}
	delete ddsSubscriber_m;
	if (releaseCB_m) delete callback_m;

	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Receiver Flow: %s @ stream: %s has been destroyed.", flowName_m.c_str(), streamName.c_str()));
}//~BulkDataNTReceiverFlow

void BulkDataNTReceiverFlow::setReceiverName(char* recvName)
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	std::string oldReceiverName = callback_m->getReceiverName();
	callback_m->setReceiverName(receiverStream_m->getReceiverName());
	ACS_LOG(LM_RUNTIME_CONTEXT, __PRETTY_FUNCTION__, (LM_DEBUG, "Receiver name on callback for flow: %s on steam: %s has been changed from: %s to %s",
			flowName_m.c_str(), receiverStream_m->getName().c_str(),
			oldReceiverName.c_str(), recvName
		));
}//setReceiverName

void AcsBulkdata::BulkDataNTReceiverFlow::enableCallingCB()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	dataReaderListener_m->enableCallingCB();
}

void AcsBulkdata::BulkDataNTReceiverFlow::disableCallingCB()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	dataReaderListener_m->disableCallingCB();
}

void AcsBulkdata::BulkDataNTReceiverFlow::dumpStatistics()
{
	DDS::DataReaderProtocolStatus drps;
	DDS::DataReaderCacheStatus drcs;

	ddsDataReader_m->get_datareader_protocol_status(drps);

	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__,
			(LM_DEBUG, "DataReader protocol status for flow: %s [sample received: %lld (%lld). Sent: HB: %lld (%lld) ACKs: %lld (%lld) NACKs: %lld (%lld).",
					flowName_m.c_str(),
					drps.received_sample_count_change, drps.received_sample_bytes_change,
					drps.received_heartbeat_count_change, drps.received_heartbeat_bytes_change,
					drps.sent_ack_count_change, drps.sent_ack_bytes_change,
					drps.sent_nack_count_change, drps.sent_nack_bytes_change,
					drps.rejected_sample_count_change));

	ddsDataReader_m->get_datareader_cache_status(drcs);
	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__,
			(LM_DEBUG, "DataReader cache Status: sample count (peak): %lld (%lld)", drcs.sample_count, drcs.sample_count_peak));

}//void dumpStatistics()

/*___oOo___*/
<commit_msg>fiexd  formating in dumpStatistics()<commit_after>/*******************************************************************************
* ALMA - Atacama Large Millimiter Array
* (c) European Southern Observatory, 2011
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*
* "@(#) $Id: bulkDataNTReceiverFlow.cpp,v 1.28 2013/02/07 13:34:04 bjeram Exp $"
*
* who       when      what
* --------  --------  ----------------------------------------------
* bjeram  2011-04-19  created
*/

#include "bulkDataNTReceiverFlow.h"
#include <iostream>
#include "bulkDataNTReceiverStream.h"
#include <AV/FlowSpec_Entry.h>  // we need it for TAO_Tokenizer ??

static char *rcsId="@(#) $Id: bulkDataNTReceiverFlow.cpp,v 1.28 2013/02/07 13:34:04 bjeram Exp $";
static void *use_rcsId = ((void)&use_rcsId,(void *) &rcsId);

using namespace AcsBulkdata;
using namespace std;

BulkDataNTReceiverFlow::BulkDataNTReceiverFlow(BulkDataNTReceiverStreamBase *receiverStream,
    const char* flowName,
    const ReceiverFlowConfiguration &rcvCfg,
    BulkDataNTCallback *cb,
    bool releaseCB) :
    BulkDataNTFlow(flowName),
    receiverStream_m(receiverStream),
    callback_m(cb), releaseCB_m(releaseCB),
    rcvCfg_m(rcvCfg)
{
  AUTO_TRACE(__PRETTY_FUNCTION__);
  std::string streamName, topicName;
  streamName = receiverStream_m->getName();
  ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Going to create Receiver Flow: %s @ Stream: %s ...", flowName_m.c_str(), streamName.c_str()));

  callback_m->setStreamName(receiverStream_m->getName().c_str());
  callback_m->setFlowName(flowName);
  callback_m->setReceiverName(receiverStream_m->getReceiverName());
  callback_m->setCBReceiveProcessTimeout(rcvCfg.getCbReceiveProcessTimeout());
  callback_m->setCBReceiveAvgProcessTimeout(rcvCfg.getCbReceiveAvgProcessTimeout());

  receiverStream->addDDSQoSProfile(rcvCfg_m);

  if (!rcvCfg_m.isEnableMulticast() &&  rcvCfg_m.getUnicastPort()==ReceiverFlowConfiguration::DEFAULT_UNICAST_PORT/*=0*/) //unicast && no unicast port was defined
  {
	  rcvCfg_m.setUnicastPort(receiverStream_m->getNextFlowUnicastPort()); //re-set unicast port on local(!) copy
  }
  // should be refactor to have just one object for comunication !! DDSDataWriter or similar
  ddsSubscriber_m = new BulkDataNTDDSSubscriber(receiverStream_m->getDDSParticipant(), rcvCfg_m);

  topicName =  streamName + "#" + flowName_m;
  ddsTopic_m = ddsSubscriber_m->createDDSTopic(topicName.c_str());

  dataReaderListener_m = new BulkDataNTReaderListener(topicName.c_str(), callback_m);

  ddsDataReader_m= ddsSubscriber_m->createDDSReader(ddsTopic_m, dataReaderListener_m);
  ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Receiver Flow: %s @ Stream: %s has been created.", flowName_m.c_str(), streamName.c_str()));
}//BulkDataNTReceiverFlow

BulkDataNTReceiverFlow::~BulkDataNTReceiverFlow()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	std::string streamName = receiverStream_m->getName();

	receiverStream_m->removeFlowFromMap(flowName_m.c_str());

	// this part can go to BulkDataNTDDSPublisher, anyway we need to refactor
	DDS::DomainParticipant *participant = receiverStream_m->getDDSParticipant();
	if (participant!=0)
	{
		ddsSubscriber_m->destroyDDSReader(ddsDataReader_m);
		ddsDataReader_m = 0;
		delete dataReaderListener_m;

		ddsSubscriber_m->destroyDDSTopic(ddsTopic_m);
		ddsTopic_m = 0;
	}
	else
	{
		ACS_SHORT_LOG((LM_ERROR, "Problem deleting data reader and topic because participant is NULL"));
	}
	delete ddsSubscriber_m;
	if (releaseCB_m) delete callback_m;

	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__, (LM_INFO, "Receiver Flow: %s @ stream: %s has been destroyed.", flowName_m.c_str(), streamName.c_str()));
}//~BulkDataNTReceiverFlow

void BulkDataNTReceiverFlow::setReceiverName(char* recvName)
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	std::string oldReceiverName = callback_m->getReceiverName();
	callback_m->setReceiverName(receiverStream_m->getReceiverName());
	ACS_LOG(LM_RUNTIME_CONTEXT, __PRETTY_FUNCTION__, (LM_DEBUG, "Receiver name on callback for flow: %s on steam: %s has been changed from: %s to %s",
			flowName_m.c_str(), receiverStream_m->getName().c_str(),
			oldReceiverName.c_str(), recvName
		));
}//setReceiverName

void AcsBulkdata::BulkDataNTReceiverFlow::enableCallingCB()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	dataReaderListener_m->enableCallingCB();
}

void AcsBulkdata::BulkDataNTReceiverFlow::disableCallingCB()
{
	AUTO_TRACE(__PRETTY_FUNCTION__);
	dataReaderListener_m->disableCallingCB();
}

void AcsBulkdata::BulkDataNTReceiverFlow::dumpStatistics()
{
	DDS::DataReaderProtocolStatus drps;
	DDS::DataReaderCacheStatus drcs;

	ddsDataReader_m->get_datareader_protocol_status(drps);

	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__,
			(LM_DEBUG, "DataReader protocol status for flow: %s [sample received: %lld (%lld). Sent: HB: %lld (%lld) ACKs: %lld (%lld) NACKs: %lld (%lld). Rejected: %lld]",
					flowName_m.c_str(),
					drps.received_sample_count_change, drps.received_sample_bytes_change,
					drps.received_heartbeat_count_change, drps.received_heartbeat_bytes_change,
					drps.sent_ack_count_change, drps.sent_ack_bytes_change,
					drps.sent_nack_count_change, drps.sent_nack_bytes_change,
					drps.rejected_sample_count_change));

	ddsDataReader_m->get_datareader_cache_status(drcs);
	ACS_LOG(LM_RUNTIME_CONTEXT, __FUNCTION__,
			(LM_DEBUG, "DataReader cache Status: sample count (peak): %lld (%lld)", drcs.sample_count, drcs.sample_count_peak));

}//void dumpStatistics()

/*___oOo___*/
<|endoftext|>


<commit_before>/*
 * opencog/learning/PatternMiner/PatternMinerDistributedWorker.cc
 *
 * Copyright (C) 2012 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Shujing Ke <rainkekekeke@gmail.com> in 2015
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include <stdlib.h>

#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <vector>
#include <sstream>
#include <thread>
#include <ifaddrs.h>

#include <opencog/atoms/base/ClassServer.h>
#include <opencog/atoms/base/Handle.h>
#include <opencog/atoms/base/atom_types.h>
#include <opencog/spacetime/atom_types.h>
#include <opencog/embodiment/atom_types.h>
#include <opencog/query/BindLinkAPI.h>
#include <opencog/util/Config.h>

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#include "DistributedPatternMiner.h"

using namespace utility;                    // Common utilities like string conversions
using namespace web;                        // Common features like URIs.
using namespace web::http;                  // Common HTTP functionality
using namespace web::http::client;          // HTTP client features
using namespace concurrency::streams;       // Asynchronous streams

using namespace opencog::PatternMining;
using namespace opencog;

void DistributedPatternMiner::launchADistributedWorker()
{

    run_as_central_server = false;
    run_as_distributed_worker = true;

    centralServerIP = config().get("PMCentralServerIP");
    centralServerPort = config().get("PMCentralServerPort");
    centralServerBaseURL = "http://" +  centralServerIP + ":" + centralServerPort + "/PatternMinerServer";

    boost::uuids::random_generator uuidGen;
    boost::uuids::uuid uid = uuidGen();
    std::stringstream ssuid;
    ssuid << uid;
    clientWorkerUID = ssuid.str();
    std::cout<<"Current client UID = "<< clientWorkerUID << std::endl;
    std::cout<<"Registering to the central server: "<< centralServerIP << std::endl;

    // Create http_client to send the request.
    httpClient = new http_client(U(centralServerBaseURL.c_str()));

    // Build request URI and start the request.
    uri_builder builder(U("/RegisterNewWorker"));
    builder.append_query(U("ClientUID"), U(clientWorkerUID));

    http_request request(methods::POST);
    request.set_request_uri(builder.to_uri());
    http_response response;
    if (sendRequest(request, response))
    {
        // std::cout << response.to_string() << std::endl;
        if (response.status_code() == status_codes::OK)
        {
            std::cout << "Registered to the central server successfully! " << std::endl;
            startMiningWork();
        }
        else
        {
            std::cout << "Registered to the central server failed! Please check network and the the state of the central server.Client application quited!" << std::endl;
            std::exit(EXIT_SUCCESS);
        }
    }
    else
    {
        std::cout << "Network problem. Cannot connect to the central server! Client application quited!" << std::endl;
        std::exit(EXIT_SUCCESS);
    }

}

void DistributedPatternMiner::notifyServerThisWorkerStop()
{

    int tryTimes = 0;
    int maxTryTimes = 10;

    while (tryTimes ++ < maxTryTimes)
    {
        // Build request URI and start the request.
        uri_builder builder(U("/ReportWorkerStop"));
        builder.append_query(U("ClientUID"), U(clientWorkerUID));

        http_request request(methods::POST);
        request.set_request_uri(builder.to_uri());
        http_response response;

        if (sendRequest(request, response))
        {
            std::cout << response.to_string() << std::endl;
            if (response.status_code() == status_codes::OK)
            {
                std::cout << "Report to the central server this worker stopped successfully! " << std::endl;
                break;
            }
            else
            {
                std::cout << "Network problem: Failed report to the central server this worker stopped!" << std::endl;
            }
        }
        else
        {
            std::cout << "Network problem: Failed report to the central server this worker stopped!" << std::endl;
        }

        if (tryTimes < maxTryTimes)
        {
            std::cout << "Wait for 5 seconds and retry..."  << std::endl;
            sleep(5);
        }
        else
        {
            std::cout << "Already tried " << maxTryTimes << " times. Enter 'y' to keep trying, enter others to quit."  << std::endl;

            string inputstr;
            cin  >> inputstr;

            if ( (inputstr == "y") or  (inputstr == "yes") )
            {
                maxTryTimes += 10;
                std::cout << "Continue trying..."  << std::endl;
                sleep(5);
            }
            else
            {
                break;
            }

        }

    }

    cout << "Client quited!" << std::endl;
    std::exit(EXIT_SUCCESS);

}

void DistributedPatternMiner::startMiningWork()
{
    // Note: Breadth_First mining is not maintained anymore. Only Depth_First is used in distributed mining.
    //    Pattern_mining_mode = config().get("Pattern_mining_mode"); // option: Breadth_First , Depth_First
    //    assert( (Pattern_mining_mode == "Breadth_First") || (Pattern_mining_mode == "Depth_First"));
        Pattern_mining_mode = "Depth_First";

        cur_worker_mined_pattern_num = 0;

        std::cout << "Start pattern mining work! Max gram = "
                  << this->MAX_GRAM << ", mode = Depth_First" << std::endl;

        int start_time = time(NULL);

        originalAtomSpace->get_handles_by_type(back_inserter(allLinks), (Type) LINK, true );

        allLinkNumber = (int)(allLinks.size());
        atomspaceSizeFloat = (float)(allLinkNumber);

        runPatternMinerDepthFirst();

        int end_time = time(NULL);
        printf("Current pattern mining worker finished working! Total time: %d seconds. \n", end_time - start_time);

        notifyServerThisWorkerStop();

        std::cout << THREAD_NUM << " threads used. "
                  <<"Corpus size: "<< allLinkNumber << " links in total. \n"
                  << cur_worker_mined_pattern_num << " patterns mined in total." << std::endl;

        // std::exit(EXIT_SUCCESS);

}

void DistributedPatternMiner::runPatternMinerDepthFirst()
{
    // observingAtomSpace is used to copy one link everytime from the originalAtomSpace
    observingAtomSpace = new AtomSpace();

//    cur_DF_ExtractedLinks = new set<string>[MAX_GRAM];

    linksPerThread = allLinkNumber / THREAD_NUM;

    processedLinkNum = 0;
    actualProcessedLinkNum = 0;

    for (unsigned int i = 0; i < THREAD_NUM; ++ i)
    {

        threads[i] = std::thread(&DistributedPatternMiner::growPatternsDepthFirstTask,this,i);
        // threads[i] = std::thread([this]{this->growPatternsDepthFirstTask(i);}); // using C++11 lambda-expression
    }

    for (unsigned int i = 0; i < THREAD_NUM; ++ i)
    {
        threads[i].join();
    }

    // release allLinks
    allLinks.clear();
    (HandleSeq()).swap(allLinks);

//    delete [] cur_DF_ExtractedLinks;
    delete [] threads;

    cout << "\nFinished mining 1~" << MAX_GRAM << " gram patterns.\n";
    cout << "\nprocessedLinkNum = " << processedLinkNum << std::endl;

    delete [] patternJsonArrays;
    cout << "Totally "<< cur_worker_mined_pattern_num << " patterns found!\n";
}

void DistributedPatternMiner::growPatternsDepthFirstTask(unsigned int thread_index)
{

    // The start index in allLinks for current thread
    unsigned int start_index = linksPerThread * thread_index;
    unsigned int end_index; // the last index for current thread (excluded)
    if (thread_index == THREAD_NUM - 1) // if this the last thread, it
                                        // needs to finish all the
                                        // rest of the links
        end_index = allLinkNumber;
    else
        end_index = linksPerThread * (thread_index + 1);

    cout << "Start thread " << thread_index << " from " << start_index
         << " to (excluded) " << end_index << std::endl;

    patternJsonArrays[thread_index] = json::value::array();

    float allLinkNumberfloat = ((float)(end_index - start_index));
    for(unsigned int t_cur_index = start_index; t_cur_index < end_index; ++t_cur_index)
    {
        readNextLinkLock.lock();
        cout<< "\r" << ((float)(t_cur_index - start_index))/allLinkNumberfloat*100.0f << "% completed in Thread " + toString(thread_index) + "."; // it's not liner
        std::cout.flush();

        processedLinkNum ++;
        Handle& cur_link = allLinks[t_cur_index];

        readNextLinkLock.unlock();

        // if this link is listlink, ignore it
        if (cur_link->getType() == opencog::LIST_LINK)
        {
            continue;
        }

        // Add this link into observingAtomSpace
        HandleSeq outgoingLinks, outVariableNodes;

        swapOneLinkBetweenTwoAtomSpace(originalAtomSpace, observingAtomSpace, cur_link, outgoingLinks, outVariableNodes);
        Handle newLink = observingAtomSpace->add_link(cur_link->getType(), outgoingLinks);
        newLink->merge(cur_link->getTruthValue());

        // Extract all the possible patterns from this originalLink, and extend till the max_gram links, not duplicating the already existing patterns
        HandleSeq lastGramLinks;
        map<Handle,Handle> lastGramValueToVarMap;
        map<Handle,Handle> patternVarMap;

        // allHTreeNodesCurTask is only used in distributed version;
        // is to store all the HTreeNode* mined in this current task, and release them after the task is finished.
        vector<HTreeNode*> allHTreeNodesCurTask;

        // allNewMinedPatternInfo is only used in distributed version, to store all the new mined patterns in this task for sending to the server.
        vector<MinedPatternInfo> allNewMinedPatternInfo;

        actualProcessedLinkLock.lock();
        actualProcessedLinkNum ++;
        actualProcessedLinkLock.unlock();

        extendAPatternForOneMoreGramRecursively(newLink, observingAtomSpace, Handle::UNDEFINED, lastGramLinks, 0, lastGramValueToVarMap,
                                                patternVarMap, false, allHTreeNodesCurTask, allNewMinedPatternInfo);

        cout << "test allNewMinedPatternInfo.size =  " << allNewMinedPatternInfo.size() << std::endl;
        // send new mined patterns to the server
        for (MinedPatternInfo& pInfo : allNewMinedPatternInfo)
            addPatternsToJsonArrayBuf(pInfo.curPatternKeyStr, pInfo.parentKeyString, pInfo.extendedLinkIndex, patternJsonArrays[thread_index]);

        // release all the HTreeNodes created in this task
        // clean up the pattern atomspace, do not need to keep patterns in atomspace when run as a distributed worker
        if (THREAD_NUM == 1)
            atomSpace->clear(); // can only clear the atomspace when only 1 thread is used

        for(unsigned int hNodeNum = 0; hNodeNum < allHTreeNodesCurTask.size(); hNodeNum ++)
        {
            delete (allHTreeNodesCurTask[hNodeNum]);
        }

    }

    if (patternJsonArrays[thread_index].size() > 0)
        sendPatternsToCentralServer(patternJsonArrays[thread_index]);

    cout<< "\r100% completed in Thread " + toString(thread_index) + ".";
    std::cout.flush();
}

void DistributedPatternMiner::addPatternsToJsonArrayBuf(string curPatternKeyStr, string parentKeyString,  unsigned int extendedLinkIndex, json::value &patternJsonArray)
{
    cout << "send pattern:" << curPatternKeyStr << std::endl;
    try
    {

        json::value patternInfo = json::value::object();
        patternInfo[U("Pattern")] = json::value(U(curPatternKeyStr));
        patternInfo[U("ParentPattern")] = json::value(U(parentKeyString));
        patternInfo[U("ExtendedLinkIndex")] = json::value(U(extendedLinkIndex));
        patternInfo[U("ClientUID")] = json::value(U(clientWorkerUID));
        patternInfo[U("ProcessedFactsNum")] = json::value(U(actualProcessedLinkNum));

        patternJsonArray[patternJsonArray.size()] = patternInfo;

        if(patternJsonArray.size() >= JSON_BUF_MAX_NUM)
            sendPatternsToCentralServer(patternJsonArray);

    }
    catch (exception const & e)
    {
       cout << e.what() << "addPatternsToJsonArrayBuf exception!: " << endl;

    }

    cur_worker_mined_pattern_num ++;
    // cout << "\nWorker added cur_worker_mined_pattern_num = " << cur_worker_mined_pattern_num << std::endl;
}

// this function will empty patternJsonArray after sent
void DistributedPatternMiner::sendPatternsToCentralServer(json::value &patternJsonArray)
{

    uri_builder builder(U("/FindNewPatterns"));

    while (true)
    {
        http_request request(methods::POST);
        request.set_request_uri(builder.to_uri());
        request.headers().add(header_names::accept, U("application/json"));
        request.set_body(patternJsonArray);

        http_response response;

        if (sendRequest(request, response))
        {
//            std::cout << response.to_string() << std::endl;
            if (response.status_code() == status_codes::OK)
            {
                break;
            }
            else
            {
                std::cout << "Network problem: Failed to send new found patterns! Retrying ... " << std::endl;
            }
        }
        else
        {
            std::cout << "Network problem: Failed to send new found patterns! Retrying ... " << std::endl;
        }

    }

    // empty the array
    patternJsonArray = json::value::array();

    usleep(20000);

}

// send request, get response back
bool DistributedPatternMiner::sendRequest(http_request &request, http_response &response)
{
    pplx::task<http_response> task = httpClient->request(request);

    try
    {
        if (task.wait() == pplx::task_status::completed)
        {
            response = task.get();
//            response.StatusCode = task.get().status_code();
//            pplx::task<web::json::value> contentTask = task.get().extract_json();
//            if (contentTask.wait() == task_status::completed)
//            {
//                response.JsonContent = contentTask.get();
//            }

            return true;
        }
        else
        {
            cout << "Network problem:http request task is not finished! " << std::endl;
            return false;
        }
    }
    catch (http_exception e)
    {
        cout << "http_exception:" << e.what() << std::endl;
        return false;
    }
    catch (exception e)
    {
        cout << "exception:" << e.what() << std::endl;
        return false;
    }

}

<commit_msg>Clean up some test output.<commit_after>/*
 * opencog/learning/PatternMiner/PatternMinerDistributedWorker.cc
 *
 * Copyright (C) 2012 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Shujing Ke <rainkekekeke@gmail.com> in 2015
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include <stdlib.h>

#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <vector>
#include <sstream>
#include <thread>
#include <ifaddrs.h>

#include <opencog/atoms/base/ClassServer.h>
#include <opencog/atoms/base/Handle.h>
#include <opencog/atoms/base/atom_types.h>
#include <opencog/spacetime/atom_types.h>
#include <opencog/embodiment/atom_types.h>
#include <opencog/query/BindLinkAPI.h>
#include <opencog/util/Config.h>

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#include "DistributedPatternMiner.h"

using namespace utility;                    // Common utilities like string conversions
using namespace web;                        // Common features like URIs.
using namespace web::http;                  // Common HTTP functionality
using namespace web::http::client;          // HTTP client features
using namespace concurrency::streams;       // Asynchronous streams

using namespace opencog::PatternMining;
using namespace opencog;

void DistributedPatternMiner::launchADistributedWorker()
{

    run_as_central_server = false;
    run_as_distributed_worker = true;

    centralServerIP = config().get("PMCentralServerIP");
    centralServerPort = config().get("PMCentralServerPort");
    centralServerBaseURL = "http://" +  centralServerIP + ":" + centralServerPort + "/PatternMinerServer";

    boost::uuids::random_generator uuidGen;
    boost::uuids::uuid uid = uuidGen();
    std::stringstream ssuid;
    ssuid << uid;
    clientWorkerUID = ssuid.str();
    std::cout<<"Current client UID = "<< clientWorkerUID << std::endl;
    std::cout<<"Registering to the central server: "<< centralServerIP << std::endl;

    // Create http_client to send the request.
    httpClient = new http_client(U(centralServerBaseURL.c_str()));

    // Build request URI and start the request.
    uri_builder builder(U("/RegisterNewWorker"));
    builder.append_query(U("ClientUID"), U(clientWorkerUID));

    http_request request(methods::POST);
    request.set_request_uri(builder.to_uri());
    http_response response;
    if (sendRequest(request, response))
    {
        // std::cout << response.to_string() << std::endl;
        if (response.status_code() == status_codes::OK)
        {
            std::cout << "Registered to the central server successfully! " << std::endl;
            startMiningWork();
        }
        else
        {
            std::cout << "Registered to the central server failed! Please check network and the the state of the central server.Client application quited!" << std::endl;
            std::exit(EXIT_SUCCESS);
        }
    }
    else
    {
        std::cout << "Network problem. Cannot connect to the central server! Client application quited!" << std::endl;
        std::exit(EXIT_SUCCESS);
    }

}

void DistributedPatternMiner::notifyServerThisWorkerStop()
{

    int tryTimes = 0;
    int maxTryTimes = 10;

    while (tryTimes ++ < maxTryTimes)
    {
        // Build request URI and start the request.
        uri_builder builder(U("/ReportWorkerStop"));
        builder.append_query(U("ClientUID"), U(clientWorkerUID));

        http_request request(methods::POST);
        request.set_request_uri(builder.to_uri());
        http_response response;

        if (sendRequest(request, response))
        {
            std::cout << response.to_string() << std::endl;
            if (response.status_code() == status_codes::OK)
            {
                std::cout << "Report to the central server this worker stopped successfully! " << std::endl;
                break;
            }
            else
            {
                std::cout << "Network problem: Failed report to the central server this worker stopped!" << std::endl;
            }
        }
        else
        {
            std::cout << "Network problem: Failed report to the central server this worker stopped!" << std::endl;
        }

        if (tryTimes < maxTryTimes)
        {
            std::cout << "Wait for 5 seconds and retry..."  << std::endl;
            sleep(5);
        }
        else
        {
            std::cout << "Already tried " << maxTryTimes << " times. Enter 'y' to keep trying, enter others to quit."  << std::endl;

            string inputstr;
            cin  >> inputstr;

            if ( (inputstr == "y") or  (inputstr == "yes") )
            {
                maxTryTimes += 10;
                std::cout << "Continue trying..."  << std::endl;
                sleep(5);
            }
            else
            {
                break;
            }

        }

    }

    cout << "Client quited!" << std::endl;
    std::exit(EXIT_SUCCESS);

}

void DistributedPatternMiner::startMiningWork()
{
    // Note: Breadth_First mining is not maintained anymore. Only Depth_First is used in distributed mining.
    //    Pattern_mining_mode = config().get("Pattern_mining_mode"); // option: Breadth_First , Depth_First
    //    assert( (Pattern_mining_mode == "Breadth_First") || (Pattern_mining_mode == "Depth_First"));
        Pattern_mining_mode = "Depth_First";

        cur_worker_mined_pattern_num = 0;

        std::cout << "Start pattern mining work! Max gram = "
                  << this->MAX_GRAM << ", mode = Depth_First" << std::endl;

        int start_time = time(NULL);

        originalAtomSpace->get_handles_by_type(back_inserter(allLinks), (Type) LINK, true );

        allLinkNumber = (int)(allLinks.size());
        atomspaceSizeFloat = (float)(allLinkNumber);

        runPatternMinerDepthFirst();

        int end_time = time(NULL);
        printf("Current pattern mining worker finished working! Total time: %d seconds. \n", end_time - start_time);

        notifyServerThisWorkerStop();

        std::cout << THREAD_NUM << " threads used. "
                  <<"Corpus size: "<< allLinkNumber << " links in total. \n"
                  << cur_worker_mined_pattern_num << " patterns mined in total." << std::endl;

        // std::exit(EXIT_SUCCESS);

}

void DistributedPatternMiner::runPatternMinerDepthFirst()
{
    // observingAtomSpace is used to copy one link everytime from the originalAtomSpace
    observingAtomSpace = new AtomSpace();

//    cur_DF_ExtractedLinks = new set<string>[MAX_GRAM];

    linksPerThread = allLinkNumber / THREAD_NUM;

    processedLinkNum = 0;
    actualProcessedLinkNum = 0;

    for (unsigned int i = 0; i < THREAD_NUM; ++ i)
    {

        threads[i] = std::thread(&DistributedPatternMiner::growPatternsDepthFirstTask,this,i);
        // threads[i] = std::thread([this]{this->growPatternsDepthFirstTask(i);}); // using C++11 lambda-expression
    }

    for (unsigned int i = 0; i < THREAD_NUM; ++ i)
    {
        threads[i].join();
    }

    // release allLinks
    allLinks.clear();
    (HandleSeq()).swap(allLinks);

//    delete [] cur_DF_ExtractedLinks;
    delete [] threads;

    cout << "\nFinished mining 1~" << MAX_GRAM << " gram patterns.\n";
    cout << "\nprocessedLinkNum = " << processedLinkNum << std::endl;

    delete [] patternJsonArrays;
    cout << "Totally "<< cur_worker_mined_pattern_num << " patterns found!\n";
}

void DistributedPatternMiner::growPatternsDepthFirstTask(unsigned int thread_index)
{

    // The start index in allLinks for current thread
    unsigned int start_index = linksPerThread * thread_index;
    unsigned int end_index; // the last index for current thread (excluded)
    if (thread_index == THREAD_NUM - 1) // if this the last thread, it
                                        // needs to finish all the
                                        // rest of the links
        end_index = allLinkNumber;
    else
        end_index = linksPerThread * (thread_index + 1);

    cout << "Start thread " << thread_index << " from " << start_index
         << " to (excluded) " << end_index << std::endl;

    patternJsonArrays[thread_index] = json::value::array();

    float allLinkNumberfloat = ((float)(end_index - start_index));
    for(unsigned int t_cur_index = start_index; t_cur_index < end_index; ++t_cur_index)
    {
        readNextLinkLock.lock();
        cout<< "\r" << ((float)(t_cur_index - start_index))/allLinkNumberfloat*100.0f << "% completed in Thread " + toString(thread_index) + "."; // it's not liner
        std::cout.flush();

        processedLinkNum ++;
        Handle& cur_link = allLinks[t_cur_index];

        readNextLinkLock.unlock();

        // if this link is listlink, ignore it
        if (cur_link->getType() == opencog::LIST_LINK)
        {
            continue;
        }

        // Add this link into observingAtomSpace
        HandleSeq outgoingLinks, outVariableNodes;

        swapOneLinkBetweenTwoAtomSpace(originalAtomSpace, observingAtomSpace, cur_link, outgoingLinks, outVariableNodes);
        Handle newLink = observingAtomSpace->add_link(cur_link->getType(), outgoingLinks);
        newLink->merge(cur_link->getTruthValue());

        // Extract all the possible patterns from this originalLink, and extend till the max_gram links, not duplicating the already existing patterns
        HandleSeq lastGramLinks;
        map<Handle,Handle> lastGramValueToVarMap;
        map<Handle,Handle> patternVarMap;

        // allHTreeNodesCurTask is only used in distributed version;
        // is to store all the HTreeNode* mined in this current task, and release them after the task is finished.
        vector<HTreeNode*> allHTreeNodesCurTask;

        // allNewMinedPatternInfo is only used in distributed version, to store all the new mined patterns in this task for sending to the server.
        vector<MinedPatternInfo> allNewMinedPatternInfo;

        actualProcessedLinkLock.lock();
        actualProcessedLinkNum ++;
        actualProcessedLinkLock.unlock();

        extendAPatternForOneMoreGramRecursively(newLink, observingAtomSpace, Handle::UNDEFINED, lastGramLinks, 0, lastGramValueToVarMap,
                                                patternVarMap, false, allHTreeNodesCurTask, allNewMinedPatternInfo);

        // send new mined patterns to the server
        for (MinedPatternInfo& pInfo : allNewMinedPatternInfo)
            addPatternsToJsonArrayBuf(pInfo.curPatternKeyStr, pInfo.parentKeyString, pInfo.extendedLinkIndex, patternJsonArrays[thread_index]);

        // release all the HTreeNodes created in this task
        // clean up the pattern atomspace, do not need to keep patterns in atomspace when run as a distributed worker
        if (THREAD_NUM == 1)
            atomSpace->clear(); // can only clear the atomspace when only 1 thread is used

        for(unsigned int hNodeNum = 0; hNodeNum < allHTreeNodesCurTask.size(); hNodeNum ++)
        {
            delete (allHTreeNodesCurTask[hNodeNum]);
        }

    }

    if (patternJsonArrays[thread_index].size() > 0)
        sendPatternsToCentralServer(patternJsonArrays[thread_index]);

    cout<< "\r100% completed in Thread " + toString(thread_index) + ".";
    std::cout.flush();
}

void DistributedPatternMiner::addPatternsToJsonArrayBuf(string curPatternKeyStr, string parentKeyString,  unsigned int extendedLinkIndex, json::value &patternJsonArray)
{

    try
    {

        json::value patternInfo = json::value::object();
        patternInfo[U("Pattern")] = json::value(U(curPatternKeyStr));
        patternInfo[U("ParentPattern")] = json::value(U(parentKeyString));
        patternInfo[U("ExtendedLinkIndex")] = json::value(U(extendedLinkIndex));
        patternInfo[U("ClientUID")] = json::value(U(clientWorkerUID));
        patternInfo[U("ProcessedFactsNum")] = json::value(U(actualProcessedLinkNum));

        patternJsonArray[patternJsonArray.size()] = patternInfo;

        if(patternJsonArray.size() >= JSON_BUF_MAX_NUM)
            sendPatternsToCentralServer(patternJsonArray);

    }
    catch (exception const & e)
    {
       cout << e.what() << "addPatternsToJsonArrayBuf exception!: " << endl;

    }

    cur_worker_mined_pattern_num ++;
    // cout << "\nWorker added cur_worker_mined_pattern_num = " << cur_worker_mined_pattern_num << std::endl;
}

// this function will empty patternJsonArray after sent
void DistributedPatternMiner::sendPatternsToCentralServer(json::value &patternJsonArray)
{

    uri_builder builder(U("/FindNewPatterns"));

    while (true)
    {
        http_request request(methods::POST);
        request.set_request_uri(builder.to_uri());
        request.headers().add(header_names::accept, U("application/json"));
        request.set_body(patternJsonArray);

        http_response response;

        if (sendRequest(request, response))
        {
//            std::cout << response.to_string() << std::endl;
            if (response.status_code() == status_codes::OK)
            {
                break;
            }
            else
            {
                std::cout << "Network problem: Failed to send new found patterns! Retrying ... " << std::endl;
            }
        }
        else
        {
            std::cout << "Network problem: Failed to send new found patterns! Retrying ... " << std::endl;
        }

    }

    // empty the array
    patternJsonArray = json::value::array();

    usleep(20000);

}

// send request, get response back
bool DistributedPatternMiner::sendRequest(http_request &request, http_response &response)
{
    pplx::task<http_response> task = httpClient->request(request);

    try
    {
        if (task.wait() == pplx::task_status::completed)
        {
            response = task.get();
//            response.StatusCode = task.get().status_code();
//            pplx::task<web::json::value> contentTask = task.get().extract_json();
//            if (contentTask.wait() == task_status::completed)
//            {
//                response.JsonContent = contentTask.get();
//            }

            return true;
        }
        else
        {
            cout << "Network problem:http request task is not finished! " << std::endl;
            return false;
        }
    }
    catch (http_exception e)
    {
        cout << "http_exception:" << e.what() << std::endl;
        return false;
    }
    catch (exception e)
    {
        cout << "exception:" << e.what() << std::endl;
        return false;
    }

}

<|endoftext|>


<commit_before>/*
 * opencog/learning/PatternMiner/PatternMinerDistributedWorker.cc
 *
 * Copyright (C) 2012 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Shujing Ke <rainkekekeke@gmail.com> in 2015
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include <stdlib.h>

#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <vector>
#include <sstream>
#include <thread>
#include <ifaddrs.h>

#include <opencog/atomspace/ClassServer.h>
#include <opencog/atomspace/Handle.h>
#include <opencog/atomspace/atom_types.h>
#include <opencog/spacetime/atom_types.h>
#include <opencog/embodiment/AtomSpaceExtensions/atom_types.h>
#include <opencog/query/BindLinkAPI.h>
#include <opencog/util/Config.h>
#include <opencog/util/StringManipulator.h>

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_io.hpp>

#include "HTree.h"
#include "PatternMiner.h"

using namespace utility;                    // Common utilities like string conversions
using namespace web;                        // Common features like URIs.
using namespace web::http;                  // Common HTTP functionality
using namespace web::http::client;          // HTTP client features
using namespace concurrency::streams;       // Asynchronous streams

using namespace opencog::PatternMining;
using namespace opencog;

void PatternMiner::launchADistributedWorker()
{
    centralServerIP = config().get("PMCentralServerIP");
    centralServerPort = config().get("PMCentralServerPort");
    centralServerBaseURL = "http://" +  centralServerIP + ":" + centralServerPort + "/PatternMinerServer";

    boost::uuids::uuid uid;
    std::stringstream ssuid;
    ssuid << uid;
    clientWorkerUID = ssuid.str();
    std::cout<<"Current client UID = "<< clientWorkerUID << std::endl;
    std::cout<<"Registering to the central server: "<< centralServerIP << std::endl;

    // Create http_client to send the request.
    httpClient = new http_client(U(centralServerBaseURL.c_str()));

    // Build request URI and start the request.
    uri_builder builder(U("/RegisterNewWorker"));
    builder.append_query(U("ClientUID"), U(clientWorkerUID));
    std::cout << httpClient->request(methods::GET, builder.to_string()).get().extract_json().get() ;

//// Note: Breadth_First mining is not maintained anymore. Only Depth_First is used in distributed mining.
////    Pattern_mining_mode = config().get("Pattern_mining_mode"); // option: Breadth_First , Depth_First
////    assert( (Pattern_mining_mode == "Breadth_First") || (Pattern_mining_mode == "Depth_First"));
//    Pattern_mining_mode = "Depth_First";

//    std::cout<<"A new PatternMining worker launched! Max gram = " + toString(this->MAX_GRAM) << ", mode = Depth_First" << std::endl;

//    int start_time = time(NULL);

//    originalAtomSpace->get_handles_by_type(back_inserter(allLinks), (Type) LINK, true );

//    allLinkNumber = (int)(allLinks.size());
//    atomspaceSizeFloat = (float)(allLinkNumber);

//    if (Pattern_mining_mode == "Breadth_First")
//        runPatternMinerBreadthFirst();
//    else
//    {
//        runPatternMinerDepthFirst();

//        if (enable_Frequent_Pattern)
//        {
//            std::cout<<"Debug: PatternMiner:  done frequent pattern mining for 1 to "<< MAX_GRAM <<"gram patterns!\n";

//            for(unsigned int gram = 1; gram <= MAX_GRAM; gram ++)
//            {
//                // sort by frequency
//                std::sort((patternsForGram[gram-1]).begin(), (patternsForGram[gram-1]).end(),compareHTreeNodeByFrequency );

//                // Finished mining gram patterns; output to file
//                std::cout<<"gram = " + toString(gram) + ": " + toString((patternsForGram[gram-1]).size()) + " patterns found! ";

//                OutPutFrequentPatternsToFile(gram);

//                std::cout<< std::endl;
//            }
//        }

//        if (enable_Interesting_Pattern)
//        {
//            for(cur_gram = 2; cur_gram <= MAX_GRAM - 1; cur_gram ++)
//            {
//                cout << "\nCalculating interestingness for " << cur_gram << " gram patterns by evaluating " << interestingness_Evaluation_method << std::endl;
//                cur_index = -1;
//                threads = new thread[THREAD_NUM];
//                num_of_patterns_without_superpattern_cur_gram = 0;

//                for (unsigned int i = 0; i < THREAD_NUM; ++ i)
//                {
//                    threads[i] = std::thread([this]{this->evaluateInterestingnessTask();}); // using C++11 lambda-expression
//                }

//                for (unsigned int i = 0; i < THREAD_NUM; ++ i)
//                {
//                    threads[i].join();
//                }

//                delete [] threads;

//                std::cout<<"Debug: PatternMiner:  done (gram = " + toString(cur_gram) + ") interestingness evaluation!" + toString((patternsForGram[cur_gram-1]).size()) + " patterns found! ";
//                std::cout<<"Outputting to file ... ";

//                if (interestingness_Evaluation_method == "Interaction_Information")
//                {
//                    // sort by interaction information
//                    std::sort((patternsForGram[cur_gram-1]).begin(), (patternsForGram[cur_gram-1]).end(),compareHTreeNodeByInteractionInformation);
//                    OutPutInterestingPatternsToFile(patternsForGram[cur_gram-1], cur_gram);
//                }
//                else if (interestingness_Evaluation_method == "surprisingness")
//                {
//                    // sort by surprisingness_I first
//                    std::sort((patternsForGram[cur_gram-1]).begin(), (patternsForGram[cur_gram-1]).end(),compareHTreeNodeBySurprisingness_I);
//                    OutPutInterestingPatternsToFile(patternsForGram[cur_gram-1], cur_gram,1);

//                    vector<HTreeNode*> curGramPatterns = patternsForGram[cur_gram-1];

//                    // and then sort by surprisingness_II
//                    std::sort(curGramPatterns.begin(), curGramPatterns.end(),compareHTreeNodeBySurprisingness_II);
//                    OutPutInterestingPatternsToFile(curGramPatterns,cur_gram,2);

//                    // Get the min threshold of surprisingness_II
//                    int threshold_index_II;
//                    threshold_index_II = SURPRISINGNESS_II_TOP_THRESHOLD * (float)(curGramPatterns.size() - num_of_patterns_without_superpattern_cur_gram);
//                    int looptimes = 0;
//                    while (true)
//                    {

//                        surprisingness_II_threshold = (curGramPatterns[threshold_index_II])->nII_Surprisingness;
//                        if (surprisingness_II_threshold <= 0.00000f)
//                        {
//                            if (++ looptimes > 8)
//                            {
//                                surprisingness_II_threshold = 0.00000f;
//                                break;
//                            }

//                            threshold_index_II = ((float)threshold_index_II) * SURPRISINGNESS_II_TOP_THRESHOLD;
//                        }
//                        else
//                            break;
//                    }

//                    cout<< "surprisingness_II_threshold for " << cur_gram << " gram = "<< surprisingness_II_threshold;

//                    // go through the top N patterns of surprisingness_I, pick the patterns with surprisingness_II higher than threshold
//                    int threshold_index_I = SURPRISINGNESS_I_TOP_THRESHOLD * (float)(curGramPatterns.size());
//                    for (int p = 0; p <= threshold_index_I; p ++)
//                    {
//                        HTreeNode* pNode = (patternsForGram[cur_gram-1])[p];

//                        // for patterns have no superpatterns, nII_Surprisingness == -1.0, which should be taken into account
//                        if ( (pNode->nII_Surprisingness < 0 ) || (pNode->nII_Surprisingness > surprisingness_II_threshold ) )
//                            finalPatternsForGram[cur_gram-1].push_back(pNode);
//                    }

//                    // sort by frequency
//                    std::sort((finalPatternsForGram[cur_gram-1]).begin(), (finalPatternsForGram[cur_gram-1]).end(),compareHTreeNodeByFrequency );

//                    OutPutFinalPatternsToFile(cur_gram);

//                }

//                std::cout<< std::endl;
//            }
//        }
//    }

//    int end_time = time(NULL);
//    printf("Pattern Mining Finish one round! Total time: %d seconds. \n", end_time - start_time);
//    std::cout<< THREAD_NUM << " threads used. \n";
//    std::cout<<"Corpus size: "<< allLinkNumber << " links in total. \n";

////   testPatternMatcher2();

////   selectSubsetFromCorpus();

}
<commit_msg>Register a worker to central server<commit_after>/*
 * opencog/learning/PatternMiner/PatternMinerDistributedWorker.cc
 *
 * Copyright (C) 2012 by OpenCog Foundation
 * All Rights Reserved
 *
 * Written by Shujing Ke <rainkekekeke@gmail.com> in 2015
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <math.h>
#include <stdlib.h>

#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <vector>
#include <sstream>
#include <thread>
#include <ifaddrs.h>

#include <opencog/atomspace/ClassServer.h>
#include <opencog/atomspace/Handle.h>
#include <opencog/atomspace/atom_types.h>
#include <opencog/spacetime/atom_types.h>
#include <opencog/embodiment/AtomSpaceExtensions/atom_types.h>
#include <opencog/query/BindLinkAPI.h>
#include <opencog/util/Config.h>
#include <opencog/util/StringManipulator.h>

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>

#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#include "HTree.h"
#include "PatternMiner.h"

using namespace utility;                    // Common utilities like string conversions
using namespace web;                        // Common features like URIs.
using namespace web::http;                  // Common HTTP functionality
using namespace web::http::client;          // HTTP client features
using namespace concurrency::streams;       // Asynchronous streams

using namespace opencog::PatternMining;
using namespace opencog;

void PatternMiner::launchADistributedWorker()
{
    centralServerIP = config().get("PMCentralServerIP");
    centralServerPort = config().get("PMCentralServerPort");
    centralServerBaseURL = "http://" +  centralServerIP + ":" + centralServerPort + "/PatternMinerServer";

    boost::uuids::random_generator uuidGen;
    boost::uuids::uuid uid = uuidGen();
    std::stringstream ssuid;
    ssuid << uid;
    clientWorkerUID = ssuid.str();
    std::cout<<"Current client UID = "<< clientWorkerUID << std::endl;
    std::cout<<"Registering to the central server: "<< centralServerIP << std::endl;

    // Create http_client to send the request.
    httpClient = new http_client(U(centralServerBaseURL.c_str()));

    // Build request URI and start the request.
    uri_builder builder(U("/RegisterNewWorker"));
    builder.append_query(U("ClientUID"), U(clientWorkerUID));

    httpClient->request(methods::GET, builder.to_string()).then([=](http_response response)
    {
        std::cout << response.to_string() << std::endl;
        if (response.status_code() == status_codes::OK)
        {
            std::cout << "Registered to the central server successfully!" << std::endl;
        }
        else
        {
            std::cout << "Registered to the central server failed! Please check network and the the state of the central server." << std::endl;
        }

        return;

    });

    std::cout << "Registered to the central server failed! Please check network and the the state of the central server." << std::endl;

//// Note: Breadth_First mining is not maintained anymore. Only Depth_First is used in distributed mining.
////    Pattern_mining_mode = config().get("Pattern_mining_mode"); // option: Breadth_First , Depth_First
////    assert( (Pattern_mining_mode == "Breadth_First") || (Pattern_mining_mode == "Depth_First"));
//    Pattern_mining_mode = "Depth_First";

//    std::cout<<"A new PatternMining worker launched! Max gram = " + toString(this->MAX_GRAM) << ", mode = Depth_First" << std::endl;

//    int start_time = time(NULL);

//    originalAtomSpace->get_handles_by_type(back_inserter(allLinks), (Type) LINK, true );

//    allLinkNumber = (int)(allLinks.size());
//    atomspaceSizeFloat = (float)(allLinkNumber);

//    if (Pattern_mining_mode == "Breadth_First")
//        runPatternMinerBreadthFirst();
//    else
//    {
//        runPatternMinerDepthFirst();

//        if (enable_Frequent_Pattern)
//        {
//            std::cout<<"Debug: PatternMiner:  done frequent pattern mining for 1 to "<< MAX_GRAM <<"gram patterns!\n";

//            for(unsigned int gram = 1; gram <= MAX_GRAM; gram ++)
//            {
//                // sort by frequency
//                std::sort((patternsForGram[gram-1]).begin(), (patternsForGram[gram-1]).end(),compareHTreeNodeByFrequency );

//                // Finished mining gram patterns; output to file
//                std::cout<<"gram = " + toString(gram) + ": " + toString((patternsForGram[gram-1]).size()) + " patterns found! ";

//                OutPutFrequentPatternsToFile(gram);

//                std::cout<< std::endl;
//            }
//        }

//        if (enable_Interesting_Pattern)
//        {
//            for(cur_gram = 2; cur_gram <= MAX_GRAM - 1; cur_gram ++)
//            {
//                cout << "\nCalculating interestingness for " << cur_gram << " gram patterns by evaluating " << interestingness_Evaluation_method << std::endl;
//                cur_index = -1;
//                threads = new thread[THREAD_NUM];
//                num_of_patterns_without_superpattern_cur_gram = 0;

//                for (unsigned int i = 0; i < THREAD_NUM; ++ i)
//                {
//                    threads[i] = std::thread([this]{this->evaluateInterestingnessTask();}); // using C++11 lambda-expression
//                }

//                for (unsigned int i = 0; i < THREAD_NUM; ++ i)
//                {
//                    threads[i].join();
//                }

//                delete [] threads;

//                std::cout<<"Debug: PatternMiner:  done (gram = " + toString(cur_gram) + ") interestingness evaluation!" + toString((patternsForGram[cur_gram-1]).size()) + " patterns found! ";
//                std::cout<<"Outputting to file ... ";

//                if (interestingness_Evaluation_method == "Interaction_Information")
//                {
//                    // sort by interaction information
//                    std::sort((patternsForGram[cur_gram-1]).begin(), (patternsForGram[cur_gram-1]).end(),compareHTreeNodeByInteractionInformation);
//                    OutPutInterestingPatternsToFile(patternsForGram[cur_gram-1], cur_gram);
//                }
//                else if (interestingness_Evaluation_method == "surprisingness")
//                {
//                    // sort by surprisingness_I first
//                    std::sort((patternsForGram[cur_gram-1]).begin(), (patternsForGram[cur_gram-1]).end(),compareHTreeNodeBySurprisingness_I);
//                    OutPutInterestingPatternsToFile(patternsForGram[cur_gram-1], cur_gram,1);

//                    vector<HTreeNode*> curGramPatterns = patternsForGram[cur_gram-1];

//                    // and then sort by surprisingness_II
//                    std::sort(curGramPatterns.begin(), curGramPatterns.end(),compareHTreeNodeBySurprisingness_II);
//                    OutPutInterestingPatternsToFile(curGramPatterns,cur_gram,2);

//                    // Get the min threshold of surprisingness_II
//                    int threshold_index_II;
//                    threshold_index_II = SURPRISINGNESS_II_TOP_THRESHOLD * (float)(curGramPatterns.size() - num_of_patterns_without_superpattern_cur_gram);
//                    int looptimes = 0;
//                    while (true)
//                    {

//                        surprisingness_II_threshold = (curGramPatterns[threshold_index_II])->nII_Surprisingness;
//                        if (surprisingness_II_threshold <= 0.00000f)
//                        {
//                            if (++ looptimes > 8)
//                            {
//                                surprisingness_II_threshold = 0.00000f;
//                                break;
//                            }

//                            threshold_index_II = ((float)threshold_index_II) * SURPRISINGNESS_II_TOP_THRESHOLD;
//                        }
//                        else
//                            break;
//                    }

//                    cout<< "surprisingness_II_threshold for " << cur_gram << " gram = "<< surprisingness_II_threshold;

//                    // go through the top N patterns of surprisingness_I, pick the patterns with surprisingness_II higher than threshold
//                    int threshold_index_I = SURPRISINGNESS_I_TOP_THRESHOLD * (float)(curGramPatterns.size());
//                    for (int p = 0; p <= threshold_index_I; p ++)
//                    {
//                        HTreeNode* pNode = (patternsForGram[cur_gram-1])[p];

//                        // for patterns have no superpatterns, nII_Surprisingness == -1.0, which should be taken into account
//                        if ( (pNode->nII_Surprisingness < 0 ) || (pNode->nII_Surprisingness > surprisingness_II_threshold ) )
//                            finalPatternsForGram[cur_gram-1].push_back(pNode);
//                    }

//                    // sort by frequency
//                    std::sort((finalPatternsForGram[cur_gram-1]).begin(), (finalPatternsForGram[cur_gram-1]).end(),compareHTreeNodeByFrequency );

//                    OutPutFinalPatternsToFile(cur_gram);

//                }

//                std::cout<< std::endl;
//            }
//        }
//    }

//    int end_time = time(NULL);
//    printf("Pattern Mining Finish one round! Total time: %d seconds. \n", end_time - start_time);
//    std::cout<< THREAD_NUM << " threads used. \n";
//    std::cout<<"Corpus size: "<< allLinkNumber << " links in total. \n";

////   testPatternMatcher2();

////   selectSubsetFromCorpus();

}
<|endoftext|>


<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2014.  All rights reserved
 */
#include    "../../StroikaPreComp.h"

#include    <cstdio>

#if     qPlatform_POSIX
#include    <unistd.h>
#include    <sys/types.h>
#include    <sys/socket.h>
#include    <sys/ioctl.h>
#include    <net/if.h>
#include    <netinet/in.h>
#include    <netdb.h>
#include    <arpa/inet.h>
#include    <netinet/in.h>
#include    <linux/netlink.h>
#include    <linux/rtnetlink.h>
#elif   qPlatform_Windows
#include    <WinSock2.h>
#include    <WS2tcpip.h>
#include    <Iphlpapi.h>
#include    <netioapi.h>
#endif

#include    "../../Characters/CString/Utilities.h"
#include    "../../Containers/Collection.h"
#include    "../../Execution/ErrNoException.h"
#include    "../../Execution/Thread.h"
#if     qPlatform_Windows
#include    "../../../Foundation/Execution/Platform/Windows/Exception.h"
#endif

#include    "Socket.h"

#include    "LinkMonitor.h"

using   namespace   Stroika::Foundation;
using   namespace   Stroika::Foundation::Memory;
using   namespace   Stroika::Foundation::IO;
using   namespace   Stroika::Foundation::IO::Network;

#if 0
// FOR POSIX DO SOMETHING LIKE THIS:
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/in.h>
#include <arpa/inet.h>

static const char* flags (int sd, const char* name)
{
    static char buf[1024];

    static struct ifreq ifreq;
    strcpy (ifreq.ifr_name, name);

    int r = ioctl (sd, SIOCGIFFLAGS, (char*)&ifreq);
    assert (r == 0);

    int l = 0;
#define FLAG(b) if(ifreq.ifr_flags & b) l += snprintf(buf + l, sizeof(buf) - l, #b " ")
    FLAG (IFF_UP);
    FLAG (IFF_BROADCAST);
    FLAG (IFF_DEBUG);
    FLAG (IFF_LOOPBACK);
    FLAG (IFF_POINTOPOINT);
    FLAG (IFF_RUNNING);
    FLAG (IFF_NOARP);
    FLAG (IFF_PROMISC);
    FLAG (IFF_NOTRAILERS);
    FLAG (IFF_ALLMULTI);
    FLAG (IFF_MASTER);
    FLAG (IFF_SLAVE);
    FLAG (IFF_MULTICAST);
    FLAG (IFF_PORTSEL);
    FLAG (IFF_AUTOMEDIA);
    FLAG (IFF_DYNAMIC);
#undef FLAG

    return buf;
}

int main (void)
{
    static struct ifreq ifreqs[32];
    struct ifconf ifconf;
    memset (&ifconf, 0, sizeof(ifconf));
    ifconf.ifc_req = ifreqs;
    ifconf.ifc_len = sizeof(ifreqs);

    int sd = socket (PF_INET, SOCK_STREAM, 0);
    assert (sd >= 0);

    int r = ioctl (sd, SIOCGIFCONF, (char*)&ifconf);
    assert (r == 0);

    for (int i = 0; i < ifconf.ifc_len / sizeof(struct ifreq); ++i) {
        printf ("%s: %s\n", ifreqs[i].ifr_name, inet_ntoa (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr));
        printf (" flags: %s\n", flags (sd, ifreqs[i].ifr_name));
    }

    close (sd);

    return 0;
}
#endif

#if     qPlatform_Windows
// /SEE THIS COD EOFR WINDOWS
//http ://support.microsoft.com/default.aspx?scid=http://support.microsoft.com:80/support/kb/articles/Q129/3/15.asp&NoWebContent=1
#endif

#if     qPlatform_Windows
namespace {
    bool    sStartedUp_ = false;
    bool    sAutoSetup_ =   true;

    void    CheckStarup_ ()
    {
        if (not sStartedUp_) {
            WSADATA wsaData;        // Initialize Winsock
            int iResult = WSAStartup (MAKEWORD (2, 2), &wsaData);
            if (iResult != 0) {
                Execution::Platform::Windows::Exception::DoThrow (::WSAGetLastError ());
            }
            sStartedUp_ = true;
        }
    }
}
#endif

InternetAddress Network::GetPrimaryInternetAddress ()
{
    /// HORRIBLY KLUDGY BAD IMPL!!!
#if     qPlatform_Windows
    CheckStarup_ ();
#if 0
    DWORD TEST = GetComputerNameEx((COMPUTER_NAME_FORMAT)cnf, buffer, &dwSize))
#endif
    char ac[1024];
    if (gethostname (ac, sizeof(ac)) == SOCKET_ERROR) {
        return InternetAddress ();
    }
    struct hostent* phe = gethostbyname (ac);
    if (phe == nullptr) {
    return InternetAddress ();
    }
    for (int i = 0; phe->h_addr_list[i] != 0; ++i) {
        struct in_addr addr;
        memcpy (&addr, phe->h_addr_list[i], sizeof(struct in_addr));
        return InternetAddress (addr);
    }
    return InternetAddress ();
#elif   qPlatform_POSIX
    auto getFlags = [] (int sd, const char* name) {
        char buf[1024];

        struct ifreq ifreq;
        memset(&ifreq, 0, sizeof (ifreq));
        strcpy (ifreq.ifr_name, name);

        int r = ioctl (sd, SIOCGIFFLAGS, (char*)&ifreq);
        Assert (r == 0);
        return (ifreq.ifr_flags);
    };

    struct ifreq ifreqs[32];
    struct ifconf ifconf;
    memset (&ifconf, 0, sizeof(ifconf));
    ifconf.ifc_req = ifreqs;
    ifconf.ifc_len = sizeof(ifreqs);

    int sd = socket (PF_INET, SOCK_STREAM, 0);
    Assert (sd >= 0);

    int r = ioctl (sd, SIOCGIFCONF, (char*)&ifconf);
    Assert (r == 0);

    InternetAddress result;
    for (int i = 0; i < ifconf.ifc_len / sizeof(struct ifreq); ++i) {
        int flags = getFlags (sd, ifreqs[i].ifr_name);
        if ((flags & IFF_UP)
                and (not (flags & IFF_LOOPBACK))
                and (flags & IFF_RUNNING)
           ) {
            result = InternetAddress (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr);
            break;
        }
        //printf ("%s: %s\n", ifreqs[i].ifr_name, inet_ntoa (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr));
        //printf (" flags: %s\n", flags (sd, ifreqs[i].ifr_name));
    }
    close (sd);
    return result;
#endif
}

String  Network::GetPrimaryNetworkDeviceMacAddress ()
{
    auto printMacAddr = [](const uint8_t macaddrBytes[6]) -> String {
        char     buf[100];
        (void)::memset (buf, 0, sizeof (buf));
        (void)snprintf (buf, sizeof (buf), "%02x-%02x-%02x-%02x-%02x-%02x",
        macaddrBytes[0], macaddrBytes[1],
        macaddrBytes[2], macaddrBytes[3],
        macaddrBytes[4], macaddrBytes[5]
                       );
        return String::FromAscii (buf);
    };
#if     qPlatform_Windows
    IP_ADAPTER_INFO adapterInfo[10];
    DWORD dwBufLen = sizeof(adapterInfo);
    Execution::Platform::Windows::ThrowIfNotERROR_SUCCESS (::GetAdaptersInfo (adapterInfo, &dwBufLen));
    for (PIP_ADAPTER_INFO pi = adapterInfo; pi != nullptr; pi = pi->Next) {
        // check attributes - IF TEST to see if good adaptoer
        // @todo
        return printMacAddr (pi->Address);
    }
#elif qPlatform_POSIX
    Socket s = Socket (Socket::SocketKind::DGRAM);

    ifconf   ifc;
    char buf[10 * 1024];
    ifc.ifc_len = sizeof(buf);
    ifc.ifc_buf = buf;
    Execution::ThrowErrNoIfNegative (::ioctl (s.GetNativeSocket (), SIOCGIFCONF, &ifc));

    const struct ifreq* const end = ifc.ifc_req + (ifc.ifc_len / sizeof(struct ifreq));
    for (const ifreq* it = ifc.ifc_req ; it != end; ++it) {
        struct ifreq    ifr;
        memset (&ifr, 0, sizeof (ifr));
        Characters::CString::Copy (ifr.ifr_name, NEltsOf (ifr.ifr_name), it->ifr_name);
        if (::ioctl (s.GetNativeSocket (), SIOCGIFFLAGS, &ifr) == 0) {
            if (!(ifr.ifr_flags & IFF_LOOPBACK)) { // don't count loopback
                if (::ioctl (s.GetNativeSocket (), SIOCGIFHWADDR, &ifr) == 0) {
                    return printMacAddr (reinterpret_cast<const uint8_t*> (ifr.ifr_hwaddr.sa_data));
                }
            }
        }
    }
#endif
    return String ();
}

struct  LinkMonitor::Rep_ {
    void    AddCallback (const function<void(LinkChange, const String& linkName, const String& ipAddr)>& callback)
    {
        fCallbacks_.Add (callback);
        StartMonitorIfNeeded_();
    }
    Containers::Collection<function<void(LinkChange, const String& linkName, const String& ipAddr)>>  fCallbacks_;
#if     qPlatform_POSIX
    Execution::Thread   fMonitorThread_;
#endif
#if     qPlatform_Windows
    HANDLE fMonitorHandler_ = INVALID_HANDLE_VALUE;
#endif

    void    SendNotifies (LinkChange lc, const String& linkName, const String& ipAddr)
    {
        for (auto cb : fCallbacks_) {
            cb (lc, linkName, ipAddr);
        }
    }

#if     qPlatform_Windows
    // cannot use LAMBDA cuz we need WINAPI call convention
    static     void    WINAPI  CB_ (void* callerContext, PMIB_UNICASTIPADDRESS_ROW Address, MIB_NOTIFICATION_TYPE NotificationType)
    {
        Rep_*   rep =   reinterpret_cast<Rep_*> (callerContext);
        if (Address != NULL) {
            char    ipAddrBuf[1024];
            snprintf (ipAddrBuf, NEltsOf(ipAddrBuf), "%d.%d.%d.%d", Address->Address.Ipv4.sin_addr.s_net,
                      Address->Address.Ipv4.sin_addr.s_host,
                      Address->Address.Ipv4.sin_addr.s_lh,
                      Address->Address.Ipv4.sin_addr.s_impno);
            rep->SendNotifies (LinkChange::eAdded, String (), String::FromAscii (ipAddrBuf));
        }
    }
#endif

    void    StartMonitorIfNeeded_()
    {
#if     qPlatform_Windows
        if (fMonitorHandler_ == INVALID_HANDLE_VALUE) {
            Execution::Platform::Windows::ThrowIfNotERROR_SUCCESS (::NotifyUnicastIpAddressChange (AF_INET, &CB_, NULL, FALSE, &fMonitorHandler_));
        }
#elif   qPlatform_POSIX
        /// WRONG - not really if posix - if LINUX - must have sep define for LINUX or at least for NETLINK!!!
        if (fMonitorThread_.GetStatus () == Execution::Thread::Status::eNull) {
            // very slight race starting this but not worth worrying about
            fMonitorThread_ = Execution::Thread ([this] () {

                // for now - only handle adds, but removes SB easy too...

                Socket  sock    =   Socket::Attach (Execution::ThrowErrNoIfNegative (::socket (PF_NETLINK, SOCK_RAW, NETLINK_ROUTE)));

                {
                    sockaddr_nl addr;
                    memset(&addr, 0, sizeof(addr));
                    addr.nl_family = AF_NETLINK;
                    addr.nl_groups = RTMGRP_IPV4_IFADDR;
                    Execution::ThrowErrNoIfNegative (bind (sock.GetNativeSocket (), (struct sockaddr*)&addr, sizeof(addr)));
                }

                //
                /// @todo - PROBABLY REDO USING Socket::Recv () - but we have none right now!!!
                //          -- LGP 2014-01-23
                //

                int len;
                char buffer[4096];
                struct nlmsghdr* nlh;
                nlh = (struct nlmsghdr*)buffer;
                while ((len = recv(sock.GetNativeSocket (), nlh, 4096, 0)) > 0) {
                    while ((NLMSG_OK(nlh, len)) && (nlh->nlmsg_type != NLMSG_DONE)) {
                        if (nlh->nlmsg_type == RTM_NEWADDR) {
                            struct ifaddrmsg* ifa = (struct ifaddrmsg*) NLMSG_DATA(nlh);
                            struct rtattr* rth = IFA_RTA(ifa);
                            int rtl = IFA_PAYLOAD(nlh);
                            while (rtl && RTA_OK(rth, rtl)) {
                                if (rth->rta_type == IFA_LOCAL) {
                                    uint32_t ipaddr = htonl(*((uint32_t*)RTA_DATA(rth)));
                                    char name[IFNAMSIZ];
                                    if_indextoname(ifa->ifa_index, name);

                                    for (auto cb : this->fCallbacks_) {
                                        char    ipAddrBuf[1024];
                                        snprintf (ipAddrBuf, NEltsOf(ipAddrBuf), "%d.%d.%d.%d", (ipaddr >> 24) & 0xff,  (ipaddr >> 16) & 0xff, (ipaddr >> 8) & 0xff, ipaddr & 0xff);
                                        cb (LinkChange::eAdded, String::FromNarrowSDKString (name), String::FromAscii (ipAddrBuf));
                                    }
                                }
                                rth = RTA_NEXT(rth, rtl);
                            }
                        }
                        nlh = NLMSG_NEXT(nlh, len);
                    }
                }
            });
        }
#else
        AssertNotImplemented ();
#endif
    }

    ~Rep_ ()
    {
#if     qPlatform_Windows
        if (fMonitorHandler_ != INVALID_HANDLE_VALUE) {
            // @todo should check error result, but then do what?
            // also - does this blcok until pending notifies done?
            // assuming so!!!
            ::CancelMibChangeNotify2 (fMonitorHandler_);
        }
#elif   qPlatform_POSIX
        Execution::Thread::SuppressAbortInContext  suppressAbort;  // critical to wait til done cuz captures this
        fMonitorThread_.AbortAndWaitForDone ();
#endif
    }
};

/*
 ********************************************************************************
 ************************* IO::Network::LinkMonitor *****************************
 ********************************************************************************
 */
LinkMonitor::LinkMonitor ()
    : fRep_ (new Rep_ ())
{
}

void    LinkMonitor::AddCallback (const function<void(LinkChange, const String& linkName, const String& ipAddr)>& callback)
{
    fRep_->AddCallback (callback);
}
<commit_msg>LinkMonitor cleanups<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2014.  All rights reserved
 */
#include    "../../StroikaPreComp.h"

#include    <cstdio>

#if     qPlatform_POSIX
#include    <unistd.h>
#include    <sys/types.h>
#include    <sys/socket.h>
#include    <sys/ioctl.h>
#include    <net/if.h>
#include    <netinet/in.h>
#include    <netdb.h>
#include    <arpa/inet.h>
#include    <netinet/in.h>
#include    <linux/netlink.h>
#include    <linux/rtnetlink.h>
#elif   qPlatform_Windows
#include    <WinSock2.h>
#include    <WS2tcpip.h>
#include    <Iphlpapi.h>
#include    <netioapi.h>
#endif

#include    "../../Characters/CString/Utilities.h"
#include    "../../Containers/Collection.h"
#include    "../../Execution/ErrNoException.h"
#include    "../../Execution/Thread.h"
#if     qPlatform_Windows
#include    "../../../Foundation/Execution/Platform/Windows/Exception.h"
#endif

#include    "Socket.h"

#include    "LinkMonitor.h"

using   namespace   Stroika::Foundation;
using   namespace   Stroika::Foundation::Memory;
using   namespace   Stroika::Foundation::IO;
using   namespace   Stroika::Foundation::IO::Network;

#if 0
// FOR POSIX DO SOMETHING LIKE THIS:
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/in.h>
#include <arpa/inet.h>

static const char* flags (int sd, const char* name)
{
    static char buf[1024];

    static struct ifreq ifreq;
    strcpy (ifreq.ifr_name, name);

    int r = ioctl (sd, SIOCGIFFLAGS, (char*)&ifreq);
    assert (r == 0);

    int l = 0;
#define FLAG(b) if(ifreq.ifr_flags & b) l += snprintf(buf + l, sizeof(buf) - l, #b " ")
    FLAG (IFF_UP);
    FLAG (IFF_BROADCAST);
    FLAG (IFF_DEBUG);
    FLAG (IFF_LOOPBACK);
    FLAG (IFF_POINTOPOINT);
    FLAG (IFF_RUNNING);
    FLAG (IFF_NOARP);
    FLAG (IFF_PROMISC);
    FLAG (IFF_NOTRAILERS);
    FLAG (IFF_ALLMULTI);
    FLAG (IFF_MASTER);
    FLAG (IFF_SLAVE);
    FLAG (IFF_MULTICAST);
    FLAG (IFF_PORTSEL);
    FLAG (IFF_AUTOMEDIA);
    FLAG (IFF_DYNAMIC);
#undef FLAG

    return buf;
}

int main (void)
{
    static struct ifreq ifreqs[32];
    struct ifconf ifconf;
    memset (&ifconf, 0, sizeof(ifconf));
    ifconf.ifc_req = ifreqs;
    ifconf.ifc_len = sizeof(ifreqs);

    int sd = socket (PF_INET, SOCK_STREAM, 0);
    assert (sd >= 0);

    int r = ioctl (sd, SIOCGIFCONF, (char*)&ifconf);
    assert (r == 0);

    for (int i = 0; i < ifconf.ifc_len / sizeof(struct ifreq); ++i) {
        printf ("%s: %s\n", ifreqs[i].ifr_name, inet_ntoa (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr));
        printf (" flags: %s\n", flags (sd, ifreqs[i].ifr_name));
    }

    close (sd);

    return 0;
}
#endif

#if     qPlatform_Windows
// /SEE THIS COD EOFR WINDOWS
//http ://support.microsoft.com/default.aspx?scid=http://support.microsoft.com:80/support/kb/articles/Q129/3/15.asp&NoWebContent=1
#endif

#if     qPlatform_Windows
namespace {
    bool    sStartedUp_ = false;
    bool    sAutoSetup_ =   true;

    void    CheckStarup_ ()
    {
        if (not sStartedUp_) {
            WSADATA wsaData;        // Initialize Winsock
            int iResult = WSAStartup (MAKEWORD (2, 2), &wsaData);
            if (iResult != 0) {
                Execution::Platform::Windows::Exception::DoThrow (::WSAGetLastError ());
            }
            sStartedUp_ = true;
        }
    }
}
#endif

InternetAddress Network::GetPrimaryInternetAddress ()
{
    /// HORRIBLY KLUDGY BAD IMPL!!!
#if     qPlatform_Windows
    CheckStarup_ ();
#if 0
    DWORD TEST = GetComputerNameEx((COMPUTER_NAME_FORMAT)cnf, buffer, &dwSize))
#endif
    char ac[1024];
    if (gethostname (ac, sizeof(ac)) == SOCKET_ERROR) {
        return InternetAddress ();
    }
    struct hostent* phe = gethostbyname (ac);
    if (phe == nullptr) {
    return InternetAddress ();
    }
    for (int i = 0; phe->h_addr_list[i] != 0; ++i) {
        struct in_addr addr;
        memcpy (&addr, phe->h_addr_list[i], sizeof(struct in_addr));
        return InternetAddress (addr);
    }
    return InternetAddress ();
#elif   qPlatform_POSIX
    auto getFlags = [] (int sd, const char* name) {
        char buf[1024];

        struct ifreq ifreq;
        memset(&ifreq, 0, sizeof (ifreq));
        strcpy (ifreq.ifr_name, name);

        int r = ioctl (sd, SIOCGIFFLAGS, (char*)&ifreq);
        Assert (r == 0);
        return (ifreq.ifr_flags);
    };

    struct ifreq ifreqs[32];
    struct ifconf ifconf;
    memset (&ifconf, 0, sizeof(ifconf));
    ifconf.ifc_req = ifreqs;
    ifconf.ifc_len = sizeof(ifreqs);

    int sd = socket (PF_INET, SOCK_STREAM, 0);
    Assert (sd >= 0);

    int r = ioctl (sd, SIOCGIFCONF, (char*)&ifconf);
    Assert (r == 0);

    InternetAddress result;
    for (int i = 0; i < ifconf.ifc_len / sizeof(struct ifreq); ++i) {
        int flags = getFlags (sd, ifreqs[i].ifr_name);
        if ((flags & IFF_UP)
                and (not (flags & IFF_LOOPBACK))
                and (flags & IFF_RUNNING)
           ) {
            result = InternetAddress (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr);
            break;
        }
        //printf ("%s: %s\n", ifreqs[i].ifr_name, inet_ntoa (((struct sockaddr_in*)&ifreqs[i].ifr_addr)->sin_addr));
        //printf (" flags: %s\n", flags (sd, ifreqs[i].ifr_name));
    }
    close (sd);
    return result;
#endif
}

String  Network::GetPrimaryNetworkDeviceMacAddress ()
{
    auto printMacAddr = [](const uint8_t macaddrBytes[6]) -> String {
        char     buf[100];
        (void)::memset (buf, 0, sizeof (buf));
        (void)snprintf (buf, sizeof (buf), "%02x-%02x-%02x-%02x-%02x-%02x",
        macaddrBytes[0], macaddrBytes[1],
        macaddrBytes[2], macaddrBytes[3],
        macaddrBytes[4], macaddrBytes[5]
                       );
        return String::FromAscii (buf);
    };
#if     qPlatform_Windows
    IP_ADAPTER_INFO adapterInfo[10];
    DWORD dwBufLen = sizeof(adapterInfo);
    Execution::Platform::Windows::ThrowIfNotERROR_SUCCESS (::GetAdaptersInfo (adapterInfo, &dwBufLen));
    for (PIP_ADAPTER_INFO pi = adapterInfo; pi != nullptr; pi = pi->Next) {
        // check attributes - IF TEST to see if good adaptoer
        // @todo
        return printMacAddr (pi->Address);
    }
#elif qPlatform_POSIX
    Socket s = Socket (Socket::SocketKind::DGRAM);

    ifconf   ifc;
    char buf[10 * 1024];
    ifc.ifc_len = sizeof(buf);
    ifc.ifc_buf = buf;
    Execution::ThrowErrNoIfNegative (::ioctl (s.GetNativeSocket (), SIOCGIFCONF, &ifc));

    const struct ifreq* const end = ifc.ifc_req + (ifc.ifc_len / sizeof(struct ifreq));
    for (const ifreq* it = ifc.ifc_req ; it != end; ++it) {
        struct ifreq    ifr;
        memset (&ifr, 0, sizeof (ifr));
        Characters::CString::Copy (ifr.ifr_name, NEltsOf (ifr.ifr_name), it->ifr_name);
        if (::ioctl (s.GetNativeSocket (), SIOCGIFFLAGS, &ifr) == 0) {
            if (!(ifr.ifr_flags & IFF_LOOPBACK)) { // don't count loopback
                if (::ioctl (s.GetNativeSocket (), SIOCGIFHWADDR, &ifr) == 0) {
                    return printMacAddr (reinterpret_cast<const uint8_t*> (ifr.ifr_hwaddr.sa_data));
                }
            }
        }
    }
#endif
    return String ();
}

struct  LinkMonitor::Rep_ {
    void    AddCallback (const function<void(LinkChange, const String& linkName, const String& ipAddr)>& callback)
    {
        fCallbacks_.Add (callback);
        StartMonitorIfNeeded_();
    }
    Containers::Collection<function<void(LinkChange, const String& linkName, const String& ipAddr)>>  fCallbacks_;
#if     qPlatform_POSIX
    Execution::Thread   fMonitorThread_;
#endif
#if     qPlatform_Windows
    HANDLE fMonitorHandler_ = INVALID_HANDLE_VALUE;
#endif

    void    SendNotifies (LinkChange lc, const String& linkName, const String& ipAddr)
    {
        for (auto cb : fCallbacks_) {
            cb (lc, linkName, ipAddr);
        }
    }

#if     qPlatform_Windows
    // cannot use LAMBDA cuz we need WINAPI call convention
    static     void    WINAPI  CB_ (void* callerContext, PMIB_UNICASTIPADDRESS_ROW Address, MIB_NOTIFICATION_TYPE NotificationType)
    {
        Rep_*   rep =   reinterpret_cast<Rep_*> (callerContext);
        if (Address != NULL) {
            char    ipAddrBuf[1024];
            snprintf (ipAddrBuf, NEltsOf(ipAddrBuf), "%d.%d.%d.%d", Address->Address.Ipv4.sin_addr.s_net,
                      Address->Address.Ipv4.sin_addr.s_host,
                      Address->Address.Ipv4.sin_addr.s_lh,
                      Address->Address.Ipv4.sin_addr.s_impno);
            rep->SendNotifies (LinkChange::eAdded, String (), String::FromAscii (ipAddrBuf));
        }
    }
#endif

    void    StartMonitorIfNeeded_()
    {
#if     qPlatform_Windows
        if (fMonitorHandler_ == INVALID_HANDLE_VALUE) {
            Execution::Platform::Windows::ThrowIfNotERROR_SUCCESS (::NotifyUnicastIpAddressChange (AF_INET, &CB_, NULL, FALSE, &fMonitorHandler_));
        }
#elif   qPlatform_POSIX
        /// WRONG - not really if posix - if LINUX - must have sep define for LINUX or at least for NETLINK!!!
        if (fMonitorThread_.GetStatus () == Execution::Thread::Status::eNull) {
            // very slight race starting this but not worth worrying about
            fMonitorThread_ = Execution::Thread ([this] () {

                // for now - only handle adds, but removes SB easy too...

                Socket  sock    =   Socket::Attach (Execution::ThrowErrNoIfNegative (::socket (PF_NETLINK, SOCK_RAW, NETLINK_ROUTE)));

                {
                    sockaddr_nl addr;
                    memset(&addr, 0, sizeof(addr));
                    addr.nl_family = AF_NETLINK;
                    addr.nl_groups = RTMGRP_IPV4_IFADDR;
                    Execution::ThrowErrNoIfNegative (bind (sock.GetNativeSocket (), (struct sockaddr*)&addr, sizeof(addr)));
                }

                //
                /// @todo - PROBABLY REDO USING Socket::Recv () - but we have none right now!!!
                //          -- LGP 2014-01-23
                //

                int len;
                char buffer[4096];
                struct nlmsghdr* nlh;
                nlh = (struct nlmsghdr*)buffer;
                while ((len = recv(sock.GetNativeSocket (), nlh, 4096, 0)) > 0) {
                    while ((NLMSG_OK(nlh, len)) && (nlh->nlmsg_type != NLMSG_DONE)) {
                        if (nlh->nlmsg_type == RTM_NEWADDR) {
                            struct ifaddrmsg* ifa = (struct ifaddrmsg*) NLMSG_DATA(nlh);
                            struct rtattr* rth = IFA_RTA(ifa);
                            int rtl = IFA_PAYLOAD(nlh);
                            while (rtl && RTA_OK(rth, rtl)) {
                                if (rth->rta_type == IFA_LOCAL) {
                                    uint32_t ipaddr = htonl(*((uint32_t*)RTA_DATA(rth)));
                                    char name[IFNAMSIZ];
                                    if_indextoname(ifa->ifa_index, name);
                                    {
                                        char    ipAddrBuf[1024];
                                        snprintf (ipAddrBuf, NEltsOf(ipAddrBuf), "%d.%d.%d.%d", (ipaddr >> 24) & 0xff,  (ipaddr >> 16) & 0xff, (ipaddr >> 8) & 0xff, ipaddr & 0xff);
                                        SendNotifies (LinkChange::eAdded, String::FromNarrowSDKString (name), String::FromAscii (ipAddrBuf));
                                    }
                                }
                                rth = RTA_NEXT(rth, rtl);
                            }
                        }
                        nlh = NLMSG_NEXT(nlh, len);
                    }
                }
            });
        }
#else
        AssertNotImplemented ();
#endif
    }

    ~Rep_ ()
    {
#if     qPlatform_Windows
        if (fMonitorHandler_ != INVALID_HANDLE_VALUE) {
            // @todo should check error result, but then do what?
            // also - does this blcok until pending notifies done?
            // assuming so!!!
            ::CancelMibChangeNotify2 (fMonitorHandler_);
        }
#elif   qPlatform_POSIX
        Execution::Thread::SuppressAbortInContext  suppressAbort;  // critical to wait til done cuz captures this
        fMonitorThread_.AbortAndWaitForDone ();
#endif
    }
};

/*
 ********************************************************************************
 ************************* IO::Network::LinkMonitor *****************************
 ********************************************************************************
 */
LinkMonitor::LinkMonitor ()
    : fRep_ (new Rep_ ())
{
}

void    LinkMonitor::AddCallback (const function<void(LinkChange, const String& linkName, const String& ipAddr)>& callback)
{
    fRep_->AddCallback (callback);
}
<|endoftext|>


<commit_before><commit_msg>Make show all triples the default.<commit_after><|endoftext|>