<commit_before>// =============================================================================
// PROJECT CHRONO - http://projectchrono.org
//
// Copyright (c) 2014 projectchrono.org
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file at the top level of the distribution and at
// http://projectchrono.org/license-chrono.txt.
//
// =============================================================================
// Authors: Alessandro Tasora
// =============================================================================
//
// FEA for 3D beams
//
// =============================================================================

#include "chrono/physics/ChSystemNSC.h"
#include "chrono/physics/ChLinkMate.h"
#include "chrono/physics/ChBodyEasy.h"
#include "chrono/timestepper/ChTimestepper.h"
#include "chrono/solver/ChSolverPMINRES.h"
#include "chrono/solver/ChSolverMINRES.h"

#include "chrono_fea/ChElementBeamIGA.h"
#include "chrono_fea/ChBuilderBeam.h"
#include "chrono_fea/ChMesh.h"
#include "chrono_fea/ChVisualizationFEAmesh.h"
#include "chrono_fea/ChLinkPointFrame.h"
#include "chrono_fea/ChLinkDirFrame.h"

#include "chrono_irrlicht/ChIrrApp.h"

using namespace chrono;
using namespace chrono::fea;
using namespace chrono::irrlicht;

using namespace irr;

int main(int argc, char* argv[]) {
    GetLog() << "Copyright (c) 2017 projectchrono.org\nChrono version: " << CHRONO_VERSION << "\n\n";

    // Create a Chrono::Engine physical system
    ChSystemNSC my_system;

    // Create a mesh, that is a container for groups
    // of elements and their referenced nodes.
    auto my_mesh = std::make_shared<ChMesh>();

    // Create a section, i.e. thickness and material properties
    // for beams. This will be shared among some beams.

    auto msection = std::make_shared<ChBeamSectionAdvanced>();

    double beam_wy = 0.012;
    double beam_wz = 0.025;
    msection->SetAsRectangularSection(beam_wy, beam_wz);
    msection->SetYoungModulus(0.01e9);
    msection->SetGshearModulus(0.01e9 * 0.3);
    msection->SetBeamRaleyghDamping(0.000);
    // msection->SetCentroid(0,0.02);
    // msection->SetShearCenter(0,0.1);
    // msection->SetSectionRotation(45*CH_C_RAD_TO_DEG);

    //
    // Add some IGA BEAMS:
    //

    double beam_L = 0.1;

    auto hnode1 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(0, 0, 0)));
    auto hnode2 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L, 0, 0)));
    auto hnode3 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L * 2, 0, 0)));

    my_mesh->AddNode(hnode1);
    my_mesh->AddNode(hnode2);
    my_mesh->AddNode(hnode3);

    // spline lineare con tre control points e 5 knots= {0 0 1 2 2}
    auto belement1 = std::make_shared<ChElementBeamIGA>();

    belement1->SetNodesLinear(hnode1, hnode2, 0,0, 1,2);
    belement1->SetSection(msection);

    my_mesh->AddElement(belement1);

    auto belement2 = std::make_shared<ChElementBeamIGA>();

    belement2->SetNodesLinear(hnode2, hnode3, 0,1, 2,2);
    belement2->SetSection(msection);

    my_mesh->AddElement(belement2);

    // Apply a force or a torque to a node:
    // hnode2->SetForce(ChVector<>(4, 2, 0));
    // hnode3->SetTorque( ChVector<>(0, -0.04, 0));

    // Fix a node to ground:
    hnode1->SetFixed(true);

    //
    // Final touches..
    //

    // We do not want gravity effect on FEA elements in this demo
    my_mesh->SetAutomaticGravity(false);

    // Remember to add the mesh to the system!
    my_system.Add(my_mesh);

    // ==Asset== attach a visualization of the FEM mesh.
    // This will automatically update a triangle mesh (a ChTriangleMeshShape
    // asset that is internally managed) by setting  proper
    // coordinates and vertex colors as in the FEM elements.
    // Such triangle mesh can be rendered by Irrlicht or POVray or whatever
    // postprocessor that can handle a colored ChTriangleMeshShape).
    // Do not forget AddAsset() at the end!

    /*
    auto mvisualizebeamA = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamA->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_SURFACE);
    mvisualizebeamA->SetSmoothFaces(true);
    my_mesh->AddAsset(mvisualizebeamA);
    */

    auto mvisualizebeamA = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamA->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_ELEM_BEAM_MZ);
    mvisualizebeamA->SetColorscaleMinMax(-0.4, 0.4);
    mvisualizebeamA->SetSmoothFaces(true);
    mvisualizebeamA->SetWireframe(false);
    my_mesh->AddAsset(mvisualizebeamA);

    auto mvisualizebeamC = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamC->SetFEMglyphType(ChVisualizationFEAmesh::E_GLYPH_NODE_CSYS);
    mvisualizebeamC->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_NONE);
    mvisualizebeamC->SetSymbolsThickness(0.006);
    mvisualizebeamC->SetSymbolsScale(0.01);
    mvisualizebeamC->SetZbufferHide(false);
    my_mesh->AddAsset(mvisualizebeamC);

    // Create the Irrlicht visualization (open the Irrlicht device,
    // bind a simple user interface, etc. etc.)
    ChIrrApp application(&my_system, L"IGA beams (SPACE for dynamics, F10 / F11 statics)", core::dimension2d<u32>(800, 600),
                         false, true);

    // Easy shortcuts to add camera, lights, logo and sky in Irrlicht scene:
    application.AddTypicalLogo();
    application.AddTypicalSky();
    application.AddTypicalLights();
    application.AddTypicalCamera(core::vector3df(-0.1f, 0.2f, -0.2f));

    // ==IMPORTANT!== Use this function for adding a ChIrrNodeAsset to all items
    // in the system. These ChIrrNodeAsset assets are 'proxies' to the Irrlicht meshes.
    // If you need a finer control on which item really needs a visualization proxy in
    // Irrlicht, just use application.AssetBind(myitem); on a per-item basis.

    application.AssetBindAll();

    // ==IMPORTANT!== Use this function for 'converting' into Irrlicht meshes the assets
    // that you added to the bodies into 3D shapes, they can be visualized by Irrlicht!

    application.AssetUpdateAll();

    // Mark completion of system construction
    my_system.SetupInitial();

    //
    // THE SOFT-REAL-TIME CYCLE
    //
    my_system.SetSolverType(ChSolver::Type::MINRES);
    my_system.SetSolverWarmStarting(true);  // this helps a lot to speedup convergence in this class of problems
    my_system.SetMaxItersSolverSpeed(460);
    my_system.SetMaxItersSolverStab(460);
    my_system.SetTolForce(1e-13);

    auto msolver = std::static_pointer_cast<ChSolverMINRES>(my_system.GetSolver());
    msolver->SetVerbose(false);
    msolver->SetDiagonalPreconditioning(true);

    application.SetTimestep(0.001);

    GetLog() << "\n\n\n===========STATICS======== \n\n\n";

//    application.GetSystem()->DoStaticLinear();

    GetLog() << "BEAM RESULTS (LINEAR STATIC ANALYSIS) \n\n";

    ChVector<> F, M;
    ChMatrixDynamic<> displ;

    belement1->GetStateBlock(displ);
    GetLog() << displ;
    for (double eta = -1; eta <= 1; eta += 0.4) {
        belement1->EvaluateSectionForceTorque(eta, F, M);
        GetLog() << "  b1_at " << eta << " Mx=" << M.x() << " My=" << M.y() << " Mz=" << M.z() << " Tx=" << F.x()
                 << " Ty=" << F.y() << " Tz=" << F.z() << "\n";
    }
    GetLog() << "\n";
    belement2->GetStateBlock(displ);
    for (double eta = -1; eta <= 1; eta += 0.4) {
        belement2->EvaluateSectionForceTorque(eta, F, M);
        GetLog() << "  b2_at " << eta << " Mx=" << M.x() << " My=" << M.y() << " Mz=" << M.z() << " Tx=" << F.x()
                 << " Ty=" << F.y() << " Tz=" << F.z() << "\n";
    }

    GetLog() << "Node 3 coordinate x= " << hnode3->Frame().GetPos().x() << "    y=" << hnode3->Frame().GetPos().y()
             << "    z=" << hnode3->Frame().GetPos().z() << "\n\n";

    GetLog() << "Press SPACE bar to start/stop dynamic simulation \n\n";
    GetLog() << "Press F10 for nonlinear static solution \n\n";
    GetLog() << "Press F11 for linear static solution \n\n";

    application.SetPaused(true);

    while (application.GetDevice()->run()) {
        application.BeginScene();

        application.DrawAll();

        application.DoStep();

        application.EndScene();
    }

    return 0;
}
<commit_msg>Demo for IGA modified to test the new ChBuilderBeamIGA class in ChBuilderBeam.h<commit_after>// =============================================================================
// PROJECT CHRONO - http://projectchrono.org
//
// Copyright (c) 2014 projectchrono.org
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file at the top level of the distribution and at
// http://projectchrono.org/license-chrono.txt.
//
// =============================================================================
// Authors: Alessandro Tasora
// =============================================================================
//
// FEA for 3D beams
//
// =============================================================================

#include "chrono/physics/ChSystemNSC.h"
#include "chrono/physics/ChLinkMate.h"
#include "chrono/physics/ChBodyEasy.h"
#include "chrono/timestepper/ChTimestepper.h"
#include "chrono/solver/ChSolverPMINRES.h"
#include "chrono/solver/ChSolverMINRES.h"

#include "chrono_fea/ChElementBeamIGA.h"
#include "chrono_fea/ChBuilderBeam.h"
#include "chrono_fea/ChMesh.h"
#include "chrono_fea/ChVisualizationFEAmesh.h"
#include "chrono_fea/ChLinkPointFrame.h"
#include "chrono_fea/ChLinkDirFrame.h"

#include "chrono_irrlicht/ChIrrApp.h"

using namespace chrono;
using namespace chrono::fea;
using namespace chrono::irrlicht;

using namespace irr;

int main(int argc, char* argv[]) {
	GetLog() << "Copyright (c) 2017 projectchrono.org\nChrono version: " << CHRONO_VERSION << "\n\n";

	// Create a Chrono::Engine physical system
	ChSystemNSC my_system;

	// Create a mesh, that is a container for groups
	// of elements and their referenced nodes.
	auto my_mesh = std::make_shared<ChMesh>();

	// Create a section, i.e. thickness and material properties
	// for beams. This will be shared among some beams.

	auto msection = std::make_shared<ChBeamSectionAdvanced>();

	double beam_wy = 0.012;
	double beam_wz = 0.025;
	msection->SetAsRectangularSection(beam_wy, beam_wz);
	msection->SetYoungModulus(0.01e9);
	msection->SetGshearModulus(0.01e9 * 0.3);
	msection->SetBeamRaleyghDamping(0.000);
	// msection->SetCentroid(0,0.02);
	// msection->SetShearCenter(0,0.1);
	// msection->SetSectionRotation(45*CH_C_RAD_TO_DEG);

	//
	// Add some IGA BEAMS:
	//

    //
    // Example A: create nodes and elements (low level approach):
    //

	double beam_L = 0.1;

	auto hnode1 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L*0,           0, 0)));
    auto hnode2 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L*0.5,	  0.00,	0)));
    auto hnode3 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L*1.0,      0.00,	0)));
	auto hnode4 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L*1.5,      0.00, 0)));
	auto hnode5 = std::make_shared<ChNodeFEAxyzrot>(ChFrame<>(ChVector<>(beam_L*2.0,      0.00, 0)));

    my_mesh->AddNode(hnode1);
    my_mesh->AddNode(hnode2);
    my_mesh->AddNode(hnode3);
	my_mesh->AddNode(hnode4);
	my_mesh->AddNode(hnode5);

    // spline cubica con 2 span, 5 points e 9 knots= {0 0 0 0 1/2 1 1 1 1}

    auto belement1 = std::make_shared<ChElementBeamIGA>();

	belement1->SetNodesCubic(hnode1, hnode2, hnode3, hnode4, 0, 0, 0, 0, 1./2., 1, 1, 1);
	belement1->SetSection(msection);

	my_mesh->AddElement(belement1);

    auto belement2 = std::make_shared<ChElementBeamIGA>();

	belement2->SetNodesCubic(hnode2, hnode3, hnode4, hnode5, 0, 0, 0, 1./2., 1, 1, 1, 1);
    belement2->SetSection(msection);

    my_mesh->AddElement(belement2);

    //
    // Example B: Automatic creation of the nodes and knots
    // using the ChBuilderBeamIGA tool for creating a straight rod divided in Nel elements:
    //

    ChBuilderBeamIGA builder;
    builder.BuildBeam(      my_mesh,            // the mesh to put the elements in
                            msection,           // section of the beam
                            8,                  // number of sections (spans)
                            ChVector<>(0,  0,1),// start point
                            ChVector<>(0.5,0,1),// end point
                            VECT_Y,             // suggested Y direction of section
                            3);                 // order (3 = cubic, etc)

    // in case you want to acces one of the created nodes, ex. to apply a tip force:
    builder.GetLastBeamNodes().back()->SetFixed(true);
    builder.GetLastBeamNodes().front()->SetForce(ChVector<>(0, -1, 0));

    /*
    ChBuilderBeamIGA builder;
    builder.BuildBeam(      my_mesh,            // the mesh to put the elements in
                            msection,           // section of the beam
                            1,                  // number of sections (spans)
                            ChVector<>(0,  0,1),// start point
                            ChVector<>(0.2,0,1),// end point
                            VECT_Y,             // suggested Y direction of section
                            1);                 // order (3 = cubic, etc)
    builder.GetLastBeamNodes().back()->SetPos(ChVector<>(0.21,0,1)); // move a bit the 2nd node respect to initial X0 state to test strain computation
    */

    //
    // Example C: Automatic creation of the nodes and knots using the
    // ChBuilderBeamIGA tool for creating a generic curved rod that matches a Bspline:
    //

    std::vector< ChVector<> > my_points = { {0,0,0}, {0,0.1,0}, {0,0.2,0}, {0,0.3,0.1} };

    geometry::ChLineBspline my_spline(  3,          // order (3 = cubic, etc)
                                        my_points); // control points, will become the IGA nodes

    builder.BuildBeam(      my_mesh,            // the mesh to put the elements in
                            msection,           // section of the beam
                            my_spline,          // Bspline to match (also order will be matched)
                            VECT_Z);            // suggested Y direction of section

    //
    // Final touches..
    //

    // We do not want gravity effect on FEA elements in this demo
    my_mesh->SetAutomaticGravity(false);

    // Remember to add the mesh to the system!
    my_system.Add(my_mesh);

    // ==Asset== attach a visualization of the FEM mesh.
    // This will automatically update a triangle mesh (a ChTriangleMeshShape
    // asset that is internally managed) by setting  proper
    // coordinates and vertex colors as in the FEM elements.
    // Such triangle mesh can be rendered by Irrlicht or POVray or whatever
    // postprocessor that can handle a colored ChTriangleMeshShape).
    // Do not forget AddAsset() at the end!

    /*
    auto mvisualizebeamA = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamA->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_SURFACE);
    mvisualizebeamA->SetSmoothFaces(true);
    my_mesh->AddAsset(mvisualizebeamA);
    */

    auto mvisualizebeamA = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamA->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_ELEM_BEAM_MZ);
    mvisualizebeamA->SetColorscaleMinMax(-0.4, 0.4);
    mvisualizebeamA->SetSmoothFaces(true);
    mvisualizebeamA->SetWireframe(false);
    my_mesh->AddAsset(mvisualizebeamA);

    auto mvisualizebeamC = std::make_shared<ChVisualizationFEAmesh>(*(my_mesh.get()));
    mvisualizebeamC->SetFEMglyphType(ChVisualizationFEAmesh::E_GLYPH_NODE_CSYS);
    mvisualizebeamC->SetFEMdataType(ChVisualizationFEAmesh::E_PLOT_NONE);
    mvisualizebeamC->SetSymbolsThickness(0.006);
    mvisualizebeamC->SetSymbolsScale(0.01);
    mvisualizebeamC->SetZbufferHide(false);
    my_mesh->AddAsset(mvisualizebeamC);

    // Create the Irrlicht visualization (open the Irrlicht device,
    // bind a simple user interface, etc. etc.)
    ChIrrApp application(&my_system, L"IGA beams DEMO (SPACE for dynamics, F10 / F11 statics)", core::dimension2d<u32>(800, 600),
                         false, true);

    // Easy shortcuts to add camera, lights, logo and sky in Irrlicht scene:
    application.AddTypicalLogo();
    application.AddTypicalSky();
    application.AddTypicalLights();
    application.AddTypicalCamera(core::vector3df(-0.1f, 0.2f, -0.2f));

    // ==IMPORTANT!== Use this function for adding a ChIrrNodeAsset to all items
    // in the system. These ChIrrNodeAsset assets are 'proxies' to the Irrlicht meshes.
    // If you need a finer control on which item really needs a visualization proxy in
    // Irrlicht, just use application.AssetBind(myitem); on a per-item basis.

    application.AssetBindAll();

    // ==IMPORTANT!== Use this function for 'converting' into Irrlicht meshes the assets
    // that you added to the bodies into 3D shapes, they can be visualized by Irrlicht!

    application.AssetUpdateAll();

    // Mark completion of system construction
    my_system.SetupInitial();

    //
    // THE SOFT-REAL-TIME CYCLE
    //
    my_system.SetSolverType(ChSolver::Type::MINRES);
    my_system.SetSolverWarmStarting(true);  // this helps a lot to speedup convergence in this class of problems
    my_system.SetMaxItersSolverSpeed(460);
    my_system.SetMaxItersSolverStab(460);
    my_system.SetTolForce(1e-13);

    auto msolver = std::static_pointer_cast<ChSolverMINRES>(my_system.GetSolver());
    msolver->SetVerbose(false);
    msolver->SetDiagonalPreconditioning(true);

    application.SetTimestep(0.001);

    GetLog() << "\n\n\n===========STATICS======== \n\n\n";

//    application.GetSystem()->DoStaticLinear();

    GetLog() << "BEAM RESULTS (LINEAR STATIC ANALYSIS) \n\n";

    ChVector<> F, M;
    ChMatrixDynamic<> displ;
	/*
    belement1->GetStateBlock(displ);
    GetLog() << displ;
    for (double eta = -1; eta <= 1; eta += 0.4) {
        belement1->EvaluateSectionForceTorque(eta, F, M);
        GetLog() << "  b1_at " << eta << " Mx=" << M.x() << " My=" << M.y() << " Mz=" << M.z() << " Tx=" << F.x()
                 << " Ty=" << F.y() << " Tz=" << F.z() << "\n";
    }
	*/
    GetLog() << "\n";
	/*
    belement2->GetStateBlock(displ);
    for (double eta = -1; eta <= 1; eta += 0.4) {
        belement2->EvaluateSectionForceTorque(eta, F, M);
        GetLog() << "  b2_at " << eta << " Mx=" << M.x() << " My=" << M.y() << " Mz=" << M.z() << " Tx=" << F.x()
                 << " Ty=" << F.y() << " Tz=" << F.z() << "\n";
    }
	*/
    GetLog() << "Node 3 coordinate x= " << hnode3->Frame().GetPos().x() << "    y=" << hnode3->Frame().GetPos().y()
             << "    z=" << hnode3->Frame().GetPos().z() << "\n\n";

    GetLog() << "Press SPACE bar to start/stop dynamic simulation \n\n";
    GetLog() << "Press F10 for nonlinear static solution \n\n";
    GetLog() << "Press F11 for linear static solution \n\n";

    application.SetPaused(true);

    while (application.GetDevice()->run()) {
        application.BeginScene();

        application.DrawAll();

        application.DoStep();

        application.EndScene();
    }

    return 0;
}
<|endoftext|>


<commit_before>#include <opencv2/opencv.hpp>
#include <vector>
#include <iostream>

using namespace std;
using namespace cv;

static void help()
{
    cout << "\n This program demonstrates how to use BLOB and MSER to detect region \n"
        "Usage: \n"
        "  ./BLOB_MSER <image1(../data/basketball1.png as default)>\n"
        "Press a key when image window is active to change descriptor";
}

struct MSERParams
    {
    MSERParams(int _delta = 5, int _min_area = 60, int _max_area = 14400,
    double _max_variation = 0.25, double _min_diversity = .2,
    int _max_evolution = 200, double _area_threshold = 1.01,
    double _min_margin = 0.003, int _edge_blur_size = 5)
        {
        delta = _delta;
        minArea = _min_area;
        maxArea = _max_area;
        maxVariation = _max_variation;
        minDiversity = _min_diversity;
        maxEvolution = _max_evolution;
        areaThreshold = _area_threshold;
        minMargin = _min_margin;
        edgeBlurSize = _edge_blur_size;
        pass2Only = false;
        }

    int delta;
    int minArea;
    int maxArea;
    double maxVariation;
    double minDiversity;
    bool pass2Only;

    int maxEvolution;
    double areaThreshold;
    double minMargin;
    int edgeBlurSize;
    };

int main(int argc, char *argv[])
{
    vector<String> fileName;
    if (argc == 1)
        {
        fileName.push_back("../data/forme.jpg");
        }
    else if (argc == 2)
        {
        fileName.push_back(argv[1]);
        }
    else
        {
        help();
        return(0);
        }
    Mat imgOrig = imread(fileName[0], IMREAD_UNCHANGED),img;
    if (imgOrig.rows*imgOrig.cols <= 0)
        {
        cout << "Image " << fileName[0] << " is empty or cannot be found\n";
        return(0);
        }
    GaussianBlur(imgOrig,img,Size(11,11),5,5);
    SimpleBlobDetector::Params pDefaultBLOB;
    // This is default parameters for SimpleBlobDetector
    pDefaultBLOB.thresholdStep = 10;
    pDefaultBLOB.minThreshold = 1;
    pDefaultBLOB.maxThreshold = 220;
    pDefaultBLOB.minRepeatability = 2;
    pDefaultBLOB.minDistBetweenBlobs = 10;
    pDefaultBLOB.filterByColor = false;
    pDefaultBLOB.blobColor = 0;
    pDefaultBLOB.filterByArea = false;
    pDefaultBLOB.minArea = 25;
    pDefaultBLOB.maxArea = 5000;
    pDefaultBLOB.filterByCircularity = false;
    pDefaultBLOB.minCircularity = 0.8f;
    pDefaultBLOB.maxCircularity = std::numeric_limits<float>::max();
    pDefaultBLOB.filterByInertia = false;
    pDefaultBLOB.minInertiaRatio = 0.1f;
    pDefaultBLOB.maxInertiaRatio = std::numeric_limits<float>::max();
    pDefaultBLOB.filterByConvexity = false;
    pDefaultBLOB.minConvexity = 0.95f;
    pDefaultBLOB.maxConvexity = std::numeric_limits<float>::max();
    MSERParams pDefaultMSER;
    // Descriptor array (BLOB or MSER)
    vector<String> typeDesc;
    // Param array for BLOB
    vector<SimpleBlobDetector::Params> pBLOB;
    vector<SimpleBlobDetector::Params>::iterator itBLOB;
    // Param array for MSER
    vector<MSERParams> pMSER;

    // Color palette
    vector<Vec3b>  palette;
    for (int i=0;i<65536;i++)
        palette.push_back(Vec3b(rand(),rand(),rand()));
    help();
    // This descriptor are going to be detect and compute 4 BLOBS with 4 differents params
    typeDesc.push_back("BLOB");    // see http://docs.opencv.org/trunk/d0/d7a/classcv_1_1SimpleBlobDetector.html
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByArea = true;
    pBLOB.back().minArea = 1;
    pBLOB.back().maxArea = img.rows*img.cols;
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByArea = true;
    pBLOB.back().maxArea = img.rows*img.cols;
    pBLOB.back().filterByCircularity = true;
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByInertia = true;
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByColor = true;
    pBLOB.back().blobColor = 60;
    typeDesc.push_back("MSER");
    itBLOB=pBLOB.begin();
    vector<double> desMethCmp;
    Ptr<Feature2D> b;

    // Descriptor loop
    vector<String>::iterator itDesc;
    for (itDesc = typeDesc.begin(); itDesc != typeDesc.end(); itDesc++)
    {
        vector<KeyPoint> keyImg1;
        if (*itDesc == "BLOB"){
            b = SimpleBlobDetector::create(*itBLOB);
            itBLOB++;
        }
        if (*itDesc == "MSER"){
            b = MSER::create();
        }
        try {
            // We can detect keypoint with detect method
            vector<KeyPoint>  keyImg;
            vector<Rect>  zone;
            vector<vector <Point>>  region;
            Mat     desc, result;
            int nb = img.channels();
            if (img.channels() == 3)
            {
                img.copyTo(result);
            }
            else
                {
                vector<Mat> plan;
                plan.push_back(img);
                plan.push_back(img);
                plan.push_back(img);
                merge(plan, result);
                }
            if (b.dynamicCast<SimpleBlobDetector>() != NULL)
            {
                Ptr<SimpleBlobDetector> sbd = b.dynamicCast<SimpleBlobDetector>();
                sbd->detect(img, keyImg, Mat());
                drawKeypoints(img,keyImg,result);
                int i=0;
                for (vector<KeyPoint>::iterator k=keyImg.begin();k!=keyImg.end();k++,i++)
                    circle(result,k->pt,k->size,palette[i%65536]);

            }
            if (b.dynamicCast<MSER>() != NULL)
            {
                Ptr<MSER> sbd = b.dynamicCast<MSER>();
                sbd->detectRegions(img,  region, zone);
                int i = 0;

                for (vector<Rect>::iterator r = zone.begin(); r != zone.end();r++,i++)
                {
                rectangle(result, *r, palette[i % 65536],2);
                }
                i=0;
                for (vector<vector <Point>>::iterator itr = region.begin(); itr != region.end(); itr++, i++)
                    {
                    for (vector <Point>::iterator itp = region[i].begin(); itp != region[i].end(); itp++)
                        {

                        result.at<Vec3b>(itp->y, itp->x) = Vec3b(0,0,0);
                        }
                    }
                i = 0;
                for (vector<vector <Point>>::iterator itr = region.begin(); itr != region.end(); itr++, i++)
                    {
                    for (vector <Point>::iterator itp = region[i].begin(); itp != region[i].end(); itp++)
                        {

                        result.at<Vec3b>(itp->y, itp->x) += palette[i % 65536];
                        }
                    }
                }
            namedWindow(*itDesc , WINDOW_AUTOSIZE);
            imshow(*itDesc, result);
            imshow("Original", img);
            FileStorage fs(*itDesc + "_" + fileName[0] + ".xml", FileStorage::WRITE);
            fs<<*itDesc<<keyImg;
                waitKey();
        }
        catch (Exception& e)
        {
            cout << "Feature : " << *itDesc << "\n";
            cout<<e.msg<<endl;
        }
    }
    return 0;
}
<commit_msg>Example BLOB_MSER<commit_after>#include <opencv2/opencv.hpp>
#include <vector>
#include <iostream>

using namespace std;
using namespace cv;

static void help()
{
    cout << "\n This program demonstrates how to use BLOB and MSER to detect region \n"
        "Usage: \n"
        "  ./BLOB_MSER <image1(../data/forme2.jpg as default)>\n"
        "Press a key when image window is active to change descriptor";
}

struct MSERParams
    {
    MSERParams(int _delta = 5, int _min_area = 60, int _max_area = 14400,
    double _max_variation = 0.25, double _min_diversity = .2,
    int _max_evolution = 200, double _area_threshold = 1.01,
    double _min_margin = 0.003, int _edge_blur_size = 5)
        {
        delta = _delta;
        minArea = _min_area;
        maxArea = _max_area;
        maxVariation = _max_variation;
        minDiversity = _min_diversity;
        maxEvolution = _max_evolution;
        areaThreshold = _area_threshold;
        minMargin = _min_margin;
        edgeBlurSize = _edge_blur_size;
        pass2Only = false;
        }

    int delta;
    int minArea;
    int maxArea;
    double maxVariation;
    double minDiversity;
    bool pass2Only;

    int maxEvolution;
    double areaThreshold;
    double minMargin;
    int edgeBlurSize;
    };

String Legende(SimpleBlobDetector::Params &pAct)
{
    String s="";
    if (pAct.filterByArea)
        {
        String inf = static_cast<ostringstream*>(&(ostringstream() << pAct.minArea))->str();
        String sup = static_cast<ostringstream*>(&(ostringstream() << pAct.maxArea))->str();
        s = " Area range [" + inf + " to  " + sup + "]";
        }
    if (pAct.filterByCircularity)
        {
        String inf = static_cast<ostringstream*>(&(ostringstream() << pAct.minCircularity))->str();
        String sup = static_cast<ostringstream*>(&(ostringstream() << pAct.maxCircularity))->str();
        if (s.length()==0)
            s = " Circularity range [" + inf + " to  " + sup + "]";
        else
            s += " AND Circularity range [" + inf + " to  " + sup + "]";
        }
    if (pAct.filterByColor)
        {
        String inf = static_cast<ostringstream*>(&(ostringstream() << pAct.blobColor))->str();
        if (s.length() == 0)
            s = " Blob color " + inf;
        else
            s += " AND Blob color " + inf;
        }
    if (pAct.filterByConvexity)
        {
        String inf = static_cast<ostringstream*>(&(ostringstream() << pAct.minConvexity))->str();
        String sup = static_cast<ostringstream*>(&(ostringstream() << pAct.maxConvexity))->str();
        if (s.length() == 0)
            s = " Convexity range[" + inf + " to  " + sup + "]";
        else
            s += " AND  Convexity range[" + inf + " to  " + sup + "]";
        }
    if (pAct.filterByInertia)
        {
        String inf = static_cast<ostringstream*>(&(ostringstream() << pAct.minInertiaRatio))->str();
        String sup = static_cast<ostringstream*>(&(ostringstream() << pAct.maxInertiaRatio))->str();
        if (s.length() == 0)
            s = " Inertia ratio range [" + inf + " to  " + sup + "]";
        else
            s += " AND  Inertia ratio range [" + inf + " to  " + sup + "]";
        }
    return s;
}

int main(int argc, char *argv[])
{
    vector<String> fileName;
    if (argc == 1)
        {
        fileName.push_back("../data/BLOB_MSER.bmp");
        }
    else if (argc == 2)
        {
        fileName.push_back(argv[1]);
        }
    else
        {
        help();
        return(0);
        }
    Mat imgOrig = imread(fileName[0], IMREAD_UNCHANGED),img;
    if (imgOrig.rows*imgOrig.cols <= 0)
        {
        cout << "Image " << fileName[0] << " is empty or cannot be found\n";
        return(0);
        }
    GaussianBlur(imgOrig,img,Size(11,11),0.1,0.1);

    SimpleBlobDetector::Params pDefaultBLOB;
    MSERParams pDefaultMSER;
    // This is default parameters for SimpleBlobDetector
    pDefaultBLOB.thresholdStep = 10;
    pDefaultBLOB.minThreshold = 10;
    pDefaultBLOB.maxThreshold = 220;
    pDefaultBLOB.minRepeatability = 2;
    pDefaultBLOB.minDistBetweenBlobs = 10;
    pDefaultBLOB.filterByColor = false;
    pDefaultBLOB.blobColor = 0;
    pDefaultBLOB.filterByArea = false;
    pDefaultBLOB.minArea = 25;
    pDefaultBLOB.maxArea = 5000;
    pDefaultBLOB.filterByCircularity = false;
    pDefaultBLOB.minCircularity = 0.9f;
    pDefaultBLOB.maxCircularity = std::numeric_limits<float>::max();
    pDefaultBLOB.filterByInertia = false;
    pDefaultBLOB.minInertiaRatio = 0.1f;
    pDefaultBLOB.maxInertiaRatio = std::numeric_limits<float>::max();
    pDefaultBLOB.filterByConvexity = false;
    pDefaultBLOB.minConvexity = 0.95f;
    pDefaultBLOB.maxConvexity = std::numeric_limits<float>::max();
    // Descriptor array (BLOB or MSER)
    vector<String> typeDesc;
    // Param array for BLOB
    vector<SimpleBlobDetector::Params> pBLOB;
    vector<SimpleBlobDetector::Params>::iterator itBLOB;
    // Param array for MSER
    vector<MSERParams> pMSER;
    vector<MSERParams>::iterator itMSER;

    // Color palette
    vector<Vec3b>  palette;
    for (int i=0;i<65536;i++)
        palette.push_back(Vec3b(rand(),rand(),rand()));
    help();
    typeDesc.push_back("MSER");
    pMSER.push_back(pDefaultMSER);
    pMSER.back().minArea = 1;
    pMSER.back().maxArea = img.rows*img.cols;

    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByColor = true;
    pBLOB.back().blobColor = 255;
    // This descriptor are going to be detect and compute 4 BLOBS with 4 differents params
    // Param for first BLOB detector we want all
    typeDesc.push_back("BLOB");    // see http://docs.opencv.org/trunk/d0/d7a/classcv_1_1SimpleBlobDetector.html
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByArea = true;
    pBLOB.back().minArea = 1;
    pBLOB.back().maxArea = img.rows*img.cols;
    // Param for second BLOB detector we want area between 500 and 2900 pixels
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByArea = true;
    pBLOB.back().minArea = 500;
    pBLOB.back().maxArea = 2900;
    // Param for third BLOB detector we want only circular object
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByCircularity = true;
    // Param for Fourth BLOB detector we want ratio inertia
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByInertia = true;
    pBLOB.back().minInertiaRatio = 0;
    pBLOB.back().maxInertiaRatio = 0.2;
    // Param for Fourth BLOB detector we want ratio inertia
    typeDesc.push_back("BLOB");
    pBLOB.push_back(pDefaultBLOB);
    pBLOB.back().filterByConvexity = true;
    pBLOB.back().minConvexity = 0.;
    pBLOB.back().maxConvexity = 0.9;

    itBLOB = pBLOB.begin();
    itMSER = pMSER.begin();
    vector<double> desMethCmp;
    Ptr<Feature2D> b;
    String label;
    // Descriptor loop
    vector<String>::iterator itDesc;
    for (itDesc = typeDesc.begin(); itDesc != typeDesc.end(); itDesc++)
    {
        vector<KeyPoint> keyImg1;
        if (*itDesc == "BLOB"){
            b = SimpleBlobDetector::create(*itBLOB);
            label=Legende(*itBLOB);

            itBLOB++;
        }
        if (*itDesc == "MSER"){
            b = MSER::create(itMSER->delta, itMSER->minArea,itMSER->maxArea,itMSER->maxVariation,itMSER->minDiversity,itMSER->maxEvolution,
                itMSER->areaThreshold,itMSER->minMargin,itMSER->edgeBlurSize);
            }
        try {
            // We can detect keypoint with detect method
            vector<KeyPoint>  keyImg;
            vector<Rect>  zone;
            vector<vector <Point>>  region;
            Mat     desc, result;
            int nb = img.channels();
            if (img.channels() == 3)
            {
                img.copyTo(result);
            }
            else
                {
                vector<Mat> plan;
                plan.push_back(img);
                plan.push_back(img);
                plan.push_back(img);
                merge(plan, result);
                }
            if (b.dynamicCast<SimpleBlobDetector>() != NULL)
            {
                Ptr<SimpleBlobDetector> sbd = b.dynamicCast<SimpleBlobDetector>();
                sbd->detect(img, keyImg, Mat());
                drawKeypoints(img,keyImg,result);
                int i=0;
                for (vector<KeyPoint>::iterator k=keyImg.begin();k!=keyImg.end();k++,i++)
                    circle(result,k->pt,k->size,palette[i%65536]);
            }
            if (b.dynamicCast<MSER>() != NULL)
            {
                Ptr<MSER> sbd = b.dynamicCast<MSER>();
                sbd->detectRegions(img,  region, zone);
                int i = 0;

                for (vector<Rect>::iterator r = zone.begin(); r != zone.end();r++,i++)
                {
                rectangle(result, *r, palette[i % 65536],2);
                }
                i=0;
                for (vector<vector <Point>>::iterator itr = region.begin(); itr != region.end(); itr++, i++)
                    {
                    for (vector <Point>::iterator itp = region[i].begin(); itp != region[i].end(); itp++)
                        {

                        result.at<Vec3b>(itp->y, itp->x) = Vec3b(0,0,0);
                        }
                    }
                i = 0;
                for (vector<vector <Point>>::iterator itr = region.begin(); itr != region.end(); itr++, i++)
                    {
                    for (vector <Point>::iterator itp = region[i].begin(); itp != region[i].end(); itp++)
                        {

                        result.at<Vec3b>(itp->y, itp->x) = Vec3b(0,255,255);
                        }
                    }
                }
            namedWindow(*itDesc+label , WINDOW_AUTOSIZE);
            imshow(*itDesc + label, result);
            imshow("Original", img);
            FileStorage fs(*itDesc + "_" + fileName[0] + ".xml", FileStorage::WRITE);
            fs<<*itDesc<<keyImg;
                waitKey();
        }
        catch (Exception& e)
        {
            cout << "Feature : " << *itDesc << "\n";
            cout<<e.msg<<endl;
        }
    }
    return 0;
}
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/framework/common_shape_fns.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/shape_inference.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {

using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

// Configuring a distributed TPU system is achieved by running
// the following Ops:
//
// 1 Run _DisconnectHostFromDistributedTPUSystem on the TPU_SYSTEM of each
// host. This is needed in case the system had previously been configured. It
// returns, for each host, the number of TPU chips on the host.
//
// 2 Run _ConfigureDistributedTPU on TPU_SYSTEM of worker 0. Takes as input the
// number of chips on each host. Validates that all hosts have the same number
// of chips, and that the chips are consistent with the topology set by
// flags. Has a single output which is a proto describing the requested system
// configuration, which is sent to all hosts.
//
// 3 Run _InitializeHostForDistributedTPU on the TPU_SYSTEM of each host, taking
// as input the output from ConfigureDistributedTPU. Has a single Tensor output
// which is a vector of int32 indicating, for each TPU on the host, what its
// global TPU system id is.
//
// 4 Run _WaitForDistributedTPU on TPU_SYSTEM, taking as input the
// outputs from all the _InitializeHostForDistributedTPU
// Ops. _These partial specs are combined in the Op with the outputs from
// the host initialization Ops to construct a mapping from full TPU device
// specs to global TPU ids. Has a single Tensor output which is a
// matrix of int32 indicating, for each host (outer dimension) and for
// each TPU on the host (inner dimension) what that TPU's global id
// is. _WaitForDistributedTPU also waits for the TPU distributed
// system to initialize fully, which may take several minutes for a
// large system.
//
// 5 Run _SetGlobalTPUArray on the TPU_SYSTEM of each host, taking as input the
// output from _WaitForDistributedTPU. This Op tells each host the global Id of
// every TPU on every host.
//
// Most user code works by placing the ConfigureDistributedTPU Op on the desired
// TPU_SYSTEM device, and a graph rewrite replaces it by the subgraph described
// above.
//
//
// A distributed TPU system can be cleanly shut down by running the following
// Ops:
//
// 1 Run _DisconnectHostFromDistributedTPUSystem on the TPU_SYSTEM of each host.
//
// 2 Run _ShutdownDistributedTPU on the TPU_SYSTEM where
// _ConfigureDistributedTPU was run. The Op will return an error if no system is
// configured.
//
//
// Most user code works by placing the ShutdownDistributedTPU Op on the desired
// TPU_SYSTEM device, and a graph rewrite replaces it by the subgraph described
// above.

REGISTER_OP("_ConfigureDistributedTPU")
    .Input("inputs: N * int32")
    .Output("output: string")
    .Attr("N: int >= 1")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      // Validate that all the inputs are scalars.
      for (int i = 0; i < c->num_inputs(); ++i) {
        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &input));
      }
      c->set_output(0, c->Scalar());
      return Status::OK();
    })
    .Doc(R"doc(
An op that sets up the centralized structures for a distributed TPU
system.

inputs: A scalar tensor for each host indicating how many TPU chips
there are on the host.
output: A tensor containing a TPUHostConfiguration proto serialized to
a string, containing the information necessary to initialize the chips
in a host.
enable_whole_mesh_compilations: Usually the master TPU worker is the only
worker compile ops are sent, and the master worker is the only one which
can execute them. Other TPU clients distribute TPU compilation across all
the hosts of the mesh, and setting this flag to True enables such mesh
initialization mode.
)doc");

REGISTER_OP("_WaitForDistributedTPU")
    .Input("inputs: N * int32")
    .Output("topology: string")
    .Attr("startup_timeout_sec: int = 20")
    .Attr("N: int")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      // Validate that all the inputs have the same vector shape.
      for (int i = 0; i < c->num_inputs(); ++i) {
        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 1, &input));
      }
      c->set_output(0, c->Scalar());
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that blocks execution until a distributed TPU system has
started up. This Op must be run on the same TPU_SYSTEM device as
_ConfigureDistributedTPU, and takes an inputs the outputs from the
_InitializeHostForDistributedTPU Ops.

inputs: For each initialized host, a vector giving the global TPU id
of each TPU on the host.
topology: A serialized tensorflow.tpu.TopologyProto that describes the TPU
topology.
startup_timeout_sec: The number of seconds to wait for the TPU system
to stabilize.
)doc");

REGISTER_OP("_SetGlobalTPUArray")
    .Input("topology: string")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that informs a host of the global ids of all the of TPUs in the
system.

topology: A serialized tensorflow.tpu.TopologyProto that describes the TPU
topology.
)doc");

REGISTER_OP("_ShutdownDistributedTPU")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape)
    .Doc(R"doc(
An op that shuts down a running distributed TPU system. The Op returns
an error if no system is running. This Op must be run on the same
TPU_SYSTEM device as the corresponding _ConfigureDistributedTPU was run
to start the system, and must be run only after
_DisconnectHostFromDistributedTPUSystem has completed on every host in
the system.
)doc");

REGISTER_OP("_InitializeHostForDistributedTPU")
    .Input("input: string")
    .Output("tpu_ids: int32")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
      c->set_output(0, c->Vector(c->UnknownDim()));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that connects each chip on the host to a centralized UberDriver to allow
them to operate as a distributed system with chips in other hosts.

input: A string containing the address of the UberDriver to connect to.
tpu_ids: A vector containing the global TPU id of each TPU on the host.
enable_whole_mesh_compilations: Usually the master TPU worker is the only
worker compile ops are sent, and the master worker is the only one which
can execute them. Other TPU clients distribute TPU compilation across all
the hosts of the mesh, and setting this flag to True enables such mesh
initialization mode.
)doc");

REGISTER_OP("_DisconnectHostFromDistributedTPUSystem")
    .Output("number_of_tpu_chips: int32")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape)
    .Doc(R"doc(
An op that disconnects the TPUs on a host from a running distributed
TPU system.

number_of_tpu_chips: A scalar tensor containing the number of TPU
chips on the host.
)doc");

REGISTER_OP("ConfigureDistributedTPU")
    .Output("topology: string")
    .Attr("embedding_config: string = ''")
    .Attr("tpu_embedding_config: string = ''")
    .Attr("is_global_init: bool = false")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .Attr("compilation_failure_closes_chips: bool = true")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

REGISTER_OP("ShutdownDistributedTPU")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

REGISTER_OP("ConfigureTPUEmbedding")
    .Attr("config: string")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

}  // end namespace tensorflow
<commit_msg>Clarify multi-client TPU configuration.<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/framework/common_shape_fns.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/shape_inference.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {

using shape_inference::InferenceContext;
using shape_inference::ShapeHandle;

// Configuring a distributed TPU system is achieved by running
// the following Ops:
//
// 1 Run _DisconnectHostFromDistributedTPUSystem on the TPU_SYSTEM of each
// host. This is needed in case the system had previously been configured. It
// returns, for each host, the number of TPU chips on the host.
//
// 2 Run _ConfigureDistributedTPU on TPU_SYSTEM of worker 0. Takes as input the
// number of chips on each host. Validates that all hosts have the same number
// of chips, and that the chips are consistent with the topology set by
// flags. Has a single output which is a proto describing the requested system
// configuration, which is sent to all hosts. Note that for multi-client setups
// the input to _ConfigureDistributedTPU refers only to hosts controlled by the
// local process/client; the topology set by flags determines the total number
// of hosts across all clients, and this is reflected in the return value.
//
// 3 Run _InitializeHostForDistributedTPU on the TPU_SYSTEM of each host, taking
// as input the output from ConfigureDistributedTPU. Has a single Tensor output
// which is a vector of int32 indicating, for each TPU on the host, what its
// global TPU system id is.
//
// 4 Run _WaitForDistributedTPU on TPU_SYSTEM, taking as input the
// outputs from all the _InitializeHostForDistributedTPU
// Ops. _These partial specs are combined in the Op with the outputs from
// the host initialization Ops to construct a mapping from full TPU device
// specs to global TPU ids. Has a single Tensor output which is a
// matrix of int32 indicating, for each host (outer dimension) and for
// each TPU on the host (inner dimension) what that TPU's global id
// is. _WaitForDistributedTPU also waits for the TPU distributed
// system to initialize fully, which may take several minutes for a
// large system.
//
// 5 Run _SetGlobalTPUArray on the TPU_SYSTEM of each host, taking as input the
// output from _WaitForDistributedTPU. This Op tells each host the global Id of
// every TPU on every host.
//
// Most user code works by placing the ConfigureDistributedTPU Op on the desired
// TPU_SYSTEM device, and a graph rewrite replaces it by the subgraph described
// above.
//
//
// A distributed TPU system can be cleanly shut down by running the following
// Ops:
//
// 1 Run _DisconnectHostFromDistributedTPUSystem on the TPU_SYSTEM of each host.
//
// 2 Run _ShutdownDistributedTPU on the TPU_SYSTEM where
// _ConfigureDistributedTPU was run. The Op will return an error if no system is
// configured.
//
//
// Most user code works by placing the ShutdownDistributedTPU Op on the desired
// TPU_SYSTEM device, and a graph rewrite replaces it by the subgraph described
// above.

REGISTER_OP("_ConfigureDistributedTPU")
    .Input("inputs: N * int32")
    .Output("output: string")
    .Attr("N: int >= 1")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      // Validate that all the inputs are scalars.
      for (int i = 0; i < c->num_inputs(); ++i) {
        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &input));
      }
      c->set_output(0, c->Scalar());
      return Status::OK();
    })
    .Doc(R"doc(
An op that sets up the centralized structures for a distributed TPU
system.

inputs: A scalar tensor for each host indicating how many TPU chips
there are on the host.
output: A tensor containing a TPUHostConfiguration proto serialized to
a string, containing the information necessary to initialize the chips
in a host.
enable_whole_mesh_compilations: Usually the master TPU worker is the only
worker compile ops are sent, and the master worker is the only one which
can execute them. Other TPU clients distribute TPU compilation across all
the hosts of the mesh, and setting this flag to True enables such mesh
initialization mode.
)doc");

REGISTER_OP("_WaitForDistributedTPU")
    .Input("inputs: N * int32")
    .Output("topology: string")
    .Attr("startup_timeout_sec: int = 20")
    .Attr("N: int")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      // Validate that all the inputs have the same vector shape.
      for (int i = 0; i < c->num_inputs(); ++i) {
        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 1, &input));
      }
      c->set_output(0, c->Scalar());
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that blocks execution until a distributed TPU system has
started up. This Op must be run on the same TPU_SYSTEM device as
_ConfigureDistributedTPU, and takes an inputs the outputs from the
_InitializeHostForDistributedTPU Ops.

inputs: For each initialized host, a vector giving the global TPU id
of each TPU on the host.
topology: A serialized tensorflow.tpu.TopologyProto that describes the TPU
topology.
startup_timeout_sec: The number of seconds to wait for the TPU system
to stabilize.
)doc");

REGISTER_OP("_SetGlobalTPUArray")
    .Input("topology: string")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that informs a host of the global ids of all the of TPUs in the
system.

topology: A serialized tensorflow.tpu.TopologyProto that describes the TPU
topology.
)doc");

REGISTER_OP("_ShutdownDistributedTPU")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape)
    .Doc(R"doc(
An op that shuts down a running distributed TPU system. The Op returns
an error if no system is running. This Op must be run on the same
TPU_SYSTEM device as the corresponding _ConfigureDistributedTPU was run
to start the system, and must be run only after
_DisconnectHostFromDistributedTPUSystem has completed on every host in
the system.
)doc");

REGISTER_OP("_InitializeHostForDistributedTPU")
    .Input("input: string")
    .Output("tpu_ids: int32")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .SetIsStateful()
    .SetShapeFn([](InferenceContext* c) {
      ShapeHandle input;
      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
      c->set_output(0, c->Vector(c->UnknownDim()));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
An op that connects each chip on the host to a centralized UberDriver to allow
them to operate as a distributed system with chips in other hosts.

input: A string containing the address of the UberDriver to connect to.
tpu_ids: A vector containing the global TPU id of each TPU on the host.
enable_whole_mesh_compilations: Usually the master TPU worker is the only
worker compile ops are sent, and the master worker is the only one which
can execute them. Other TPU clients distribute TPU compilation across all
the hosts of the mesh, and setting this flag to True enables such mesh
initialization mode.
)doc");

REGISTER_OP("_DisconnectHostFromDistributedTPUSystem")
    .Output("number_of_tpu_chips: int32")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape)
    .Doc(R"doc(
An op that disconnects the TPUs on a host from a running distributed
TPU system.

number_of_tpu_chips: A scalar tensor containing the number of TPU
chips on the host.
)doc");

REGISTER_OP("ConfigureDistributedTPU")
    .Output("topology: string")
    .Attr("embedding_config: string = ''")
    .Attr("tpu_embedding_config: string = ''")
    .Attr("is_global_init: bool = false")
    .Attr("enable_whole_mesh_compilations: bool = false")
    .Attr("compilation_failure_closes_chips: bool = true")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

REGISTER_OP("ShutdownDistributedTPU")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

REGISTER_OP("ConfigureTPUEmbedding")
    .Attr("config: string")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape);

}  // end namespace tensorflow
<|endoftext|>


<commit_before>/*----------------------------------------------------------------------------*/
/* Copyright (c) FIRST 2015. All Rights Reserved.                             */
/* Open Source Software - may be modified and shared by FRC teams. The code   */
/* must be accompanied by the FIRST BSD license file in the root directory of */
/* the project.                                                               */
/*----------------------------------------------------------------------------*/

#include "Storage.h"

#include "gtest/gtest.h"

namespace nt {

class StorageTest : public ::testing::Test {
 public:
  void PopulateTrue() {
    storage.SetEntryTypeValue("foo", Value::MakeBoolean(true));
    ASSERT_EQ(1u, updates().size());
    updates().pop();
  }

  Storage::EntriesMap& entries() { return storage.m_entries; }
  Storage::UpdateQueue& updates() { return storage.updates(); }
  Storage storage;
};

TEST_F(StorageTest, Construct) {
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, FindEntryNotExist) {
  ASSERT_FALSE(storage.FindEntry("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTest, FindEntryExist) {
  auto entry1 = storage.GetEntry("foo");
  auto entry = storage.FindEntry("foo");
  ASSERT_TRUE(bool(entry));
  ASSERT_EQ(entry1, entry);
  ASSERT_FALSE(storage.FindEntry("bar"));
}

TEST_F(StorageTest, GetEntryNotExist) {
  auto entry = storage.GetEntry("foo");
  ASSERT_TRUE(bool(entry));
  ASSERT_EQ(1u, entries().size());

  EXPECT_FALSE(entry->value());
  EXPECT_EQ(0u, entry->flags());
  EXPECT_EQ(0xffffu, entry->id());
  EXPECT_EQ(SequenceNumber(), entry->seq_num());
}

TEST_F(StorageTest, GetEntryValueNotExist) {
  ASSERT_FALSE(storage.GetEntryValue("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTest, GetEntryValueExist) {
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  ASSERT_EQ(1u, updates().size());
  updates().pop();
  ASSERT_EQ(value, storage.GetEntryValue("foo"));
}

TEST_F(StorageTest, SetEntryTypeValueAssignNew) {
  // brand new entry
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTest, SetEntryTypeValueAssignTypeChange) {
  PopulateTrue();
  // update with different type results in assignment message
  auto value = Value::MakeDouble(0.0);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTest, SetEntryTypeValueEqualValue) {
  PopulateTrue();
  // update with same type and same value: change value contents but no update
  // message is issued (minimizing bandwidth usage)
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryTypeValueDifferentValue) {
  PopulateTrue();
  // update with same type and different value results in value update message
  auto value = Value::MakeBoolean(false);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kValueUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryTypeValueEmptyName) {
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("", value);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryTypeValueEmptyValue) {
  storage.SetEntryTypeValue("foo", nullptr);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueAssignNew) {
  // brand new entry
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTest, SetEntryValueAssignTypeChange) {
  PopulateTrue();
  // update with different type results in error and no message
  auto value = Value::MakeDouble(0.0);
  EXPECT_FALSE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_NE(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueEqualValue) {
  PopulateTrue();
  // update with same type and same value: change value contents but no update
  // message is issued (minimizing bandwidth usage)
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueDifferentValue) {
  PopulateTrue();
  // update with same type and different value results in value update message
  auto value = Value::MakeBoolean(false);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kValueUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryValueEmptyName) {
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("", value));
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueEmptyValue) {
  EXPECT_TRUE(storage.SetEntryValue("foo", nullptr));
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryFlagsNew) {
  // flags setting doesn't create an entry
  storage.SetEntryFlags("foo", 0u);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryFlagsEqualValue) {
  PopulateTrue();
  // update with same value: no update message is issued (minimizing bandwidth
  // usage)
  storage.SetEntryFlags("foo", 0u);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(0u, entry->flags());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryFlagsDifferentValue) {
  PopulateTrue();
  // update with different value results in flags update message
  storage.SetEntryFlags("foo", 1u);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(1u, entry->flags());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kFlagsUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryFlagsEmptyName) {
  storage.SetEntryFlags("", 0u);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, GetEntryFlagsNotExist) {
  ASSERT_EQ(0u, storage.GetEntryFlags("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTest, GetEntryFlagsExist) {
  PopulateTrue();
  storage.SetEntryFlags("foo", 1u);
  while (!updates().empty()) updates().pop();
  ASSERT_EQ(1u, storage.GetEntryFlags("foo"));
}

TEST_F(StorageTest, DeleteEntryNotExist) {
  storage.DeleteEntry("foo");
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, DeleteEntryExist) {
  PopulateTrue();
  auto entry = storage.GetEntry("foo");
  storage.DeleteEntry("foo");
  ASSERT_TRUE(entries().empty());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kDelete, update.kind);
}

TEST_F(StorageTest, DeleteAllEntriesEmpty) {
  storage.DeleteAllEntries();
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, DeleteAllEntries) {
  PopulateTrue();
  storage.DeleteAllEntries();
  ASSERT_TRUE(entries().empty());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(nullptr, update.entry);
  EXPECT_EQ(Storage::Update::kDeleteAll, update.kind);
}

}  // namespace nt
<commit_msg>StorageTest: Add fixture classes, add GetEntryInfo test.<commit_after>/*----------------------------------------------------------------------------*/
/* Copyright (c) FIRST 2015. All Rights Reserved.                             */
/* Open Source Software - may be modified and shared by FRC teams. The code   */
/* must be accompanied by the FIRST BSD license file in the root directory of */
/* the project.                                                               */
/*----------------------------------------------------------------------------*/

#include "Storage.h"

#include "gtest/gtest.h"

namespace nt {

class StorageTest : public ::testing::Test {
 public:
  Storage::EntriesMap& entries() { return storage.m_entries; }
  Storage::UpdateQueue& updates() { return storage.updates(); }
  Storage storage;
};

class StorageTestPopulateOne : public StorageTest {
 public:
  StorageTestPopulateOne() {
    storage.SetEntryTypeValue("foo", Value::MakeBoolean(true));
    while (!updates().empty()) updates().pop();
  }
};

class StorageTestPopulated : public StorageTest {
 public:
  StorageTestPopulated() {
    storage.SetEntryTypeValue("foo", Value::MakeBoolean(true));
    storage.SetEntryTypeValue("foo2", Value::MakeDouble(0.0));
    storage.SetEntryTypeValue("bar", Value::MakeDouble(1.0));
    storage.SetEntryTypeValue("bar2", Value::MakeBoolean(false));
    while (!updates().empty()) updates().pop();
  }
};

TEST_F(StorageTest, Construct) {
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, FindEntryNotExist) {
  ASSERT_FALSE(storage.FindEntry("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTest, FindEntryExist) {
  auto entry1 = storage.GetEntry("foo");
  auto entry = storage.FindEntry("foo");
  ASSERT_TRUE(bool(entry));
  ASSERT_EQ(entry1, entry);
  ASSERT_FALSE(storage.FindEntry("bar"));
}

TEST_F(StorageTest, GetEntryNotExist) {
  auto entry = storage.GetEntry("foo");
  ASSERT_TRUE(bool(entry));
  ASSERT_EQ(1u, entries().size());

  EXPECT_FALSE(entry->value());
  EXPECT_EQ(0u, entry->flags());
  EXPECT_EQ(0xffffu, entry->id());
  EXPECT_EQ(SequenceNumber(), entry->seq_num());
}

TEST_F(StorageTest, GetEntryValueNotExist) {
  ASSERT_FALSE(storage.GetEntryValue("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTest, GetEntryValueExist) {
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  ASSERT_EQ(1u, updates().size());
  updates().pop();
  ASSERT_EQ(value, storage.GetEntryValue("foo"));
}

TEST_F(StorageTest, SetEntryTypeValueAssignNew) {
  // brand new entry
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTestPopulateOne, SetEntryTypeValueAssignTypeChange) {
  // update with different type results in assignment message
  auto value = Value::MakeDouble(0.0);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTestPopulateOne, SetEntryTypeValueEqualValue) {
  // update with same type and same value: change value contents but no update
  // message is issued (minimizing bandwidth usage)
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, SetEntryTypeValueDifferentValue) {
  // update with same type and different value results in value update message
  auto value = Value::MakeBoolean(false);
  storage.SetEntryTypeValue("foo", value);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kValueUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryTypeValueEmptyName) {
  auto value = Value::MakeBoolean(true);
  storage.SetEntryTypeValue("", value);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryTypeValueEmptyValue) {
  storage.SetEntryTypeValue("foo", nullptr);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueAssignNew) {
  // brand new entry
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kAssign, update.kind);
}

TEST_F(StorageTestPopulateOne, SetEntryValueAssignTypeChange) {
  // update with different type results in error and no message
  auto value = Value::MakeDouble(0.0);
  EXPECT_FALSE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_NE(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, SetEntryValueEqualValue) {
  // update with same type and same value: change value contents but no update
  // message is issued (minimizing bandwidth usage)
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, SetEntryValueDifferentValue) {
  // update with same type and different value results in value update message
  auto value = Value::MakeBoolean(false);
  EXPECT_TRUE(storage.SetEntryValue("foo", value));
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(value, entry->value());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kValueUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryValueEmptyName) {
  auto value = Value::MakeBoolean(true);
  EXPECT_TRUE(storage.SetEntryValue("", value));
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryValueEmptyValue) {
  EXPECT_TRUE(storage.SetEntryValue("foo", nullptr));
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, SetEntryFlagsNew) {
  // flags setting doesn't create an entry
  storage.SetEntryFlags("foo", 0u);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, SetEntryFlagsEqualValue) {
  // update with same value: no update message is issued (minimizing bandwidth
  // usage)
  storage.SetEntryFlags("foo", 0u);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(0u, entry->flags());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, SetEntryFlagsDifferentValue) {
  // update with different value results in flags update message
  storage.SetEntryFlags("foo", 1u);
  auto entry = storage.GetEntry("foo");
  EXPECT_EQ(1u, entry->flags());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kFlagsUpdate, update.kind);
}

TEST_F(StorageTest, SetEntryFlagsEmptyName) {
  storage.SetEntryFlags("", 0u);
  ASSERT_TRUE(entries().empty());
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTest, GetEntryFlagsNotExist) {
  ASSERT_EQ(0u, storage.GetEntryFlags("foo"));
  ASSERT_TRUE(entries().empty());
}

TEST_F(StorageTestPopulateOne, GetEntryFlagsExist) {
  storage.SetEntryFlags("foo", 1u);
  while (!updates().empty()) updates().pop();
  ASSERT_EQ(1u, storage.GetEntryFlags("foo"));
}

TEST_F(StorageTest, DeleteEntryNotExist) {
  storage.DeleteEntry("foo");
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulateOne, DeleteEntryExist) {
  auto entry = storage.GetEntry("foo");
  storage.DeleteEntry("foo");
  ASSERT_TRUE(entries().empty());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(entry, update.entry);
  EXPECT_EQ(Storage::Update::kDelete, update.kind);
}

TEST_F(StorageTest, DeleteAllEntriesEmpty) {
  storage.DeleteAllEntries();
  ASSERT_TRUE(updates().empty());
}

TEST_F(StorageTestPopulated, DeleteAllEntries) {
  storage.DeleteAllEntries();
  ASSERT_TRUE(entries().empty());
  ASSERT_EQ(1u, updates().size());
  auto update = updates().pop();
  EXPECT_EQ(nullptr, update.entry);
  EXPECT_EQ(Storage::Update::kDeleteAll, update.kind);
}

TEST_F(StorageTestPopulated, GetEntryInfoAll) {
  auto info = storage.GetEntryInfo("", 0u);
  ASSERT_EQ(4u, info.size());
}

TEST_F(StorageTestPopulated, GetEntryInfoPrefix) {
  auto info = storage.GetEntryInfo("foo", 0u);
  ASSERT_EQ(2u, info.size());
  if (info[0].name == "foo") {
    EXPECT_EQ("foo", info[0].name);
    EXPECT_EQ(NT_BOOLEAN, info[0].type);
    EXPECT_EQ("foo2", info[1].name);
    EXPECT_EQ(NT_DOUBLE, info[1].type);
  } else {
    EXPECT_EQ("foo2", info[0].name);
    EXPECT_EQ(NT_DOUBLE, info[0].type);
    EXPECT_EQ("foo", info[1].name);
    EXPECT_EQ(NT_BOOLEAN, info[1].type);
  }
}

TEST_F(StorageTestPopulated, GetEntryInfoTypes) {
  auto info = storage.GetEntryInfo("", NT_DOUBLE);
  ASSERT_EQ(2u, info.size());
  EXPECT_EQ(NT_DOUBLE, info[0].type);
  EXPECT_EQ(NT_DOUBLE, info[1].type);
  if (info[0].name == "foo2") {
    EXPECT_EQ("foo2", info[0].name);
    EXPECT_EQ("bar", info[1].name);
  } else {
    EXPECT_EQ("bar", info[0].name);
    EXPECT_EQ("foo2", info[1].name);
  }
}

TEST_F(StorageTestPopulated, GetEntryInfoPrefixTypes) {
  auto info = storage.GetEntryInfo("bar", NT_BOOLEAN);
  ASSERT_EQ(1u, info.size());
  EXPECT_EQ("bar2", info[0].name);
  EXPECT_EQ(NT_BOOLEAN, info[0].type);
}

}  // namespace nt
<|endoftext|>


<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2012 Preferred Infrastracture and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#ifndef JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_
#define JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_

#include <iosfwd>
#include <string>
#include <utility>
#include <vector>
#include <msgpack.hpp>
#include "jubatus/util/data/unordered_map.h"
#include "jubatus/util/data/unordered_set.h"
#include "lsh_vector.hpp"
#include "storage_type.hpp"
#include "../common/key_manager.hpp"
#include "../common/unordered_map.hpp"
#include "../framework/mixable.hpp"

namespace jubatus {
namespace core {
namespace storage {

struct lsh_entry {
  std::vector<uint64_t> lsh_hash;
  bit_vector simhash_bv;
  float norm;

  MSGPACK_DEFINE(lsh_hash, simhash_bv, norm);

  template <typename Ar>
  void serialize(Ar& ar) {
    ar & JUBA_MEMBER(lsh_hash) & JUBA_MEMBER(simhash_bv) & JUBA_MEMBER(norm);
  }
};

typedef jubatus::util::data::unordered_map<std::string, lsh_entry>
  lsh_master_table_t;

typedef jubatus::util::data::unordered_map<uint64_t, std::vector<uint64_t> >
  lsh_table_t;

class lsh_index_storage {
 public:
  lsh_index_storage();
  lsh_index_storage(size_t lsh_num, size_t table_num, uint32_t seed);
  lsh_index_storage(size_t table_num, const std::vector<float>& shift);
  virtual ~lsh_index_storage();

  // hash is a randomly-projected and scaled hash values without shifting
  void set_row(
      const std::string& row,
      const std::vector<float>& hash,
      float norm);
  void remove_row(const std::string& row);
  void clear();
  void get_all_row_ids(std::vector<std::string>& ids) const;

  void similar_row(
      const std::vector<float>& hash,
      float norm,
      uint64_t probe_num,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  void similar_row(
      const std::string& id,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  std::string name() const;

  size_t table_num() const {
    return table_num_;
  }

  size_t all_lsh_num() const {
    return shift_.size();
  }
  storage::version get_version() const {
    return storage::version();
  }

  void pack(msgpack::packer<msgpack::sbuffer>& packer) const;
  void unpack(msgpack::object o);

  void get_diff(lsh_master_table_t& diff) const;
  bool put_diff(const lsh_master_table_t& mixed_diff);
  void mix(const lsh_master_table_t& lhs, lsh_master_table_t& rhs) const;

  MSGPACK_DEFINE(master_table_, master_table_diff_, lsh_table_,
      lsh_table_diff_, shift_, table_num_, key_manager_);

 private:
  friend class jubatus::util::data::serialization::access;
  template <class Ar>
  void serialize(Ar& ar) {
    ar
        & JUBA_MEMBER(master_table_)
        & JUBA_MEMBER(master_table_diff_)
        & JUBA_MEMBER(lsh_table_)
        & JUBA_MEMBER(lsh_table_diff_)
        & JUBA_MEMBER(shift_)
        & JUBA_MEMBER(table_num_)
        & JUBA_MEMBER(key_manager_);
  }

  lsh_master_table_t::iterator remove_and_get_row(const std::string& row);
  void put_empty_entry(uint64_t row_id, const lsh_entry& entry);

  std::vector<float> make_entry(
      const std::vector<float>& hash,
      float norm,
      lsh_entry& entry) const;
  bool retrieve_hit_rows(
      uint64_t hash,
      size_t ret_num,
      jubatus::util::data::unordered_set<uint64_t>& cands) const;

  void get_sorted_similar_rows(
      const jubatus::util::data::unordered_set<uint64_t>& cands,
      const bit_vector& query_simhash,
      float query_norm,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  const lsh_entry* get_lsh_entry(const std::string& row) const;
  void remove_model_row(const std::string& row);
  void set_mixed_row(const std::string& row, const lsh_entry& entry);

  lsh_master_table_t master_table_;
  lsh_master_table_t master_table_diff_;

  lsh_table_t lsh_table_;
  lsh_table_t lsh_table_diff_;

  std::vector<float> shift_;
  uint64_t table_num_;
  common::key_manager key_manager_;
};

typedef framework::delegating_mixable<lsh_index_storage, lsh_master_table_t>
    mixable_lsh_index_storage;

}  // namespace storage
}  // namespace core
}  // namespace jubatus

#endif  // JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_
<commit_msg>Fix lsh_index_storage to use mixable helper<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2012 Preferred Infrastracture and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#ifndef JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_
#define JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_

#include <iosfwd>
#include <string>
#include <utility>
#include <vector>
#include <msgpack.hpp>
#include "jubatus/util/data/unordered_map.h"
#include "jubatus/util/data/unordered_set.h"
#include "lsh_vector.hpp"
#include "storage_type.hpp"
#include "../common/key_manager.hpp"
#include "../common/unordered_map.hpp"
#include "../framework/mixable_helper.hpp"

namespace jubatus {
namespace core {
namespace storage {

struct lsh_entry {
  std::vector<uint64_t> lsh_hash;
  bit_vector simhash_bv;
  float norm;

  MSGPACK_DEFINE(lsh_hash, simhash_bv, norm);

  template <typename Ar>
  void serialize(Ar& ar) {
    ar & JUBA_MEMBER(lsh_hash) & JUBA_MEMBER(simhash_bv) & JUBA_MEMBER(norm);
  }
};

typedef jubatus::util::data::unordered_map<std::string, lsh_entry>
  lsh_master_table_t;

typedef jubatus::util::data::unordered_map<uint64_t, std::vector<uint64_t> >
  lsh_table_t;

class lsh_index_storage {
 public:
  lsh_index_storage();
  lsh_index_storage(size_t lsh_num, size_t table_num, uint32_t seed);
  lsh_index_storage(size_t table_num, const std::vector<float>& shift);
  virtual ~lsh_index_storage();

  // hash is a randomly-projected and scaled hash values without shifting
  void set_row(
      const std::string& row,
      const std::vector<float>& hash,
      float norm);
  void remove_row(const std::string& row);
  void clear();
  void get_all_row_ids(std::vector<std::string>& ids) const;

  void similar_row(
      const std::vector<float>& hash,
      float norm,
      uint64_t probe_num,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  void similar_row(
      const std::string& id,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  std::string name() const;

  size_t table_num() const {
    return table_num_;
  }

  size_t all_lsh_num() const {
    return shift_.size();
  }
  storage::version get_version() const {
    return storage::version();
  }

  void pack(msgpack::packer<msgpack::sbuffer>& packer) const;
  void unpack(msgpack::object o);

  void get_diff(lsh_master_table_t& diff) const;
  bool put_diff(const lsh_master_table_t& mixed_diff);
  void mix(const lsh_master_table_t& lhs, lsh_master_table_t& rhs) const;

  MSGPACK_DEFINE(master_table_, master_table_diff_, lsh_table_,
      lsh_table_diff_, shift_, table_num_, key_manager_);

 private:
  friend class jubatus::util::data::serialization::access;
  template <class Ar>
  void serialize(Ar& ar) {
    ar
        & JUBA_MEMBER(master_table_)
        & JUBA_MEMBER(master_table_diff_)
        & JUBA_MEMBER(lsh_table_)
        & JUBA_MEMBER(lsh_table_diff_)
        & JUBA_MEMBER(shift_)
        & JUBA_MEMBER(table_num_)
        & JUBA_MEMBER(key_manager_);
  }

  lsh_master_table_t::iterator remove_and_get_row(const std::string& row);
  void put_empty_entry(uint64_t row_id, const lsh_entry& entry);

  std::vector<float> make_entry(
      const std::vector<float>& hash,
      float norm,
      lsh_entry& entry) const;
  bool retrieve_hit_rows(
      uint64_t hash,
      size_t ret_num,
      jubatus::util::data::unordered_set<uint64_t>& cands) const;

  void get_sorted_similar_rows(
      const jubatus::util::data::unordered_set<uint64_t>& cands,
      const bit_vector& query_simhash,
      float query_norm,
      uint64_t ret_num,
      std::vector<std::pair<std::string, float> >& ids) const;
  const lsh_entry* get_lsh_entry(const std::string& row) const;
  void remove_model_row(const std::string& row);
  void set_mixed_row(const std::string& row, const lsh_entry& entry);

  lsh_master_table_t master_table_;
  lsh_master_table_t master_table_diff_;

  lsh_table_t lsh_table_;
  lsh_table_t lsh_table_diff_;

  std::vector<float> shift_;
  uint64_t table_num_;
  common::key_manager key_manager_;
};

typedef framework::linear_mixable_helper<lsh_index_storage, lsh_master_table_t>
    mixable_lsh_index_storage;

}  // namespace storage
}  // namespace core
}  // namespace jubatus

#endif  // JUBATUS_CORE_STORAGE_LSH_INDEX_STORAGE_HPP_
<|endoftext|>