<commit_before>#include "RandomWorld.hpp"
#include "../Simulator.hpp"
#include "../agent/Ant.hpp"
#include "../gui/GuiEventManager.hpp"
#include "../sim/nav/Node.hpp"
#include "../sim/nav/Edge.hpp"
#include "../sim/agent/Ant.hpp"
#include "../sim/worldobject/AntFoodPile.hpp"
#include "../sim/worldobject/SolidObject.hpp"
#include "../sim/worldobject/AntHome.hpp"
#include "../util/Random.hpp"

#include <vector>
#include <memory>
#include <cassert>

// Christopher D. Canfield
// November 2013
// RandomWorld.cpp

using cdc::RandomWorld;
using cdc::GuiEventManager;
using cdc::Node;
using cdc::Edge;
using cdc::GridLocation;
using cdc::AntFoodPile;
using cdc::AntHome;
using cdc::Random;
using cdc::SolidObject;
using cdc::Ant;

using namespace sf;
using namespace std;

// Creates the world's navigation graph.
void createNavGraph(vector<Node>& navGraph);

// Adds food piles to the world.
void addFood(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntFoodPile>& antFoodPiles);

// Adds obstructions to the world.
void addObstructions(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<Sprite>& obstructions);

// Adds the ant hill to the world.
// Returns the ant hill location within the nav graph.
int addAntHill(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntHome>& antHills);

// Adds a random number of ants to the world.
void addAnts(vector<Node>& navGraph, AntHome& antHill, GuiEventManager& eventManager, vector<Ant>& ants);

// Finds an unoccupied location between the min and max node (inclusive).
// Returns the index of the node within the nav graph.
int findValidLocation(vector<Node>& navGraph, vector<Node*>& occupiedAreas, int minAllowedNode, int maxAllowedNode, Random& rand);

// Returns true if the node is occupied, or false if not.
bool isOccupied(vector<Node*> occupiedAreas, Node& location);

// TODO: the size of the world should be accessible outside of the class,
// or the size should be passed into the world.
const int nav_graph_rows = 30;
const int nav_graph_columns = 30;

const int side_offset = 50;
const int node_offset = 100;

RandomWorld::RandomWorld(GuiEventManager& eventManager) :
	eventManager(eventManager)
{
}

RandomWorld::~RandomWorld()
{
}

void RandomWorld::create(GuiEventManager& eventManager)
{
	createNavGraph(navGraph);
	vector<Node*> offLimitAreas;

	addFood(navGraph, offLimitAreas, antFoodPiles);
	addObstructions(navGraph, offLimitAreas, obstructions);
	int antHillLocation = addAntHill(navGraph, offLimitAreas, antHills);
	addAnts(navGraph, antHills[0], eventManager, ants);
}

void createNavGraph(vector<Node>& navGraph)
{
	navGraph.reserve(nav_graph_rows * nav_graph_rows);

	for (int row = 0; row < nav_graph_rows; ++row)
	{
		for (int column = 0; column < nav_graph_columns; ++column)
		{
			int pixelX = side_offset + (column * node_offset);
			int pixelY = side_offset + (row * node_offset);
			navGraph.push_back(Node(GridLocation(row, column), pixelX, pixelY));
		}
	}

	for (int row = 0; row < nav_graph_rows; ++row)
	{
		for (int column = 0; column < nav_graph_columns; ++column)
		{
			auto& startNode = navGraph[(row * nav_graph_columns) + column];
			// Add up connection.
			if (row > 0)
			{
				auto& endNode = navGraph[(row - 1) * nav_graph_columns + column];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add down connection.
			if (row < nav_graph_columns - 1)
			{
				auto& endNode = navGraph[(row + 1) * nav_graph_columns + column];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add left connection.
			if (column > 0)
			{
				auto& endNode = navGraph[row * nav_graph_columns + column - 1];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add right connection.
			if (column < nav_graph_columns - 1)
			{
				auto& endNode = navGraph[row * nav_graph_columns + column + 1];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add diagonal connection 1.

			// Add diagonal connection 2.
		}
	}
}

void addFood(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntFoodPile>& antFoodPiles)
{
	Random rand;

	// Add between 3 and 10 piles of food.
	const int minFoodPiles = 3;
	const int maxFoodPiles = 10;

	int foodPileCount = rand.getInteger(minFoodPiles, maxFoodPiles);
	for (int i = 0; i < maxFoodPiles; ++i)
	{
		int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, navGraph.size(), rand);
		int foodInPile = rand.getInteger(50, 750);
		antFoodPiles.push_back(AntFoodPile(foodInPile, navGraph[nodeLocation]));
		occupiedAreas.push_back(&navGraph[nodeLocation]);
	}
}

void addObstructions(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<Sprite>& obstructions)
{
	Random rand;

	// Add between 10 and 20 obstructions.
	const int maxObstructions = 20;
	const int minObstructions = 10;

	int obstructionCount = rand.getInteger(minObstructions, maxObstructions);
	for (int i = 0; i < maxObstructions; ++i)
	{
		int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, navGraph.size(), rand);
		auto rock = SolidObject::createRock(navGraph, navGraph[nodeLocation].getPixelX(), navGraph[nodeLocation].getPixelY());
		obstructions.push_back(rock);
		occupiedAreas.push_back(&navGraph[nodeLocation]);
	}
}

int addAntHill(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntHome>& antHills)
{
	Random rand;

	// Ant hill can be in any node within the first three rows.
	int antHillLocation = rand.getInteger(0, 3 * nav_graph_columns);
	int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, 3 * nav_graph_columns, rand);

	antHills.push_back(AntHome(navGraph[antHillLocation], navGraph));
	occupiedAreas.push_back(&navGraph[antHillLocation]);
	return antHillLocation;
}

void addAnts(vector<Node>& navGraph, AntHome& antHill, GuiEventManager& eventManager, vector<Ant>& ants)
{
	Random rand;

	const int minAnts = 30;
	const int maxAnts = 60;

	int antCount = rand.getInteger(minAnts, maxAnts);
	for (int i = 0; i < antCount; ++i)
	{
		ants.push_back(Ant(eventManager, antHill, antHill.getNavGraphHelper(), antHill.getNode()));
	}
}

int findValidLocation(vector<Node>& navGraph, vector<Node*>& occupiedAreas, int minAllowedNode, int maxAllowedNode, Random& rand)
{
	bool validLocationFound = false;
	while (!validLocationFound)
	{
		int nodeLocation = rand.getInteger(minAllowedNode, maxAllowedNode);
		if (!isOccupied(occupiedAreas, navGraph[nodeLocation]))
		{
			return nodeLocation;
		}
	}
	return -1;
}

bool isOccupied(vector<Node*> occupiedAreas, Node& location)
{
	for (auto& occupied : occupiedAreas)
	{
		if (occupied == &location)
		{
			return true;
		}
	}
	return false;
}<commit_msg>Added diagonal edges to nodes<commit_after>#include "RandomWorld.hpp"
#include "../Simulator.hpp"
#include "../agent/Ant.hpp"
#include "../gui/GuiEventManager.hpp"
#include "../sim/nav/Node.hpp"
#include "../sim/nav/Edge.hpp"
#include "../sim/agent/Ant.hpp"
#include "../sim/worldobject/AntFoodPile.hpp"
#include "../sim/worldobject/SolidObject.hpp"
#include "../sim/worldobject/AntHome.hpp"
#include "../util/Random.hpp"

#include <vector>
#include <memory>
#include <cassert>

// Christopher D. Canfield
// November 2013
// RandomWorld.cpp

using cdc::RandomWorld;
using cdc::GuiEventManager;
using cdc::Node;
using cdc::Edge;
using cdc::GridLocation;
using cdc::AntFoodPile;
using cdc::AntHome;
using cdc::Random;
using cdc::SolidObject;
using cdc::Ant;

using namespace sf;
using namespace std;

// Creates the world's navigation graph.
void createNavGraph(vector<Node>& navGraph);

// Adds food piles to the world.
void addFood(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntFoodPile>& antFoodPiles);

// Adds obstructions to the world.
void addObstructions(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<Sprite>& obstructions);

// Adds the ant hill to the world.
// Returns the ant hill location within the nav graph.
int addAntHill(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntHome>& antHills);

// Adds a random number of ants to the world.
void addAnts(vector<Node>& navGraph, AntHome& antHill, GuiEventManager& eventManager, vector<Ant>& ants);

// Finds an unoccupied location between the min and max node (inclusive).
// Returns the index of the node within the nav graph.
int findValidLocation(vector<Node>& navGraph, vector<Node*>& occupiedAreas, int minAllowedNode, int maxAllowedNode, Random& rand);

// Returns true if the node is occupied, or false if not.
bool isOccupied(vector<Node*> occupiedAreas, Node& location);

// TODO: the size of the world should be accessible outside of the class,
// or the size should be passed into the world.
const int nav_graph_rows = 30;
const int nav_graph_columns = 30;

const int side_offset = 50;
const int node_offset = 100;

RandomWorld::RandomWorld(GuiEventManager& eventManager) :
	eventManager(eventManager)
{
}

RandomWorld::~RandomWorld()
{
}

void RandomWorld::create(GuiEventManager& eventManager)
{
	createNavGraph(navGraph);
	vector<Node*> offLimitAreas;

	addFood(navGraph, offLimitAreas, antFoodPiles);
	addObstructions(navGraph, offLimitAreas, obstructions);
	int antHillLocation = addAntHill(navGraph, offLimitAreas, antHills);
	addAnts(navGraph, antHills[0], eventManager, ants);
}

void createNavGraph(vector<Node>& navGraph)
{
	navGraph.reserve(nav_graph_rows * nav_graph_rows);
	vector<vector<Node*>> navGraphEdgeHelper(nav_graph_rows, vector<Node*>(nav_graph_columns, nullptr));

	for (int row = 0; row < nav_graph_rows; ++row)
	{
		for (int column = 0; column < nav_graph_columns; ++column)
		{
			int pixelX = side_offset + (column * node_offset);
			int pixelY = side_offset + (row * node_offset);
			navGraph.push_back(Node(GridLocation(row, column), pixelX, pixelY));
			navGraphEdgeHelper[row][column] = &navGraph.back();
		}
	}

	for (int row = 0; row < nav_graph_rows; ++row)
	{
		for (int column = 0; column < nav_graph_columns; ++column)
		{
			auto& startNode = *navGraphEdgeHelper[row][column];
			// Add up connection.
			if (row > 0)
			{
				auto& endNode = *navGraphEdgeHelper[row - 1][column];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add down connection.
			if (row < nav_graph_columns - 1)
			{
				auto& endNode = *navGraphEdgeHelper[row + 1][column];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add left connection.
			if (column > 0)
			{
				auto& endNode = *navGraphEdgeHelper[row][column - 1];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add right connection.
			if (column < nav_graph_columns - 1)
			{
				auto& endNode = *navGraphEdgeHelper[row][column + 1];
				auto edge = make_shared<Edge>(startNode, endNode, 1);
				if (!startNode.edgeExists(edge))
				{
					startNode.addEdge(edge);
				}
			}
			// Add diagonal connections
			if (row > 0 && row < (nav_graph_rows - 1) && column > 0 && column < (nav_graph_columns - 1))
			{
				auto& diagonal1Node = *navGraphEdgeHelper[row + 1][column - 1];
				auto edge1 = make_shared<Edge>(startNode, diagonal1Node, 1);

				auto& diagonal2Node = *navGraphEdgeHelper[row + 1][column + 1];
				auto edge2 = make_shared<Edge>(startNode, diagonal2Node, 1);

				auto& diagonal3Node = *navGraphEdgeHelper[row - 1][column - 1];
				auto edge3 = make_shared<Edge>(startNode, diagonal3Node, 1);

				auto& diagonal4Node = *navGraphEdgeHelper[row - 1][column + 1];
				auto edge4 = make_shared<Edge>(startNode, diagonal4Node, 1);

				if (!startNode.edgeExists(edge1))
				{
					startNode.addEdge(edge1);
				}
				else if (!startNode.edgeExists(edge2))
				{
					startNode.addEdge(edge2);
				}
				else if (!startNode.edgeExists(edge3))
				{
					startNode.addEdge(edge3);
				}
				else if (!startNode.edgeExists(edge4))
				{
					startNode.addEdge(edge4);
				}
			}
		}
	}
}

void addFood(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntFoodPile>& antFoodPiles)
{
	Random rand;

	// Add between 3 and 10 piles of food.
	const int minFoodPiles = 3;
	const int maxFoodPiles = 10;

	int foodPileCount = rand.getInteger(minFoodPiles, maxFoodPiles);
	for (int i = 0; i < maxFoodPiles; ++i)
	{
		int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, navGraph.size(), rand);
		int foodInPile = rand.getInteger(50, 750);
		antFoodPiles.push_back(AntFoodPile(foodInPile, navGraph[nodeLocation]));
		occupiedAreas.push_back(&navGraph[nodeLocation]);
	}
}

void addObstructions(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<Sprite>& obstructions)
{
	Random rand;

	// Add between 10 and 20 obstructions.
	const int maxObstructions = 20;
	const int minObstructions = 10;

	int obstructionCount = rand.getInteger(minObstructions, maxObstructions);
	for (int i = 0; i < maxObstructions; ++i)
	{
		int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, navGraph.size(), rand);
		auto rock = SolidObject::createRock(navGraph, navGraph[nodeLocation].getPixelX(), navGraph[nodeLocation].getPixelY());
		obstructions.push_back(rock);
		occupiedAreas.push_back(&navGraph[nodeLocation]);
	}
}

int addAntHill(vector<Node>& navGraph, vector<Node*>& occupiedAreas, vector<AntHome>& antHills)
{
	Random rand;

	// Ant hill can be in any node within the first three rows.
	int antHillLocation = rand.getInteger(0, 3 * nav_graph_columns);
	int nodeLocation = findValidLocation(navGraph, occupiedAreas, 0, 3 * nav_graph_columns, rand);

	antHills.push_back(AntHome(navGraph[antHillLocation], navGraph));
	occupiedAreas.push_back(&navGraph[antHillLocation]);
	return antHillLocation;
}

void addAnts(vector<Node>& navGraph, AntHome& antHill, GuiEventManager& eventManager, vector<Ant>& ants)
{
	Random rand;

	const int minAnts = 30;
	const int maxAnts = 60;

	int antCount = rand.getInteger(minAnts, maxAnts);
	for (int i = 0; i < antCount; ++i)
	{
		ants.push_back(Ant(eventManager, antHill, antHill.getNavGraphHelper(), antHill.getNode()));
	}
}

int findValidLocation(vector<Node>& navGraph, vector<Node*>& occupiedAreas, int minAllowedNode, int maxAllowedNode, Random& rand)
{
	bool validLocationFound = false;
	while (!validLocationFound)
	{
		int nodeLocation = rand.getInteger(minAllowedNode, maxAllowedNode);
		if (!isOccupied(occupiedAreas, navGraph[nodeLocation]))
		{
			return nodeLocation;
		}
	}
	return -1;
}

bool isOccupied(vector<Node*> occupiedAreas, Node& location)
{
	for (auto& occupied : occupiedAreas)
	{
		if (occupied == &location)
		{
			return true;
		}
	}
	return false;
}<|endoftext|>


<commit_before>#include <jni.h>
#include <stdlib.h>
#include <time.h>
#include <locale.h>

#include <mpv/client.h>
#include <mpv/opengl_cb.h>

#include <EGL/egl.h>

#include "main.h"

extern "C" {
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_init(JNIEnv* env, jobject obj);
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_resize(JNIEnv* env, jobject obj, jint width, jint height);
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_step(JNIEnv* env, jobject obj);
};

static void die(const char *msg)
{
    ALOGE("%s", msg);
    exit(1);
}

static void *get_proc_address_mpv(void *fn_ctx, const char *name)
{
    return (void*)eglGetProcAddress(name);
}

mpv_handle *mpv;
mpv_opengl_cb_context *mpv_gl;

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_init(JNIEnv* env, jobject obj) {
    ALOGE("init");

    setlocale(LC_NUMERIC, "C");

    mpv = mpv_create();
    if (!mpv)
        die("context init failed");

    int res;
    int terminal = 1;
    res = mpv_set_option(mpv, "terminal", MPV_FORMAT_FLAG, &terminal);
    ALOGE("mpv_set_option terminal ret %d", res);
    res = mpv_set_option_string(mpv, "msg-level", "all=v");
    ALOGE("mpv_set_option_string msg-level ret %d", res);

    if (mpv_initialize(mpv) < 0)
        die("mpv init failed");
    mpv_gl = (mpv_opengl_cb_context*)mpv_get_sub_api(mpv, MPV_SUB_API_OPENGL_CB);
    if (!mpv_gl)
        die("failed to create mpv GL API handle");

    if (mpv_opengl_cb_init_gl(mpv_gl, NULL, get_proc_address_mpv, NULL) < 0)
        die("failed to initialize mpv GL context");

    if (mpv_set_option_string(mpv, "vo", "opengl-cb") < 0)
        die("failed to set VO");
    if (mpv_set_option_string(mpv, "ao", "null") < 0)
        die("failed to set AO");

    const char *cmd[] = {"loadfile", "/sdcard1/1.mp4", NULL};
    res = mpv_command(mpv, cmd);
    ALOGE("mpv_command returns %d", res);
}

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_resize(JNIEnv* env, jobject obj, jint width, jint height) {
}

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_step(JNIEnv* env, jobject obj) {
    mpv_opengl_cb_draw(mpv_gl, 0, 1000, -1000);
}
<commit_msg>ao=openal<commit_after>#include <jni.h>
#include <stdlib.h>
#include <time.h>
#include <locale.h>

#include <mpv/client.h>
#include <mpv/opengl_cb.h>

#include <EGL/egl.h>

#include "main.h"

extern "C" {
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_init(JNIEnv* env, jobject obj);
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_resize(JNIEnv* env, jobject obj, jint width, jint height);
    JNIEXPORT void JNICALL Java_is_xyz_mpv_MPVLib_step(JNIEnv* env, jobject obj);
};

static void die(const char *msg)
{
    ALOGE("%s", msg);
    exit(1);
}

static void *get_proc_address_mpv(void *fn_ctx, const char *name)
{
    return (void*)eglGetProcAddress(name);
}

mpv_handle *mpv;
mpv_opengl_cb_context *mpv_gl;

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_init(JNIEnv* env, jobject obj) {
    ALOGE("init");

    setlocale(LC_NUMERIC, "C");

    mpv = mpv_create();
    if (!mpv)
        die("context init failed");

    int res;
    int terminal = 1;
    res = mpv_set_option(mpv, "terminal", MPV_FORMAT_FLAG, &terminal);
    ALOGE("mpv_set_option terminal ret %d", res);
    res = mpv_set_option_string(mpv, "msg-level", "all=v");
    ALOGE("mpv_set_option_string msg-level ret %d", res);

    if (mpv_initialize(mpv) < 0)
        die("mpv init failed");
    mpv_gl = (mpv_opengl_cb_context*)mpv_get_sub_api(mpv, MPV_SUB_API_OPENGL_CB);
    if (!mpv_gl)
        die("failed to create mpv GL API handle");

    if (mpv_opengl_cb_init_gl(mpv_gl, NULL, get_proc_address_mpv, NULL) < 0)
        die("failed to initialize mpv GL context");

    if (mpv_set_option_string(mpv, "vo", "opengl-cb") < 0)
        die("failed to set VO");
    if (mpv_set_option_string(mpv, "ao", "openal") < 0)
        die("failed to set AO");

    const char *cmd[] = {"loadfile", "/sdcard1/1.mp4", NULL};
    res = mpv_command(mpv, cmd);
    ALOGE("mpv_command returns %d", res);
}

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_resize(JNIEnv* env, jobject obj, jint width, jint height) {
}

JNIEXPORT void JNICALL
Java_is_xyz_mpv_MPVLib_step(JNIEnv* env, jobject obj) {
    mpv_opengl_cb_draw(mpv_gl, 0, 1000, -1000);
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: ary.hxx,v $
 *
 *  $Revision: 1.5 $
 *
 *  last change: $Author: hr $ $Date: 2007-11-02 14:36:51 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef ARY_ARY_HXX
#define ARY_ARY_HXX

// USED SERVICES
    // BASE CLASSES
    // OTHER

namespace ary
{
namespace idl
{
    class Gate;
}

namespace cpp
{
    class Gate;
}
}

namespace ary
{

/** Starting point for all work with the
    Autodoc Sourcecode Repository.

    Create and destroy the repository and
    give access to the "Gates" for different tasks.

    @collab ::ary::cpp::Gate
    @collab ::ary::idl::Gate
*/

class Repository
{
  public:
    //  LIFECYCLE
    virtual             ~Repository() {}
    static DYN Repository &
                        Create_();
    // INQUIRY
    virtual const String &
                        Title() const = 0;
    virtual const ::ary::cpp::Gate &
                        Gate_Cpp() const = 0;
    virtual const ::ary::idl::Gate &
                        Gate_Idl() const = 0;
    // ACCESS
    virtual ::ary::cpp::Gate &
                        Gate_Cpp() = 0;
    virtual ::ary::idl::Gate &
                        Gate_Idl() = 0;
    virtual void        Set_Title(
                            const String &      i_sName ) = 0;
};

} // namespace ary
#endif
<commit_msg>INTEGRATION: CWS changefileheader (1.5.22); FILE MERGED 2008/03/28 16:00:56 rt 1.5.22.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: ary.hxx,v $
 * $Revision: 1.6 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef ARY_ARY_HXX
#define ARY_ARY_HXX

// USED SERVICES
    // BASE CLASSES
    // OTHER

namespace ary
{
namespace idl
{
    class Gate;
}

namespace cpp
{
    class Gate;
}
}

namespace ary
{

/** Starting point for all work with the
    Autodoc Sourcecode Repository.

    Create and destroy the repository and
    give access to the "Gates" for different tasks.

    @collab ::ary::cpp::Gate
    @collab ::ary::idl::Gate
*/

class Repository
{
  public:
    //  LIFECYCLE
    virtual             ~Repository() {}
    static DYN Repository &
                        Create_();
    // INQUIRY
    virtual const String &
                        Title() const = 0;
    virtual const ::ary::cpp::Gate &
                        Gate_Cpp() const = 0;
    virtual const ::ary::idl::Gate &
                        Gate_Idl() const = 0;
    // ACCESS
    virtual ::ary::cpp::Gate &
                        Gate_Cpp() = 0;
    virtual ::ary::idl::Gate &
                        Gate_Idl() = 0;
    virtual void        Set_Title(
                            const String &      i_sName ) = 0;
};

} // namespace ary
#endif
<|endoftext|>


<commit_before>#include "StdAfx.h"
#include "Rw2Decoder.h"

/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

    http://www.klauspost.com
*/
namespace RawSpeed {

Rw2Decoder::Rw2Decoder(TiffIFD *rootIFD, FileMap* file) :
    RawDecoder(file), mRootIFD(rootIFD), input_start(0) {
      decoderVersion = 1;
}
Rw2Decoder::~Rw2Decoder(void) {
  if (input_start)
    delete input_start;
  input_start = 0;
  if (mRootIFD)
    delete mRootIFD;
  mRootIFD = NULL;
}

RawImage Rw2Decoder::decodeRawInternal() {

  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(PANASONIC_STRIPOFFSET);

  bool isOldPanasonic = FALSE;

  if (data.empty()) {
    if (!mRootIFD->hasEntryRecursive(STRIPOFFSETS))
      ThrowRDE("RW2 Decoder: No image data found");
    isOldPanasonic = TRUE;
    data = mRootIFD->getIFDsWithTag(STRIPOFFSETS);
  }

  TiffIFD* raw = data[0];
  uint32 height = raw->getEntry((TiffTag)3)->getShort();
  uint32 width = raw->getEntry((TiffTag)2)->getShort();

  if (isOldPanasonic) {
    ThrowRDE("Cannot decode old-style Panasonic RAW files");
    TiffEntry *offsets = raw->getEntry(STRIPOFFSETS);
    TiffEntry *counts = raw->getEntry(STRIPBYTECOUNTS);

    if (offsets->count != 1) {
      ThrowRDE("RW2 Decoder: Multiple Strips found: %u", offsets->count);
    }
    int off = offsets->getInt();
    if (!mFile->isValid(off))
      ThrowRDE("Panasonic RAW Decoder: Invalid image data offset, cannot decode.");

    int count = counts->getInt();
    if (count != (int)(width*height*2))
      ThrowRDE("Panasonic RAW Decoder: Byte count is wrong.");

    if (!mFile->isValid(off+count))
      ThrowRDE("Panasonic RAW Decoder: Invalid image data offset, cannot decode.");

    mRaw->dim = iPoint2D(width, height);
    mRaw->createData();
    ByteStream input_start(mFile->getData(off), mFile->getSize() - off);
    iPoint2D pos(0, 0);
    readUncompressedRaw(input_start, mRaw->dim,pos, width*2, 16, BitOrder_Plain);

  } else {

    mRaw->dim = iPoint2D(width, height);
    mRaw->createData();
    TiffEntry *offsets = raw->getEntry(PANASONIC_STRIPOFFSET);

    if (offsets->count != 1) {
      ThrowRDE("RW2 Decoder: Multiple Strips found: %u", offsets->count);
    }

    load_flags = 0x2008;
    int off = offsets->getInt();

    if (!mFile->isValid(off))
      ThrowRDE("RW2 Decoder: Invalid image data offset, cannot decode.");

    input_start = new ByteStream(mFile->getData(off), mFile->getSize() - off);
    DecodeRw2();
  }
  return mRaw;
}

void Rw2Decoder::DecodeRw2() {
  startThreads();
}

void Rw2Decoder::decodeThreaded(RawDecoderThread * t) {
  int x, i, j, sh = 0, pred[2], nonz[2];
  int w = mRaw->dim.x / 14;
  uint32 y;

  bool zero_is_bad = false;
  map<string,string>::iterator zero_hint = hints.find("zero_is_bad");
  if (zero_hint != hints.end())
    zero_is_bad = true;

  /* 9 + 1/7 bits per pixel */
  int skip = w * 14 * t->start_y * 9;
  skip += w * 2 * t->start_y;
  skip /= 8;

  PanaBitpump bits(new ByteStream(input_start));
  bits.load_flags = load_flags;
  bits.skipBytes(skip);

  vector<uint32> zero_pos;
  for (y = t->start_y; y < t->end_y; y++) {
    ushort16* dest = (ushort16*)mRaw->getData(0, y);
    for (x = 0; x < w; x++) {
      pred[0] = pred[1] = nonz[0] = nonz[1] = 0;
      int u = 0;
      for (i = 0; i < 14; i++) {
        // Even pixels
        if (u == 2)
        {
          sh = 4 >> (3 - bits.getBits(2));
          u = -1;
        }
        if (nonz[0]) {
          if ((j = bits.getBits(8))) {
            if ((pred[0] -= 0x80 << sh) < 0 || sh == 4)
              pred[0] &= ~(-1 << sh);
            pred[0] += j << sh;
          }
        } else if ((nonz[0] = bits.getBits(8)) || i > 11)
          pred[0] = nonz[0] << 4 | bits.getBits(4);
        *dest++ = pred[0];
        if (zero_is_bad && 0 == pred[0])
          zero_pos.push_back((y<<16) | (x*14+i));

        // Odd pixels
        i++;
        u++;
        if (u == 2)
        {
          sh = 4 >> (3 - bits.getBits(2));
          u = -1;
        }
        if (nonz[1]) {
          if ((j = bits.getBits(8))) {
            if ((pred[1] -= 0x80 << sh) < 0 || sh == 4)
              pred[1] &= ~(-1 << sh);
            pred[1] += j << sh;
          }
        } else if ((nonz[1] = bits.getBits(8)) || i > 11)
          pred[1] = nonz[1] << 4 | bits.getBits(4);
        *dest++ = pred[1];
        if (zero_is_bad && 0 == pred[1])
          zero_pos.push_back((y<<16) | (x*14+i));
        u++;
      }
    }
  }
  if (zero_is_bad && !zero_pos.empty()) {
    pthread_mutex_lock(&mRaw->mBadPixelMutex);
    mRaw->mBadPixelPositions.insert(mRaw->mBadPixelPositions.end(), zero_pos.begin(), zero_pos.end());
    pthread_mutex_unlock(&mRaw->mBadPixelMutex);
  }
}

void Rw2Decoder::checkSupportInternal(CameraMetaData *meta) {
  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(MODEL);
  if (data.empty())
    ThrowRDE("RW2 Support check: Model name found");

  string make = data[0]->getEntry(MAKE)->getString();
  string model = data[0]->getEntry(MODEL)->getString();
  if (!this->checkCameraSupported(meta, make, model, guessMode()))
    this->checkCameraSupported(meta, make, model, "");
}

void Rw2Decoder::decodeMetaDataInternal(CameraMetaData *meta) {
  mRaw->cfa.setCFA(CFA_BLUE, CFA_GREEN, CFA_GREEN2, CFA_RED);
  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(MODEL);

  if (data.empty())
    ThrowRDE("RW2 Meta Decoder: Model name not found");
  if (!data[0]->hasEntry(MAKE))
    ThrowRDE("RW2 Support: Make name not found");

  string make = data[0]->getEntry(MAKE)->getString();
  string model = data[0]->getEntry(MODEL)->getString();
  string mode = guessMode();
  int iso = 0;
  if (mRootIFD->hasEntryRecursive(PANASONIC_ISO_SPEED))
    iso = mRootIFD->getEntryRecursive(PANASONIC_ISO_SPEED)->getInt();

  if (this->checkCameraSupported(meta, make, model, mode)) {
    setMetaData(meta, make, model, mode, iso);
  } else {
    mRaw->mode = mode;
    _RPT1(0, "Mode not found in DB: %s", mode.c_str());
    setMetaData(meta, make, model, "", iso);
  }
}

std::string Rw2Decoder::guessMode() {
  float ratio = 3.0f / 2.0f;  // Default

  if (!mRaw->isAllocated())
    return "";

  ratio = (float)mRaw->dim.x / (float)mRaw->dim.y;

  float min_diff = fabs(ratio - 16.0f / 9.0f);
  std::string closest_match = "16:9";

  float t = fabs(ratio - 3.0f / 2.0f);
  if (t < min_diff) {
    closest_match = "3:2";
    min_diff  = t;
  }

  t = fabs(ratio - 4.0f / 3.0f);
  if (t < min_diff) {
    closest_match =  "4:3";
    min_diff  = t;
  }

  t = fabs(ratio - 1.0f);
  if (t < min_diff) {
    closest_match = "1:1";
    min_diff  = t;
  }
  _RPT1(0, "Mode guess: '%s'\n", closest_match.c_str());
  return closest_match;
}

PanaBitpump::PanaBitpump(ByteStream* _input) : input(_input), vbits(0) {
}

PanaBitpump::~PanaBitpump() {
  if (input)
    delete input;
  input = 0;
}

void PanaBitpump::skipBytes(int bytes) {
  int blocks = (bytes / 0x4000) * 0x4000;
  input->skipBytes(blocks);
  for (int i = blocks; i < bytes; i++)
    getBits(8);
}

uint32 PanaBitpump::getBits(int nbits) {
  int byte;

  if (!vbits) {
    /* On truncated files this routine will just return just for the truncated
    * part of the file. Since there is no chance of affecting output buffer
    * size we allow the decoder to decode this
    */
    if (input->getRemainSize() < 0x4000 - load_flags) {
      memcpy(buf + load_flags, input->getData(), input->getRemainSize());
      input->skipBytes(input->getRemainSize());
    } else {
      memcpy(buf + load_flags, input->getData(), 0x4000 - load_flags);
      input->skipBytes(0x4000 - load_flags);
      if (input->getRemainSize() < load_flags) {
        memcpy(buf, input->getData(), input->getRemainSize());
        input->skipBytes(input->getRemainSize());
      } else {
        memcpy(buf, input->getData(), load_flags);
        input->skipBytes(load_flags);
      }
    }
  }
  vbits = (vbits - nbits) & 0x1ffff;
  byte = vbits >> 3 ^ 0x3ff0;
  return (buf[byte] | buf[byte+1] << 8) >> (vbits & 7) & ~(-1 << nbits);
}

} // namespace RawSpeed
<commit_msg>Read blacklevels from Panasonic metadata.<commit_after>#include "StdAfx.h"
#include "Rw2Decoder.h"

/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

    http://www.klauspost.com
*/
namespace RawSpeed {

Rw2Decoder::Rw2Decoder(TiffIFD *rootIFD, FileMap* file) :
    RawDecoder(file), mRootIFD(rootIFD), input_start(0) {
      decoderVersion = 1;
}
Rw2Decoder::~Rw2Decoder(void) {
  if (input_start)
    delete input_start;
  input_start = 0;
  if (mRootIFD)
    delete mRootIFD;
  mRootIFD = NULL;
}

RawImage Rw2Decoder::decodeRawInternal() {

  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(PANASONIC_STRIPOFFSET);

  bool isOldPanasonic = FALSE;

  if (data.empty()) {
    if (!mRootIFD->hasEntryRecursive(STRIPOFFSETS))
      ThrowRDE("RW2 Decoder: No image data found");
    isOldPanasonic = TRUE;
    data = mRootIFD->getIFDsWithTag(STRIPOFFSETS);
  }

  TiffIFD* raw = data[0];
  uint32 height = raw->getEntry((TiffTag)3)->getShort();
  uint32 width = raw->getEntry((TiffTag)2)->getShort();

  if (isOldPanasonic) {
    ThrowRDE("Cannot decode old-style Panasonic RAW files");
    TiffEntry *offsets = raw->getEntry(STRIPOFFSETS);
    TiffEntry *counts = raw->getEntry(STRIPBYTECOUNTS);

    if (offsets->count != 1) {
      ThrowRDE("RW2 Decoder: Multiple Strips found: %u", offsets->count);
    }
    int off = offsets->getInt();
    if (!mFile->isValid(off))
      ThrowRDE("Panasonic RAW Decoder: Invalid image data offset, cannot decode.");

    int count = counts->getInt();
    if (count != (int)(width*height*2))
      ThrowRDE("Panasonic RAW Decoder: Byte count is wrong.");

    if (!mFile->isValid(off+count))
      ThrowRDE("Panasonic RAW Decoder: Invalid image data offset, cannot decode.");

    mRaw->dim = iPoint2D(width, height);
    mRaw->createData();
    ByteStream input_start(mFile->getData(off), mFile->getSize() - off);
    iPoint2D pos(0, 0);
    readUncompressedRaw(input_start, mRaw->dim,pos, width*2, 16, BitOrder_Plain);

  } else {

    mRaw->dim = iPoint2D(width, height);
    mRaw->createData();
    TiffEntry *offsets = raw->getEntry(PANASONIC_STRIPOFFSET);

    if (offsets->count != 1) {
      ThrowRDE("RW2 Decoder: Multiple Strips found: %u", offsets->count);
    }

    load_flags = 0x2008;
    int off = offsets->getInt();

    if (!mFile->isValid(off))
      ThrowRDE("RW2 Decoder: Invalid image data offset, cannot decode.");

    input_start = new ByteStream(mFile->getData(off), mFile->getSize() - off);
    DecodeRw2();
  }
  // Read blacklevels
  if (raw->hasEntry((TiffTag)0x1c) && raw->hasEntry((TiffTag)0x1d) && raw->hasEntry((TiffTag)0x1e)) {
    mRaw->blackLevelSeparate[0] = raw->getEntry((TiffTag)0x1c)->getInt() + 15;
    mRaw->blackLevelSeparate[1] = mRaw->blackLevelSeparate[2] = raw->getEntry((TiffTag)0x1d)->getInt() + 15;
    mRaw->blackLevelSeparate[3] = raw->getEntry((TiffTag)0x1e)->getInt() + 15;
  }
  return mRaw;
}

void Rw2Decoder::DecodeRw2() {
  startThreads();
}

void Rw2Decoder::decodeThreaded(RawDecoderThread * t) {
  int x, i, j, sh = 0, pred[2], nonz[2];
  int w = mRaw->dim.x / 14;
  uint32 y;

  bool zero_is_bad = false;
  map<string,string>::iterator zero_hint = hints.find("zero_is_bad");
  if (zero_hint != hints.end())
    zero_is_bad = true;

  /* 9 + 1/7 bits per pixel */
  int skip = w * 14 * t->start_y * 9;
  skip += w * 2 * t->start_y;
  skip /= 8;

  PanaBitpump bits(new ByteStream(input_start));
  bits.load_flags = load_flags;
  bits.skipBytes(skip);

  vector<uint32> zero_pos;
  for (y = t->start_y; y < t->end_y; y++) {
    ushort16* dest = (ushort16*)mRaw->getData(0, y);
    for (x = 0; x < w; x++) {
      pred[0] = pred[1] = nonz[0] = nonz[1] = 0;
      int u = 0;
      for (i = 0; i < 14; i++) {
        // Even pixels
        if (u == 2)
        {
          sh = 4 >> (3 - bits.getBits(2));
          u = -1;
        }
        if (nonz[0]) {
          if ((j = bits.getBits(8))) {
            if ((pred[0] -= 0x80 << sh) < 0 || sh == 4)
              pred[0] &= ~(-1 << sh);
            pred[0] += j << sh;
          }
        } else if ((nonz[0] = bits.getBits(8)) || i > 11)
          pred[0] = nonz[0] << 4 | bits.getBits(4);
        *dest++ = pred[0];
        if (zero_is_bad && 0 == pred[0])
          zero_pos.push_back((y<<16) | (x*14+i));

        // Odd pixels
        i++;
        u++;
        if (u == 2)
        {
          sh = 4 >> (3 - bits.getBits(2));
          u = -1;
        }
        if (nonz[1]) {
          if ((j = bits.getBits(8))) {
            if ((pred[1] -= 0x80 << sh) < 0 || sh == 4)
              pred[1] &= ~(-1 << sh);
            pred[1] += j << sh;
          }
        } else if ((nonz[1] = bits.getBits(8)) || i > 11)
          pred[1] = nonz[1] << 4 | bits.getBits(4);
        *dest++ = pred[1];
        if (zero_is_bad && 0 == pred[1])
          zero_pos.push_back((y<<16) | (x*14+i));
        u++;
      }
    }
  }
  if (zero_is_bad && !zero_pos.empty()) {
    pthread_mutex_lock(&mRaw->mBadPixelMutex);
    mRaw->mBadPixelPositions.insert(mRaw->mBadPixelPositions.end(), zero_pos.begin(), zero_pos.end());
    pthread_mutex_unlock(&mRaw->mBadPixelMutex);
  }
}

void Rw2Decoder::checkSupportInternal(CameraMetaData *meta) {
  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(MODEL);
  if (data.empty())
    ThrowRDE("RW2 Support check: Model name found");

  string make = data[0]->getEntry(MAKE)->getString();
  string model = data[0]->getEntry(MODEL)->getString();
  if (!this->checkCameraSupported(meta, make, model, guessMode()))
    this->checkCameraSupported(meta, make, model, "");
}

void Rw2Decoder::decodeMetaDataInternal(CameraMetaData *meta) {
  mRaw->cfa.setCFA(CFA_BLUE, CFA_GREEN, CFA_GREEN2, CFA_RED);
  vector<TiffIFD*> data = mRootIFD->getIFDsWithTag(MODEL);

  if (data.empty())
    ThrowRDE("RW2 Meta Decoder: Model name not found");
  if (!data[0]->hasEntry(MAKE))
    ThrowRDE("RW2 Support: Make name not found");

  string make = data[0]->getEntry(MAKE)->getString();
  string model = data[0]->getEntry(MODEL)->getString();
  string mode = guessMode();
  int iso = 0;
  if (mRootIFD->hasEntryRecursive(PANASONIC_ISO_SPEED))
    iso = mRootIFD->getEntryRecursive(PANASONIC_ISO_SPEED)->getInt();

  if (this->checkCameraSupported(meta, make, model, mode)) {
    setMetaData(meta, make, model, mode, iso);
  } else {
    mRaw->mode = mode;
    _RPT1(0, "Mode not found in DB: %s", mode.c_str());
    setMetaData(meta, make, model, "", iso);
  }
}

std::string Rw2Decoder::guessMode() {
  float ratio = 3.0f / 2.0f;  // Default

  if (!mRaw->isAllocated())
    return "";

  ratio = (float)mRaw->dim.x / (float)mRaw->dim.y;

  float min_diff = fabs(ratio - 16.0f / 9.0f);
  std::string closest_match = "16:9";

  float t = fabs(ratio - 3.0f / 2.0f);
  if (t < min_diff) {
    closest_match = "3:2";
    min_diff  = t;
  }

  t = fabs(ratio - 4.0f / 3.0f);
  if (t < min_diff) {
    closest_match =  "4:3";
    min_diff  = t;
  }

  t = fabs(ratio - 1.0f);
  if (t < min_diff) {
    closest_match = "1:1";
    min_diff  = t;
  }
  _RPT1(0, "Mode guess: '%s'\n", closest_match.c_str());
  return closest_match;
}

PanaBitpump::PanaBitpump(ByteStream* _input) : input(_input), vbits(0) {
}

PanaBitpump::~PanaBitpump() {
  if (input)
    delete input;
  input = 0;
}

void PanaBitpump::skipBytes(int bytes) {
  int blocks = (bytes / 0x4000) * 0x4000;
  input->skipBytes(blocks);
  for (int i = blocks; i < bytes; i++)
    getBits(8);
}

uint32 PanaBitpump::getBits(int nbits) {
  int byte;

  if (!vbits) {
    /* On truncated files this routine will just return just for the truncated
    * part of the file. Since there is no chance of affecting output buffer
    * size we allow the decoder to decode this
    */
    if (input->getRemainSize() < 0x4000 - load_flags) {
      memcpy(buf + load_flags, input->getData(), input->getRemainSize());
      input->skipBytes(input->getRemainSize());
    } else {
      memcpy(buf + load_flags, input->getData(), 0x4000 - load_flags);
      input->skipBytes(0x4000 - load_flags);
      if (input->getRemainSize() < load_flags) {
        memcpy(buf, input->getData(), input->getRemainSize());
        input->skipBytes(input->getRemainSize());
      } else {
        memcpy(buf, input->getData(), load_flags);
        input->skipBytes(load_flags);
      }
    }
  }
  vbits = (vbits - nbits) & 0x1ffff;
  byte = vbits >> 3 ^ 0x3ff0;
  return (buf[byte] | buf[byte+1] << 8) >> (vbits & 7) & ~(-1 << nbits);
}

} // namespace RawSpeed
<|endoftext|>


<commit_before>/*
Copyright (c) 2012-2015 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include "split.h"

void welcome(){
	printf("ssdb-split - Split ssdb-server\n");
	printf("Copyright (c) 2012-2015 ssdb.io\n");
	printf("\n");
}

void usage(int argc, char **argv){
	printf("Usage:\n");
	printf("    %sr\n", argv[0]);
	printf("\n");
	printf("Options:\n");
	printf("    xx - \n");
}

int main(int argc, char **argv){
	welcome();

	int ret;
	Split split;
	ret = split.init("127.0.0.1", 9000, "127.0.0.1", 8888, "127.0.0.1", 8889);
	if(ret == -1){
		fprintf(stderr, "%d error!\n", __LINE__);
		exit(1);
	}

	int64_t src_dbsize;
	ssdb::Status s;
	s = split.src_client->dbsize(&src_dbsize);
	if(!s.ok()){
		fprintf(stderr, "%d error!\n", __LINE__);
		exit(1);
	}

	printf("src dbsize: %lld\n", src_dbsize);

	int64_t total_moved = 0;
	std::string min_key;
	std::string max_key;
	while(1){
		int64_t size = split.move_some();
		if(size == -1){
			fprintf(stderr, "error!\n");
			exit(1);
		}
		if(size == 0){
			fprintf(stderr, "no data to move, end.\n");
			split.finish();
			break;
		}
		total_moved += size;

		int64_t src_dbsize_new;
		ssdb::Status s;
		s = split.src_client->dbsize(&src_dbsize_new);
		if(!s.ok()){
			fprintf(stderr, "error!\n");
			exit(1);
		}

		printf("moved: %lld, src_dbsize_old: %lld, src_dbsize_new: %lld\n", total_moved/1024, src_dbsize/1024, src_dbsize_new/1024);

		if(total_moved > src_dbsize/2 || (src_dbsize_new - src_dbsize) > src_dbsize/10){
			fprintf(stderr, "split end.\n");
			split.finish();
			break;
		}

		getchar();
	}

	return 0;
}

<commit_msg>update<commit_after>/*
Copyright (c) 2012-2015 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include "split.h"

void welcome(){
	printf("ssdb-split - Split ssdb-server\n");
	printf("Copyright (c) 2012-2015 ssdb.io\n");
	printf("\n");
}

void usage(int argc, char **argv){
	printf("Usage:\n");
	printf("    %sr\n", argv[0]);
	printf("\n");
	printf("Options:\n");
	printf("    xx - \n");
}

int main(int argc, char **argv){
	welcome();

	int ret;
	Split split;
	ret = split.init("127.0.0.1", 9000, "127.0.0.1", 8888, "127.0.0.1", 8889);
	if(ret == -1){
		fprintf(stderr, "%d error!\n", __LINE__);
		exit(1);
	}

	ssdb::Status s;
	int64_t src_dbsize;
	s = split.src_client->dbsize(&src_dbsize);
	if(!s.ok()){
		fprintf(stderr, "%d error!\n", __LINE__);
		exit(1);
	}

	printf("src dbsize: %lld\n", src_dbsize);

	int64_t total_moved = 0;
	std::string min_key;
	std::string max_key;
	while(1){
		int64_t size = split.move_some();
		if(size == -1){
			fprintf(stderr, "error!\n");
			exit(1);
		}
		if(size == 0){
			fprintf(stderr, "no data to move, end.\n");
			split.finish();
			break;
		}
		total_moved += size;

		int64_t src_dbsize_new;
		s = split.src_client->dbsize(&src_dbsize_new);
		if(!s.ok()){
			fprintf(stderr, "error!\n");
			exit(1);
		}

		int64_t dst_dbsize_new;
		s = split.dst_client->dbsize(&dst_dbsize_new);
		if(!s.ok()){
			fprintf(stderr, "error!\n");
			exit(1);
		}

		printf("moved: %lld, src_dbsize_old: %lld, src_dbsize_new: %lld, dst_dbsize_new: %lld\n",
			total_moved/1024, src_dbsize/1024, src_dbsize_new/1024, dst_dbsize_new/1024);

		if(total_moved > src_dbsize/2 || dst_dbsize_new > src_dbsize/2 || (src_dbsize_new - src_dbsize) > src_dbsize/10){
			fprintf(stderr, "split end.\n");
			split.finish();
			break;
		}

		getchar();
	}

	return 0;
}

<|endoftext|>


<commit_before>// ======================================================================== //
// Copyright 2009-2016 Intel Corporation                                    //
//                                                                          //
// Licensed under the Apache License, Version 2.0 (the "License");          //
// you may not use this file except in compliance with the License.         //
// You may obtain a copy of the License at                                  //
//                                                                          //
//     http://www.apache.org/licenses/LICENSE-2.0                           //
//                                                                          //
// Unless required by applicable law or agreed to in writing, software      //
// distributed under the License is distributed on an "AS IS" BASIS,        //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and      //
// limitations under the License.                                           //
// ======================================================================== //

#include "taskschedulerinternal.h"
#include "../math/math.h"
#include "../sys/sysinfo.h"
#include <algorithm>

namespace embree
{
  size_t TaskScheduler::g_numThreads = 0;
  __thread TaskScheduler* TaskScheduler::g_instance = nullptr;
  __thread TaskScheduler::Thread* TaskScheduler::thread_local_thread = nullptr;
  TaskScheduler::ThreadPool* TaskScheduler::threadPool = nullptr;

  template<typename Predicate, typename Body>
  __forceinline void TaskScheduler::steal_loop(Thread& thread, const Predicate& pred, const Body& body)
  {
    while (true)
    {
      /*! some rounds that yield */
      for (size_t i=0; i<32; i++)
      {
        /*! some spinning rounds */
        const size_t threadCount = thread.threadCount();
        for (size_t j=0; j<1024; j+=threadCount)
        {
          if (!pred()) return;
          if (thread.scheduler->steal_from_other_threads(thread)) {
            i=j=0;
            body();
          }
        }
        yield();
      }
    }
  }

  /*! run this task */
  __dllexport void TaskScheduler::Task::run (Thread& thread) // FIXME: avoid as many __dllexports as possible
  {
    /* try to run if not already stolen */
    if (try_switch_state(INITIALIZED,DONE))
    {
      Task* prevTask = thread.task;
      thread.task = this;
      try {
        if (thread.scheduler->cancellingException == nullptr)
          closure->execute();
      } catch (...) {
        if (thread.scheduler->cancellingException == nullptr)
          thread.scheduler->cancellingException = std::current_exception();
      }
      thread.task = prevTask;
      add_dependencies(-1);
    }

    /* steal until all dependencies have completed */
    steal_loop(thread,
               [&] () { return dependencies>0; },
               [&] () { while (thread.tasks.execute_local(thread,this)); });

    /* now signal our parent task that we are finished */
    if (parent)
      parent->add_dependencies(-1);
  }

  __dllexport bool TaskScheduler::TaskQueue::execute_local(Thread& thread, Task* parent)
  {
    /* stop if we run out of local tasks or reach the waiting task */
    if (right == 0 || &tasks[right-1] == parent)
      return false;

    /* execute task */
    size_t oldRight = right;
    tasks[right-1].run(thread);
    if (right != oldRight) {
      THROW_RUNTIME_ERROR("you have to wait for spawned subtasks");
    }

    /* pop task and closure from stack */
    right--;
    if (tasks[right].stackPtr != size_t(-1))
      stackPtr = tasks[right].stackPtr;

    /* also move left pointer */
    if (left >= right) left.store(right.load());

    return right != 0;
  }

  bool TaskScheduler::TaskQueue::steal(Thread& thread)
  {
    size_t l = left;
    if (l < right)
      l = left++;
    else
      return false;

    if (!tasks[l].try_steal(thread.tasks.tasks[thread.tasks.right]))
      return false;

    thread.tasks.right++;
    return true;
  }

  /* we steal from the left */
  size_t TaskScheduler::TaskQueue::getTaskSizeAtLeft()
  {
    if (left >= right) return 0;
    return tasks[left].N;
  }

  static MutexSys g_mutex;
  static BarrierSys g_barrier(2);

  void threadPoolFunction(std::pair<TaskScheduler::ThreadPool*,size_t>* pair)
  {
    TaskScheduler::ThreadPool* pool = pair->first;
    size_t threadIndex = pair->second;
    g_barrier.wait();
    pool->thread_loop(threadIndex);
  }

  TaskScheduler::ThreadPool::ThreadPool(bool set_affinity)
    : numThreads(0), numThreadsRunning(0), set_affinity(set_affinity), running(false) {}

  __dllexport void TaskScheduler::ThreadPool::startThreads()
  {
    if (running) return;
    setNumThreads(numThreads,true);
  }

  void TaskScheduler::ThreadPool::setNumThreads(size_t newNumThreads, bool startThreads)
  {
    Lock<MutexSys> lock(g_mutex);

    if (newNumThreads == 0)
      newNumThreads = getNumberOfLogicalThreads();

    numThreads = newNumThreads;
    if (!startThreads && !running) return;
    running = true;
    size_t numThreadsActive = numThreadsRunning;

    mutex.lock();
    numThreadsRunning = newNumThreads;
    mutex.unlock();
    condition.notify_all();

    /* start new threads */
    for (size_t t=numThreadsActive; t<numThreads; t++)
    {
      if (t == 0) continue;
      auto pair = std::make_pair(this,t);
      threads.push_back(createThread((thread_func)threadPoolFunction,&pair,4*1024*1024,set_affinity ? t : -1));
      g_barrier.wait();
    }

    /* stop some threads if we reduce the number of threads */
    for (ssize_t t=numThreadsActive-1; t>=ssize_t(numThreadsRunning); t--) {
      if (t == 0) continue;
      embree::join(threads.back());
      threads.pop_back();
    }
  }

  TaskScheduler::ThreadPool::~ThreadPool()
  {
    /* leave all taskschedulers */
    mutex.lock();
    numThreadsRunning = 0;
    mutex.unlock();
    condition.notify_all();

    /* wait for threads to terminate */
    for (size_t i=0; i<threads.size(); i++)
      embree::join(threads[i]);
  }

  __dllexport void TaskScheduler::ThreadPool::add(const Ref<TaskScheduler>& scheduler)
  {
    mutex.lock();
    schedulers.push_back(scheduler);
    mutex.unlock();
    condition.notify_all();
  }

  __dllexport void TaskScheduler::ThreadPool::remove(const Ref<TaskScheduler>& scheduler)
  {
    Lock<MutexSys> lock(mutex);
    for (std::list<Ref<TaskScheduler> >::iterator it = schedulers.begin(); it != schedulers.end(); it++) {
      if (scheduler == *it) {
        schedulers.erase(it);
        return;
      }
    }
  }

  void TaskScheduler::ThreadPool::thread_loop(size_t globalThreadIndex)
  {
    while (globalThreadIndex < numThreadsRunning)
    {
      Ref<TaskScheduler> scheduler = NULL;
      ssize_t threadIndex = -1;
      {
        Lock<MutexSys> lock(mutex);
        condition.wait(mutex, [&] () { return globalThreadIndex >= numThreadsRunning || !schedulers.empty(); });
        if (globalThreadIndex >= numThreadsRunning) break;
        scheduler = schedulers.front();
        threadIndex = scheduler->allocThreadIndex();
      }
      scheduler->thread_loop(threadIndex);
    }
  }

  TaskScheduler::TaskScheduler()
    : threadCounter(0), anyTasksRunning(0), hasRootTask(false)
  {
    threadLocal.resize(2*getNumberOfLogicalThreads()); // FIXME: this has to be 2x as in the join mode the worker threads also join
    for (size_t i=0; i<threadLocal.size(); i++)
      threadLocal[i].store(nullptr);
  }

  TaskScheduler::~TaskScheduler()
  {
    assert(threadCounter == 0);
  }

  __dllexport size_t TaskScheduler::threadIndex()
  {
    Thread* thread = TaskScheduler::thread();
    if (thread) return thread->threadIndex;
    else        return 0;
  }

  __dllexport size_t TaskScheduler::threadCount() {
    return threadPool->size();
  }

  __dllexport TaskScheduler* TaskScheduler::instance()
  {
    if (g_instance == NULL) {
      g_instance = new TaskScheduler;
      g_instance->refInc();
    }
    return g_instance;
  }

  void TaskScheduler::create(size_t numThreads, bool set_affinity)
  {
    if (!threadPool) threadPool = new TaskScheduler::ThreadPool(set_affinity);
    threadPool->setNumThreads(numThreads,false);
  }

  void TaskScheduler::destroy() {
    delete threadPool; threadPool = nullptr;
  }

  __dllexport ssize_t TaskScheduler::allocThreadIndex()
  {
    size_t threadIndex = threadCounter++;
    assert(threadIndex < threadLocal.size());
    return threadIndex;
  }

  void TaskScheduler::join()
  {
    mutex.lock();
    size_t threadIndex = allocThreadIndex();
    condition.wait(mutex, [&] () { return hasRootTask.load(); });
    mutex.unlock();
    std::exception_ptr except = thread_loop(threadIndex);
    if (except != nullptr) std::rethrow_exception(except);
  }

  void TaskScheduler::reset() {
    hasRootTask = false;
  }

  void TaskScheduler::wait_for_threads(size_t threadCount)
  {
    while (threadCounter < threadCount-1)
      __pause_cpu();
  }

  __dllexport TaskScheduler::Thread* TaskScheduler::thread() {
    return thread_local_thread;
  }

  __dllexport TaskScheduler::Thread* TaskScheduler::swapThread(Thread* thread)
  {
    Thread* old = thread_local_thread;
    thread_local_thread = thread;
    return old;
  }

  __dllexport bool TaskScheduler::wait()
  {
    Thread* thread = TaskScheduler::thread();
    if (thread == nullptr) return true;
    while (thread->tasks.execute_local(*thread,thread->task)) {};
    return thread->scheduler->cancellingException == nullptr;
  }

  std::exception_ptr TaskScheduler::thread_loop(size_t threadIndex)
  {
    /* allocate thread structure */
    std::unique_ptr<Thread> mthread(new Thread(threadIndex,this)); // too large for stack allocation
    Thread& thread = *mthread;
    threadLocal[threadIndex].store(&thread);
    Thread* oldThread = swapThread(&thread);

    /* main thread loop */
    while (anyTasksRunning)
    {
      steal_loop(thread,
                 [&] () { return anyTasksRunning > 0; },
                 [&] () {
                   anyTasksRunning++;
                   while (thread.tasks.execute_local(thread,nullptr));
                   anyTasksRunning--;
                 });
    }
    threadLocal[threadIndex].store(nullptr);
    swapThread(oldThread);

    /* remember exception to throw */
    std::exception_ptr except = nullptr;
    if (cancellingException != nullptr) except = cancellingException;

    /* wait for all threads to terminate */
    threadCounter--;
#if defined(__WIN32__)
	size_t loopIndex = 1;
#endif
#define LOOP_YIELD_THRESHOLD 4096
	while (threadCounter > 0) {
#if defined(__WIN32__)
		if ((loopIndex % LOOP_YIELD_THRESHOLD) == 0)
			yield();
		else
			_mm_pause();
#endif
	  loopIndex++;
	}
    return except;
  }

  bool TaskScheduler::steal_from_other_threads(Thread& thread)
  {
    const size_t threadIndex = thread.threadIndex;
    const size_t threadCount = this->threadCounter;

    for (size_t i=1; i<threadCount; i++)
    {
      __pause_cpu(32);
      size_t otherThreadIndex = threadIndex+i;
      if (otherThreadIndex >= threadCount) otherThreadIndex -= threadCount;

      Thread* othread = threadLocal[otherThreadIndex].load();
      if (!othread)
        continue;

      if (othread->tasks.steal(thread))
        return true;
    }

    return false;
  }

  __dllexport void TaskScheduler::startThreads() {
    threadPool->startThreads();
  }

  __dllexport void TaskScheduler::addScheduler(const Ref<TaskScheduler>& scheduler) {
    threadPool->add(scheduler);
  }

  __dllexport void TaskScheduler::removeScheduler(const Ref<TaskScheduler>& scheduler) {
    threadPool->remove(scheduler);
  }
}
<commit_msg>compile fix<commit_after>// ======================================================================== //
// Copyright 2009-2016 Intel Corporation                                    //
//                                                                          //
// Licensed under the Apache License, Version 2.0 (the "License");          //
// you may not use this file except in compliance with the License.         //
// You may obtain a copy of the License at                                  //
//                                                                          //
//     http://www.apache.org/licenses/LICENSE-2.0                           //
//                                                                          //
// Unless required by applicable law or agreed to in writing, software      //
// distributed under the License is distributed on an "AS IS" BASIS,        //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and      //
// limitations under the License.                                           //
// ======================================================================== //

#include "taskschedulerinternal.h"
#include "../math/math.h"
#include "../sys/sysinfo.h"
#include <algorithm>

namespace embree
{
  size_t TaskScheduler::g_numThreads = 0;
  __thread TaskScheduler* TaskScheduler::g_instance = nullptr;
  __thread TaskScheduler::Thread* TaskScheduler::thread_local_thread = nullptr;
  TaskScheduler::ThreadPool* TaskScheduler::threadPool = nullptr;

  template<typename Predicate, typename Body>
  __forceinline void TaskScheduler::steal_loop(Thread& thread, const Predicate& pred, const Body& body)
  {
    while (true)
    {
      /*! some rounds that yield */
      for (size_t i=0; i<32; i++)
      {
        /*! some spinning rounds */
        const size_t threadCount = thread.threadCount();
        for (size_t j=0; j<1024; j+=threadCount)
        {
          if (!pred()) return;
          if (thread.scheduler->steal_from_other_threads(thread)) {
            i=j=0;
            body();
          }
        }
        yield();
      }
    }
  }

  /*! run this task */
  __dllexport void TaskScheduler::Task::run (Thread& thread) // FIXME: avoid as many __dllexports as possible
  {
    /* try to run if not already stolen */
    if (try_switch_state(INITIALIZED,DONE))
    {
      Task* prevTask = thread.task;
      thread.task = this;
      try {
        if (thread.scheduler->cancellingException == nullptr)
          closure->execute();
      } catch (...) {
        if (thread.scheduler->cancellingException == nullptr)
          thread.scheduler->cancellingException = std::current_exception();
      }
      thread.task = prevTask;
      add_dependencies(-1);
    }

    /* steal until all dependencies have completed */
    steal_loop(thread,
               [&] () { return dependencies>0; },
               [&] () { while (thread.tasks.execute_local(thread,this)); });

    /* now signal our parent task that we are finished */
    if (parent)
      parent->add_dependencies(-1);
  }

  __dllexport bool TaskScheduler::TaskQueue::execute_local(Thread& thread, Task* parent)
  {
    /* stop if we run out of local tasks or reach the waiting task */
    if (right == 0 || &tasks[right-1] == parent)
      return false;

    /* execute task */
    size_t oldRight = right;
    tasks[right-1].run(thread);
    if (right != oldRight) {
      THROW_RUNTIME_ERROR("you have to wait for spawned subtasks");
    }

    /* pop task and closure from stack */
    right--;
    if (tasks[right].stackPtr != size_t(-1))
      stackPtr = tasks[right].stackPtr;

    /* also move left pointer */
    if (left >= right) left.store(right.load());

    return right != 0;
  }

  bool TaskScheduler::TaskQueue::steal(Thread& thread)
  {
    size_t l = left;
    if (l < right)
      l = left++;
    else
      return false;

    if (!tasks[l].try_steal(thread.tasks.tasks[thread.tasks.right]))
      return false;

    thread.tasks.right++;
    return true;
  }

  /* we steal from the left */
  size_t TaskScheduler::TaskQueue::getTaskSizeAtLeft()
  {
    if (left >= right) return 0;
    return tasks[left].N;
  }

  static MutexSys g_mutex;
  static BarrierSys g_barrier(2);

  void threadPoolFunction(std::pair<TaskScheduler::ThreadPool*,size_t>* pair)
  {
    TaskScheduler::ThreadPool* pool = pair->first;
    size_t threadIndex = pair->second;
    g_barrier.wait();
    pool->thread_loop(threadIndex);
  }

  TaskScheduler::ThreadPool::ThreadPool(bool set_affinity)
    : numThreads(0), numThreadsRunning(0), set_affinity(set_affinity), running(false) {}

  __dllexport void TaskScheduler::ThreadPool::startThreads()
  {
    if (running) return;
    setNumThreads(numThreads,true);
  }

  void TaskScheduler::ThreadPool::setNumThreads(size_t newNumThreads, bool startThreads)
  {
    Lock<MutexSys> lock(g_mutex);

    if (newNumThreads == 0)
      newNumThreads = getNumberOfLogicalThreads();

    numThreads = newNumThreads;
    if (!startThreads && !running) return;
    running = true;
    size_t numThreadsActive = numThreadsRunning;

    mutex.lock();
    numThreadsRunning = newNumThreads;
    mutex.unlock();
    condition.notify_all();

    /* start new threads */
    for (size_t t=numThreadsActive; t<numThreads; t++)
    {
      if (t == 0) continue;
      auto pair = std::make_pair(this,t);
      threads.push_back(createThread((thread_func)threadPoolFunction,&pair,4*1024*1024,set_affinity ? t : -1));
      g_barrier.wait();
    }

    /* stop some threads if we reduce the number of threads */
    for (ssize_t t=numThreadsActive-1; t>=ssize_t(numThreadsRunning); t--) {
      if (t == 0) continue;
      embree::join(threads.back());
      threads.pop_back();
    }
  }

  TaskScheduler::ThreadPool::~ThreadPool()
  {
    /* leave all taskschedulers */
    mutex.lock();
    numThreadsRunning = 0;
    mutex.unlock();
    condition.notify_all();

    /* wait for threads to terminate */
    for (size_t i=0; i<threads.size(); i++)
      embree::join(threads[i]);
  }

  __dllexport void TaskScheduler::ThreadPool::add(const Ref<TaskScheduler>& scheduler)
  {
    mutex.lock();
    schedulers.push_back(scheduler);
    mutex.unlock();
    condition.notify_all();
  }

  __dllexport void TaskScheduler::ThreadPool::remove(const Ref<TaskScheduler>& scheduler)
  {
    Lock<MutexSys> lock(mutex);
    for (std::list<Ref<TaskScheduler> >::iterator it = schedulers.begin(); it != schedulers.end(); it++) {
      if (scheduler == *it) {
        schedulers.erase(it);
        return;
      }
    }
  }

  void TaskScheduler::ThreadPool::thread_loop(size_t globalThreadIndex)
  {
    while (globalThreadIndex < numThreadsRunning)
    {
      Ref<TaskScheduler> scheduler = NULL;
      ssize_t threadIndex = -1;
      {
        Lock<MutexSys> lock(mutex);
        condition.wait(mutex, [&] () { return globalThreadIndex >= numThreadsRunning || !schedulers.empty(); });
        if (globalThreadIndex >= numThreadsRunning) break;
        scheduler = schedulers.front();
        threadIndex = scheduler->allocThreadIndex();
      }
      scheduler->thread_loop(threadIndex);
    }
  }

  TaskScheduler::TaskScheduler()
    : threadCounter(0), anyTasksRunning(0), hasRootTask(false)
  {
    threadLocal.resize(2*getNumberOfLogicalThreads()); // FIXME: this has to be 2x as in the join mode the worker threads also join
    for (size_t i=0; i<threadLocal.size(); i++)
      threadLocal[i].store(nullptr);
  }

  TaskScheduler::~TaskScheduler()
  {
    assert(threadCounter == 0);
  }

  __dllexport size_t TaskScheduler::threadIndex()
  {
    Thread* thread = TaskScheduler::thread();
    if (thread) return thread->threadIndex;
    else        return 0;
  }

  __dllexport size_t TaskScheduler::threadCount() {
    return threadPool->size();
  }

  __dllexport TaskScheduler* TaskScheduler::instance()
  {
    if (g_instance == NULL) {
      g_instance = new TaskScheduler;
      g_instance->refInc();
    }
    return g_instance;
  }

  void TaskScheduler::create(size_t numThreads, bool set_affinity)
  {
    if (!threadPool) threadPool = new TaskScheduler::ThreadPool(set_affinity);
    threadPool->setNumThreads(numThreads,false);
  }

  void TaskScheduler::destroy() {
    delete threadPool; threadPool = nullptr;
  }

  __dllexport ssize_t TaskScheduler::allocThreadIndex()
  {
    size_t threadIndex = threadCounter++;
    assert(threadIndex < threadLocal.size());
    return threadIndex;
  }

  void TaskScheduler::join()
  {
    mutex.lock();
    size_t threadIndex = allocThreadIndex();
    condition.wait(mutex, [&] () { return hasRootTask.load(); });
    mutex.unlock();
    std::exception_ptr except = thread_loop(threadIndex);
    if (except != nullptr) std::rethrow_exception(except);
  }

  void TaskScheduler::reset() {
    hasRootTask = false;
  }

  void TaskScheduler::wait_for_threads(size_t threadCount)
  {
    while (threadCounter < threadCount-1)
      __pause_cpu();
  }

  __dllexport TaskScheduler::Thread* TaskScheduler::thread() {
    return thread_local_thread;
  }

  __dllexport TaskScheduler::Thread* TaskScheduler::swapThread(Thread* thread)
  {
    Thread* old = thread_local_thread;
    thread_local_thread = thread;
    return old;
  }

  __dllexport bool TaskScheduler::wait()
  {
    Thread* thread = TaskScheduler::thread();
    if (thread == nullptr) return true;
    while (thread->tasks.execute_local(*thread,thread->task)) {};
    return thread->scheduler->cancellingException == nullptr;
  }

  std::exception_ptr TaskScheduler::thread_loop(size_t threadIndex)
  {
    /* allocate thread structure */
    std::unique_ptr<Thread> mthread(new Thread(threadIndex,this)); // too large for stack allocation
    Thread& thread = *mthread;
    threadLocal[threadIndex].store(&thread);
    Thread* oldThread = swapThread(&thread);

    /* main thread loop */
    while (anyTasksRunning)
    {
      steal_loop(thread,
                 [&] () { return anyTasksRunning > 0; },
                 [&] () {
                   anyTasksRunning++;
                   while (thread.tasks.execute_local(thread,nullptr));
                   anyTasksRunning--;
                 });
    }
    threadLocal[threadIndex].store(nullptr);
    swapThread(oldThread);

    /* remember exception to throw */
    std::exception_ptr except = nullptr;
    if (cancellingException != nullptr) except = cancellingException;

    /* wait for all threads to terminate */
    threadCounter--;
#if defined(__WIN32__)
	size_t loopIndex = 1;
#endif
#define LOOP_YIELD_THRESHOLD 4096
	while (threadCounter > 0) {
#if defined(__WIN32__)
          if ((loopIndex % LOOP_YIELD_THRESHOLD) == 0)
            yield();
          else
            _mm_pause();
	  loopIndex++;
#else
          yield();
#endif
	}
    return except;
  }

  bool TaskScheduler::steal_from_other_threads(Thread& thread)
  {
    const size_t threadIndex = thread.threadIndex;
    const size_t threadCount = this->threadCounter;

    for (size_t i=1; i<threadCount; i++)
    {
      __pause_cpu(32);
      size_t otherThreadIndex = threadIndex+i;
      if (otherThreadIndex >= threadCount) otherThreadIndex -= threadCount;

      Thread* othread = threadLocal[otherThreadIndex].load();
      if (!othread)
        continue;

      if (othread->tasks.steal(thread))
        return true;
    }

    return false;
  }

  __dllexport void TaskScheduler::startThreads() {
    threadPool->startThreads();
  }

  __dllexport void TaskScheduler::addScheduler(const Ref<TaskScheduler>& scheduler) {
    threadPool->add(scheduler);
  }

  __dllexport void TaskScheduler::removeScheduler(const Ref<TaskScheduler>& scheduler) {
    threadPool->remove(scheduler);
  }
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/thread/queue.h"
#include "fnord-base/wallclock.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-eventdb/TableRepository.h"
#include "fnord-eventdb/LogTableTail.h"
#include "fnord-msg/MessageEncoder.h"
#include "fnord-msg/MessagePrinter.h"
#include "logjoin/LogJoinBackfill.h"
#include "IndexReader.h"

#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

fnord::thread::EventLoop ev;
std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

struct BackfillData {
  Option<String> shop_name;
  Option<String> shop_id;
  Option<uint64_t> category1;
  Option<uint64_t> category2;
  Option<uint64_t> category3;
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "datadir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "worker_threads",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "4",
      "threads",
      "<num>");

  flags.defineFlag(
      "upload_threads",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1",
      "threads",
      "<num>");

  flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "replica",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "replica id",
      "<id>");

  flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* args */
  auto index_path = flags.getString("index");
  auto batch_size = flags.getInt("batch_size");
  auto datadir = flags.getString("datadir");
  auto dry_run = !flags.isSet("no_dryrun");
  if (!FileUtil::isDirectory(datadir)) {
    RAISEF(kIOError, "no such directory: $0", datadir);
  }

  URI target_uri("http://localhost:8000/eventdb/insert?table=joined_sessions-dawanda");

  /* event loop, http */
  http::HTTPConnectionPool http(&ev);
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* open index */
  auto index = cm::IndexReader::openIndex(index_path);

  /* open table */
  auto schema = joinedSessionsSchema();
  auto table = eventdb::TableReader::open(
      "dawanda_joined_sessions",
      flags.getString("replica"),
      datadir,
      schema);

  auto queries_fid = schema.id("queries");
  auto queryitems_fid = schema.id("queries.items");
  auto qi_id_fid = schema.id("queries.items.item_id");
  auto qi_sid_fid = schema.id("queries.items.shop_id");
  auto qi_sname_fid = schema.id("queries.items.shop_name");
  auto qi_c1_fid = schema.id("queries.items.category1");
  auto qi_c2_fid = schema.id("queries.items.category2");
  auto qi_c3_fid = schema.id("queries.items.category3");

  HashMap<String, BackfillData> cache;

  /* backfill fn */
  auto get_backfill_data = [&cache, &index] (const String& item_id) -> BackfillData {
    auto cached = cache.find(item_id);
    if (!(cached == cache.end())) {
      return cached->second;
    }

    BackfillData data;

    DocID docid { .docid = item_id };
    data.shop_id = index->docIndex()->getField(docid, "shop_id");
    data.shop_name = index->docIndex()->getField(docid, "shop_name");

    auto category1 = index->docIndex()->getField(docid, "category1");
    if (!category1.isEmpty()) {
      data.category1 = Some((uint64_t) std::stoull(category1.get()));
    }

    auto category2 = index->docIndex()->getField(docid, "category2");
    if (!category2.isEmpty()) {
      data.category2 = Some((uint64_t) std::stoull(category2.get()));
    }

    auto category3 = index->docIndex()->getField(docid, "category3");
    if (!category3.isEmpty()) {
      data.category3 = Some((uint64_t) std::stoull(category3.get()));
    }

    cache[item_id] = data;
    return data;
  };

  auto backfill_fn = [
    &schema,
    &queries_fid,
    &queryitems_fid,
    &qi_id_fid,
    &qi_sid_fid,
    &qi_sname_fid,
    &qi_c1_fid,
    &qi_c2_fid,
    &qi_c3_fid,
    &get_backfill_data
  ] (msg::MessageObject* record) {
    auto msg = msg::MessagePrinter::print(*record, schema);

    for (auto& q : record->asObject()) {
      if (q.id != queries_fid) {
        continue;
      }

      for (auto& qi : q.asObject()) {
        if (qi.id != queryitems_fid) {
          continue;
        }

        String item_id;
        for (auto cur = qi.asObject().begin(); cur != qi.asObject().end(); ) {
          auto id = cur->id;

          if (id == qi_id_fid) {
            item_id = cur->asString();
            ++cur;
          } else if (
              id == qi_sid_fid ||
              id == qi_sname_fid ||
              id == qi_c1_fid ||
              id == qi_c2_fid ||
              id == qi_c3_fid) {
            cur = qi.asObject().erase(cur);
          } else {
            ++cur;
          }
        }

        auto bdata = get_backfill_data(item_id);

        if (!bdata.shop_id.isEmpty()) {
          qi.addChild(qi_sid_fid, bdata.shop_id.get());
        }

        if (!bdata.shop_name.isEmpty()) {
          qi.addChild(qi_sname_fid, bdata.shop_name.get());
        }

        if (!bdata.category1.isEmpty()) {
          qi.addChild(qi_c1_fid, bdata.category1.get());
        }

        if (!bdata.category2.isEmpty()) {
          qi.addChild(qi_c2_fid, bdata.category2.get());
        }

        if (!bdata.category3.isEmpty()) {
          qi.addChild(qi_c3_fid, bdata.category3.get());
        }
      }
    }

    fnord::iputs("backfill: $0", msg);
  };

  /* run backfill */
  cm::LogJoinBackfill backfill(
      table,
      backfill_fn,
      "/tmp/logjoin-backfill-state",
      dry_run,
      target_uri,
      &http);

  backfill.start();

  while (backfill.process(batch_size)) {
    if (cm_logjoin_shutdown.load()) {
      break;
    }
  }

  backfill.shutdown();
  ev.shutdown();
  evloop_thread.join();

  return 0;
}

<commit_msg>print backfilled record<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/thread/queue.h"
#include "fnord-base/wallclock.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-eventdb/TableRepository.h"
#include "fnord-eventdb/LogTableTail.h"
#include "fnord-msg/MessageEncoder.h"
#include "fnord-msg/MessagePrinter.h"
#include "logjoin/LogJoinBackfill.h"
#include "IndexReader.h"

#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

fnord::thread::EventLoop ev;
std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

struct BackfillData {
  Option<String> shop_name;
  Option<String> shop_id;
  Option<uint64_t> category1;
  Option<uint64_t> category2;
  Option<uint64_t> category3;
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "datadir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "worker_threads",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "4",
      "threads",
      "<num>");

  flags.defineFlag(
      "upload_threads",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1",
      "threads",
      "<num>");

  flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "replica",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "replica id",
      "<id>");

  flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* args */
  auto index_path = flags.getString("index");
  auto batch_size = flags.getInt("batch_size");
  auto datadir = flags.getString("datadir");
  auto dry_run = !flags.isSet("no_dryrun");
  if (!FileUtil::isDirectory(datadir)) {
    RAISEF(kIOError, "no such directory: $0", datadir);
  }

  URI target_uri("http://localhost:8000/eventdb/insert?table=joined_sessions-dawanda");

  /* event loop, http */
  http::HTTPConnectionPool http(&ev);
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* open index */
  auto index = cm::IndexReader::openIndex(index_path);

  /* open table */
  auto schema = joinedSessionsSchema();
  auto table = eventdb::TableReader::open(
      "dawanda_joined_sessions",
      flags.getString("replica"),
      datadir,
      schema);

  auto queries_fid = schema.id("queries");
  auto queryitems_fid = schema.id("queries.items");
  auto qi_id_fid = schema.id("queries.items.item_id");
  auto qi_sid_fid = schema.id("queries.items.shop_id");
  auto qi_sname_fid = schema.id("queries.items.shop_name");
  auto qi_c1_fid = schema.id("queries.items.category1");
  auto qi_c2_fid = schema.id("queries.items.category2");
  auto qi_c3_fid = schema.id("queries.items.category3");

  HashMap<String, BackfillData> cache;

  /* backfill fn */
  auto get_backfill_data = [&cache, &index] (const String& item_id) -> BackfillData {
    auto cached = cache.find(item_id);
    if (!(cached == cache.end())) {
      return cached->second;
    }

    BackfillData data;

    DocID docid { .docid = item_id };
    data.shop_id = index->docIndex()->getField(docid, "shop_id");
    data.shop_name = index->docIndex()->getField(docid, "shop_name");

    auto category1 = index->docIndex()->getField(docid, "category1");
    if (!category1.isEmpty()) {
      data.category1 = Some((uint64_t) std::stoull(category1.get()));
    }

    auto category2 = index->docIndex()->getField(docid, "category2");
    if (!category2.isEmpty()) {
      data.category2 = Some((uint64_t) std::stoull(category2.get()));
    }

    auto category3 = index->docIndex()->getField(docid, "category3");
    if (!category3.isEmpty()) {
      data.category3 = Some((uint64_t) std::stoull(category3.get()));
    }

    cache[item_id] = data;
    return data;
  };

  auto backfill_fn = [
    &schema,
    &queries_fid,
    &queryitems_fid,
    &qi_id_fid,
    &qi_sid_fid,
    &qi_sname_fid,
    &qi_c1_fid,
    &qi_c2_fid,
    &qi_c3_fid,
    &get_backfill_data
  ] (msg::MessageObject* record) {
    for (auto& q : record->asObject()) {
      if (q.id != queries_fid) {
        continue;
      }

      for (auto& qi : q.asObject()) {
        if (qi.id != queryitems_fid) {
          continue;
        }

        String item_id;
        for (auto cur = qi.asObject().begin(); cur != qi.asObject().end(); ) {
          auto id = cur->id;

          if (id == qi_id_fid) {
            item_id = cur->asString();
            ++cur;
          } else if (
              id == qi_sid_fid ||
              id == qi_sname_fid ||
              id == qi_c1_fid ||
              id == qi_c2_fid ||
              id == qi_c3_fid) {
            cur = qi.asObject().erase(cur);
          } else {
            ++cur;
          }
        }

        auto bdata = get_backfill_data(item_id);

        if (!bdata.shop_id.isEmpty()) {
          qi.addChild(qi_sid_fid, bdata.shop_id.get());
        }

        if (!bdata.shop_name.isEmpty()) {
          qi.addChild(qi_sname_fid, bdata.shop_name.get());
        }

        if (!bdata.category1.isEmpty()) {
          qi.addChild(qi_c1_fid, bdata.category1.get());
        }

        if (!bdata.category2.isEmpty()) {
          qi.addChild(qi_c2_fid, bdata.category2.get());
        }

        if (!bdata.category3.isEmpty()) {
          qi.addChild(qi_c3_fid, bdata.category3.get());
        }
      }
    }

    auto msg = msg::MessagePrinter::print(*record, schema);
    fnord::iputs("backfill: $0", msg);
  };

  /* run backfill */
  cm::LogJoinBackfill backfill(
      table,
      backfill_fn,
      "/tmp/logjoin-backfill-state",
      dry_run,
      target_uri,
      &http);

  backfill.start();

  while (backfill.process(batch_size)) {
    if (cm_logjoin_shutdown.load()) {
      break;
    }
  }

  backfill.shutdown();
  ev.shutdown();
  evloop_thread.join();

  return 0;
}

<|endoftext|>