<commit_before>// Copyright 2005-2016 The Mumble Developers. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file at the root of the
// Mumble source tree or at <https://www.mumble.info/LICENSE>.

#include "../mumble_plugin_win32.h" // Include standard plugin header.
#include "../mumble_plugin_utils.h" // Include plugin header for special functions, like "escape".

static int fetch(float *avatar_pos, float *avatar_front, float *avatar_top, float *camera_pos, float *camera_front, float *camera_top, std::string &context, std::wstring &identity) {
	for (int i=0;i<3;i++) {
		avatar_pos[i] = avatar_front[i] = avatar_top[i] = camera_pos[i] = camera_front[i] = camera_top[i] = 0.0f;
	}

	// Boolean value to check if game addresses retrieval is successful
	bool ok;
	// Create containers to stuff our raw data into, so we can convert it to Mumble's coordinate system
	float avatar_pos_corrector[3], camera_pos_corrector[3], avatar_front_corrector[3], avatar_top_corrector[3], camera_front_corrector[3], camera_top_corrector[3];
	// Char values for extra features
	char state, in_game, player[50], vehicle[50], location[50], street[50];

	// Avatar pointer
	BYTE *avatar_base = peekProc<BYTE *>(pModule + 0x01B2C3D0);
	if (!avatar_base) return false;

	// Peekproc and assign game addresses to our containers, so we can retrieve positional data
	ok = peekProc((BYTE *) pModule + 0x264DE00, &state, 1) && // Magical state value: 0 when in single player, 2 when online and 3 when in a lobby.
			peekProc((BYTE *) pModule + 0x1B5C5EF, &in_game, 1) && // 0 when loading or not in-game, 1 when in-game.
			peekProc((BYTE *) pModule + 0x1B9E620, avatar_pos_corrector, 12) && // Avatar Position values (Z, X and Y).
			peekProc((BYTE *) pModule + 0x1BEDBE0, camera_pos_corrector, 12) && // Camera Position values (Z, X and Y).
			peekProc((BYTE *) avatar_base + 0x70, avatar_front_corrector, 12) && // Avatar Front Vector values (Z, X and Y).
			peekProc((BYTE *) avatar_base + 0x80, avatar_top_corrector, 12) && // Avatar Top Vector values (Z, X and Y).
			peekProc((BYTE *) pModule + 0x1BEF6A0, camera_front_corrector, 12) && // Camera Front Vector values (Z, X and Y).
			peekProc((BYTE *) pModule + 0x1B6F710, camera_top_corrector, 12) && // Camera Top Vector values (Z, X and Y).
			peekProc((BYTE *) pModule + 0x265857C, player) && // Player nickname.
			peekProc((BYTE *) pModule + 0x22506D0, vehicle) && // Vehicle name if in a vehicle, empty if not.
			peekProc((BYTE *) pModule + 0x22500AB, location) && // Location name.
			peekProc((BYTE *) pModule + 0x224CE70, street); // Street name if on a street, empty if not.

	// This prevents the plugin from linking to the game in case something goes wrong during values retrieval from memory addresses.
	if (! ok)
		return false;

	// State
	if (state != 2 || in_game == 0) { // If not in-game
		context.clear(); // Clear context
		identity.clear(); // Clear identity
		// Set vectors values to 0.
		for (int i=0;i<3;i++)
			avatar_pos[i] = avatar_front[i] = avatar_top[i] = camera_pos[i] =  camera_front[i] = camera_top[i] = 0.0f;

		return true; // This tells Mumble to ignore all vectors.
	}
	/*
	// Begin context
	std::ostringstream ocontext;

	// Host
	host[sizeof(host)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(host);
	if (strcmp(host, "") != 0 && strstr(host, "127.0.0.1") == NULL) { // Set host string as empty if "127.0.0.1" is found in it.
		ocontext << " {\"Host\": \"" << host << "\"}"; // Set context with IP address and port
	}

	context = ocontext.str();
	// End context
	*/
	// Begin identity
	std::wostringstream oidentity;
	oidentity << "{";

	// Player
	player[sizeof(player)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(player);
	if (strcmp(player, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Player\": \"" << player << "\","; // Set player nickname in identity.
	} else {
		oidentity << std::endl << "\"Player\": null,";
	}

	// Vehicle
	vehicle[sizeof(vehicle)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(vehicle);
	if (strcmp(vehicle, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Vehicle\": \"" << vehicle << "\","; // Set vehicle name in identity.
	} else {
		oidentity << std::endl << "\"Vehicle\": null,";
	}

	// Location
	location[sizeof(location)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(location);
	if (strcmp(location, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Location\": \"" << location << "\","; // Set location name in identity.
	} else {
		oidentity << std::endl << "\"Location\": null,";
	}

	// Street
	street[sizeof(street)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(street);
	if (strcmp(street, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Street\": \"" << street << "\","; // Set street name in identity.
	} else {
		oidentity << std::endl << "\"Street\": null";
	}

	oidentity << std::endl << "}";
	identity = oidentity.str();
	// End identity

	/*
	Mumble | Game
	X      | Z
	Y      | X
	Z      | Y
	*/
	avatar_pos[0] = avatar_pos_corrector[1];
	avatar_pos[1] = avatar_pos_corrector[2];
	avatar_pos[2] = avatar_pos_corrector[0];

	camera_pos[0] = camera_pos_corrector[1];
	camera_pos[1] = camera_pos_corrector[2];
	camera_pos[2] = camera_pos_corrector[0];

	avatar_front[0] = avatar_front_corrector[1];
	avatar_front[1] = avatar_front_corrector[2];
	avatar_front[2] = avatar_front_corrector[0];

	avatar_top[0] = avatar_top_corrector[1];
	avatar_top[1] = avatar_top_corrector[2];
	avatar_top[2] = avatar_top_corrector[0];

	camera_front[0] = camera_front_corrector[1];
	camera_front[1] = camera_front_corrector[2];
	camera_front[2] = camera_front_corrector[0];

	camera_top[0] = camera_top_corrector[1];
	camera_top[1] = camera_top_corrector[2];
	camera_top[2] = camera_top_corrector[0];

	return true;
}

static int trylock(const std::multimap<std::wstring, unsigned long long int> &pids) {

	if (! initialize(pids, L"GTA5.exe")) { // Link the game executable
		return false;
	}

	// Check if we can get meaningful data from it
	float apos[3], afront[3], atop[3], cpos[3], cfront[3], ctop[3];
	std::wstring sidentity;
	std::string scontext;

	if (fetch(apos, afront, atop, cpos, cfront, ctop, scontext, sidentity)) {
		return true;
	} else {
		generic_unlock();
		return false;
	}
}

static const std::wstring longdesc() {
	return std::wstring(L"Supports Grand Theft Auto V with identity support."); // Plugin long description
}

static std::wstring description(L"Grand Theft Auto V version 1.33"); // Plugin short description
static std::wstring shortname(L"Grand Theft Auto V"); // Plugin short name

static int trylock1() {
	return trylock(std::multimap<std::wstring, unsigned long long int>());
}

static MumblePlugin gtavplug = {
	MUMBLE_PLUGIN_MAGIC,
	description,
	shortname,
	NULL,
	NULL,
	trylock1,
	generic_unlock,
	longdesc,
	fetch
};

static MumblePlugin2 gtavplug2 = {
	MUMBLE_PLUGIN_MAGIC_2,
	MUMBLE_PLUGIN_VERSION,
	trylock
};

extern "C" __declspec(dllexport) MumblePlugin *getMumblePlugin() {
	return &gtavplug;
}

extern "C" __declspec(dllexport) MumblePlugin2 *getMumblePlugin2() {
	return &gtavplug2;
}
<commit_msg>plugins/gtav: Plugin update for game's latest version<commit_after>// Copyright 2005-2016 The Mumble Developers. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file at the root of the
// Mumble source tree or at <https://www.mumble.info/LICENSE>.

#include "../mumble_plugin_win32.h" // Include standard plugin header.
#include "../mumble_plugin_utils.h" // Include plugin header for special functions, like "escape".

static int fetch(float *avatar_pos, float *avatar_front, float *avatar_top, float *camera_pos, float *camera_front, float *camera_top, std::string &context, std::wstring &identity) {
	for (int i=0;i<3;i++) {
		avatar_pos[i] = avatar_front[i] = avatar_top[i] = camera_pos[i] = camera_front[i] = camera_top[i] = 0.0f;
	}

	// Boolean value to check if game addresses retrieval is successful
	bool ok;
	// Create containers to stuff our raw data into, so we can convert it to Mumble's coordinate system
	float avatar_pos_corrector[3], camera_pos_corrector[3], avatar_front_corrector[3], avatar_top_corrector[3], camera_front_corrector[3], camera_top_corrector[3];
	// Char values for extra features
	char state, in_game, player[50], vehicle[50], location[50], street[50];

	// Avatar pointer
	BYTE *avatar_base = peekProc<BYTE *>(pModule + 0x01B3D3F0);
	if (!avatar_base) return false;

	// Peekproc and assign game addresses to our containers, so we can retrieve positional data
	ok = peekProc((BYTE *) pModule + 0x267FD50, &state, 1) && // Magical state value: 0 when in single player, 2 when online and 3 when in a lobby.
			peekProc((BYTE *) pModule + 0x1B6D757, &in_game, 1) && // 0 when loading or not in-game, 1 when in-game.
			peekProc((BYTE *) pModule + 0x1EC2830, avatar_pos_corrector, 12) && // Avatar Position values (X, Z and Y).
			peekProc((BYTE *) pModule + 0x1BFEDA0, camera_pos_corrector, 12) && // Camera Position values (X, Z and Y).
			peekProc((BYTE *) avatar_base + 0x70, avatar_front_corrector, 12) && // Avatar Front Vector values (X, Z and Y).
			peekProc((BYTE *) avatar_base + 0x80, avatar_top_corrector, 12) && // Avatar Top Vector values (X, Z and Y).
			peekProc((BYTE *) pModule + 0x1C00860, camera_front_corrector, 12) && // Camera Front Vector values (X, Z and Y).
			peekProc((BYTE *) pModule + 0x1ED0E30, camera_top_corrector, 12) && // Camera Top Vector values (X, Z and Y).
			peekProc((BYTE *) pModule + 0x268A4EC, player) && // Player nickname.
			peekProc((BYTE *) pModule + 0x2282400, vehicle) && // Vehicle name if in a vehicle, empty if not.
			peekProc((BYTE *) pModule + 0x2281DDB, location) && // Location name.
			peekProc((BYTE *) pModule + 0x227EBA0, street); // Street name if on a street, empty if not.

	// This prevents the plugin from linking to the game in case something goes wrong during values retrieval from memory addresses.
	if (! ok)
		return false;

	// State
	if (state != 2 || in_game == 0) { // If not in-game
		context.clear(); // Clear context
		identity.clear(); // Clear identity
		// Set vectors values to 0.
		for (int i=0;i<3;i++)
			avatar_pos[i] = avatar_front[i] = avatar_top[i] = camera_pos[i] =  camera_front[i] = camera_top[i] = 0.0f;

		return true; // This tells Mumble to ignore all vectors.
	}
	/*
	// Begin context
	std::ostringstream ocontext;

	// Host
	host[sizeof(host)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(host);
	if (strcmp(host, "") != 0 && strstr(host, "127.0.0.1") == NULL) { // Set host string as empty if "127.0.0.1" is found in it.
		ocontext << " {\"Host\": \"" << host << "\"}"; // Set context with IP address and port
	}

	context = ocontext.str();
	// End context
	*/
	// Begin identity
	std::wostringstream oidentity;
	oidentity << "{";

	// Player
	player[sizeof(player)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(player);
	if (strcmp(player, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Player\": \"" << player << "\","; // Set player nickname in identity.
	} else {
		oidentity << std::endl << "\"Player\": null,";
	}

	// Vehicle
	vehicle[sizeof(vehicle)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(vehicle);
	if (strcmp(vehicle, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Vehicle\": \"" << vehicle << "\","; // Set vehicle name in identity.
	} else {
		oidentity << std::endl << "\"Vehicle\": null,";
	}

	// Location
	location[sizeof(location)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(location);
	if (strcmp(location, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Location\": \"" << location << "\","; // Set location name in identity.
	} else {
		oidentity << std::endl << "\"Location\": null,";
	}

	// Street
	street[sizeof(street)-1] = 0; // NUL terminate queried C strings. We do this to ensure the strings from the game are NUL terminated. They should be already, but we can't take any chances.
	escape(street);
	if (strcmp(street, "") != 0) {
		oidentity << std::endl;
		oidentity << "\"Street\": \"" << street << "\""; // Set street name in identity.
	} else {
		oidentity << std::endl << "\"Street\": null";
	}

	oidentity << std::endl << "}";
	identity = oidentity.str();
	// End identity

	/*
	Mumble | Game
	X      | X
	Y      | Z
	Z      | Y
	*/
	avatar_pos[0] = avatar_pos_corrector[0];
	avatar_pos[1] = avatar_pos_corrector[2];
	avatar_pos[2] = avatar_pos_corrector[1];

	camera_pos[0] = camera_pos_corrector[0];
	camera_pos[1] = camera_pos_corrector[2];
	camera_pos[2] = camera_pos_corrector[1];

	avatar_front[0] = avatar_front_corrector[0];
	avatar_front[1] = avatar_front_corrector[2];
	avatar_front[2] = avatar_front_corrector[1];

	avatar_top[0] = avatar_top_corrector[0];
	avatar_top[1] = avatar_top_corrector[2];
	avatar_top[2] = avatar_top_corrector[1];

	camera_front[0] = camera_front_corrector[0];
	camera_front[1] = camera_front_corrector[2];
	camera_front[2] = camera_front_corrector[1];

	camera_top[0] = camera_top_corrector[0];
	camera_top[1] = camera_top_corrector[2];
	camera_top[2] = camera_top_corrector[1];

	return true;
}

static int trylock(const std::multimap<std::wstring, unsigned long long int> &pids) {

	if (! initialize(pids, L"GTA5.exe")) { // Retrieve game executable's memory address
		return false;
	}

	// Check if we can get meaningful data from it
	float apos[3], afront[3], atop[3], cpos[3], cfront[3], ctop[3];
	std::wstring sidentity;
	std::string scontext;

	if (fetch(apos, afront, atop, cpos, cfront, ctop, scontext, sidentity)) {
		return true;
	} else {
		generic_unlock();
		return false;
	}
}

static const std::wstring longdesc() {
	return std::wstring(L"Supports Grand Theft Auto V version 1.34 with identity support."); // Plugin long description
}

static std::wstring description(L"Grand Theft Auto V (v1.34)"); // Plugin short description
static std::wstring shortname(L"Grand Theft Auto V"); // Plugin short name

static int trylock1() {
	return trylock(std::multimap<std::wstring, unsigned long long int>());
}

static MumblePlugin gtavplug = {
	MUMBLE_PLUGIN_MAGIC,
	description,
	shortname,
	NULL,
	NULL,
	trylock1,
	generic_unlock,
	longdesc,
	fetch
};

static MumblePlugin2 gtavplug2 = {
	MUMBLE_PLUGIN_MAGIC_2,
	MUMBLE_PLUGIN_VERSION,
	trylock
};

extern "C" __declspec(dllexport) MumblePlugin *getMumblePlugin() {
	return &gtavplug;
}

extern "C" __declspec(dllexport) MumblePlugin2 *getMumblePlugin2() {
	return &gtavplug2;
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "runtime/client-cache.h"

#include <sstream>
#include <thrift/server/TServer.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/transport/TSocket.h>
#include <thrift/transport/TTransportUtils.h>
#include <memory>

#include "common/logging.h"
#include "util/container-util.h"
#include "util/network-util.h"
#include "rpc/thrift-util.h"
#include "gen-cpp/ImpalaInternalService.h"

#include "common/names.h"
using namespace apache::thrift;
using namespace apache::thrift::server;
using namespace apache::thrift::transport;
using namespace apache::thrift::protocol;

namespace impala {

Status ClientCacheHelper::GetClient(const TNetworkAddress& address,
    ClientFactory factory_method, ClientKey* client_key) {
  shared_ptr<PerHostCache> host_cache;
  {
    lock_guard<mutex> lock(cache_lock_);
    VLOG(2) << "GetClient(" << address << ")";
    shared_ptr<PerHostCache>* ptr = &per_host_caches_[address];
    if (ptr->get() == NULL) ptr->reset(new PerHostCache());
    host_cache = *ptr;
  }

  {
    lock_guard<mutex> lock(host_cache->lock);
    if (!host_cache->clients.empty()) {
      *client_key = host_cache->clients.front();
      VLOG(2) << "GetClient(): returning cached client for " << address;
      host_cache->clients.pop_front();
      if (metrics_enabled_) clients_in_use_metric_->Increment(1);
      return Status::OK();
    }
  }

  // Only get here if host_cache->clients.empty(). No need for the lock.
  RETURN_IF_ERROR(CreateClient(address, factory_method, client_key));
  if (metrics_enabled_) clients_in_use_metric_->Increment(1);
  return Status::OK();
}

Status ClientCacheHelper::ReopenClient(ClientFactory factory_method,
    ClientKey* client_key) {
  // Clients are not ordinarily removed from the cache completely (in the future, they may
  // be); this is the only method where a client may be deleted and replaced with another.
  shared_ptr<ThriftClientImpl> client_impl;
  ClientMap::iterator client;
  {
    lock_guard<mutex> lock(client_map_lock_);
    client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(1) << "ReopenClient(): re-creating client for " << client_impl->address();

  client_impl->Close();

  // TODO: Thrift TBufferedTransport cannot be re-opened after Close() because it does not
  // clean up internal buffers it reopens. To work around this issue, create a new client
  // instead.
  ClientKey old_client_key = *client_key;
  if (metrics_enabled_) total_clients_metric_->Increment(-1);
  Status status = CreateClient(client_impl->address(), factory_method, client_key);
  // Only erase the existing client from the map if creation of the new one succeeded.
  // This helps to ensure the proper accounting of metrics in the presence of
  // re-connection failures (the original client should be released as usual).
  if (status.ok()) {
    lock_guard<mutex> lock(client_map_lock_);
    client_map_.erase(client);
  } else {
    // Restore the client used before the failed re-opening attempt, so the caller can
    // properly release it.
    *client_key = old_client_key;
  }
  return status;
}

Status ClientCacheHelper::CreateClient(const TNetworkAddress& address,
    ClientFactory factory_method, ClientKey* client_key) {
  shared_ptr<ThriftClientImpl> client_impl(factory_method(address, client_key));
  VLOG(2) << "CreateClient(): creating new client for " << client_impl->address();
  Status status = client_impl->OpenWithRetry(num_tries_, wait_ms_);
  if (!status.ok()) {
    *client_key = NULL;
    return status;
  }
  // Set the TSocket's send and receive timeouts.
  client_impl->setRecvTimeout(recv_timeout_ms_);
  client_impl->setSendTimeout(send_timeout_ms_);

  // Because the client starts life 'checked out', we don't add it to its host cache.
  {
    lock_guard<mutex> lock(client_map_lock_);
    client_map_[*client_key] = client_impl;
  }

  if (metrics_enabled_) total_clients_metric_->Increment(1);
  return Status::OK();
}

void ClientCacheHelper::ReleaseClient(ClientKey* client_key) {
  DCHECK(*client_key != NULL) << "Trying to release NULL client";
  shared_ptr<ThriftClientImpl> client_impl;
  {
    lock_guard<mutex> lock(client_map_lock_);
    ClientMap::iterator client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(2) << "Releasing client for " << client_impl->address() << " back to cache";
  {
    lock_guard<mutex> lock(cache_lock_);
    PerHostCacheMap::iterator cache = per_host_caches_.find(client_impl->address());
    DCHECK(cache != per_host_caches_.end());
    lock_guard<mutex> entry_lock(cache->second->lock);
    cache->second->clients.push_back(*client_key);
  }
  if (metrics_enabled_) clients_in_use_metric_->Increment(-1);
  *client_key = NULL;
}

void ClientCacheHelper::DestroyClient(ClientKey* client_key) {
  DCHECK(*client_key != NULL) << "Trying to destroy NULL client";
  shared_ptr<ThriftClientImpl> client_impl;
  ClientMap::iterator client;
  {
    lock_guard<mutex> lock(client_map_lock_);
    client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(1) << "Broken Connection, destroy client for " << client_impl->address();

  client_impl->Close();
  if (metrics_enabled_) total_clients_metric_->Increment(-1);
  if (metrics_enabled_) clients_in_use_metric_->Increment(-1);
  lock_guard<mutex> lock(client_map_lock_);
  client_map_.erase(client);
  *client_key = NULL;
}

void ClientCacheHelper::CloseConnections(const TNetworkAddress& address) {
  PerHostCache* cache;
  {
    lock_guard<mutex> lock(cache_lock_);
    PerHostCacheMap::iterator cache_it = per_host_caches_.find(address);
    if (cache_it == per_host_caches_.end()) return;
    cache = cache_it->second.get();
  }

  {
    VLOG(2) << "Invalidating all " << cache->clients.size() << " clients for: "
            << address;
    lock_guard<mutex> entry_lock(cache->lock);
    lock_guard<mutex> map_lock(client_map_lock_);
    for (ClientKey client_key: cache->clients) {
      ClientMap::iterator client_map_entry = client_map_.find(client_key);
      DCHECK(client_map_entry != client_map_.end());
      client_map_entry->second->Close();
    }
  }
}

string ClientCacheHelper::DebugString() {
  lock_guard<mutex> lock(cache_lock_);
  stringstream out;
  out << "ClientCacheHelper(#hosts=" << per_host_caches_.size()
      << " [";
  bool first = true;
  for (const PerHostCacheMap::value_type& cache: per_host_caches_) {
    lock_guard<mutex> host_cache_lock(cache.second->lock);
    if (!first) out << " ";
    out << cache.first << ":" << cache.second->clients.size();
    first = false;
  }
  out << "])";
  return out.str();
}

void ClientCacheHelper::TestShutdown() {
  vector<TNetworkAddress> addresses;
  {
    lock_guard<mutex> lock(cache_lock_);
    for (const PerHostCacheMap::value_type& cache_entry: per_host_caches_) {
      addresses.push_back(cache_entry.first);
    }
  }
  for (const TNetworkAddress& address: addresses) {
    CloseConnections(address);
  }
}

void ClientCacheHelper::InitMetrics(MetricGroup* metrics, const string& key_prefix) {
  DCHECK(metrics != NULL);
  // Not strictly needed if InitMetrics is called before any cache usage, but ensures that
  // metrics_enabled_ is published.
  lock_guard<mutex> lock(cache_lock_);
  stringstream count_ss;
  count_ss << key_prefix << ".client-cache.clients-in-use";
  clients_in_use_metric_ = metrics->AddGauge<int64_t>(count_ss.str(), 0);

  stringstream max_ss;
  max_ss << key_prefix << ".client-cache.total-clients";
  total_clients_metric_ = metrics->AddGauge<int64_t>(max_ss.str(), 0);
  metrics_enabled_ = true;
}

}
<commit_msg>IMPALA-4253: impala-server.backends.client-cache.total-clients shows negative value<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "runtime/client-cache.h"

#include <sstream>
#include <thrift/server/TServer.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/transport/TSocket.h>
#include <thrift/transport/TTransportUtils.h>
#include <memory>

#include "common/logging.h"
#include "util/container-util.h"
#include "util/network-util.h"
#include "rpc/thrift-util.h"
#include "gen-cpp/ImpalaInternalService.h"

#include "common/names.h"
using namespace apache::thrift;
using namespace apache::thrift::server;
using namespace apache::thrift::transport;
using namespace apache::thrift::protocol;

namespace impala {

Status ClientCacheHelper::GetClient(const TNetworkAddress& address,
    ClientFactory factory_method, ClientKey* client_key) {
  shared_ptr<PerHostCache> host_cache;
  {
    lock_guard<mutex> lock(cache_lock_);
    VLOG(2) << "GetClient(" << address << ")";
    shared_ptr<PerHostCache>* ptr = &per_host_caches_[address];
    if (ptr->get() == NULL) ptr->reset(new PerHostCache());
    host_cache = *ptr;
  }

  {
    lock_guard<mutex> lock(host_cache->lock);
    if (!host_cache->clients.empty()) {
      *client_key = host_cache->clients.front();
      VLOG(2) << "GetClient(): returning cached client for " << address;
      host_cache->clients.pop_front();
      if (metrics_enabled_) clients_in_use_metric_->Increment(1);
      return Status::OK();
    }
  }

  // Only get here if host_cache->clients.empty(). No need for the lock.
  RETURN_IF_ERROR(CreateClient(address, factory_method, client_key));
  if (metrics_enabled_) clients_in_use_metric_->Increment(1);
  return Status::OK();
}

Status ClientCacheHelper::ReopenClient(ClientFactory factory_method,
    ClientKey* client_key) {
  // Clients are not ordinarily removed from the cache completely (in the future, they may
  // be); this is the only method where a client may be deleted and replaced with another.
  shared_ptr<ThriftClientImpl> client_impl;
  ClientMap::iterator client;
  {
    lock_guard<mutex> lock(client_map_lock_);
    client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(1) << "ReopenClient(): re-creating client for " << client_impl->address();

  client_impl->Close();

  // TODO: Thrift TBufferedTransport cannot be re-opened after Close() because it does not
  // clean up internal buffers it reopens. To work around this issue, create a new client
  // instead.
  ClientKey old_client_key = *client_key;
  Status status = CreateClient(client_impl->address(), factory_method, client_key);
  // Only erase the existing client from the map if creation of the new one succeeded.
  // This helps to ensure the proper accounting of metrics in the presence of
  // re-connection failures (the original client should be released as usual).
  if (status.ok()) {
    // CreateClient() will increment total_clients_metric_ if succeed.
    if (metrics_enabled_) {
      total_clients_metric_->Increment(-1);
      DCHECK_GE(total_clients_metric_->value(), 0);
    }
    lock_guard<mutex> lock(client_map_lock_);
    client_map_.erase(client);
  } else {
    // Restore the client used before the failed re-opening attempt, so the caller can
    // properly release it.
    *client_key = old_client_key;
  }
  return status;
}

Status ClientCacheHelper::CreateClient(const TNetworkAddress& address,
    ClientFactory factory_method, ClientKey* client_key) {
  shared_ptr<ThriftClientImpl> client_impl(factory_method(address, client_key));
  VLOG(2) << "CreateClient(): creating new client for " << client_impl->address();
  Status status = client_impl->OpenWithRetry(num_tries_, wait_ms_);
  if (!status.ok()) {
    *client_key = NULL;
    return status;
  }
  // Set the TSocket's send and receive timeouts.
  client_impl->setRecvTimeout(recv_timeout_ms_);
  client_impl->setSendTimeout(send_timeout_ms_);

  // Because the client starts life 'checked out', we don't add it to its host cache.
  {
    lock_guard<mutex> lock(client_map_lock_);
    client_map_[*client_key] = client_impl;
  }

  if (metrics_enabled_) total_clients_metric_->Increment(1);
  return Status::OK();
}

void ClientCacheHelper::ReleaseClient(ClientKey* client_key) {
  DCHECK(*client_key != NULL) << "Trying to release NULL client";
  shared_ptr<ThriftClientImpl> client_impl;
  {
    lock_guard<mutex> lock(client_map_lock_);
    ClientMap::iterator client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(2) << "Releasing client for " << client_impl->address() << " back to cache";
  {
    lock_guard<mutex> lock(cache_lock_);
    PerHostCacheMap::iterator cache = per_host_caches_.find(client_impl->address());
    DCHECK(cache != per_host_caches_.end());
    lock_guard<mutex> entry_lock(cache->second->lock);
    cache->second->clients.push_back(*client_key);
  }
  if (metrics_enabled_) clients_in_use_metric_->Increment(-1);
  *client_key = NULL;
}

void ClientCacheHelper::DestroyClient(ClientKey* client_key) {
  DCHECK(*client_key != NULL) << "Trying to destroy NULL client";
  shared_ptr<ThriftClientImpl> client_impl;
  ClientMap::iterator client;
  {
    lock_guard<mutex> lock(client_map_lock_);
    client = client_map_.find(*client_key);
    DCHECK(client != client_map_.end());
    client_impl = client->second;
  }
  VLOG(1) << "Broken Connection, destroy client for " << client_impl->address();

  client_impl->Close();
  if (metrics_enabled_) total_clients_metric_->Increment(-1);
  if (metrics_enabled_) clients_in_use_metric_->Increment(-1);
  lock_guard<mutex> lock(client_map_lock_);
  client_map_.erase(client);
  *client_key = NULL;
}

void ClientCacheHelper::CloseConnections(const TNetworkAddress& address) {
  PerHostCache* cache;
  {
    lock_guard<mutex> lock(cache_lock_);
    PerHostCacheMap::iterator cache_it = per_host_caches_.find(address);
    if (cache_it == per_host_caches_.end()) return;
    cache = cache_it->second.get();
  }

  {
    VLOG(2) << "Invalidating all " << cache->clients.size() << " clients for: "
            << address;
    lock_guard<mutex> entry_lock(cache->lock);
    lock_guard<mutex> map_lock(client_map_lock_);
    for (ClientKey client_key: cache->clients) {
      ClientMap::iterator client_map_entry = client_map_.find(client_key);
      DCHECK(client_map_entry != client_map_.end());
      client_map_entry->second->Close();
    }
  }
}

string ClientCacheHelper::DebugString() {
  lock_guard<mutex> lock(cache_lock_);
  stringstream out;
  out << "ClientCacheHelper(#hosts=" << per_host_caches_.size()
      << " [";
  bool first = true;
  for (const PerHostCacheMap::value_type& cache: per_host_caches_) {
    lock_guard<mutex> host_cache_lock(cache.second->lock);
    if (!first) out << " ";
    out << cache.first << ":" << cache.second->clients.size();
    first = false;
  }
  out << "])";
  return out.str();
}

void ClientCacheHelper::TestShutdown() {
  vector<TNetworkAddress> addresses;
  {
    lock_guard<mutex> lock(cache_lock_);
    for (const PerHostCacheMap::value_type& cache_entry: per_host_caches_) {
      addresses.push_back(cache_entry.first);
    }
  }
  for (const TNetworkAddress& address: addresses) {
    CloseConnections(address);
  }
}

void ClientCacheHelper::InitMetrics(MetricGroup* metrics, const string& key_prefix) {
  DCHECK(metrics != NULL);
  // Not strictly needed if InitMetrics is called before any cache usage, but ensures that
  // metrics_enabled_ is published.
  lock_guard<mutex> lock(cache_lock_);
  stringstream count_ss;
  count_ss << key_prefix << ".client-cache.clients-in-use";
  clients_in_use_metric_ = metrics->AddGauge<int64_t>(count_ss.str(), 0);

  stringstream max_ss;
  max_ss << key_prefix << ".client-cache.total-clients";
  total_clients_metric_ = metrics->AddGauge<int64_t>(max_ss.str(), 0);
  metrics_enabled_ = true;
}

}
<|endoftext|>


<commit_before>/**
*  Copyright (C) 2016 3D Repo Ltd
*
*  This program is free software: you can redistribute it and/or modify
*  it under the terms of the GNU Affero General Public License as
*  published by the Free Software Foundation, either version 3 of the
*  License, or (at your option) any later version.
*
*  This program is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU Affero General Public License for more details.
*
*  You should have received a copy of the GNU Affero General Public License
*  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/**
* Allows Export functionality from 3D Repo World to asset bundle
*/

#include "repo_model_export_asset.h"
#include "../../../core/model/bson/repo_bson_factory.h"
#include "../../../lib/repo_log.h"

using namespace repo::manipulator::modelconvertor;
const static size_t SRC_MAX_VERTEX_LIMIT = 65535;
//Labels for multipart JSON descriptor files
const static std::string MP_LABEL_APPEARANCE = "appearance";
const static std::string MP_LABEL_MAT_DIFFUSE = "diffuseColor";
const static std::string MP_LABEL_MAT_EMISSIVE = "emissiveColor";
const static std::string MP_LABEL_MATERIAL = "material";
const static std::string MP_LABEL_MAPPING = "mapping";
const static std::string MP_LABEL_MAT_SHININESS = "shininess";
const static std::string MP_LABEL_MAT_SPECULAR = "specularColor";
const static std::string MP_LABEL_MAT_TRANSPARENCY = "transparency";
const static std::string MP_LABEL_MAX = "max";
const static std::string MP_LABEL_MAX_GEO_COUNT = "maxGeoCount";
const static std::string MP_LABEL_MIN = "min";
const static std::string MP_LABEL_NAME = "name";
const static std::string MP_LABEL_NUM_IDs = "numberOfIDs";
const static std::string MP_LABEL_USAGE = "usage";

const static std::string MP_LABEL_ASSETS = "assets";
const static std::string MP_LABEL_JSONS = "jsonFiles";
const static std::string MP_LABEL_OFFSET = "offset";
const static std::string MP_LABEL_DATABASE = "database";
const static std::string MP_LABEL_PROJECT = "project";

AssetModelExport::AssetModelExport(
	const repo::core::model::RepoScene *scene
	) : WebModelExport(scene)
{
	//Considering all newly imported models should have a stash graph, we only need to support stash graph?
	if (convertSuccess)
	{
		if (gType == repo::core::model::RepoScene::GraphType::OPTIMIZED)
		{
			convertSuccess = generateTreeRepresentation();
		}
		else  if (!(convertSuccess = !scene->getAllMeshes(repo::core::model::RepoScene::GraphType::DEFAULT).size()))
		{
			repoError << "Scene has no optimised graph and it is not a federation graph. SRC Exporter relies on this.";
		}
	}
	else
	{
		repoError << "Unable to export to SRC : Empty scene graph!";
	}
}

AssetModelExport::~AssetModelExport()
{
}

repo_web_buffers_t AssetModelExport::getAllFilesExportedAsBuffer() const
{
	return{ std::unordered_map<std::string, std::vector<uint8_t>>(), getJSONFilesAsBuffer() };
}

bool AssetModelExport::generateJSONMapping(
	const repo::core::model::MeshNode  *mesh,
	const repo::core::model::RepoScene *scene,
	const std::unordered_map<repo::lib::RepoUUID, std::vector<uint32_t>, repo::lib::RepoUUIDHasher> &splitMapping)
{
	bool success;
	if (success = mesh)
	{
		repo::lib::PropertyTree jsonTree;
		std::vector<repo_mesh_mapping_t> mappings = mesh->getMeshMapping();
		std::sort(mappings.begin(), mappings.end(),
			[](repo_mesh_mapping_t const& a, repo_mesh_mapping_t const& b) { return a.vertFrom < b.vertFrom; });

		size_t mappingLength = mappings.size();

		jsonTree.addToTree(MP_LABEL_NUM_IDs, mappingLength);
		jsonTree.addToTree(MP_LABEL_MAX_GEO_COUNT, mappingLength);

		std::vector<repo::lib::PropertyTree> mappingTrees;
		std::string meshUID = mesh->getUniqueID().toString();
		//Could get the mesh split function to pass a mapping out so we don't do this again.
		for (size_t i = 0; i < mappingLength; ++i)
		{
			auto mapIt = splitMapping.find(mappings[i].mesh_id);
			if (mapIt != splitMapping.end())
			{
				for (const uint32_t &subMeshID : mapIt->second)
				{
					repo::lib::PropertyTree mappingTree;

					mappingTree.addToTree(MP_LABEL_NAME, mappings[i].mesh_id.toString());
					mappingTree.addToTree(MP_LABEL_MIN, mappings[i].min);
					mappingTree.addToTree(MP_LABEL_MAX, mappings[i].max);
					std::vector<std::string> usageArr = { meshUID + "_" + std::to_string(subMeshID) };
					mappingTree.addToTree(MP_LABEL_USAGE, usageArr);
					mappingTrees.push_back(mappingTree);
				}
			}
			else
			{
				repoError << "Failed to find split mapping for id: " << mappings[i].mesh_id;
			}
		}

		jsonTree.addArrayObjects(MP_LABEL_MAPPING, mappingTrees);

		std::string jsonFileName = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/" + mesh->getUniqueID().toString() + "_unity.json.mpc";

		jsonTrees[jsonFileName] = jsonTree;
	}
	else
	{
		repoError << "Unable to generate JSON file mapping : null pointer to mesh!";
	}

	return success;
}

bool AssetModelExport::generateTreeRepresentation(
	)
{
	bool success;
	if (success = scene->hasRoot(gType))
	{
		auto meshes = scene->getAllMeshes(gType);

		std::vector<std::string> assetFiles, jsons;
		for (const repo::core::model::RepoNode* node : meshes)
		{
			auto mesh = dynamic_cast<const repo::core::model::MeshNode*>(node);
			if (!mesh)
			{
				repoError << "Failed to cast a Repo Node of type mesh into a MeshNode(" << node->getUniqueID() << "). Skipping...";
			}
			std::string textureID = scene->getTextureIDForMesh(gType, mesh->getSharedID());

			repo::manipulator::modelutility::MeshMapReorganiser *reSplitter =
				new repo::manipulator::modelutility::MeshMapReorganiser(mesh, SRC_MAX_VERTEX_LIMIT);
			reorganisedMeshes.push_back(std::make_shared<repo::core::model::MeshNode>(reSplitter->getRemappedMesh()));

			if (success = !(reorganisedMeshes.back()->isEmpty()))
			{
				serialisedFaceBuf.push_back(reSplitter->getSerialisedFaces());
				idMapBuf.push_back(reSplitter->getIDMapArrays());
				meshMappings.push_back(reSplitter->getMappingsPerSubMesh());
				std::unordered_map<repo::lib::RepoUUID, std::vector<uint32_t>, repo::lib::RepoUUIDHasher> splitMapping = reSplitter->getSplitMapping();

				std::string fNamePrefix = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/" + mesh->getUniqueID().toString();
				assetFiles.push_back(fNamePrefix + ".unity3d");
				jsons.push_back(fNamePrefix + "_unity.json.mpc");

				success &= generateJSONMapping(mesh, scene, reSplitter->getSplitMapping());
				delete reSplitter;
			}
			else
			{
				repoError << "Failed to generate a remapped mesh for mesh with ID : " << mesh->getUniqueID();
				break;
			}
		}

		std::string assetListFile = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/revision/" + scene->getRevisionID().toString() + "/unityAssets.json";
		repo::lib::PropertyTree assetListTree;
		assetListTree.addToTree(MP_LABEL_ASSETS, assetFiles);
		assetListTree.addToTree(MP_LABEL_JSONS, jsons);
		assetListTree.addToTree(MP_LABEL_OFFSET, scene->getWorldOffset());
		assetListTree.addToTree(MP_LABEL_DATABASE, scene->getDatabaseName());
		assetListTree.addToTree(MP_LABEL_PROJECT, scene->getProjectName());
		jsonTrees[assetListFile] = assetListTree;
	}

	return success;
}<commit_msg>ISSUE #189 write sharedID into json file<commit_after>/**
*  Copyright (C) 2016 3D Repo Ltd
*
*  This program is free software: you can redistribute it and/or modify
*  it under the terms of the GNU Affero General Public License as
*  published by the Free Software Foundation, either version 3 of the
*  License, or (at your option) any later version.
*
*  This program is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU Affero General Public License for more details.
*
*  You should have received a copy of the GNU Affero General Public License
*  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/**
* Allows Export functionality from 3D Repo World to asset bundle
*/

#include "repo_model_export_asset.h"
#include "../../../core/model/bson/repo_bson_factory.h"
#include "../../../lib/repo_log.h"

using namespace repo::manipulator::modelconvertor;
const static size_t SRC_MAX_VERTEX_LIMIT = 65535;
//Labels for multipart JSON descriptor files
const static std::string MP_LABEL_APPEARANCE = "appearance";
const static std::string MP_LABEL_MAT_DIFFUSE = "diffuseColor";
const static std::string MP_LABEL_MAT_EMISSIVE = "emissiveColor";
const static std::string MP_LABEL_MATERIAL = "material";
const static std::string MP_LABEL_MAPPING = "mapping";
const static std::string MP_LABEL_MAT_SHININESS = "shininess";
const static std::string MP_LABEL_MAT_SPECULAR = "specularColor";
const static std::string MP_LABEL_MAT_TRANSPARENCY = "transparency";
const static std::string MP_LABEL_MAX = "max";
const static std::string MP_LABEL_MAX_GEO_COUNT = "maxGeoCount";
const static std::string MP_LABEL_MIN = "min";
const static std::string MP_LABEL_NAME = "name";
const static std::string MP_LABEL_NUM_IDs = "numberOfIDs";
const static std::string MP_LABEL_USAGE = "usage";

const static std::string MP_LABEL_ASSETS = "assets";
const static std::string MP_LABEL_SHARED = "sharedID";
const static std::string MP_LABEL_JSONS = "jsonFiles";
const static std::string MP_LABEL_OFFSET = "offset";
const static std::string MP_LABEL_DATABASE = "database";
const static std::string MP_LABEL_PROJECT = "project";

AssetModelExport::AssetModelExport(
	const repo::core::model::RepoScene *scene
	) : WebModelExport(scene)
{
	//Considering all newly imported models should have a stash graph, we only need to support stash graph?
	if (convertSuccess)
	{
		if (gType == repo::core::model::RepoScene::GraphType::OPTIMIZED)
		{
			convertSuccess = generateTreeRepresentation();
		}
		else  if (!(convertSuccess = !scene->getAllMeshes(repo::core::model::RepoScene::GraphType::DEFAULT).size()))
		{
			repoError << "Scene has no optimised graph and it is not a federation graph. SRC Exporter relies on this.";
		}
	}
	else
	{
		repoError << "Unable to export to SRC : Empty scene graph!";
	}
}

AssetModelExport::~AssetModelExport()
{
}

repo_web_buffers_t AssetModelExport::getAllFilesExportedAsBuffer() const
{
	return{ std::unordered_map<std::string, std::vector<uint8_t>>(), getJSONFilesAsBuffer() };
}

bool AssetModelExport::generateJSONMapping(
	const repo::core::model::MeshNode  *mesh,
	const repo::core::model::RepoScene *scene,
	const std::unordered_map<repo::lib::RepoUUID, std::vector<uint32_t>, repo::lib::RepoUUIDHasher> &splitMapping)
{
	bool success;
	if (success = mesh)
	{
		repo::lib::PropertyTree jsonTree;
		std::vector<repo_mesh_mapping_t> mappings = mesh->getMeshMapping();
		std::sort(mappings.begin(), mappings.end(),
			[](repo_mesh_mapping_t const& a, repo_mesh_mapping_t const& b) { return a.vertFrom < b.vertFrom; });

		size_t mappingLength = mappings.size();

		jsonTree.addToTree(MP_LABEL_NUM_IDs, mappingLength);
		jsonTree.addToTree(MP_LABEL_MAX_GEO_COUNT, mappingLength);

		std::vector<repo::lib::PropertyTree> mappingTrees;
		std::string meshUID = mesh->getUniqueID().toString();
		//Could get the mesh split function to pass a mapping out so we don't do this again.
		for (size_t i = 0; i < mappingLength; ++i)
		{
			auto mapIt = splitMapping.find(mappings[i].mesh_id);
			if (mapIt != splitMapping.end())
			{
				for (const uint32_t &subMeshID : mapIt->second)
				{
					repo::lib::PropertyTree mappingTree;

					mappingTree.addToTree(MP_LABEL_NAME, mappings[i].mesh_id.toString());

					repo::lib::RepoUUID id(mappings[i].mesh_id.toString());
					auto mesh = scene->getNodeByUniqueID(repo::core::model::RepoScene::GraphType::DEFAULT, id);
					if (mesh)
						mappingTree.addToTree(MP_LABEL_SHARED, mesh->getSharedID().toString());
					mappingTree.addToTree(MP_LABEL_MIN, mappings[i].min);
					mappingTree.addToTree(MP_LABEL_MAX, mappings[i].max);
					std::vector<std::string> usageArr = { meshUID + "_" + std::to_string(subMeshID) };
					mappingTree.addToTree(MP_LABEL_USAGE, usageArr);
					mappingTrees.push_back(mappingTree);
				}
			}
			else
			{
				repoError << "Failed to find split mapping for id: " << mappings[i].mesh_id;
			}
		}

		jsonTree.addArrayObjects(MP_LABEL_MAPPING, mappingTrees);

		std::string jsonFileName = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/" + mesh->getUniqueID().toString() + "_unity.json.mpc";

		jsonTrees[jsonFileName] = jsonTree;
	}
	else
	{
		repoError << "Unable to generate JSON file mapping : null pointer to mesh!";
	}

	return success;
}

bool AssetModelExport::generateTreeRepresentation(
	)
{
	bool success;
	if (success = scene->hasRoot(gType))
	{
		auto meshes = scene->getAllMeshes(gType);

		std::vector<std::string> assetFiles, jsons;
		for (const repo::core::model::RepoNode* node : meshes)
		{
			auto mesh = dynamic_cast<const repo::core::model::MeshNode*>(node);
			if (!mesh)
			{
				repoError << "Failed to cast a Repo Node of type mesh into a MeshNode(" << node->getUniqueID() << "). Skipping...";
			}
			std::string textureID = scene->getTextureIDForMesh(gType, mesh->getSharedID());

			repo::manipulator::modelutility::MeshMapReorganiser *reSplitter =
				new repo::manipulator::modelutility::MeshMapReorganiser(mesh, SRC_MAX_VERTEX_LIMIT);
			reorganisedMeshes.push_back(std::make_shared<repo::core::model::MeshNode>(reSplitter->getRemappedMesh()));

			if (success = !(reorganisedMeshes.back()->isEmpty()))
			{
				serialisedFaceBuf.push_back(reSplitter->getSerialisedFaces());
				idMapBuf.push_back(reSplitter->getIDMapArrays());
				meshMappings.push_back(reSplitter->getMappingsPerSubMesh());
				std::unordered_map<repo::lib::RepoUUID, std::vector<uint32_t>, repo::lib::RepoUUIDHasher> splitMapping = reSplitter->getSplitMapping();

				std::string fNamePrefix = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/" + mesh->getUniqueID().toString();
				assetFiles.push_back(fNamePrefix + ".unity3d");
				jsons.push_back(fNamePrefix + "_unity.json.mpc");

				success &= generateJSONMapping(mesh, scene, reSplitter->getSplitMapping());
				delete reSplitter;
			}
			else
			{
				repoError << "Failed to generate a remapped mesh for mesh with ID : " << mesh->getUniqueID();
				break;
			}
		}

		std::string assetListFile = "/" + scene->getDatabaseName() + "/" + scene->getProjectName() + "/revision/" + scene->getRevisionID().toString() + "/unityAssets.json";
		repo::lib::PropertyTree assetListTree;
		assetListTree.addToTree(MP_LABEL_ASSETS, assetFiles);
		assetListTree.addToTree(MP_LABEL_JSONS, jsons);
		assetListTree.addToTree(MP_LABEL_OFFSET, scene->getWorldOffset());
		assetListTree.addToTree(MP_LABEL_DATABASE, scene->getDatabaseName());
		assetListTree.addToTree(MP_LABEL_PROJECT, scene->getProjectName());
		jsonTrees[assetListFile] = assetListTree;
	}

	return success;
}<|endoftext|>


<commit_before><commit_msg>works<commit_after><|endoftext|>


<commit_before>#include "numpy.hpp"

namespace python {

  void numpy::module() {
	// numpy requires this
	import_array();
  }

  boost::python::object numpy::ndarray(double* data, shape_type shape) {
	using namespace boost::python;

	PyObject* obj = PyArray_New(&PyArray_Type, shape.size(),
								const_cast<npy_intp*>(shape.begin()), NPY_DOUBLE, // data type
								NULL, data, // data pointer
								0, NPY_ARRAY_CARRAY, // NPY_ARRAY_CARRAY_RO for readonly
								NULL);
	handle<> array( obj );
	return object(array);
  }

}
<commit_msg>pouf compilation with python3<commit_after>#include "numpy.hpp"

namespace python {

  void numpy::module() {
	// numpy requires this
    [] {
      import_array();
    }();
  }

  boost::python::object numpy::ndarray(double* data, shape_type shape) {
	using namespace boost::python;

	PyObject* obj = PyArray_New(&PyArray_Type, shape.size(),
								const_cast<npy_intp*>(shape.begin()), NPY_DOUBLE, // data type
								NULL, data, // data pointer
								0, NPY_ARRAY_CARRAY, // NPY_ARRAY_CARRAY_RO for readonly
								NULL);
	handle<> array( obj );
	return object(array);
  }

}
<|endoftext|>


<commit_before>// Copyright 2008 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
  An example running of this program:

  ./lda           \
  --num_topics 2 \
  --alpha 0.1    \
  --beta 0.01                                           \
  --training_data_file ./testdata/test_data.txt \
  --model_file /tmp/lda_model.txt                       \
  --burn_in_iterations 100                              \
  --total_iterations 150
*/

#include <fstream>
#include <set>
#include <sstream>
#include <string>
#include <map>

#include "common.h"
#include "document.h"
#include "model.h"
#include "accumulative_model.h"
#include "sampler.h"
#include "cmd_flags.h"

namespace learning_lda {

using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::set;
using std::map;

int LoadAndInitTrainingCorpus(std::vector<string> &lines,
                              int num_topics,
                              LDACorpus* corpus,
                              map<string, int>* word_index_map) {
  corpus->clear();
  word_index_map->clear();
  string line;
  for (int i=0; i < lines.size(); i++){  // Each line is a training document.
    string line = lines[i];
    if (line.size() > 0 &&      // Skip empty lines.
        line[0] != '\r' &&      // Skip empty lines.
        line[0] != '\n' &&      // Skip empty lines.
        line[0] != '#') {       // Skip comment lines.
      istringstream ss(line);
      DocumentWordTopicsPB document;
      string word;
      int count;
      while (ss >> word >> count) {  // Load and init a document.
        vector<int32> topics;
        for (int i = 0; i < count; ++i) {
          topics.push_back(RandInt(num_topics));
        }
        int word_index;
        map<string, int>::const_iterator iter = word_index_map->find(word);
        if (iter == word_index_map->end()) {
          word_index = word_index_map->size();
          (*word_index_map)[word] = word_index;
        } else {
          word_index = iter->second;
        }
        document.add_wordtopics(word, word_index, topics);
      }
      corpus->push_back(new LDADocument(document, num_topics));
    }
  }
  return corpus->size();

}

int LoadAndInitTrainingCorpus(const string& corpus_file,
                              int num_topics,
                              LDACorpus* corpus,
                              map<string, int>* word_index_map) {
  ifstream fin(corpus_file.c_str());
  string line;
  vector<string> lines;
  while (getline(fin, line)) {  // Each line is a training document.
    if (line.size() > 0 &&      // Skip empty lines.
        line[0] != '\r' &&      // Skip empty lines.
        line[0] != '\n' &&      // Skip empty lines.
        line[0] != '#') {       // Skip comment lines.
      lines.push_back(line);
    }
  }
  return LoadAndInitTrainingCorpus(lines,num_topics,corpus,word_index_map);
}

void FreeCorpus(LDACorpus* corpus) {
  for (list<LDADocument*>::iterator iter = corpus->begin();
       iter != corpus->end();
       ++iter) {
    if (*iter != NULL) {
      delete *iter;
      *iter = NULL;
    }
  }
}

}  // namespace learning_lda

int main(int argc, char** argv) {
  using learning_lda::LDACorpus;
  using learning_lda::LDAModel;
  using learning_lda::LDAAccumulativeModel;
  using learning_lda::LDASampler;
  using learning_lda::LDADocument;
  using learning_lda::LoadAndInitTrainingCorpus;
  using learning_lda::LDACmdLineFlags;
  using std::list;

  LDACmdLineFlags flags;
  flags.ParseCmdFlags(argc, argv);
  if (!flags.CheckTrainingValidity()) {
    return -1;
  }
  srand(time(NULL));
  LDACorpus corpus;
  map<string, int> word_index_map;
  CHECK_GT(LoadAndInitTrainingCorpus(flags.training_data_file_,
                                     flags.num_topics_,
                                     &corpus, &word_index_map), 0);
  LDAModel model(flags.num_topics_, word_index_map);
  LDAAccumulativeModel accum_model(flags.num_topics_, word_index_map.size());
  LDASampler sampler(flags.alpha_, flags.beta_, &model, &accum_model);

  sampler.InitModelGivenTopics(corpus);

  for (int iter = 0; iter < flags.total_iterations_; ++iter) {
    std::cout << "Iteration " << iter << " ...\n";
    if (flags.compute_likelihood_ == "true") {
      double loglikelihood = 0;
      for (list<LDADocument*>::const_iterator iterator = corpus.begin();
           iterator != corpus.end();
           ++iterator) {
        loglikelihood += sampler.LogLikelihood(*iterator);
      }
      std::cout << "Loglikelihood: " << loglikelihood << std::endl;
    }
    sampler.DoIteration(&corpus, true, iter < flags.burn_in_iterations_);
  }
  accum_model.AverageModel(
      flags.total_iterations_ - flags.burn_in_iterations_);

  FreeCorpus(&corpus);

  std::ofstream fout(flags.model_file_.c_str());
  accum_model.AppendAsString(word_index_map, fout);

  return 0;
}
<commit_msg>More refactoring<commit_after>// Copyright 2008 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
  An example running of this program:

  ./lda           \
  --num_topics 2 \
  --alpha 0.1    \
  --beta 0.01                                           \
  --training_data_file ./testdata/test_data.txt \
  --model_file /tmp/lda_model.txt                       \
  --burn_in_iterations 100                              \
  --total_iterations 150
*/

#include <fstream>
#include <set>
#include <sstream>
#include <string>
#include <map>

#include "common.h"
#include "document.h"
#include "model.h"
#include "accumulative_model.h"
#include "sampler.h"
#include "cmd_flags.h"

namespace learning_lda {

using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::set;
using std::map;

int LoadAndInitTrainingCorpus(std::vector<string> &lines,
                              int num_topics,
                              LDACorpus* corpus,
                              map<string, int>* word_index_map) {
  corpus->clear();
  word_index_map->clear();
  string line;
  for (int i=0; i < lines.size(); i++){  // Each line is a training document.
    string line = lines[i];
    if (line.size() > 0 &&      // Skip empty lines.
        line[0] != '\r' &&      // Skip empty lines.
        line[0] != '\n' &&      // Skip empty lines.
        line[0] != '#') {       // Skip comment lines.
      istringstream ss(line);
      DocumentWordTopicsPB document;
      string word;
      int count;
      while (ss >> word >> count) {  // Load and init a document.
        vector<int32> topics;
        for (int i = 0; i < count; ++i) {
          topics.push_back(RandInt(num_topics));
        }
        int word_index;
        map<string, int>::const_iterator iter = word_index_map->find(word);
        if (iter == word_index_map->end()) {
          word_index = word_index_map->size();
          (*word_index_map)[word] = word_index;
        } else {
          word_index = iter->second;
        }
        document.add_wordtopics(word, word_index, topics);
      }
      corpus->push_back(new LDADocument(document, num_topics));
    }
  }
  return corpus->size();

}

int LoadAndInitTrainingCorpus(const string& corpus_file,
                              int num_topics,
                              LDACorpus* corpus,
                              map<string, int>* word_index_map) {
  ifstream fin(corpus_file.c_str());
  string line;
  vector<string> lines;
  while (getline(fin, line)) {  // Each line is a training document.
    if (line.size() > 0 &&      // Skip empty lines.
        line[0] != '\r' &&      // Skip empty lines.
        line[0] != '\n' &&      // Skip empty lines.
        line[0] != '#') {       // Skip comment lines.
      lines.push_back(line);
    }
  }
  return LoadAndInitTrainingCorpus(lines,num_topics,corpus,word_index_map);
}

void FreeCorpus(LDACorpus* corpus) {
  for (list<LDADocument*>::iterator iter = corpus->begin();
       iter != corpus->end();
       ++iter) {
    if (*iter != NULL) {
      delete *iter;
      *iter = NULL;
    }
  }
}

LDAAccumulativeModel* LearnFromFile(const string& corpus_file,int num_topics,map<string, int>* word_index_map, int total_iterations, int burn_in_iterations, double alpha, double beta)
{
  using learning_lda::LDACorpus;
  using learning_lda::LDAModel;
  using learning_lda::LDAAccumulativeModel;
  using learning_lda::LDASampler;
  using learning_lda::LDADocument;
  using learning_lda::LoadAndInitTrainingCorpus;
  using std::list;
  LDACorpus corpus;
  int ret = LoadAndInitTrainingCorpus(corpus_file,
                                     num_topics,
                                     &corpus, word_index_map);

  LDAAccumulativeModel *accum_model = new LDAAccumulativeModel(num_topics, (*word_index_map).size());
  LDAModel model(num_topics, *word_index_map);
  LDASampler sampler(alpha, beta, &model, accum_model);
  sampler.InitModelGivenTopics(corpus);

  for (int iter = 0; iter < total_iterations; ++iter) {
    std::cout << "Iteration " << iter << " ...\n";
    double loglikelihood = 0;
    for (list<LDADocument*>::const_iterator iterator = corpus.begin();
        iterator != corpus.end();
        ++iterator) {
      loglikelihood += sampler.LogLikelihood(*iterator);
    }
    std::cout << "Loglikelihood: " << loglikelihood << std::endl;
    sampler.DoIteration(&corpus, true, iter < burn_in_iterations);
    std::cout << "DONEa" << "\n";
  }
  std::cout << "DONE1" << "\n";
  (*accum_model).AverageModel(total_iterations - burn_in_iterations);
  std::cout << "DONE1" << "\n";
  FreeCorpus(&corpus);
  std::cout << "DONE1" << "\n";
  return accum_model;
}

}  // namespace learning_lda

int main(int argc, char** argv) {

  using learning_lda::LDACmdLineFlags;
  using learning_lda::LearnFromFile;
  using learning_lda::LDAAccumulativeModel;
  map<string, int> word_index_map;
  srand(time(NULL));

  LDACmdLineFlags flags;
  flags.ParseCmdFlags(argc, argv);

  if (!flags.CheckTrainingValidity()) {
    return -1;
  }

  LDAAccumulativeModel *accum_model = LearnFromFile(flags.training_data_file_,flags.num_topics_,&word_index_map, flags.total_iterations_, flags.burn_in_iterations_, flags.alpha_, flags.beta_);

  std::cout << "OUTPUTTING TO " << flags.model_file_ << "\n";
  std::ofstream fout(flags.model_file_.c_str());
  (*accum_model).AppendAsString(word_index_map, fout);

  return 0;
}
<|endoftext|>


<commit_before>// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/instruction/replace_type_function.h>
#include <vespa/eval/instruction/fast_rename_optimizer.h>
#include <vespa/eval/eval/test/tensor_model.hpp>
#include <vespa/eval/eval/test/eval_fixture.h>

#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::eval::test;
using namespace vespalib::eval::tensor_function;

const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get();

EvalFixture::ParamRepo make_params() {
    return EvalFixture::ParamRepo()
        .add("x5", spec({x(5)}, N()))
        .add("x5f", spec(float_cells({x(5)}), N()))
        .add("x_m", spec({x({"a", "b", "c"})}, N()))
        .add("xy_mm", spec({x({"a", "b", "c"}),y({"d","e"})}, N()))
        .add("x5y3z_m", spec({x(5),y(3),z({"a","b"})}, N()))
        .add("x5yz_m", spec({x(5),y({"a","b"}),z({"d","e"})}, N()))
        .add("x5y3", spec({x(5),y(3)}, N()));
}
EvalFixture::ParamRepo param_repo = make_params();

void verify_optimized(const vespalib::string &expr) {
    EvalFixture fixture(prod_factory, expr, param_repo, true);
    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
    auto info = fixture.find_all<ReplaceTypeFunction>();
    EXPECT_EQUAL(info.size(), 1u);
}

void verify_not_optimized(const vespalib::string &expr) {
    EvalFixture fixture(prod_factory, expr, param_repo, true);
    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
    auto info = fixture.find_all<ReplaceTypeFunction>();
    EXPECT_TRUE(info.empty());
}

TEST("require that non-transposing dense renames are optimized") {
    TEST_DO(verify_optimized("rename(x5,x,y)"));
    TEST_DO(verify_optimized("rename(x5,x,a)"));
    TEST_DO(verify_optimized("rename(x5y3,y,z)"));
    TEST_DO(verify_optimized("rename(x5y3,x,a)"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(a,b))"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(z,zz))"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(y,z))"));
    TEST_DO(verify_optimized("rename(x5y3,(y,x),(b,a))"));
}

TEST("require that transposing dense renames are not optimized") {
    TEST_DO(verify_not_optimized("rename(x5y3,x,z)"));
    TEST_DO(verify_not_optimized("rename(x5y3,y,a)"));
    TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(y,x))"));
    TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(b,a))"));
    TEST_DO(verify_not_optimized("rename(x5y3,(y,x),(a,b))"));
}

TEST("require that non-dense renames may be optimized") {
    TEST_DO(verify_optimized("rename(x_m,x,y)"));
    TEST_DO(verify_optimized("rename(xy_mm,(x,y),(a,b))"));
    TEST_DO(verify_optimized("rename(xy_mm,(x,y),(y,z))"));
    TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(b,a))"));
    TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(y,x))"));

    TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))"));
    TEST_DO(verify_not_optimized("rename(x5y3z_m,(y),(a))"));
    TEST_DO(verify_not_optimized("rename(x5y3z_m,(x,z),(z,x))"));

    TEST_DO(verify_optimized("rename(x5yz_m,(x,y),(y,x))"));
    TEST_DO(verify_optimized("rename(x5yz_m,(x,y,z),(c,a,b))"));
    TEST_DO(verify_optimized("rename(x5yz_m,(y,z),(a,b))"));
    TEST_DO(verify_not_optimized("rename(x5yz_m,(z),(a))"));
    TEST_DO(verify_not_optimized("rename(x5yz_m,(y,z),(z,y))"));
}

TEST("require that chained optimized renames are compacted into a single operation") {
    TEST_DO(verify_optimized("rename(rename(x5,x,y),y,z)"));
}

TEST("require that optimization works for float cells") {
    TEST_DO(verify_optimized("rename(x5f,x,y)"));
}

bool is_stable(const vespalib::string &from_spec, const vespalib::string &to_spec,
               const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to)
{
    auto from_type = ValueType::from_spec(from_spec);
    auto to_type = ValueType::from_spec(to_spec);
    return FastRenameOptimizer::is_stable_rename(from_type, to_type, from, to);
}

TEST("require that rename is stable if dimension order is preserved") {
    EXPECT_TRUE(is_stable("tensor(a{},b{})", "tensor(a{},c{})", {"b"}, {"c"}));
    EXPECT_TRUE(is_stable("tensor(c[3],d[5])", "tensor(c[3],e[5])", {"d"}, {"e"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a{},b{},c[3],e[5])", {"d"}, {"e"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(e{},f{},g[3],h[5])", {"a", "b", "c", "d"}, {"e", "f", "g", "h"}));
}

TEST("require that rename is unstable if nontrivial indexed dimensions change order") {
    EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(d[5],e[3])", {"c"}, {"e"}));
    EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(c[5],d[3])", {"c", "d"}, {"d", "c"}));
}

TEST("require that rename is unstable if mapped dimensions change order") {
    EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(b{},c{})", {"a"}, {"c"}));
    EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(a{},b{})", {"a", "b"}, {"b", "a"}));
}

TEST("require that rename can be stable if indexed and mapped dimensions change order") {
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a[3],b[5],c{},d{})", {"a", "b", "c", "d"}, {"c", "d", "a", "b"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(c[3],d[5],e{},f{})", {"a", "b"}, {"e", "f"}));
}

TEST("require that rename can be stable if trivial dimension is moved") {
    EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},bb[1],c[3])", {"a"}, {"bb"}));
    EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},c[3],cc[1])", {"a"}, {"cc"}));
}

TEST_MAIN() { TEST_RUN_ALL(); }
<commit_msg>use GenSpec in fast_rename_optimizer_test<commit_after>// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/eval/eval/tensor_function.h>
#include <vespa/eval/instruction/replace_type_function.h>
#include <vespa/eval/instruction/fast_rename_optimizer.h>
#include <vespa/eval/eval/test/gen_spec.h>
#include <vespa/eval/eval/test/eval_fixture.h>

#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>

using namespace vespalib;
using namespace vespalib::eval;
using namespace vespalib::eval::test;
using namespace vespalib::eval::tensor_function;

const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get();

EvalFixture::ParamRepo make_params() {
    return EvalFixture::ParamRepo()
        .add("x5", GenSpec().idx("x", 5).gen())
        .add("x5f", GenSpec().idx("x", 5).cells_float().gen())
        .add("x_m", GenSpec().map("x", {"a", "b", "c"}).gen())
        .add("xy_mm", GenSpec().map("x", {"a", "b", "c"}).map("y", {"d","e"}).gen())
        .add("x5y3z_m", GenSpec().idx("x", 5).idx("y", 3).map("z", {"a","b"}).gen())
        .add("x5yz_m", GenSpec().idx("x", 5).map("y", {"a","b"}).map("z", {"d","e"}).gen())
        .add("x5y3", GenSpec().idx("x", 5).idx("y", 3).gen());
}
EvalFixture::ParamRepo param_repo = make_params();

void verify_optimized(const vespalib::string &expr) {
    EvalFixture fixture(prod_factory, expr, param_repo, true);
    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
    auto info = fixture.find_all<ReplaceTypeFunction>();
    EXPECT_EQUAL(info.size(), 1u);
}

void verify_not_optimized(const vespalib::string &expr) {
    EvalFixture fixture(prod_factory, expr, param_repo, true);
    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
    auto info = fixture.find_all<ReplaceTypeFunction>();
    EXPECT_TRUE(info.empty());
}

TEST("require that non-transposing dense renames are optimized") {
    TEST_DO(verify_optimized("rename(x5,x,y)"));
    TEST_DO(verify_optimized("rename(x5,x,a)"));
    TEST_DO(verify_optimized("rename(x5y3,y,z)"));
    TEST_DO(verify_optimized("rename(x5y3,x,a)"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(a,b))"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(z,zz))"));
    TEST_DO(verify_optimized("rename(x5y3,(x,y),(y,z))"));
    TEST_DO(verify_optimized("rename(x5y3,(y,x),(b,a))"));
}

TEST("require that transposing dense renames are not optimized") {
    TEST_DO(verify_not_optimized("rename(x5y3,x,z)"));
    TEST_DO(verify_not_optimized("rename(x5y3,y,a)"));
    TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(y,x))"));
    TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(b,a))"));
    TEST_DO(verify_not_optimized("rename(x5y3,(y,x),(a,b))"));
}

TEST("require that non-dense renames may be optimized") {
    TEST_DO(verify_optimized("rename(x_m,x,y)"));
    TEST_DO(verify_optimized("rename(xy_mm,(x,y),(a,b))"));
    TEST_DO(verify_optimized("rename(xy_mm,(x,y),(y,z))"));
    TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(b,a))"));
    TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(y,x))"));

    TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))"));
    TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))"));
    TEST_DO(verify_not_optimized("rename(x5y3z_m,(y),(a))"));
    TEST_DO(verify_not_optimized("rename(x5y3z_m,(x,z),(z,x))"));

    TEST_DO(verify_optimized("rename(x5yz_m,(x,y),(y,x))"));
    TEST_DO(verify_optimized("rename(x5yz_m,(x,y,z),(c,a,b))"));
    TEST_DO(verify_optimized("rename(x5yz_m,(y,z),(a,b))"));
    TEST_DO(verify_not_optimized("rename(x5yz_m,(z),(a))"));
    TEST_DO(verify_not_optimized("rename(x5yz_m,(y,z),(z,y))"));
}

TEST("require that chained optimized renames are compacted into a single operation") {
    TEST_DO(verify_optimized("rename(rename(x5,x,y),y,z)"));
}

TEST("require that optimization works for float cells") {
    TEST_DO(verify_optimized("rename(x5f,x,y)"));
}

bool is_stable(const vespalib::string &from_spec, const vespalib::string &to_spec,
               const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to)
{
    auto from_type = ValueType::from_spec(from_spec);
    auto to_type = ValueType::from_spec(to_spec);
    return FastRenameOptimizer::is_stable_rename(from_type, to_type, from, to);
}

TEST("require that rename is stable if dimension order is preserved") {
    EXPECT_TRUE(is_stable("tensor(a{},b{})", "tensor(a{},c{})", {"b"}, {"c"}));
    EXPECT_TRUE(is_stable("tensor(c[3],d[5])", "tensor(c[3],e[5])", {"d"}, {"e"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a{},b{},c[3],e[5])", {"d"}, {"e"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(e{},f{},g[3],h[5])", {"a", "b", "c", "d"}, {"e", "f", "g", "h"}));
}

TEST("require that rename is unstable if nontrivial indexed dimensions change order") {
    EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(d[5],e[3])", {"c"}, {"e"}));
    EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(c[5],d[3])", {"c", "d"}, {"d", "c"}));
}

TEST("require that rename is unstable if mapped dimensions change order") {
    EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(b{},c{})", {"a"}, {"c"}));
    EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(a{},b{})", {"a", "b"}, {"b", "a"}));
}

TEST("require that rename can be stable if indexed and mapped dimensions change order") {
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a[3],b[5],c{},d{})", {"a", "b", "c", "d"}, {"c", "d", "a", "b"}));
    EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(c[3],d[5],e{},f{})", {"a", "b"}, {"e", "f"}));
}

TEST("require that rename can be stable if trivial dimension is moved") {
    EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},bb[1],c[3])", {"a"}, {"bb"}));
    EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},c[3],cc[1])", {"a"}, {"cc"}));
}

TEST_MAIN() { TEST_RUN_ALL(); }
<|endoftext|>


<commit_before>/*
    Copyright (c) 2013 250bpm s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom
    the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.
*/

#ifndef NN_HPP_INCLUDED
#define NN_HPP_INCLUDED

#include <nanomsg/nn.h>

#include <cassert>
#include <cstring>
#include <algorithm>
#include <exception>

#if defined __GNUC__
#define nn_slow(x) __builtin_expect ((x), 0)
#else
#define nn_slow(x) (x)
#endif

namespace nn
{

    class exception : public std::exception
    {
    public:

        exception () : err (nn_errno ()) {}

        virtual const char *what () const throw ()
        {
            return nn_strerror (err);
        }

        int num () const
        {
            return err;
        }

    private:

        int err;
    };

    inline void version (int *major, int *minor, int *patch)
    {
        nn_version (major, minor, patch);
    }

    inline void *allocmsg (size_t size, int type)
    {
        void *msg = nn_allocmsg (size, type);
        if (nn_slow (!msg))
            throw nn::exception ();
    }

    inline int freemsg (void *msg)
    {
        int rc = nn_freemsg (msg);
        if (nn_slow (rc != 0))
            throw nn::exception ();
    }

    class socket
    {
    public:

        inline socket (int domain, int protocol)
        {
            s = nn_socket (domain, protocol);
            if (nn_slow (s < 0))
                throw nn::exception ();
        }

        inline ~socket ()
        {
            int rc = nn_close (s);
            assert (rc == 0);
        }

        inline void setsockopt (int level, int option, const void *optval,
            size_t optvallen)
        {
            int rc = nn_setsockopt (s, level, option, optval, optvallen);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline void getsockopt (int level, int option, void *optval,
            size_t *optvallen)
        {
            int rc = nn_getsockopt (s, level, option, optval, optvallen);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline int bind (const char *addr)
        {
            int rc = nn_bind (s, addr);
            if (nn_slow (rc < 0))
                throw nn::exception ();
            return rc;
        }

        inline int connect (const char *addr)
        {
            int rc = nn_connect (s, addr);
            if (nn_slow (rc < 0))
                throw nn::exception ();
            return rc;
        }

        inline void shutdown (int how)
        {
            int rc = nn_shutdown (s, how);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline int send (const void *buf, size_t len, int flags)
        {
            int rc = nn_send (s, buf, len, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int recv (void *buf, size_t len, int flags)
        {
            int rc = nn_recv (s, buf, len, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int sendmsg (const struct nn_msghdr *msghdr, int flags)
        {
            int rc = nn_sendmsg (s, msghdr, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int recvmsg (struct nn_msghdr *msghdr, int flags)
        {
            int rc = nn_recvmsg (s, msghdr, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

    private:

        int s;

        /*  Prevent making copies of the socket by accident. */
        socket (const socket&);
        void operator = (const socket&);
    };

    inline void term ()
    {
        nn_term ();
    }

}

#undef nn_slow

#endif

<commit_msg>nn_symbol() exposed via nn:symbol()<commit_after>/*
    Copyright (c) 2013 250bpm s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom
    the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.
*/

#ifndef NN_HPP_INCLUDED
#define NN_HPP_INCLUDED

#include <nanomsg/nn.h>

#include <cassert>
#include <cstring>
#include <algorithm>
#include <exception>

#if defined __GNUC__
#define nn_slow(x) __builtin_expect ((x), 0)
#else
#define nn_slow(x) (x)
#endif

namespace nn
{

    class exception : public std::exception
    {
    public:

        exception () : err (nn_errno ()) {}

        virtual const char *what () const throw ()
        {
            return nn_strerror (err);
        }

        int num () const
        {
            return err;
        }

    private:

        int err;
    };

    inline const char *symbol (int i, int *value)
    {
        return nn_symbol (i, value);
    }

    inline void *allocmsg (size_t size, int type)
    {
        void *msg = nn_allocmsg (size, type);
        if (nn_slow (!msg))
            throw nn::exception ();
    }

    inline int freemsg (void *msg)
    {
        int rc = nn_freemsg (msg);
        if (nn_slow (rc != 0))
            throw nn::exception ();
    }

    class socket
    {
    public:

        inline socket (int domain, int protocol)
        {
            s = nn_socket (domain, protocol);
            if (nn_slow (s < 0))
                throw nn::exception ();
        }

        inline ~socket ()
        {
            int rc = nn_close (s);
            assert (rc == 0);
        }

        inline void setsockopt (int level, int option, const void *optval,
            size_t optvallen)
        {
            int rc = nn_setsockopt (s, level, option, optval, optvallen);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline void getsockopt (int level, int option, void *optval,
            size_t *optvallen)
        {
            int rc = nn_getsockopt (s, level, option, optval, optvallen);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline int bind (const char *addr)
        {
            int rc = nn_bind (s, addr);
            if (nn_slow (rc < 0))
                throw nn::exception ();
            return rc;
        }

        inline int connect (const char *addr)
        {
            int rc = nn_connect (s, addr);
            if (nn_slow (rc < 0))
                throw nn::exception ();
            return rc;
        }

        inline void shutdown (int how)
        {
            int rc = nn_shutdown (s, how);
            if (nn_slow (rc != 0))
                throw nn::exception ();
        }

        inline int send (const void *buf, size_t len, int flags)
        {
            int rc = nn_send (s, buf, len, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int recv (void *buf, size_t len, int flags)
        {
            int rc = nn_recv (s, buf, len, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int sendmsg (const struct nn_msghdr *msghdr, int flags)
        {
            int rc = nn_sendmsg (s, msghdr, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

        inline int recvmsg (struct nn_msghdr *msghdr, int flags)
        {
            int rc = nn_recvmsg (s, msghdr, flags);
            if (nn_slow (rc < 0)) {
                if (nn_slow (nn_errno () != EAGAIN))
                    throw nn::exception ();
                return -1;
            }
            return rc;
        }

    private:

        int s;

        /*  Prevent making copies of the socket by accident. */
        socket (const socket&);
        void operator = (const socket&);
    };

    inline void term ()
    {
        nn_term ();
    }

}

#undef nn_slow

#endif

<|endoftext|>


<commit_before>#include <stdlib.h>
#include <sstream>
#include <string>
#include <algorithm>
#include <vector>
#include <iostream>
#include "tinyxml.h"

// ----------------------------------------------------------------------
// STDOUT dump and indenting utility functions
// ----------------------------------------------------------------------
const unsigned int NUM_INDENTS_PER_SPACE=2;

const char * getIndent( unsigned int numIndents )
{
    static const char * pINDENT="                                      + ";
    static const unsigned int LENGTH=strlen( pINDENT );
    unsigned int n=numIndents*NUM_INDENTS_PER_SPACE;
    if ( n > LENGTH ) n = LENGTH;

    return &pINDENT[ LENGTH-n ];
}

// same as getIndent but no "+" at the end
const char * getIndentAlt( unsigned int numIndents )
{
    static const char * pINDENT="                                        ";
    static const unsigned int LENGTH=strlen( pINDENT );
    unsigned int n=numIndents*NUM_INDENTS_PER_SPACE;
    if ( n > LENGTH ) n = LENGTH;

    return &pINDENT[ LENGTH-n ];
}

int dump_attribs_to_stdout(TiXmlElement* pElement, unsigned int indent)
{
    if ( !pElement ) return 0;

    TiXmlAttribute* pAttrib=pElement->FirstAttribute();
    int i=0;
    int ival;
    double dval;
    const char* pIndent=getIndent(indent);
    printf("\n");
    while (pAttrib)
    {
        printf( "%s%s: value=[%s]", pIndent, pAttrib->Name(), pAttrib->Value());

        if (pAttrib->QueryIntValue(&ival)==TIXML_SUCCESS)    printf( " int=%d", ival);
        if (pAttrib->QueryDoubleValue(&dval)==TIXML_SUCCESS) printf( " d=%1.1f", dval);
        printf( "\n" );
        i++;
        pAttrib=pAttrib->Next();
    }
    return i;
}

void dump_to_stdout( TiXmlNode* pParent, unsigned int indent = 0 )
{
    if ( !pParent ) return;

    TiXmlNode* pChild;
    TiXmlText* pText;
    int t = pParent->Type();
    printf( "%s", getIndent(indent));
    int num;

    switch ( t )
    {
    case TiXmlNode::TINYXML_DOCUMENT:
        printf( "Document" );
        break;

    case TiXmlNode::TINYXML_ELEMENT:
        printf( "Element [%s]", pParent->Value() );
        num=dump_attribs_to_stdout(pParent->ToElement(), indent+1);
        switch(num)
        {
            case 0:  printf( " (No attributes)"); break;
            case 1:  printf( "%s1 attribute", getIndentAlt(indent)); break;
            default: printf( "%s%d attributes", getIndentAlt(indent), num); break;
        }
        break;

    case TiXmlNode::TINYXML_COMMENT:
        printf( "Comment: [%s]", pParent->Value());
        break;

    case TiXmlNode::TINYXML_UNKNOWN:
        printf( "Unknown" );
        break;

    case TiXmlNode::TINYXML_TEXT:
        pText = pParent->ToText();
        printf( "Text: [%s]", pText->Value() );
        break;

    case TiXmlNode::TINYXML_DECLARATION:
        printf( "Declaration" );
        break;
    default:
        break;
    }
    printf( "\n" );
    for ( pChild = pParent->FirstChild(); pChild != 0; pChild = pChild->NextSibling())
    {
        dump_to_stdout( pChild, indent+1 );
    }
}

std::string lowercase(const std::string& in)
{
  std::string out = in;
  std::transform(out.begin(), out.end(), out.begin(), ::tolower);
  return out;
}

std::string lowercase(const char* in)
{
  std::string ins = in;
  return lowercase(ins);
}

struct Command
{
   char type;
   double data[6]; // not all used
};

struct Path
{
   std::vector<Command> cmds;
   std::string id;
   std::string style;
};

std::vector<std::string> &split(const std::string &s, char delim, std::vector<std::string> &elems) {
    std::stringstream ss(s);
    std::string item;
    while (std::getline(ss, item, delim)) {
        elems.push_back(item);
    }
    return elems;
}

void get_path_commands(std::vector<std::string> &tokens, std::vector<Command> &cmds)
{
     std::string lookup = "cCmMlLvVhHzZ";
     unsigned int size = tokens.size();

/*
     int i =0;
     for(std::string token: tokens)
     {
        std::cout << i << "/ " << size <<  "] " << token << std::endl;
        i++;
     }
*/
     char lastCmd = 'x';
     std::vector<double> numbers;

     for(std::string token: tokens)
     {
       // new command?
       if(lookup.find(token[0]) == std::string::npos)
       {
         // its just numbers
         std::cout << std::endl;
         std::vector<std::string> numberStrs;
         split(token, ',', numberStrs);
         for(std::string numberStr : numberStrs)
         {
           std::cout << "    " << lastCmd  << " NB STR"  << numberStr << std::endl;
           double f = atof(numberStr.c_str());
           numbers.push_back(f);
         }
       }
       else
       {

std::cout << " [[[[[#$%435\n";
         if(lastCmd != 'x')
         {
           if (lastCmd == 'c' || lastCmd == 'C')
           {
std::cout << "#$%435\n";
              unsigned int i = 0;
	      size_t size = numbers.size();
	      while( i < size)
	      {
                Command cmd;
                cmd.type = lastCmd;
		cmd.data[0] = numbers[i];
		cmd.data[1] = numbers[i+1];
		cmd.data[2] = numbers[i+2];
		cmd.data[3] = numbers[i+3];
		cmd.data[4] = numbers[i+4];
		cmd.data[5] = numbers[i+5];
                cmds.push_back(cmd);
                i += 6;
	      }
           }
//           else if ()
         }
         // its new command
         lastCmd = token[0];
         numbers.resize(0);
         std::cout << "  CMD:" << lastCmd << std::endl;
       }
     }

}

void get_path_attribs(TiXmlElement* pElement, Path &path)
{
    if ( !pElement ) return;

    TiXmlAttribute* pAttrib=pElement->FirstAttribute();
    while (pAttrib)
    {
        std::string name = lowercase(pAttrib->Name());
        std::string value= lowercase(pAttrib->Value());
        if (name == "style")
        {
            path.style = value;
        }
        if (name == "id")
        {
            path.id = value;
        }
        if (name == "d")
        {
            using namespace std;
            // this attribute contains a list of coordinates
            std::vector<std::string> tokens;
            split(value, ' ', tokens);
            get_path_commands(tokens, path.cmds);

        }
        // int ival;
        // double dval;
        // if (pAttrib->QueryIntValue(&ival)==TIXML_SUCCESS)    printf( " int=%d", ival);
        // if (pAttrib->QueryDoubleValue(&dval)==TIXML_SUCCESS) printf( " d=%1.1f", dval);
        pAttrib=pAttrib->Next();
    }
}

void get_svg_paths(TiXmlNode* pParent, std::vector<Path> &paths)
{
    if ( !pParent ) return;

    TiXmlNode* pChild;
    TiXmlText* pText;
    int t = pParent->Type();
    int num;
    std::string name;
    switch ( t )
    {
      case TiXmlNode::TINYXML_ELEMENT:
        name = lowercase(pParent->Value());
        if (name == "path")
        {
          Path p;
          get_path_attribs(pParent->ToElement(), p);
          paths.push_back(p);
        }
        break;

     default:
        break;
   }

  for ( pChild = pParent->FirstChild(); pChild != 0; pChild = pChild->NextSibling())
  {
      get_svg_paths( pChild, paths );
  }
}

// load the named file and dump its structure to STDOUT
void dump_to_stdout(const char* pFilename)
{
    TiXmlDocument doc(pFilename);
    bool loadOkay = doc.LoadFile();
    if (!loadOkay)
    {
        printf("Failed to load file \"%s\"\n", pFilename);
    }
    else
    {
      printf("\n%s:\n", pFilename);
//      dump_to_stdout( &doc ); // defined later in the tutorial
      std::vector<Path> paths;

      std::cout << "\n\n\n****\n\n\n*****\n\n\n****" << std::endl;

      get_svg_paths( &doc, paths);

      std::cout << "PATHS: " << std::endl;
      for (Path path : paths)
      {
        std::cout << " -" << path.id << " " << path.style << std::endl;
        for (Command cmd: path.cmds)
	{
	  std::cout << "    " << cmd.type << std::endl;
	}
     }
  }
}

// ----------------------------------------------------------------------
// main() for printing files named on the command line
// ----------------------------------------------------------------------
int main(int argc, char* argv[])
{
    for (int i=1; i<argc; i++)
    {
        dump_to_stdout(argv[i]);
    }
    return 0;
}

<commit_msg>adding subpaths<commit_after>#include <stdlib.h>
#include <exception>
#include <sstream>
#include <string>
#include <algorithm>
#include <vector>
#include <iostream>
#include "tinyxml.h"

class SvgError: public std::runtime_error
{ public: SvgError(const std::string& what_arg): std::runtime_error(what_arg){};};

struct Point
{
    double x;
    double y;
};

struct Command
{
   char type;
   std::vector<double> numbers;
/*
   double data[6]; // not all used
   Command()
   {
     data[0] = 0;
     data[1] = 0;
     data[2] = 0;
     data[3] = 0;
     data[4] = 0;
     data[5] = 0;
   }
*/

   std::string tostr()
   {
     std::ostringstream os;
     os << type << "[";
     for (double d : numbers)
     {
       os << d << ", ";
     }
     os << "]";
/*
     os << data[0] << ", ";
     os << data[1] << ", ";
     os << data[2] << ", ";
     os << data[3] << ", ";
     os << data[4] << ", ";
     os << data[5] << "]";
*/
     return os.str();
   }
};

struct Path
{
//   std::vector<Command> cmds;
   std::string id;
   std::string style;

    std::vector< std::vector<Command> > subPaths;
};

class SvgReader
{

  public: SvgReader(){}

  public: void Parse(const char*path, std::vector<Path> &paths);
  public: void Dump_paths(const std::vector<Path> paths ) const;

  private: void make_commands(char cmd, const std::vector<double> &numbers, std::vector<Command> &cmds);
  private: void get_path_commands(const std::vector<std::string> &tokens, Path &path);
  private: void get_path_attribs(TiXmlElement* pElement, Path &path);
  private: void get_svg_paths(TiXmlNode* pParent, std::vector<Path> &paths);

  private: void SplitCommands(const std::vector<Command> cmds, double resolution, std::vector< std::vector<Command> > &split_cmds);
  private: void PathToPoints(const Path &path, double resolution, std::vector< std::vector<Point> > &polys);

};

std::string lowercase(const std::string& in)
{
  std::string out = in;
  std::transform(out.begin(), out.end(), out.begin(), ::tolower);
  return out;
}

std::string lowercase(const char* in)
{
  std::string ins = in;
  return lowercase(ins);
}

std::vector<std::string> &split(const std::string &s, char delim, std::vector<std::string> &elems) {
    std::stringstream ss(s);
    std::string item;
    while (std::getline(ss, item, delim)) {
        elems.push_back(item);
    }
    return elems;
}

void SvgReader::SplitCommands(const std::vector<Command> cmds, double resolution, std::vector< std::vector<Command> > &split_cmds)
{
  for(Command cmd: cmds)
  {
    if( tolower(cmd.type) == 'm')
    {

    }
  }

}

void SvgReader::make_commands(char cmd, const std::vector<double> &numbers, std::vector<Command> &cmds)
{
  if(cmd != 'x')
  {
    Command c;
    c.type = cmd;
    c.numbers = numbers;
    cmds.push_back(c);
  }

/*
  if (tolower(cmd) == 'c')
  {
    unsigned int i = 0;
	size_t size = numbers.size();
	while( i < size)
	{
      Command c;
      c.type = cmd;

	  c.data[0] = numbers[i+0];
	  c.data[1] = numbers[i+1];
	  c.data[2] = numbers[i+2];
	  c.data[3] = numbers[i+3];
	  c.data[4] = numbers[i+4];
	  c.data[5] = numbers[i+5];
      cmds.push_back(c);
      i += 6;
	}
    return;
  }

  if (tolower(cmd) == 'm' || tolower(cmd) == 'l')
  {
    unsigned int i = 0;
    size_t size = numbers.size();
    std::cout << "MCOMMAND " << size << std::endl;
    while (i < size)
    {
      Command c;
      c.type = cmd;
      size_t size =  numbers.size();
      c.data[0] = numbers[i+0];
      c.data[1] = numbers[i+1];
      cmds.push_back(c);
      i += 2;
    }
    return;
  }

  if (tolower(cmd) == 'v' || tolower(cmd) == 'h')
  {
    unsigned int i = 0;
    size_t size = numbers.size();
    while (i < size)
    {
      Command c;
      c.type = cmd;
      c.data[0] = numbers[0];
      c.data[1] = numbers[1];
      cmds.push_back(c);
      i += 1;
    }
    return;
  }

  if (tolower(cmd) == 'z')
  {
    Command c;
    c.type = cmd;
    cmds.push_back(c);
    return;
  }

  std::string s = "Unknown cmd " + cmd;
  SvgError x(s.c_str());
  throw(x);
 */

}

void SvgReader::get_path_commands(const std::vector<std::string> &tokens, Path &path)
{
     std::vector <Command> cmds;

     std::string lookup = "cCmMlLvVhHzZ";
     char lastCmd = 'x';
     std::vector<double> numbers;

     for(std::string token: tokens)
     {
       // new command?
       if(lookup.find(token[0]) == std::string::npos)
       {
         // its just numbers
         std::cout << std::endl;
         std::vector<std::string> numberStrs;
         split(token, ',', numberStrs);
         for(std::string numberStr : numberStrs)
         {
           double f = atof(numberStr.c_str());
           numbers.push_back(f);
         }
       }
       else
       {

         if(lastCmd != 'x')
         {
           Command c;
           c.type = lastCmd;
           c.numbers = numbers;
           cmds.push_back(c);
         }

         // its new command
         lastCmd = token[0];
         numbers.resize(0);
       }
     }
     // the last command
     if(lastCmd != 'x')
     {
       Command c;
       c.type = lastCmd;
       c.numbers = numbers;
       cmds.push_back(c);
     }
    // split the commands into sub_paths

}

void SvgReader::get_path_attribs(TiXmlElement* pElement, Path &path)
{
    if ( !pElement ) return;

    TiXmlAttribute* pAttrib=pElement->FirstAttribute();
    while (pAttrib)
    {
        std::string name = lowercase(pAttrib->Name());
        std::string value= lowercase(pAttrib->Value());
        if (name == "style")
        {
            path.style = value;
        }
        if (name == "id")
        {
            path.id = value;
        }
        if (name == "d")
        {
            using namespace std;
            // this attribute contains a list of coordinates
            std::vector<std::string> tokens;
            split(value, ' ', tokens);
            get_path_commands(tokens, path);

        }
        // int ival;
        // double dval;
        // if (pAttrib->QueryIntValue(&ival)==TIXML_SUCCESS)    printf( " int=%d", ival);
        // if (pAttrib->QueryDoubleValue(&dval)==TIXML_SUCCESS) printf( " d=%1.1f", dval);
        pAttrib=pAttrib->Next();
    }
}

void SvgReader::get_svg_paths(TiXmlNode* pParent, std::vector<Path> &paths)
{
    if ( !pParent ) return;

    TiXmlNode* pChild;
    TiXmlText* pText;
    int t = pParent->Type();
    int num;
    std::string name;
    switch ( t )
    {
      case TiXmlNode::TINYXML_ELEMENT:
        name = lowercase(pParent->Value());
        if (name == "path")
        {
          Path p;
          get_path_attribs(pParent->ToElement(), p);
          paths.push_back(p);
        }
        break;

     default:
        break;
   }

  for ( pChild = pParent->FirstChild(); pChild != 0; pChild = pChild->NextSibling())
  {
      get_svg_paths( pChild, paths );
  }
}

// load the named file and dump its structure to STDOUT
void SvgReader::Parse(const char* pFilename, std::vector<Path> &paths)
{
    TiXmlDocument doc(pFilename);
    bool loadOkay = doc.LoadFile();
    if (!loadOkay)
    {
      std::ostringstream os;
      os << "Failed to load file " <<  pFilename;
      SvgError x(os.str());
      throw x;
    }

  get_svg_paths( &doc, paths);

}

void SvgReader::Dump_paths(const std::vector<Path> paths ) const
{
  std::cout << "PATHS: " << std::endl;
  for (Path path : paths)
  {
    std::cout << " -" << path.id << " " << path.style << std::endl;
    for (std::vector<Command> subPath : path.subPaths)
    {
      std::cout << "   subpath (" << subPath.size() << " cmds)" << std::endl;
      for (Command cmd: subPath)
  	  {
	    std::cout << "    " << cmd.tostr() << std::endl;
	  }
   }
 }

}

// ----------------------------------------------------------------------
// main() for printing files named on the command line
// ----------------------------------------------------------------------
int main(int argc, char* argv[])
{
    std::vector<Path> paths;

    for (int i=1; i<argc; i++)
    {
      std::cout << "=========\nFILE: " << argv[i] << std::endl;
      std::vector<Path> paths;

      SvgReader svg;
      svg.Parse(argv[i], paths);
      svg.Dump_paths(paths);
    }

    return 0;
}

<|endoftext|>