<commit_before>#include "libtorrent/tracker_manager.hpp"
#include "libtorrent/http_tracker_connection.hpp"
#include "libtorrent/buffer.hpp"
#include "libtorrent/xml_parse.hpp"
#include "libtorrent/upnp.hpp"
#include "libtorrent/entry.hpp"
#include "libtorrent/torrent_info.hpp"
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include <boost/bind.hpp>

#include "test.hpp"

using namespace libtorrent;
using namespace boost::tuples;
using boost::bind;

tuple<int, int> feed_bytes(http_parser& parser, char const* str)
{
	tuple<int, int> ret(0, 0);
	buffer::const_interval recv_buf(str, str + 1);
	for (; *str; ++str)
	{
		recv_buf.end = str + 1;
		int payload, protocol;
		tie(payload, protocol) = parser.incoming(recv_buf);
		ret.get<0>() += payload;
		ret.get<1>() += protocol;
	}
	return ret;
}

void parser_callback(std::string& out, int token, char const* s, char const* val)
{
	switch (token)
	{
		case xml_start_tag: out += "B"; break;
		case xml_end_tag: out += "F"; break;
		case xml_empty_tag: out += "E"; break;
		case xml_declaration_tag: out += "D"; break;
		case xml_comment: out += "C"; break;
		case xml_string: out += "S"; break;
		case xml_attribute: out += "A"; break;
		case xml_parse_error: out += "P"; break;
		default: TEST_CHECK(false);
	}
	out += s;
	if (token == xml_attribute)
	{
		TEST_CHECK(val != 0);
		out += "V";
		out += val;
	}
	else
	{
		TEST_CHECK(val == 0);
	}
}

int test_main()
{
	using namespace libtorrent;

	TEST_CHECK(parse_url_components("http://foo:bar@host.com:80/path/to/file")
		== make_tuple("http", "foo:bar", "host.com", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://host.com/path/to/file")
		== make_tuple("http", "", "host.com", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("ftp://host.com:21/path/to/file")
		== make_tuple("ftp", "", "host.com", 21, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://host.com/path?foo:bar@foo:")
		== make_tuple("http", "", "host.com", 80, "/path?foo:bar@foo:"));

	TEST_CHECK(parse_url_components("http://192.168.0.1/path/to/file")
		== make_tuple("http", "", "192.168.0.1", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://[::1]/path/to/file")
		== make_tuple("http", "", "[::1]", 80, "/path/to/file"));

	// base64 test vectors from http://www.faqs.org/rfcs/rfc4648.html

	TEST_CHECK(base64encode("") == "");
	TEST_CHECK(base64encode("f") == "Zg==");
	TEST_CHECK(base64encode("fo") == "Zm8=");
	TEST_CHECK(base64encode("foo") == "Zm9v");
	TEST_CHECK(base64encode("foob") == "Zm9vYg==");
	TEST_CHECK(base64encode("fooba") == "Zm9vYmE=");
	TEST_CHECK(base64encode("foobar") == "Zm9vYmFy");

	// HTTP request parser

	http_parser parser;
	boost::tuple<int, int> received = feed_bytes(parser
		, "HTTP/1.1 200 OK\r\n"
		"Content-Length: 4\r\n"
		"Content-Type: text/plain\r\n"
		"\r\n"
		"test");

	TEST_CHECK(received == make_tuple(4, 64));
	TEST_CHECK(parser.finished());
	TEST_CHECK(std::equal(parser.get_body().begin, parser.get_body().end, "test"));
	TEST_CHECK(parser.header<std::string>("content-type") == "text/plain");
	TEST_CHECK(parser.header<int>("content-length") == 4);

	parser.reset();

	TEST_CHECK(!parser.finished());

	char const* upnp_response =
		"HTTP/1.1 200 OK\r\n"
		"ST:upnp:rootdevice\r\n"
		"USN:uuid:000f-66d6-7296000099dc::upnp:rootdevice\r\n"
		"Location: http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc\r\n"
		"Server: Custom/1.0 UPnP/1.0 Proc/Ver\r\n"
		"EXT:\r\n"
		"Cache-Control:max-age=180\r\n"
		"DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n";

	received = feed_bytes(parser, upnp_response);

	TEST_CHECK(received == make_tuple(0, int(strlen(upnp_response))));
	TEST_CHECK(parser.get_body().left() == 0);
	TEST_CHECK(parser.header<std::string>("st") == "upnp:rootdevice");
	TEST_CHECK(parser.header<std::string>("location")
		== "http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc");
	TEST_CHECK(parser.header<std::string>("ext") == "");
	TEST_CHECK(parser.header<std::string>("date") == "Fri, 02 Jan 1970 08:10:38 GMT");

	parser.reset();
	TEST_CHECK(!parser.finished());

	char const* upnp_notify =
		"NOTIFY * HTTP/1.1\r\n"
		"Host:239.255.255.250:1900\r\n"
		"NT:urn:schemas-upnp-org:device:MediaServer:1\r\n"
		"NTS:ssdp:alive\r\n"
		"Location:http://10.0.1.15:2353/upnphost/udhisapi.dll?content=uuid:c17f2c31-d19b-4912-af94-651945c8a84e\r\n"
		"USN:uuid:c17f0c32-d1db-4be8-ae94-25f94583026e::urn:schemas-upnp-org:device:MediaServer:1\r\n"
		"Cache-Control:max-age=900\r\n"
		"Server:Microsoft-Windows-NT/5.1 UPnP/1.0 UPnP-Device-Host/1.0\r\n";

	received = feed_bytes(parser, upnp_notify);

	TEST_CHECK(received == make_tuple(0, int(strlen(upnp_notify))));
	TEST_CHECK(parser.method() == "notify");
	TEST_CHECK(parser.path() == "*");

	// test xml parser

	char xml1[] = "foobar";
	std::string out1;

	xml_parse(xml1, xml1 + sizeof(xml1) - 1, bind(&parser_callback
		, boost::ref(out1), _1, _2, _3));
	std::cerr << out1 << std::endl;
	TEST_CHECK(out1 == "BaSfooEbSbarFa");

	char xml2[] = "<?xml version = \"1.0\"?><c x=\"1\" \t y=\"3\"/><d foo='bar'></d boo='foo'>";
	std::string out2;

	xml_parse(xml2, xml2 + sizeof(xml2) - 1, bind(&parser_callback
		, boost::ref(out2), _1, _2, _3));
	std::cerr << out2 << std::endl;
	TEST_CHECK(out2 == "DxmlAversionV1.0EcAxV1AyV3BdAfooVbarFdAbooVfooCcomment");

	char xml3[] = "foo";
	std::string out3;

	xml_parse(xml3, xml3 + sizeof(xml3) - 1, bind(&parser_callback
		, boost::ref(out3), _1, _2, _3));
	std::cerr << out3 << std::endl;
	TEST_CHECK(out3 == "BaPunquoted attribute valueSfooFaPmissing end quote on attribute");

	char xml4[] = "foo";
	std::string out4;

	xml_parse(xml4, xml4 + sizeof(xml4) - 1, bind(&parser_callback
		, boost::ref(out4), _1, _2, _3));
	std::cerr << out4 << std::endl;
	TEST_CHECK(out4 == "BaPgarbage inside element bracketsSfooFaPgarbage inside element brackets");

	// test network functions

	TEST_CHECK(is_local(address::from_string("192.168.0.1")));
	TEST_CHECK(is_local(address::from_string("10.1.1.56")));
	TEST_CHECK(!is_local(address::from_string("14.14.251.63")));

	// test torrent parsing

	entry info;
	info["pieces"] = "aaaaaaaaaaaaaaaaaaaa";
	info["name.utf-8"] = "test1";
	info["name"] = "test__";
	info["piece length"] = 16 * 1024;
	info["length"] = 3245;
	entry torrent;
	torrent["info"] = info;

	torrent_info ti(torrent);

	TEST_CHECK(ti.name() == "test1");

	return 0;
}

<commit_msg>added more address utility function tests<commit_after>#include "libtorrent/tracker_manager.hpp"
#include "libtorrent/http_tracker_connection.hpp"
#include "libtorrent/buffer.hpp"
#include "libtorrent/xml_parse.hpp"
#include "libtorrent/upnp.hpp"
#include "libtorrent/entry.hpp"
#include "libtorrent/torrent_info.hpp"
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include <boost/bind.hpp>

#include "test.hpp"

using namespace libtorrent;
using namespace boost::tuples;
using boost::bind;

tuple<int, int> feed_bytes(http_parser& parser, char const* str)
{
	tuple<int, int> ret(0, 0);
	buffer::const_interval recv_buf(str, str + 1);
	for (; *str; ++str)
	{
		recv_buf.end = str + 1;
		int payload, protocol;
		tie(payload, protocol) = parser.incoming(recv_buf);
		ret.get<0>() += payload;
		ret.get<1>() += protocol;
	}
	return ret;
}

void parser_callback(std::string& out, int token, char const* s, char const* val)
{
	switch (token)
	{
		case xml_start_tag: out += "B"; break;
		case xml_end_tag: out += "F"; break;
		case xml_empty_tag: out += "E"; break;
		case xml_declaration_tag: out += "D"; break;
		case xml_comment: out += "C"; break;
		case xml_string: out += "S"; break;
		case xml_attribute: out += "A"; break;
		case xml_parse_error: out += "P"; break;
		default: TEST_CHECK(false);
	}
	out += s;
	if (token == xml_attribute)
	{
		TEST_CHECK(val != 0);
		out += "V";
		out += val;
	}
	else
	{
		TEST_CHECK(val == 0);
	}
}

int test_main()
{
	using namespace libtorrent;

	TEST_CHECK(parse_url_components("http://foo:bar@host.com:80/path/to/file")
		== make_tuple("http", "foo:bar", "host.com", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://host.com/path/to/file")
		== make_tuple("http", "", "host.com", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("ftp://host.com:21/path/to/file")
		== make_tuple("ftp", "", "host.com", 21, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://host.com/path?foo:bar@foo:")
		== make_tuple("http", "", "host.com", 80, "/path?foo:bar@foo:"));

	TEST_CHECK(parse_url_components("http://192.168.0.1/path/to/file")
		== make_tuple("http", "", "192.168.0.1", 80, "/path/to/file"));

	TEST_CHECK(parse_url_components("http://[::1]/path/to/file")
		== make_tuple("http", "", "[::1]", 80, "/path/to/file"));

	// base64 test vectors from http://www.faqs.org/rfcs/rfc4648.html

	TEST_CHECK(base64encode("") == "");
	TEST_CHECK(base64encode("f") == "Zg==");
	TEST_CHECK(base64encode("fo") == "Zm8=");
	TEST_CHECK(base64encode("foo") == "Zm9v");
	TEST_CHECK(base64encode("foob") == "Zm9vYg==");
	TEST_CHECK(base64encode("fooba") == "Zm9vYmE=");
	TEST_CHECK(base64encode("foobar") == "Zm9vYmFy");

	// HTTP request parser

	http_parser parser;
	boost::tuple<int, int> received = feed_bytes(parser
		, "HTTP/1.1 200 OK\r\n"
		"Content-Length: 4\r\n"
		"Content-Type: text/plain\r\n"
		"\r\n"
		"test");

	TEST_CHECK(received == make_tuple(4, 64));
	TEST_CHECK(parser.finished());
	TEST_CHECK(std::equal(parser.get_body().begin, parser.get_body().end, "test"));
	TEST_CHECK(parser.header<std::string>("content-type") == "text/plain");
	TEST_CHECK(parser.header<int>("content-length") == 4);

	parser.reset();

	TEST_CHECK(!parser.finished());

	char const* upnp_response =
		"HTTP/1.1 200 OK\r\n"
		"ST:upnp:rootdevice\r\n"
		"USN:uuid:000f-66d6-7296000099dc::upnp:rootdevice\r\n"
		"Location: http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc\r\n"
		"Server: Custom/1.0 UPnP/1.0 Proc/Ver\r\n"
		"EXT:\r\n"
		"Cache-Control:max-age=180\r\n"
		"DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n";

	received = feed_bytes(parser, upnp_response);

	TEST_CHECK(received == make_tuple(0, int(strlen(upnp_response))));
	TEST_CHECK(parser.get_body().left() == 0);
	TEST_CHECK(parser.header<std::string>("st") == "upnp:rootdevice");
	TEST_CHECK(parser.header<std::string>("location")
		== "http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc");
	TEST_CHECK(parser.header<std::string>("ext") == "");
	TEST_CHECK(parser.header<std::string>("date") == "Fri, 02 Jan 1970 08:10:38 GMT");

	parser.reset();
	TEST_CHECK(!parser.finished());

	char const* upnp_notify =
		"NOTIFY * HTTP/1.1\r\n"
		"Host:239.255.255.250:1900\r\n"
		"NT:urn:schemas-upnp-org:device:MediaServer:1\r\n"
		"NTS:ssdp:alive\r\n"
		"Location:http://10.0.1.15:2353/upnphost/udhisapi.dll?content=uuid:c17f2c31-d19b-4912-af94-651945c8a84e\r\n"
		"USN:uuid:c17f0c32-d1db-4be8-ae94-25f94583026e::urn:schemas-upnp-org:device:MediaServer:1\r\n"
		"Cache-Control:max-age=900\r\n"
		"Server:Microsoft-Windows-NT/5.1 UPnP/1.0 UPnP-Device-Host/1.0\r\n";

	received = feed_bytes(parser, upnp_notify);

	TEST_CHECK(received == make_tuple(0, int(strlen(upnp_notify))));
	TEST_CHECK(parser.method() == "notify");
	TEST_CHECK(parser.path() == "*");

	// test xml parser

	char xml1[] = "foobar";
	std::string out1;

	xml_parse(xml1, xml1 + sizeof(xml1) - 1, bind(&parser_callback
		, boost::ref(out1), _1, _2, _3));
	std::cerr << out1 << std::endl;
	TEST_CHECK(out1 == "BaSfooEbSbarFa");

	char xml2[] = "<?xml version = \"1.0\"?><c x=\"1\" \t y=\"3\"/><d foo='bar'></d boo='foo'>";
	std::string out2;

	xml_parse(xml2, xml2 + sizeof(xml2) - 1, bind(&parser_callback
		, boost::ref(out2), _1, _2, _3));
	std::cerr << out2 << std::endl;
	TEST_CHECK(out2 == "DxmlAversionV1.0EcAxV1AyV3BdAfooVbarFdAbooVfooCcomment");

	char xml3[] = "foo";
	std::string out3;

	xml_parse(xml3, xml3 + sizeof(xml3) - 1, bind(&parser_callback
		, boost::ref(out3), _1, _2, _3));
	std::cerr << out3 << std::endl;
	TEST_CHECK(out3 == "BaPunquoted attribute valueSfooFaPmissing end quote on attribute");

	char xml4[] = "foo";
	std::string out4;

	xml_parse(xml4, xml4 + sizeof(xml4) - 1, bind(&parser_callback
		, boost::ref(out4), _1, _2, _3));
	std::cerr << out4 << std::endl;
	TEST_CHECK(out4 == "BaPgarbage inside element bracketsSfooFaPgarbage inside element brackets");

	// test network functions

	asio::error_code ec;
	TEST_CHECK(is_local(address::from_string("192.168.0.1", ec)));
	TEST_CHECK(is_local(address::from_string("10.1.1.56", ec)));
	TEST_CHECK(!is_local(address::from_string("14.14.251.63", ec)));
	TEST_CHECK(is_loopback(address::from_string("127.0.0.1", ec)));
	TEST_CHECK(is_loopback(address::from_string("::1", ec)));
	TEST_CHECK(is_any(address_v6::any()));
	TEST_CHECK(is_any(address_v4::any()));
	TEST_CHECK(!is_any(address::from_string("31.53.21.64", ec)));

	// test torrent parsing

	entry info;
	info["pieces"] = "aaaaaaaaaaaaaaaaaaaa";
	info["name.utf-8"] = "test1";
	info["name"] = "test__";
	info["piece length"] = 16 * 1024;
	info["length"] = 3245;
	entry torrent;
	torrent["info"] = info;

	torrent_info ti(torrent);

	TEST_CHECK(ti.name() == "test1");

	return 0;
}

<|endoftext|>


<commit_before>// tests the time for a single batch to go forward and backward through the layers

#include <iostream>
#include <random>

#include "NeuralNet.h"

#include "gtest/gtest.h"

#include "test/gtest_supp.h"
#include "test/Sampler.h"
#include "Timer.h"

using namespace std;

TEST( testsinglebatch, main ) {
    const int batchSize = 128;
    const float learningRate = 0.1f;

    NeuralNet *net = NeuralNet::maker()->planes(1)->boardSize(28)->instance();
    net->convolutionalMaker()->numFilters(32)->filterSize(5)->relu()->biased()->insert();
    net->convolutionalMaker()->numFilters(32)->filterSize(5)->relu()->biased()->insert();
    net->convolutionalMaker()->numFilters(10)->filterSize(20)->tanh()->biased()->insert();
    net->setBatchSize(batchSize);

    mt19937 random;
    random.seed(0); // so always gives same results
    const int inputsSize = net->getInputSizePerExample() * batchSize;
    float *inputData = new float[ inputsSize ];
    for( int i = 0; i < inputsSize; i++ ) {
        inputData[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
    }
    const int resultsSize = net->getLastLayer()->getResultsSize();
    float *expectedResults = new float[resultsSize];
    for( int i = 0; i < resultsSize; i++ ) {
        expectedResults[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
    }

    for (int layerIndex = 1; layerIndex <= 3; layerIndex++ ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        int weightsSize = layer->getWeightsSize();
        for( int i = 0; i < weightsSize; i++ ) {
            layer->weights[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
        }
        layer->weightsWrapper->copyToDevice();
        int biasWeightsSize = layer->getBiasWeightsSize();
        for( int i = 0; i < biasWeightsSize; i++ ) {
            layer->biasWeights[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
        }
//        layer->biasWeightsWrapper->copyToDevice();
    }

    net->propagate( inputData );
    for (int layerIndex = 1; layerIndex <= 3; layerIndex++ ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        float const*results = layer->getResults();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "results", resultsSize, (float*)results, 3 );
    }
float *results = (float*)(net->layers[1]->getResults());
EXPECT_FLOAT_NEAR( 0.0767364, results[684] );
EXPECT_FLOAT_NEAR( 0.0294366, results[559] );
EXPECT_FLOAT_NEAR( 0.0244578, results[373] );
results = (float*)(net->layers[2]->getResults());
EXPECT_FLOAT_NEAR( 0.0867873, results[684] );
EXPECT_FLOAT_NEAR( 0.0496127, results[559] );
EXPECT_FLOAT_NEAR( 0, results[373] );
results = (float*)(net->layers[3]->getResults());
EXPECT_FLOAT_NEAR( -0.232493, results[684] );
EXPECT_FLOAT_NEAR( 0.179215, results[559] );
EXPECT_FLOAT_NEAR( 0.14498, results[373] );

float *weights = net->layers[3]->weights;
int weightsSize = net->layers[3]->getWeightsSize();
Sampler::printSamples( "weights", weightsSize, (float*)weights, 3 );

EXPECT_FLOAT_NEAR( -0.0688307, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0310387, weights[72239] );
EXPECT_FLOAT_NEAR( -0.0746839, weights[98933] );

cout << "=============== backprop ..." << endl;
float *errors = new float[100000];
ConvolutionalLayer *layer3 = dynamic_cast<ConvolutionalLayer*>( net->layers[3] );
layer3->calcErrors( expectedResults, errors );
int layer3ResultsSize = layer3->getResultsSize();
Sampler::printSamples( "errors", layer3ResultsSize, errors, 3 );

EXPECT_FLOAT_NEAR( -0.296495, errors[684] );
EXPECT_FLOAT_NEAR( 0.214934, errors[559] );
EXPECT_FLOAT_NEAR( 0.1246, errors[373] );

cout << endl;

ConvolutionalLayer *layer2 = dynamic_cast<ConvolutionalLayer*>( net->layers[2] );
int layer2ResultsSize = layer2->getResultsSize();
float *layer2errors = new float[ layer2ResultsSize ];
layer3->backPropErrors( learningRate, errors, layer2errors );
Sampler::printSamples( "layer2errors", layer2ResultsSize, layer2errors );

EXPECT_FLOAT_NEAR( -0.296495, errors[684] );
EXPECT_FLOAT_NEAR( 0.214934, errors[559] );
EXPECT_FLOAT_NEAR( 0.1246, errors[373] );

EXPECT_FLOAT_NEAR( 0.00070503, layer2errors[1116844] );
EXPECT_FLOAT_NEAR( 0.0220331, layer2errors[174639] );
EXPECT_FLOAT_NEAR( -0.00368077, layer2errors[1353333] );
EXPECT_FLOAT_NEAR( -0.0292891, layer2errors[314560] );
EXPECT_FLOAT_NEAR( 0.0361823, layer2errors[176963] );

cout << endl;

ConvolutionalLayer *layer1 = dynamic_cast<ConvolutionalLayer*>( net->layers[1] );
int layer1ResultsSize = layer1->getResultsSize();
float *layer1errors = new float[ layer1ResultsSize ];
layer2->backPropErrors( learningRate, layer2errors, layer1errors );
Sampler::printSamples( "layer1errors", layer1ResultsSize, layer1errors );

EXPECT_FLOAT_NEAR( -0.0137842, layer1errors[199340] );
EXPECT_FLOAT_NEAR( -0.015897, layer1errors[567855] );
EXPECT_FLOAT_NEAR( 0.0170709, layer1errors[2270837] );

cout << endl;

    net->backProp( learningRate, expectedResults );
    for (int layerIndex = 3; layerIndex >= 1; layerIndex-- ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        weights = layer->weights;
        weightsSize = layer->getWeightsSize();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "weights", weightsSize, (float*)weights, 3 );
        float *biasWeights = layer->biasWeights;
        int biasWeightsSize = layer->getBiasWeightsSize();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "biasWeights", biasWeightsSize, (float*)biasWeights, 3 );
    }

dynamic_cast<ConvolutionalLayer*>(net->layers[3])->weightsWrapper->copyToHost();
weights = net->layers[3]->weights;
EXPECT_FLOAT_NEAR( -0.0683787, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0297281, weights[72239] );
EXPECT_FLOAT_NEAR( -0.0748054, weights[98933] );
float *biasWeights = net->layers[3]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0729806, biasWeights[4] );
EXPECT_FLOAT_NEAR( -0.0621146, biasWeights[9] );
EXPECT_FLOAT_NEAR( -0.0397368, biasWeights[3] );
dynamic_cast<ConvolutionalLayer*>(net->layers[2])->weightsWrapper->copyToHost();
weights = net->layers[2]->weights;
EXPECT_FLOAT_NEAR( 0.0501235, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0982721, weights[21039] );
EXPECT_FLOAT_NEAR( -0.0942265, weights[22133] );
biasWeights = net->layers[2]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0552651, biasWeights[12] );
EXPECT_FLOAT_NEAR( -0.0572661, biasWeights[15] );
EXPECT_FLOAT_NEAR( -0.0277963, biasWeights[21] );
weights = net->layers[1]->weights;
EXPECT_FLOAT_NEAR( -0.022328, weights[44] );
EXPECT_FLOAT_NEAR( -0.065826, weights[239] );
EXPECT_FLOAT_NEAR( -0.0419509, weights[533] );
biasWeights = net->layers[1]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0563379, biasWeights[12] );
EXPECT_FLOAT_NEAR( -0.0601005, biasWeights[15] );
EXPECT_FLOAT_NEAR( 0.00422995, biasWeights[21] );

    Timer timer;
    for( int i = 0; i < 4; i++ ) {
        net->learnBatch( learningRate, inputData, expectedResults );
    }
    timer.timeCheck("batch time");
    StatefulTimer::dump(true);

    results = (float*)(net->getResults());
    Sampler::printSamples( "net->getResults()", resultsSize, (float*)results );

EXPECT_FLOAT_NEAR( -0.121662, net->getResults()[684] );
EXPECT_FLOAT_NEAR( 0.0783329, net->getResults()[559] );
EXPECT_FLOAT_NEAR( -0.0549671, net->getResults()[373] );
EXPECT_FLOAT_NEAR( 0.0715649, net->getResults()[960] );
EXPECT_FLOAT_NEAR( -0.00818501, net->getResults()[323] );

    delete[] expectedResults;
    delete[] inputData;
    delete net;
}

<commit_msg>update testsinglebatch, so can test against it (values come from pre-break git version, hopefully correct-er)<commit_after>// tests the time for a single batch to go forward and backward through the layers

#include <iostream>
#include <random>

#include "NeuralNet.h"

#include "gtest/gtest.h"

#include "test/gtest_supp.h"
#include "test/Sampler.h"
#include "Timer.h"

using namespace std;

TEST( testsinglebatch, main ) {
    const int batchSize = 128;
    const float learningRate = 0.1f;

    NeuralNet *net = NeuralNet::maker()->planes(1)->boardSize(28)->instance();
    net->convolutionalMaker()->numFilters(32)->filterSize(5)->relu()->biased()->insert();
    net->convolutionalMaker()->numFilters(32)->filterSize(5)->relu()->biased()->insert();
    net->convolutionalMaker()->numFilters(10)->filterSize(20)->tanh()->biased()->insert();
    net->setBatchSize(batchSize);

    mt19937 random;
    random.seed(0); // so always gives same results
    const int inputsSize = net->getInputSizePerExample() * batchSize;
    float *inputData = new float[ inputsSize ];
    for( int i = 0; i < inputsSize; i++ ) {
        inputData[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
    }
    const int resultsSize = net->getLastLayer()->getResultsSize();
    float *expectedResults = new float[resultsSize];
    for( int i = 0; i < resultsSize; i++ ) {
        expectedResults[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
    }

    for (int layerIndex = 1; layerIndex <= 3; layerIndex++ ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        int weightsSize = layer->getWeightsSize();
        for( int i = 0; i < weightsSize; i++ ) {
            layer->weights[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
        }
        layer->weightsWrapper->copyToDevice();
        int biasWeightsSize = layer->getBiasWeightsSize();
        for( int i = 0; i < biasWeightsSize; i++ ) {
            layer->biasWeights[i] = random() / (float)mt19937::max() * 0.2f - 0.1f;
        }
//        layer->biasWeightsWrapper->copyToDevice();
    }

    net->propagate( inputData );
    for (int layerIndex = 1; layerIndex <= 3; layerIndex++ ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        float const*results = layer->getResults();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "results", resultsSize, (float*)results, 3 );
    }
float *results = (float*)(net->layers[1]->getResults());
EXPECT_FLOAT_NEAR( 0.0767364, results[684] );
EXPECT_FLOAT_NEAR( 0.0294366, results[559] );
EXPECT_FLOAT_NEAR( 0.0244578, results[373] );
results = (float*)(net->layers[2]->getResults());
EXPECT_FLOAT_NEAR( 0.0867873, results[684] );
EXPECT_FLOAT_NEAR( 0.0496127, results[559] );
EXPECT_FLOAT_NEAR( 0, results[373] );
results = (float*)(net->layers[3]->getResults());
EXPECT_FLOAT_NEAR( -0.232493, results[684] );
EXPECT_FLOAT_NEAR( 0.179215, results[559] );
EXPECT_FLOAT_NEAR( 0.14498, results[373] );

float *weights = net->layers[3]->weights;
int weightsSize = net->layers[3]->getWeightsSize();
Sampler::printSamples( "weights", weightsSize, (float*)weights, 3 );

EXPECT_FLOAT_NEAR( -0.0688307, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0310387, weights[72239] );
EXPECT_FLOAT_NEAR( -0.0746839, weights[98933] );

cout << "=============== backprop ..." << endl;
float *errors = new float[100000];
ConvolutionalLayer *layer3 = dynamic_cast<ConvolutionalLayer*>( net->layers[3] );
layer3->calcErrors( expectedResults, errors );
int layer3ResultsSize = layer3->getResultsSize();
Sampler::printSamples( "errors", layer3ResultsSize, errors, 3 );

EXPECT_FLOAT_NEAR( -0.296495, errors[684] );
EXPECT_FLOAT_NEAR( 0.214934, errors[559] );
EXPECT_FLOAT_NEAR( 0.1246, errors[373] );

cout << endl;

ConvolutionalLayer *layer2 = dynamic_cast<ConvolutionalLayer*>( net->layers[2] );
int layer2ResultsSize = layer2->getResultsSize();
float *layer2errors = new float[ layer2ResultsSize ];
layer3->backPropErrors( learningRate, errors, layer2errors );
Sampler::printSamples( "layer2errors", layer2ResultsSize, layer2errors );

EXPECT_FLOAT_NEAR( -0.296495, errors[684] );
EXPECT_FLOAT_NEAR( 0.214934, errors[559] );
EXPECT_FLOAT_NEAR( 0.1246, errors[373] );

EXPECT_FLOAT_NEAR( 0.00070503, layer2errors[1116844] );
EXPECT_FLOAT_NEAR( 0.0220331, layer2errors[174639] );
EXPECT_FLOAT_NEAR( -0.00368077, layer2errors[1353333] );
EXPECT_FLOAT_NEAR( -0.0292891, layer2errors[314560] );
EXPECT_FLOAT_NEAR( 0.0361823, layer2errors[176963] );

cout << endl;

ConvolutionalLayer *layer1 = dynamic_cast<ConvolutionalLayer*>( net->layers[1] );
int layer1ResultsSize = layer1->getResultsSize();
float *layer1errors = new float[ layer1ResultsSize ];
layer2->backPropErrors( learningRate, layer2errors, layer1errors );
Sampler::printSamples( "layer1errors", layer1ResultsSize, layer1errors );

EXPECT_FLOAT_NEAR( -0.0137842, layer1errors[199340] );
EXPECT_FLOAT_NEAR( -0.015897, layer1errors[567855] );
EXPECT_FLOAT_NEAR( 0.0170709, layer1errors[2270837] );

cout << endl;

layer1->backPropErrors( learningRate, layer1errors, 0 );

    net->backProp( learningRate, expectedResults );
    for (int layerIndex = 3; layerIndex >= 1; layerIndex-- ) {
        ConvolutionalLayer *layer = dynamic_cast<ConvolutionalLayer*>( net->layers[layerIndex] );
        weights = layer->weights;
        weightsSize = layer->getWeightsSize();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "weights", weightsSize, (float*)weights, 3 );
        float *biasWeights = layer->biasWeights;
        int biasWeightsSize = layer->getBiasWeightsSize();
        cout << "layer " << layerIndex << endl;
        Sampler::printSamples( "biasWeights", biasWeightsSize, (float*)biasWeights, 3 );
    }

weights = net->layers[3]->weights;
EXPECT_FLOAT_NEAR( -0.0679266, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0284175, weights[72239] );
EXPECT_FLOAT_NEAR( -0.0749269, weights[98933] );
float *biasWeights = net->layers[3]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0605856, biasWeights[4] );
EXPECT_FLOAT_NEAR( -0.0663593, biasWeights[9] );
EXPECT_FLOAT_NEAR( -0.0573801, biasWeights[3] );
weights = net->layers[2]->weights;
EXPECT_FLOAT_NEAR( 0.0507008, weights[16044] );
EXPECT_FLOAT_NEAR( 0.0982873, weights[21039] );
EXPECT_FLOAT_NEAR( -0.094224, weights[22133] );
biasWeights = net->layers[2]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0552651, biasWeights[12] );
EXPECT_FLOAT_NEAR( -0.0571462, biasWeights[15] );
EXPECT_FLOAT_NEAR( -0.0304532, biasWeights[21] );
weights = net->layers[1]->weights;
EXPECT_FLOAT_NEAR( -0.0223737, weights[44] );
EXPECT_FLOAT_NEAR( -0.0658144, weights[239] );
EXPECT_FLOAT_NEAR( -0.0419252, weights[533] );
biasWeights = net->layers[1]->biasWeights;
EXPECT_FLOAT_NEAR( -0.0563513, biasWeights[12] );
EXPECT_FLOAT_NEAR( -0.0601025, biasWeights[15] );
EXPECT_FLOAT_NEAR( 0.000941529, biasWeights[21] );

    Timer timer;
    for( int i = 0; i < 3; i++ ) {
        net->learnBatch( learningRate, inputData, expectedResults );
    }
    timer.timeCheck("batch time");
    StatefulTimer::dump(true);

    results = (float*)(net->getResults());
    Sampler::printSamples( "net->getResults()", resultsSize, (float*)results );

EXPECT_FLOAT_NEAR( -0.15081, net->getResults()[684] );
EXPECT_FLOAT_NEAR( -0.0236106, net->getResults()[559] );
EXPECT_FLOAT_NEAR( -0.0585419, net->getResults()[373] );
EXPECT_FLOAT_NEAR( 0.168737, net->getResults()[960] );
EXPECT_FLOAT_NEAR( -0.00845184, net->getResults()[323] );

    delete[] expectedResults;
    delete[] inputData;
    delete net;
}

<|endoftext|>


<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vector>
#include <fstream>
#include <iostream>
#include <sstream>
#include <cassert>

#include <zorba/zorba.h>
#include <zorba/store_manager.h>
#include <zorba/external_module.h>
#include <zorba/external_function.h>
#include <zorba/serialization_callback.h>
#include <zorba/empty_sequence.h>

#include "system/properties.h"

using namespace zorba;

namespace zorba { namespace ext_in_opt {

bool theGetExtFuncParamDidntWork = false;

class ExtFunction : public NonePureStatelessExternalFunction
{
public:
  String getURI() const { return "http://www.zorba-xquery.com/mod2"; }

  String getLocalName() const { return "bar"; }

  ItemSequence_t evaluate(
        const StatelessExternalFunction::Arguments_t& args,
        const StaticContext* sctx,
        const DynamicContext* dctx) const
  {
    void* lParam;
    // testing if the function paramegter num is available or not
    if (!dctx->getExternalFunctionParam("num", lParam))
      theGetExtFuncParamDidntWork = true;
    int* lNum = static_cast<int*>(lParam);
    std::cout << "ExtFunction::evaluate: " << *lNum << std::endl;
    return ItemSequence_t(new EmptySequence());
  }
};

class ExtModule : public ExternalModule
{
protected:
  ExtFunction lFunc;

public:
  String getURI() const { return "http://www.zorba-xquery.com/mod2"; }

  StatelessExternalFunction* getExternalFunction(String aLocalname) const
  {
    return const_cast<ExtFunction*>(&lFunc);
  }
};

class MySerializationCallback : public SerializationCallback
{
  protected:
    ExtModule theMod;

  public:
    virtual ExternalModule*
    getExternalModule(const String& aURI) const
    {
      return const_cast<ExtModule*>(&theMod);
    }
};

} /* namespace nofolding */ } /* namespace zorba */

int
ext_in_opt(int argc, char* argv[])
{
  void* lStore = zorba::StoreManager::getStore();
  Zorba* lZorba = Zorba::getInstance(lStore);
  bool lCorrectError = false;

  zorba::Properties::load(0, NULL);

  try
  {
    std::ifstream lIn("ext_in_opt.xq");
    assert(lIn.good());
    std::ostringstream lOut;
    ext_in_opt::ExtModule lMod;

    {
      StaticContext_t lSctx = lZorba->createStaticContext();
      lSctx->registerModule(&lMod);

      XQuery_t lQuery = lZorba->compileQuery(lIn, lSctx);

      // value to pass to an external function
      int lNum = 1000;

      zorba::DynamicContext* lDynContext = lQuery->getDynamicContext();
      lDynContext->setVariable("local:foo",
                               lZorba->getItemFactory()->createString("default"));
      lDynContext->addExternalFunctionParam("num", &lNum);
      std::cout << lQuery << std::endl;
    }
  } catch (QueryException& qe) {
    if (qe.getErrorCode() == XPTY0004)
      // the error XPTY0004 is the correct result of the query
      lCorrectError = true;
    else
      std::cerr << qe << std::endl;
      return 3;
  } catch (ZorbaException& e) {
    std::cerr << e << std::endl;
    return 4;
  }

  lZorba->shutdown();
  zorba::StoreManager::shutdownStore(lStore);
  if (ext_in_opt::theGetExtFuncParamDidntWork)
    // the external function parameter "num" was not available
    return 5;
  else if (!lCorrectError)
    // query didn't end with the correct error code
    return 6;
  else
    return 0;
}

<commit_msg>fix for build on mac<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vector>
#include <fstream>
#include <iostream>
#include <sstream>
#include <cassert>

#include <zorba/zorba.h>
#include <zorba/store_manager.h>
#include <zorba/external_module.h>
#include <zorba/external_function.h>
#include <zorba/serialization_callback.h>
#include <zorba/empty_sequence.h>

#include "system/properties.h"

using namespace zorba;

namespace zorba { namespace ext_in_opt_ns {

bool theGetExtFuncParamDidntWork = false;

class ExtFunction : public NonePureStatelessExternalFunction
{
public:
  String getURI() const { return "http://www.zorba-xquery.com/mod2"; }

  String getLocalName() const { return "bar"; }

  ItemSequence_t evaluate(
        const StatelessExternalFunction::Arguments_t& args,
        const StaticContext* sctx,
        const DynamicContext* dctx) const
  {
    void* lParam;
    // testing if the function paramegter num is available or not
    if (!dctx->getExternalFunctionParam("num", lParam))
      theGetExtFuncParamDidntWork = true;
    int* lNum = static_cast<int*>(lParam);
    std::cout << "ExtFunction::evaluate: " << *lNum << std::endl;
    return ItemSequence_t(new EmptySequence());
  }
};

class ExtModule : public ExternalModule
{
protected:
  ExtFunction lFunc;

public:
  String getURI() const { return "http://www.zorba-xquery.com/mod2"; }

  StatelessExternalFunction* getExternalFunction(String aLocalname) const
  {
    return const_cast<ExtFunction*>(&lFunc);
  }
};

class MySerializationCallback : public SerializationCallback
{
  protected:
    ExtModule theMod;

  public:
    virtual ExternalModule*
    getExternalModule(const String& aURI) const
    {
      return const_cast<ExtModule*>(&theMod);
    }
};

} /* namespace nofolding */ } /* namespace zorba */

int
ext_in_opt(int argc, char* argv[])
{
  void* lStore = zorba::StoreManager::getStore();
  Zorba* lZorba = Zorba::getInstance(lStore);
  bool lCorrectError = false;

  zorba::Properties::load(0, NULL);

  try
  {
    std::ifstream lIn("ext_in_opt.xq");
    assert(lIn.good());
    std::ostringstream lOut;
    ext_in_opt_ns::ExtModule lMod;

    {
      StaticContext_t lSctx = lZorba->createStaticContext();
      lSctx->registerModule(&lMod);

      XQuery_t lQuery = lZorba->compileQuery(lIn, lSctx);

      // value to pass to an external function
      int lNum = 1000;

      zorba::DynamicContext* lDynContext = lQuery->getDynamicContext();
      lDynContext->setVariable("local:foo",
                               lZorba->getItemFactory()->createString("default"));
      lDynContext->addExternalFunctionParam("num", &lNum);
      std::cout << lQuery << std::endl;
    }
  } catch (QueryException& qe) {
    if (qe.getErrorCode() == XPTY0004)
      // the error XPTY0004 is the correct result of the query
      lCorrectError = true;
    else
      std::cerr << qe << std::endl;
      return 3;
  } catch (ZorbaException& e) {
    std::cerr << e << std::endl;
    return 4;
  }

  lZorba->shutdown();
  zorba::StoreManager::shutdownStore(lStore);
  if (ext_in_opt_ns::theGetExtFuncParamDidntWork)
    // the external function parameter "num" was not available
    return 5;
  else if (!lCorrectError)
    // query didn't end with the correct error code
    return 6;
  else
    return 0;
}

<|endoftext|>


<commit_before>/**
 * This file is part of the CernVM File System.
 */

#include <gtest/gtest.h>

#include "prng.h"

class T_Prng : public ::testing::Test {
 protected:
  virtual void SetUp() {
    for (unsigned i = 0; i < buf_size; ++i) {
      buffer[i] = -1;
    }
    for (unsigned i = 0; i < rnd_range; ++i) {
      validation[i] = 0;
    }
    prng_ = new Prng();
  }

  virtual void TearDown() {
    delete prng_;
  }

 protected:
  static const uint32_t buf_size = 100000;
  static const uint32_t rnd_range = 100;
  int32_t buffer[buf_size];
  int32_t validation[rnd_range];
  Prng *prng_;
};

TEST_F(T_Prng, FullRange) {
  for (unsigned i = 0; i < buf_size; ++i) {
    uint32_t random_number = prng_->Next(rnd_range);
    validation[random_number] = 1;
    // printf("%d\n", random_number);
  }
  uint32_t test = 0;
  for (unsigned i = 0; i < rnd_range; ++i) {
    test += validation[i];
  }
  uint32_t compare = rnd_range;
  EXPECT_EQ(test, compare);
}

TEST_F(T_Prng, FullRangeTimeseed) {
  prng_->InitLocaltime();
  for (unsigned i = 0; i < buf_size; ++i) {
    uint32_t random_number = prng_->Next(rnd_range);
    validation[random_number] = 1;
    // printf("%d\n", random_number);
  }
  uint32_t test = 0;
  for (unsigned i = 0; i < rnd_range; ++i) {
    test += validation[i];
  }
  uint32_t compare = rnd_range;
  EXPECT_EQ(test, compare);
}

TEST_F(T_Prng, Deterministic) {
  prng_->InitSeed(7);
  for (unsigned i = 0; i < buf_size; ++i) {
    buffer[i] = prng_->Next(0xFFFFFFF);
  }
  prng_->InitSeed(7);
  for (unsigned i = 0; i < buf_size; ++i) {
    ASSERT_EQ(prng_->Next(0xFFFFFFF), (uint32_t)buffer[i]);
  }
}
<commit_msg>Unit test for Prng::NextNormal<commit_after>/**
 * This file is part of the CernVM File System.
 */

#include <gtest/gtest.h>

#include "prng.h"

class T_Prng : public ::testing::Test {
 protected:
  virtual void SetUp() {
    for (unsigned i = 0; i < buf_size; ++i) {
      buffer[i] = -1;
    }
    for (unsigned i = 0; i < rnd_range; ++i) {
      validation[i] = 0;
    }
    prng_ = new Prng();
  }

  virtual void TearDown() {
    delete prng_;
  }

 protected:
  static const uint32_t buf_size = 100000;
  static const uint32_t rnd_range = 100;
  int32_t buffer[buf_size];
  int32_t validation[rnd_range];
  Prng *prng_;
};

TEST_F(T_Prng, FullRange) {
  for (unsigned i = 0; i < buf_size; ++i) {
    uint32_t random_number = prng_->Next(rnd_range);
    validation[random_number] = 1;
    // printf("%d\n", random_number);
  }
  uint32_t test = 0;
  for (unsigned i = 0; i < rnd_range; ++i) {
    test += validation[i];
  }
  uint32_t compare = rnd_range;
  EXPECT_EQ(test, compare);
}

TEST_F(T_Prng, FullRangeTimeseed) {
  prng_->InitLocaltime();
  for (unsigned i = 0; i < buf_size; ++i) {
    uint32_t random_number = prng_->Next(rnd_range);
    validation[random_number] = 1;
    // printf("%d\n", random_number);
  }
  uint32_t test = 0;
  for (unsigned i = 0; i < rnd_range; ++i) {
    test += validation[i];
  }
  uint32_t compare = rnd_range;
  EXPECT_EQ(test, compare);
}

TEST_F(T_Prng, Deterministic) {
  prng_->InitSeed(7);
  for (unsigned i = 0; i < buf_size; ++i) {
    buffer[i] = prng_->Next(0xFFFFFFF);
  }
  prng_->InitSeed(7);
  for (unsigned i = 0; i < buf_size; ++i) {
    ASSERT_EQ(prng_->Next(0xFFFFFFF), (uint32_t)buffer[i]);
  }
}

TEST_F(T_Prng, NormalDistribution) {
  const int n = 100000;
  prng_->InitLocaltime();
  double rands[n];
  for (int i = 0; i < n; ++i) {
    rands[i] = prng_->NextNormal();
  }
  double mean = 0;
  for (int i = 0; i < n; ++i) {
    mean += rands[i];
  }
  mean /= n;
  // Margin of error wide enough to pass every time
  EXPECT_TRUE(-0.05 < mean && mean < 0.05);
  double variance = 0;
  for (int i = 0; i < n; ++i) {
    variance += (rands[i] - mean)*(rands[i] - mean);
  }
  variance /= n-1;
  // Margin of error wide enough to pass every time
  EXPECT_TRUE(0.95 < variance && variance < 1.05);
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 * REALM CONFIDENTIAL
 * __________________
 *
 *  [2011] - [2015] Realm Inc
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of Realm Incorporated and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to Realm Incorporated
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Realm Incorporated.
 *
 **************************************************************************/
#ifndef REALM_TEST_UTIL_TEST_TYPES_HPP
#define REALM_TEST_UTIL_TEST_TYPES_HPP

#include "unit_test.hpp"
#include "demangle.hpp"

#define TEST_TYPES(name, ...) \
    TEST_TYPES_IF(name, true, __VA_ARGS__)

#define TEST_TYPES_IF(name, enabled, ...) \
    TEST_TYPES_EX(name, realm::test_util::unit_test::get_default_test_list(), enabled, __VA_ARGS__)

#define TEST_TYPES_EX(name, list, enabled, ...) \
    template<class> \
    struct Realm_UnitTest__##name: realm::test_util::unit_test::Test { \
        bool test_enabled() const { return bool(enabled); } \
        void test_run(); \
    }; \
    realm::test_util::unit_test::TestCons<Realm_UnitTest__##name, __VA_ARGS__> realm_unit_test__##name; \
    realm::test_util::unit_test::RegisterTests \
        realm_unit_test_reg__##name((list), realm_unit_test__##name, \
                                    "DefaultSuite", #name, __FILE__, __LINE__); \
    template<class TEST_TYPE> \
    void Realm_UnitTest__##name<TEST_TYPE>::test_run()

namespace realm {
namespace test_util {
namespace unit_test {

template<template<class> class Test, class...>
class TestCons;

template<template<class> class Test, class Type, class... Types>
class TestCons<Test, Type, Types...> {
public:
    Test<Type> head;
    TestCons<Test, Types...> tail;
};
template<template<class> class Test>
class TestCons<Test> {};

struct RegisterTests {
    template<template<class> class Test, class... Types>
    RegisterTests(TestList& list, TestCons<Test, Types...>& tests, const char* suite,
                 const char* name, const char* file, long line)
    {
        register_tests(list, tests, suite, name, file, line);
    }
    template<template<class> class Test, class Type, class... Types>
    static void register_tests(TestList& list, TestCons<Test, Type, Types...>& tests,
                               const char* suite, const char* name, const char* file, long line)
    {
        std::string name_2 = name;
        name_2 += '<';
        name_2 += get_type_name<Type>();
        name_2 += '>';
        RegisterTest::register_test(list, tests.head, suite, name_2, file, line);
        register_tests(list, tests.tail, suite, name, file, line);
    }
    template<template<class> class Test>
    static void register_tests(TestList&, TestCons<Test>&,
                               const char*, const char*, const char*, long)
    {
    }
};

} // namespace unit_test
} // namespace test_util
} // namespace realm

#endif // REALM_TEST_UTIL_TEST_TYPES_HPP
<commit_msg>Get rid of spaces in test names produced via TEST_TYPES() macro<commit_after>/*************************************************************************
 *
 * REALM CONFIDENTIAL
 * __________________
 *
 *  [2011] - [2015] Realm Inc
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of Realm Incorporated and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to Realm Incorporated
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Realm Incorporated.
 *
 **************************************************************************/
#ifndef REALM_TEST_UTIL_TEST_TYPES_HPP
#define REALM_TEST_UTIL_TEST_TYPES_HPP

#include "unit_test.hpp"
#include "demangle.hpp"

#define TEST_TYPES(name, ...) \
    TEST_TYPES_IF(name, true, __VA_ARGS__)

#define TEST_TYPES_IF(name, enabled, ...) \
    TEST_TYPES_EX(name, realm::test_util::unit_test::get_default_test_list(), enabled, __VA_ARGS__)

#define TEST_TYPES_EX(name, list, enabled, ...) \
    template<class> \
    struct Realm_UnitTest__##name: realm::test_util::unit_test::Test { \
        bool test_enabled() const { return bool(enabled); } \
        void test_run(); \
    }; \
    realm::test_util::unit_test::TestCons<Realm_UnitTest__##name, __VA_ARGS__> realm_unit_test__##name; \
    realm::test_util::unit_test::RegisterTests \
        realm_unit_test_reg__##name((list), realm_unit_test__##name, \
                                    "DefaultSuite", #name, __FILE__, __LINE__); \
    template<class TEST_TYPE> \
    void Realm_UnitTest__##name<TEST_TYPE>::test_run()

namespace realm {
namespace test_util {
namespace unit_test {

template<template<class> class Test, class...>
class TestCons;

template<template<class> class Test, class Type, class... Types>
class TestCons<Test, Type, Types...> {
public:
    Test<Type> head;
    TestCons<Test, Types...> tail;
};
template<template<class> class Test>
class TestCons<Test> {};

struct RegisterTests {
    template<template<class> class Test, class... Types>
    RegisterTests(TestList& list, TestCons<Test, Types...>& tests, const char* suite,
                 const char* name, const char* file, long line)
    {
        register_tests(list, tests, suite, name, file, line);
    }
    template<template<class> class Test, class Type, class... Types>
    static void register_tests(TestList& list, TestCons<Test, Type, Types...>& tests,
                               const char* suite, const char* name, const char* file, long line)
    {
        std::string type_name = get_type_name<Type>();
        auto replace = [&](std::string a, std::string b) {
            std::string::size_type offset = 0;
            for (;;) {
                std::string::size_type i = type_name.find(a, offset);
                if (i == std::string::npos)
                    break;
                type_name.replace(i, a.size(), b);
                offset = i = b.size();
            }
        };
        replace("(anonymous namespace)", "anon"); // GCC specific
        replace(" >", ">");
        replace(" ", "+"); // "long double" -> "long+double"
        std::string name_2 = name;
        name_2 += '<';
        name_2 += type_name;
        name_2 += '>';
        RegisterTest::register_test(list, tests.head, suite, name_2, file, line);
        register_tests(list, tests.tail, suite, name, file, line);
    }
    template<template<class> class Test>
    static void register_tests(TestList&, TestCons<Test>&,
                               const char*, const char*, const char*, long)
    {
    }
};

} // namespace unit_test
} // namespace test_util
} // namespace realm

#endif // REALM_TEST_UTIL_TEST_TYPES_HPP
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>
#include <cpprest/json.h>
#include <cpprest/uri.h>
#include <string>
#include <windows.h>
#include "Base64.h"
#include "StringUtils.h"
#include "REST.h"
#include "Gzip.h"
#include "QueryCache.h"
#include "JsonConverter.h"

#include <ctime>
#include <fcntl.h>
#include <io.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdarg.h>

using namespace utility;
using namespace web::http;
using namespace web::http::client;
using namespace concurrency::streams;
using namespace web;
using namespace web::json;

/// <summary>
/// Find the longest length
/// </summary>
/// <param name="results"></param>
/// <param name="column"></param>
/// <returns></returns>
int ScanForLength ( std::vector <SQLRowContent*> results, int column )
{
    int max = 0;

    for ( auto p = results . begin (); p < results . end (); p++ )
    {
        SQLRowContent* result = *p;
        int length = result -> contents[column] . size ();

        if ( length > max )
        {
            max = length;
        }
    }

    return max;
}

/// <summary>
/// Scale is Maximum number of digits to the right of the decimal point.
/// Find the largest scale.
/// </summary>
/// <param name="results"></param>
/// <param name="column"></param>
/// <returns></returns>
int ScanForScale ( std::vector <SQLRowContent*> results, int column )
{
    int max = 0;

    for ( auto p = results . begin (); p < results . end (); p++ )
    {
        SQLRowContent* result = *p;
        int length = result -> contents[column] . size ();
        int dotLocation = result -> contents[column] . find ( L"." );

        if ( dotLocation != string::npos )
        {
            int scale = length - 1 - dotLocation;

            if ( scale > max )
            {
                max = scale;
            }
        }
    }

    return max;
}

void overwrite ( SQLResponse* res )
{
    for ( int i = 0; i < ( int ) res -> columnMetas . size (); ++i )
    {
        SelectedColumnMeta* meta = res -> columnMetas[i];
        ODBCTypes t = ( ODBCTypes ) meta -> columnType;
        int scale = 0;
        int length = 0;

        switch ( t )
        {
            case ODBCTypes::ODBC_Numeric :
            case ODBCTypes::ODBC_Decimal :
            case ODBCTypes::ODBC_Double :
            case ODBCTypes::ODBC_Real :
            case ODBCTypes::ODBC_Float :
                scale = ScanForScale ( res -> results, i );
                meta -> scale = scale;
                meta -> scale = 4;
                break;

            case ODBCTypes::ODBC_Char :
            case ODBCTypes::ODBC_VarChar :
            case ODBCTypes::ODBC_LongVarChar :
            case ODBCTypes::ODBC_WChar :
            case ODBCTypes::ODBC_WVarChar :
            case ODBCTypes::ODBC_WLongVarChar :
            case ODBCTypes::ODBC_DateTime :
            case ODBCTypes::ODBC_Type_Date :
            case ODBCTypes::ODBC_Type_Time :
            case ODBCTypes::ODBC_Type_Timestamp :
                length = ScanForLength ( res -> results, i );
				if (length > meta -> displaySize)
				{
					meta -> displaySize = length;
				}

				if (length > meta -> precision)
				{
					meta -> precision = length;
				}
                break;

            default :
                break;
        }
    }
}

std::wstring completeServerStr ( char* serverStr, long port )
{
    //concat the whole server string
    char completeServerAddr[256];
    char portSuffix[10];
    sprintf ( portSuffix, ":%d", port );

    if ( strstr ( serverStr, "https://" ) == serverStr ||
        strstr ( serverStr, "http://" ) == serverStr )
    {
        sprintf ( completeServerAddr, "%s", serverStr );
    }

    else
    {
        // by default use https
        sprintf ( completeServerAddr, "https://%s", serverStr );
    }

    if ( strstr ( serverStr, portSuffix ) == NULL )
    {
        strcat ( completeServerAddr, portSuffix );
    }

    return string2wstring ( std::string ( completeServerAddr ) );
}

http_request makeRequest ( const char* username, const char* passwd, const wchar_t* uriStr, http::method method )
{
    http_request request;
    char s[128];
    sprintf ( s, "%s:%s", username, passwd );
    std::string b64 = base64_encode ( ( unsigned char const* ) s, strlen ( s ) );
    request . set_method ( method );
    request . set_request_uri ( uri ( uri::encode_uri ( uriStr ) ) );
    request . headers () . add ( header_names::authorization, string2wstring ( "Basic " + b64 ) );
	request . headers () . add ( header_names::accept, "application/json" );
    request . headers () . add ( header_names::content_type, "application/json" );
    return request;
}

bool restAuthenticate ( char* serverAddr, long port, char* username, char* passwd )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
    http_client session ( serverAddrW, config );
    //can get project list only when correct username/password is given
    http_request request = makeRequest ( username, passwd, L"/kylin/api/projects", methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        return true;
    }

    else
    {
        return false;
    }
}

void restListProjects ( char* serverAddr, long port, char* username, char* passwd, std::vector <string>& holder )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
    http_client session ( serverAddrW, config );
    http_request request = makeRequest ( username, passwd, L"/kylin/api/projects", methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        web::json::value projects = response . extract_json () . get ();

        for ( auto iter = projects . as_array () . begin (); iter != projects . as_array () . end (); ++iter )
        {
            holder . push_back ( wstring2string ( ( *iter )[U ( "name" )] . as_string () ) );
        }

        if ( holder . size () == 0 )
        {
            throw exception ( "There is no project available in this server" );
        }
    }

    else if ( response . status_code () == status_codes::InternalError )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( response . extract_json () . get () );
        string errorMsg = wstring2string ( em -> msg );
        throw exception ( errorMsg . c_str () );
    }

    else
    {
        throw exception ( "REST request(listproject) Invalid Response status code : " + response . status_code () );
    }
}

std::unique_ptr <MetadataResponse> restGetMeta ( char* serverAddr, long port, char* username, char* passwd,
                                                 char* project )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
    http_client session ( serverAddrW, config );
    std::wstringstream wss;
    wss << L"/kylin/api/tables_and_columns" << L"?project=" << project;
    http_request request = makeRequest ( username, passwd, wss . str () . c_str (), methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        return MetadataResponseFromJSON ( response . extract_json () . get () );
    }

    else if ( response . status_code () == status_codes::Unauthorized )
    {
        throw exception ( "Username/Password Unauthorized." );
    }

    else if ( response . status_code () == status_codes::InternalError )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( response . extract_json () . get () );
        string errorMsg = wstring2string ( em -> msg );
        throw exception ( errorMsg . c_str () );
    }

    else
    {
        throw exception ( "REST request(getmeta) Invalid Response status code : " + response . status_code () );
    }
}

wstring cookQuery ( wchar_t* p )
{
    std::wstringstream wss;

	int l = wcslen ( p );
    for ( int i = 0; i < l; i++ )
    {
        if ( p[i] == L'\r' || p[i] == L'\n' || p[i] == L'\t' )
        {
            wss << L' ';
        }

        else if (p[i] == L'"')
		{
			wss << L"\\\"";
		}

		else
		{
			wss << p[i];
		}
    }

	return wss.str();
}

wstring getBodyString ( http_response& response )
{
    bool isGzipped = response . headers () . has ( L"Content-Encoding" );

    if ( isGzipped )
    {
        isGzipped = false;
        http_headers::iterator iterator = response . headers () . find ( L"Content-Encoding" );

        if ( iterator != response . headers () . end () )
        {
            wstring contentEncoding = iterator -> second;

            if ( contentEncoding . find ( L"gzip" ) != std::string::npos )
            {
                isGzipped = true;
            }
        }
    }

    container_buffer <std::string> bodyBuffer;
    response . body () . read_to_end ( bodyBuffer ) . get ();
    const std::string& raw = bodyBuffer . collection ();
    std::string uncompressed;

    if ( isGzipped )
    {
        bool decompressStatus = gzipInflate ( raw, uncompressed );

        if ( !decompressStatus )
        {
            throw exception ( "gzip decompress failed" );
        }
    }

    else
    {
        uncompressed = raw;
    }

    //convert the string from utf8 to wchar
    int size_needed = ::MultiByteToWideChar ( CP_UTF8, 0, ( char* ) uncompressed . c_str (), uncompressed . size (), NULL, 0 );
    std::wstring ret ( size_needed, 0 );
    ::MultiByteToWideChar ( CP_UTF8, 0, ( char* ) uncompressed . c_str (), uncompressed . size (), &ret[0], size_needed );
    return ret;
}

std::unique_ptr <SQLResponse> convertToSQLResponse ( int statusFlag,
										  wstring responseStr )
{
    if ( statusFlag == 1 )
    {
        //convert to json
        web::json::value actualRes = web::json::value::parse ( responseStr );
        std::unique_ptr <SQLResponse> r = SQLResponseFromJSON ( actualRes );

        if ( r -> isException == true )
        {
            string expMsg = wstring2string ( r -> exceptionMessage );
            throw exception ( expMsg . c_str () );
        }

        overwrite ( r . get () );
        return r;
    }

    else if ( statusFlag == 0 )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( web::json::value::parse ( responseStr ) );
        string expMsg = wstring2string ( em -> msg );
        throw exception ( expMsg . c_str () );
    }

    return NULL;
}

wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
                                          char* passwd,
                                          char* project,
										  bool isPrepare,
										  int* statusFlag)
{
    //using local cache to intercept probing queries
    const wchar_t* cachedQueryRes = NULL;

	if (isPrepare) {
		cachedQueryRes = loadCache ( rawSql  );
	}

    if ( cachedQueryRes != NULL )
    {
		*statusFlag = 1;
        return cachedQueryRes;
    }

    //real requesting
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 36000 ) );

	//uncomment these lines for debug with proxy
	//wstring p = L"http://127.0.0.1:8888";
	//config.set_proxy(web_proxy(p));

    http_client session ( serverAddrW, config );
    http_request request;

	if (!isPrepare)
	{
		request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
	}

	else
	{
		request = makeRequest ( username, passwd, L"/kylin/api/query/prestate", methods::POST );
	}

    wstring sql = cookQuery ( rawSql );
    std::wstringstream wss;
    wss << L"{ \"acceptPartial\": false, \"project\" : \"" << project << L"\", " << " \"sql\" : \"" << sql << L"\"";

	// backward compatible, Apache Kylin <=2.0
	if (isPrepare)
	{
		wss << L", \"params\" : [] ";
	}

	wss << L"}" ;

    request . set_body ( wss . str (), L"application/json" );
    request . headers () . add ( header_names::accept_encoding, "gzip,deflate" );
    http_response response;
	http::status_code status;

	try
    {
        response = session . request ( request ) . get ();
        status = response . status_code ();
    }

    catch ( std::exception& e )
    {
        std::stringstream ss;
        ss << "An exception is throw Error message: " << e . what ();
        throw exception ( ss . str () . c_str () );
    }

	if ( status == status_codes::OK )
    {
        *statusFlag = 1;
    }

    else if ( status == status_codes::InternalError )
    {
        *statusFlag = 0;
    }

    else
    {
        throw exception ( "Unknown exception in rest query with return code " + status );
    }

	wstring ret = getBodyString ( response );

    if (*statusFlag == 1 && isPrepare)
    {
        storeCache(rawSql, ret.c_str());
    }
	return ret;
}

<commit_msg>minor, allow self-signed SSL for ODBC client<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <cpprest/http_client.h>
#include <cpprest/filestream.h>
#include <cpprest/json.h>
#include <cpprest/uri.h>
#include <string>
#include <windows.h>
#include "Base64.h"
#include "StringUtils.h"
#include "REST.h"
#include "Gzip.h"
#include "QueryCache.h"
#include "JsonConverter.h"

#include <ctime>
#include <fcntl.h>
#include <io.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdarg.h>

using namespace utility;
using namespace web::http;
using namespace web::http::client;
using namespace concurrency::streams;
using namespace web;
using namespace web::json;

/// <summary>
/// Find the longest length
/// </summary>
/// <param name="results"></param>
/// <param name="column"></param>
/// <returns></returns>
int ScanForLength ( std::vector <SQLRowContent*> results, int column )
{
    int max = 0;

    for ( auto p = results . begin (); p < results . end (); p++ )
    {
        SQLRowContent* result = *p;
        int length = result -> contents[column] . size ();

        if ( length > max )
        {
            max = length;
        }
    }

    return max;
}

/// <summary>
/// Scale is Maximum number of digits to the right of the decimal point.
/// Find the largest scale.
/// </summary>
/// <param name="results"></param>
/// <param name="column"></param>
/// <returns></returns>
int ScanForScale ( std::vector <SQLRowContent*> results, int column )
{
    int max = 0;

    for ( auto p = results . begin (); p < results . end (); p++ )
    {
        SQLRowContent* result = *p;
        int length = result -> contents[column] . size ();
        int dotLocation = result -> contents[column] . find ( L"." );

        if ( dotLocation != string::npos )
        {
            int scale = length - 1 - dotLocation;

            if ( scale > max )
            {
                max = scale;
            }
        }
    }

    return max;
}

void overwrite ( SQLResponse* res )
{
    for ( int i = 0; i < ( int ) res -> columnMetas . size (); ++i )
    {
        SelectedColumnMeta* meta = res -> columnMetas[i];
        ODBCTypes t = ( ODBCTypes ) meta -> columnType;
        int scale = 0;
        int length = 0;

        switch ( t )
        {
            case ODBCTypes::ODBC_Numeric :
            case ODBCTypes::ODBC_Decimal :
            case ODBCTypes::ODBC_Double :
            case ODBCTypes::ODBC_Real :
            case ODBCTypes::ODBC_Float :
                scale = ScanForScale ( res -> results, i );
                meta -> scale = scale;
                meta -> scale = 4;
                break;

            case ODBCTypes::ODBC_Char :
            case ODBCTypes::ODBC_VarChar :
            case ODBCTypes::ODBC_LongVarChar :
            case ODBCTypes::ODBC_WChar :
            case ODBCTypes::ODBC_WVarChar :
            case ODBCTypes::ODBC_WLongVarChar :
            case ODBCTypes::ODBC_DateTime :
            case ODBCTypes::ODBC_Type_Date :
            case ODBCTypes::ODBC_Type_Time :
            case ODBCTypes::ODBC_Type_Timestamp :
                length = ScanForLength ( res -> results, i );
				if (length > meta -> displaySize)
				{
					meta -> displaySize = length;
				}

				if (length > meta -> precision)
				{
					meta -> precision = length;
				}
                break;

            default :
                break;
        }
    }
}

std::wstring completeServerStr ( char* serverStr, long port )
{
    //concat the whole server string
    char completeServerAddr[256];
    char portSuffix[10];
    sprintf ( portSuffix, ":%d", port );

    if ( strstr ( serverStr, "https://" ) == serverStr ||
        strstr ( serverStr, "http://" ) == serverStr )
    {
        sprintf ( completeServerAddr, "%s", serverStr );
    }

    else
    {
        // by default use https
        sprintf ( completeServerAddr, "https://%s", serverStr );
    }

    if ( strstr ( serverStr, portSuffix ) == NULL )
    {
        strcat ( completeServerAddr, portSuffix );
    }

    return string2wstring ( std::string ( completeServerAddr ) );
}

http_request makeRequest ( const char* username, const char* passwd, const wchar_t* uriStr, http::method method )
{
    http_request request;
    char s[128];
    sprintf ( s, "%s:%s", username, passwd );
    std::string b64 = base64_encode ( ( unsigned char const* ) s, strlen ( s ) );
    request . set_method ( method );
    request . set_request_uri ( uri ( uri::encode_uri ( uriStr ) ) );
    request . headers () . add ( header_names::authorization, string2wstring ( "Basic " + b64 ) );
	request . headers () . add ( header_names::accept, "application/json" );
    request . headers () . add ( header_names::content_type, "application/json" );
    return request;
}

bool restAuthenticate ( char* serverAddr, long port, char* username, char* passwd )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
	config . set_validate_certificates ( false );
    http_client session ( serverAddrW, config );
    //can get project list only when correct username/password is given
    http_request request = makeRequest ( username, passwd, L"/kylin/api/projects", methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        return true;
    }

    else
    {
        return false;
    }
}

void restListProjects ( char* serverAddr, long port, char* username, char* passwd, std::vector <string>& holder )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
	config . set_validate_certificates ( false );
    http_client session ( serverAddrW, config );
    http_request request = makeRequest ( username, passwd, L"/kylin/api/projects", methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        web::json::value projects = response . extract_json () . get ();

        for ( auto iter = projects . as_array () . begin (); iter != projects . as_array () . end (); ++iter )
        {
            holder . push_back ( wstring2string ( ( *iter )[U ( "name" )] . as_string () ) );
        }

        if ( holder . size () == 0 )
        {
            throw exception ( "There is no project available in this server" );
        }
    }

    else if ( response . status_code () == status_codes::InternalError )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( response . extract_json () . get () );
        string errorMsg = wstring2string ( em -> msg );
        throw exception ( errorMsg . c_str () );
    }

    else
    {
        throw exception ( "REST request(listproject) Invalid Response status code : " + response . status_code () );
    }
}

std::unique_ptr <MetadataResponse> restGetMeta ( char* serverAddr, long port, char* username, char* passwd,
                                                 char* project )
{
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 300 ) );
	config . set_validate_certificates ( false );
    http_client session ( serverAddrW, config );
    std::wstringstream wss;
    wss << L"/kylin/api/tables_and_columns" << L"?project=" << project;
    http_request request = makeRequest ( username, passwd, wss . str () . c_str (), methods::GET );
    http_response response = session . request ( request ) . get ();

    if ( response . status_code () == status_codes::OK )
    {
        return MetadataResponseFromJSON ( response . extract_json () . get () );
    }

    else if ( response . status_code () == status_codes::Unauthorized )
    {
        throw exception ( "Username/Password Unauthorized." );
    }

    else if ( response . status_code () == status_codes::InternalError )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( response . extract_json () . get () );
        string errorMsg = wstring2string ( em -> msg );
        throw exception ( errorMsg . c_str () );
    }

    else
    {
        throw exception ( "REST request(getmeta) Invalid Response status code : " + response . status_code () );
    }
}

wstring cookQuery ( wchar_t* p )
{
    std::wstringstream wss;

	int l = wcslen ( p );
    for ( int i = 0; i < l; i++ )
    {
        if ( p[i] == L'\r' || p[i] == L'\n' || p[i] == L'\t' )
        {
            wss << L' ';
        }

        else if (p[i] == L'"')
		{
			wss << L"\\\"";
		}

		else
		{
			wss << p[i];
		}
    }

	return wss.str();
}

wstring getBodyString ( http_response& response )
{
    bool isGzipped = response . headers () . has ( L"Content-Encoding" );

    if ( isGzipped )
    {
        isGzipped = false;
        http_headers::iterator iterator = response . headers () . find ( L"Content-Encoding" );

        if ( iterator != response . headers () . end () )
        {
            wstring contentEncoding = iterator -> second;

            if ( contentEncoding . find ( L"gzip" ) != std::string::npos )
            {
                isGzipped = true;
            }
        }
    }

    container_buffer <std::string> bodyBuffer;
    response . body () . read_to_end ( bodyBuffer ) . get ();
    const std::string& raw = bodyBuffer . collection ();
    std::string uncompressed;

    if ( isGzipped )
    {
        bool decompressStatus = gzipInflate ( raw, uncompressed );

        if ( !decompressStatus )
        {
            throw exception ( "gzip decompress failed" );
        }
    }

    else
    {
        uncompressed = raw;
    }

    //convert the string from utf8 to wchar
    int size_needed = ::MultiByteToWideChar ( CP_UTF8, 0, ( char* ) uncompressed . c_str (), uncompressed . size (), NULL, 0 );
    std::wstring ret ( size_needed, 0 );
    ::MultiByteToWideChar ( CP_UTF8, 0, ( char* ) uncompressed . c_str (), uncompressed . size (), &ret[0], size_needed );
    return ret;
}

std::unique_ptr <SQLResponse> convertToSQLResponse ( int statusFlag,
										  wstring responseStr )
{
    if ( statusFlag == 1 )
    {
        //convert to json
        web::json::value actualRes = web::json::value::parse ( responseStr );
        std::unique_ptr <SQLResponse> r = SQLResponseFromJSON ( actualRes );

        if ( r -> isException == true )
        {
            string expMsg = wstring2string ( r -> exceptionMessage );
            throw exception ( expMsg . c_str () );
        }

        overwrite ( r . get () );
        return r;
    }

    else if ( statusFlag == 0 )
    {
        std::unique_ptr <ErrorMessage> em = ErrorMessageFromJSON ( web::json::value::parse ( responseStr ) );
        string expMsg = wstring2string ( em -> msg );
        throw exception ( expMsg . c_str () );
    }

    return NULL;
}

wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
                                          char* passwd,
                                          char* project,
										  bool isPrepare,
										  int* statusFlag)
{
    //using local cache to intercept probing queries
    const wchar_t* cachedQueryRes = NULL;

	if (isPrepare) {
		cachedQueryRes = loadCache ( rawSql  );
	}

    if ( cachedQueryRes != NULL )
    {
		*statusFlag = 1;
        return cachedQueryRes;
    }

    //real requesting
    wstring serverAddrW = completeServerStr ( serverAddr, port );
    http_client_config config;
    config . set_timeout ( utility::seconds ( 36000 ) );
	config . set_validate_certificates ( false );

	//uncomment these lines for debug with proxy
	//wstring p = L"http://127.0.0.1:8888";
	//config.set_proxy(web_proxy(p));

    http_client session ( serverAddrW, config );
    http_request request;

	if (!isPrepare)
	{
		request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
	}

	else
	{
		request = makeRequest ( username, passwd, L"/kylin/api/query/prestate", methods::POST );
	}

    wstring sql = cookQuery ( rawSql );
    std::wstringstream wss;
    wss << L"{ \"acceptPartial\": false, \"project\" : \"" << project << L"\", " << " \"sql\" : \"" << sql << L"\"";

	// backward compatible, Apache Kylin <=2.0
	if (isPrepare)
	{
		wss << L", \"params\" : [] ";
	}

	wss << L"}" ;

    request . set_body ( wss . str (), L"application/json" );
    request . headers () . add ( header_names::accept_encoding, "gzip,deflate" );
    http_response response;
	http::status_code status;

	try
    {
        response = session . request ( request ) . get ();
        status = response . status_code ();
    }

    catch ( std::exception& e )
    {
        std::stringstream ss;
        ss << "An exception is throw Error message: " << e . what ();
        throw exception ( ss . str () . c_str () );
    }

	if ( status == status_codes::OK )
    {
        *statusFlag = 1;
    }

    else if ( status == status_codes::InternalError )
    {
        *statusFlag = 0;
    }

    else
    {
        throw exception ( "Unknown exception in rest query with return code " + status );
    }

	wstring ret = getBodyString ( response );

    if (*statusFlag == 1 && isPrepare)
    {
        storeCache(rawSql, ret.c_str());
    }
	return ret;
}

<|endoftext|>


<commit_before>#ifndef MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR
#define MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR
#include <mjolnir/core/RandomNumberGenerator.hpp>
#include <mjolnir/core/System.hpp>
#include <mjolnir/core/ForceField.hpp>
#include <mjolnir/core/Unit.hpp>

namespace mjolnir
{

// Simple underdamped Langevin integrator.
// The implementation is based on the article written by J. D. Honeycutt and
// D. Thirumalai (1992) Biopolymers and also the article written by Z. Guo and
// D. Thirumalai (1995) Biopolymers. Same algorithm used as default Langevin
// integrator in CafeMol that is developed by H. Kenzaki et al., (2011) JCTC.
template<typename traitsT>
class UnderdampedLangevinIntegrator
{
  public:
    using traits_type     = traitsT;
    using boundary_type   = typename traits_type::boundary_type;
    using real_type       = typename traits_type::real_type;
    using coordinate_type = typename traits_type::coordinate_type;
    using system_type     = System<traitsT>;
    using forcefield_type = ForceField<traitsT>;
    using rng_type        = RandomNumberGenerator<traits_type>;

    struct parameter_set
    {
        real_type       gamma;
        real_type       sqrt_gamma_over_mass;
        coordinate_type accel;
    };

  public:

    UnderdampedLangevinIntegrator(const real_type dt,
            std::vector<real_type> gamma, rng_type&& rng)
        : dt_(dt), halfdt_(dt / 2), halfdt2_(dt * dt / 2),
          rng_(std::move(rng)),
          sqrt_gamma_over_mass_(gamma.size()),
          acceleration_(gamma.size())
    {
        gammas_ = std::move(gamma);
    }
    ~UnderdampedLangevinIntegrator() = default;

    void initialize(system_type& sys, forcefield_type& ff);
    real_type step(const real_type time, system_type& sys, forcefield_type& ff);

    real_type delta_t() const noexcept {return dt_;}
    void  set_delta_t(const real_type dt) noexcept
    {
        dt_ = dt; halfdt_ = dt / 2; halfdt2_ = dt * dt / 2;
    }

    void update(const system_type& sys)
    {
        if(!sys.has_attribute("temperature"))
        {
            throw std::out_of_range("mjolnir::UnderdampedLangevinIntegrator: "
                "Langevin Integrator requires reference temperature, but "
                "`temperature` is not found in `system.attribute`.");
        }

        this->temperature_ = sys.attribute("temperature");
        this->noise_coef_  = std::sqrt(2 * physics::constants<real_type>::kB() *
                                       this->temperature_ / dt_);
    }

    std::vector<real_type> const& parameters() const noexcept {return gammas_;}

  private:

    coordinate_type gen_gaussian_vec(const real_type coef)
    {
        return coordinate_type(this->rng_.gaussian(0.0, coef),
                               this->rng_.gaussian(0.0, coef),
                               this->rng_.gaussian(0.0, coef));
    }

  private:
    real_type dt_;
    real_type halfdt_;
    real_type halfdt2_;
    real_type temperature_;
    real_type noise_coef_;
    rng_type  rng_;

    std::vector<real_type>       gammas_;
    std::vector<real_type>       sqrt_gamma_over_mass_;
    std::vector<coordinate_type> acceleration_;
};

template<typename traitsT>
void UnderdampedLangevinIntegrator<traitsT>::initialize(
        system_type& system, forcefield_type& ff)
{
    // initialize temperature and noise intensity
    this->update(system);
    for(std::size_t i=0; i<system.size(); ++i)
    {
        system.force(i) = coordinate_type(0, 0, 0);
    }

    // calculate force
    ff.calc_force(system);

    for(std::size_t i=0; i<system.size(); ++i)
    {
        const auto  rmass = system.rmass(i);
        const auto& force = system.force(i);

        sqrt_gamma_over_mass_[i] = std::sqrt(gammas_[i] * rmass);
        acceleration_[i] = force * rmass +
            this->gen_gaussian_vec(this->noise_coef_ * sqrt_gamma_over_mass_[i]);
    }
    return;
}

template<typename traitsT>
typename UnderdampedLangevinIntegrator<traitsT>::real_type
UnderdampedLangevinIntegrator<traitsT>::step(
        const real_type time, system_type& sys, forcefield_type& ff)
{
    real_type largest_disp2(0.0);
    for(std::size_t i=0; i<sys.size(); ++i)
    {
        auto&       p = sys.position(i); // position of i-th particle
        auto&       v = sys.velocity(i); // ditto
        auto&       f = sys.force(i);    // ditto
        const auto& a = this->acceleration_[i];
        auto    gamma = this->gammas_[i];

        const real_type gamma_dt_over_2           = gamma * halfdt_;
        const real_type one_minus_gamma_dt_over_2 = 1 - gamma_dt_over_2;

        const auto displacement =
            (dt_ * one_minus_gamma_dt_over_2) * v + halfdt2_ * a;

        p = sys.adjust_position(p + displacement);

        v *= one_minus_gamma_dt_over_2 * (one_minus_gamma_dt_over_2 *
             one_minus_gamma_dt_over_2 + gamma_dt_over_2);
        v += (halfdt_ * one_minus_gamma_dt_over_2) * a;

        f = coordinate_type(0, 0, 0);

        largest_disp2 = std::max(largest_disp2, length_sq(displacement));
    }

    // update neighbor list; reduce margin, reconstruct the list if needed
    ff.update_margin(2 * std::sqrt(largest_disp2), sys);

    // calc f(t+dt)
    ff.calc_force(sys);

    // calc a(t+dt) and v(t+dt), generate noise
    for(std::size_t i=0; i<sys.size(); ++i)
    {
        const auto  rm = sys.rmass(i);
        auto&       v  = sys.velocity(i);
        const auto& f  = sys.force(i);
        auto&       a  = this->acceleration_[i];

        a  = f * rm + gen_gaussian_vec(noise_coef_ * sqrt_gamma_over_mass_[i]);
        v += halfdt_ * (1 - gammas_[i] * halfdt_) * a;
    }

    return time + dt_;
}

} // mjolnir
#endif /* MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR */
<commit_msg>remove needless struct<commit_after>#ifndef MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR
#define MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR
#include <mjolnir/core/RandomNumberGenerator.hpp>
#include <mjolnir/core/System.hpp>
#include <mjolnir/core/ForceField.hpp>
#include <mjolnir/core/Unit.hpp>

namespace mjolnir
{

// Simple underdamped Langevin integrator.
// The implementation is based on the article written by J. D. Honeycutt and
// D. Thirumalai (1992) Biopolymers and also the article written by Z. Guo and
// D. Thirumalai (1995) Biopolymers. Same algorithm used as default Langevin
// integrator in CafeMol that is developed by H. Kenzaki et al., (2011) JCTC.
template<typename traitsT>
class UnderdampedLangevinIntegrator
{
  public:
    using traits_type     = traitsT;
    using boundary_type   = typename traits_type::boundary_type;
    using real_type       = typename traits_type::real_type;
    using coordinate_type = typename traits_type::coordinate_type;
    using system_type     = System<traitsT>;
    using forcefield_type = ForceField<traitsT>;
    using rng_type        = RandomNumberGenerator<traits_type>;

  public:

    UnderdampedLangevinIntegrator(const real_type dt,
            std::vector<real_type> gamma, rng_type&& rng)
        : dt_(dt), halfdt_(dt / 2), halfdt2_(dt * dt / 2),
          rng_(std::move(rng)),
          sqrt_gamma_over_mass_(gamma.size()),
          acceleration_(gamma.size())
    {
        gammas_ = std::move(gamma);
    }
    ~UnderdampedLangevinIntegrator() = default;

    void initialize(system_type& sys, forcefield_type& ff);
    real_type step(const real_type time, system_type& sys, forcefield_type& ff);

    real_type delta_t() const noexcept {return dt_;}
    void  set_delta_t(const real_type dt) noexcept
    {
        dt_ = dt; halfdt_ = dt / 2; halfdt2_ = dt * dt / 2;
    }

    void update(const system_type& sys)
    {
        if(!sys.has_attribute("temperature"))
        {
            throw std::out_of_range("mjolnir::UnderdampedLangevinIntegrator: "
                "Langevin Integrator requires reference temperature, but "
                "`temperature` is not found in `system.attribute`.");
        }

        this->temperature_ = sys.attribute("temperature");
        this->noise_coef_  = std::sqrt(2 * physics::constants<real_type>::kB() *
                                       this->temperature_ / dt_);
    }

    std::vector<real_type> const& parameters() const noexcept {return gammas_;}

  private:

    coordinate_type gen_gaussian_vec(const real_type coef)
    {
        return coordinate_type(this->rng_.gaussian(0.0, coef),
                               this->rng_.gaussian(0.0, coef),
                               this->rng_.gaussian(0.0, coef));
    }

  private:
    real_type dt_;
    real_type halfdt_;
    real_type halfdt2_;
    real_type temperature_;
    real_type noise_coef_;
    rng_type  rng_;

    std::vector<real_type>       gammas_;
    std::vector<real_type>       sqrt_gamma_over_mass_;
    std::vector<coordinate_type> acceleration_;
};

template<typename traitsT>
void UnderdampedLangevinIntegrator<traitsT>::initialize(
        system_type& system, forcefield_type& ff)
{
    // initialize temperature and noise intensity
    this->update(system);
    for(std::size_t i=0; i<system.size(); ++i)
    {
        system.force(i) = coordinate_type(0, 0, 0);
    }

    // calculate force
    ff.calc_force(system);

    for(std::size_t i=0; i<system.size(); ++i)
    {
        const auto  rmass = system.rmass(i);
        const auto& force = system.force(i);

        sqrt_gamma_over_mass_[i] = std::sqrt(gammas_[i] * rmass);
        acceleration_[i] = force * rmass +
            this->gen_gaussian_vec(this->noise_coef_ * sqrt_gamma_over_mass_[i]);
    }
    return;
}

template<typename traitsT>
typename UnderdampedLangevinIntegrator<traitsT>::real_type
UnderdampedLangevinIntegrator<traitsT>::step(
        const real_type time, system_type& sys, forcefield_type& ff)
{
    real_type largest_disp2(0.0);
    for(std::size_t i=0; i<sys.size(); ++i)
    {
        auto&       p = sys.position(i); // position of i-th particle
        auto&       v = sys.velocity(i); // ditto
        auto&       f = sys.force(i);    // ditto
        const auto& a = this->acceleration_[i];
        auto    gamma = this->gammas_[i];

        const real_type gamma_dt_over_2           = gamma * halfdt_;
        const real_type one_minus_gamma_dt_over_2 = 1 - gamma_dt_over_2;

        const auto displacement =
            (dt_ * one_minus_gamma_dt_over_2) * v + halfdt2_ * a;

        p = sys.adjust_position(p + displacement);

        v *= one_minus_gamma_dt_over_2 * (one_minus_gamma_dt_over_2 *
             one_minus_gamma_dt_over_2 + gamma_dt_over_2);
        v += (halfdt_ * one_minus_gamma_dt_over_2) * a;

        f = coordinate_type(0, 0, 0);

        largest_disp2 = std::max(largest_disp2, length_sq(displacement));
    }

    // update neighbor list; reduce margin, reconstruct the list if needed
    ff.update_margin(2 * std::sqrt(largest_disp2), sys);

    // calc f(t+dt)
    ff.calc_force(sys);

    // calc a(t+dt) and v(t+dt), generate noise
    for(std::size_t i=0; i<sys.size(); ++i)
    {
        const auto  rm = sys.rmass(i);
        auto&       v  = sys.velocity(i);
        const auto& f  = sys.force(i);
        auto&       a  = this->acceleration_[i];

        a  = f * rm + gen_gaussian_vec(noise_coef_ * sqrt_gamma_over_mass_[i]);
        v += halfdt_ * (1 - gammas_[i] * halfdt_) * a;
    }

    return time + dt_;
}

} // mjolnir
#endif /* MJOLNIR_UNDERDAMPED_LANGEVIN_INTEGRATOR */
<|endoftext|>