<commit_before>#include <ContentForgeTool.h>
#include <Utils.h>
//#include <ParamsUtils.h>
//#include <CLI11.hpp>
#include <Recast.h>
#include <DetourNavMesh.h> // For max vertices per polygon constant
#include <DetourNavMeshBuilder.h>

namespace fs = std::filesystem;

// Set working directory to $(TargetDir)
// Example args:
// -s src/FIXME

class CNavmeshTool : public CContentForgeTool
{
protected:

public:

	CNavmeshTool(const std::string& Name, const std::string& Desc, CVersion Version)
		: CContentForgeTool(Name, Desc, Version)
	{
	}

	virtual bool SupportsMultithreading() const override
	{
		// FIXME: must handle duplicate targets (for example 2 metafiles writing the same resulting file, like depth_atest_ps)
		// FIXME: CStrID
		return false;
	}

	virtual int Init() override
	{
		return 0;
	}

	virtual void ProcessCommandLine(CLI::App& CLIApp) override
	{
		CContentForgeTool::ProcessCommandLine(CLIApp);
	}

	virtual bool ProcessTask(CContentForgeTask& Task) override
	{
		// TODO: check whether the metafile can be processed by this tool

		const std::string TaskName = GetValidResourceName(Task.TaskID.ToString());

		//???use level + world or incoming data must be a ready-made list of .scn + root world tfm for each?
		//can feed from game or editor, usiversal, requires no ECS knowledge and suports ofmeshes well
		// iterate all scn with static data for the level
		// collect collision shapes and/or visible geometry
		// add geometry from static entities (static is a flag in a CSceneComponent)
		// add offmesh links
		// add typed areas

		constexpr int MAX_CONVEXVOL_PTS = 12;
		struct CConvexVolume
		{
			float verts[MAX_CONVEXVOL_PTS*3];
			float hmin, hmax;
			int nverts;
			int areaId;
		};

		static const int MAX_OFFMESH_CONNECTIONS = 256;
		float offMeshConVerts[MAX_OFFMESH_CONNECTIONS*3*2];
		float offMeshConRads[MAX_OFFMESH_CONNECTIONS];
		unsigned char offMeshConDirs[MAX_OFFMESH_CONNECTIONS];
		unsigned char offMeshConAreas[MAX_OFFMESH_CONNECTIONS];
		unsigned short offMeshConFlags[MAX_OFFMESH_CONNECTIONS];
		unsigned int offMeshConId[MAX_OFFMESH_CONNECTIONS];
		int offMeshConCount = 0;

		const float* verts = nullptr;
		const int nverts = 0;
		const int* tris = nullptr;
		const int ntris = 0;
		std::vector<CConvexVolume> vols;
		float bmax[3];
		float bmin[3];

		// NB: the code below is copied from RecastDemo (Sample_SoloMesh.cpp) with slight changes

		// Step 1. Initialize build config.

		rcContext ctx;

		//!!!build navmesh for each agent's R+h! agent selects all h >= its h and from them the closest R >= its R
		const float agentHeight = 1.8f;
		const float agentRadius = 0.3f;
		const float agentMaxClimb = 0.2f;

		//!!!level's interactive bounds must be used! also can use rcCalcBounds
		const float MaxHorzBound = std::max(bmax[0] - bmin[0], bmax[2] - bmin[2]);

		//!!!TODO: most of these things must be in settings!
		const float cellSize = MaxHorzBound * 0.00029f; //???how to choose good value? demo has 0.1 to 1.0, default 0.3
		const float cellHeight = 0.2f;
		const float edgeMaxLen = 12.f;
		const float edgeMaxError = 1.3f;
		const int regionMinSize = (int)(MaxHorzBound * 0.0075f + 0.5f); // 0.5f to round 1.5 to 2; demo has 0 to 150, default 8
		const int regionMergeSize = 20; // demo has 0 to 150, default 20
		const float detailSampleDist = 6.f;
		const float detailSampleMaxError = 1.f;

		rcConfig cfg;
		memset(&cfg, 0, sizeof(cfg));
		cfg.cs = cellSize;
		cfg.ch = cellHeight;
		cfg.walkableSlopeAngle = 60.f;
		cfg.walkableHeight = (int)ceilf(agentHeight / cfg.ch);
		cfg.walkableClimb = (int)floorf(agentMaxClimb / cfg.ch);
		cfg.walkableRadius = (int)ceilf(agentRadius / cfg.cs);
		cfg.maxEdgeLen = (int)(edgeMaxLen / cfg.cs);
		cfg.maxSimplificationError = edgeMaxError;
		cfg.minRegionArea = rcSqr(regionMinSize); // Note: area = size*size
		cfg.mergeRegionArea = rcSqr(regionMergeSize); // Note: area = size*size
		cfg.maxVertsPerPoly = DT_VERTS_PER_POLYGON;
		cfg.detailSampleDist = detailSampleDist < 0.9f ? 0.f : cfg.cs * detailSampleDist;
		cfg.detailSampleMaxError = cfg.ch * detailSampleMaxError;

		//!!!level's interactive bounds must be used!
		rcVcopy(cfg.bmin, bmin);
		rcVcopy(cfg.bmax, bmax);
		rcCalcGridSize(cfg.bmin, cfg.bmax, cfg.cs, &cfg.width, &cfg.height);

		// Step 2. Rasterize input polygon soup.

		auto solid = rcAllocHeightfield();
		if (!rcCreateHeightfield(&ctx, *solid, cfg.width, cfg.height, cfg.bmin, cfg.bmax, cfg.cs, cfg.ch))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not create solid heightfield.");
			return false;
		}

		std::unique_ptr<unsigned char[]> triareas(new unsigned char[ntris]);
		memset(triareas.get(), 0, ntris * sizeof(unsigned char));
		rcMarkWalkableTriangles(&ctx, cfg.walkableSlopeAngle, verts, nverts, tris, ntris, triareas.get());
		if (!rcRasterizeTriangles(&ctx, verts, nverts, tris, triareas.get(), ntris, *solid, cfg.walkableClimb))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not rasterize triangles.");
			return false;
		}

		triareas.reset();

		// Step 3. Filter walkables surfaces.

		rcFilterLowHangingWalkableObstacles(&ctx, cfg.walkableClimb, *solid);
		rcFilterLedgeSpans(&ctx, cfg.walkableHeight, cfg.walkableClimb, *solid);
		rcFilterWalkableLowHeightSpans(&ctx, cfg.walkableHeight, *solid);

		// Step 4. Partition walkable surface to simple regions.

		auto chf = rcAllocCompactHeightfield();
		if (!rcBuildCompactHeightfield(&ctx, cfg.walkableHeight, cfg.walkableClimb, *solid, *chf))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not build compact data.");
			return false;
		}

		rcFreeHeightField(solid);

		if (!rcErodeWalkableArea(&ctx, cfg.walkableRadius, *chf))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not erode.");
			return false;
		}

		for (const auto& vol : vols)
			rcMarkConvexPolyArea(&ctx, vol.verts, vol.nverts, vol.hmin, vol.hmax, (unsigned char)vol.areaId, *chf);

		if (!rcBuildDistanceField(&ctx, *chf))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not build distance field.");
			return false;
		}

		if (!rcBuildRegions(&ctx, *chf, 0, cfg.minRegionArea, cfg.mergeRegionArea))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not build watershed regions.");
			return false;
		}

		// Step 5. Trace and simplify region contours.

		auto cset = rcAllocContourSet();
		if (!rcBuildContours(&ctx, *chf, cfg.maxSimplificationError, cfg.maxEdgeLen, *cset))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not create contours.");
			return false;
		}

		// Step 6. Build polygons mesh from contours.

		auto pmesh = rcAllocPolyMesh();
		if (!rcBuildPolyMesh(&ctx, *cset, cfg.maxVertsPerPoly, *pmesh))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not triangulate contours.");
			return false;
		}

		// Step 7. Create detail mesh which allows to access approximate height on each polygon.
		//???optional?

		auto dmesh = rcAllocPolyMeshDetail();
		if (!rcBuildPolyMeshDetail(&ctx, *pmesh, *chf, cfg.detailSampleDist, cfg.detailSampleMaxError, *dmesh))
		{
			//ctx->log(RC_LOG_ERROR, "buildNavigation: Could not build detail mesh.");
			return false;
		}

		rcFreeCompactHeightfield(chf);
		rcFreeContourSet(cset);

		// (Optional) Step 8. Create Detour data from Recast poly mesh.

		// Update poly flags from areas.
		for (int i = 0; i < pmesh->npolys; ++i)
		{
			/*
			if (pmesh->areas[i] == RC_WALKABLE_AREA)
				pmesh->areas[i] = SAMPLE_POLYAREA_GROUND;

			if (pmesh->areas[i] == SAMPLE_POLYAREA_GROUND ||
				pmesh->areas[i] == SAMPLE_POLYAREA_GRASS ||
				pmesh->areas[i] == SAMPLE_POLYAREA_ROAD)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_WALK;
			}
			else if (pmesh->areas[i] == SAMPLE_POLYAREA_WATER)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_SWIM;
			}
			else if (pmesh->areas[i] == SAMPLE_POLYAREA_DOOR)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_WALK | SAMPLE_POLYFLAGS_DOOR;
			}
			*/
		}

		dtNavMeshCreateParams params;
		memset(&params, 0, sizeof(params));
		params.verts = pmesh->verts;
		params.vertCount = pmesh->nverts;
		params.polys = pmesh->polys;
		params.polyAreas = pmesh->areas;
		params.polyFlags = pmesh->flags;
		params.polyCount = pmesh->npolys;
		params.nvp = pmesh->nvp;
		params.detailMeshes = dmesh->meshes;
		params.detailVerts = dmesh->verts;
		params.detailVertsCount = dmesh->nverts;
		params.detailTris = dmesh->tris;
		params.detailTriCount = dmesh->ntris;
		params.offMeshConVerts = offMeshConVerts;
		params.offMeshConRad = offMeshConRads;
		params.offMeshConDir = offMeshConDirs;
		params.offMeshConAreas = offMeshConAreas;
		params.offMeshConFlags = offMeshConFlags;
		params.offMeshConUserID = offMeshConId;
		params.offMeshConCount = offMeshConCount;
		params.walkableHeight = agentHeight;
		params.walkableRadius = agentRadius;
		params.walkableClimb = agentMaxClimb;
		rcVcopy(params.bmin, pmesh->bmin);
		rcVcopy(params.bmax, pmesh->bmax);
		params.cs = cfg.cs;
		params.ch = cfg.ch;
		params.buildBvTree = true;

		unsigned char* navData = nullptr;
		int navDataSize = 0;
		if (!dtCreateNavMeshData(&params, &navData, &navDataSize))
		{
			//ctx->log(RC_LOG_ERROR, "Could not build Detour navmesh.");
			return false;
		}

		// create output folder
		// save separate nm file for each R+h
		// Old format was:
		// Magic
		// Version
		// NM count
		// Per NM:
		//  R
		//  h
		//  Data size
		//  Data
		//  Named region count
		//  Per named convex volume:
		//   Poly count
		//   Poly refs

		dtFree(navData);

		return true;
	}
};

int main(int argc, const char** argv)
{
	CNavmeshTool Tool("cf-navmesh", "Navigation mesh generator for static level geometry", { 0, 1, 0 });
	return Tool.Execute(argc, argv);
}
<commit_msg>Tools: navmesh config reading<commit_after>#include <ContentForgeTool.h>
#include <Utils.h>
#include <ParamsUtils.h>
//#include <CLI11.hpp>
#include <Recast.h>
#include <DetourNavMesh.h> // For max vertices per polygon constant
#include <DetourNavMeshBuilder.h>

namespace fs = std::filesystem;

// Set working directory to $(TargetDir)
// Example args:
// -s src/game/levels

class CNavmeshTool : public CContentForgeTool
{
protected:

public:

	CNavmeshTool(const std::string& Name, const std::string& Desc, CVersion Version)
		: CContentForgeTool(Name, Desc, Version)
	{
	}

	virtual bool SupportsMultithreading() const override
	{
		// FIXME: must handle duplicate targets (for example 2 metafiles writing the same resulting file, like depth_atest_ps)
		// FIXME: CStrID
		return false;
	}

	virtual int Init() override
	{
		return 0;
	}

	virtual void ProcessCommandLine(CLI::App& CLIApp) override
	{
		CContentForgeTool::ProcessCommandLine(CLIApp);
	}

	virtual bool ProcessTask(CContentForgeTask& Task) override
	{
		// TODO: check whether the metafile can be processed by this tool
		if (ParamsUtils::GetParam(Task.Params, "Tools", std::string{}) != "cf-navmesh")
		{
			// FIXME: skip sliently without error. To CF tool base class???
			return false;
		}

		const std::string TaskName = GetValidResourceName(Task.TaskID.ToString());

		// Read navmesh source HRD

		Data::CParams Desc;
		if (!ParamsUtils::LoadParamsFromHRD(Task.SrcFilePath.string().c_str(), Desc))
		{
			if (_LogVerbosity >= EVerbosity::Errors)
				Task.Log.LogError(Task.SrcFilePath.generic_string() + " HRD loading or parsing error");
			return false;
		}

		//???use level + world or incoming data must be a ready-made list of .scn + root world tfm for each?
		//can feed from game or editor, usiversal, requires no ECS knowledge and suports ofmeshes well
		// iterate all scn with static data for the level
		// collect collision shapes and/or visible geometry
		// add geometry from static entities (static is a flag in a CSceneComponent)
		// add offmesh links
		// add typed areas

		constexpr int MAX_CONVEXVOL_PTS = 12;
		struct CConvexVolume
		{
			float verts[MAX_CONVEXVOL_PTS*3];
			float hmin, hmax;
			int nverts;
			int areaId;
		};

		static const int MAX_OFFMESH_CONNECTIONS = 256;
		float offMeshConVerts[MAX_OFFMESH_CONNECTIONS*3*2];
		float offMeshConRads[MAX_OFFMESH_CONNECTIONS];
		unsigned char offMeshConDirs[MAX_OFFMESH_CONNECTIONS];
		unsigned char offMeshConAreas[MAX_OFFMESH_CONNECTIONS];
		unsigned short offMeshConFlags[MAX_OFFMESH_CONNECTIONS];
		unsigned int offMeshConId[MAX_OFFMESH_CONNECTIONS];
		int offMeshConCount = 0;

		const float* verts = nullptr;
		const int nverts = 0;
		const int* tris = nullptr;
		const int ntris = 0;
		std::vector<CConvexVolume> vols;

		//!!!level's interactive bounds must be used! also can use rcCalcBounds
		float bmax[3];
		float bmin[3];

		// NB: the code below is copied from RecastDemo (Sample_SoloMesh.cpp) with slight changes

		// Step 1. Initialize build config.

		rcContext ctx;

		//!!!build navmesh for each agent's R+h! agent selects all h >= its h and from them the closest R >= its R
		const float agentHeight = ParamsUtils::GetParam(Desc, "AgentHeight", 1.8f);
		const float agentRadius = ParamsUtils::GetParam(Desc, "AgentRadius", 0.3f);
		const float agentMaxClimb = ParamsUtils::GetParam(Desc, "AgentMaxClimb", 0.2f);
		const float agentWalkableSlope = ParamsUtils::GetParam(Desc, "AgentWalkableSlope", 60.f);

		//!!!TODO: most of these things must be in settings!
		const float cellSize = ParamsUtils::GetParam(Desc, "CellSize", agentRadius / 3.f);
		const float cellHeight = ParamsUtils::GetParam(Desc, "CellHeight", cellSize);
		const float edgeMaxLen = ParamsUtils::GetParam(Desc, "EdgeMaxLength", 12.f);
		const float edgeMaxError = ParamsUtils::GetParam(Desc, "EdgeMaxError", 1.3f);
		const int regionMinSize = ParamsUtils::GetParam(Desc, "RegionMinSize", 8);
		const int regionMergeSize = ParamsUtils::GetParam(Desc, "RegionMergeSize", 8);
		const float detailSampleDist = ParamsUtils::GetParam(Desc, "DetailSampleDistance", 6.f);
		const float detailSampleMaxError = ParamsUtils::GetParam(Desc, "DetailSampleMaxError", 1.f);

		rcConfig cfg;
		memset(&cfg, 0, sizeof(cfg));
		cfg.cs = cellSize;
		cfg.ch = cellHeight;
		cfg.walkableSlopeAngle = agentWalkableSlope;
		cfg.walkableHeight = (int)ceilf(agentHeight / cfg.ch);
		cfg.walkableClimb = (int)floorf(agentMaxClimb / cfg.ch);
		cfg.walkableRadius = (int)ceilf(agentRadius / cfg.cs);
		cfg.maxEdgeLen = (int)(edgeMaxLen / cfg.cs);
		cfg.maxSimplificationError = edgeMaxError;
		cfg.minRegionArea = rcSqr(regionMinSize); // Note: area = size*size
		cfg.mergeRegionArea = rcSqr(regionMergeSize); // Note: area = size*size
		cfg.maxVertsPerPoly = DT_VERTS_PER_POLYGON;
		cfg.detailSampleDist = detailSampleDist < 0.9f ? 0.f : cfg.cs * detailSampleDist;
		cfg.detailSampleMaxError = cfg.ch * detailSampleMaxError;

		rcVcopy(cfg.bmin, bmin);
		rcVcopy(cfg.bmax, bmax);
		rcCalcGridSize(cfg.bmin, cfg.bmax, cfg.cs, &cfg.width, &cfg.height);

		// Step 2. Rasterize input polygon soup.

		auto solid = rcAllocHeightfield();
		if (!rcCreateHeightfield(&ctx, *solid, cfg.width, cfg.height, cfg.bmin, cfg.bmax, cfg.cs, cfg.ch))
		{
			Task.Log.LogError("buildNavigation: Could not create solid heightfield.");
			return false;
		}

		std::unique_ptr<unsigned char[]> triareas(new unsigned char[ntris]);
		memset(triareas.get(), 0, ntris * sizeof(unsigned char));
		rcMarkWalkableTriangles(&ctx, cfg.walkableSlopeAngle, verts, nverts, tris, ntris, triareas.get());
		if (!rcRasterizeTriangles(&ctx, verts, nverts, tris, triareas.get(), ntris, *solid, cfg.walkableClimb))
		{
			Task.Log.LogError("buildNavigation: Could not rasterize triangles.");
			return false;
		}

		triareas.reset();

		// Step 3. Filter walkables surfaces.

		rcFilterLowHangingWalkableObstacles(&ctx, cfg.walkableClimb, *solid);
		rcFilterLedgeSpans(&ctx, cfg.walkableHeight, cfg.walkableClimb, *solid);
		rcFilterWalkableLowHeightSpans(&ctx, cfg.walkableHeight, *solid);

		// Step 4. Partition walkable surface to simple regions.

		auto chf = rcAllocCompactHeightfield();
		if (!rcBuildCompactHeightfield(&ctx, cfg.walkableHeight, cfg.walkableClimb, *solid, *chf))
		{
			Task.Log.LogError("buildNavigation: Could not build compact data.");
			return false;
		}

		rcFreeHeightField(solid);

		if (!rcErodeWalkableArea(&ctx, cfg.walkableRadius, *chf))
		{
			Task.Log.LogError("buildNavigation: Could not erode.");
			return false;
		}

		for (const auto& vol : vols)
			rcMarkConvexPolyArea(&ctx, vol.verts, vol.nverts, vol.hmin, vol.hmax, (unsigned char)vol.areaId, *chf);

		if (!rcBuildDistanceField(&ctx, *chf))
		{
			Task.Log.LogError("buildNavigation: Could not build distance field.");
			return false;
		}

		if (!rcBuildRegions(&ctx, *chf, 0, cfg.minRegionArea, cfg.mergeRegionArea))
		{
			Task.Log.LogError("buildNavigation: Could not build watershed regions.");
			return false;
		}

		// Step 5. Trace and simplify region contours.

		auto cset = rcAllocContourSet();
		if (!rcBuildContours(&ctx, *chf, cfg.maxSimplificationError, cfg.maxEdgeLen, *cset))
		{
			Task.Log.LogError("buildNavigation: Could not create contours.");
			return false;
		}

		// Step 6. Build polygons mesh from contours.

		auto pmesh = rcAllocPolyMesh();
		if (!rcBuildPolyMesh(&ctx, *cset, cfg.maxVertsPerPoly, *pmesh))
		{
			Task.Log.LogError("buildNavigation: Could not triangulate contours.");
			return false;
		}

		// Step 7. Create detail mesh which allows to access approximate height on each polygon.
		//???optional?

		auto dmesh = rcAllocPolyMeshDetail();
		if (!rcBuildPolyMeshDetail(&ctx, *pmesh, *chf, cfg.detailSampleDist, cfg.detailSampleMaxError, *dmesh))
		{
			Task.Log.LogError("buildNavigation: Could not build detail mesh.");
			return false;
		}

		rcFreeCompactHeightfield(chf);
		rcFreeContourSet(cset);

		// (Optional) Step 8. Create Detour data from Recast poly mesh.

		// Update poly flags from areas.
		for (int i = 0; i < pmesh->npolys; ++i)
		{
			/*
			if (pmesh->areas[i] == RC_WALKABLE_AREA)
				pmesh->areas[i] = SAMPLE_POLYAREA_GROUND;

			if (pmesh->areas[i] == SAMPLE_POLYAREA_GROUND ||
				pmesh->areas[i] == SAMPLE_POLYAREA_GRASS ||
				pmesh->areas[i] == SAMPLE_POLYAREA_ROAD)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_WALK;
			}
			else if (pmesh->areas[i] == SAMPLE_POLYAREA_WATER)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_SWIM;
			}
			else if (pmesh->areas[i] == SAMPLE_POLYAREA_DOOR)
			{
				pmesh->flags[i] = SAMPLE_POLYFLAGS_WALK | SAMPLE_POLYFLAGS_DOOR;
			}
			*/
		}

		dtNavMeshCreateParams params;
		memset(&params, 0, sizeof(params));
		params.verts = pmesh->verts;
		params.vertCount = pmesh->nverts;
		params.polys = pmesh->polys;
		params.polyAreas = pmesh->areas;
		params.polyFlags = pmesh->flags;
		params.polyCount = pmesh->npolys;
		params.nvp = pmesh->nvp;
		params.detailMeshes = dmesh->meshes;
		params.detailVerts = dmesh->verts;
		params.detailVertsCount = dmesh->nverts;
		params.detailTris = dmesh->tris;
		params.detailTriCount = dmesh->ntris;
		params.offMeshConVerts = offMeshConVerts;
		params.offMeshConRad = offMeshConRads;
		params.offMeshConDir = offMeshConDirs;
		params.offMeshConAreas = offMeshConAreas;
		params.offMeshConFlags = offMeshConFlags;
		params.offMeshConUserID = offMeshConId;
		params.offMeshConCount = offMeshConCount;
		params.walkableHeight = agentHeight;
		params.walkableRadius = agentRadius;
		params.walkableClimb = agentMaxClimb;
		rcVcopy(params.bmin, pmesh->bmin);
		rcVcopy(params.bmax, pmesh->bmax);
		params.cs = cfg.cs;
		params.ch = cfg.ch;
		params.buildBvTree = true;

		unsigned char* navData = nullptr;
		int navDataSize = 0;
		if (!dtCreateNavMeshData(&params, &navData, &navDataSize))
		{
			Task.Log.LogError("Could not build Detour navmesh.");
			return false;
		}

		// create output folder
		// save separate nm file for each R+h
		// Old format was:
		// Magic
		// Version
		// NM count
		// Per NM:
		//  R
		//  h
		//  Data size
		//  Data
		//  Named region count
		//  Per named convex volume:
		//   Poly count
		//   Poly refs

		dtFree(navData);

		return true;
	}
};

int main(int argc, const char** argv)
{
	CNavmeshTool Tool("cf-navmesh", "Navigation mesh generator for static level geometry", { 0, 1, 0 });
	return Tool.Execute(argc, argv);
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) , Arm Limited and affiliates.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "SocketAddress.h"
#include "NetworkInterface.h"
#include "NetworkStack.h"
#include <string.h>
#include "mbed.h"

static bool ipv6_is_valid(const char *addr)
{
    return false;
}

static int ipv6_scan_chunk(uint16_t *shorts, const char *chunk)
{
    return 0;
}

SocketAddress::SocketAddress(nsapi_addr_t addr, uint16_t port)
{
}

SocketAddress::SocketAddress(const char *addr, uint16_t port)
{
}

SocketAddress::SocketAddress(const void *bytes, nsapi_version_t version, uint16_t port)
{
}

SocketAddress::SocketAddress(const SocketAddress &addr)
{
}

SocketAddress::~SocketAddress()
{
}

bool SocketAddress::set_ip_address(const char *addr)
{
    return false;
}

void SocketAddress::set_ip_bytes(const void *bytes, nsapi_version_t version)
{
}

void SocketAddress::set_addr(nsapi_addr_t addr)
{
}

void SocketAddress::set_port(uint16_t port)
{
}

const char *SocketAddress::get_ip_address() const
{
    return NULL;
}

const void *SocketAddress::get_ip_bytes() const
{
    return NULL;
}

nsapi_version_t SocketAddress::get_ip_version() const
{
    nsapi_version_t ver = NSAPI_IPv6;
    return ver;
}

nsapi_addr_t SocketAddress::get_addr() const
{
    nsapi_addr_t addr;
    addr.version = NSAPI_IPv6;
    return _addr;
}

uint16_t SocketAddress::get_port() const
{
    return 0;
}

SocketAddress::operator bool() const
{
    return false;
}

SocketAddress &SocketAddress::operator=(const SocketAddress &addr)
{
    set_addr(addr.get_addr());
    set_port(addr.get_port());
    return *this;
}

bool operator==(const SocketAddress &a, const SocketAddress &b)
{
    return false;
}

bool operator!=(const SocketAddress &a, const SocketAddress &b)
{
    return false;
}
<commit_msg>Netsocket: Fix SocketAddress stub for new API<commit_after>/*
 * Copyright (c) , Arm Limited and affiliates.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "SocketAddress.h"

SocketAddress::SocketAddress(const nsapi_addr_t &addr, uint16_t port)
{
}

SocketAddress::SocketAddress(const char *addr, uint16_t port)
{
}

SocketAddress::SocketAddress(const void *bytes, nsapi_version_t version, uint16_t port)
{
}

SocketAddress::SocketAddress(const SocketAddress &addr)
{
}

bool SocketAddress::set_ip_address(const char *addr)
{
    return false;
}

void SocketAddress::set_ip_bytes(const void *bytes, nsapi_version_t version)
{
}

void SocketAddress::set_addr(const nsapi_addr_t &addr)
{
}

const char *SocketAddress::get_ip_address() const
{
    return NULL;
}

SocketAddress::operator bool() const
{
    return false;
}

SocketAddress &SocketAddress::operator=(const SocketAddress &addr)
{
    set_addr(addr.get_addr());
    set_port(addr.get_port());
    return *this;
}

bool operator==(const SocketAddress &a, const SocketAddress &b)
{
    return false;
}

bool operator!=(const SocketAddress &a, const SocketAddress &b)
{
    return false;
}
<|endoftext|>


<commit_before>/* -------------------------------------------------------------------------- */
/* Copyright 2002-2009, Distributed Systems Architecture Group, Universidad   */
/* Complutense de Madrid (dsa-research.org)                                   */
/*                                                                            */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
/* not use this file except in compliance with the License. You may obtain    */
/* a copy of the License at                                                   */
/*                                                                            */
/* http://www.apache.org/licenses/LICENSE-2.0                                 */
/*                                                                            */
/* Unless required by applicable law or agreed to in writing, software        */
/* distributed under the License is distributed on an "AS IS" BASIS,          */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
/* See the License for the specific language governing permissions and        */
/* limitations under the License.                                             */
/* -------------------------------------------------------------------------- */

#include <limits.h>
#include <string.h>

#include <iostream>
#include <sstream>

#include "SchedulerHost.h"
#include "Scheduler.h"

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

void SchedulerHost::get_capacity(int& cpu, int& memory, int threshold)
{
    int total_cpu;

    memory = get_share_free_mem();
    cpu    = get_share_free_cpu();

    total_cpu = get_share_max_cpu();

    /* eg. 96.7 >= 0.9 * 100, We need to round */
    if ( cpu >= threshold * total_cpu )
    {
        cpu = (int) ceil((float)cpu/100.0) * 100;
    }
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::insert(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not insert hosts in database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::update(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not update hosts in database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::drop(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not delete hosts from database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHostPool::allocate(
    PoolObjectSQL   *objsql)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not allocate hosts in database");

    return -1;
};

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

void SchedulerHostPool::bootstrap()
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not bootstrap database");
}

/* ************************************************************************** */
/* SchedulerHostPool                                                          */
/* ************************************************************************** */

extern "C"
{
    static int set_up_cb (
        void *                  _hids,
        int                     num,
        char **                 values,
        char **                 names)
    {
        vector<int> *    hids;

        hids = static_cast<vector<int> *>(_hids);

        if ((hids==0)||(num<=0)||(values[0]==0))
        {
            return -1;
        }

        hids->push_back(atoi(values[0]));

        return 0;
    };
}

/* -------------------------------------------------------------------------- */

int SchedulerHostPool::set_up()
{
    ostringstream   oss;
    int             rc;

    // -------------------------------------------------------------------------
    // Clean the pool to get updated data from db
    // -------------------------------------------------------------------------

    clean();

    hids.clear();

    // -------------------------------------------------------------------------
    // Load the ids (to get an updated list of hosts)
    // -------------------------------------------------------------------------

    lock();

    oss << "SELECT oid FROM " << Host::table
    	<< " WHERE state != " << Host::DISABLED;

    rc = db->exec(oss,set_up_cb,(void *) &hids);

    if ( rc != 0 )
    {
        unlock();

        return -1;
    }

    oss.str("");
    oss << "Discovered Hosts (enabled):";

    for (unsigned int i=0 ; i < hids.size() ; i++)
    {
        oss << " " << hids[i];
    }

    Scheduler::log("HOST",Log::DEBUG,oss);

    unlock();

    return 0;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

<commit_msg>#136: Only look for enabled and non-error hosts to schedule VMs<commit_after>/* -------------------------------------------------------------------------- */
/* Copyright 2002-2009, Distributed Systems Architecture Group, Universidad   */
/* Complutense de Madrid (dsa-research.org)                                   */
/*                                                                            */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
/* not use this file except in compliance with the License. You may obtain    */
/* a copy of the License at                                                   */
/*                                                                            */
/* http://www.apache.org/licenses/LICENSE-2.0                                 */
/*                                                                            */
/* Unless required by applicable law or agreed to in writing, software        */
/* distributed under the License is distributed on an "AS IS" BASIS,          */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
/* See the License for the specific language governing permissions and        */
/* limitations under the License.                                             */
/* -------------------------------------------------------------------------- */

#include <limits.h>
#include <string.h>

#include <iostream>
#include <sstream>

#include "SchedulerHost.h"
#include "Scheduler.h"

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

void SchedulerHost::get_capacity(int& cpu, int& memory, int threshold)
{
    int total_cpu;

    memory = get_share_free_mem();
    cpu    = get_share_free_cpu();

    total_cpu = get_share_max_cpu();

    /* eg. 96.7 >= 0.9 * 100, We need to round */
    if ( cpu >= threshold * total_cpu )
    {
        cpu = (int) ceil((float)cpu/100.0) * 100;
    }
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::insert(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not insert hosts in database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::update(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not update hosts in database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHost::drop(SqliteDB *db)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not delete hosts from database");

    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int SchedulerHostPool::allocate(
    PoolObjectSQL   *objsql)
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not allocate hosts in database");

    return -1;
};

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

void SchedulerHostPool::bootstrap()
{
    Scheduler::log("HOST",Log::ERROR,
                   "Scheduler can not bootstrap database");
}

/* ************************************************************************** */
/* SchedulerHostPool                                                          */
/* ************************************************************************** */

extern "C"
{
    static int set_up_cb (
        void *                  _hids,
        int                     num,
        char **                 values,
        char **                 names)
    {
        vector<int> *    hids;

        hids = static_cast<vector<int> *>(_hids);

        if ((hids==0)||(num<=0)||(values[0]==0))
        {
            return -1;
        }

        hids->push_back(atoi(values[0]));

        return 0;
    };
}

/* -------------------------------------------------------------------------- */

int SchedulerHostPool::set_up()
{
    ostringstream   oss;
    int             rc;

    // -------------------------------------------------------------------------
    // Clean the pool to get updated data from db
    // -------------------------------------------------------------------------

    clean();

    hids.clear();

    // -------------------------------------------------------------------------
    // Load the ids (to get an updated list of hosts)
    // -------------------------------------------------------------------------

    lock();

    oss << "SELECT oid FROM " << Host::table
    	<< " WHERE state != " << Host::DISABLED
        << " AND state != " << Host::ERROR;

    rc = db->exec(oss,set_up_cb,(void *) &hids);

    if ( rc != 0 )
    {
        unlock();

        return -1;
    }

    oss.str("");
    oss << "Discovered Hosts (enabled):";

    for (unsigned int i=0 ; i < hids.size() ; i++)
    {
        oss << " " << hids[i];
    }

    Scheduler::log("HOST",Log::DEBUG,oss);

    unlock();

    return 0;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

<|endoftext|>


<commit_before>// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Copyright (c) 2014-2019 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chainparams.h>
#include <init.h>
#include <interfaces/chain.h>
#include <net.h>
#include <scheduler.h>
#include <outputtype.h>
#include <util/error.h>
#include <util/system.h>
#include <util/moneystr.h>
#include <validation.h>
#include <walletinitinterface.h>
#include <wallet/rpcwallet.h>
#include <wallet/wallet.h>
#include <wallet/walletutil.h>

class WalletInit : public WalletInitInterface {
public:

    //! Return the wallets help message.
    void AddWalletOptions() const override;

    //! Was the wallet component compiled in.
    bool HasWalletSupport() const override {return true;}

    //! Wallets parameter interaction
    bool ParameterInteraction() const override;

    //! Add wallets that should be opened to list of init interfaces.
    void Construct(InitInterfaces& interfaces) const override;
};

const WalletInitInterface& g_wallet_init_interface = WalletInit();

void WalletInit::AddWalletOptions() const
{
    gArgs.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-avoidpartialspends", strprintf("Group outputs by address, selecting all or none, instead of selecting on a per-output basis. Privacy is improved as an address is only used once (unless someone sends to it after spending from it), but may result in slightly higher fees as suboptimal coin selection may result due to the added limitation (default: %u)", DEFAULT_AVOIDPARTIALSPENDS), false, OptionsCategory::WALLET);
    gArgs.AddArg("-changetype", "What type of change to use (\"legacy\", \"p2sh-segwit\", or \"bech32\"). Default is same as -addresstype, except when -addresstype=p2sh-segwit a native segwit output is used when sending to a native segwit address)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-disablewallet", "Do not load the wallet and disable wallet RPC calls", false, OptionsCategory::WALLET);
    gArgs.AddArg("-discardfee=<amt>", strprintf("The fee rate (in %s/kB) that indicates your tolerance for discarding change by adding it to the fee (default: %s). "
                                                                "Note: An output is discarded if it is dust at this rate, but we will always discard up to the dust relay fee and a discard fee above that is limited by the fee estimate for the longest target",
                                                              CURRENCY_UNIT, FormatMoney(DEFAULT_DISCARD_FEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)",
                                                               CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), false, OptionsCategory::WALLET);
    gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)",
        CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST);
    gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)",
                                                            CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)",
                                                            CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), false, OptionsCategory::WALLET);
    gArgs.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), false, OptionsCategory::WALLET);
    gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), false, OptionsCategory::WALLET);
    gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletbroadcast",  strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletnotify=<cmd>", "Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), false, OptionsCategory::WALLET);
    gArgs.AddArg("-zapwallettxes=<mode>", "Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup"
                               " (1 = keep tx meta data e.g. payment request information, 2 = drop tx meta data)", false, OptionsCategory::WALLET);

    gArgs.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), true, OptionsCategory::WALLET_DEBUG_TEST);
}

bool WalletInit::ParameterInteraction() const
{
    if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
        for (const std::string& wallet : gArgs.GetArgs("-wallet")) {
            LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet);
        }

        return true;
    }

    const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1;

    if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && gArgs.SoftSetBoolArg("-walletbroadcast", false)) {
        LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__);
    }

    if (gArgs.GetBoolArg("-salvagewallet", false)) {
        if (is_multiwallet) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet"));
        }
        // Rewrite just private keys: rescan to find transactions
        if (gArgs.SoftSetBoolArg("-rescan", true)) {
            LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__);
        }
    }

    bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false);
    // -zapwallettxes implies dropping the mempool on startup
    if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) {
        LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -persistmempool=0\n", __func__);
    }

    // -zapwallettxes implies a rescan
    if (zapwallettxes) {
        if (is_multiwallet) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes"));
        }
        if (gArgs.SoftSetBoolArg("-rescan", true)) {
            LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -rescan=1\n", __func__);
        }
    }

    if (is_multiwallet) {
        if (gArgs.GetBoolArg("-upgradewallet", false)) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet"));
        }
    }

    if (gArgs.GetBoolArg("-sysperms", false))
        return InitError("-sysperms is not allowed in combination with enabled wallet functionality");
    if (gArgs.GetArg("-prune", 0) && gArgs.GetBoolArg("-rescan", false))
        return InitError(_("Rescans are not possible in pruned mode. You will need to use -reindex which will download the whole blockchain again."));

    return true;
}

void WalletInit::Construct(InitInterfaces& interfaces) const
{
    if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
        LogPrintf("Wallet disabled!\n");
        return;
    }
    gArgs.SoftSetArg("-wallet", "");
    interfaces.chain_clients.emplace_back(interfaces::MakeWalletClient(*interfaces.chain, gArgs.GetArgs("-wallet")));
}<commit_msg>Bitcoin: 5ebc6b0eb267e0552c66fffc5e5afe7df8becf80 (bitcoind: update -avoidpartialspends description to account for auto-enable for avoid_reuse wallets).<commit_after>// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Copyright (c) 2014-2019 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chainparams.h>
#include <init.h>
#include <interfaces/chain.h>
#include <net.h>
#include <scheduler.h>
#include <outputtype.h>
#include <util/error.h>
#include <util/system.h>
#include <util/moneystr.h>
#include <validation.h>
#include <walletinitinterface.h>
#include <wallet/rpcwallet.h>
#include <wallet/wallet.h>
#include <wallet/walletutil.h>

class WalletInit : public WalletInitInterface {
public:

    //! Return the wallets help message.
    void AddWalletOptions() const override;

    //! Was the wallet component compiled in.
    bool HasWalletSupport() const override {return true;}

    //! Wallets parameter interaction
    bool ParameterInteraction() const override;

    //! Add wallets that should be opened to list of init interfaces.
    void Construct(InitInterfaces& interfaces) const override;
};

const WalletInitInterface& g_wallet_init_interface = WalletInit();

void WalletInit::AddWalletOptions() const
{
    gArgs.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-avoidpartialspends", strprintf("Group outputs by address, selecting all or none, instead of selecting on a per-output basis. Privacy is improved as an address is only used once (unless someone sends to it after spending from it), but may result in slightly higher fees as suboptimal coin selection may result due to the added limitation (default: %u (always enabled for wallets with \"avoid_reuse\" enabled))", DEFAULT_AVOIDPARTIALSPENDS), false, OptionsCategory::WALLET);
    gArgs.AddArg("-changetype", "What type of change to use (\"legacy\", \"p2sh-segwit\", or \"bech32\"). Default is same as -addresstype, except when -addresstype=p2sh-segwit a native segwit output is used when sending to a native segwit address)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-disablewallet", "Do not load the wallet and disable wallet RPC calls", false, OptionsCategory::WALLET);
    gArgs.AddArg("-discardfee=<amt>", strprintf("The fee rate (in %s/kB) that indicates your tolerance for discarding change by adding it to the fee (default: %s). "
                                                                "Note: An output is discarded if it is dust at this rate, but we will always discard up to the dust relay fee and a discard fee above that is limited by the fee estimate for the longest target",
                                                              CURRENCY_UNIT, FormatMoney(DEFAULT_DISCARD_FEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)",
                                                               CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), false, OptionsCategory::WALLET);
    gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)",
        CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST);
    gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)",
                                                            CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), false, OptionsCategory::WALLET);
    gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)",
                                                            CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), false, OptionsCategory::WALLET);
    gArgs.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), false, OptionsCategory::WALLET);
    gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), false, OptionsCategory::WALLET);
    gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", false, OptionsCategory::WALLET);
    gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletbroadcast",  strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletnotify=<cmd>", "Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)", false, OptionsCategory::WALLET);
    gArgs.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), false, OptionsCategory::WALLET);
    gArgs.AddArg("-zapwallettxes=<mode>", "Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup"
                               " (1 = keep tx meta data e.g. payment request information, 2 = drop tx meta data)", false, OptionsCategory::WALLET);

    gArgs.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), true, OptionsCategory::WALLET_DEBUG_TEST);
    gArgs.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), true, OptionsCategory::WALLET_DEBUG_TEST);
}

bool WalletInit::ParameterInteraction() const
{
    if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
        for (const std::string& wallet : gArgs.GetArgs("-wallet")) {
            LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet);
        }

        return true;
    }

    const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1;

    if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && gArgs.SoftSetBoolArg("-walletbroadcast", false)) {
        LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__);
    }

    if (gArgs.GetBoolArg("-salvagewallet", false)) {
        if (is_multiwallet) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet"));
        }
        // Rewrite just private keys: rescan to find transactions
        if (gArgs.SoftSetBoolArg("-rescan", true)) {
            LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__);
        }
    }

    bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false);
    // -zapwallettxes implies dropping the mempool on startup
    if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) {
        LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -persistmempool=0\n", __func__);
    }

    // -zapwallettxes implies a rescan
    if (zapwallettxes) {
        if (is_multiwallet) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes"));
        }
        if (gArgs.SoftSetBoolArg("-rescan", true)) {
            LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -rescan=1\n", __func__);
        }
    }

    if (is_multiwallet) {
        if (gArgs.GetBoolArg("-upgradewallet", false)) {
            return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet"));
        }
    }

    if (gArgs.GetBoolArg("-sysperms", false))
        return InitError("-sysperms is not allowed in combination with enabled wallet functionality");
    if (gArgs.GetArg("-prune", 0) && gArgs.GetBoolArg("-rescan", false))
        return InitError(_("Rescans are not possible in pruned mode. You will need to use -reindex which will download the whole blockchain again."));

    return true;
}

void WalletInit::Construct(InitInterfaces& interfaces) const
{
    if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
        LogPrintf("Wallet disabled!\n");
        return;
    }
    gArgs.SoftSetArg("-wallet", "");
    interfaces.chain_clients.emplace_back(interfaces::MakeWalletClient(*interfaces.chain, gArgs.GetArgs("-wallet")));
}<|endoftext|>


<commit_before>/*
 * Qt4 bitcoin GUI.
 *
 * W.J. van der Laan 2011-2012
 * The Bitcoin Developers 2011-2013
 */
#include "walletstack.h"
#include "walletview.h"
#include "bitcoingui.h"

#include <QMap>
#include <QMessageBox>

WalletStack::WalletStack(QWidget *parent) :
    QStackedWidget(parent),
    clientModel(0),
    bOutOfSync(true)
{
}

WalletStack::~WalletStack()
{
}

bool WalletStack::addWalletView(const QString& name, WalletModel *walletModel)
{
    if (!gui || !clientModel || mapWalletViews.count(name) > 0)
        return false;

    WalletView *walletView = new WalletView(this, gui);
    walletView->setBitcoinGUI(gui);
    walletView->setClientModel(clientModel);
    walletView->setWalletModel(walletModel);
    walletView->showOutOfSyncWarning(bOutOfSync);
    addWidget(walletView);
    mapWalletViews[name] = walletView;
    return true;

}

bool WalletStack::removeWalletView(const QString& name)
{
    if (mapWalletViews.count(name) == 0) return false;
    WalletView *walletView = mapWalletViews.take(name);
    removeWidget(walletView);
    return true;
}

bool WalletStack::handleURI(const QString &uri)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (!walletView) return false;

    return walletView->handleURI(uri);
}

void WalletStack::showOutOfSyncWarning(bool fShow)
{
    bOutOfSync = fShow;
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->showOutOfSyncWarning(fShow);
}

void WalletStack::gotoOverviewPage()
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->gotoOverviewPage();
}

void WalletStack::gotoHistoryPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
        if ((WalletView*)currentWidget() == i.value())
          i.value()->gotoHistoryPage(false, true, false);
        else
          i.value()->gotoHistoryPage(false, false, false);
    }
}

void WalletStack::gotoAddressBookPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
       if ((WalletView*)currentWidget() == i.value())
         i.value()->gotoAddressBookPage(false, true, false);
       else
         i.value()->gotoAddressBookPage(false, false, false);
    }
}

void WalletStack::gotoReceiveCoinsPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
       if ((WalletView*)currentWidget() == i.value())
         i.value()->gotoReceiveCoinsPage(false, true, false);
       else
         i.value()->gotoReceiveCoinsPage(false, false, false);
    }
}

void WalletStack::gotoSendCoinsPage()
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->gotoSendCoinsPage();
}

void WalletStack::gotoSignMessageTab(QString addr)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->gotoSignMessageTab(addr);
}

void WalletStack::gotoVerifyMessageTab(QString addr)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->gotoVerifyMessageTab(addr);
}

void WalletStack::encryptWallet(bool status)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->encryptWallet(status);
}

void WalletStack::backupWallet()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->backupWallet();
}

void WalletStack::changePassphrase()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->changePassphrase();
}

void WalletStack::unlockWallet()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->unlockWallet();
}

void WalletStack::setEncryptionStatus()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->setEncryptionStatus();
}

QString WalletStack::getCurrentWallet()
{
  return QString (mapWalletViews.key((WalletView*)currentWidget()));
}

void WalletStack::setCurrentWalletView(const QString& name)
{
    if (mapWalletViews.count(name) == 0) return;
    WalletView *walletView = mapWalletViews.value(name);
    setCurrentWidget(walletView);
    walletView->setEncryptionStatus();

    //Call the pages that have export and only set that connect
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
        bool x;
        if ( mapWalletViews.constBegin() == (i) )
            x=true;
        else
            x=false;

        if ((WalletView*)currentWidget() == i.value())
        {
          i.value()->gotoReceiveCoinsPage(true, true, x);
          i.value()->gotoHistoryPage(true, true, x);
          i.value()->gotoAddressBookPage(true, true, x);
        }
        else
        {
          i.value()->gotoReceiveCoinsPage(true, false, x);
          i.value()->gotoHistoryPage(true, false, x);
          i.value()->gotoAddressBookPage(true, false, x );
        }
     }
}
<commit_msg>V1.3<commit_after>/*
 * Qt4 bitcoin GUI.
 *
 * W.J. van der Laan 2011-2012
 * The Bitcoin Developers 2011-2013
 */
#include "walletstack.h"
#include "walletview.h"
#include "bitcoingui.h"

#include <QMap>
#include <QMessageBox>

WalletStack::WalletStack(QWidget *parent) :
    QStackedWidget(parent),
    gui(0),
    clientModel(0),
    bOutOfSync(true)
{
}

WalletStack::~WalletStack()
{
}

bool WalletStack::addWalletView(const QString& name, WalletModel *walletModel)
{
    if (!gui || !clientModel || mapWalletViews.count(name) > 0)
        return false;

    WalletView *walletView = new WalletView(this, gui);
    walletView->setBitcoinGUI(gui);
    walletView->setClientModel(clientModel);
    walletView->setWalletModel(walletModel);
    walletView->showOutOfSyncWarning(bOutOfSync);
    addWidget(walletView);
    mapWalletViews[name] = walletView;

    // Ensure a walletView is able to show the main window
    connect(walletView, SIGNAL(showNormalIfMinimized()), gui, SLOT(showNormalIfMinimized()));
    return true;

}

bool WalletStack::removeWalletView(const QString& name)
{
    if (mapWalletViews.count(name) == 0) return false;
    WalletView *walletView = mapWalletViews.take(name);
    removeWidget(walletView);
    return true;
}

bool WalletStack::handleURI(const QString &uri)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (!walletView) return false;

    return walletView->handleURI(uri);
}

void WalletStack::showOutOfSyncWarning(bool fShow)
{
    bOutOfSync = fShow;
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->showOutOfSyncWarning(fShow);
}

void WalletStack::gotoOverviewPage()
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->gotoOverviewPage();
}

void WalletStack::gotoHistoryPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
        if ((WalletView*)currentWidget() == i.value())
          i.value()->gotoHistoryPage(false, true, false);
        else
          i.value()->gotoHistoryPage(false, false, false);
    }
}

void WalletStack::gotoAddressBookPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
       if ((WalletView*)currentWidget() == i.value())
         i.value()->gotoAddressBookPage(false, true, false);
       else
         i.value()->gotoAddressBookPage(false, false, false);
    }
}

void WalletStack::gotoReceiveCoinsPage(bool fExportOnly, bool fExportConnect, bool fExportFirstTime)
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
       if ((WalletView*)currentWidget() == i.value())
         i.value()->gotoReceiveCoinsPage(false, true, false);
       else
         i.value()->gotoReceiveCoinsPage(false, false, false);
    }
}

void WalletStack::gotoSendCoinsPage()
{
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
        i.value()->gotoSendCoinsPage();
}

void WalletStack::gotoSignMessageTab(QString addr)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->gotoSignMessageTab(addr);
}

void WalletStack::gotoVerifyMessageTab(QString addr)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->gotoVerifyMessageTab(addr);
}

void WalletStack::encryptWallet(bool status)
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->encryptWallet(status);
}

void WalletStack::backupWallet()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->backupWallet();
}

void WalletStack::changePassphrase()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->changePassphrase();
}

void WalletStack::unlockWallet()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->unlockWallet();
}

void WalletStack::setEncryptionStatus()
{
    WalletView *walletView = (WalletView*)currentWidget();
    if (walletView) walletView->setEncryptionStatus();
}

QString WalletStack::getCurrentWallet()
{
  return QString (mapWalletViews.key((WalletView*)currentWidget()));
}

void WalletStack::setCurrentWalletView(const QString& name)
{
    if (mapWalletViews.count(name) == 0) return;
    WalletView *walletView = mapWalletViews.value(name);
    setCurrentWidget(walletView);
    walletView->setEncryptionStatus();

    //Call the pages that have export and only set that connect
    QMap<QString, WalletView*>::const_iterator i;
    for (i = mapWalletViews.constBegin(); i != mapWalletViews.constEnd(); ++i)
    {
        bool x;
        if ( mapWalletViews.constBegin() == (i) )
            x=true;
        else
            x=false;

        if ((WalletView*)currentWidget() == i.value())
        {
          i.value()->gotoReceiveCoinsPage(true, true, x);
          i.value()->gotoHistoryPage(true, true, x);
          i.value()->gotoAddressBookPage(true, true, x);
        }
        else
        {
          i.value()->gotoReceiveCoinsPage(true, false, x);
          i.value()->gotoHistoryPage(true, false, x);
          i.value()->gotoAddressBookPage(true, false, x );
        }
     }
}
<|endoftext|>


<commit_before>#include "clstm.h"
#include <assert.h>
#include <math.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <memory>
#include <regex>
#include <set>
#include <sstream>
#include <vector>
#include "clstmhl.h"
#include "extras.h"
#include "pstring.h"
#include "utils.h"

using namespace Eigen;
using namespace ocropus;
using std::vector;
using std::map;
using std::make_pair;
using std::shared_ptr;
using std::unique_ptr;
using std::cout;
using std::ifstream;
using std::set;
using std::to_string;
using std_string = std::string;
using std_wstring = std::wstring;
using std::regex;
using std::regex_replace;
#define string std_string
#define wstring std_wstring

#ifndef NODISPLAY
void show(PyServer &py, Sequence &s, int subplot = 0, int batch = 0) {
  Tensor<float, 2> temp;
  temp.resize(s.size(), s.rows());
  for (int i = 0; i < s.size(); i++)
    for (int j = 0; j < s.rows(); j++) temp(i, j) = s[i].v(j, batch);
  if (subplot > 0) py.evalf("subplot(%d)", subplot);
  py.imshowT(temp, "cmap=cm.hot");
}
#endif

wstring separate_chars(const wstring &s, const wstring &charsep) {
  if (charsep == L"") return s;
  wstring result;
  for (int i = 0; i < s.size(); i++) {
    if (i > 0) result.push_back(charsep[0]);
    result.push_back(s[i]);
  }
  return result;
}

struct Dataset {
  vector<string> fnames;
  wstring charsep = utf8_to_utf32(getsenv("charsep", ""));
  int size() { return fnames.size(); }
  Dataset() {}
  Dataset(string file_list) { readFileList(file_list); }
  void readFileList(string file_list) { read_lines(fnames, file_list); }
  void getCodec(Codec &codec) {
    vector<string> gtnames;
    for (auto s : fnames) gtnames.push_back(basename(s) + ".gt.txt");
    codec.build(gtnames, charsep);
  }

  string readSample(Tensor2 &raw, wstring &gt, int index, bool drouOut = false) {

	string fname = fnames[index];
	try{

		string base = basename(fname);
		gt = separate_chars(read_text32(base + ".gt.txt"), charsep);
		read_png(raw, fname.c_str());
		raw() = -raw() + Float(1);
		if( drouOut )
		{
			int N = raw.dimension(0);
			int d = raw.dimension(1);
			for (int t = 0; t < N; t++)
				for (int i = 0; i < d; i++){
					if(lrand48() % 10 > 5)
						raw(t, i) = 0;
				}
		}
	}catch(...){
		std::cout << "Bad sample: " << fname << std::endl;
	}
	return fname;
  }
};

pair<double, double> test_set_error(CLSTMOCR &clstm, Dataset &testset, double& test50) {
  double count = 0.0;
  double errors = 0.0;
  test50 = 0;
  double count50;
  for (int test = 0; test < testset.size(); test++) {
    Tensor2 raw;
    wstring gt;
    testset.readSample(raw, gt, test);
    wstring pred = clstm.predict(raw());
    count += gt.size();
    int dist = levenshtein(pred, gt);
    errors += dist;
    if( test < 50 ){
    	test50 += dist;
    	count50 += gt.size();
    }
  }
  test50 /= count50;
  return make_pair(errors, count);
}

int main1(int argc, char **argv) {
  int ntrain = getienv("ntrain", 10000000);
  string save_name = getsenv("save_name", "_ocr");
  int report_time = getienv("report_time", 0);

  if (argc < 2 || argc > 3) THROW("... training [testing]");
  Dataset trainingset(argv[1]);
  assert(trainingset.size() > 0);
  Dataset testset;
  if (argc > 2) testset.readFileList(argv[2]);
  print("got", trainingset.size(), "files,", testset.size(), "tests");

  string load_name = getsenv("load", "");

  CLSTMOCR clstm;

  if (load_name != "") {
    clstm.load(load_name);
  } else {
    Codec codec;
    trainingset.getCodec(codec);
    print("got", codec.size(), "classes");

    clstm.target_height = int(getrenv("target_height", 48));
    clstm.createBidi2(codec.codec, getienv("nhidden", 100));
    clstm.setLearningRate(getdenv("lrate", 1e-4), getdenv("momentum", 0.9));
  }
  network_info(clstm.net);

  double test_error = 9999.0;
  double best_error = 1e38;
  double best_test50 = 1e38;

#ifndef NODISPLAY
  PyServer py;
  if (display_every > 0) py.open();
#endif
  double start_time = now();
  int start = clstm.net->attr.get("trial", getienv("start", -1)) + 1;
  if (start > 0) print("start", start);

  Trigger test_trigger(getienv("test_every", 10000), -1, start);
  test_trigger.skip0();
  Trigger save_trigger(getienv("save_every", 10000), ntrain, start);
  save_trigger.enable(save_name != "").skip0();
  Trigger report_trigger(getienv("report_every", 100), ntrain, start);
  Trigger display_trigger(getienv("display_every", 0), ntrain, start);

 int fromSamples = 450 + 10000;

for (int trial = start; trial < ntrain; trial++) {
    int sample = lrand48() % trainingset.size();
    if( true )
    {
		fromSamples += 1;
		fromSamples = std::min(fromSamples, trainingset.size() );
    	sample = lrand48() % fromSamples;
    }
    Tensor2 raw;
    wstring gt;
    string fname = trainingset.readSample(raw, gt, sample);

    wstring pred = clstm.train(raw(), gt);

    if (report_trigger(trial)) {
      std::cout << fname << std::endl;
      double dist = levenshtein(pred, gt);

      print(trial);
      print("TRU", gt);
      print("ALN", clstm.aligned_utf8());
      print("OUT", utf32_to_utf8(pred), dist / gt.size());
      if (trial > 0 && report_time)
        print("steptime", (now() - start_time) / report_trigger.since());
      start_time = now();
    }

#ifndef NODISPLAY
    if (display_trigger(trial)) {
      py.evalf("clf");
      show(py, clstm.net->inputs, 411);
      show(py, clstm.net->outputs, 412);
      show(py, clstm.targets, 413);
      show(py, clstm.aligned, 414);
    }
#endif

    if (test_trigger(trial) && trial > start + 1) {
      double test50;
      auto tse = test_set_error(clstm, testset, test50);
      double errors = tse.first;
      double count = tse.second;
      test_error = errors / count;
      print("ERROR", trial, test_error, "   ", errors, count,  errors/(float)count, test50);
      if (test_error < best_error) {
        best_error = test_error;
        string fname = save_name + ".clstm";
        print("saving best performing network so far", fname, "error rate: ",
              best_error, test50);
        clstm.net->attr.set("trial", trial);
        clstm.save(fname);
	best_test50 = std::min(test50, best_test50);
      }else if( test50 < best_test50  )
      {
    	  string fname = save_name + ".clstm";
    	  print("saving best performing network so far", fname, "error rate: ",
    			  best_error, test50);
    	  clstm.net->attr.set("trial", trial);
    	  clstm.save(fname);
    	  best_test50 = test50;
      }
    }
    /*
    if (save_trigger(trial)) {
      string fname = save_name + "-" + to_string(trial) + ".clstm";
      print("saving", fname);
      clstm.net->attr.set("trial", trial);
      clstm.save(fname);
    }*/
  }

  return 0;
}

int main(int argc, char **argv) {
  TRY { main1(argc, argv); }
  CATCH(const char *message) { cerr << "FATAL: " << message << endl; }
}
<commit_msg>test dropout<commit_after>#include "clstm.h"
#include <assert.h>
#include <math.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <memory>
#include <regex>
#include <set>
#include <sstream>
#include <vector>
#include "clstmhl.h"
#include "extras.h"
#include "pstring.h"
#include "utils.h"

using namespace Eigen;
using namespace ocropus;
using std::vector;
using std::map;
using std::make_pair;
using std::shared_ptr;
using std::unique_ptr;
using std::cout;
using std::ifstream;
using std::set;
using std::to_string;
using std_string = std::string;
using std_wstring = std::wstring;
using std::regex;
using std::regex_replace;
#define string std_string
#define wstring std_wstring

#ifndef NODISPLAY
void show(PyServer &py, Sequence &s, int subplot = 0, int batch = 0) {
  Tensor<float, 2> temp;
  temp.resize(s.size(), s.rows());
  for (int i = 0; i < s.size(); i++)
    for (int j = 0; j < s.rows(); j++) temp(i, j) = s[i].v(j, batch);
  if (subplot > 0) py.evalf("subplot(%d)", subplot);
  py.imshowT(temp, "cmap=cm.hot");
}
#endif

wstring separate_chars(const wstring &s, const wstring &charsep) {
  if (charsep == L"") return s;
  wstring result;
  for (int i = 0; i < s.size(); i++) {
    if (i > 0) result.push_back(charsep[0]);
    result.push_back(s[i]);
  }
  return result;
}

struct Dataset {
  vector<string> fnames;
  wstring charsep = utf8_to_utf32(getsenv("charsep", ""));
  int size() { return fnames.size(); }
  Dataset() {}
  Dataset(string file_list) { readFileList(file_list); }
  void readFileList(string file_list) { read_lines(fnames, file_list); }
  void getCodec(Codec &codec) {
    vector<string> gtnames;
    for (auto s : fnames) gtnames.push_back(basename(s) + ".gt.txt");
    codec.build(gtnames, charsep);
  }

  string readSample(Tensor2 &raw, wstring &gt, int index, bool drouOut = false) {

	string fname = fnames[index];
	try{

		string base = basename(fname);
		gt = separate_chars(read_text32(base + ".gt.txt"), charsep);
		read_png(raw, fname.c_str());
		raw() = -raw() + Float(1);
		if( drouOut )
		{
			int N = raw.dimension(0);
			int d = raw.dimension(1);
			for (int t = 0; t < N; t++)
				for (int i = 0; i < d; i++){
					if(lrand48() % 10 > 5)
						raw(t, i) = 0;
				}
		}
	}catch(...){
		std::cout << "Bad sample: " << fname << std::endl;
	}
	return fname;
  }
};

pair<double, double> test_set_error(CLSTMOCR &clstm, Dataset &testset, double& test50) {
  double count = 0.0;
  double errors = 0.0;
  test50 = 0;
  double count50;
  for (int test = 0; test < testset.size(); test++) {
    Tensor2 raw;
    wstring gt;
    testset.readSample(raw, gt, test);
    wstring pred = clstm.predict(raw());
    count += gt.size();
    int dist = levenshtein(pred, gt);
    errors += dist;
    if( test < 50 ){
    	test50 += dist;
    	count50 += gt.size();
    }
  }
  test50 /= count50;
  return make_pair(errors, count);
}

int main1(int argc, char **argv) {
  int ntrain = getienv("ntrain", 10000000);
  string save_name = getsenv("save_name", "_ocr");
  int report_time = getienv("report_time", 0);

  if (argc < 2 || argc > 3) THROW("... training [testing]");
  Dataset trainingset(argv[1]);
  assert(trainingset.size() > 0);
  Dataset testset;
  if (argc > 2) testset.readFileList(argv[2]);
  print("got", trainingset.size(), "files,", testset.size(), "tests");

  string load_name = getsenv("load", "");

  CLSTMOCR clstm;

  if (load_name != "") {
    clstm.load(load_name);
  } else {
    Codec codec;
    trainingset.getCodec(codec);
    print("got", codec.size(), "classes");

    clstm.target_height = int(getrenv("target_height", 48));
    clstm.createBidi2(codec.codec, getienv("nhidden", 100));
    clstm.setLearningRate(getdenv("lrate", 1e-4), getdenv("momentum", 0.9));
  }
  network_info(clstm.net);

  double test_error = 9999.0;
  double best_error = 1e38;
  double best_test50 = 1e38;

#ifndef NODISPLAY
  PyServer py;
  if (display_every > 0) py.open();
#endif
  double start_time = now();
  int start = clstm.net->attr.get("trial", getienv("start", -1)) + 1;
  if (start > 0) print("start", start);

  Trigger test_trigger(getienv("test_every", 10000), -1, start);
  test_trigger.skip0();
  Trigger save_trigger(getienv("save_every", 10000), ntrain, start);
  save_trigger.enable(save_name != "").skip0();
  Trigger report_trigger(getienv("report_every", 100), ntrain, start);
  Trigger display_trigger(getienv("display_every", 0), ntrain, start);

 int fromSamples = 450 + 10000;

for (int trial = start; trial < ntrain; trial++) {
    int sample = lrand48() % trainingset.size();
    if( true )
    {
		fromSamples += 1;
		fromSamples = std::min(fromSamples, trainingset.size() );
    	sample = lrand48() % fromSamples;
    }
    Tensor2 raw;
    wstring gt;
    string fname = trainingset.readSample(raw, gt, sample, true);

    wstring pred = clstm.train(raw(), gt);

    if (report_trigger(trial)) {
      std::cout << fname << std::endl;
      double dist = levenshtein(pred, gt);

      print(trial);
      print("TRU", gt);
      print("ALN", clstm.aligned_utf8());
      print("OUT", utf32_to_utf8(pred), dist / gt.size());
      if (trial > 0 && report_time)
        print("steptime", (now() - start_time) / report_trigger.since());
      start_time = now();
    }

#ifndef NODISPLAY
    if (display_trigger(trial)) {
      py.evalf("clf");
      show(py, clstm.net->inputs, 411);
      show(py, clstm.net->outputs, 412);
      show(py, clstm.targets, 413);
      show(py, clstm.aligned, 414);
    }
#endif

    if (test_trigger(trial) && trial > start + 1) {
      double test50;
      auto tse = test_set_error(clstm, testset, test50);
      double errors = tse.first;
      double count = tse.second;
      test_error = errors / count;
      print("ERROR", trial, test_error, "   ", errors, count,  errors/(float)count, test50);
      if (test_error < best_error) {
        best_error = test_error;
        string fname = save_name + ".clstm";
        print("saving best performing network so far", fname, "error rate: ",
              best_error, test50);
        clstm.net->attr.set("trial", trial);
        clstm.save(fname);
	best_test50 = std::min(test50, best_test50);
      }else if( test50 < best_test50  )
      {
    	  string fname = save_name + ".clstm";
    	  print("saving best performing network so far", fname, "error rate: ",
    			  best_error, test50);
    	  clstm.net->attr.set("trial", trial);
    	  clstm.save(fname);
    	  best_test50 = test50;
      }
    }
    /*
    if (save_trigger(trial)) {
      string fname = save_name + "-" + to_string(trial) + ".clstm";
      print("saving", fname);
      clstm.net->attr.set("trial", trial);
      clstm.save(fname);
    }*/
  }

  return 0;
}

int main(int argc, char **argv) {
  TRY { main1(argc, argv); }
  CATCH(const char *message) { cerr << "FATAL: " << message << endl; }
}
<|endoftext|>