<commit_before>#ifndef __se3_urdf_hpp__
#define __se3_urdf_hpp__

#include <urdf_model/model.h>
#include <urdf_parser/urdf_parser.h>

#include <iostream>
#include <boost/foreach.hpp>
#include "pinocchio/multibody/model.hpp"

namespace urdf
{
  typedef boost::shared_ptr<ModelInterface> ModelInterfacePtr;
  typedef boost::shared_ptr<const Joint> JointConstPtr;
  typedef boost::shared_ptr<const Link> LinkConstPtr;
  typedef boost::shared_ptr<Link> LinkPtr;

  typedef boost::shared_ptr<const Inertial> InertialConstPtr;
}

namespace se3
{
  namespace urdf
  {

    Inertia convertFromUrdf( const ::urdf::Inertial& Y )
    {
      const ::urdf::Vector3 & p = Y.origin.position;
      const ::urdf::Rotation & q = Y.origin.rotation;

      const Eigen::Vector3d com(p.x,p.y,p.z);
      const Eigen::Matrix3d & R = Eigen::Quaterniond(q.w,q.x,q.y,q.z).matrix();

      Eigen::Matrix3d I; I << Y.ixx,Y.ixy,Y.ixz
			   ,  Y.ixy,Y.iyy,Y.iyz
			   ,  Y.ixz,Y.iyz,Y.izz;

      return Inertia(Y.mass,com,R*I*R.transpose());
    }

    SE3 convertFromUrdf( const ::urdf::Pose & M )
    {
      const ::urdf::Vector3 & p = M.position;
      const ::urdf::Rotation & q = M.rotation;
      return SE3( Eigen::Quaterniond(q.w,q.x,q.y,q.z).matrix(), Eigen::Vector3d(p.x,p.y,p.z));
    }

    enum AxisCartesian { AXIS_X, AXIS_Y, AXIS_Z, AXIS_UNALIGNED };
    AxisCartesian extractCartesianAxis( const ::urdf::Vector3 & axis )
    {
      if( (axis.x==1.0)&&(axis.y==0.0)&&(axis.z==0.0) )
	return AXIS_X;
      else if( (axis.x==0.0)&&(axis.y==1.0)&&(axis.z==0.0) )
	return AXIS_Y;
      else if( (axis.x==0.0)&&(axis.y==0.0)&&(axis.z==1.0) )
	return AXIS_Z;
      else
	return AXIS_UNALIGNED;
    }

    void parseTree( ::urdf::LinkConstPtr link, Model & model, bool freeFlyer )
    {
      ::urdf::JointConstPtr joint = link->parent_joint;

      // std::cout << " *** " << link->name << "    < attached by joint ";
      // if(joint)
      //   std::cout << "#" << link->parent_joint->name << std::endl;
      // else std::cout << "###ROOT" << std::endl;

      //assert(link->inertial && "The parser cannot accept trivial mass");
      const Inertia & Y = (link->inertial) ?
	convertFromUrdf(*link->inertial)
	: Inertia::Identity();
      //std::cout << "Inertia: " << Y << std::endl;

      bool visual = (link->visual) ? true : false;

      if(joint!=NULL)
	{
	  assert(link->getParent()!=NULL);

	  Model::Index parent
	    = (link->getParent()->parent_joint==NULL) ?
	    (freeFlyer ? 1 : 0)
	    : model.getBodyId( link->getParent()->parent_joint->name );
	  //std::cout << joint->name << " === " << parent << std::endl;

	  const SE3 & jointPlacement = convertFromUrdf(joint->parent_to_joint_origin_transform);

	  //std::cout << "Parent = " << parent << std::endl;
	  //std::cout << "Placement = " << (Matrix4)jointPlacement << std::endl;

	  switch(joint->type)
	    {
	    case ::urdf::Joint::REVOLUTE:
	    case ::urdf::Joint::CONTINUOUS: // Revolute with no joint limits
	      {
	    Eigen::Vector3d jointAxis(Eigen::Vector3d::Zero());
		AxisCartesian axis = extractCartesianAxis(joint->axis);
		switch(axis)
		  {
		  case AXIS_X:
		    model.addBody( parent, JointModelRX(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Y:
		    model.addBody( parent, JointModelRY(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Z:
		    model.addBody( parent, JointModelRZ(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_UNALIGNED:
		  	jointAxis= Eigen::Vector3d( joint->axis.x,joint->axis.y,joint->axis.z );
		    jointAxis.normalize();
		    model.addBody( parent, JointModelRevoluteUnaligned(jointAxis),
				   jointPlacement, Y, joint->name,link->name, visual );
		  	break;
		  default:
		    assert( false && "Fatal Error while extracting revolute joint axis");
		    break;
		  }
		break;
	      }
	    case ::urdf::Joint::PRISMATIC:
	      {
	    AxisCartesian axis = extractCartesianAxis(joint->axis);
		switch(axis)
		  {
		  case AXIS_X:
		    model.addBody( parent, JointModelPX(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Y:
		    model.addBody( parent, JointModelPY(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Z:
		    model.addBody( parent, JointModelPZ(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_UNALIGNED:
		  	std::cerr << "Bad axis = (" <<joint->axis.x<<","<<joint->axis.y
			      <<","<<joint->axis.z<<")" << std::endl;
		    assert(false && "Only X, Y or Z axis are accepted." );
		  	break;
		  default:
		    assert( false && "Fatal Error while extracting revolute joint axis");
		    break;
		  }
		break;
	      }
	    case ::urdf::Joint::FIXED:
	      {
	    model.mergeFixedBody(parent, jointPlacement, Y);
	    assert( (link->child_links.empty()) && "Fixed body has joint child. Shouldn't happen");
	    // For the moment, fixed bodies are handled only if end of chain (and only once)
		break;
	      }

	    default:
	      {
		std::cerr << "The joint type " << joint->type << " is not supported." << std::endl;
		assert(false && "Only revolute joint are accepted." );
		break;
	      }
	    }
	    }
      else if(freeFlyer)
	{ /* The link is the root of the body. */
	  model.addBody( 0, JointModelFreeFlyer(), SE3::Identity(), Y, "root", link->name, true );
	}

      BOOST_FOREACH(::urdf::LinkConstPtr child,link->child_links)
	{
	  parseTree( child,model,freeFlyer );
	}
    }

    Model buildModel( const std::string & filename, bool freeFlyer = false )
    {
      Model model;

      ::urdf::ModelInterfacePtr urdfTree = ::urdf::parseURDFFile (filename);
      parseTree(urdfTree->getRoot(),model,freeFlyer);
      return model;
    }

  } // namespace urdf
} // namespace se3

#endif // ifndef __se3_urdf_hpp__

<commit_msg>[Major] Handling case of urdf::Fixed joints during parsing of urdf file.<commit_after>#ifndef __se3_urdf_hpp__
#define __se3_urdf_hpp__

#include <urdf_model/model.h>
#include <urdf_parser/urdf_parser.h>

#include <iostream>
#include <boost/foreach.hpp>
#include "pinocchio/multibody/model.hpp"

namespace urdf
{
  typedef boost::shared_ptr<ModelInterface> ModelInterfacePtr;
  typedef boost::shared_ptr<const Joint> JointConstPtr;
  typedef boost::shared_ptr<const Link> LinkConstPtr;
  typedef boost::shared_ptr<Link> LinkPtr;
  typedef boost::shared_ptr<const Inertial> InertialConstPtr;
}

namespace se3
{
  namespace urdf
  {
    Inertia convertFromUrdf( const ::urdf::Inertial& Y )
    {
      const ::urdf::Vector3 & p = Y.origin.position;
      const ::urdf::Rotation & q = Y.origin.rotation;

      const Eigen::Vector3d com(p.x,p.y,p.z);
      const Eigen::Matrix3d & R = Eigen::Quaterniond(q.w,q.x,q.y,q.z).matrix();

      Eigen::Matrix3d I; I << Y.ixx,Y.ixy,Y.ixz
			   ,  Y.ixy,Y.iyy,Y.iyz
			   ,  Y.ixz,Y.iyz,Y.izz;
      return Inertia(Y.mass,com,R*I*R.transpose());
    }

    SE3 convertFromUrdf( const ::urdf::Pose & M )
    {
      const ::urdf::Vector3 & p = M.position;
      const ::urdf::Rotation & q = M.rotation;
      return SE3( Eigen::Quaterniond(q.w,q.x,q.y,q.z).matrix(), Eigen::Vector3d(p.x,p.y,p.z));
    }

    enum AxisCartesian { AXIS_X, AXIS_Y, AXIS_Z, AXIS_UNALIGNED };
    AxisCartesian extractCartesianAxis( const ::urdf::Vector3 & axis )
    {
      if( (axis.x==1.0)&&(axis.y==0.0)&&(axis.z==0.0) )
	return AXIS_X;
      else if( (axis.x==0.0)&&(axis.y==1.0)&&(axis.z==0.0) )
	return AXIS_Y;
      else if( (axis.x==0.0)&&(axis.y==0.0)&&(axis.z==1.0) )
	return AXIS_Z;
      else
	return AXIS_UNALIGNED;
    }

    void parseTree( ::urdf::LinkConstPtr link, Model & model, bool freeFlyer, SE3 placementOffset = SE3::Identity() )
    {

      ::urdf::JointConstPtr joint = link->parent_joint;
      SE3 nextPlacementOffset=SE3::Identity(); // in most cases, no offset for the next link

      // std::cout << " *** " << link->name << "    < attached by joint ";
      // if(joint)
      //   std::cout << "#" << link->parent_joint->name << std::endl;
      // else std::cout << "###ROOT" << std::endl;

      //std::cout << " *** " << link->name << "    < attached by joint ";

      //assert(link->inertial && "The parser cannot accept trivial mass");
      const Inertia & Y = (link->inertial) ?
	convertFromUrdf(*link->inertial)
	: Inertia::Identity();

     // std::cout << "placementOffset: " << placementOffset << std::endl;

      bool visual = (link->visual) ? true : false;

      if(joint!=NULL)
	{
	  assert(link->getParent()!=NULL);

	  Model::Index parent
	    = (link->getParent()->parent_joint==NULL) ?
	    (freeFlyer ? 1 : 0)
	    : model.getBodyId( link->getParent()->parent_joint->name );
	  //std::cout << joint->name << " === " << parent << std::endl;

      const SE3 & jointPlacement = placementOffset*convertFromUrdf(joint->parent_to_joint_origin_transform);

	  //std::cout << "Parent = " << parent << std::endl;
	  //std::cout << "Placement = " << (Matrix4)jointPlacement << std::endl;

	  switch(joint->type)
	    {
	    case ::urdf::Joint::REVOLUTE:
	    case ::urdf::Joint::CONTINUOUS: // Revolute with no joint limits
	      {
	    Eigen::Vector3d jointAxis(Eigen::Vector3d::Zero());
		AxisCartesian axis = extractCartesianAxis(joint->axis);
		switch(axis)
		  {
		  case AXIS_X:
		    model.addBody( parent, JointModelRX(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Y:
		    model.addBody( parent, JointModelRY(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Z:
		    model.addBody( parent, JointModelRZ(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_UNALIGNED:
		  	jointAxis= Eigen::Vector3d( joint->axis.x,joint->axis.y,joint->axis.z );
		    jointAxis.normalize();
		    model.addBody( parent, JointModelRevoluteUnaligned(jointAxis),
				   jointPlacement, Y, joint->name,link->name, visual );
		  	break;
		  default:
		    assert( false && "Fatal Error while extracting revolute joint axis");
		    break;
		  }
		break;
	      }
	    case ::urdf::Joint::PRISMATIC:
	      {
	    AxisCartesian axis = extractCartesianAxis(joint->axis);
		switch(axis)
		  {
		  case AXIS_X:
		    model.addBody( parent, JointModelPX(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Y:
		    model.addBody( parent, JointModelPY(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_Z:
		    model.addBody( parent, JointModelPZ(), jointPlacement, Y, joint->name,link->name, visual );
		    break;
		  case AXIS_UNALIGNED:
		  	std::cerr << "Bad axis = (" <<joint->axis.x<<","<<joint->axis.y
			      <<","<<joint->axis.z<<")" << std::endl;
		    assert(false && "Only X, Y or Z axis are accepted." );
		  	break;
		  default:
		    assert( false && "Fatal Error while extracting revolute joint axis");
		    break;
		  }
		break;
	      }
	    case ::urdf::Joint::FIXED:
	      {
            /* In case of fixed join: 	-add the inertia of the link to his parent in the model
			 * 							-let all the children become children of parent
             * 							-inform the parser of the offset to apply
			 * */
            model.mergeFixedBody(parent, jointPlacement, Y); //Modify the parent inertia in the model
            SE3 ptjot_se3 = convertFromUrdf(link->parent_joint->parent_to_joint_origin_transform);
            //transformation of the current placement offset (important if several fixed join following)
            nextPlacementOffset=placementOffset*ptjot_se3;
			BOOST_FOREACH(::urdf::LinkPtr child_link,link->child_links)
			{
                child_link->setParent(link->getParent() ); 	//skip the fixed generation
			}
			break;
	      }

	    default:
	      {
		std::cerr << "The joint type " << joint->type << " is not supported." << std::endl;
		assert(false && "Only revolute joint are accepted." );
		break;
	      }
	    }
	    }
      else if(freeFlyer)
	{ /* The link is the root of the body. */
	  model.addBody( 0, JointModelFreeFlyer(), SE3::Identity(), Y, "root", link->name, true );
	}

    BOOST_FOREACH(::urdf::LinkConstPtr child,link->child_links)
	{
      parseTree( child,model,freeFlyer,nextPlacementOffset );
	}
    }

    Model buildModel( const std::string & filename, bool freeFlyer = false )
    {
      Model model;

      ::urdf::ModelInterfacePtr urdfTree = ::urdf::parseURDFFile (filename);
      parseTree(urdfTree->getRoot(),model,freeFlyer);
      return model;
    }

  } // namespace urdf
} // namespace se3

#endif // ifndef __se3_urdf_hpp__

<|endoftext|>


<commit_before>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2017-2019 Hugo Beauzée-Luyssen, Videolabs, VideoLAN
 *
 * Authors: Hugo Beauzée-Luyssen <hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Url.h"
#include "Filename.h"
#include "medialibrary/filesystem/Errors.h"

#include <algorithm>
#include <cstring>
#include <iterator>
#include <stdexcept>

namespace
{
inline bool isSafe( char c )
{
    return strchr( ".-_~/"
#ifdef _WIN32
                    "\\"
#endif
                   , c ) != nullptr;
}

std::string::const_iterator encodeSegment( std::string& res,
                                           std::string::const_iterator begin,
                                           std::string::const_iterator end,
                                           const char* extraChars )
{
    for ( auto i = begin; i < end; ++i )
    {
        // This must be an unsigned char for the bits operations below to work.
        // auto will yield a signed char here.
        const unsigned char c = *i;
        if ( ( c >= 32 && c <= 126 ) && (
                 ( c >= 'a' && c <= 'z' ) ||
                 ( c >= 'A' && c <= 'Z' ) ||
                 ( c >= '0' && c <= '9' ) ||
                 isSafe( c ) == true ||
                 ( extraChars != nullptr && strchr( extraChars, c ) != nullptr ) )
             )
        {
            res.push_back( c );
        }
        else
            res.append({ '%', "0123456789ABCDEF"[c >> 4], "0123456789ABCDEF"[c & 0xF] });
    }
    return end;
}

inline uint16_t charToVal( char c )
{
    if ( c >= '0' && c <= '9' )
        return c - '0';
    if ( c >= 'a' && c <= 'f' )
        return c - 'a' + 10;
    if ( c >= 'A' && c <= 'F' )
        return c - 'A' + 10;
    throw std::runtime_error( "Incomplete/invalid character sequence" );
}

}

namespace medialibrary
{
namespace utils
{
namespace url
{

parts split( const std::string& url )
{
    parts res{};
    auto schemePos = url.find( "://" );
    if ( schemePos == std::string::npos )
    {
        res.path = url;
        return res;
    }
    std::copy( url.cbegin(), url.cbegin() + schemePos,
               std::back_inserter( res.scheme ) );

    if ( res.scheme == "file" )
    {
        std::copy( url.cbegin() + schemePos + 3, url.cend(),
                   std::back_inserter( res.path ) );
        return res;
    }

    const auto authorityBegin = url.cbegin() + schemePos + 3;
    auto authorityEnd = std::find( authorityBegin, url.cend(), '/' );
    auto queryBegin = std::find( authorityBegin, url.cend(), '?' );
    const auto fragmentBegin = std::find( authorityBegin, url.cend(), '#' );

    /*
     * The fragment must come after the query parameters. Since we use both
     * values at a later point, sanitize them now.
     */
    if ( fragmentBegin != url.cend() && queryBegin != url.cend() &&
         fragmentBegin < queryBegin )
    {
        queryBegin = url.cend();
    }

    /* RFC 3986 §3.2:
     * The authority component is preceded by a double slash ("//") and is
     * terminated by the next slash ("/"), question mark ("?"), or number
     * sign ("#") character, or by the end of the URI.
     */
    if ( authorityEnd == url.cend() )
    {
        if ( queryBegin != url.cend() )
            authorityEnd = queryBegin;
        else
            authorityEnd = fragmentBegin;
    }
    else
    {
        if ( queryBegin != url.cend() && queryBegin < authorityEnd )
            authorityEnd = queryBegin;
        if ( fragmentBegin != url.cend() && fragmentBegin < authorityEnd )
            authorityEnd = fragmentBegin;
    }

    {
        /* Split the authority into its actual components */
        auto userInfoEnd = std::find( authorityBegin, authorityEnd, '@' );
        if ( userInfoEnd != authorityEnd )
        {
            std::copy( authorityBegin, userInfoEnd,
                       std::back_inserter( res.userInfo ) );
            /* Skip the '@' when copying the host */
            ++userInfoEnd;
            res.hostMarker = true;
        }
        else
            userInfoEnd = authorityBegin;

        auto portBegin = std::find( userInfoEnd, authorityEnd, ':' );
        if ( portBegin != authorityEnd )
            std::copy( portBegin + 1, authorityEnd, std::back_inserter( res.port ) );
        else
            portBegin = authorityEnd;
        std::copy( userInfoEnd, portBegin, std::back_inserter( res.host ) );

    }
    if ( authorityEnd == url.cend() )
    {
        /*
         * If we don't have a clear end for the authority segment, it means the
         * end is the url end, so we don't have anything else to split
         */
        return res;
    }

    auto pathEnd = queryBegin != url.cend() ? queryBegin : fragmentBegin;
    std::copy( authorityEnd, pathEnd,
               std::back_inserter( res.path ) );
    if ( queryBegin != url.cend() )
    {
        std::copy( queryBegin + 1,
                   fragmentBegin != url.cend() ? fragmentBegin : url.cend(),
                   std::back_inserter( res.query ) );
    }
    if ( fragmentBegin != url.cend() )
    {
        std::copy( fragmentBegin + 1, url.cend(),
                   std::back_inserter( res.fragments ) );
    }
    return res;
}

std::string decode( const std::string& str )
{
    std::string res;
    res.reserve( str.size() );
    auto it = str.cbegin();
    auto ite = str.cend();
    for ( ; it != ite; ++it )
    {
        if ( *it == '%' )
        {
            ++it;
            const auto c1 = charToVal( *it );
            const auto c2 = charToVal( *( it + 1 ) );
            auto val = c1 * 16 + c2;
            res.push_back( static_cast<std::string::value_type>( val ) );
            ++it;
        }
        else
            res.push_back( *it );
    }
    return res;
}

std::string encode( const std::string& str )
{
    auto parts = split( str );
    std::string res{};
    res.reserve( str.length() );
    /*
     * If the file is a local path, we need to encode everything as the
     * characters won't have any URL related meaning
     */
    if ( parts.scheme == "file" || parts.scheme.empty() == true )
    {
        if ( parts.scheme.empty() == false )
            res += "file://";
        auto pathBegin = parts.path.cbegin();
#ifdef _WIN32
        if ( *pathBegin == '/' && isalpha( *(pathBegin + 1) ) && *(pathBegin + 2) == ':' )
        {
            // Don't encode the ':' after the drive letter.
            // All other ':' need to be encoded, there are not allowed in a file path
            // on windows, but they might appear in urls
            std::copy( pathBegin, pathBegin + 3, std::back_inserter( res ) );
            pathBegin += 3;
        }
#endif
        encodeSegment( res, pathBegin, parts.path.cend(), nullptr );
        return res;
    }
    /*
     * We already accept any character in ".-_~/" through isSafe(), but we need
     * to accept more depending on the URL segments:
     */
    encodeSegment( res, parts.scheme.cbegin(), parts.scheme.cend(), "+" );
    res += "://";
    if ( parts.userInfo.empty() == false )
    {
        encodeSegment( res, parts.userInfo.cbegin(), parts.userInfo.cend(),
                       "!$&'()*+,;=:" );
    }
    if ( parts.hostMarker == true )
        res += '@';
    encodeSegment( res, parts.host.cbegin(), parts.host.cend(), "[]" );
    if ( parts.port.empty() == false )
    {
        res += ':';
        res += parts.port;
    }

    encodeSegment( res, parts.path.cbegin(), parts.path.cend(), "!$&'()*+,;=:@" );

    if ( parts.query.empty() == false )
    {
        res += '?';
        encodeSegment( res, parts.query.cbegin(), parts.query.cend(),
                       "!$&'()*+,;=:@?" );
    }
    if ( parts.fragments.empty() == false )
    {
        res += '#';
        encodeSegment( res, parts.query.cbegin(), parts.query.cend(),
                       "!$&'()*+,;=:@?" );
    }
    return res;
}

std::string stripScheme( const std::string& mrl )
{
    auto pos = mrl.find( "://" );
    if ( pos == std::string::npos )
        throw fs::errors::UnhandledScheme( "<empty scheme>" );
    return mrl.substr( pos + 3 );
}

std::string scheme( const std::string& mrl )
{
    auto pos = mrl.find( "://" );
    if ( pos == std::string::npos )
        throw fs::errors::UnhandledScheme( "<empty scheme>" );
    return mrl.substr( 0, pos + 3 );
}

#ifndef _WIN32

std::string toLocalPath( const std::string& mrl )
{
    if ( mrl.compare( 0, 7, "file://" ) != 0 )
        throw fs::errors::UnhandledScheme( scheme( mrl ) );
    return utils::url::decode( mrl.substr( 7 ) );
}

#else

std::string toLocalPath( const std::string& mrl )
{
    if ( mrl.compare( 0, 7, "file://" ) != 0 )
        throw fs::errors::UnhandledScheme( utils::url::scheme( mrl ) );
    auto localPath = mrl.substr( 7 );
    // If the path is a local path (ie. X:\path\to and not \\path\to) skip the
    // initial backslash, as it is only part of our representation, and not
    // understood by the win32 API functions
    // Note that the initial '/' (after the 2 forward slashes from the scheme)
    // is not a backslash, as it is not a path separator, so do not use
    // DIR_SEPARATOR_CHAR here.
    if ( localPath[0] == '/' && isalpha( localPath[1] ) )
        localPath.erase( 0, 1 );
    std::replace( begin( localPath ), end( localPath ), '/', '\\' );
    return utils::url::decode( localPath );
}

#endif

bool schemeIs( const std::string& scheme, const std::string& mrl )
{
    return mrl.compare( 0, scheme.size(), scheme ) == 0;
}

std::string path( const std::string &mrl )
{
    auto schemelessMrl = stripScheme( mrl );
    auto host = utils::file::firstFolder( schemelessMrl );
    return utils::file::removePath( schemelessMrl, host );
}

}
}
}
<commit_msg>utils: Url: Fix GCC13 build by including cstdint<commit_after>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2017-2019 Hugo Beauzée-Luyssen, Videolabs, VideoLAN
 *
 * Authors: Hugo Beauzée-Luyssen <hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Url.h"
#include "Filename.h"
#include "medialibrary/filesystem/Errors.h"

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <iterator>
#include <stdexcept>

namespace
{
inline bool isSafe( char c )
{
    return strchr( ".-_~/"
#ifdef _WIN32
                    "\\"
#endif
                   , c ) != nullptr;
}

std::string::const_iterator encodeSegment( std::string& res,
                                           std::string::const_iterator begin,
                                           std::string::const_iterator end,
                                           const char* extraChars )
{
    for ( auto i = begin; i < end; ++i )
    {
        // This must be an unsigned char for the bits operations below to work.
        // auto will yield a signed char here.
        const unsigned char c = *i;
        if ( ( c >= 32 && c <= 126 ) && (
                 ( c >= 'a' && c <= 'z' ) ||
                 ( c >= 'A' && c <= 'Z' ) ||
                 ( c >= '0' && c <= '9' ) ||
                 isSafe( c ) == true ||
                 ( extraChars != nullptr && strchr( extraChars, c ) != nullptr ) )
             )
        {
            res.push_back( c );
        }
        else
            res.append({ '%', "0123456789ABCDEF"[c >> 4], "0123456789ABCDEF"[c & 0xF] });
    }
    return end;
}

inline uint16_t charToVal( char c )
{
    if ( c >= '0' && c <= '9' )
        return c - '0';
    if ( c >= 'a' && c <= 'f' )
        return c - 'a' + 10;
    if ( c >= 'A' && c <= 'F' )
        return c - 'A' + 10;
    throw std::runtime_error( "Incomplete/invalid character sequence" );
}

}

namespace medialibrary
{
namespace utils
{
namespace url
{

parts split( const std::string& url )
{
    parts res{};
    auto schemePos = url.find( "://" );
    if ( schemePos == std::string::npos )
    {
        res.path = url;
        return res;
    }
    std::copy( url.cbegin(), url.cbegin() + schemePos,
               std::back_inserter( res.scheme ) );

    if ( res.scheme == "file" )
    {
        std::copy( url.cbegin() + schemePos + 3, url.cend(),
                   std::back_inserter( res.path ) );
        return res;
    }

    const auto authorityBegin = url.cbegin() + schemePos + 3;
    auto authorityEnd = std::find( authorityBegin, url.cend(), '/' );
    auto queryBegin = std::find( authorityBegin, url.cend(), '?' );
    const auto fragmentBegin = std::find( authorityBegin, url.cend(), '#' );

    /*
     * The fragment must come after the query parameters. Since we use both
     * values at a later point, sanitize them now.
     */
    if ( fragmentBegin != url.cend() && queryBegin != url.cend() &&
         fragmentBegin < queryBegin )
    {
        queryBegin = url.cend();
    }

    /* RFC 3986 §3.2:
     * The authority component is preceded by a double slash ("//") and is
     * terminated by the next slash ("/"), question mark ("?"), or number
     * sign ("#") character, or by the end of the URI.
     */
    if ( authorityEnd == url.cend() )
    {
        if ( queryBegin != url.cend() )
            authorityEnd = queryBegin;
        else
            authorityEnd = fragmentBegin;
    }
    else
    {
        if ( queryBegin != url.cend() && queryBegin < authorityEnd )
            authorityEnd = queryBegin;
        if ( fragmentBegin != url.cend() && fragmentBegin < authorityEnd )
            authorityEnd = fragmentBegin;
    }

    {
        /* Split the authority into its actual components */
        auto userInfoEnd = std::find( authorityBegin, authorityEnd, '@' );
        if ( userInfoEnd != authorityEnd )
        {
            std::copy( authorityBegin, userInfoEnd,
                       std::back_inserter( res.userInfo ) );
            /* Skip the '@' when copying the host */
            ++userInfoEnd;
            res.hostMarker = true;
        }
        else
            userInfoEnd = authorityBegin;

        auto portBegin = std::find( userInfoEnd, authorityEnd, ':' );
        if ( portBegin != authorityEnd )
            std::copy( portBegin + 1, authorityEnd, std::back_inserter( res.port ) );
        else
            portBegin = authorityEnd;
        std::copy( userInfoEnd, portBegin, std::back_inserter( res.host ) );

    }
    if ( authorityEnd == url.cend() )
    {
        /*
         * If we don't have a clear end for the authority segment, it means the
         * end is the url end, so we don't have anything else to split
         */
        return res;
    }

    auto pathEnd = queryBegin != url.cend() ? queryBegin : fragmentBegin;
    std::copy( authorityEnd, pathEnd,
               std::back_inserter( res.path ) );
    if ( queryBegin != url.cend() )
    {
        std::copy( queryBegin + 1,
                   fragmentBegin != url.cend() ? fragmentBegin : url.cend(),
                   std::back_inserter( res.query ) );
    }
    if ( fragmentBegin != url.cend() )
    {
        std::copy( fragmentBegin + 1, url.cend(),
                   std::back_inserter( res.fragments ) );
    }
    return res;
}

std::string decode( const std::string& str )
{
    std::string res;
    res.reserve( str.size() );
    auto it = str.cbegin();
    auto ite = str.cend();
    for ( ; it != ite; ++it )
    {
        if ( *it == '%' )
        {
            ++it;
            const auto c1 = charToVal( *it );
            const auto c2 = charToVal( *( it + 1 ) );
            auto val = c1 * 16 + c2;
            res.push_back( static_cast<std::string::value_type>( val ) );
            ++it;
        }
        else
            res.push_back( *it );
    }
    return res;
}

std::string encode( const std::string& str )
{
    auto parts = split( str );
    std::string res{};
    res.reserve( str.length() );
    /*
     * If the file is a local path, we need to encode everything as the
     * characters won't have any URL related meaning
     */
    if ( parts.scheme == "file" || parts.scheme.empty() == true )
    {
        if ( parts.scheme.empty() == false )
            res += "file://";
        auto pathBegin = parts.path.cbegin();
#ifdef _WIN32
        if ( *pathBegin == '/' && isalpha( *(pathBegin + 1) ) && *(pathBegin + 2) == ':' )
        {
            // Don't encode the ':' after the drive letter.
            // All other ':' need to be encoded, there are not allowed in a file path
            // on windows, but they might appear in urls
            std::copy( pathBegin, pathBegin + 3, std::back_inserter( res ) );
            pathBegin += 3;
        }
#endif
        encodeSegment( res, pathBegin, parts.path.cend(), nullptr );
        return res;
    }
    /*
     * We already accept any character in ".-_~/" through isSafe(), but we need
     * to accept more depending on the URL segments:
     */
    encodeSegment( res, parts.scheme.cbegin(), parts.scheme.cend(), "+" );
    res += "://";
    if ( parts.userInfo.empty() == false )
    {
        encodeSegment( res, parts.userInfo.cbegin(), parts.userInfo.cend(),
                       "!$&'()*+,;=:" );
    }
    if ( parts.hostMarker == true )
        res += '@';
    encodeSegment( res, parts.host.cbegin(), parts.host.cend(), "[]" );
    if ( parts.port.empty() == false )
    {
        res += ':';
        res += parts.port;
    }

    encodeSegment( res, parts.path.cbegin(), parts.path.cend(), "!$&'()*+,;=:@" );

    if ( parts.query.empty() == false )
    {
        res += '?';
        encodeSegment( res, parts.query.cbegin(), parts.query.cend(),
                       "!$&'()*+,;=:@?" );
    }
    if ( parts.fragments.empty() == false )
    {
        res += '#';
        encodeSegment( res, parts.query.cbegin(), parts.query.cend(),
                       "!$&'()*+,;=:@?" );
    }
    return res;
}

std::string stripScheme( const std::string& mrl )
{
    auto pos = mrl.find( "://" );
    if ( pos == std::string::npos )
        throw fs::errors::UnhandledScheme( "<empty scheme>" );
    return mrl.substr( pos + 3 );
}

std::string scheme( const std::string& mrl )
{
    auto pos = mrl.find( "://" );
    if ( pos == std::string::npos )
        throw fs::errors::UnhandledScheme( "<empty scheme>" );
    return mrl.substr( 0, pos + 3 );
}

#ifndef _WIN32

std::string toLocalPath( const std::string& mrl )
{
    if ( mrl.compare( 0, 7, "file://" ) != 0 )
        throw fs::errors::UnhandledScheme( scheme( mrl ) );
    return utils::url::decode( mrl.substr( 7 ) );
}

#else

std::string toLocalPath( const std::string& mrl )
{
    if ( mrl.compare( 0, 7, "file://" ) != 0 )
        throw fs::errors::UnhandledScheme( utils::url::scheme( mrl ) );
    auto localPath = mrl.substr( 7 );
    // If the path is a local path (ie. X:\path\to and not \\path\to) skip the
    // initial backslash, as it is only part of our representation, and not
    // understood by the win32 API functions
    // Note that the initial '/' (after the 2 forward slashes from the scheme)
    // is not a backslash, as it is not a path separator, so do not use
    // DIR_SEPARATOR_CHAR here.
    if ( localPath[0] == '/' && isalpha( localPath[1] ) )
        localPath.erase( 0, 1 );
    std::replace( begin( localPath ), end( localPath ), '/', '\\' );
    return utils::url::decode( localPath );
}

#endif

bool schemeIs( const std::string& scheme, const std::string& mrl )
{
    return mrl.compare( 0, scheme.size(), scheme ) == 0;
}

std::string path( const std::string &mrl )
{
    auto schemelessMrl = stripScheme( mrl );
    auto host = utils::file::firstFolder( schemelessMrl );
    return utils::file::removePath( schemelessMrl, host );
}

}
}
}
<|endoftext|>


<commit_before>#include "zip_int_store.hpp"
#include <nark/fsa/nest_trie_dawg.hpp>
#include <nark/io/FileStream.hpp>
#include <nark/io/DataIO.hpp>
#include <nark/util/mmap.hpp>

namespace nark { namespace db {

ZipIntStore::ZipIntStore() {
	m_mmapBase = nullptr;
	m_mmapSize = 0;
}
ZipIntStore::~ZipIntStore() {
	if (m_mmapBase) {
		m_dedup.risk_release_ownership();
		m_index.risk_release_ownership();
		mmap_close(m_mmapBase, m_mmapSize);
	}
}

llong ZipIntStore::dataStorageSize() const {
	return m_dedup.mem_size() + m_index.mem_size();
}

llong ZipIntStore::numDataRows() const {
	return m_index.size() ? m_index.size() : m_dedup.size();
}

template<class Int>
void ZipIntStore::valueAppend(size_t recIdx, valvec<byte>* key) const {
	if (m_index.size()) {
		size_t idx = m_index.get(recIdx);
		assert(idx < m_dedup.size());
		Int iValue = Int(m_minValue + m_dedup.get(idx));
		unaligned_save<Int>(key->grow_no_init(sizeof(Int)), iValue);
	}
	else {
		Int iValue = Int(m_minValue + m_dedup.get(recIdx));
		unaligned_save<Int>(key->grow_no_init(sizeof(Int)), iValue);
	}
}

void ZipIntStore::getValueAppend(llong id, valvec<byte>* val, DbContext*) const {
	assert(id >= 0);
	size_t idx = size_t(id);
	assert(idx < m_dedup.size());
	switch (m_intType) {
	default:
		THROW_STD(invalid_argument, "Bad m_intType=%s", Schema::columnTypeStr(m_intType));
	case ColumnType::Sint08: valueAppend< int8_t >(idx, val); break;
	case ColumnType::Uint08: valueAppend<uint8_t >(idx, val); break;
	case ColumnType::Sint16: valueAppend< int16_t>(idx, val); break;
	case ColumnType::Uint16: valueAppend<uint16_t>(idx, val); break;
	case ColumnType::Sint32: valueAppend< int32_t>(idx, val); break;
	case ColumnType::Uint32: valueAppend<uint32_t>(idx, val); break;
	case ColumnType::Sint64: valueAppend< int64_t>(idx, val); break;
	case ColumnType::Uint64: valueAppend<uint64_t>(idx, val); break;
	case ColumnType::VarSint: {
		byte  buf[16];
		byte* end = save_var_int64(buf, int64_t(m_minValue + m_dedup.get(idx)));
		val->append(buf, end - buf);
		break; }
	case ColumnType::VarUint: {
		byte  buf[16];
		byte* end = save_var_uint64(buf, uint64_t(m_minValue + m_dedup.get(idx)));
		val->append(buf, end - buf);
		break; }
	}
}

StoreIterator* ZipIntStore::createStoreIterForward(DbContext*) const {
	return nullptr; // not needed
}

StoreIterator* ZipIntStore::createStoreIterBackward(DbContext*) const {
	return nullptr; // not needed
}

template<class Int>
void ZipIntStore::zipValues(const void* data, size_t size) {
	size_t rows = size / sizeof(Int);
	assert(size % sizeof(Int) == 0);
	const Int* values = (Int*)data;

	UintVecMin0 dup;
	m_minValue = dup.build_from(values, rows);

	valvec<Int> dedup(values, rows);
	std::sort(dedup.begin(), dedup.end());
	dedup.trim(std::unique(dedup.begin(), dedup.end()));

	size_t indexBits = nark_bsr_u32(dedup.size()-1) + 1;
	size_t indexSize = (indexBits      * rows + 7) / 8;
	size_t dedupSize = (dup.uintbits() * rows + 7) / 8;
	if (dedupSize + indexSize < dup.mem_size()) {
		valvec<uint32_t> index(rows, valvec_no_init());
		for(size_t i = 0; i < rows; ++i) {
			size_t j = lower_bound_a(dedup, values[i]);
			assert(values[i] == dedup[j]);
			index[i] = j;
		}
		m_index.build_from(index);
		m_dedup.build_from(dedup);
	}
}

void ZipIntStore::build(ColumnType keyType, SortableStrVec& strVec) {
	assert(strVec.m_index.size() == 0);
	m_intType = keyType;
	void*  data = strVec.m_strpool.data();
	size_t size = strVec.m_strpool.size();
	switch (keyType) {
	default:
		THROW_STD(invalid_argument, "Bad keyType=%s", Schema::columnTypeStr(keyType));
	case ColumnType::Sint08: zipValues< int8_t >(data, size); break;
	case ColumnType::Uint08: zipValues<uint8_t >(data, size); break;
	case ColumnType::Sint16: zipValues< int16_t>(data, size); break;
	case ColumnType::Uint16: zipValues<uint16_t>(data, size); break;
	case ColumnType::Sint32: zipValues< int32_t>(data, size); break;
	case ColumnType::Uint32: zipValues<uint32_t>(data, size); break;
	case ColumnType::Sint64: zipValues< int64_t>(data, size); break;
	case ColumnType::Uint64: zipValues<uint64_t>(data, size); break;
	case ColumnType::VarSint: {
		valvec<llong> tmp;
		const byte* pos = strVec.m_strpool.data();
		const byte* end = strVec.m_strpool.end();
		while (pos < end) {
			const byte* next = nullptr;
			llong key = load_var_int64(pos, &next);
			tmp.push_back(key);
			pos = next;
		}
		zipValues<int64_t>(tmp.data(), tmp.used_mem_size());
		break; }
	case ColumnType::VarUint: {
		valvec<ullong> tmp;
		const byte* pos = strVec.m_strpool.data();
		const byte* end = strVec.m_strpool.end();
		while (pos < end) {
			const byte* next = nullptr;
			ullong key = load_var_uint64(pos, &next);
			tmp.push_back(key);
			pos = next;
		}
		zipValues<uint64_t>(tmp.data(), tmp.used_mem_size());
		break; }
	}
}

namespace {
	struct ZipIntStoreHeader {
		uint32_t rows;
		uint32_t uniqNum;
		uint8_t  intBits;
		uint8_t  intType;
		uint16_t padding1;
		uint32_t padding2;
		uint32_t padding3;
		 int64_t minValue;
	};
	BOOST_STATIC_ASSERT(sizeof(ZipIntStoreHeader) == 32);
}

void ZipIntStore::load(fstring path) {
	std::string fpath = path + ".zint";
	m_mmapBase = (byte_t*)mmap_load(fpath.c_str(), &m_mmapSize);
	auto header = (const ZipIntStoreHeader*)m_mmapBase;
	size_t rows = header->rows;
	m_intType = ColumnType(header->intType);
	m_minValue  = header->minValue;
	m_dedup.risk_set_data((byte*)(header+1), header->uniqNum, header->intBits);
	if (header->uniqNum != rows) {
		assert(header->uniqNum < rows);
		size_t indexBits = nark_bsr_u32(rows - 1) + 1;
		auto   indexData = (byte*)(header+1) + m_dedup.mem_size();
		m_index.risk_set_data(indexData, rows, indexBits);
	}
}

void ZipIntStore::save(fstring path) const {
	std::string fpath = path + ".zint";
	NativeDataOutput<FileStream> dio;
	dio.open(fpath.c_str(), "wb");
	ZipIntStoreHeader header;
	header.rows = uint32_t(numDataRows());
	header.uniqNum = m_dedup.size();
	header.intBits = byte(m_dedup.uintbits());
	header.intType = byte(m_intType);
	header.padding1 = 0;
	header.padding2 = 0;
	header.padding3 = 0;
	header.minValue = int64_t(m_minValue);
	dio.ensureWrite(&header, sizeof(header));
	dio.ensureWrite(m_dedup.data(), m_dedup.mem_size());
	dio.ensureWrite(m_index.data(), m_index.mem_size());
}

}} // namespace nark::db
<commit_msg>ZipIntStore: fix bugs<commit_after>#include "zip_int_store.hpp"
#include <nark/fsa/nest_trie_dawg.hpp>
#include <nark/io/FileStream.hpp>
#include <nark/io/DataIO.hpp>
#include <nark/util/mmap.hpp>

namespace nark { namespace db {

ZipIntStore::ZipIntStore() {
	m_mmapBase = nullptr;
	m_mmapSize = 0;
}
ZipIntStore::~ZipIntStore() {
	if (m_mmapBase) {
		m_dedup.risk_release_ownership();
		m_index.risk_release_ownership();
		mmap_close(m_mmapBase, m_mmapSize);
	}
}

llong ZipIntStore::dataStorageSize() const {
	return m_dedup.mem_size() + m_index.mem_size();
}

llong ZipIntStore::numDataRows() const {
	return m_index.size() ? m_index.size() : m_dedup.size();
}

template<class Int>
void ZipIntStore::valueAppend(size_t recIdx, valvec<byte>* key) const {
	if (m_index.size()) {
		size_t idx = m_index.get(recIdx);
		assert(idx < m_dedup.size());
		Int iValue = Int(m_minValue + m_dedup.get(idx));
		unaligned_save<Int>(key->grow_no_init(sizeof(Int)), iValue);
	}
	else {
		Int iValue = Int(m_minValue + m_dedup.get(recIdx));
		unaligned_save<Int>(key->grow_no_init(sizeof(Int)), iValue);
	}
}

void ZipIntStore::getValueAppend(llong id, valvec<byte>* val, DbContext*) const {
	assert(id >= 0);
	size_t idx = size_t(id);
	assert(idx < m_dedup.size());
	switch (m_intType) {
	default:
		THROW_STD(invalid_argument, "Bad m_intType=%s", Schema::columnTypeStr(m_intType));
	case ColumnType::Sint08: valueAppend< int8_t >(idx, val); break;
	case ColumnType::Uint08: valueAppend<uint8_t >(idx, val); break;
	case ColumnType::Sint16: valueAppend< int16_t>(idx, val); break;
	case ColumnType::Uint16: valueAppend<uint16_t>(idx, val); break;
	case ColumnType::Sint32: valueAppend< int32_t>(idx, val); break;
	case ColumnType::Uint32: valueAppend<uint32_t>(idx, val); break;
	case ColumnType::Sint64: valueAppend< int64_t>(idx, val); break;
	case ColumnType::Uint64: valueAppend<uint64_t>(idx, val); break;
	case ColumnType::VarSint: {
		byte  buf[16];
		byte* end = save_var_int64(buf, int64_t(m_minValue + m_dedup.get(idx)));
		val->append(buf, end - buf);
		break; }
	case ColumnType::VarUint: {
		byte  buf[16];
		byte* end = save_var_uint64(buf, uint64_t(m_minValue + m_dedup.get(idx)));
		val->append(buf, end - buf);
		break; }
	}
}

StoreIterator* ZipIntStore::createStoreIterForward(DbContext*) const {
	return nullptr; // not needed
}

StoreIterator* ZipIntStore::createStoreIterBackward(DbContext*) const {
	return nullptr; // not needed
}

template<class Int>
void ZipIntStore::zipValues(const void* data, size_t size) {
	size_t rows = size / sizeof(Int);
	assert(size % sizeof(Int) == 0);
	const Int* values = (Int*)data;

	UintVecMin0 dup;
	m_minValue = dup.build_from(values, rows);

	valvec<Int> dedup(values, rows);
	std::sort(dedup.begin(), dedup.end());
	dedup.trim(std::unique(dedup.begin(), dedup.end()));

	size_t indexBits = nark_bsr_u32(dedup.size()-1) + 1;
	size_t indexSize = (indexBits      * rows         + 7) / 8;
	size_t dedupSize = (dup.uintbits() * dedup.size() + 7) / 8;
	if (dedupSize + indexSize < dup.mem_size()) {
		valvec<uint32_t> index(rows, valvec_no_init());
		for(size_t i = 0; i < rows; ++i) {
			size_t j = lower_bound_a(dedup, values[i]);
			assert(values[i] == dedup[j]);
			index[i] = j;
		}
		m_index.build_from(index);
		m_dedup.build_from(dedup);
	}
	else {
		m_dedup.swap(dup);
	}
}

void ZipIntStore::build(ColumnType keyType, SortableStrVec& strVec) {
	assert(strVec.m_index.size() == 0);
	m_intType = keyType;
	void*  data = strVec.m_strpool.data();
	size_t size = strVec.m_strpool.size();
	switch (keyType) {
	default:
		THROW_STD(invalid_argument, "Bad keyType=%s", Schema::columnTypeStr(keyType));
	case ColumnType::Sint08: zipValues< int8_t >(data, size); break;
	case ColumnType::Uint08: zipValues<uint8_t >(data, size); break;
	case ColumnType::Sint16: zipValues< int16_t>(data, size); break;
	case ColumnType::Uint16: zipValues<uint16_t>(data, size); break;
	case ColumnType::Sint32: zipValues< int32_t>(data, size); break;
	case ColumnType::Uint32: zipValues<uint32_t>(data, size); break;
	case ColumnType::Sint64: zipValues< int64_t>(data, size); break;
	case ColumnType::Uint64: zipValues<uint64_t>(data, size); break;
	case ColumnType::VarSint: {
		valvec<llong> tmp;
		const byte* pos = strVec.m_strpool.data();
		const byte* end = strVec.m_strpool.end();
		while (pos < end) {
			const byte* next = nullptr;
			llong key = load_var_int64(pos, &next);
			tmp.push_back(key);
			pos = next;
		}
		zipValues<int64_t>(tmp.data(), tmp.used_mem_size());
		break; }
	case ColumnType::VarUint: {
		valvec<ullong> tmp;
		const byte* pos = strVec.m_strpool.data();
		const byte* end = strVec.m_strpool.end();
		while (pos < end) {
			const byte* next = nullptr;
			ullong key = load_var_uint64(pos, &next);
			tmp.push_back(key);
			pos = next;
		}
		zipValues<uint64_t>(tmp.data(), tmp.used_mem_size());
		break; }
	}
}

namespace {
	struct ZipIntStoreHeader {
		uint32_t rows;
		uint32_t uniqNum;
		uint8_t  intBits;
		uint8_t  intType;
		uint16_t padding1;
		uint32_t padding2;
		uint32_t padding3;
		 int64_t minValue;
	};
	BOOST_STATIC_ASSERT(sizeof(ZipIntStoreHeader) == 32);
}

void ZipIntStore::load(fstring path) {
	std::string fpath = path + ".zint";
	m_mmapBase = (byte_t*)mmap_load(fpath.c_str(), &m_mmapSize);
	auto header = (const ZipIntStoreHeader*)m_mmapBase;
	size_t rows = header->rows;
	m_intType = ColumnType(header->intType);
	m_minValue  = header->minValue;
	m_dedup.risk_set_data((byte*)(header+1), header->uniqNum, header->intBits);
	if (header->uniqNum != rows) {
		assert(header->uniqNum < rows);
		size_t indexBits = nark_bsr_u32(rows - 1) + 1;
		auto   indexData = (byte*)(header+1) + m_dedup.mem_size();
		m_index.risk_set_data(indexData, rows, indexBits);
	}
}

void ZipIntStore::save(fstring path) const {
	std::string fpath = path + ".zint";
	NativeDataOutput<FileStream> dio;
	dio.open(fpath.c_str(), "wb");
	ZipIntStoreHeader header;
	header.rows = uint32_t(numDataRows());
	header.uniqNum = m_dedup.size();
	header.intBits = byte(m_dedup.uintbits());
	header.intType = byte(m_intType);
	header.padding1 = 0;
	header.padding2 = 0;
	header.padding3 = 0;
	header.minValue = int64_t(m_minValue);
	dio.ensureWrite(&header, sizeof(header));
	dio.ensureWrite(m_dedup.data(), m_dedup.mem_size());
	dio.ensureWrite(m_index.data(), m_index.mem_size());
}

}} // namespace nark::db
<|endoftext|>


<commit_before>#ifdef STXXL_BOOST_CONFIG

#include "stxxl/bits/io/boostfd_file.h"
#include "stxxl/bits/common/debug.h"

 #include "boost/filesystem/operations.hpp"
 #include "boost/filesystem/fstream.hpp"

__STXXL_BEGIN_NAMESPACE

boost::iostreams::file_descriptor
boostfd_file::get_file_des() const
{
    return file_des;
}

boostfd_request::boostfd_request (
    boostfd_file * f,
    void * buf,
    stxxl::int64 off,
    size_t b,
    request_type t,
    completion_handler on_cmpl) :
    request (on_cmpl, f, buf, off, b, t),
    _state (OP)
{ }

bool boostfd_request::add_waiter (onoff_switch * sw)
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
 #else
    waiters_mutex.lock ();
 #endif

    if (poll ())   // request already finished
    {
 #ifndef STXXL_BOOST_THREADS
        waiters_mutex.unlock ();
 #endif
        return true;
    }

    waiters.insert (sw);
 #ifndef STXXL_BOOST_THREADS
    waiters_mutex.unlock ();
 #endif

    return false;
}
void boostfd_request::delete_waiter (onoff_switch * sw)
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
    waiters.erase (sw);
 #else
    waiters_mutex.lock ();
    waiters.erase (sw);
    waiters_mutex.unlock ();
 #endif
}
int boostfd_request::nwaiters ()     // returns number of waiters
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
    return waiters.size();
 #else
    waiters_mutex.lock ();
    int size = waiters.size ();
    waiters_mutex.unlock ();
    return size;
 #endif
}
void boostfd_request::check_aligning ()
{
    if (offset % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Offset is not aligned: modulo "
                << BLOCK_ALIGN << " = " <<
                      offset % BLOCK_ALIGN);

    if (bytes % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Size is not a multiple of " <<
                      BLOCK_ALIGN << ", = " << bytes % BLOCK_ALIGN);

    if (long (buffer) % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Buffer is not aligned: modulo "
                << BLOCK_ALIGN << " = " <<
                      long (buffer) % BLOCK_ALIGN << " (" <<
                      std::hex << buffer << std::dec << ")");
}

boostfd_request::~boostfd_request ()
{
    STXXL_VERBOSE3("boostfd_request " << unsigned (this) << ": deletion, cnt: " << ref_cnt);

    assert(_state() == DONE || _state() == READY2DIE);
}

void boostfd_request::wait ()
{
    STXXL_VERBOSE3("boostfd_request : " << unsigned (this) << " wait ");

    START_COUNT_WAIT_TIME

 #ifdef NO_OVERLAPPING
    enqueue();
 #endif

    _state.wait_for (READY2DIE);

    END_COUNT_WAIT_TIME

    check_errors();
}
bool boostfd_request::poll()
{
 #ifdef NO_OVERLAPPING
    /*if(_state () < DONE)*/ wait();
 #endif

    const bool s = _state() >= DONE;

    check_errors();

    return s;
}
const char * boostfd_request::io_type ()
{
    return "boostfd";
}

boostfd_file::boostfd_file (
    const std::string & filename,
    int mode,
    int disk) : file (disk), mode_(mode)
{
    BOOST_IOS::openmode boostfd_mode;

 #ifndef STXXL_DIRECT_IO_OFF
    if (mode & DIRECT)
    {
        // direct mode not supported in Boost
    }
 #endif

    if (mode & RDONLY)
    {
        boostfd_mode = BOOST_IOS::in;
    }

    if (mode & WRONLY)
    {
        boostfd_mode = BOOST_IOS::out;
    }

    if (mode & RDWR)
    {
        boostfd_mode = BOOST_IOS::out | BOOST_IOS::in;
    }

    const boost::filesystem::path fspath(filename,
                                         boost::filesystem::native);

    if (mode & TRUNC)
    {
        if (boost::filesystem::exists(fspath))
        {
            boost::filesystem::remove(fspath);
            boost::filesystem::ofstream f(fspath);
            f.close();
            assert(boost::filesystem::exists(fspath));
        }
    }

    if (mode & CREAT)
    {
        // need to be emulated:
        if (!boost::filesystem::exists(fspath))
        {
            boost::filesystem::ofstream f(fspath);
            f.close();
            assert(boost::filesystem::exists(fspath));
        }
    }

    file_des.open(filename, boostfd_mode, boostfd_mode);
}

boostfd_file::~boostfd_file ()
{
    file_des.close();
}
stxxl::int64 boostfd_file::size ()
{
    stxxl::int64 size_ = file_des.seek(0, BOOST_IOS::end);
    return size_;
}
void boostfd_file::set_size (stxxl::int64 newsize)
{
    stxxl::int64 oldsize = size();
    file_des.seek(newsize, BOOST_IOS::beg);
    file_des.seek(0, BOOST_IOS::beg); // not important ?
    assert(size() >= oldsize);
}

void boostfd_request::serve ()
{
    stats * iostats = stats::get_instance();
    if (nref() < 2)
    {
        STXXL_ERRMSG("WARNING: serious error, reference to the request is lost before serve (nref="
        << nref() << ") " <<
                     " this=" << long (this) << " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes
        << " type=" << ((type == READ) ? "READ" : "WRITE") );
    }
    STXXL_VERBOSE2("boostfd_request::serve(): Buffer at " << ((void *)buffer)
                                                          << " offset: " << offset << " bytes: " << bytes << ((type == READ) ? " READ" : " WRITE")
    );

    boostfd_file::fd_type fd = static_cast<boostfd_file *>(file_)->get_file_des();

    try
    {
        fd.seek(offset, BOOST_IOS::beg);
    }
    catch (const std::exception & ex)
    {
        STXXL_FORMAT_ERROR_MSG(msg, "seek() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                               buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") << " : " << ex.what() )

        error_occured(msg.str());
    }

    {
        if (type == READ)
        {
 #if STXXL_IO_STATS
            iostats->read_started (size());
 #endif

            debugmon::get_instance()->io_started((char *)buffer);

            try
            {
                fd.read((char *)buffer, bytes);
            }
            catch (const std::exception & ex)
            {
                STXXL_FORMAT_ERROR_MSG(msg, "read() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                                       buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") <<
                                       " nref= " << nref() << " : " << ex.what() )

                error_occured(msg.str());
            }

            debugmon::get_instance()->io_finished((char *)buffer);

 #if STXXL_IO_STATS
            iostats->read_finished ();
 #endif
        }
        else
        {
 #if STXXL_IO_STATS
            iostats->write_started (size());
 #endif

            debugmon::get_instance()->io_started((char *)buffer);

            try
            {
                fd.write((char *)buffer, bytes);
            }
            catch (const std::exception & ex)
            {
                STXXL_FORMAT_ERROR_MSG(msg, "write() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                                       buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") <<
                                       " nref= " << nref() << " : " << ex.what() )

                error_occured(msg.str());
            }

            debugmon::get_instance()->io_finished((char *)buffer);

 #if STXXL_IO_STATS
            iostats->write_finished ();
 #endif
        }
    }

    if (nref() < 2)
    {
        STXXL_ERRMSG("WARNING: reference to the request is lost after serve (nref=" << nref() << ") " <<
                     " this=" << long (this) <<
                     " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes <<
                     " type=" << ((type == READ) ? "READ" : "WRITE"));
    }

    _state.set_to (DONE);

 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
 #else
    waiters_mutex.lock ();
 #endif
    // << notification >>
    std::for_each(
        waiters.begin(),
        waiters.end(),
        std::mem_fun(&onoff_switch::on) );

 #ifdef STXXL_BOOST_THREADS
    Lock.unlock();
 #else
    waiters_mutex.unlock ();
 #endif

    completed ();
    _state.set_to (READY2DIE);
}

request_ptr boostfd_file::aread (
    void * buffer,
    stxxl::int64 pos,
    size_t bytes,
    completion_handler on_cmpl)
{
    request_ptr req = new boostfd_request(this,
                                          buffer, pos, bytes,
                                          request::READ, on_cmpl);

    if (!req.get())
        stxxl_function_error(io_error);

 #ifndef NO_OVERLAPPING
    disk_queues::get_instance ()->add_readreq(req, get_id());
 #endif
    return req;
}
request_ptr boostfd_file::awrite (
    void * buffer,
    stxxl::int64 pos,
    size_t bytes,
    completion_handler on_cmpl)
{
    request_ptr req = new boostfd_request(this, buffer, pos, bytes,
                                          request::WRITE, on_cmpl);

    if (!req.get())
        stxxl_function_error(io_error);

 #ifndef NO_OVERLAPPING
    disk_queues::get_instance ()->add_writereq(req, get_id());
 #endif
    return req;
}

__STXXL_END_NAMESPACE

#endif // BOOST_VERSION

<commit_msg>added semicolons<commit_after>#ifdef STXXL_BOOST_CONFIG

#include "stxxl/bits/io/boostfd_file.h"
#include "stxxl/bits/common/debug.h"

 #include "boost/filesystem/operations.hpp"
 #include "boost/filesystem/fstream.hpp"

__STXXL_BEGIN_NAMESPACE

boost::iostreams::file_descriptor
boostfd_file::get_file_des() const
{
    return file_des;
}

boostfd_request::boostfd_request (
    boostfd_file * f,
    void * buf,
    stxxl::int64 off,
    size_t b,
    request_type t,
    completion_handler on_cmpl) :
    request (on_cmpl, f, buf, off, b, t),
    _state (OP)
{ }

bool boostfd_request::add_waiter (onoff_switch * sw)
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
 #else
    waiters_mutex.lock ();
 #endif

    if (poll ())   // request already finished
    {
 #ifndef STXXL_BOOST_THREADS
        waiters_mutex.unlock ();
 #endif
        return true;
    }

    waiters.insert (sw);
 #ifndef STXXL_BOOST_THREADS
    waiters_mutex.unlock ();
 #endif

    return false;
}
void boostfd_request::delete_waiter (onoff_switch * sw)
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
    waiters.erase (sw);
 #else
    waiters_mutex.lock ();
    waiters.erase (sw);
    waiters_mutex.unlock ();
 #endif
}
int boostfd_request::nwaiters ()     // returns number of waiters
{
 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
    return waiters.size();
 #else
    waiters_mutex.lock ();
    int size = waiters.size ();
    waiters_mutex.unlock ();
    return size;
 #endif
}
void boostfd_request::check_aligning ()
{
    if (offset % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Offset is not aligned: modulo "
                << BLOCK_ALIGN << " = " <<
                      offset % BLOCK_ALIGN);

    if (bytes % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Size is not a multiple of " <<
                      BLOCK_ALIGN << ", = " << bytes % BLOCK_ALIGN);

    if (long (buffer) % BLOCK_ALIGN != 0)
        STXXL_ERRMSG ("Buffer is not aligned: modulo "
                << BLOCK_ALIGN << " = " <<
                      long (buffer) % BLOCK_ALIGN << " (" <<
                      std::hex << buffer << std::dec << ")");
}

boostfd_request::~boostfd_request ()
{
    STXXL_VERBOSE3("boostfd_request " << unsigned (this) << ": deletion, cnt: " << ref_cnt);

    assert(_state() == DONE || _state() == READY2DIE);
}

void boostfd_request::wait ()
{
    STXXL_VERBOSE3("boostfd_request : " << unsigned (this) << " wait ");

    START_COUNT_WAIT_TIME

 #ifdef NO_OVERLAPPING
    enqueue();
 #endif

    _state.wait_for (READY2DIE);

    END_COUNT_WAIT_TIME

    check_errors();
}
bool boostfd_request::poll()
{
 #ifdef NO_OVERLAPPING
    /*if(_state () < DONE)*/ wait();
 #endif

    const bool s = _state() >= DONE;

    check_errors();

    return s;
}
const char * boostfd_request::io_type ()
{
    return "boostfd";
}

boostfd_file::boostfd_file (
    const std::string & filename,
    int mode,
    int disk) : file (disk), mode_(mode)
{
    BOOST_IOS::openmode boostfd_mode;

 #ifndef STXXL_DIRECT_IO_OFF
    if (mode & DIRECT)
    {
        // direct mode not supported in Boost
    }
 #endif

    if (mode & RDONLY)
    {
        boostfd_mode = BOOST_IOS::in;
    }

    if (mode & WRONLY)
    {
        boostfd_mode = BOOST_IOS::out;
    }

    if (mode & RDWR)
    {
        boostfd_mode = BOOST_IOS::out | BOOST_IOS::in;
    }

    const boost::filesystem::path fspath(filename,
                                         boost::filesystem::native);

    if (mode & TRUNC)
    {
        if (boost::filesystem::exists(fspath))
        {
            boost::filesystem::remove(fspath);
            boost::filesystem::ofstream f(fspath);
            f.close();
            assert(boost::filesystem::exists(fspath));
        }
    }

    if (mode & CREAT)
    {
        // need to be emulated:
        if (!boost::filesystem::exists(fspath))
        {
            boost::filesystem::ofstream f(fspath);
            f.close();
            assert(boost::filesystem::exists(fspath));
        }
    }

    file_des.open(filename, boostfd_mode, boostfd_mode);
}

boostfd_file::~boostfd_file ()
{
    file_des.close();
}
stxxl::int64 boostfd_file::size ()
{
    stxxl::int64 size_ = file_des.seek(0, BOOST_IOS::end);
    return size_;
}
void boostfd_file::set_size (stxxl::int64 newsize)
{
    stxxl::int64 oldsize = size();
    file_des.seek(newsize, BOOST_IOS::beg);
    file_des.seek(0, BOOST_IOS::beg); // not important ?
    assert(size() >= oldsize);
}

void boostfd_request::serve ()
{
    stats * iostats = stats::get_instance();
    if (nref() < 2)
    {
        STXXL_ERRMSG("WARNING: serious error, reference to the request is lost before serve (nref="
        << nref() << ") " <<
                     " this=" << long (this) << " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes
        << " type=" << ((type == READ) ? "READ" : "WRITE") );
    }
    STXXL_VERBOSE2("boostfd_request::serve(): Buffer at " << ((void *)buffer)
                                                          << " offset: " << offset << " bytes: " << bytes << ((type == READ) ? " READ" : " WRITE")
    );

    boostfd_file::fd_type fd = static_cast<boostfd_file *>(file_)->get_file_des();

    try
    {
        fd.seek(offset, BOOST_IOS::beg);
    }
    catch (const std::exception & ex)
    {
        STXXL_FORMAT_ERROR_MSG(msg, "seek() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                               buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") << " : " << ex.what() );

        error_occured(msg.str());
    }

    {
        if (type == READ)
        {
 #if STXXL_IO_STATS
            iostats->read_started (size());
 #endif

            debugmon::get_instance()->io_started((char *)buffer);

            try
            {
                fd.read((char *)buffer, bytes);
            }
            catch (const std::exception & ex)
            {
                STXXL_FORMAT_ERROR_MSG(msg, "read() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                                       buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") <<
                                       " nref= " << nref() << " : " << ex.what() );

                error_occured(msg.str());
            }

            debugmon::get_instance()->io_finished((char *)buffer);

 #if STXXL_IO_STATS
            iostats->read_finished ();
 #endif
        }
        else
        {
 #if STXXL_IO_STATS
            iostats->write_started (size());
 #endif

            debugmon::get_instance()->io_started((char *)buffer);

            try
            {
                fd.write((char *)buffer, bytes);
            }
            catch (const std::exception & ex)
            {
                STXXL_FORMAT_ERROR_MSG(msg, "write() in boostfd_request::serve() offset=" << offset
                                                                                << " this=" << long (this) << " buffer=" <<
                                       buffer << " bytes=" << bytes
                                                                                << " type=" << ((type == READ) ? "READ" : "WRITE") <<
                                       " nref= " << nref() << " : " << ex.what() );

                error_occured(msg.str());
            }

            debugmon::get_instance()->io_finished((char *)buffer);

 #if STXXL_IO_STATS
            iostats->write_finished ();
 #endif
        }
    }

    if (nref() < 2)
    {
        STXXL_ERRMSG("WARNING: reference to the request is lost after serve (nref=" << nref() << ") " <<
                     " this=" << long (this) <<
                     " offset=" << offset << " buffer=" << buffer << " bytes=" << bytes <<
                     " type=" << ((type == READ) ? "READ" : "WRITE"));
    }

    _state.set_to (DONE);

 #ifdef STXXL_BOOST_THREADS
    boost::mutex::scoped_lock Lock(waiters_mutex);
 #else
    waiters_mutex.lock ();
 #endif
    // << notification >>
    std::for_each(
        waiters.begin(),
        waiters.end(),
        std::mem_fun(&onoff_switch::on) );

 #ifdef STXXL_BOOST_THREADS
    Lock.unlock();
 #else
    waiters_mutex.unlock ();
 #endif

    completed ();
    _state.set_to (READY2DIE);
}

request_ptr boostfd_file::aread (
    void * buffer,
    stxxl::int64 pos,
    size_t bytes,
    completion_handler on_cmpl)
{
    request_ptr req = new boostfd_request(this,
                                          buffer, pos, bytes,
                                          request::READ, on_cmpl);

    if (!req.get())
        stxxl_function_error(io_error);

 #ifndef NO_OVERLAPPING
    disk_queues::get_instance ()->add_readreq(req, get_id());
 #endif
    return req;
}
request_ptr boostfd_file::awrite (
    void * buffer,
    stxxl::int64 pos,
    size_t bytes,
    completion_handler on_cmpl)
{
    request_ptr req = new boostfd_request(this, buffer, pos, bytes,
                                          request::WRITE, on_cmpl);

    if (!req.get())
        stxxl_function_error(io_error);

 #ifndef NO_OVERLAPPING
    disk_queues::get_instance ()->add_writereq(req, get_id());
 #endif
    return req;
}

__STXXL_END_NAMESPACE

#endif // BOOST_VERSION

<|endoftext|>


<commit_before>// #include <compare>
// #include <concepts>

#include "acmacs-base/argv.hh"
#include "acmacs-base/range-v3.hh"
#include "acmacs-base/enumerate.hh"
#include "acmacs-base/quicklook.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/chart.hh"
#include "acmacs-chart-2/stress.hh"
#include "acmacs-chart-2/randomizer.hh"
#include "acmacs-chart-2/bounding-ball.hh"
#include "acmacs-draw/surface-cairo.hh"

// ----------------------------------------------------------------------

// template <typename T> requires requires (const T& a, const T& b)
// {
//     a.compare(b);
//     // {
//     //     a.compare(b)
//     //         } -> concepts::convertible_to<int>;
// }
// inline std::strong_ordering operator<=>(const T& lhs, const T& rhs)
// {
//     if (const auto res = lhs.compare(rhs); res == 0)
//         return std::strong_ordering::equal;
//     else if (res < 0)
//         return std::strong_ordering::less;
//     else
//         return std::strong_ordering::greater;
// }

struct TiterRef
{
    std::string serum;
    std::string antigen;
    size_t table_no;
    acmacs::chart::Titer titer;

    bool operator<(const TiterRef& rhs) const
    {
        if (const auto r1 = serum.compare(rhs.serum); r1 != 0)
            return r1 < 0;
        if (const auto r1 = antigen.compare(rhs.antigen); r1 != 0)
            return r1 < 0;
        return table_no < rhs.table_no;
    }

    // std::strong_ordering operator<=>(const TiterRef&) const = default;
};

struct TiterRefCollapsed
{
    std::string serum;
    std::string antigen;
    std::vector<acmacs::chart::Titer> titers;

    TiterRefCollapsed(const std::string& a_serum, const std::string& a_antigen, size_t num_tables) : serum{a_serum}, antigen{a_antigen}, titers(num_tables) {}

    static inline bool valid(const acmacs::chart::Titer& titer) { return !titer.is_dont_care(); }

    auto num_tables() const
    {
        return ranges::count_if(titers, valid);
    }

    auto mean_logged_titer() const
    {
        return ranges::accumulate(titers | ranges::views::filter(valid), 0.0, [](double sum, const auto& titer) { return sum + titer.logged_with_thresholded(); }) / static_cast<double>(num_tables());
    }

    bool eq(const TiterRef& raw) const { return serum == raw.serum && antigen == raw.antigen; }
};

class ChartData
{
  public:
    void scan(const acmacs::chart::Chart& chart)
    {
        const auto table_no = tables_.size();
        tables_.push_back(chart.info()->date());
        auto chart_antigens = chart.antigens();
        auto chart_sera = chart.sera();
        auto chart_titers = chart.titers();
        for (auto [ag_no, ag] : acmacs::enumerate(*chart_antigens)) {
            for (auto [sr_no, sr] : acmacs::enumerate(*chart_sera)) {
                if (const auto& titer = chart_titers->titer(ag_no, sr_no); !titer.is_dont_care())
                    raw_.push_back(TiterRef{.serum = sr->full_name(), .antigen = ag->full_name(), .table_no = table_no, .titer = titer});
            }
        }
    }

    void collapse()
    {
        ranges::sort(raw_, &TiterRef::operator<);
        for (const auto& raw : raw_) {
            if (collapsed_.empty())
                collapsed_.emplace_back(raw.serum, raw.antigen, tables_.size());
            else if (!collapsed_.back().eq(raw)) {
                if (collapsed_.back().num_tables() < 2)
                    collapsed_.pop_back();
                collapsed_.emplace_back(raw.serum, raw.antigen, tables_.size());
            }
            collapsed_.back().titers[raw.table_no] = raw.titer;
        }
    }

    void report_deviation_from_mean() const
    {
        for (const auto& en : collapsed_) {
            fmt::print("{}\n{}\n", en.serum, en.antigen);
            const auto mean = en.mean_logged_titer();
            for (size_t t_no = 0; t_no < tables_.size(); ++t_no) {
                if (!en.titers[t_no].is_dont_care())
                    fmt::print(" {}  {:>7s}  {:.2f}\n", tables_[t_no], en.titers[t_no], std::abs(en.titers[t_no].logged_with_thresholded() - mean));
                else
                    fmt::print(" {}\n", tables_[t_no]);
            }
            fmt::print("            {:.2f}\n\n", mean);
        }
    }

    void report_average_deviation_from_mean_per_table() const
    {
        std::vector<std::pair<double, size_t>> deviations(tables_.size(), {0.0, 0});
        for (const auto& en : collapsed_) {
            const auto mean = en.mean_logged_titer();
            ranges::for_each(ranges::views::iota(0ul, deviations.size()) //
                                 | ranges::views::filter([&en](size_t t_no) { return !en.titers[t_no].is_dont_care(); }),
                             [&en, &deviations, mean](size_t t_no) {
                                 deviations[t_no].first += std::abs(en.titers[t_no].logged_with_thresholded() - mean);
                                 ++deviations[t_no].second;
                             });
        }
        for (size_t t_no = 0; t_no < tables_.size(); ++t_no)
            fmt::print("{:2d} {}  {:.2f}  {:4d}\n", t_no, tables_[t_no], deviations[t_no].first / static_cast<double>(deviations[t_no].second), deviations[t_no].second);
    }

    auto make_distance_matrix(bool report = false) const
    {
        std::vector<std::vector<double>> matrix(tables_.size() * tables_.size());
        for (const auto& en : collapsed_) {
            for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
                for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                    if (!en.titers[t1].is_dont_care() && !en.titers[t2].is_dont_care())
                        matrix[t1 * tables_.size() + t2].push_back(std::abs(en.titers[t1].logged_with_thresholded() - en.titers[t2].logged_with_thresholded()));
                }
            }
        }

        if (report) {
            for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
                for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                    auto& distances = matrix[t1 * tables_.size() + t2];
                    ranges::sort(distances);
                    fmt::print("{} {} mean:{} median:{} max:{} {}\n", tables_[t1], tables_[t2], ranges::accumulate(distances, 0.0) / static_cast<double>(distances.size()),
                               distances[distances.size() / 2], distances.back(), distances);
                }
            }
        }

        acmacs::chart::TableDistances table_distances;
        for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
            for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                auto& distances = matrix[t1 * tables_.size() + t2];
                const auto mean = ranges::accumulate(distances, 0.0) / static_cast<double>(distances.size());
                table_distances.add_value(acmacs::chart::Titer::Regular, t1, t2, mean);
            }
        }
        return table_distances;
    }

    size_t num_tables() const { return tables_.size(); }
    constexpr const auto& tables() const { return tables_; }

  private:
    std::vector<acmacs::chart::TableDate> tables_;
    std::vector<TiterRef> raw_;
    std::vector<TiterRefCollapsed> collapsed_;
};

// ----------------------------------------------------------------------

using namespace acmacs::argv;
struct Options : public argv
{
    Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

    std::string_view help_pre() const override { return "compare titers of mutiple tables"; }
    argument<str_array> charts{*this, arg_name{"chart"}, mandatory};

    option<str> output{*this, 'o'};

};

int main(int argc, char* const argv[])
{
    int exit_code = 0;
    try {
        Options opt(argc, argv);

        ChartData data;
        for (const auto& fn : opt.charts)
            data.scan(*acmacs::chart::import_from_file(fn));
        data.collapse();
        // data.report_deviation_from_mean();
        data.report_average_deviation_from_mean_per_table();

        acmacs::chart::Stress stress(acmacs::number_of_dimensions_t{2}, data.num_tables());
        stress.table_distances() = data.make_distance_matrix(true);

        double best_stress{9e99};
        acmacs::Layout best_layout(data.num_tables(), stress.number_of_dimensions());
        for (auto attempt : acmacs::range(1ul)) {
            acmacs::Layout layout(data.num_tables(), stress.number_of_dimensions());
            acmacs::chart::LayoutRandomizerPlain rnd(10.0, std::nullopt);
            for (auto point_no : acmacs::range(layout.number_of_points()))
                layout.update(point_no, rnd.get(stress.number_of_dimensions()));

            const auto status1 =
                acmacs::chart::optimize(acmacs::chart::optimization_method::alglib_cg_pca, stress, layout.data(), layout.data() + layout.size(), acmacs::chart::optimization_precision::rough);
            // fmt::print("{:3d} stress: {} <- {} iterations: {}\n", attempt, status1.final_stress, status1.initial_stress, status1.number_of_iterations);
            if (status1.final_stress < best_stress) {
                best_stress = status1.final_stress;
                best_layout = layout;
            }
        }
        fmt::print("best_stress: {}\n", best_stress);

        if (opt.output) {
            const std::array colors_of_tables{
                GREEN,   // 20180206 Leah G/Heidi
                RED,     // 20180227 Tasuola
                BLUE,   // 20180528 Rob
                RED,     // 20180605 Tasuola
                BLUE,   // 20180626 Rob
                GREEN,   // 20180821 Heidi/Malet
                GREEN,   // 20180918 Heidi/Tas
                GREEN,   // 20181127 Heidi
                RED,   // 20181213 Tasuola
                RED,   // 20190211 Tasuola/Rob
                RED,   // 20190409 Tasuola
                GREEN,   // 20190514 Leah
                GREEN,   // 20190521 Leah
                GREEN,   // 20190716 Leah/Heidi
                GREEN,    // 20190820 Leah
                GREEN,   // 20190903 Leah
                GREEN,    // 20190918 Leah

                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
                GREEN,
            };
            const double size = 800.0;
            acmacs::surface::PdfCairo surface(opt.output, size, size, size);
            const acmacs::BoundingBall bb{minimum_bounding_ball(best_layout)};
            acmacs::Viewport viewport;
            viewport.set_from_center_size(bb.center(), bb.diameter());
            viewport.whole_width();
            fmt::print("{}\n", viewport);
            acmacs::surface::Surface& rescaled_surface = surface.subsurface(acmacs::PointCoordinates::zero2D, Scaled{surface.viewport().size.width}, viewport, true);
            rescaled_surface.grid(Scaled{1.0}, GREY, Pixels{1});
            for (size_t t1 = 0; t1 < data.num_tables(); ++t1) {
                rescaled_surface.circle_filled(best_layout[t1], Pixels{10}, AspectNormal, NoRotation, BLACK, Pixels{1}, acmacs::surface::Dash::NoDash, colors_of_tables[t1]);
                rescaled_surface.text(best_layout[t1] + acmacs::PointCoordinates{-0.05, 0.05}, data.tables()[t1], RED, Pixels{10});
            }
            acmacs::open(opt.output, 1, 1);
        }
    }
    catch (std::exception& err) {
        AD_ERROR("{}", err);
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>chart-table-compare<commit_after>// #include <compare>
// #include <concepts>

#include "acmacs-base/argv.hh"
#include "acmacs-base/range-v3.hh"
#include "acmacs-base/enumerate.hh"
#include "acmacs-base/quicklook.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/chart.hh"
#include "acmacs-chart-2/stress.hh"
#include "acmacs-chart-2/randomizer.hh"
#include "acmacs-chart-2/bounding-ball.hh"
#include "acmacs-draw/surface-cairo.hh"

// ----------------------------------------------------------------------

// template <typename T> requires requires (const T& a, const T& b)
// {
//     a.compare(b);
//     // {
//     //     a.compare(b)
//     //         } -> concepts::convertible_to<int>;
// }
// inline std::strong_ordering operator<=>(const T& lhs, const T& rhs)
// {
//     if (const auto res = lhs.compare(rhs); res == 0)
//         return std::strong_ordering::equal;
//     else if (res < 0)
//         return std::strong_ordering::less;
//     else
//         return std::strong_ordering::greater;
// }

struct TiterRef
{
    std::string serum;
    std::string antigen;
    size_t table_no;
    acmacs::chart::Titer titer;

    bool operator<(const TiterRef& rhs) const
    {
        if (const auto r1 = serum.compare(rhs.serum); r1 != 0)
            return r1 < 0;
        if (const auto r1 = antigen.compare(rhs.antigen); r1 != 0)
            return r1 < 0;
        return table_no < rhs.table_no;
    }

    // std::strong_ordering operator<=>(const TiterRef&) const = default;
};

struct TiterRefCollapsed
{
    std::string serum;
    std::string antigen;
    std::vector<acmacs::chart::Titer> titers;

    TiterRefCollapsed(const std::string& a_serum, const std::string& a_antigen, size_t num_tables) : serum{a_serum}, antigen{a_antigen}, titers(num_tables) {}

    static inline bool valid(const acmacs::chart::Titer& titer) { return !titer.is_dont_care(); }

    auto num_tables() const
    {
        return ranges::count_if(titers, valid);
    }

    auto mean_logged_titer() const
    {
        return ranges::accumulate(titers | ranges::views::filter(valid), 0.0, [](double sum, const auto& titer) { return sum + titer.logged_with_thresholded(); }) / static_cast<double>(num_tables());
    }

    bool eq(const TiterRef& raw) const { return serum == raw.serum && antigen == raw.antigen; }
};

class ChartData
{
  public:
    void scan(const acmacs::chart::Chart& chart)
    {
        const auto table_no = tables_.size();
        tables_.push_back(chart.info()->date());
        auto chart_antigens = chart.antigens();
        auto chart_sera = chart.sera();
        auto chart_titers = chart.titers();
        for (auto [ag_no, ag] : acmacs::enumerate(*chart_antigens)) {
            for (auto [sr_no, sr] : acmacs::enumerate(*chart_sera)) {
                if (const auto& titer = chart_titers->titer(ag_no, sr_no); !titer.is_dont_care())
                    raw_.push_back(TiterRef{.serum = sr->full_name(), .antigen = ag->full_name(), .table_no = table_no, .titer = titer});
            }
        }
    }

    void collapse()
    {
        ranges::sort(raw_, &TiterRef::operator<);
        for (const auto& raw : raw_) {
            if (collapsed_.empty())
                collapsed_.emplace_back(raw.serum, raw.antigen, tables_.size());
            else if (!collapsed_.back().eq(raw)) {
                if (collapsed_.back().num_tables() < 2)
                    collapsed_.pop_back();
                collapsed_.emplace_back(raw.serum, raw.antigen, tables_.size());
            }
            collapsed_.back().titers[raw.table_no] = raw.titer;
        }
    }

    void report_deviation_from_mean() const
    {
        for (const auto& en : collapsed_) {
            fmt::print("{}\n{}\n", en.serum, en.antigen);
            const auto mean = en.mean_logged_titer();
            for (size_t t_no = 0; t_no < tables_.size(); ++t_no) {
                if (!en.titers[t_no].is_dont_care())
                    fmt::print(" {}  {:>7s}  {:.2f}\n", tables_[t_no], en.titers[t_no], std::abs(en.titers[t_no].logged_with_thresholded() - mean));
                else
                    fmt::print(" {}\n", tables_[t_no]);
            }
            fmt::print("            {:.2f}\n\n", mean);
        }
    }

    void report_average_deviation_from_mean_per_table() const
    {
        std::vector<std::pair<double, size_t>> deviations(tables_.size(), {0.0, 0});
        for (const auto& en : collapsed_) {
            const auto mean = en.mean_logged_titer();
            ranges::for_each(ranges::views::iota(0ul, deviations.size()) //
                                 | ranges::views::filter([&en](size_t t_no) { return !en.titers[t_no].is_dont_care(); }),
                             [&en, &deviations, mean](size_t t_no) {
                                 deviations[t_no].first += std::abs(en.titers[t_no].logged_with_thresholded() - mean);
                                 ++deviations[t_no].second;
                             });
        }
        for (size_t t_no = 0; t_no < tables_.size(); ++t_no)
            fmt::print("{:2d} {}  {:.2f}  {:4d}\n", t_no, tables_[t_no], deviations[t_no].first / static_cast<double>(deviations[t_no].second), deviations[t_no].second);
    }

    auto make_distance_matrix(bool report = false) const
    {
        std::vector<std::vector<double>> matrix(tables_.size() * tables_.size());
        for (const auto& en : collapsed_) {
            for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
                for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                    if (!en.titers[t1].is_dont_care() && !en.titers[t2].is_dont_care())
                        matrix[t1 * tables_.size() + t2].push_back(std::abs(en.titers[t1].logged_with_thresholded() - en.titers[t2].logged_with_thresholded()));
                }
            }
        }

        if (report) {
            for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
                for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                    if (auto& distances = matrix[t1 * tables_.size() + t2]; !distances.empty()) {
                        ranges::sort(distances);
                        fmt::print("{} {} mean:{} median:{} max:{} {}\n", tables_[t1], tables_[t2], ranges::accumulate(distances, 0.0) / static_cast<double>(distances.size()),
                                   distances[distances.size() / 2], distances.back(), distances);
                    }
                    else
                        fmt::print("{} {}: *no distances*\n", tables_[t1], tables_[t2]);
                }
            }
        }

        acmacs::chart::TableDistances table_distances;
        for (size_t t1 = 0; t1 < (tables_.size() - 1); ++t1) {
            for (size_t t2 = t1 + 1; t2 < tables_.size(); ++t2) {
                if (auto& distances = matrix[t1 * tables_.size() + t2]; !distances.empty()) {
                    const auto mean = ranges::accumulate(distances, 0.0) / static_cast<double>(distances.size());
                    table_distances.add_value(acmacs::chart::Titer::Regular, t1, t2, mean);
                }
            }
        }
        return table_distances;
    }

    size_t num_tables() const { return tables_.size(); }
    constexpr const auto& tables() const { return tables_; }

  private:
    std::vector<acmacs::chart::TableDate> tables_;
    std::vector<TiterRef> raw_;
    std::vector<TiterRefCollapsed> collapsed_;
};

// ----------------------------------------------------------------------

using namespace acmacs::argv;
struct Options : public argv
{
    Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

    std::string_view help_pre() const override { return "compare titers of mutiple tables"; }
    argument<str_array> charts{*this, arg_name{"chart"}, mandatory};

    option<str> output{*this, 'o'};
    option<size_t> number_of_optimizations{*this, 'n', dflt{100ul}};
};

int main(int argc, char* const argv[])
{
    int exit_code = 0;
    try {
        Options opt(argc, argv);

        ChartData data;
        for (const auto& fn : opt.charts)
            data.scan(*acmacs::chart::import_from_file(fn));
        data.collapse();
        // data.report_deviation_from_mean();
        data.report_average_deviation_from_mean_per_table();

        acmacs::chart::Stress stress(acmacs::number_of_dimensions_t{2}, data.num_tables());
        stress.table_distances() = data.make_distance_matrix();

        double best_stress{9e99};
        acmacs::Layout best_layout(data.num_tables(), stress.number_of_dimensions());
        for ([[maybe_unused]] auto attempt : acmacs::range(*opt.number_of_optimizations)) {
            acmacs::Layout layout(data.num_tables(), stress.number_of_dimensions());
            acmacs::chart::LayoutRandomizerPlain rnd(10.0, std::nullopt);
            for (auto point_no : acmacs::range(layout.number_of_points()))
                layout.update(point_no, rnd.get(stress.number_of_dimensions()));

            const auto status1 =
                acmacs::chart::optimize(acmacs::chart::optimization_method::alglib_cg_pca, stress, layout.data(), layout.data() + layout.size(), acmacs::chart::optimization_precision::rough);
            // fmt::print("{:3d} stress: {} <- {} iterations: {}\n", attempt, status1.final_stress, status1.initial_stress, status1.number_of_iterations);
            if (status1.final_stress < best_stress) {
                best_stress = status1.final_stress;
                best_layout = layout;
            }
        }
        fmt::print("best_stress: {}\n", best_stress);

        if (opt.output) {
            const std::array colors_of_tables{
                GREEN,   // 20180206 Leah G/Heidi
                RED,     // 20180227 Tasuola
                BLUE,   // 20180528 Rob
                RED,     // 20180605 Tasuola
                BLUE,   // 20180626 Rob
                GREEN,   // 20180821 Heidi/Malet
                GREEN,   // 20180918 Heidi/Tas
                GREEN,   // 20181127 Heidi
                RED,   // 20181213 Tasuola
                RED,   // 20190211 Tasuola/Rob
                RED,   // 20190409 Tasuola
                GREEN,   // 20190514 Leah
                GREEN,   // 20190521 Leah
                GREEN,   // 20190716 Leah/Heidi
                GREEN,    // 20190820 Leah
                GREEN,   // 20190903 Leah
                GREEN,    // 20190918 Leah
            };
            const double size = 800.0;
            acmacs::surface::PdfCairo surface(opt.output, size, size, size);
            const acmacs::BoundingBall bb{minimum_bounding_ball(best_layout)};
            acmacs::Viewport viewport;
            viewport.set_from_center_size(bb.center(), bb.diameter());
            viewport.whole_width();
            fmt::print("{}\n", viewport);
            acmacs::surface::Surface& rescaled_surface = surface.subsurface(acmacs::PointCoordinates::zero2D, Scaled{surface.viewport().size.width}, viewport, true);
            rescaled_surface.grid(Scaled{1.0}, GREY, Pixels{1});
            for (size_t t1 = 0; t1 < data.num_tables(); ++t1) {
                const auto fill = t1 < colors_of_tables.size() ? colors_of_tables[t1] : GREEN;
                rescaled_surface.circle_filled(best_layout[t1], Pixels{10}, AspectNormal, NoRotation, BLACK, Pixels{1}, acmacs::surface::Dash::NoDash, fill);
                rescaled_surface.text(best_layout[t1] + acmacs::PointCoordinates{-0.05, 0.05}, data.tables()[t1], BLACK, Pixels{10});
            }
            acmacs::open(opt.output, 1, 1);
        }
    }
    catch (std::exception& err) {
        AD_ERROR("{}", err);
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>#include "gtest/gtest.h"
#include "filesystem/Path.h"
#include <iostream>

using namespace std::literals::string_literals;

::std::ostream& operator<<(::std::ostream& os, const Path& path) {
  return os << std::string(path).c_str();
}

TEST(PathAppendTests, ReturnsAppendedPathWithFileNAme)
{
	Path path("/home/serengeor/Coding");
	Path path2("TheEngine2");
	Path correctPath("/home/serengeor/Coding/TheEngine2");

	Path appendPathResult = path.Append(path2);

	ASSERT_EQ(correctPath, appendPathResult);
}

TEST(PathAppendTests, ReturnsAppendedPathWithoutAdditionalSeparators)
{
	//REFACTOR: Clarify this test
	Path path("/home/serengeor/Coding");
	Path pathS("/home/serengeor/Coding/");
	Path path2("TheEngine2");
	Path path2S("/TheEngine2");
	Path correctPath("/home/serengeor/Coding/TheEngine2");

	Path appendPathResult1 = path.Append(path2);
	Path appendPathResult2 = pathS.Append(path2);
	Path appendPathResult3 = path.Append(path2S);
	Path appendPathResult4 = pathS.Append(path2S);

	ASSERT_EQ(correctPath, appendPathResult1);
	ASSERT_EQ(correctPath, appendPathResult2);
	ASSERT_EQ(correctPath, appendPathResult3);
	ASSERT_EQ(correctPath, appendPathResult4);
}

TEST(PathFileNameTests, ReturnsFileNameWithoutPath)
{
	Path path("/home/serengeor/Coding/TheEngine2");
	Path path2("/TheEngine2");

	ASSERT_EQ("TheEngine2"s, path.GetFileName());
	ASSERT_EQ("TheEngine2"s, path2.GetFileName());
}

TEST(PathFileNameTests, ReturnsFileNameIfNotInDirectory)
{
	Path path("TheEngine2");

	ASSERT_EQ("TheEngine2"s, path.GetFileName());
}

TEST(PathFileNameTests, ReturnsFileNameWithExtension)
{
	Path path("/home/serengeor/Coding/TheEngine2.exe");

	ASSERT_EQ("TheEngine2.exe"s, path.GetFileName());
}

TEST(PathParentDirectoryTests, ReturnsParentDirWhenPathWithFile)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2";
	Path path(pathString+fileString);

	ASSERT_EQ(pathString, path.GetParentDirectory());
}

TEST(PathParentDirectoryTests, ReturnsParentDirWhenPathWithFileAndExtension)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2.exe";
	Path path(pathString+fileString);

	ASSERT_EQ(pathString, path.GetParentDirectory());
}

TEST(PathExtensionTests, ReturnsExtensionWithDotWhenFileWithPath)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2.exe";
	Path path(pathString+fileString);

	ASSERT_EQ(".exe"s, path.GetExtension());
}

TEST(PathExtensionTests, ReturnsExtensionWithDotWhenFileWithout)
{
	std::string fileString = "TheEngine2.exe";
	Path path(fileString);

	ASSERT_EQ(".exe"s, path.GetExtension());
}

TEST(PathExtensionTests, ReturnsEmptyStringForExtensionWhenNoExtension)
{
	std::string fileString = "TheEngine2";
	Path path(fileString);

	ASSERT_EQ(""s, path.GetExtension());
}

TEST(PathNormalizationTests, DuplicatedDirSeparatorsAreNormalizedOnConstruction)
{
	std::string goodPath = "C:/SomeRelativePath/BlaBla/SomeFile.exec";
	std::string badPath = "C://SomeRelativePath///////BlaBla//SomeFile.exec";

	Path normalizedPath(badPath);

	ASSERT_EQ(goodPath, normalizedPath.AsString());
}<commit_msg>Windows separator test case.<commit_after>#include "gtest/gtest.h"
#include "filesystem/Path.h"
#include <iostream>

using namespace std::literals::string_literals;

::std::ostream& operator<<(::std::ostream& os, const Path& path) {
  return os << std::string(path).c_str();
}

TEST(PathAppendTests, ReturnsAppendedPathWithFileNAme)
{
	Path path("/home/serengeor/Coding");
	Path path2("TheEngine2");
	Path correctPath("/home/serengeor/Coding/TheEngine2");

	Path appendPathResult = path.Append(path2);

	ASSERT_EQ(correctPath, appendPathResult);
}

TEST(PathAppendTests, ReturnsAppendedPathWithoutAdditionalSeparators)
{
	//REFACTOR: Clarify this test
	Path path("/home/serengeor/Coding");
	Path pathS("/home/serengeor/Coding/");
	Path path2("TheEngine2");
	Path path2S("/TheEngine2");
	Path correctPath("/home/serengeor/Coding/TheEngine2");

	Path appendPathResult1 = path.Append(path2);
	Path appendPathResult2 = pathS.Append(path2);
	Path appendPathResult3 = path.Append(path2S);
	Path appendPathResult4 = pathS.Append(path2S);

	ASSERT_EQ(correctPath, appendPathResult1);
	ASSERT_EQ(correctPath, appendPathResult2);
	ASSERT_EQ(correctPath, appendPathResult3);
	ASSERT_EQ(correctPath, appendPathResult4);
}

TEST(PathFileNameTests, ReturnsFileNameWithoutPath)
{
	Path path("/home/serengeor/Coding/TheEngine2");
	Path path2("/TheEngine2");

	ASSERT_EQ("TheEngine2"s, path.GetFileName());
	ASSERT_EQ("TheEngine2"s, path2.GetFileName());
}

TEST(PathFileNameTests, ReturnsFileNameIfNotInDirectory)
{
	Path path("TheEngine2");

	ASSERT_EQ("TheEngine2"s, path.GetFileName());
}

TEST(PathFileNameTests, ReturnsFileNameWithExtension)
{
	Path path("/home/serengeor/Coding/TheEngine2.exe");

	ASSERT_EQ("TheEngine2.exe"s, path.GetFileName());
}

TEST(PathParentDirectoryTests, ReturnsParentDirWhenPathWithFile)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2";
	Path path(pathString+fileString);

	ASSERT_EQ(pathString, path.GetParentDirectory());
}

TEST(PathParentDirectoryTests, ReturnsParentDirWhenPathWithFileAndExtension)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2.exe";
	Path path(pathString+fileString);

	ASSERT_EQ(pathString, path.GetParentDirectory());
}

TEST(PathExtensionTests, ReturnsExtensionWithDotWhenFileWithPath)
{
	std::string pathString = "/home/serengeor/Coding";
	std::string fileString = "/TheEngine2.exe";
	Path path(pathString+fileString);

	ASSERT_EQ(".exe"s, path.GetExtension());
}

TEST(PathExtensionTests, ReturnsExtensionWithDotWhenFileWithout)
{
	std::string fileString = "TheEngine2.exe";
	Path path(fileString);

	ASSERT_EQ(".exe"s, path.GetExtension());
}

TEST(PathExtensionTests, ReturnsEmptyStringForExtensionWhenNoExtension)
{
	std::string fileString = "TheEngine2";
	Path path(fileString);

	ASSERT_EQ(""s, path.GetExtension());
}

TEST(PathNormalizationTests, DuplicatedDirSeparatorsAreNormalizedOnConstruction)
{
	std::string goodPath = "C:/SomeRelativePath/BlaBla/SomeFile.exec";
	std::string badUnixSeparatorPath = "C://SomeRelativePath///////BlaBla//SomeFile.exec";
	std::string badWinSeparatorPath = "C:\\\\SomeRelativePath\\\\\\\\BlaBla\\\\SomeFile.exec";

	Path normalizedUnixPath(badUnixSeparatorPath);
	Path normalizedWinPath(badWinSeparatorPath);

	ASSERT_EQ(goodPath, normalizedUnixPath.AsString());
	ASSERT_EQ(goodPath, normalizedWinPath.AsString());
}<|endoftext|>


<commit_before>/***************************************************************************
 *  @file       horner_rule.hpp
 *  @author     Alan.W
 *  @date       27  Aug 2014
 *  @version    2
 *  @remark     CLRS Algorithms implementation in C++ templates.
 ***************************************************************************/
//!
//! @problem 2-3 Correctness of Horner’s rule
//!

#ifndef HORNER_RULE_HPP
#define HORNER_RULE_HPP

#include "iterator.hpp"

namespace clrs {namespace ch2 {

/**
 * @brief horner_rule
 * @param first
 * @param last
 * @param x
 *
 * @ex  problem 2-3
 * @complx  O(n)
 */
template<typename Iter>
inline clrs::IterValue<Iter>
horner_rule(Iter first, Iter last, const clrs::IterValue<Iter>& x)
{
    clrs::IterValue<Iter> ret = 0;
    for(auto it = last - 1; it != first - 1; --it)
        ret = *it + x * ret;

    return ret;
}

}}//namespace

#endif // HORNER_RULE_HPP

//! @test   horner_rule for problem 2-3
//!
//#include <iostream>
//#include <vector>
//#include "alan.hpp"
//#include "horner_rule.hpp"

//int main()
//{
//    std::vector<int> v{1,6,88,2,3,77};
//    auto ret = clrs::ch2::horner_rule(v.begin(), v.end(),3);
//    std::cout << ret;

//    alan::end();
//    return 0;
//}
//! @output
//!
//19819
//exit normally
<commit_msg>refactor : polynomial_evaluate, not tested 	modified:   horner_rule.hpp<commit_after>/***************************************************************************
 *  @file       horner_rule.hpp
 *  @author     Alan.W
 *  @date       27  Aug 2014
 *  @version    2
 *  @remark     CLRS Algorithms implementation in C++ templates.
 ***************************************************************************/
//!
//! @problem 2-3 Correctness of Horner’s rule
//!

#ifndef HORNER_RULE_HPP
#define HORNER_RULE_HPP

#include "iterator.hpp"

namespace clrs {namespace ch2 {

/**
 * @brief horner_rule
 * @param first
 * @param last
 * @param x
 *
 * @ex  problem 2-3
 * @complx  O(n)
 */
template<typename Iter>
inline clrs::IterValue<Iter>
horner_rule(Iter first, Iter last, const clrs::IterValue<Iter>& x)
{
    clrs::IterValue<Iter> ret = 0;
    for(auto it = last - 1; it != first - 1; --it)
        ret = *it + x * ret;

    return ret;
}

template<typename ValueType>
inline ValueType
power(int order, const ValueType& x)
{
    return  order > 0?    x * power(order - 1, x) :   1;
}

template<typename Iter>
clrs::IterValue<Iter>
polynomial_evaluate(Iter first, Iter last, const clrs::IterValue<Iter>& x)
{
    IterValue<Iter> ret = 0;
    int order = 0;
    for(auto it = first; it != last; ++it)
        ret += *it  *   power(order++, x);

    return ret;
}

}}//namespace

#endif // HORNER_RULE_HPP

//! @test   horner_rule for problem 2-3
//!
//#include <iostream>
//#include <vector>
//#include "alan.hpp"
//#include "horner_rule.hpp"

//int main()
//{
//    std::vector<int> v{1,6,88,2,3,77};
//    auto ret = clrs::ch2::horner_rule(v.begin(), v.end(),3);
//    std::cout << ret;

//    alan::end();
//    return 0;
//}
//! @output
//!
//19819
//exit normally
<|endoftext|>


<commit_before><commit_msg>Improve shadow cascade frustum bounds<commit_after><|endoftext|>