<commit_before>#include <bts/chain/asset.hpp>
#include <fc/uint128.hpp>

namespace bts { namespace chain {
      bool operator < ( const asset& a, const asset& b )
      {
         return std::tie( a.asset_id, a.amount ) < std::tie( b.asset_id, b.amount);
      }
      bool operator <= ( const asset& a, const asset& b )
      {
         return std::tie( a.asset_id, a.amount ) <= std::tie( b.asset_id, b.amount);
      }
      bool operator < ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         auto bmult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         assert( (a.to_real() < b.to_real()) == (amult < bmult) );
         return amult < bmult;
      }
      bool operator <= ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         auto bmult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         assert( (a.to_real() <= b.to_real()) == (amult <= bmult) );
         return amult <= bmult;
      }
      bool operator == ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         auto bmult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         return amult == bmult;
      }
      bool operator != ( const price& a, const price& b )
      {
         return !(a==b);
      }

      bool operator >= ( const price& a, const price& b )
      {
         return !(a < b);
      }
      bool operator > ( const price& a, const price& b )
      {
         return !(a <= b);
      }

      asset operator * ( const asset& a, const price& b )
      {
         if( a.asset_id == b.base.asset_id )
         {
            FC_ASSERT( b.base.amount.value > 0 );
            auto result = (fc::uint128(a.amount.value) * b.quote.amount.value)/b.base.amount.value;
            FC_ASSERT( result <= BTS_MAX_SHARE_SUPPLY );
            return asset( result.to_uint64(), b.quote.asset_id );
         }
         else if( a.asset_id == b.quote.asset_id )
         {
            FC_ASSERT( b.quote.amount.value > 0 );
            auto result = (fc::uint128(a.amount.value) * b.base.amount.value)/b.quote.amount.value;
            FC_ASSERT( result <= BTS_MAX_SHARE_SUPPLY );
            return asset( result.to_uint64(), b.base.asset_id );
         }
         FC_ASSERT( !"invalid asset * price", "", ("asset",a)("price",b) );
      }

      price operator / ( const asset& base, const asset& quote )
      {
         assert( base.asset_id != quote.asset_id );
         return price{base,quote};
      }
      price price::max( asset_id_type base, asset_id_type quote ) { return asset( share_type(BTS_MAX_SHARE_SUPPLY), base ) / asset( share_type(1), quote); }
      price price::min( asset_id_type base, asset_id_type quote ) { return asset( 1, base ) / asset( BTS_MAX_SHARE_SUPPLY, quote); }

      price price::call_price(const asset& debt, const asset& collateral, uint16_t collateral_ratio)
      {
         fc::uint128 tmp( collateral.amount.value );
         tmp *= collateral_ratio - 1000;
         tmp /= 1000;
         FC_ASSERT( tmp <= BTS_MAX_SHARE_SUPPLY );
         return asset( tmp.to_uint64(), collateral.asset_id) / debt;
      }

      bool price::is_null() const { return *this == price(); }

      void price::validate() const
      { try {
         FC_ASSERT( base.amount > share_type(0) );
         FC_ASSERT( quote.amount > share_type(0) );
         FC_ASSERT( base.asset_id != quote.asset_id );
      } FC_CAPTURE_AND_RETHROW() }

      void price_feed::validate() const
      { try {
         if( !call_limit.is_null() )
            call_limit.validate();
         if( !short_limit.is_null() )
            short_limit.validate();
         if( !settlement_price.is_null() )
            settlement_price.validate();
         FC_ASSERT( call_limit.is_null() == short_limit.is_null() );
         FC_ASSERT( call_limit.base.asset_id == short_limit.quote.asset_id );
         FC_ASSERT( call_limit.quote.asset_id == short_limit.base.asset_id );
         FC_ASSERT( max_margin_period_sec > 0 );
         FC_ASSERT( required_maintenance_collateral < required_initial_collateral );
         FC_ASSERT( required_maintenance_collateral >= 1000 );
         FC_ASSERT( call_limit.is_null() || call_limit < ~short_limit );
      } FC_CAPTURE_AND_RETHROW() }

} } // bts::chain
<commit_msg>Don't crash if dividing an unexpected price<commit_after>#include <bts/chain/asset.hpp>
#include <fc/uint128.hpp>

namespace bts { namespace chain {
      bool operator < ( const asset& a, const asset& b )
      {
         return std::tie( a.asset_id, a.amount ) < std::tie( b.asset_id, b.amount);
      }
      bool operator <= ( const asset& a, const asset& b )
      {
         return std::tie( a.asset_id, a.amount ) <= std::tie( b.asset_id, b.amount);
      }
      bool operator < ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         auto bmult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         assert( (a.to_real() < b.to_real()) == (amult < bmult) );
         return amult < bmult;
      }
      bool operator <= ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         auto bmult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         assert( (a.to_real() <= b.to_real()) == (amult <= bmult) );
         return amult <= bmult;
      }
      bool operator == ( const price& a, const price& b )
      {
         if( a.base.asset_id < b.base.asset_id ) return true;
         if( a.base.asset_id > b.base.asset_id ) return false;
         if( a.quote.asset_id < b.quote.asset_id ) return true;
         if( a.quote.asset_id > b.quote.asset_id ) return false;
         auto amult = fc::uint128(a.quote.amount.value) * b.base.amount.value;
         auto bmult = fc::uint128(b.quote.amount.value) * a.base.amount.value;
         return amult == bmult;
      }
      bool operator != ( const price& a, const price& b )
      {
         return !(a==b);
      }

      bool operator >= ( const price& a, const price& b )
      {
         return !(a < b);
      }
      bool operator > ( const price& a, const price& b )
      {
         return !(a <= b);
      }

      asset operator * ( const asset& a, const price& b )
      {
         if( a.asset_id == b.base.asset_id )
         {
            FC_ASSERT( b.base.amount.value > 0 );
            auto result = (fc::uint128(a.amount.value) * b.quote.amount.value)/b.base.amount.value;
            FC_ASSERT( result <= BTS_MAX_SHARE_SUPPLY );
            return asset( result.to_uint64(), b.quote.asset_id );
         }
         else if( a.asset_id == b.quote.asset_id )
         {
            FC_ASSERT( b.quote.amount.value > 0 );
            auto result = (fc::uint128(a.amount.value) * b.base.amount.value)/b.quote.amount.value;
            FC_ASSERT( result <= BTS_MAX_SHARE_SUPPLY );
            return asset( result.to_uint64(), b.base.asset_id );
         }
         FC_ASSERT( !"invalid asset * price", "", ("asset",a)("price",b) );
      }

      price operator / ( const asset& base, const asset& quote )
      {
         FC_ASSERT( base.asset_id != quote.asset_id );
         return price{base,quote};
      }
      price price::max( asset_id_type base, asset_id_type quote ) { return asset( share_type(BTS_MAX_SHARE_SUPPLY), base ) / asset( share_type(1), quote); }
      price price::min( asset_id_type base, asset_id_type quote ) { return asset( 1, base ) / asset( BTS_MAX_SHARE_SUPPLY, quote); }

      price price::call_price(const asset& debt, const asset& collateral, uint16_t collateral_ratio)
      {
         fc::uint128 tmp( collateral.amount.value );
         tmp *= collateral_ratio - 1000;
         tmp /= 1000;
         FC_ASSERT( tmp <= BTS_MAX_SHARE_SUPPLY );
         return asset( tmp.to_uint64(), collateral.asset_id) / debt;
      }

      bool price::is_null() const { return *this == price(); }

      void price::validate() const
      { try {
         FC_ASSERT( base.amount > share_type(0) );
         FC_ASSERT( quote.amount > share_type(0) );
         FC_ASSERT( base.asset_id != quote.asset_id );
      } FC_CAPTURE_AND_RETHROW() }

      void price_feed::validate() const
      { try {
         if( !call_limit.is_null() )
            call_limit.validate();
         if( !short_limit.is_null() )
            short_limit.validate();
         if( !settlement_price.is_null() )
            settlement_price.validate();
         FC_ASSERT( call_limit.is_null() == short_limit.is_null() );
         FC_ASSERT( call_limit.base.asset_id == short_limit.quote.asset_id );
         FC_ASSERT( call_limit.quote.asset_id == short_limit.base.asset_id );
         FC_ASSERT( max_margin_period_sec > 0 );
         FC_ASSERT( required_maintenance_collateral < required_initial_collateral );
         FC_ASSERT( required_maintenance_collateral >= 1000 );
         FC_ASSERT( call_limit.is_null() || call_limit < ~short_limit );
      } FC_CAPTURE_AND_RETHROW() }

} } // bts::chain
<|endoftext|>


<commit_before>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright 2019-2021 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#include <hip/hip_ext.h>
#include <hip/hip_runtime.h>

#include <cstddef>

#include <Tensile/Debug.hpp>
#include <Tensile/EmbeddedData.hpp>
#include <Tensile/hip/HipSolutionAdapter.hpp>
#include <Tensile/hip/HipUtils.hpp>

namespace Tensile
{
    namespace hip
    {
        SolutionAdapter::SolutionAdapter()
            : m_debug(Debug::Instance().printKernelArguments())
            , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch())
        {
        }

        SolutionAdapter::SolutionAdapter(bool debug)
            : m_debug(debug)
        {
            m_debug = debug || Debug::Instance().printKernelArguments();
        }

        SolutionAdapter::SolutionAdapter(bool debug, std::string const& name)
            : m_debug(debug)
            , m_name(name)
        {
            m_debug = debug || Debug::Instance().printKernelArguments();
        }

        SolutionAdapter::~SolutionAdapter()
        {
            for(auto module : m_modules)
                hipModuleUnload(module);
        }

        hipError_t SolutionAdapter::loadCodeObjectFile(std::string const& path)
        {
            hipModule_t module;

            HIP_CHECK_RETURN(hipModuleLoad(&module, path.c_str()));

            if(m_debug)
                std::cout << "loaded code object" << path << std::endl;

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.push_back(module);
                m_loadedModuleNames.push_back(concatenate("File ", path));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::loadCodeObjectBytes(std::vector<uint8_t> const& bytes)
        {
            return loadCodeObject(bytes.data());
        }

        hipError_t SolutionAdapter::loadCodeObject(const void* image)
        {
            hipModule_t module;

            HIP_CHECK_RETURN(hipModuleLoadData(&module, image));

            if(m_debug)
                std::cout << "loaded code object data." << std::endl;

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.push_back(module);
                m_loadedModuleNames.push_back("Module from bytes");
            }
            return hipSuccess;
        }

        void SolutionAdapter::loadEmbeddedCodeObjects()
        {
            loadEmbeddedCodeObjects("");
        }

        void SolutionAdapter::loadEmbeddedCodeObjects(std::string const& key)
        {
            auto const& embeddedData = EmbeddedData<Tensile::SolutionAdapter>::Get(key);

            if(embeddedData.size() == 0)
            {
                if(m_debug || Debug::Instance().printCodeObjectInfo())
                {
                    std::cerr << "Found no embedded code objects";
                    if(key != "")
                        std::cerr << " with the key " << key;

                    std::cerr << "." << std::endl;
                }
                return;
            }

            std::vector<hipModule_t> newModules;
            newModules.reserve(embeddedData.size());

            for(size_t i = 0; i < embeddedData.size(); i++)
            {
                hipModule_t nextModule;
                try
                {
                    auto error = hipModuleLoadData(&nextModule, embeddedData[i].data());

                    if(error == hipErrorUnknown || error == hipErrorSharedObjectInitFailed)
                        continue;
                    newModules.push_back(nextModule);
                    HIP_CHECK_EXC(error);

                    if(m_debug)
                        std::cout << "Loaded code object for key " << key << std::endl;
                }
                catch(std::runtime_error const& exc)
                {
                    std::cout << exc.what() << std::endl;
                }
            }

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.insert(m_modules.end(), newModules.begin(), newModules.end());
                m_loadedModuleNames.push_back(
                    concatenate("Embedded code object ", key, " (", newModules.size(), ")"));
            }
        }

        hipError_t SolutionAdapter::initKernel(std::string const& name)
        {
            hipFunction_t function;
            return getKernel(function, name);
        }

        hipError_t SolutionAdapter::getKernel(hipFunction_t& rv, std::string const& name)
        {
            std::unique_lock<std::mutex> guard(m_access);
            hipError_t                   err = hipSuccess;

            auto it = m_kernels.find(name);
            if(it != m_kernels.end())
            {
                rv = it->second;
                return err;
            }

            for(auto module : m_modules)
            {
                err = hipModuleGetFunction(&rv, module, name.c_str());

                if(err == hipSuccess)
                {
                    m_kernels[name] = rv;
                    return err;
                }
                else if(err != hipErrorNotFound)
                {
                    return err;
                }
            }

            return err;
        }

        hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel)
        {
            return launchKernel(kernel, nullptr, nullptr, nullptr);
        }

        hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel,
                                                 hipStream_t             stream,
                                                 hipEvent_t              startEvent,
                                                 hipEvent_t              stopEvent)
        {
            if(m_debug)
            {
                std::cout << "Kernel " << kernel.kernelName << std::endl;
                std::cout << " l" << kernel.workGroupSize << " x g" << kernel.numWorkGroups << " = "
                          << kernel.numWorkItems << std::endl;
                std::cout << kernel.args;
            }
            if(m_debugSkipLaunch)
            {
                std::cout << "DEBUG: Skip kernel execution" << std::endl;
                if(startEvent != nullptr)
                    HIP_CHECK_RETURN(hipEventRecord(startEvent, stream));
                if(stopEvent != nullptr)
                    HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream));
                return hipSuccess;
            }

            hipFunction_t function;
            HIP_CHECK_RETURN(getKernel(function, kernel.kernelName));

            void*  kernelArgs = const_cast<void*>(kernel.args.data());
            size_t argsSize   = kernel.args.size();

            void* hipLaunchParams[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER,
                                       kernelArgs,
                                       HIP_LAUNCH_PARAM_BUFFER_SIZE,
                                       &argsSize,
                                       HIP_LAUNCH_PARAM_END};

            if(startEvent != nullptr)
                HIP_CHECK_RETURN(hipEventRecord(startEvent, stream));
            HIP_CHECK_RETURN(hipExtModuleLaunchKernel(function,
                                                      kernel.numWorkItems.x,
                                                      kernel.numWorkItems.y,
                                                      kernel.numWorkItems.z,
                                                      kernel.workGroupSize.x,
                                                      kernel.workGroupSize.y,
                                                      kernel.workGroupSize.z,
                                                      kernel.sharedMemBytes, // sharedMem
                                                      stream, // stream
                                                      nullptr,
                                                      (void**)&hipLaunchParams,
                                                      nullptr, // event
                                                      nullptr // event
                                                      ));
            if(stopEvent != nullptr)
                HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream));
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels)
        {
            for(auto const& k : kernels)
            {
                HIP_CHECK_RETURN(launchKernel(k));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels,
                                                  hipStream_t                          stream,
                                                  hipEvent_t                           startEvent,
                                                  hipEvent_t                           stopEvent)
        {
            auto first = kernels.begin();
            auto last  = kernels.end() - 1;

            for(auto iter = kernels.begin(); iter != kernels.end(); iter++)
            {
                hipEvent_t kStart = nullptr;
                hipEvent_t kStop  = nullptr;

                if(iter == first)
                    kStart = startEvent;
                if(iter == last)
                    kStop = stopEvent;

                HIP_CHECK_RETURN(launchKernel(*iter, stream, kStart, kStop));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels,
                                                  hipStream_t                          stream,
                                                  std::vector<hipEvent_t> const&       startEvents,
                                                  std::vector<hipEvent_t> const&       stopEvents)
        {
            if(kernels.size() != startEvents.size() || kernels.size() != stopEvents.size())
                throw std::runtime_error(concatenate("Must have an equal number of kernels (",
                                                     kernels.size(),
                                                     "), start events (",
                                                     startEvents.size(),
                                                     "), and stop events. (",
                                                     stopEvents.size(),
                                                     ")"));

            for(size_t i = 0; i < kernels.size(); i++)
            {
                HIP_CHECK_RETURN(launchKernel(kernels[i], stream, startEvents[i], stopEvents[i]));
            }
            return hipSuccess;
        }

        std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter)
        {
            stream << "hip::SolutionAdapter";

            if(adapter.m_debug)
            {
                stream << "[" << std::endl;
                for(auto const& name : adapter.m_loadedModuleNames)
                    stream << name << std::endl;

                stream << "]";
            }

            stream << " (" << adapter.name() << ", " << adapter.m_modules.size()
                   << " total modules)" << std::endl;

            return stream;
        }

        std::ostream& operator<<(std::ostream& stream, std::shared_ptr<SolutionAdapter> const& ptr)
        {
            if(ptr)
            {
                return stream << "*" << *ptr;
            }
            else
            {
                return stream << "(nullptr)";
            }
        }
    } // namespace hip
} // namespace Tensile
<commit_msg>Return hipErrorNotFound if no modules loaded (#1496)<commit_after>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright 2019-2022 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#include <hip/hip_ext.h>
#include <hip/hip_runtime.h>

#include <cstddef>

#include <Tensile/Debug.hpp>
#include <Tensile/EmbeddedData.hpp>
#include <Tensile/hip/HipSolutionAdapter.hpp>
#include <Tensile/hip/HipUtils.hpp>

namespace Tensile
{
    namespace hip
    {
        SolutionAdapter::SolutionAdapter()
            : m_debug(Debug::Instance().printKernelArguments())
            , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch())
        {
        }

        SolutionAdapter::SolutionAdapter(bool debug)
            : m_debug(debug)
        {
            m_debug = debug || Debug::Instance().printKernelArguments();
        }

        SolutionAdapter::SolutionAdapter(bool debug, std::string const& name)
            : m_debug(debug)
            , m_name(name)
        {
            m_debug = debug || Debug::Instance().printKernelArguments();
        }

        SolutionAdapter::~SolutionAdapter()
        {
            for(auto module : m_modules)
                hipModuleUnload(module);
        }

        hipError_t SolutionAdapter::loadCodeObjectFile(std::string const& path)
        {
            hipModule_t module;

            HIP_CHECK_RETURN(hipModuleLoad(&module, path.c_str()));

            if(m_debug)
                std::cout << "loaded code object" << path << std::endl;

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.push_back(module);
                m_loadedModuleNames.push_back(concatenate("File ", path));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::loadCodeObjectBytes(std::vector<uint8_t> const& bytes)
        {
            return loadCodeObject(bytes.data());
        }

        hipError_t SolutionAdapter::loadCodeObject(const void* image)
        {
            hipModule_t module;

            HIP_CHECK_RETURN(hipModuleLoadData(&module, image));

            if(m_debug)
                std::cout << "loaded code object data." << std::endl;

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.push_back(module);
                m_loadedModuleNames.push_back("Module from bytes");
            }
            return hipSuccess;
        }

        void SolutionAdapter::loadEmbeddedCodeObjects()
        {
            loadEmbeddedCodeObjects("");
        }

        void SolutionAdapter::loadEmbeddedCodeObjects(std::string const& key)
        {
            auto const& embeddedData = EmbeddedData<Tensile::SolutionAdapter>::Get(key);

            if(embeddedData.size() == 0)
            {
                if(m_debug || Debug::Instance().printCodeObjectInfo())
                {
                    std::cerr << "Found no embedded code objects";
                    if(key != "")
                        std::cerr << " with the key " << key;

                    std::cerr << "." << std::endl;
                }
                return;
            }

            std::vector<hipModule_t> newModules;
            newModules.reserve(embeddedData.size());

            for(size_t i = 0; i < embeddedData.size(); i++)
            {
                hipModule_t nextModule;
                try
                {
                    auto error = hipModuleLoadData(&nextModule, embeddedData[i].data());

                    if(error == hipErrorUnknown || error == hipErrorSharedObjectInitFailed)
                        continue;
                    newModules.push_back(nextModule);
                    HIP_CHECK_EXC(error);

                    if(m_debug)
                        std::cout << "Loaded code object for key " << key << std::endl;
                }
                catch(std::runtime_error const& exc)
                {
                    std::cout << exc.what() << std::endl;
                }
            }

            {
                std::lock_guard<std::mutex> guard(m_access);
                m_modules.insert(m_modules.end(), newModules.begin(), newModules.end());
                m_loadedModuleNames.push_back(
                    concatenate("Embedded code object ", key, " (", newModules.size(), ")"));
            }
        }

        hipError_t SolutionAdapter::initKernel(std::string const& name)
        {
            hipFunction_t function;
            return getKernel(function, name);
        }

        hipError_t SolutionAdapter::getKernel(hipFunction_t& rv, std::string const& name)
        {
            std::unique_lock<std::mutex> guard(m_access);
            hipError_t                   err = hipErrorNotFound;

            auto it = m_kernels.find(name);
            if(it != m_kernels.end())
            {
                rv = it->second;
                return hipSuccess;
            }

            for(auto module : m_modules)
            {
                err = hipModuleGetFunction(&rv, module, name.c_str());

                if(err == hipSuccess)
                {
                    m_kernels[name] = rv;
                    return err;
                }
                else if(err != hipErrorNotFound)
                {
                    return err;
                }
            }

            return err;
        }

        hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel)
        {
            return launchKernel(kernel, nullptr, nullptr, nullptr);
        }

        hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel,
                                                 hipStream_t             stream,
                                                 hipEvent_t              startEvent,
                                                 hipEvent_t              stopEvent)
        {
            if(m_debug)
            {
                std::cout << "Kernel " << kernel.kernelName << std::endl;
                std::cout << " l" << kernel.workGroupSize << " x g" << kernel.numWorkGroups << " = "
                          << kernel.numWorkItems << std::endl;
                std::cout << kernel.args;
            }
            if(m_debugSkipLaunch)
            {
                std::cout << "DEBUG: Skip kernel execution" << std::endl;
                if(startEvent != nullptr)
                    HIP_CHECK_RETURN(hipEventRecord(startEvent, stream));
                if(stopEvent != nullptr)
                    HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream));
                return hipSuccess;
            }

            hipFunction_t function;
            HIP_CHECK_RETURN(getKernel(function, kernel.kernelName));

            void*  kernelArgs = const_cast<void*>(kernel.args.data());
            size_t argsSize   = kernel.args.size();

            void* hipLaunchParams[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER,
                                       kernelArgs,
                                       HIP_LAUNCH_PARAM_BUFFER_SIZE,
                                       &argsSize,
                                       HIP_LAUNCH_PARAM_END};

            if(startEvent != nullptr)
                HIP_CHECK_RETURN(hipEventRecord(startEvent, stream));
            HIP_CHECK_RETURN(hipExtModuleLaunchKernel(function,
                                                      kernel.numWorkItems.x,
                                                      kernel.numWorkItems.y,
                                                      kernel.numWorkItems.z,
                                                      kernel.workGroupSize.x,
                                                      kernel.workGroupSize.y,
                                                      kernel.workGroupSize.z,
                                                      kernel.sharedMemBytes, // sharedMem
                                                      stream, // stream
                                                      nullptr,
                                                      (void**)&hipLaunchParams,
                                                      nullptr, // event
                                                      nullptr // event
                                                      ));
            if(stopEvent != nullptr)
                HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream));
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels)
        {
            for(auto const& k : kernels)
            {
                HIP_CHECK_RETURN(launchKernel(k));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels,
                                                  hipStream_t                          stream,
                                                  hipEvent_t                           startEvent,
                                                  hipEvent_t                           stopEvent)
        {
            auto first = kernels.begin();
            auto last  = kernels.end() - 1;

            for(auto iter = kernels.begin(); iter != kernels.end(); iter++)
            {
                hipEvent_t kStart = nullptr;
                hipEvent_t kStop  = nullptr;

                if(iter == first)
                    kStart = startEvent;
                if(iter == last)
                    kStop = stopEvent;

                HIP_CHECK_RETURN(launchKernel(*iter, stream, kStart, kStop));
            }
            return hipSuccess;
        }

        hipError_t SolutionAdapter::launchKernels(std::vector<KernelInvocation> const& kernels,
                                                  hipStream_t                          stream,
                                                  std::vector<hipEvent_t> const&       startEvents,
                                                  std::vector<hipEvent_t> const&       stopEvents)
        {
            if(kernels.size() != startEvents.size() || kernels.size() != stopEvents.size())
                throw std::runtime_error(concatenate("Must have an equal number of kernels (",
                                                     kernels.size(),
                                                     "), start events (",
                                                     startEvents.size(),
                                                     "), and stop events. (",
                                                     stopEvents.size(),
                                                     ")"));

            for(size_t i = 0; i < kernels.size(); i++)
            {
                HIP_CHECK_RETURN(launchKernel(kernels[i], stream, startEvents[i], stopEvents[i]));
            }
            return hipSuccess;
        }

        std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter)
        {
            stream << "hip::SolutionAdapter";

            if(adapter.m_debug)
            {
                stream << "[" << std::endl;
                for(auto const& name : adapter.m_loadedModuleNames)
                    stream << name << std::endl;

                stream << "]";
            }

            stream << " (" << adapter.name() << ", " << adapter.m_modules.size()
                   << " total modules)" << std::endl;

            return stream;
        }

        std::ostream& operator<<(std::ostream& stream, std::shared_ptr<SolutionAdapter> const& ptr)
        {
            if(ptr)
            {
                return stream << "*" << *ptr;
            }
            else
            {
                return stream << "(nullptr)";
            }
        }
    } // namespace hip
} // namespace Tensile
<|endoftext|>


<commit_before>/*  Sirikata Transfer -- Content Transfer mediation
 *  TransferMediator.hpp
 *
 *  Copyright (c) 2010, Jeff Terrace
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of Sirikata nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*  Created on: Jan 15, 2010 */

#ifndef SIRIKATA_TransferMediator_HPP__
#define SIRIKATA_TransferMediator_HPP__

#include <sirikata/core/transfer/TransferPool.hpp>
#include <map>
#include <vector>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/lambda/lambda.hpp>
#include <sirikata/core/network/IOService.hpp>
#include <sirikata/core/network/Asio.hpp>
#include <sirikata/core/util/Thread.hpp>
#include <sirikata/core/util/Singleton.hpp>

namespace Sirikata {
namespace Transfer {

using namespace boost::multi_index;

/*
 * Mediates requests for name lookups and chunk downloads
 */
class SIRIKATA_EXPORT TransferMediator
    : public AutoSingleton<TransferMediator> {

    /*
     * Used to aggregate requests from different clients. If multiple clients request
     * the same file, this object keeps track of the original separate requests so that
     * each client's callback can be called when the request finishes. It also uses the
     * PriorityAggregation interface to aggregate multiple client priorities into a single
     * aggregated priority.
     */
	class AggregateRequest {
	public:
	    //Stores the aggregated priority
		TransferRequest::PriorityType mPriority;

	private:
		//Maps each client's string ID to the original TransferRequest object
		std::map<std::string, std::tr1::shared_ptr<TransferRequest> > mTransferReqs;

		//Aggregated request unique identifier
		const std::string mIdentifier;

		//Updates the aggregated priority from each client's priority when needed
		void updateAggregatePriority();

	public:
		//Returns a map from each client ID to their original TransferRequest object
		const std::map<std::string, std::tr1::shared_ptr<TransferRequest> > & getTransferRequests() const;

		//Since there is overlap between requests here, this returns a single TransferRequest from the list of clients
		std::tr1::shared_ptr<TransferRequest> getSingleRequest();

		//Adds an additional client's request
		void setClientPriority(std::tr1::shared_ptr<TransferRequest> req);

        //Removes a client request from this aggregate request
        void removeClient(std::string clientID);

		//Returns a unique identifier for this aggregated request
		const std::string& getIdentifier() const;

		//Returns the aggregated priority value
		TransferRequest::PriorityType getPriority() const;

		//Pass in the first client's request
		AggregateRequest(std::tr1::shared_ptr<TransferRequest> req);
	};

	//lock this to access mAggregatedList
	boost::mutex mAggMutex;

	//tags used to index AggregateList (see boost::multi_index)
	struct tagID{};
	struct tagPriority{};

	/*
	 * This multi_index_container allows the efficient retrieval of an AggregateRequest
	 * either by its identifier or sorted by its priority
	 */
	typedef multi_index_container<
		std::tr1::shared_ptr<AggregateRequest>,
		indexed_by<
			hashed_unique<tag<tagID>, const_mem_fun<AggregateRequest,const std::string &,&AggregateRequest::getIdentifier> >,
			ordered_non_unique<tag<tagPriority>,
			member<AggregateRequest,TransferRequest::PriorityType,&AggregateRequest::mPriority>,
			std::greater<TransferRequest::PriorityType> >
		>
	> AggregateList;
	AggregateList mAggregateList;

	//access iterators for AggregateList for convenience (see boost::multi_index)
	typedef AggregateList::index<tagID>::type AggregateListByID;
	typedef AggregateList::index<tagPriority>::type AggregateListByPriority;

	/*
	 * Used to process the queue of requests coming from a single client.
	 * There is one PoolWorker for each client, and this makes it so that the client's
	 * thread doesn't have to block when inserting a request. Instead, the PoolWorkers
	 * block against each other when inserting the request into the aggregated list.
	 */
	class PoolWorker {
	private:
	    //The TransferPool associated with this worker
		std::tr1::shared_ptr<TransferPool> mTransferPool;

		//The worker's thread
		Thread * mWorkerThread;

		//Set to true when worker should shut down
		bool mCleanup;

		//Runs the worker thread
	    void run();

	public:
		//Initialize with the associated TransferPool
		PoolWorker(std::tr1::shared_ptr<TransferPool> transferPool);

		//Returns the TransferPool this was initialized with
		std::tr1::shared_ptr<TransferPool> getTransferPool() const;

		//Returns the worker's thread
		Thread * getThread() const;

		//Signal to shut down
		void cleanup();
	};

    friend class PoolWorker;
    //Helper for compatibility with compilers where TransferPool declaring
    //TransferMediator as a friend does not give PoolWorker access
    static inline std::tr1::shared_ptr<TransferRequest> getRequest(std::tr1::shared_ptr<TransferPool> pool) {
        return pool->getRequest();
    }

	//Maps a client ID string to the PoolWorker class
	typedef std::map<std::string, std::tr1::shared_ptr<PoolWorker> > PoolType;
	//Stores the list of pools
	PoolType mPools;
	//lock this to access mPools
	boost::shared_mutex mPoolMutex;

	//Set to true to signal shutdown
	bool mCleanup;
	//Number of outstanding requests
	uint32 mNumOutstanding;

	//TransferMediator's worker thread
	Thread* mThread;

    // Algorithm used to aggregate priorities of requests
    PriorityAggregationAlgorithm* mAggregationAlgorithm;

    //Main thread that handles the input pools
    void mediatorThread();

    //Callback for when an executed request finishes
    void execute_finished(std::tr1::shared_ptr<TransferRequest> req, std::string id);

    //Check our internal queue to see what request to process next
    void checkQueue();

    void registerPool(TransferPoolPtr pool);
public:
    static TransferMediator& getSingleton();
    static void destroy();

    TransferMediator();
    ~TransferMediator();

    /** Used to register a client that has a pool of requests it needs
     *  serviced by the transfer mediator
     *
     *  @param clientID Should be a string that uniquely identifies the client
     */
    template<typename PoolType>
    std::tr1::shared_ptr<PoolType> registerClient(const std::string& clientID) {
        std::tr1::shared_ptr<PoolType> ret(new PoolType(clientID));
        registerPool(ret);
        return ret;
    }

    //Call when system should be shut down
    void cleanup();
};

}
}

#endif /* SIRIKATA_TransferMediator_HPP__ */
<commit_msg>Fix accidental conflicting use of 'PoolType'.<commit_after>/*  Sirikata Transfer -- Content Transfer mediation
 *  TransferMediator.hpp
 *
 *  Copyright (c) 2010, Jeff Terrace
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of Sirikata nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*  Created on: Jan 15, 2010 */

#ifndef SIRIKATA_TransferMediator_HPP__
#define SIRIKATA_TransferMediator_HPP__

#include <sirikata/core/transfer/TransferPool.hpp>
#include <map>
#include <vector>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/lambda/lambda.hpp>
#include <sirikata/core/network/IOService.hpp>
#include <sirikata/core/network/Asio.hpp>
#include <sirikata/core/util/Thread.hpp>
#include <sirikata/core/util/Singleton.hpp>

namespace Sirikata {
namespace Transfer {

using namespace boost::multi_index;

/*
 * Mediates requests for name lookups and chunk downloads
 */
class SIRIKATA_EXPORT TransferMediator
    : public AutoSingleton<TransferMediator> {

    /*
     * Used to aggregate requests from different clients. If multiple clients request
     * the same file, this object keeps track of the original separate requests so that
     * each client's callback can be called when the request finishes. It also uses the
     * PriorityAggregation interface to aggregate multiple client priorities into a single
     * aggregated priority.
     */
	class AggregateRequest {
	public:
	    //Stores the aggregated priority
		TransferRequest::PriorityType mPriority;

	private:
		//Maps each client's string ID to the original TransferRequest object
		std::map<std::string, std::tr1::shared_ptr<TransferRequest> > mTransferReqs;

		//Aggregated request unique identifier
		const std::string mIdentifier;

		//Updates the aggregated priority from each client's priority when needed
		void updateAggregatePriority();

	public:
		//Returns a map from each client ID to their original TransferRequest object
		const std::map<std::string, std::tr1::shared_ptr<TransferRequest> > & getTransferRequests() const;

		//Since there is overlap between requests here, this returns a single TransferRequest from the list of clients
		std::tr1::shared_ptr<TransferRequest> getSingleRequest();

		//Adds an additional client's request
		void setClientPriority(std::tr1::shared_ptr<TransferRequest> req);

        //Removes a client request from this aggregate request
        void removeClient(std::string clientID);

		//Returns a unique identifier for this aggregated request
		const std::string& getIdentifier() const;

		//Returns the aggregated priority value
		TransferRequest::PriorityType getPriority() const;

		//Pass in the first client's request
		AggregateRequest(std::tr1::shared_ptr<TransferRequest> req);
	};

	//lock this to access mAggregatedList
	boost::mutex mAggMutex;

	//tags used to index AggregateList (see boost::multi_index)
	struct tagID{};
	struct tagPriority{};

	/*
	 * This multi_index_container allows the efficient retrieval of an AggregateRequest
	 * either by its identifier or sorted by its priority
	 */
	typedef multi_index_container<
		std::tr1::shared_ptr<AggregateRequest>,
		indexed_by<
			hashed_unique<tag<tagID>, const_mem_fun<AggregateRequest,const std::string &,&AggregateRequest::getIdentifier> >,
			ordered_non_unique<tag<tagPriority>,
			member<AggregateRequest,TransferRequest::PriorityType,&AggregateRequest::mPriority>,
			std::greater<TransferRequest::PriorityType> >
		>
	> AggregateList;
	AggregateList mAggregateList;

	//access iterators for AggregateList for convenience (see boost::multi_index)
	typedef AggregateList::index<tagID>::type AggregateListByID;
	typedef AggregateList::index<tagPriority>::type AggregateListByPriority;

	/*
	 * Used to process the queue of requests coming from a single client.
	 * There is one PoolWorker for each client, and this makes it so that the client's
	 * thread doesn't have to block when inserting a request. Instead, the PoolWorkers
	 * block against each other when inserting the request into the aggregated list.
	 */
	class PoolWorker {
	private:
	    //The TransferPool associated with this worker
		std::tr1::shared_ptr<TransferPool> mTransferPool;

		//The worker's thread
		Thread * mWorkerThread;

		//Set to true when worker should shut down
		bool mCleanup;

		//Runs the worker thread
	    void run();

	public:
		//Initialize with the associated TransferPool
		PoolWorker(std::tr1::shared_ptr<TransferPool> transferPool);

		//Returns the TransferPool this was initialized with
		std::tr1::shared_ptr<TransferPool> getTransferPool() const;

		//Returns the worker's thread
		Thread * getThread() const;

		//Signal to shut down
		void cleanup();
	};

    friend class PoolWorker;
    //Helper for compatibility with compilers where TransferPool declaring
    //TransferMediator as a friend does not give PoolWorker access
    static inline std::tr1::shared_ptr<TransferRequest> getRequest(std::tr1::shared_ptr<TransferPool> pool) {
        return pool->getRequest();
    }

	//Maps a client ID string to the PoolWorker class
	typedef std::map<std::string, std::tr1::shared_ptr<PoolWorker> > PoolType;
	//Stores the list of pools
	PoolType mPools;
	//lock this to access mPools
	boost::shared_mutex mPoolMutex;

	//Set to true to signal shutdown
	bool mCleanup;
	//Number of outstanding requests
	uint32 mNumOutstanding;

	//TransferMediator's worker thread
	Thread* mThread;

    // Algorithm used to aggregate priorities of requests
    PriorityAggregationAlgorithm* mAggregationAlgorithm;

    //Main thread that handles the input pools
    void mediatorThread();

    //Callback for when an executed request finishes
    void execute_finished(std::tr1::shared_ptr<TransferRequest> req, std::string id);

    //Check our internal queue to see what request to process next
    void checkQueue();

    void registerPool(TransferPoolPtr pool);
public:
    static TransferMediator& getSingleton();
    static void destroy();

    TransferMediator();
    ~TransferMediator();

    /** Used to register a client that has a pool of requests it needs
     *  serviced by the transfer mediator
     *
     *  @param clientID Should be a string that uniquely identifies the client
     */
    template<typename TPoolType>
    std::tr1::shared_ptr<TPoolType> registerClient(const std::string& clientID) {
        std::tr1::shared_ptr<TPoolType> ret(new TPoolType(clientID));
        registerPool(ret);
        return ret;
    }

    //Call when system should be shut down
    void cleanup();
};

}
}

#endif /* SIRIKATA_TransferMediator_HPP__ */
<|endoftext|>


<commit_before>#pragma once
#include <bts/blockchain/types.hpp>
#include <map>

const static std::map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {      1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    { 100000, bts::blockchain::block_id_type( "96f98d49722848a6a47ad04aece8b9f93c9e9c23" ) },
    { 200000, bts::blockchain::block_id_type( "222ddc49db592103c51ad22cdc4140185ef564d9" ) },
    { 300000, bts::blockchain::block_id_type( "0d1d0b8f7f1f4590f8e083edc03869383ed74e3e" ) },
    { 400000, bts::blockchain::block_id_type( "053d398b6597d5c61365afd100d87b824bf49f65" ) },
    { 500000, bts::blockchain::block_id_type( "f02910a7115fb826984ce3a432cb371d5d7a99b8" ) },
    { 600000, bts::blockchain::block_id_type( "f278db8722235343c9db9f077fe67c54a5f25f3b" ) },
    { 700000, bts::blockchain::block_id_type( "f120ff9b159661b9ac084a0a69c58dcbd79cbb49" ) },
    { 800000, bts::blockchain::block_id_type( "0e94ad17c598e42e5ec2f522a05bf4df6a1778da" ) },
    { 804000, bts::blockchain::block_id_type( "796a88a21e443c0655479d879e8ac4acd875d12d" ) }
};
<commit_msg>Update checkpoint<commit_after>#pragma once
#include <bts/blockchain/types.hpp>
#include <map>

const static std::map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {      1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    { 100000, bts::blockchain::block_id_type( "96f98d49722848a6a47ad04aece8b9f93c9e9c23" ) },
    { 200000, bts::blockchain::block_id_type( "222ddc49db592103c51ad22cdc4140185ef564d9" ) },
    { 300000, bts::blockchain::block_id_type( "0d1d0b8f7f1f4590f8e083edc03869383ed74e3e" ) },
    { 400000, bts::blockchain::block_id_type( "053d398b6597d5c61365afd100d87b824bf49f65" ) },
    { 500000, bts::blockchain::block_id_type( "f02910a7115fb826984ce3a432cb371d5d7a99b8" ) },
    { 600000, bts::blockchain::block_id_type( "f278db8722235343c9db9f077fe67c54a5f25f3b" ) },
    { 700000, bts::blockchain::block_id_type( "f120ff9b159661b9ac084a0a69c58dcbd79cbb49" ) },
    { 800000, bts::blockchain::block_id_type( "0e94ad17c598e42e5ec2f522a05bf4df6a1778da" ) },
    { 821000, bts::blockchain::block_id_type( "74826a14c0281ac7ff6bdc66d25a522287e565d7" ) }
};
<|endoftext|>


<commit_before>#pragma once
#include <bts/blockchain/types.hpp>
#include <map>

const static std::map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {      1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    { 100000, bts::blockchain::block_id_type( "96f98d49722848a6a47ad04aece8b9f93c9e9c23" ) },
    { 200000, bts::blockchain::block_id_type( "222ddc49db592103c51ad22cdc4140185ef564d9" ) },
    { 300000, bts::blockchain::block_id_type( "0d1d0b8f7f1f4590f8e083edc03869383ed74e3e" ) },
    { 400000, bts::blockchain::block_id_type( "053d398b6597d5c61365afd100d87b824bf49f65" ) },
    { 500000, bts::blockchain::block_id_type( "f02910a7115fb826984ce3a432cb371d5d7a99b8" ) },
    { 520000, bts::blockchain::block_id_type( "46c1ba3b7ef95586876cf609a127696ba54a4e3d" ) }
};
<commit_msg>Update checkpoint<commit_after>#pragma once
#include <bts/blockchain/types.hpp>
#include <map>

const static std::map<uint32_t, bts::blockchain::block_id_type> CHECKPOINT_BLOCKS
{
    {      1, bts::blockchain::block_id_type( "8abcfb93c52f999e3ef5288c4f837f4f15af5521" ) },
    { 100000, bts::blockchain::block_id_type( "96f98d49722848a6a47ad04aece8b9f93c9e9c23" ) },
    { 200000, bts::blockchain::block_id_type( "222ddc49db592103c51ad22cdc4140185ef564d9" ) },
    { 300000, bts::blockchain::block_id_type( "0d1d0b8f7f1f4590f8e083edc03869383ed74e3e" ) },
    { 400000, bts::blockchain::block_id_type( "053d398b6597d5c61365afd100d87b824bf49f65" ) },
    { 500000, bts::blockchain::block_id_type( "f02910a7115fb826984ce3a432cb371d5d7a99b8" ) },
    { 539000, bts::blockchain::block_id_type( "7d465b1465d8e4bc25c71cf718927eec5ce51f55" ) }
};
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: cfgmerge.hxx,v $
 *
 *  $Revision: 1.5 $
 *
 *  last change: $Author: obo $ $Date: 2006-03-29 13:24:57 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _CFG_MERGE_HXX
#define _CFG_MERGE_HXX

#include <tools/string.hxx>
#include <tools/list.hxx>
#include <hash_map>

/*struct equalByteString{
        bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
            return rKey1.CompareTo( rKey2 )==COMPARE_EQUAL;
    }
};

struct hashByteString{
    size_t operator()( const ByteString& rName ) const{
                std::hash< const char* > myHash;
                return myHash( rName.GetBuffer() );
    }
};
*/
typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
                                ByteStringHashMap;

//=============================================

//
// class CfgStackData
//

class CfgStackData
{
friend class CfgParser;
friend class CfgExport;
friend class CfgMerge;
private:
    ByteString sTagType;
    ByteString sIdentifier;

    ByteString sResTyp;

    ByteString sTextTag;
    ByteString sEndTextTag;

    //ByteString sText[ LANGUAGES ];
    ByteStringHashMap sText;
public:
    CfgStackData( const ByteString &rTag, const ByteString &rId )
            : sTagType( rTag ), sIdentifier( rId ) {};

    ByteString &GetTagType() { return sTagType; }
    ByteString &GetIdentifier() { return sIdentifier; }

    void FillInFallbacks();
};

//
// class CfgStack
//

DECLARE_LIST( CfgStackList, CfgStackData * );

class CfgStack : public CfgStackList
{
public:
    CfgStack() : CfgStackList( 10, 10 ) {}
    ~CfgStack();

    ULONG Push( CfgStackData *pStackData ) { Insert( pStackData, LIST_APPEND ); return Count() - 1; }
    CfgStackData *Push( const ByteString &rTag, const ByteString &rId )
        { CfgStackData *pD = new CfgStackData( rTag, rId ); Insert( pD, LIST_APPEND ); return pD; }
    CfgStackData *Pop() { return Remove( Count() - 1 ); }

    CfgStackData *GetStackData( ULONG nPos = LIST_APPEND );

    ByteString GetAccessPath( ULONG nPos = LIST_APPEND );
};

//
// class CfgParser
//

class CfgParser
{
protected:
    ByteString sCurrentResTyp;
    ByteString sCurrentIsoLang;
    ByteString sCurrentText;

    ByteString sLastWhitespace;

    CfgStack aStack;
    CfgStackData *pStackData;

    BOOL bLocalize;

    virtual void WorkOnText(
        ByteString &rText,
        //USHORT nLangIndex,
        ByteString nLangIndex,
        const ByteString &rResTyp );

    virtual void WorkOnRessourceEnd();

    virtual void Output( const ByteString& rOutput );

    void Error( const ByteString &rError );

private:
    int ExecuteAnalyzedToken( int nToken, char *pToken );
    std::vector<ByteString> aLanguages;
    void AddText(
        ByteString &rText,
        const ByteString &rIsoLang,
        const ByteString &rResTyp );

BOOL IsTokenClosed( const ByteString &rToken );

public:
    CfgParser();
    virtual ~CfgParser();

    int Execute( int nToken, char * pToken );
};

//
// class CfgOutputParser
//

class CfgOutputParser : public CfgParser
{
protected:
    SvFileStream *pOutputStream;
public:
    CfgOutputParser ( const ByteString &rOutputFile );
    virtual ~CfgOutputParser();
};

//
// class CfgExport
//

class CfgExport : public CfgOutputParser
{
private:
    ByteString sPrj;
    ByteString sPath;
    std::vector<ByteString> aLanguages;
protected:
    virtual void WorkOnText(
        ByteString &rText,
        USHORT nLangIndex,
        const ByteString &rResTyp );

    virtual void WorkOnRessourceEnd();

public:
    CfgExport(
        const ByteString &rOutputFile,
        const ByteString &rProject,
        const ByteString &rFilePath
    );
    virtual ~CfgExport();
};

//
// class CfgMerge
//

class CfgMerge : public CfgOutputParser
{
private:
    MergeDataFile *pMergeDataFile;
    std::vector<ByteString> aLanguages;
    ResData *pResData;
    ByteString sFilename;

    BOOL bGerman;
    BOOL bEnglish;

protected:
    virtual void WorkOnText(
        ByteString &rText,
        //USHORT nLangIndex,
        ByteString nLangIndex,
        const ByteString &rResTyp );

    virtual void WorkOnRessourceEnd();

    virtual void Output( const ByteString& rOutput );
public:
    CfgMerge(
        const ByteString &rMergeSource,
        const ByteString &rOutputFile,
        BOOL bErrorLog , ByteString &rFilename
    );
    virtual ~CfgMerge();
};

#endif
<commit_msg>INTEGRATION: CWS warnings01 (1.4.16); FILE MERGED 2006/04/07 18:34:27 sb 1.4.16.3: RESYNC: (1.4-1.5); FILE MERGED 2005/11/18 18:10:45 ihi 1.4.16.2: #i57362# Removed warnings 2005/11/07 12:47:05 ihi 1.4.16.1: #i57362# Remove warnings<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: cfgmerge.hxx,v $
 *
 *  $Revision: 1.6 $
 *
 *  last change: $Author: hr $ $Date: 2006-06-19 17:18:44 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _CFG_MERGE_HXX
#define _CFG_MERGE_HXX

#include <tools/string.hxx>
#include <tools/list.hxx>
#include <hash_map>

typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
                                ByteStringHashMap;

//
// class CfgStackData
//

class CfgStackData
{
friend class CfgParser;
friend class CfgExport;
friend class CfgMerge;
private:
    ByteString sTagType;
    ByteString sIdentifier;

    ByteString sResTyp;

    ByteString sTextTag;
    ByteString sEndTextTag;

    //ByteString sText[ LANGUAGES ];
    ByteStringHashMap sText;
public:
    CfgStackData( const ByteString &rTag, const ByteString &rId )
            : sTagType( rTag ), sIdentifier( rId ) {};

    ByteString &GetTagType() { return sTagType; }
    ByteString &GetIdentifier() { return sIdentifier; }

    //void FillInFallbacks();
};

//
// class CfgStack
//

DECLARE_LIST( CfgStackList, CfgStackData * )

class CfgStack : public CfgStackList
{
public:
    CfgStack() : CfgStackList( 10, 10 ) {}
    ~CfgStack();

    ULONG Push( CfgStackData *pStackData );
    CfgStackData *Push( const ByteString &rTag, const ByteString &rId );
    CfgStackData *Pop() { return Remove( Count() - 1 ); }

    CfgStackData *GetStackData( ULONG nPos = LIST_APPEND );

    ByteString GetAccessPath( ULONG nPos = LIST_APPEND );
};

//
// class CfgParser
//

class CfgParser
{
protected:
    ByteString sCurrentResTyp;
    ByteString sCurrentIsoLang;
    ByteString sCurrentText;

    ByteString sLastWhitespace;

    CfgStack aStack;
    CfgStackData *pStackData;

    BOOL bLocalize;

    virtual void WorkOnText(
        ByteString &rText,
        const ByteString &nLangIndex
        //const ByteString &rResTyp
        )=0;

    virtual void WorkOnRessourceEnd()=0;

    virtual void Output( const ByteString& rOutput )=0;

    void Error( const ByteString &rError );

private:
    int ExecuteAnalyzedToken( int nToken, char *pToken );
    std::vector<ByteString> aLanguages;
    void AddText(
        ByteString &rText,
        const ByteString &rIsoLang,
        const ByteString &rResTyp );

BOOL IsTokenClosed( const ByteString &rToken );

public:
    CfgParser();
    virtual ~CfgParser();

    int Execute( int nToken, char * pToken );
};

//
// class CfgOutputParser
//

class CfgOutputParser : public CfgParser
{
protected:
    SvFileStream *pOutputStream;
public:
    CfgOutputParser ( const ByteString &rOutputFile );
    virtual ~CfgOutputParser();
};

//
// class CfgExport
//

class CfgExport : public CfgOutputParser
{
private:
    ByteString sPrj;
    ByteString sPath;
    std::vector<ByteString> aLanguages;
protected:
    void WorkOnText(
        ByteString &rText,
        const ByteString &rIsoLang
        //const ByteString &rResTyp
        );

    void WorkOnRessourceEnd();
    void Output( const ByteString& rOutput );
public:
    CfgExport(
        const ByteString &rOutputFile,
        const ByteString &rProject,
        const ByteString &rFilePath
    );
    ~CfgExport();
};

//
// class CfgMerge
//

class CfgMerge : public CfgOutputParser
{
private:
    MergeDataFile *pMergeDataFile;
    std::vector<ByteString> aLanguages;
    ResData *pResData;

    BOOL bGerman;
    ByteString sFilename;
    BOOL bEnglish;

protected:
    void WorkOnText(
        ByteString &rText,
        const ByteString &nLangIndex
        //const ByteString &rResTyp
        );

    void WorkOnRessourceEnd();

    void Output( const ByteString& rOutput );
public:
    CfgMerge(
        const ByteString &rMergeSource,
        const ByteString &rOutputFile,
        ByteString &rFilename
    );
    ~CfgMerge();
};

#endif
<|endoftext|>


<commit_before>#include "multiverso/server.h"

#include <algorithm>
#include <sstream>
#include <string>
#include <vector>

#include "multiverso/actor.h"
#include "multiverso/dashboard.h"
#include "multiverso/multiverso.h"
#include "multiverso/table_interface.h"
#include "multiverso/io/io.h"
#include "multiverso/util/configure.h"
#include "multiverso/util/mt_queue.h"
#include "multiverso/zoo.h"

namespace multiverso {

MV_DEFINE_bool(sync, false, "sync or async");
MV_DEFINE_int(backup_worker_ratio, 0, "ratio% of backup workers, set 20 means 20%");

Server::Server() : Actor(actor::kServer) {
  RegisterHandler(MsgType::Request_Get, std::bind(
    &Server::ProcessGet, this, std::placeholders::_1));
  RegisterHandler(MsgType::Request_Add, std::bind(
    &Server::ProcessAdd, this, std::placeholders::_1));
}

int Server::RegisterTable(ServerTable* server_table) {
  int id = static_cast<int>(store_.size());
  store_.push_back(server_table);
  return id;
}

void Server::ProcessGet(MessagePtr& msg) {
  MONITOR_BEGIN(SERVER_PROCESS_GET);
  if (msg->data().size() != 0) {
    MessagePtr reply(msg->CreateReplyMessage());
    int table_id = msg->table_id();
    CHECK(table_id >= 0 && table_id < store_.size());
    store_[table_id]->ProcessGet(msg->data(), &reply->data());
    SendTo(actor::kCommunicator, reply);
  }
  MONITOR_END(SERVER_PROCESS_GET);
}

void Server::ProcessAdd(MessagePtr& msg) {
  MONITOR_BEGIN(SERVER_PROCESS_ADD)
  if (msg->data().size() != 0) {
    MessagePtr reply(msg->CreateReplyMessage());
    int table_id = msg->table_id();
    CHECK(table_id >= 0 && table_id < store_.size());
    store_[table_id]->ProcessAdd(msg->data());
    SendTo(actor::kCommunicator, reply);
  }
  MONITOR_END(SERVER_PROCESS_ADD)
}

// The Sync Server implement logic to support Sync SGD training
// The implementation assumes all the workers will call same number
// of Add and/or Get requests
// The server promise all workers i-th Get will get the same parameters
// If worker k has add delta to server j times when its i-th Get
// then the server will return the parameter after all K
// workers finished their j-th update

class SyncServer : public Server {
public:
  SyncServer() : Server() {
    RegisterHandler(MsgType::Server_Finish_Train, std::bind(
      &SyncServer::ProcessFinishTrain, this, std::placeholders::_1));
    int num_worker = Zoo::Get()->num_workers();
    worker_get_clocks_.reset(new VectorClock(num_worker));
    worker_add_clocks_.reset(new VectorClock(num_worker));
  }

  // make some modification to suit to the sync server
  // please not use in other place, may different with the general vector clock
  class VectorClock {
  public:
    explicit VectorClock(int n) :
      local_clock_(n, 0), global_clock_(0), size_(0) {}

    static bool except_max_int_compare(int a, int b)
    {
      return (b == std::numeric_limits<int>::max() ? false : a < b);
    }

    // Return true when all clock reach a same number
    virtual bool Update(int i) {
      Log::Debug(DebugString().c_str());
      ++local_clock_[i];
      if (global_clock_ < *(std::min_element(std::begin(local_clock_),
        std::end(local_clock_)))) {
        ++global_clock_;
        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
          std::end(local_clock_), except_max_int_compare))) {
          return true;
        }
      }
      return false;
    }

    virtual bool FinishTrain(int i) {
      local_clock_[i] = std::numeric_limits<int>::max();
      if (global_clock_ < *(std::min_element(std::begin(local_clock_),
        std::end(local_clock_)))) {
        ++global_clock_;
        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
          std::end(local_clock_), except_max_int_compare))) {
          return true;
        }
      }
      return false;
    }

    std::string DebugString() {
      std::string os = "global ";
      os += std::to_string(global_clock_) + " local: ";
      for (auto i : local_clock_) os += std::to_string(i) + " ";
      return os;
    }

    int local_clock(int i) const { return local_clock_[i]; }
    int global_clock() const { return global_clock_; }

  protected:
    std::vector<int> local_clock_;
    int global_clock_;
    int size_;
  };
protected:
  void ProcessAdd(MessagePtr& msg) override {
    // 1. Before add: cache faster worker
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    if (worker_get_clocks_->local_clock(worker) >
        worker_get_clocks_->global_clock()) {
      msg_add_cache_.Push(msg);
      return;
    }
    // 2. Process Add
    Server::ProcessAdd(msg);
    // 3. After add: process cached process get if necessary
    if (worker_add_clocks_->Update(worker)) {
      CHECK(msg_add_cache_.Empty());
      while (!msg_get_cache_.Empty()) {
        MessagePtr get_msg;
        CHECK(msg_get_cache_.TryPop(get_msg));
        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
        Server::ProcessGet(get_msg);
        worker_get_clocks_->Update(get_worker);
      }
    }
  }

  void ProcessGet(MessagePtr& msg) override {
    // 1. Before get: cache faster worker
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    if (worker_add_clocks_->local_clock(worker) >
        worker_add_clocks_->global_clock()) {
      // Will wait for other worker finished Add
      msg_get_cache_.Push(msg);
      return;
    }
    // 2. Process Get
    Server::ProcessGet(msg);
    // 3. After get: process cached process add if necessary
    if (worker_get_clocks_->Update(worker)) {
      CHECK(msg_get_cache_.Empty());
      while (!msg_add_cache_.Empty()) {
        MessagePtr add_msg;
        CHECK(msg_add_cache_.TryPop(add_msg));
        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
        Server::ProcessAdd(add_msg);
        worker_add_clocks_->Update(add_worker);
      }
    }
  }

  void ProcessFinishTrain(MessagePtr& msg) {
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    Log::Debug("[ProcessFinishTrain] Server %d, worker %d has finished training.\n", Zoo::Get()->server_rank(), worker);

    if (worker_get_clocks_->FinishTrain(worker)) {
      CHECK(msg_get_cache_.Empty());
      while (!msg_add_cache_.Empty()) {
        MessagePtr add_msg;
        CHECK(msg_add_cache_.TryPop(add_msg));
        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
        Server::ProcessAdd(add_msg);
        worker_add_clocks_->Update(add_worker);
      }
    }

    if (worker_add_clocks_->FinishTrain(worker)) {
      CHECK(msg_add_cache_.Empty());
      while (!msg_get_cache_.Empty()) {
        MessagePtr get_msg;
        CHECK(msg_get_cache_.TryPop(get_msg));
        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
        Server::ProcessGet(get_msg);
        worker_get_clocks_->Update(get_worker);
      }
    }
  }

private:
  std::unique_ptr<VectorClock> worker_get_clocks_;
  std::unique_ptr<VectorClock> worker_add_clocks_;

  MtQueue<MessagePtr> msg_add_cache_;
  MtQueue<MessagePtr> msg_get_cache_;
};

// TODO[qiwye]: Backup logic need to fix

//class WithBackupSyncServer : public Server {
//public:
//  WithBackupSyncServer() : Server() {
//    num_worker_ = Zoo::Get()->num_workers();
//    double backup_ratio = (double)MV_CONFIG_backup_worker_ratio / 100;
//    num_sync_worker_ = num_worker_ -
//                       static_cast<int>(backup_ratio * num_worker_);
//    CHECK(num_sync_worker_ > 0 && num_sync_worker_ <= num_worker_);
//    if (num_sync_worker_ == num_worker_) {
//      Log::Info("No backup worker, using the sync mode\n");
//    }
//    Log::Info("Sync with backup worker start: num_sync_worker = %d,"
//              "num_total_worker = %d\n", num_sync_worker_, num_worker_);
//    worker_get_clocks_.reset(new VectorClock(num_worker_));
//    worker_add_clocks_.reset(new VectorClock(
//      num_worker_, num_worker_ - num_sync_worker_));
//  }
//
//  // make some modification to suit to the sync server
//  // please not use in other place, may different with the general vector clock
//  class VectorClock {
//  public:
//    VectorClock(int num_worker, int num_backup_worker = 0) :
//      local_clock_(num_worker, 0), global_clock_(0), num_worker_(num_worker),
//      num_sync_worker_(num_worker - num_backup_worker), progress_(0) {}
//
//    // Return true when global clock meet the sync condition
//    // sync: all worker reach the same clock
//    // backup-worker-sync: sync-workers reach the same clock
//    virtual bool Update(int i) {
//      if (local_clock_[i]++ == global_clock_) {
//        ++progress_;
//      }
//      if (progress_ >= num_sync_worker_) {
//        ++global_clock_;
//        progress_ = 0;
//        for (auto i : local_clock_) {
//          if (i > global_clock_) ++progress_;
//        }
//        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
//          std::end(local_clock_)))) {
//          return true;
//        }
//      }
//      return false;
//    }
//
//    std::string DebugString() {
//      std::string os = "global ";
//      os += std::to_string(global_clock_) + " local: ";
//      for (auto i : local_clock_) os += std::to_string(i) + " ";
//      return os;
//    }
//
//    int local_clock(int i) const { return local_clock_[i]; }
//    int global_clock() const { return global_clock_; }
//
//  protected:
//    std::vector<int> local_clock_;
//    int global_clock_;
//    int num_worker_;
//    int num_sync_worker_;
//    int progress_;
//  };
//protected:
//  void ProcessAdd(MessagePtr& msg) override {
//    // 1. Before add: cache faster worker
//    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
//    if (worker_get_clocks_->local_clock(worker) >
//      worker_get_clocks_->global_clock()) {
//      msg_add_cache_.Push(msg);
//      return;
//    }
//    // 2. Process Add
//    if (worker_add_clocks_->local_clock(worker) >=
//        worker_add_clocks_->global_clock()) {
//      Server::ProcessAdd(msg);
//    }
//    // 3. After add: process cached process get if necessary
//    if (worker_add_clocks_->Update(worker)) {
//      CHECK(msg_add_cache_.Empty());
//      while (!msg_get_cache_.Empty()) {
//        MessagePtr get_msg;
//        CHECK(msg_get_cache_.TryPop(get_msg));
//        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
//        Server::ProcessGet(get_msg);
//        worker_get_clocks_->Update(get_worker);
//      }
//    }
//  }
//
//  void ProcessGet(MessagePtr& msg) override {
//    // 1. Before get: cache faster worker
//    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
//    if (worker_add_clocks_->local_clock(worker) >
//      worker_add_clocks_->global_clock()) {
//      // Will wait for other worker finished Add
//      msg_get_cache_.Push(msg);
//      return;
//    }
//    // 2. Process Get
//    Server::ProcessGet(msg);
//    // 3. After get: process cached process add if necessary
//    if (worker_get_clocks_->Update(worker)) {
//      CHECK(msg_get_cache_.Empty());
//      while (!msg_add_cache_.Empty()) {
//        MessagePtr add_msg;
//        CHECK(msg_add_cache_.TryPop(add_msg));
//        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
//        if (worker_add_clocks_->local_clock(add_worker) >=
//          worker_add_clocks_->global_clock()) {
//          Server::ProcessAdd(msg);
//        };
//        worker_add_clocks_->Update(add_worker);
//      }
//    }
//  }
//
//  void ProcessFinish(MessagePtr& msg) {
//
//  }
//
//private:
//  std::unique_ptr<VectorClock> worker_get_clocks_;
//  std::unique_ptr<VectorClock> worker_add_clocks_;
//
//  MtQueue<MessagePtr> msg_add_cache_;
//  MtQueue<MessagePtr> msg_get_cache_;
//
//  // num_worker_ - num_sync_worker_ = num_backup_worker_
//  int num_sync_worker_;
//  int num_worker_;
//};

Server* Server::GetServer() {
  if (!MV_CONFIG_sync) {
    Log::Info("Create a async server\n");
    return new Server();
  }
  // if (MV_CONFIG_backup_worker_ratio > 0.0) {
  Log::Info("Create a sync server\n");
  return new SyncServer();
  // }
}

}  // namespace multiverso
<commit_msg>clean unneeded info<commit_after>#include "multiverso/server.h"

#include <algorithm>
#include <sstream>
#include <string>
#include <vector>

#include "multiverso/actor.h"
#include "multiverso/dashboard.h"
#include "multiverso/multiverso.h"
#include "multiverso/table_interface.h"
#include "multiverso/io/io.h"
#include "multiverso/util/configure.h"
#include "multiverso/util/mt_queue.h"
#include "multiverso/zoo.h"

namespace multiverso {

MV_DEFINE_bool(sync, false, "sync or async");
MV_DEFINE_int(backup_worker_ratio, 0, "ratio% of backup workers, set 20 means 20%");

Server::Server() : Actor(actor::kServer) {
  RegisterHandler(MsgType::Request_Get, std::bind(
    &Server::ProcessGet, this, std::placeholders::_1));
  RegisterHandler(MsgType::Request_Add, std::bind(
    &Server::ProcessAdd, this, std::placeholders::_1));
}

int Server::RegisterTable(ServerTable* server_table) {
  int id = static_cast<int>(store_.size());
  store_.push_back(server_table);
  return id;
}

void Server::ProcessGet(MessagePtr& msg) {
  MONITOR_BEGIN(SERVER_PROCESS_GET);
  if (msg->data().size() != 0) {
    MessagePtr reply(msg->CreateReplyMessage());
    int table_id = msg->table_id();
    CHECK(table_id >= 0 && table_id < store_.size());
    store_[table_id]->ProcessGet(msg->data(), &reply->data());
    SendTo(actor::kCommunicator, reply);
  }
  MONITOR_END(SERVER_PROCESS_GET);
}

void Server::ProcessAdd(MessagePtr& msg) {
  MONITOR_BEGIN(SERVER_PROCESS_ADD)
  if (msg->data().size() != 0) {
    MessagePtr reply(msg->CreateReplyMessage());
    int table_id = msg->table_id();
    CHECK(table_id >= 0 && table_id < store_.size());
    store_[table_id]->ProcessAdd(msg->data());
    SendTo(actor::kCommunicator, reply);
  }
  MONITOR_END(SERVER_PROCESS_ADD)
}

// The Sync Server implement logic to support Sync SGD training
// The implementation assumes all the workers will call same number
// of Add and/or Get requests
// The server promise all workers i-th Get will get the same parameters
// If worker k has add delta to server j times when its i-th Get
// then the server will return the parameter after all K
// workers finished their j-th update

class SyncServer : public Server {
public:
  SyncServer() : Server() {
    RegisterHandler(MsgType::Server_Finish_Train, std::bind(
      &SyncServer::ProcessFinishTrain, this, std::placeholders::_1));
    int num_worker = Zoo::Get()->num_workers();
    worker_get_clocks_.reset(new VectorClock(num_worker));
    worker_add_clocks_.reset(new VectorClock(num_worker));
  }

  // make some modification to suit to the sync server
  // please not use in other place, may different with the general vector clock
  class VectorClock {
  public:
    explicit VectorClock(int n) :
      local_clock_(n, 0), global_clock_(0), size_(0) {}

    static bool except_max_int_compare(int a, int b)
    {
      return (b == std::numeric_limits<int>::max() ? false : a < b);
    }

    // Return true when all clock reach a same number
    virtual bool Update(int i) {
      ++local_clock_[i];
      if (global_clock_ < *(std::min_element(std::begin(local_clock_),
        std::end(local_clock_)))) {
        ++global_clock_;
        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
          std::end(local_clock_), except_max_int_compare))) {
          return true;
        }
      }
      return false;
    }

    virtual bool FinishTrain(int i) {
      local_clock_[i] = std::numeric_limits<int>::max();
      if (global_clock_ < *(std::min_element(std::begin(local_clock_),
        std::end(local_clock_)))) {
        ++global_clock_;
        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
          std::end(local_clock_), except_max_int_compare))) {
          return true;
        }
      }
      return false;
    }

    std::string DebugString() {
      std::string os = "global ";
      os += std::to_string(global_clock_) + " local: ";
      for (auto i : local_clock_) os += std::to_string(i) + " ";
      return os;
    }

    int local_clock(int i) const { return local_clock_[i]; }
    int global_clock() const { return global_clock_; }

  protected:
    std::vector<int> local_clock_;
    int global_clock_;
    int size_;
  };
protected:
  void ProcessAdd(MessagePtr& msg) override {
    // 1. Before add: cache faster worker
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    if (worker_get_clocks_->local_clock(worker) >
        worker_get_clocks_->global_clock()) {
      msg_add_cache_.Push(msg);
      return;
    }
    // 2. Process Add
    Server::ProcessAdd(msg);
    // 3. After add: process cached process get if necessary
    if (worker_add_clocks_->Update(worker)) {
      CHECK(msg_add_cache_.Empty());
      while (!msg_get_cache_.Empty()) {
        MessagePtr get_msg;
        CHECK(msg_get_cache_.TryPop(get_msg));
        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
        Server::ProcessGet(get_msg);
        worker_get_clocks_->Update(get_worker);
      }
    }
  }

  void ProcessGet(MessagePtr& msg) override {
    // 1. Before get: cache faster worker
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    if (worker_add_clocks_->local_clock(worker) >
        worker_add_clocks_->global_clock()) {
      // Will wait for other worker finished Add
      msg_get_cache_.Push(msg);
      return;
    }
    // 2. Process Get
    Server::ProcessGet(msg);
    // 3. After get: process cached process add if necessary
    if (worker_get_clocks_->Update(worker)) {
      CHECK(msg_get_cache_.Empty());
      while (!msg_add_cache_.Empty()) {
        MessagePtr add_msg;
        CHECK(msg_add_cache_.TryPop(add_msg));
        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
        Server::ProcessAdd(add_msg);
        worker_add_clocks_->Update(add_worker);
      }
    }
  }

  void ProcessFinishTrain(MessagePtr& msg) {
    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
    Log::Debug("[ProcessFinishTrain] Server %d, worker %d has finished training.\n", Zoo::Get()->server_rank(), worker);

    if (worker_get_clocks_->FinishTrain(worker)) {
      CHECK(msg_get_cache_.Empty());
      while (!msg_add_cache_.Empty()) {
        MessagePtr add_msg;
        CHECK(msg_add_cache_.TryPop(add_msg));
        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
        Server::ProcessAdd(add_msg);
        worker_add_clocks_->Update(add_worker);
      }
    }

    if (worker_add_clocks_->FinishTrain(worker)) {
      CHECK(msg_add_cache_.Empty());
      while (!msg_get_cache_.Empty()) {
        MessagePtr get_msg;
        CHECK(msg_get_cache_.TryPop(get_msg));
        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
        Server::ProcessGet(get_msg);
        worker_get_clocks_->Update(get_worker);
      }
    }
  }

private:
  std::unique_ptr<VectorClock> worker_get_clocks_;
  std::unique_ptr<VectorClock> worker_add_clocks_;

  MtQueue<MessagePtr> msg_add_cache_;
  MtQueue<MessagePtr> msg_get_cache_;
};

// TODO[qiwye]: Backup logic need to fix

//class WithBackupSyncServer : public Server {
//public:
//  WithBackupSyncServer() : Server() {
//    num_worker_ = Zoo::Get()->num_workers();
//    double backup_ratio = (double)MV_CONFIG_backup_worker_ratio / 100;
//    num_sync_worker_ = num_worker_ -
//                       static_cast<int>(backup_ratio * num_worker_);
//    CHECK(num_sync_worker_ > 0 && num_sync_worker_ <= num_worker_);
//    if (num_sync_worker_ == num_worker_) {
//      Log::Info("No backup worker, using the sync mode\n");
//    }
//    Log::Info("Sync with backup worker start: num_sync_worker = %d,"
//              "num_total_worker = %d\n", num_sync_worker_, num_worker_);
//    worker_get_clocks_.reset(new VectorClock(num_worker_));
//    worker_add_clocks_.reset(new VectorClock(
//      num_worker_, num_worker_ - num_sync_worker_));
//  }
//
//  // make some modification to suit to the sync server
//  // please not use in other place, may different with the general vector clock
//  class VectorClock {
//  public:
//    VectorClock(int num_worker, int num_backup_worker = 0) :
//      local_clock_(num_worker, 0), global_clock_(0), num_worker_(num_worker),
//      num_sync_worker_(num_worker - num_backup_worker), progress_(0) {}
//
//    // Return true when global clock meet the sync condition
//    // sync: all worker reach the same clock
//    // backup-worker-sync: sync-workers reach the same clock
//    virtual bool Update(int i) {
//      if (local_clock_[i]++ == global_clock_) {
//        ++progress_;
//      }
//      if (progress_ >= num_sync_worker_) {
//        ++global_clock_;
//        progress_ = 0;
//        for (auto i : local_clock_) {
//          if (i > global_clock_) ++progress_;
//        }
//        if (global_clock_ == *(std::max_element(std::begin(local_clock_),
//          std::end(local_clock_)))) {
//          return true;
//        }
//      }
//      return false;
//    }
//
//    std::string DebugString() {
//      std::string os = "global ";
//      os += std::to_string(global_clock_) + " local: ";
//      for (auto i : local_clock_) os += std::to_string(i) + " ";
//      return os;
//    }
//
//    int local_clock(int i) const { return local_clock_[i]; }
//    int global_clock() const { return global_clock_; }
//
//  protected:
//    std::vector<int> local_clock_;
//    int global_clock_;
//    int num_worker_;
//    int num_sync_worker_;
//    int progress_;
//  };
//protected:
//  void ProcessAdd(MessagePtr& msg) override {
//    // 1. Before add: cache faster worker
//    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
//    if (worker_get_clocks_->local_clock(worker) >
//      worker_get_clocks_->global_clock()) {
//      msg_add_cache_.Push(msg);
//      return;
//    }
//    // 2. Process Add
//    if (worker_add_clocks_->local_clock(worker) >=
//        worker_add_clocks_->global_clock()) {
//      Server::ProcessAdd(msg);
//    }
//    // 3. After add: process cached process get if necessary
//    if (worker_add_clocks_->Update(worker)) {
//      CHECK(msg_add_cache_.Empty());
//      while (!msg_get_cache_.Empty()) {
//        MessagePtr get_msg;
//        CHECK(msg_get_cache_.TryPop(get_msg));
//        int get_worker = Zoo::Get()->rank_to_worker_id(get_msg->src());
//        Server::ProcessGet(get_msg);
//        worker_get_clocks_->Update(get_worker);
//      }
//    }
//  }
//
//  void ProcessGet(MessagePtr& msg) override {
//    // 1. Before get: cache faster worker
//    int worker = Zoo::Get()->rank_to_worker_id(msg->src());
//    if (worker_add_clocks_->local_clock(worker) >
//      worker_add_clocks_->global_clock()) {
//      // Will wait for other worker finished Add
//      msg_get_cache_.Push(msg);
//      return;
//    }
//    // 2. Process Get
//    Server::ProcessGet(msg);
//    // 3. After get: process cached process add if necessary
//    if (worker_get_clocks_->Update(worker)) {
//      CHECK(msg_get_cache_.Empty());
//      while (!msg_add_cache_.Empty()) {
//        MessagePtr add_msg;
//        CHECK(msg_add_cache_.TryPop(add_msg));
//        int add_worker = Zoo::Get()->rank_to_worker_id(add_msg->src());
//        if (worker_add_clocks_->local_clock(add_worker) >=
//          worker_add_clocks_->global_clock()) {
//          Server::ProcessAdd(msg);
//        };
//        worker_add_clocks_->Update(add_worker);
//      }
//    }
//  }
//
//  void ProcessFinish(MessagePtr& msg) {
//
//  }
//
//private:
//  std::unique_ptr<VectorClock> worker_get_clocks_;
//  std::unique_ptr<VectorClock> worker_add_clocks_;
//
//  MtQueue<MessagePtr> msg_add_cache_;
//  MtQueue<MessagePtr> msg_get_cache_;
//
//  // num_worker_ - num_sync_worker_ = num_backup_worker_
//  int num_sync_worker_;
//  int num_worker_;
//};

Server* Server::GetServer() {
  if (!MV_CONFIG_sync) {
    Log::Info("Create a async server\n");
    return new Server();
  }
  // if (MV_CONFIG_backup_worker_ratio > 0.0) {
  Log::Info("Create a sync server\n");
  return new SyncServer();
  // }
}

}  // namespace multiverso
<|endoftext|>