<commit_before>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * DataManWriter.cpp
 *
 *  Created on: Jan 10, 2017
 *      Author: Jason Wang
 */

#include "DataManWriter.tcc"

namespace adios2
{
namespace core
{
namespace engine
{

DataManWriter::DataManWriter(IO &io, const std::string &name,
                             const Mode openMode, helper::Comm comm)
: Engine("DataManWriter", io, name, openMode, std::move(comm)),
  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)), m_SentSteps(0),
  m_ReplyThreadActive(true), m_PublishThreadActive(true)
{

    m_MpiRank = m_Comm.Rank();
    m_MpiSize = m_Comm.Size();

    m_Serializer.NewWriterBuffer(m_SerializerBufferSize);

    helper::GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress);
    helper::GetParameter(m_IO.m_Parameters, "Port", m_Port);
    helper::GetParameter(m_IO.m_Parameters, "Timeout", m_Timeout);
    helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity);
    helper::GetParameter(m_IO.m_Parameters, "RendezvousReaderCount",
                         m_RendezvousReaderCount);
    helper::GetParameter(m_IO.m_Parameters, "DoubleBuffer", m_DoubleBuffer);
    helper::GetParameter(m_IO.m_Parameters, "TransportMode", m_TransportMode);
    helper::GetParameter(m_IO.m_Parameters, "Monitor", m_MonitorActive);
    helper::GetParameter(m_IO.m_Parameters, "CombiningSteps", m_CombiningSteps);

    if (m_IPAddress.empty())
    {
        throw(std::invalid_argument("IP address not specified"));
    }
    m_Port += m_MpiRank;
    m_ReplierAddress = "tcp://" + m_IPAddress + ":" + std::to_string(m_Port);
    m_PublisherAddress =
        "tcp://" + m_IPAddress + ":" + std::to_string(m_Port + m_MpiSize);

    std::vector<std::string> pubVec;
    std::vector<std::string> repVec;

    if (m_MpiSize == 1)
    {
        pubVec.push_back(m_PublisherAddress);
        repVec.push_back(m_ReplierAddress);
    }
    else
    {
        std::vector<char> allPubVec(32 * m_MpiSize, '\0');
        std::vector<char> allRepVec(32 * m_MpiSize, '\0');

        m_Comm.Allgather(m_PublisherAddress.data(), m_PublisherAddress.size(),
                         allPubVec.data(), 32);
        m_Comm.Allgather(m_ReplierAddress.data(), m_ReplierAddress.size(),
                         allRepVec.data(), 32);

        for (int i = 0; i < m_MpiSize; ++i)
        {
            pubVec.push_back(std::string(allPubVec.begin() + i * 32,
                                         allPubVec.begin() + (i + 1) * 32));
            repVec.push_back(std::string(allRepVec.begin() + i * 32,
                                         allRepVec.begin() + (i + 1) * 32));
        }
    }

    nlohmann::json addJson;
    addJson["PublisherAddresses"] = pubVec;
    addJson["ReplierAddresses"] = repVec;
    m_AllAddresses = addJson.dump() + '\0';

    if (m_TransportMode == "fast")
    {
        m_Publisher.OpenPublisher(m_PublisherAddress);
    }

    m_Replier.OpenReplier(m_ReplierAddress, m_Timeout, 64);

    if (m_RendezvousReaderCount == 0 || m_TransportMode == "reliable")
    {
        m_ReplyThreadActive = true;
        m_ReplyThread = std::thread(&DataManWriter::ReplyThread, this);
    }
    else
    {
        ReplyThread();
        m_Comm.Barrier();
    }

    if (m_DoubleBuffer && m_TransportMode == "fast")
    {
        m_PublishThreadActive = true;
        m_PublishThread = std::thread(&DataManWriter::PublishThread, this);
    }
}

DataManWriter::~DataManWriter()
{
    if (not m_IsClosed)
    {
        DoClose();
    }
}

StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec)
{
    ++m_CurrentStep;
    if (m_CombiningSteps <= m_CombinedSteps)
    {
        m_Serializer.NewWriterBuffer(m_SerializerBufferSize);
    }

    if (m_MonitorActive)
    {
        m_Monitor.BeginStep(m_CurrentStep);
    }

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::BeginStep " << m_CurrentStep << std::endl;
    }

    return StepStatus::OK;
}

size_t DataManWriter::CurrentStep() const { return m_CurrentStep; }

void DataManWriter::PerformPuts() {}

void DataManWriter::EndStep()
{
    if (m_CurrentStep == 0)
    {
        m_Serializer.PutAttributes(m_IO);
    }

    ++m_CombinedSteps;

    if (m_CombinedSteps == m_CombiningSteps)
    {
        m_CombinedSteps = 0;
        m_Serializer.AttachAttributesToLocalPack();
        const auto buffer = m_Serializer.GetLocalPack();
        if (buffer->size() > m_SerializerBufferSize)
        {
            m_SerializerBufferSize = buffer->size();
        }

        if (m_MonitorActive)
        {
            m_Monitor.BeginTransport(m_CurrentStep);
        }

        if (m_DoubleBuffer || m_TransportMode == "reliable")
        {
            PushBufferQueue(buffer);
        }
        else
        {
            m_Publisher.Send(buffer);
            if (m_MonitorActive)
            {
                for (int i = 0; i < m_CombiningSteps; ++i)
                {
                    m_Monitor.EndTransport();
                }
            }
        }
    }
    else
    {
        if (m_MonitorActive)
        {
            m_Monitor.BeginTransport(m_CurrentStep);
        }
    }

    if (m_MonitorActive)
    {
        m_Monitor.EndStep(m_CurrentStep);
    }

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::EndStep " << m_CurrentStep << std::endl;
    }
}

void DataManWriter::Flush(const int transportIndex) {}

// PRIVATE functions below

#define declare_type(T)                                                        \
    void DataManWriter::DoPutSync(Variable<T> &variable, const T *values)      \
    {                                                                          \
        PutSyncCommon(variable, values);                                       \
    }                                                                          \
    void DataManWriter::DoPutDeferred(Variable<T> &variable, const T *values)  \
    {                                                                          \
        PutDeferredCommon(variable, values);                                   \
    }
ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type

void DataManWriter::DoClose(const int transportIndex)
{
    nlohmann::json endSignal;
    endSignal["FinalStep"] = static_cast<int64_t>(m_CurrentStep);
    std::string s = endSignal.dump() + '\0';
    auto cvp = std::make_shared<std::vector<char>>(s.size());
    std::memcpy(cvp->data(), s.c_str(), s.size());

    if (m_DoubleBuffer || m_TransportMode == "reliable")
    {
        PushBufferQueue(cvp);
    }
    else
    {
        m_Publisher.Send(cvp);
    }

    m_PublishThreadActive = false;

    if (m_ReplyThreadActive)
    {
        while (m_SentSteps < m_CurrentStep + 2)
        {
        }
        m_ReplyThreadActive = false;
    }

    if (m_ReplyThread.joinable())
    {
        m_ReplyThread.join();
    }

    if (m_PublishThread.joinable())
    {
        m_PublishThread.join();
    }

    m_IsClosed = true;

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::DoClose " << m_CurrentStep << std::endl;
    }
}

void DataManWriter::PushBufferQueue(std::shared_ptr<std::vector<char>> buffer)
{
    std::lock_guard<std::mutex> l(m_BufferQueueMutex);
    m_BufferQueue.push(buffer);
}

std::shared_ptr<std::vector<char>> DataManWriter::PopBufferQueue()
{
    std::lock_guard<std::mutex> l(m_BufferQueueMutex);
    if (m_BufferQueue.empty())
    {
        return nullptr;
    }
    else
    {
        auto ret = m_BufferQueue.front();
        m_BufferQueue.pop();
        return ret;
    }
}

void DataManWriter::PublishThread()
{
    while (m_PublishThreadActive)
    {
        auto buffer = PopBufferQueue();
        if (buffer != nullptr && buffer->size() > 0)
        {
            m_Publisher.Send(buffer);
            if (m_MonitorActive)
            {
                for (int i = 0; i < m_CombiningSteps; ++i)
                {
                    m_Monitor.EndTransport();
                }
            }
        }
    }
}

void DataManWriter::ReplyThread()
{
    int readerCount = 0;
    while (m_ReplyThreadActive)
    {
        auto request = m_Replier.ReceiveRequest();
        if (request != nullptr && request->size() > 0)
        {
            std::string r(request->begin(), request->end());
            if (r == "Address")
            {
                m_Replier.SendReply(m_AllAddresses.data(),
                                    m_AllAddresses.size());
            }
            else if (r == "Ready")
            {
                m_Replier.SendReply("OK", 2);
                ++readerCount;
            }
            else if (r == "Step")
            {
                auto buffer = PopBufferQueue();
                while (buffer == nullptr)
                {
                    buffer = PopBufferQueue();
                }
                if (buffer->size() > 0)
                {
                    m_Replier.SendReply(buffer);
                    ++m_SentSteps;
                    if (m_MonitorActive)
                    {
                        m_Monitor.EndTransport();
                    }
                }
            }
        }
        if (m_RendezvousReaderCount == readerCount && m_TransportMode == "fast")
        {
            m_ReplyThreadActive = false;
        }
    }
}

} // end namespace engine
} // end namespace core
} // end namespace adios2
<commit_msg>fixed missing steps at the end of a stream<commit_after>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * DataManWriter.cpp
 *
 *  Created on: Jan 10, 2017
 *      Author: Jason Wang
 */

#include "DataManWriter.tcc"

namespace adios2
{
namespace core
{
namespace engine
{

DataManWriter::DataManWriter(IO &io, const std::string &name,
                             const Mode openMode, helper::Comm comm)
: Engine("DataManWriter", io, name, openMode, std::move(comm)),
  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)), m_SentSteps(0),
  m_ReplyThreadActive(true), m_PublishThreadActive(true)
{

    m_MpiRank = m_Comm.Rank();
    m_MpiSize = m_Comm.Size();

    m_Serializer.NewWriterBuffer(m_SerializerBufferSize);

    helper::GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress);
    helper::GetParameter(m_IO.m_Parameters, "Port", m_Port);
    helper::GetParameter(m_IO.m_Parameters, "Timeout", m_Timeout);
    helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity);
    helper::GetParameter(m_IO.m_Parameters, "RendezvousReaderCount",
                         m_RendezvousReaderCount);
    helper::GetParameter(m_IO.m_Parameters, "DoubleBuffer", m_DoubleBuffer);
    helper::GetParameter(m_IO.m_Parameters, "TransportMode", m_TransportMode);
    helper::GetParameter(m_IO.m_Parameters, "Monitor", m_MonitorActive);
    helper::GetParameter(m_IO.m_Parameters, "CombiningSteps", m_CombiningSteps);

    if (m_IPAddress.empty())
    {
        throw(std::invalid_argument("IP address not specified"));
    }
    m_Port += m_MpiRank;
    m_ReplierAddress = "tcp://" + m_IPAddress + ":" + std::to_string(m_Port);
    m_PublisherAddress =
        "tcp://" + m_IPAddress + ":" + std::to_string(m_Port + m_MpiSize);

    std::vector<std::string> pubVec;
    std::vector<std::string> repVec;

    if (m_MpiSize == 1)
    {
        pubVec.push_back(m_PublisherAddress);
        repVec.push_back(m_ReplierAddress);
    }
    else
    {
        std::vector<char> allPubVec(32 * m_MpiSize, '\0');
        std::vector<char> allRepVec(32 * m_MpiSize, '\0');

        m_Comm.Allgather(m_PublisherAddress.data(), m_PublisherAddress.size(),
                         allPubVec.data(), 32);
        m_Comm.Allgather(m_ReplierAddress.data(), m_ReplierAddress.size(),
                         allRepVec.data(), 32);

        for (int i = 0; i < m_MpiSize; ++i)
        {
            pubVec.push_back(std::string(allPubVec.begin() + i * 32,
                                         allPubVec.begin() + (i + 1) * 32));
            repVec.push_back(std::string(allRepVec.begin() + i * 32,
                                         allRepVec.begin() + (i + 1) * 32));
        }
    }

    nlohmann::json addJson;
    addJson["PublisherAddresses"] = pubVec;
    addJson["ReplierAddresses"] = repVec;
    m_AllAddresses = addJson.dump() + '\0';

    if (m_TransportMode == "fast")
    {
        m_Publisher.OpenPublisher(m_PublisherAddress);
    }

    m_Replier.OpenReplier(m_ReplierAddress, m_Timeout, 64);

    if (m_RendezvousReaderCount == 0 || m_TransportMode == "reliable")
    {
        m_ReplyThreadActive = true;
        m_ReplyThread = std::thread(&DataManWriter::ReplyThread, this);
    }
    else
    {
        ReplyThread();
        m_Comm.Barrier();
    }

    if (m_DoubleBuffer && m_TransportMode == "fast")
    {
        m_PublishThreadActive = true;
        m_PublishThread = std::thread(&DataManWriter::PublishThread, this);
    }
}

DataManWriter::~DataManWriter()
{
    if (not m_IsClosed)
    {
        DoClose();
    }
}

StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec)
{
    ++m_CurrentStep;
    if (m_CombiningSteps <= m_CombinedSteps)
    {
        m_Serializer.NewWriterBuffer(m_SerializerBufferSize);
    }

    if (m_MonitorActive)
    {
        m_Monitor.BeginStep(m_CurrentStep);
    }

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::BeginStep " << m_CurrentStep << std::endl;
    }

    return StepStatus::OK;
}

size_t DataManWriter::CurrentStep() const { return m_CurrentStep; }

void DataManWriter::PerformPuts() {}

void DataManWriter::EndStep()
{
    if (m_CurrentStep == 0)
    {
        m_Serializer.PutAttributes(m_IO);
    }

    ++m_CombinedSteps;

    if (m_CombinedSteps >= m_CombiningSteps)
    {
        m_CombinedSteps = 0;
        m_Serializer.AttachAttributesToLocalPack();
        const auto buffer = m_Serializer.GetLocalPack();
        if (buffer->size() > m_SerializerBufferSize)
        {
            m_SerializerBufferSize = buffer->size();
        }

        if (m_MonitorActive)
        {
            m_Monitor.BeginTransport(m_CurrentStep);
        }

        if (m_DoubleBuffer || m_TransportMode == "reliable")
        {
            PushBufferQueue(buffer);
        }
        else
        {
            m_Publisher.Send(buffer);
            if (m_MonitorActive)
            {
                for (int i = 0; i < m_CombiningSteps; ++i)
                {
                    m_Monitor.EndTransport();
                }
            }
        }
    }
    else
    {
        if (m_MonitorActive)
        {
            m_Monitor.BeginTransport(m_CurrentStep);
        }
    }

    if (m_MonitorActive)
    {
        m_Monitor.EndStep(m_CurrentStep);
    }

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::EndStep " << m_CurrentStep << std::endl;
    }
}

void DataManWriter::Flush(const int transportIndex) {}

// PRIVATE functions below

#define declare_type(T)                                                        \
    void DataManWriter::DoPutSync(Variable<T> &variable, const T *values)      \
    {                                                                          \
        PutSyncCommon(variable, values);                                       \
    }                                                                          \
    void DataManWriter::DoPutDeferred(Variable<T> &variable, const T *values)  \
    {                                                                          \
        PutDeferredCommon(variable, values);                                   \
    }
ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type

void DataManWriter::DoClose(const int transportIndex)
{
    if (m_CombinedSteps != m_CombiningSteps)
    {
        m_CombinedSteps = m_CombiningSteps;
        EndStep();
    }
    nlohmann::json endSignal;
    endSignal["FinalStep"] = static_cast<int64_t>(m_CurrentStep);
    std::string s = endSignal.dump() + '\0';
    auto cvp = std::make_shared<std::vector<char>>(s.size());
    std::memcpy(cvp->data(), s.c_str(), s.size());

    if (m_DoubleBuffer || m_TransportMode == "reliable")
    {
        PushBufferQueue(cvp);
    }
    else
    {
        m_Publisher.Send(cvp);
    }

    m_PublishThreadActive = false;

    if (m_ReplyThreadActive)
    {
        while (m_SentSteps < m_CurrentStep + 2)
        {
        }
        m_ReplyThreadActive = false;
    }

    if (m_ReplyThread.joinable())
    {
        m_ReplyThread.join();
    }

    if (m_PublishThread.joinable())
    {
        m_PublishThread.join();
    }

    m_IsClosed = true;

    if (m_Verbosity >= 10)
    {
        std::cout << "DataManWriter::DoClose " << m_CurrentStep << std::endl;
    }
}

void DataManWriter::PushBufferQueue(std::shared_ptr<std::vector<char>> buffer)
{
    std::lock_guard<std::mutex> l(m_BufferQueueMutex);
    m_BufferQueue.push(buffer);
}

std::shared_ptr<std::vector<char>> DataManWriter::PopBufferQueue()
{
    std::lock_guard<std::mutex> l(m_BufferQueueMutex);
    if (m_BufferQueue.empty())
    {
        return nullptr;
    }
    else
    {
        auto ret = m_BufferQueue.front();
        m_BufferQueue.pop();
        return ret;
    }
}

void DataManWriter::PublishThread()
{
    while (m_PublishThreadActive)
    {
        auto buffer = PopBufferQueue();
        if (buffer != nullptr && buffer->size() > 0)
        {
            m_Publisher.Send(buffer);
            if (m_MonitorActive)
            {
                for (int i = 0; i < m_CombiningSteps; ++i)
                {
                    m_Monitor.EndTransport();
                }
            }
        }
    }
}

void DataManWriter::ReplyThread()
{
    int readerCount = 0;
    while (m_ReplyThreadActive)
    {
        auto request = m_Replier.ReceiveRequest();
        if (request != nullptr && request->size() > 0)
        {
            std::string r(request->begin(), request->end());
            if (r == "Address")
            {
                m_Replier.SendReply(m_AllAddresses.data(),
                                    m_AllAddresses.size());
            }
            else if (r == "Ready")
            {
                m_Replier.SendReply("OK", 2);
                ++readerCount;
            }
            else if (r == "Step")
            {
                auto buffer = PopBufferQueue();
                while (buffer == nullptr)
                {
                    buffer = PopBufferQueue();
                }
                if (buffer->size() > 0)
                {
                    m_Replier.SendReply(buffer);
                    ++m_SentSteps;
                    if (m_MonitorActive)
                    {
                        m_Monitor.EndTransport();
                    }
                }
            }
        }
        if (m_RendezvousReaderCount == readerCount && m_TransportMode == "fast")
        {
            m_ReplyThreadActive = false;
        }
    }
}

} // end namespace engine
} // end namespace core
} // end namespace adios2
<|endoftext|>


<commit_before>// Copyright (c) 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "source/fuzz/transformation_move_block_down.h"

#include "source/opt/basic_block.h"

namespace spvtools {
namespace fuzz {

TransformationMoveBlockDown::TransformationMoveBlockDown(
    const spvtools::fuzz::protobufs::TransformationMoveBlockDown& message)
    : message_(message) {}

TransformationMoveBlockDown::TransformationMoveBlockDown(uint32_t id) {
  message_.set_block_id(id);
}

bool TransformationMoveBlockDown::IsApplicable(
    opt::IRContext* context, const FactManager& /*unused*/) const {
  // Go through every block in every function, looking for a block whose id
  // matches that of the block we want to consider moving down.
  for (auto& function : *context->module()) {
    for (auto block_it = function.begin(); block_it != function.end();
         ++block_it) {
      if (block_it->id() == message_.block_id()) {
        // We have found a match.
        if (block_it == function.begin()) {
          // The block is the first one appearing in the function.  We are not
          // allowed to move this block down.
          return false;
        }
        // Record the block we would like to consider moving down.
        opt::BasicBlock* block_matching_id = &*block_it;
        if (!context->GetDominatorAnalysis(&function)->IsReachable(
                block_matching_id)) {
          // The block is not reachable.  We are not allowed to move it down.
          return false;
        }
        // Now see whether there is some block following that block in program
        // order.
        ++block_it;
        if (block_it == function.end()) {
          // There is no such block; i.e., the block we are considering moving
          // is the last one in the function.  The transformation thus does not
          // apply.
          return false;
        }
        opt::BasicBlock* next_block_in_program_order = &*block_it;
        // We can move the block of interest down if and only if it does not
        // dominate the block that comes next.
        return !context->GetDominatorAnalysis(&function)->Dominates(
            block_matching_id, next_block_in_program_order);
      }
    }
  }

  // We did not find a matching block, so the transformation is not applicable:
  // there is no relevant block to move.
  return false;
}

void TransformationMoveBlockDown::Apply(opt::IRContext* context,
                                        FactManager* /*unused*/) const {
  // Go through every block in every function, looking for a block whose id
  // matches that of the block we want to move down.
  for (auto& function : *context->module()) {
    for (auto block_it = function.begin(); block_it != function.end();
         ++block_it) {
      if (block_it->id() == message_.block_id()) {
        ++block_it;
        assert(block_it != function.end() &&
               "To be able to move a block down, it needs to have a "
               "program-order successor.");
        function.MoveBasicBlockToAfter(message_.block_id(), &*block_it);
        // TODO(https://github.com/KhronosGroup/SPIRV-Tools/issues/2889):
        //  revisit whether it would be OK to avoid invalidating the dominator
        //  analysis (and perhaps other analyses).
        context->InvalidateAnalysesExceptFor(
            opt::IRContext::Analysis::kAnalysisDefUse);

        return;
      }
    }
  }
  assert(false && "No block was found to move down.");
}

protobufs::Transformation TransformationMoveBlockDown::ToMessage() const {
  protobufs::Transformation result;
  *result.mutable_move_block_down() = message_;
  return result;
}

}  // namespace fuzz
}  // namespace spvtools
<commit_msg>spirv-fuzz: preserve some analyses when permuting blocks (#2918)<commit_after>// Copyright (c) 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "source/fuzz/transformation_move_block_down.h"

#include "source/opt/basic_block.h"

namespace spvtools {
namespace fuzz {

TransformationMoveBlockDown::TransformationMoveBlockDown(
    const spvtools::fuzz::protobufs::TransformationMoveBlockDown& message)
    : message_(message) {}

TransformationMoveBlockDown::TransformationMoveBlockDown(uint32_t id) {
  message_.set_block_id(id);
}

bool TransformationMoveBlockDown::IsApplicable(
    opt::IRContext* context, const FactManager& /*unused*/) const {
  // Go through every block in every function, looking for a block whose id
  // matches that of the block we want to consider moving down.
  for (auto& function : *context->module()) {
    for (auto block_it = function.begin(); block_it != function.end();
         ++block_it) {
      if (block_it->id() == message_.block_id()) {
        // We have found a match.
        if (block_it == function.begin()) {
          // The block is the first one appearing in the function.  We are not
          // allowed to move this block down.
          return false;
        }
        // Record the block we would like to consider moving down.
        opt::BasicBlock* block_matching_id = &*block_it;
        if (!context->GetDominatorAnalysis(&function)->IsReachable(
                block_matching_id)) {
          // The block is not reachable.  We are not allowed to move it down.
          return false;
        }
        // Now see whether there is some block following that block in program
        // order.
        ++block_it;
        if (block_it == function.end()) {
          // There is no such block; i.e., the block we are considering moving
          // is the last one in the function.  The transformation thus does not
          // apply.
          return false;
        }
        opt::BasicBlock* next_block_in_program_order = &*block_it;
        // We can move the block of interest down if and only if it does not
        // dominate the block that comes next.
        return !context->GetDominatorAnalysis(&function)->Dominates(
            block_matching_id, next_block_in_program_order);
      }
    }
  }

  // We did not find a matching block, so the transformation is not applicable:
  // there is no relevant block to move.
  return false;
}

void TransformationMoveBlockDown::Apply(opt::IRContext* context,
                                        FactManager* /*unused*/) const {
  // Go through every block in every function, looking for a block whose id
  // matches that of the block we want to move down.
  for (auto& function : *context->module()) {
    for (auto block_it = function.begin(); block_it != function.end();
         ++block_it) {
      if (block_it->id() == message_.block_id()) {
        ++block_it;
        assert(block_it != function.end() &&
               "To be able to move a block down, it needs to have a "
               "program-order successor.");
        function.MoveBasicBlockToAfter(message_.block_id(), &*block_it);
        // For performance, it is vital to keep the dominator analysis valid
        // (which due to https://github.com/KhronosGroup/SPIRV-Tools/issues/2889
        // requires keeping the CFG analysis valid).
        context->InvalidateAnalysesExceptFor(
            opt::IRContext::Analysis::kAnalysisDefUse |
            opt::IRContext::Analysis::kAnalysisCFG |
            opt::IRContext::Analysis::kAnalysisDominatorAnalysis);

        return;
      }
    }
  }
  assert(false && "No block was found to move down.");
}

protobufs::Transformation TransformationMoveBlockDown::ToMessage() const {
  protobufs::Transformation result;
  *result.mutable_move_block_down() = message_;
  return result;
}

}  // namespace fuzz
}  // namespace spvtools
<|endoftext|>


<commit_before>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/FieldIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/TermPositions.h>
#include <ir/index_manager/store/IndexInput.h>

#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>

using namespace std;

namespace bfs = boost::filesystem;

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

FieldIndexer::FieldIndexer(const char* field, MemCache* pCache, Indexer* pIndexer)
        :field_(field),pMemCache_(pCache),pIndexer_(pIndexer),vocFilePointer_(0),alloc_(0)
        termCount_(0),recordCount_(0),run_num_(0),pHits_(0),pHitsMax_(0),flush_(false)
{
    skipInterval_ = pIndexer_->getSkipInterval();
    maxSkipLevel_ = pIndexer_->getMaxSkipLevel();
    if (! pIndexer_->isRealTime())
    {
        //std::string sorterName = field_+".tmp";
        sorterFileName_ = field_+".tmp";
        bfs::path path(bfs::path(pIndexer->pConfigurationManager_->indexStrategy_.indexLocation_) /bfs::path(sorterFileName_));
        sorterFullPath_= path.string();
        f_ = fopen(sorterFullPath_.c_str(),"w");
        uint64_t count = 0;
        fwrite(&count, sizeof(uint64_t), 1, f_);
    }
    else
        alloc_ = new boost::scoped_alloc(recycle_);
}

FieldIndexer::~FieldIndexer()
{
    /*
        InMemoryPostingMap::iterator iter = postingMap_.begin()
        for(; iter !=postingMap_.end(); ++iter)
        {
            delete iter->second;
        }
    */
    if (alloc_) delete alloc_;
    pMemCache_ = NULL;
}

void FieldIndexer::setHitBuffer(size_t size)
{
    iHitsMax_ = size;
    iHitsMax_ = iHitsMax_/sizeof(TermId) ;
    hits_.assign(iHitsMax_);
    pHits_ = hits_;
    pHitsMax_ = hits_ + iHitsMax_;
}

void FieldIndexer::writeHitBuffer(int iHits)
{
    recordCount_ += iHits;
    bufferSort ( &hits_[0], iHits, CmpTermId_fn() );

    uint32_t output_buf_size = iHits * (sizeof(TermId)+sizeof(uint8_t));
    ///buffer size
    fwrite(&output_buf_size, sizeof(uint32_t), 1, f_);
    ///number of hits
    fwrite(&iHits, sizeof(uint32_t), 1, f_);

    uint64_t nextStart = 0;
    uint64_t nextStartPos = ftell(f_);
    ///next start
    fwrite(&nextStart, sizeof(uint64_t), 1, f_);
    FieldIndexIO ioStream(f_, "w");
    ioStream.writeRecord(&hits_[0], iHits * sizeof(TermId));
    nextStart = ftell(f_);

    ///update next start
    fseek(f_, nextStartPos, SEEK_SET);
    fwrite(&nextStart, sizeof(uint64_t), 1, f_);
    fseek(f_, nextStart, SEEK_SET);

    ++run_num_;
}

void FieldIndexer::addField(docid_t docid, boost::shared_ptr<LAInput> laInput)
{
    if (pIndexer_->isRealTime())
    {
        InMemoryPosting* curPosting;
        for (LAInput::iterator iter = laInput->begin(); iter != laInput->end(); ++iter)
        {
            InMemoryPostingMap::iterator postingIter = postingMap_.find(iter->termid_);
            if (postingIter == postingMap_.end())
            {
                //curPosting = new InMemoryPosting(pMemCache_, skipInterval_, maxSkipLevel_);
                curPosting = BOOST_NEW(*alloc_, InMemoryPosting)(pMemCache_, skipInterval_, maxSkipLevel_);
                postingMap_[iter->termid_] = curPosting;
            }
            else
                curPosting = postingIter->second;
            curPosting->add(docid, iter->wordOffset_);
        }
    }
    else
    {
        if(flush_)
        {
            hits_.assign(iHitsMax_);
            pHits_ = hits_;
            pHitsMax_ = hits_ + iHitsMax_;
            flush_ = false;
        }

        int iDocHits = laInput->size();
        TermId * pDocHits = (TermId*)&(* laInput->begin());

        while( iDocHits > 0)
        {
            int iToCopy = iDocHits < (pHitsMax_ - pHits_) ? iDocHits: (pHitsMax_ - pHits_);
            memcpy(pHits_, pDocHits, iToCopy*sizeof(TermId));
            pHits_ += iToCopy;
            pDocHits += iToCopy;
            iDocHits -= iToCopy;

            if (pHits_ < pHitsMax_)
                continue;

            int iHits = pHits_ - hits_;
            writeHitBuffer(iHits);
            pHits_ = hits_;
        }
    }
}

void FieldIndexer::addField(docid_t docid, boost::shared_ptr<ForwardIndex> forwardindex)
{
    InMemoryPosting* curPosting;

    for (ForwardIndex::iterator iter = forwardindex->begin(); iter != forwardindex->end(); ++iter)
    {
        InMemoryPostingMap::iterator postingIter = postingMap_.find(iter->first);
        if (postingIter == postingMap_.end())
        {
            curPosting = BOOST_NEW(*alloc_, InMemoryPosting)(pMemCache_, skipInterval_, maxSkipLevel_);
            postingMap_[iter->first] = curPosting;
        }
        else
            curPosting = postingIter->second;

        ForwardIndexOffset::iterator	endit = iter->second->end();
        for (ForwardIndexOffset::iterator it = iter->second->begin(); it != endit; ++it)
            curPosting->add(docid, *it);
    }
}

void FieldIndexer::reset()
{
    InMemoryPosting* pPosting;
    for (InMemoryPostingMap::iterator iter = postingMap_.begin(); iter !=postingMap_.end(); ++iter)
    {
        pPosting = iter->second;
        if (!pPosting->hasNoChunk())
        {
            pPosting->reset();		///clear posting data
        }
    }
    postingMap_.clear();
    termCount_ = 0;
}

void writeTermInfo(IndexOutput* pVocWriter, termid_t tid, const TermInfo& termInfo)
{
    pVocWriter->writeInt(tid);					///write term id
    pVocWriter->writeInt(termInfo.docFreq_);		///write df
    pVocWriter->writeInt(termInfo.ctf_);			///write ctf
    pVocWriter->writeInt(termInfo.lastDocID_);		///write last doc id
    pVocWriter->writeInt(termInfo.skipLevel_);		///write skip level
    pVocWriter->writeLong(termInfo.skipPointer_);	///write skip list offset offset
    pVocWriter->writeLong(termInfo.docPointer_);	///write document posting offset
    pVocWriter->writeInt(termInfo.docPostingLen_);	///write document posting length (without skiplist)
    pVocWriter->writeLong(termInfo.positionPointer_);	///write position posting offset
    pVocWriter->writeInt(termInfo.positionPostingLen_);///write position posting length
}

#pragma pack(push,1)
struct Record
{
    uint8_t len;
    uint32_t tid;
    uint32_t docId;
    uint32_t offset;
};
#pragma pack(pop)

fileoffset_t FieldIndexer::write(OutputDescriptor* pWriterDesc)
{
    vocFilePointer_ = pWriterDesc->getVocOutput()->getFilePointer();

    IndexOutput* pVocWriter = pWriterDesc->getVocOutput();

    termid_t tid;
    InMemoryPosting* pPosting;
    fileoffset_t vocOffset = pVocWriter->getFilePointer();
    TermInfo termInfo;

    if (pIndexer_->isRealTime())
    {
        izenelib::util::ScopedWriteLock<izenelib::util::ReadWriteLock> lock(rwLock_);
        for (InMemoryPostingMap::iterator iter = postingMap_.begin(); iter !=postingMap_.end(); ++iter)
        {
            pPosting = iter->second;
            pPosting->setDirty(true);
            if (!pPosting->hasNoChunk())
            {
                tid = iter->first;
                pPosting->write(pWriterDesc, termInfo);		///write posting data
                writeTermInfo(pVocWriter,tid,termInfo);
                pPosting->reset();								///clear posting data
                termInfo.reset();
                termCount_++;
            }
            //delete pPosting;
            //pPosting = NULL;
        }

        postingMap_.clear();

        delete alloc_;
        alloc_ = new boost::scoped_alloc(recycle_);
    }
    else
    {
        if( !boost::filesystem::exists(sorterFullPath_) )
        {
            fileoffset_t vocDescOffset = pVocWriter->getFilePointer();
            int64_t vocLength = vocDescOffset - vocOffset;

            //SF1V5_THROW(ERROR_FILEIO,"Open file error: " + sorterFullPath_);
            pVocWriter->writeLong(vocLength);	///<VocLength(Int64)>
            pVocWriter->writeLong(termCount_);	///<TermCount(Int64)>
            ///end write vocabulary descriptor

            return vocDescOffset;
        }
        int iHits = pHits_ - hits_;
        if(iHits > 0)
        {
            writeHitBuffer(iHits);
        }
        fseek(f_, 0, SEEK_SET);
        fwrite(&recordCount_, sizeof(uint64_t), 1, f_);
        fclose(f_);
        hits_.reset();

        typedef izenelib::am::SortMerger<uint32_t, uint8_t, true,SortIO<FieldIndexIO> > merge_t;

        struct timeval tvafter, tvpre;
        struct timezone tz;
        gettimeofday (&tvpre , &tz);

        merge_t* merger = new merge_t(sorterFullPath_.c_str(), run_num_, 100000000, 2);
        merger->run();

        gettimeofday (&tvafter , &tz);
        std::cout<<"\nIt takes "<<((tvafter.tv_sec-tvpre.tv_sec)*1000+(tvafter.tv_usec-tvpre.tv_usec)/1000.)/60000
		 <<" minutes to sort("<<recordCount_<<")\n";
        delete merger;
        recordCount_ = 0;
        run_num_ = 0;
        flush_ = true;

        FILE* f = fopen(sorterFullPath_.c_str(),"r");
        uint64_t count;

        FieldIndexIO ioStream(f);
        Record r;
        ioStream._readBytes((char*)(&count),sizeof(uint64_t));
        pPosting = new InMemoryPosting(pMemCache_, skipInterval_, maxSkipLevel_);
        termid_t lastTerm = BAD_DOCID;
        try
        {
        for (uint64_t i = 0; i < count; ++i)
        {
            ioStream._read((char *)(&r), 13);
            if(r.tid != lastTerm && lastTerm != BAD_DOCID)
            {
                if (!pPosting->hasNoChunk())
                {
                    pPosting->write(pWriterDesc, termInfo); 	///write posting data
                    writeTermInfo(pVocWriter, lastTerm, termInfo);
                    pPosting->reset();								///clear posting data
                    pMemCache_->flushMem();
                    termInfo.reset();
                    termCount_++;
                }
            }
            pPosting->add(r.docId, r.offset);
            lastTerm = r.tid;
        }
        }catch(std::exception& e)
        {
        cout<<e.what()<<endl;
        }

        if (!pPosting->hasNoChunk())
        {
            pPosting->write(pWriterDesc, termInfo);	///write posting data
            writeTermInfo(pVocWriter, lastTerm, termInfo);
            pPosting->reset(); 							///clear posting data
            pMemCache_->flushMem();
            termInfo.reset();
            termCount_++;
        }
        fclose(f);
        boost::filesystem::remove(sorterFullPath_);
        delete pPosting;
    }

    fileoffset_t vocDescOffset = pVocWriter->getFilePointer();
    int64_t vocLength = vocDescOffset - vocOffset;
    ///begin write vocabulary descriptor
    pVocWriter->writeLong(vocLength);	///<VocLength(Int64)>
    pVocWriter->writeLong(termCount_);	///<TermCount(Int64)>
    ///end write vocabulary descriptor

    return vocDescOffset;
}

TermReader* FieldIndexer::termReader()
{
    izenelib::util::ScopedReadLock<izenelib::util::ReadWriteLock> lock(rwLock_);
    return new InMemoryTermReader(getField(),this);
}

}

NS_IZENELIB_IR_END

<commit_msg>clear izene sort<commit_after>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/FieldIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/TermPositions.h>
#include <ir/index_manager/store/IndexInput.h>

#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>

using namespace std;

namespace bfs = boost::filesystem;

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

FieldIndexer::FieldIndexer(const char* field, MemCache* pCache, Indexer* pIndexer)
        :field_(field),pMemCache_(pCache),pIndexer_(pIndexer),vocFilePointer_(0),alloc_(0),
        termCount_(0),recordCount_(0),run_num_(0),pHits_(0),pHitsMax_(0),flush_(false)
{
    skipInterval_ = pIndexer_->getSkipInterval();
    maxSkipLevel_ = pIndexer_->getMaxSkipLevel();
    if (! pIndexer_->isRealTime())
    {
        //std::string sorterName = field_+".tmp";
        sorterFileName_ = field_+".tmp";
        bfs::path path(bfs::path(pIndexer->pConfigurationManager_->indexStrategy_.indexLocation_) /bfs::path(sorterFileName_));
        sorterFullPath_= path.string();
        f_ = fopen(sorterFullPath_.c_str(),"w");
        uint64_t count = 0;
        fwrite(&count, sizeof(uint64_t), 1, f_);
    }
    else
        alloc_ = new boost::scoped_alloc(recycle_);
}

FieldIndexer::~FieldIndexer()
{
    /*
        InMemoryPostingMap::iterator iter = postingMap_.begin()
        for(; iter !=postingMap_.end(); ++iter)
        {
            delete iter->second;
        }
    */
    if (alloc_) delete alloc_;
    pMemCache_ = NULL;
}

void FieldIndexer::setHitBuffer(size_t size)
{
    iHitsMax_ = size;
    iHitsMax_ = iHitsMax_/sizeof(TermId) ;
    hits_.assign(iHitsMax_);
    pHits_ = hits_;
    pHitsMax_ = hits_ + iHitsMax_;
}

void FieldIndexer::writeHitBuffer(int iHits)
{
    recordCount_ += iHits;
    bufferSort ( &hits_[0], iHits, CmpTermId_fn() );

    uint32_t output_buf_size = iHits * (sizeof(TermId)+sizeof(uint8_t));
    ///buffer size
    fwrite(&output_buf_size, sizeof(uint32_t), 1, f_);
    ///number of hits
    fwrite(&iHits, sizeof(uint32_t), 1, f_);

    uint64_t nextStart = 0;
    uint64_t nextStartPos = ftell(f_);
    ///next start
    fwrite(&nextStart, sizeof(uint64_t), 1, f_);
    FieldIndexIO ioStream(f_, "w");
    ioStream.writeRecord(&hits_[0], iHits * sizeof(TermId));
    nextStart = ftell(f_);

    ///update next start
    fseek(f_, nextStartPos, SEEK_SET);
    fwrite(&nextStart, sizeof(uint64_t), 1, f_);
    fseek(f_, nextStart, SEEK_SET);

    ++run_num_;
}

void FieldIndexer::addField(docid_t docid, boost::shared_ptr<LAInput> laInput)
{
    if (pIndexer_->isRealTime())
    {
        InMemoryPosting* curPosting;
        for (LAInput::iterator iter = laInput->begin(); iter != laInput->end(); ++iter)
        {
            InMemoryPostingMap::iterator postingIter = postingMap_.find(iter->termid_);
            if (postingIter == postingMap_.end())
            {
                //curPosting = new InMemoryPosting(pMemCache_, skipInterval_, maxSkipLevel_);
                curPosting = BOOST_NEW(*alloc_, InMemoryPosting)(pMemCache_, skipInterval_, maxSkipLevel_);
                postingMap_[iter->termid_] = curPosting;
            }
            else
                curPosting = postingIter->second;
            curPosting->add(docid, iter->wordOffset_);
        }
    }
    else
    {
        if(flush_)
        {
            hits_.assign(iHitsMax_);
            pHits_ = hits_;
            pHitsMax_ = hits_ + iHitsMax_;
            flush_ = false;
        }

        int iDocHits = laInput->size();
        TermId * pDocHits = (TermId*)&(* laInput->begin());

        while( iDocHits > 0)
        {
            int iToCopy = iDocHits < (pHitsMax_ - pHits_) ? iDocHits: (pHitsMax_ - pHits_);
            memcpy(pHits_, pDocHits, iToCopy*sizeof(TermId));
            pHits_ += iToCopy;
            pDocHits += iToCopy;
            iDocHits -= iToCopy;

            if (pHits_ < pHitsMax_)
                continue;

            int iHits = pHits_ - hits_;
            writeHitBuffer(iHits);
            pHits_ = hits_;
        }
    }
}

void FieldIndexer::addField(docid_t docid, boost::shared_ptr<ForwardIndex> forwardindex)
{
    InMemoryPosting* curPosting;

    for (ForwardIndex::iterator iter = forwardindex->begin(); iter != forwardindex->end(); ++iter)
    {
        InMemoryPostingMap::iterator postingIter = postingMap_.find(iter->first);
        if (postingIter == postingMap_.end())
        {
            curPosting = BOOST_NEW(*alloc_, InMemoryPosting)(pMemCache_, skipInterval_, maxSkipLevel_);
            postingMap_[iter->first] = curPosting;
        }
        else
            curPosting = postingIter->second;

        ForwardIndexOffset::iterator	endit = iter->second->end();
        for (ForwardIndexOffset::iterator it = iter->second->begin(); it != endit; ++it)
            curPosting->add(docid, *it);
    }
}

void FieldIndexer::reset()
{
    InMemoryPosting* pPosting;
    for (InMemoryPostingMap::iterator iter = postingMap_.begin(); iter !=postingMap_.end(); ++iter)
    {
        pPosting = iter->second;
        if (!pPosting->hasNoChunk())
        {
            pPosting->reset();		///clear posting data
        }
    }
    postingMap_.clear();
    termCount_ = 0;
}

void writeTermInfo(IndexOutput* pVocWriter, termid_t tid, const TermInfo& termInfo)
{
    pVocWriter->writeInt(tid);					///write term id
    pVocWriter->writeInt(termInfo.docFreq_);		///write df
    pVocWriter->writeInt(termInfo.ctf_);			///write ctf
    pVocWriter->writeInt(termInfo.lastDocID_);		///write last doc id
    pVocWriter->writeInt(termInfo.skipLevel_);		///write skip level
    pVocWriter->writeLong(termInfo.skipPointer_);	///write skip list offset offset
    pVocWriter->writeLong(termInfo.docPointer_);	///write document posting offset
    pVocWriter->writeInt(termInfo.docPostingLen_);	///write document posting length (without skiplist)
    pVocWriter->writeLong(termInfo.positionPointer_);	///write position posting offset
    pVocWriter->writeInt(termInfo.positionPostingLen_);///write position posting length
}

#pragma pack(push,1)
struct Record
{
    uint8_t len;
    uint32_t tid;
    uint32_t docId;
    uint32_t offset;
};
#pragma pack(pop)

fileoffset_t FieldIndexer::write(OutputDescriptor* pWriterDesc)
{
    vocFilePointer_ = pWriterDesc->getVocOutput()->getFilePointer();

    IndexOutput* pVocWriter = pWriterDesc->getVocOutput();

    termid_t tid;
    InMemoryPosting* pPosting;
    fileoffset_t vocOffset = pVocWriter->getFilePointer();
    TermInfo termInfo;

    if (pIndexer_->isRealTime())
    {
        izenelib::util::ScopedWriteLock<izenelib::util::ReadWriteLock> lock(rwLock_);
        for (InMemoryPostingMap::iterator iter = postingMap_.begin(); iter !=postingMap_.end(); ++iter)
        {
            pPosting = iter->second;
            pPosting->setDirty(true);
            if (!pPosting->hasNoChunk())
            {
                tid = iter->first;
                pPosting->write(pWriterDesc, termInfo);		///write posting data
                writeTermInfo(pVocWriter,tid,termInfo);
                pPosting->reset();								///clear posting data
                termInfo.reset();
                termCount_++;
            }
            //delete pPosting;
            //pPosting = NULL;
        }

        postingMap_.clear();

        delete alloc_;
        alloc_ = new boost::scoped_alloc(recycle_);
    }
    else
    {
        if( !boost::filesystem::exists(sorterFullPath_) )
        {
            fileoffset_t vocDescOffset = pVocWriter->getFilePointer();
            int64_t vocLength = vocDescOffset - vocOffset;

            //SF1V5_THROW(ERROR_FILEIO,"Open file error: " + sorterFullPath_);
            pVocWriter->writeLong(vocLength);	///<VocLength(Int64)>
            pVocWriter->writeLong(termCount_);	///<TermCount(Int64)>
            ///end write vocabulary descriptor

            return vocDescOffset;
        }
        int iHits = pHits_ - hits_;
        if(iHits > 0)
        {
            writeHitBuffer(iHits);
        }
        fseek(f_, 0, SEEK_SET);
        fwrite(&recordCount_, sizeof(uint64_t), 1, f_);
        fclose(f_);
        hits_.reset();

        typedef izenelib::am::SortMerger<uint32_t, uint8_t, true,SortIO<FieldIndexIO> > merge_t;

        struct timeval tvafter, tvpre;
        struct timezone tz;
        gettimeofday (&tvpre , &tz);

        merge_t* merger = new merge_t(sorterFullPath_.c_str(), run_num_, 100000000, 2);
        merger->run();

        gettimeofday (&tvafter , &tz);
        std::cout<<"\nIt takes "<<((tvafter.tv_sec-tvpre.tv_sec)*1000+(tvafter.tv_usec-tvpre.tv_usec)/1000.)/60000
		 <<" minutes to sort("<<recordCount_<<")\n";
        delete merger;
        recordCount_ = 0;
        run_num_ = 0;
        flush_ = true;

        FILE* f = fopen(sorterFullPath_.c_str(),"r");
        uint64_t count;

        FieldIndexIO ioStream(f);
        Record r;
        ioStream._readBytes((char*)(&count),sizeof(uint64_t));
        pPosting = new InMemoryPosting(pMemCache_, skipInterval_, maxSkipLevel_);
        termid_t lastTerm = BAD_DOCID;
        try
        {
        for (uint64_t i = 0; i < count; ++i)
        {
            ioStream._read((char *)(&r), 13);
            if(r.tid != lastTerm && lastTerm != BAD_DOCID)
            {
                if (!pPosting->hasNoChunk())
                {
                    pPosting->write(pWriterDesc, termInfo); 	///write posting data
                    writeTermInfo(pVocWriter, lastTerm, termInfo);
                    pPosting->reset();								///clear posting data
                    pMemCache_->flushMem();
                    termInfo.reset();
                    termCount_++;
                }
            }
            pPosting->add(r.docId, r.offset);
            lastTerm = r.tid;
        }
        }catch(std::exception& e)
        {
        cout<<e.what()<<endl;
        }

        if (!pPosting->hasNoChunk())
        {
            pPosting->write(pWriterDesc, termInfo);	///write posting data
            writeTermInfo(pVocWriter, lastTerm, termInfo);
            pPosting->reset(); 							///clear posting data
            pMemCache_->flushMem();
            termInfo.reset();
            termCount_++;
        }
        fclose(f);
        boost::filesystem::remove(sorterFullPath_);
        delete pPosting;
    }

    fileoffset_t vocDescOffset = pVocWriter->getFilePointer();
    int64_t vocLength = vocDescOffset - vocOffset;
    ///begin write vocabulary descriptor
    pVocWriter->writeLong(vocLength);	///<VocLength(Int64)>
    pVocWriter->writeLong(termCount_);	///<TermCount(Int64)>
    ///end write vocabulary descriptor

    return vocDescOffset;
}

TermReader* FieldIndexer::termReader()
{
    izenelib::util::ScopedReadLock<izenelib::util::ReadWriteLock> lock(rwLock_);
    return new InMemoryTermReader(getField(),this);
}

}

NS_IZENELIB_IR_END

<|endoftext|>


<commit_before>#include "NetworkClient.h"
#include <stdexcept>
#include <boost/bind.hpp>
#include "core/global.h"
#include "config/ConfigVariable.h"
using namespace std;
using boost::asio::ip::tcp;
namespace ssl = boost::asio::ssl;

NetworkClient::NetworkClient() : m_socket(nullptr), m_secure_socket(nullptr),
    m_async_timer(io_service), m_send_queue()
{
}

NetworkClient::NetworkClient(tcp::socket *socket) : m_socket(socket), m_secure_socket(nullptr),
    m_async_timer(io_service), m_send_queue()
{
    start_receive();
}

NetworkClient::NetworkClient(ssl::stream<tcp::socket>* stream) :
    m_socket(&stream->next_layer()), m_secure_socket(stream),
    m_async_timer(io_service), m_send_queue()
{
    start_receive();
}

NetworkClient::~NetworkClient()
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    send_disconnect();

    if(m_secure_socket) {
        // This also deletes m_socket:
        delete m_secure_socket;
    } else {
        // ONLY delete m_socket if we must do so directly. If it's wrapped in
        // an SSL stream, the stream "owns" the socket.
        delete m_socket;
    }

    delete [] m_data_buf;
    delete [] m_send_buf;
}

void NetworkClient::set_socket(tcp::socket *socket)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    if(m_socket) {
        throw std::logic_error("Trying to set a socket of a network client whose socket was already set.");
    }
    m_socket = socket;

    boost::asio::socket_base::keep_alive keepalive(true);
    m_socket->set_option(keepalive);

    boost::asio::ip::tcp::no_delay nodelay(true);
    m_socket->set_option(nodelay);

    start_receive();
}

void NetworkClient::set_socket(ssl::stream<tcp::socket> *stream)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    if(m_socket) {
        throw std::logic_error("Trying to set a socket of a network client whose socket was already set.");
    }

    m_secure_socket = stream;

    set_socket(&stream->next_layer());
}

void NetworkClient::set_write_timeout(unsigned int timeout)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    m_write_timeout = timeout;
}

void NetworkClient::set_write_buffer(uint64_t max_bytes)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    m_max_queue_size = max_bytes;
}

void NetworkClient::send_datagram(DatagramHandle dg)
{
    lock_guard<recursive_mutex> lock(m_lock);

    if(m_is_sending)
    {
        m_send_queue.push(dg);
        m_total_queue_size += dg->size();
        if(m_total_queue_size > m_max_queue_size && m_max_queue_size != 0)
        {
            boost::system::error_code enobufs(boost::system::errc::errc_t::no_buffer_space, boost::system::system_category());
            send_disconnect(enobufs);
        }
    }
    else
    {
        m_is_sending = true;
        async_send(dg);
    }
}

bool NetworkClient::is_connected()
{
    lock_guard<recursive_mutex> lock(m_lock);
    return m_socket->is_open();
}

void NetworkClient::start_receive()
{
    // Lock not needed: This is only called from the constructor.

    try {
        m_remote = m_socket->remote_endpoint();
    } catch(const boost::system::system_error&) {
        // A client might disconnect immediately after connecting.
        // Since we are in the constructor, ignore it. Resolves #122.
        // When the owner of the NetworkClient attempts to send or receive,
        // the error will occur and we'll cleanup then;
    }
    async_receive();
}

void NetworkClient::async_receive()
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    try {
        if(m_is_data) { // Read data
            socket_read(m_data_buf, m_data_size, &NetworkClient::receive_data);
        } else { // Read length
            socket_read(m_size_buf, sizeof(dgsize_t), &NetworkClient::receive_size);
        }
    } catch(const boost::system::system_error &err) {
        // An exception happening when trying to initiate a read is a clear
        // indicator that something happened to the connection. Therefore:
        send_disconnect(err.code());
    }
}

void NetworkClient::send_disconnect()
{
    boost::system::error_code ec;
    send_disconnect(ec);
}

void NetworkClient::send_disconnect(const boost::system::error_code &ec)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(m_local_disconnect || m_disconnect_handled) {
        // We've already set the error code and closed the socket; wait.
        return;
    }

    m_local_disconnect = true;
    m_disconnect_error = ec;

    m_socket->cancel();
    m_socket->close();
}

void NetworkClient::handle_disconnect(const boost::system::error_code &ec)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    if(m_disconnect_handled) {
        return;
    }

    m_disconnect_handled = true;

    if(m_local_disconnect) {
        receive_disconnect(m_disconnect_error);
    } else {
        receive_disconnect(ec);
    }
}

void NetworkClient::receive_size(const boost::system::error_code &ec, size_t bytes_transferred)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(ec) {
        handle_disconnect(ec);
        return;
    }

    if(bytes_transferred != sizeof(m_size_buf)) {
        boost::system::error_code epipe(boost::system::errc::errc_t::broken_pipe, boost::system::system_category());
        handle_disconnect(epipe);
        return;
    }

    // required to disable strict-aliasing optimizations, which can break the code
    dgsize_t* new_size_p = (dgsize_t*)m_size_buf;
    m_data_size = swap_le(*new_size_p);
    m_data_buf = new uint8_t[m_data_size];
    m_is_data = true;
    async_receive();
}

void NetworkClient::receive_data(const boost::system::error_code &ec, size_t bytes_transferred)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(ec) {
        handle_disconnect(ec);
        return;
    }

    if(bytes_transferred != m_data_size) {
        boost::system::error_code epipe(boost::system::errc::errc_t::broken_pipe, boost::system::system_category());
        handle_disconnect(epipe);
        return;
    }

    DatagramPtr dg = Datagram::create(m_data_buf, m_data_size); // Datagram makes a copy
    receive_datagram(dg);

    delete [] m_data_buf;
    m_data_buf = nullptr;

    m_is_data = false;
    async_receive();
}

void NetworkClient::async_send(DatagramHandle dg)
{
    lock_guard<recursive_mutex> lock(m_lock);

    size_t buffer_size = sizeof(dgsize_t) + dg->size();
    dgsize_t len = swap_le(dg->size());
    m_send_buf = new uint8_t[buffer_size];
    memcpy(m_send_buf, (uint8_t*)&len, sizeof(dgsize_t));
    memcpy(m_send_buf+sizeof(dgsize_t), dg->get_data(), dg->size());

    try
    {
        socket_write(m_send_buf, buffer_size);
    }
    catch(const boost::system::system_error& err)
    {
        // An exception happening when trying to initiate a send is a clear
        // indicator that something happened to the connection, therefore:
        send_disconnect(err.code());
    }
}

void NetworkClient::send_finished(const boost::system::error_code &ec)
{
    lock_guard<recursive_mutex> lock(m_lock);

    // Cancel the outstanding timeout
    m_async_timer.cancel();

    // Discard the buffer we just used:
    delete [] m_send_buf;
    m_send_buf = nullptr;

    // Check if the write had errors
    if(ec.value() != 0)
    {
        m_is_sending = false;
        send_disconnect(ec);
        return;
    }

    // Check if we have more items in the queue
    if(m_send_queue.size() > 0)
    {
        // Send the next item in the queue
        DatagramHandle dg = m_send_queue.front();
        m_total_queue_size -= dg->size();
        m_send_queue.pop();
        async_send(dg);
        return;
    }

    // Nothing left in the queue to send, lets open up for another write
    m_is_sending = false;
}

void NetworkClient::send_expired(const boost::system::error_code& ec)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    // operation_aborted is received if the the timer is cancelled,
    //     ie. if the send completed before it expires, so don't do anything
    if(ec != boost::asio::error::operation_aborted)
    {
        boost::system::error_code etimeout(boost::system::errc::errc_t::timed_out, boost::system::system_category());
        send_disconnect(etimeout);
    }
}

void NetworkClient::socket_read(uint8_t* buf, size_t length, receive_handler_t callback)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    if(m_secure_socket) {
        async_read(*m_secure_socket, boost::asio::buffer(buf, length),
                   boost::bind(callback, this,
                               boost::asio::placeholders::error,
                               boost::asio::placeholders::bytes_transferred));
    } else {
        async_read(*m_socket, boost::asio::buffer(buf, length),
                   boost::bind(callback, this,
                               boost::asio::placeholders::error,
                               boost::asio::placeholders::bytes_transferred));
    }
}

void NetworkClient::socket_write(const uint8_t* buf, size_t length)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    // Start async timeout, a value of 0 indicates the writes shouldn't timeout (used in debugging)
    if(m_write_timeout > 0)
    {
        m_async_timer.expires_from_now(boost::posix_time::milliseconds(m_write_timeout));
        m_async_timer.async_wait(boost::bind(&NetworkClient::send_expired, this,
                                             boost::asio::placeholders::error));
    }

    // Start async write
    if(m_secure_socket)
    {
        async_write(*m_secure_socket, boost::asio::buffer(buf, length),
                    boost::bind(&NetworkClient::send_finished, this,
                    boost::asio::placeholders::error));
    }
    else
    {
        async_write(*m_socket, boost::asio::buffer(buf, length),
                    boost::bind(&NetworkClient::send_finished, this,
                    boost::asio::placeholders::error));
    }
}
<commit_msg>net: Cancel async send timeout when shutting down the connection.<commit_after>#include "NetworkClient.h"
#include <stdexcept>
#include <boost/bind.hpp>
#include "core/global.h"
#include "config/ConfigVariable.h"
using namespace std;
using boost::asio::ip::tcp;
namespace ssl = boost::asio::ssl;

NetworkClient::NetworkClient() : m_socket(nullptr), m_secure_socket(nullptr),
    m_async_timer(io_service), m_send_queue()
{
}

NetworkClient::NetworkClient(tcp::socket *socket) : m_socket(socket), m_secure_socket(nullptr),
    m_async_timer(io_service), m_send_queue()
{
    start_receive();
}

NetworkClient::NetworkClient(ssl::stream<tcp::socket>* stream) :
    m_socket(&stream->next_layer()), m_secure_socket(stream),
    m_async_timer(io_service), m_send_queue()
{
    start_receive();
}

NetworkClient::~NetworkClient()
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    send_disconnect();

    if(m_secure_socket) {
        // This also deletes m_socket:
        delete m_secure_socket;
    } else {
        // ONLY delete m_socket if we must do so directly. If it's wrapped in
        // an SSL stream, the stream "owns" the socket.
        delete m_socket;
    }

    delete [] m_data_buf;
    delete [] m_send_buf;
}

void NetworkClient::set_socket(tcp::socket *socket)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    if(m_socket) {
        throw std::logic_error("Trying to set a socket of a network client whose socket was already set.");
    }
    m_socket = socket;

    boost::asio::socket_base::keep_alive keepalive(true);
    m_socket->set_option(keepalive);

    boost::asio::ip::tcp::no_delay nodelay(true);
    m_socket->set_option(nodelay);

    start_receive();
}

void NetworkClient::set_socket(ssl::stream<tcp::socket> *stream)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    if(m_socket) {
        throw std::logic_error("Trying to set a socket of a network client whose socket was already set.");
    }

    m_secure_socket = stream;

    set_socket(&stream->next_layer());
}

void NetworkClient::set_write_timeout(unsigned int timeout)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    m_write_timeout = timeout;
}

void NetworkClient::set_write_buffer(uint64_t max_bytes)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    m_max_queue_size = max_bytes;
}

void NetworkClient::send_datagram(DatagramHandle dg)
{
    lock_guard<recursive_mutex> lock(m_lock);

    if(m_is_sending)
    {
        m_send_queue.push(dg);
        m_total_queue_size += dg->size();
        if(m_total_queue_size > m_max_queue_size && m_max_queue_size != 0)
        {
            boost::system::error_code enobufs(boost::system::errc::errc_t::no_buffer_space, boost::system::system_category());
            send_disconnect(enobufs);
        }
    }
    else
    {
        m_is_sending = true;
        async_send(dg);
    }
}

bool NetworkClient::is_connected()
{
    lock_guard<recursive_mutex> lock(m_lock);
    return m_socket->is_open();
}

void NetworkClient::start_receive()
{
    // Lock not needed: This is only called from the constructor.

    try {
        m_remote = m_socket->remote_endpoint();
    } catch(const boost::system::system_error&) {
        // A client might disconnect immediately after connecting.
        // Since we are in the constructor, ignore it. Resolves #122.
        // When the owner of the NetworkClient attempts to send or receive,
        // the error will occur and we'll cleanup then;
    }
    async_receive();
}

void NetworkClient::async_receive()
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);
    try {
        if(m_is_data) { // Read data
            socket_read(m_data_buf, m_data_size, &NetworkClient::receive_data);
        } else { // Read length
            socket_read(m_size_buf, sizeof(dgsize_t), &NetworkClient::receive_size);
        }
    } catch(const boost::system::system_error &err) {
        // An exception happening when trying to initiate a read is a clear
        // indicator that something happened to the connection. Therefore:
        send_disconnect(err.code());
    }
}

void NetworkClient::send_disconnect()
{
    boost::system::error_code ec;
    send_disconnect(ec);
}

void NetworkClient::send_disconnect(const boost::system::error_code &ec)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(m_local_disconnect || m_disconnect_handled) {
        // We've already set the error code and closed the socket; wait.
        return;
    }

    m_local_disconnect = true;
    m_disconnect_error = ec;

    m_socket->cancel();
    m_socket->close();

    m_async_timer.cancel();
}

void NetworkClient::handle_disconnect(const boost::system::error_code &ec)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    if(m_disconnect_handled) {
        return;
    }

    m_disconnect_handled = true;

    if(m_local_disconnect) {
        receive_disconnect(m_disconnect_error);
    } else {
        receive_disconnect(ec);
    }
}

void NetworkClient::receive_size(const boost::system::error_code &ec, size_t bytes_transferred)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(ec) {
        handle_disconnect(ec);
        return;
    }

    if(bytes_transferred != sizeof(m_size_buf)) {
        boost::system::error_code epipe(boost::system::errc::errc_t::broken_pipe, boost::system::system_category());
        handle_disconnect(epipe);
        return;
    }

    // required to disable strict-aliasing optimizations, which can break the code
    dgsize_t* new_size_p = (dgsize_t*)m_size_buf;
    m_data_size = swap_le(*new_size_p);
    m_data_buf = new uint8_t[m_data_size];
    m_is_data = true;
    async_receive();
}

void NetworkClient::receive_data(const boost::system::error_code &ec, size_t bytes_transferred)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    if(ec) {
        handle_disconnect(ec);
        return;
    }

    if(bytes_transferred != m_data_size) {
        boost::system::error_code epipe(boost::system::errc::errc_t::broken_pipe, boost::system::system_category());
        handle_disconnect(epipe);
        return;
    }

    DatagramPtr dg = Datagram::create(m_data_buf, m_data_size); // Datagram makes a copy
    receive_datagram(dg);

    delete [] m_data_buf;
    m_data_buf = nullptr;

    m_is_data = false;
    async_receive();
}

void NetworkClient::async_send(DatagramHandle dg)
{
    lock_guard<recursive_mutex> lock(m_lock);

    size_t buffer_size = sizeof(dgsize_t) + dg->size();
    dgsize_t len = swap_le(dg->size());
    m_send_buf = new uint8_t[buffer_size];
    memcpy(m_send_buf, (uint8_t*)&len, sizeof(dgsize_t));
    memcpy(m_send_buf+sizeof(dgsize_t), dg->get_data(), dg->size());

    try
    {
        socket_write(m_send_buf, buffer_size);
    }
    catch(const boost::system::system_error& err)
    {
        // An exception happening when trying to initiate a send is a clear
        // indicator that something happened to the connection, therefore:
        send_disconnect(err.code());
    }
}

void NetworkClient::send_finished(const boost::system::error_code &ec)
{
    lock_guard<recursive_mutex> lock(m_lock);

    // Cancel the outstanding timeout
    m_async_timer.cancel();

    // Discard the buffer we just used:
    delete [] m_send_buf;
    m_send_buf = nullptr;

    // Check if the write had errors
    if(ec.value() != 0)
    {
        m_is_sending = false;
        send_disconnect(ec);
        return;
    }

    // Check if we have more items in the queue
    if(m_send_queue.size() > 0)
    {
        // Send the next item in the queue
        DatagramHandle dg = m_send_queue.front();
        m_total_queue_size -= dg->size();
        m_send_queue.pop();
        async_send(dg);
        return;
    }

    // Nothing left in the queue to send, lets open up for another write
    m_is_sending = false;
}

void NetworkClient::send_expired(const boost::system::error_code& ec)
{
    std::lock_guard<std::recursive_mutex> lock(m_lock);

    // operation_aborted is received if the the timer is cancelled,
    //     ie. if the send completed before it expires, so don't do anything
    if(ec != boost::asio::error::operation_aborted)
    {
        boost::system::error_code etimeout(boost::system::errc::errc_t::timed_out, boost::system::system_category());
        send_disconnect(etimeout);
    }
}

void NetworkClient::socket_read(uint8_t* buf, size_t length, receive_handler_t callback)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    if(m_secure_socket) {
        async_read(*m_secure_socket, boost::asio::buffer(buf, length),
                   boost::bind(callback, this,
                               boost::asio::placeholders::error,
                               boost::asio::placeholders::bytes_transferred));
    } else {
        async_read(*m_socket, boost::asio::buffer(buf, length),
                   boost::bind(callback, this,
                               boost::asio::placeholders::error,
                               boost::asio::placeholders::bytes_transferred));
    }
}

void NetworkClient::socket_write(const uint8_t* buf, size_t length)
{
    // Lock not needed: This is only called internally, from a function that
    // already holds the lock.

    // Start async timeout, a value of 0 indicates the writes shouldn't timeout (used in debugging)
    if(m_write_timeout > 0)
    {
        m_async_timer.expires_from_now(boost::posix_time::milliseconds(m_write_timeout));
        m_async_timer.async_wait(boost::bind(&NetworkClient::send_expired, this,
                                             boost::asio::placeholders::error));
    }

    // Start async write
    if(m_secure_socket)
    {
        async_write(*m_secure_socket, boost::asio::buffer(buf, length),
                    boost::bind(&NetworkClient::send_finished, this,
                    boost::asio::placeholders::error));
    }
    else
    {
        async_write(*m_socket, boost::asio::buffer(buf, length),
                    boost::bind(&NetworkClient::send_finished, this,
                    boost::asio::placeholders::error));
    }
}
<|endoftext|>


<commit_before>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2016 ScyllaDB.
 */
/*! \file
    \brief Some non-INET-specific socket address code

    Extracted from inet_address.cc.
 */
#include <ostream>
#include <arpa/inet.h>
#include <seastar/net/socket_defs.hh>
#include <seastar/net/inet_address.hh>
#include <seastar/net/ip.hh>
#include <seastar/core/print.hh>
#include <boost/functional/hash.hpp>

using namespace std::string_literals;

size_t std::hash<seastar::socket_address>::operator()(const seastar::socket_address& a) const {
    auto h = std::hash<seastar::net::inet_address>()(a.addr());
    boost::hash_combine(h, a.as_posix_sockaddr_in().sin_port);
    return h;
}

namespace seastar {

socket_address::socket_address(const net::inet_address& a, uint16_t p)
    : socket_address(a.is_ipv6() ? socket_address(ipv6_addr(a, p), a.scope()) : socket_address(ipv4_addr(a, p)))
{}

socket_address::socket_address(const unix_domain_addr& s) {
    u.un.sun_family = AF_UNIX;
    memset(u.un.sun_path, '\0', sizeof(u.un.sun_path));
    auto path_length = std::min((int)sizeof(u.un.sun_path), s.path_length());
    memcpy(u.un.sun_path, s.path_bytes(), path_length);
    addr_length = path_length + offsetof(struct ::sockaddr_un, sun_path);
}

std::string unix_domain_addr_text(const socket_address& sa) {
    if (sa.length() <= ((size_t) (((struct ::sockaddr_un *) 0)->sun_path))) {
        return "{unnamed}"s;
    }
    if (sa.u.un.sun_path[0]) {
        // regular (filesystem-namespace) path
        return std::string{sa.u.un.sun_path};
    }

    const size_t  path_length{sa.length() - ((size_t) (((struct ::sockaddr_un *) 0)->sun_path))};
    char ud_path[1 + path_length];
    char* targ = ud_path;
    *targ++ = '@';
    const char* src = sa.u.un.sun_path + 1;
    int k = (int)path_length;

    for (; --k > 0; src++) {
        *targ++ = std::isprint(*src) ? *src : '_';
    }
    return std::string{ud_path, path_length};
}

std::ostream& operator<<(std::ostream& os, const socket_address& a) {
    if (a.is_af_unix()) {
        return os << unix_domain_addr_text(a);
    }

    auto addr = a.addr();
    // CMH. maybe skip brackets for ipv4-mapped
    auto bracket = addr.in_family() == seastar::net::inet_address::family::INET6;

    if (bracket) {
        os << '[';
    }
    os << addr;
    if (bracket) {
        os << ']';
    }

    return os << ':' << ntohs(a.u.in.sin_port);
}

} // namespace seastar
<commit_msg>net: Don't use variable length arrays<commit_after>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2016 ScyllaDB.
 */
/*! \file
    \brief Some non-INET-specific socket address code

    Extracted from inet_address.cc.
 */
#include <ostream>
#include <arpa/inet.h>
#include <seastar/net/socket_defs.hh>
#include <seastar/net/inet_address.hh>
#include <seastar/net/ip.hh>
#include <seastar/core/print.hh>
#include <boost/functional/hash.hpp>

using namespace std::string_literals;

size_t std::hash<seastar::socket_address>::operator()(const seastar::socket_address& a) const {
    auto h = std::hash<seastar::net::inet_address>()(a.addr());
    boost::hash_combine(h, a.as_posix_sockaddr_in().sin_port);
    return h;
}

namespace seastar {

socket_address::socket_address(const net::inet_address& a, uint16_t p)
    : socket_address(a.is_ipv6() ? socket_address(ipv6_addr(a, p), a.scope()) : socket_address(ipv4_addr(a, p)))
{}

socket_address::socket_address(const unix_domain_addr& s) {
    u.un.sun_family = AF_UNIX;
    memset(u.un.sun_path, '\0', sizeof(u.un.sun_path));
    auto path_length = std::min((int)sizeof(u.un.sun_path), s.path_length());
    memcpy(u.un.sun_path, s.path_bytes(), path_length);
    addr_length = path_length + offsetof(struct ::sockaddr_un, sun_path);
}

std::string unix_domain_addr_text(const socket_address& sa) {
    if (sa.length() <= ((size_t) (((struct ::sockaddr_un *) 0)->sun_path))) {
        return "{unnamed}"s;
    }
    if (sa.u.un.sun_path[0]) {
        // regular (filesystem-namespace) path
        return std::string{sa.u.un.sun_path};
    }

    const size_t  path_length{sa.length() - ((size_t) (((struct ::sockaddr_un *) 0)->sun_path))};
    std::string ud_path(path_length, 0);
    char* targ = ud_path.data();
    *targ++ = '@';
    const char* src = sa.u.un.sun_path + 1;
    int k = (int)path_length;

    for (; --k > 0; src++) {
        *targ++ = std::isprint(*src) ? *src : '_';
    }
    return ud_path;
}

std::ostream& operator<<(std::ostream& os, const socket_address& a) {
    if (a.is_af_unix()) {
        return os << unix_domain_addr_text(a);
    }

    auto addr = a.addr();
    // CMH. maybe skip brackets for ipv4-mapped
    auto bracket = addr.in_family() == seastar::net::inet_address::family::INET6;

    if (bracket) {
        os << '[';
    }
    os << addr;
    if (bracket) {
        os << ']';
    }

    return os << ':' << ntohs(a.u.in.sin_port);
}

} // namespace seastar
<|endoftext|>


<commit_before>/*
 *  input.cpp
 *
 *  Copyright (c) 2010 Motiejus Jakštys
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 3.

 *  This program is distributed in the hope that it will be usefu
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.

 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include "input.h"
#include "soundpatty.h"

void Input::its_over(const char *port_name, double place) {
	char msg[50];
	sprintf(msg,"FOUND for %s, processed %.6f sec", port_name, place);
    printf("%s", msg);
    LOG_INFO(msg);
    // Call the over.sh function
    char command[300];
    sprintf(command, "./over.sh \"%s\" \"%.6f\"", port_name, place);
    int ret = system(command);
    if (ret != 0) {
        printf("System command failed! Return number: %d\n", ret);
    }
};

void Input::new_port_created(
        action_t action, const char *port_name,
        Input *input, all_cfg_t *cfg, void *sp_params) {

    // Hum, must create new SomeInput instance here
    // So we have new port created here. In case of jack -
    // it's safe to call jack server here, since it's not a caller thread :-)

    LOG_DEBUG("Creating SoundPatty instance for new port");
    SoundPatty *pat = new SoundPatty(action, input, cfg, sp_params);

    pthread_t tmp; pthread_attr_t attr; pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
    if (int err = pthread_create(&tmp, &attr, SoundPatty::go_thread, (void*)pat)) {
        LOG_ERROR("Failed to create thread for %s, error %d", port_name, err);
    }
    LOG_INFO("Launched new SoundPatty thread for %s", port_name);
}
<commit_msg>Removed useless printf<commit_after>/*
 *  input.cpp
 *
 *  Copyright (c) 2010 Motiejus Jakštys
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 3.

 *  This program is distributed in the hope that it will be usefu
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.

 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include "input.h"
#include "soundpatty.h"

void Input::its_over(const char *port_name, double place) {
	char msg[50];
	sprintf(msg,"FOUND for %s, processed %.6f sec", port_name, place);
    LOG_INFO(msg);
    // Call the over.sh function
    char command[300];
    sprintf(command, "./over.sh \"%s\" \"%.6f\"", port_name, place);
    int ret = system(command);
    if (ret != 0) {
        printf("System command failed! Return number: %d\n", ret);
    }
};

void Input::new_port_created(
        action_t action, const char *port_name,
        Input *input, all_cfg_t *cfg, void *sp_params) {

    // Hum, must create new SomeInput instance here
    // So we have new port created here. In case of jack -
    // it's safe to call jack server here, since it's not a caller thread :-)

    LOG_DEBUG("Creating SoundPatty instance for new port");
    SoundPatty *pat = new SoundPatty(action, input, cfg, sp_params);

    pthread_t tmp; pthread_attr_t attr; pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
    if (int err = pthread_create(&tmp, &attr, SoundPatty::go_thread, (void*)pat)) {
        LOG_ERROR("Failed to create thread for %s, error %d", port_name, err);
    }
    LOG_INFO("Launched new SoundPatty thread for %s", port_name);
}
<|endoftext|>